@denizsvnc/ai-rag-engine 0.0.0-stage → 2.2.0

package/dist/core/security/pii-guard.d.ts

@@ -0,0 +1,38 @@
+export interface PIIPolicy {
+    enabled?: boolean;
+    maskTCKN?: boolean;
+    maskCreditCards?: boolean;
+    maskIBAN?: boolean;
+    maskEmails?: boolean;
+    maskPhones?: boolean;
+    allowUnmaskedForLocalLLM?: boolean;
+    providerType?: 'cloud' | 'local' | 'custom';
+}
+export interface PIISanitizationResult {
+    sanitizedText: string;
+    hasPII: boolean;
+    detectedEntities: Array<{
+        type: 'TCKN' | 'CREDIT_CARD' | 'IBAN' | 'EMAIL' | 'PHONE';
+        originalValue: string;
+        maskedValue: string;
+    }>;
+    bypassedForLocalLLM: boolean;
+}
+export declare class PIIGuard {
+    /**
+     * Validates Turkish Republic Identity Number (TCKN) using official 11-digit checksum rules:
+     * 1. 11 digits, cannot start with 0.
+     * 2. 10th digit = ((d1+d3+d5+d7+d9)*7 - (d2+d4+d6+d8)) mod 10
+     * 3. 11th digit = (d1+d2+...+d10) mod 10
+     */
+    static isValidTCKN(candidate: string): boolean;
+    /**
+     * Validates Credit Card number using the Luhn mod-10 algorithm.
+     */
+    static isValidCreditCard(candidate: string): boolean;
+    /**
+     * Sanitizes input text, detecting and masking all sensitive PII entities.
+     * Respects data sovereignty policies: strictly masks for Cloud LLMs.
+     */
+    static sanitize(text: string, policy?: PIIPolicy): PIISanitizationResult;
+}

package/dist/core/security/sanitizer.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Cleanses and neutralizes raw untrusted visitor input.
+ */
+export declare class InputSanitizer {
+    /**
+     * Regex matching zero-width invisible characters, byte-order marks, and unsafe control chars.
+     * Frequently leveraged by attackers to hide prompt injection payloads from simple filters.
+     */
+    private static readonly INVISIBLE_CHARS_REGEX;
+    /**
+     * Tags and delimiters commonly used in prompt sandboxing and LLM chat templates.
+     */
+    private static readonly DELIMITER_ESCAPE_REGEX;
+    /**
+     * Sanitizes raw visitor input by stripping invisible chars, neutralizing delimiter breakout tags,
+     * and bounding input length.
+     */
+    /**
+     * Strips invisible zero-width unicode characters, byte-order marks, and control bytes.
+     */
+    static stripZeroWidth(rawInput: string): string;
+    static sanitize(rawInput: string, maxLength?: number): string;
+}

package/dist/core/security/secrets.d.ts

@@ -0,0 +1,2 @@
+/** Development keys are unpredictable and process-local. Production never falls back. */
+export declare function resolveSecret(name: string, supplied?: string): string;

package/dist/core/security/security-guard.d.ts

@@ -0,0 +1,18 @@
+import type { SecurityConfig, InjectionCheckResult } from './security.types.js';
+/**
+ * Enterprise Security Guardrail for the RAG Engine.
+ * Provides pre-flight input sanitization, heuristic attack detection,
+ * PII masking (TCKN, Credit Cards, IBAN), and untrusted visitor boundary formatting.
+ */
+export declare class SecurityGuard {
+    /**
+     * Inspects, sanitizes, redacts PII, and assesses security risk of incoming visitor input.
+     */
+    static inspect(rawInput: string, config?: SecurityConfig): InjectionCheckResult;
+    /**
+     * Wraps visitor text into an explicit untrusted XML sandbox tag.
+     * Tells the LLM that this text represents unverified data from a website visitor
+     * and MUST NOT be treated as system-level instructions or behavioral overrides.
+     */
+    static frameVisitorMessage(sanitizedText: string): string;
+}

package/dist/core/security/security.types.d.ts

@@ -0,0 +1,38 @@
+import type { PIIPolicy } from './pii-guard.js';
+export type RiskLevel = 'NONE' | 'LOW' | 'MEDIUM' | 'HIGH';
+export interface InjectionCheckResult {
+    isRisk: boolean;
+    riskLevel: RiskLevel;
+    matchedPatterns: string[];
+    sanitizedInput: string;
+}
+export interface SecurityConfig {
+    /**
+     * Whether prompt injection defense and input sanitization are enabled.
+     * Defaults to true.
+     */
+    enabled?: boolean | undefined;
+    /**
+     * If true, queries detected as high-risk prompt injection attacks are blocked
+     * immediately without invoking LLM or tool execution.
+     * Defaults to true.
+     */
+    blockHighRiskInjections?: boolean | undefined;
+    /**
+     * Maximum character length allowed for user input to prevent token flooding DoS.
+     * Defaults to 4000.
+     */
+    maxInputLength?: number | undefined;
+    /**
+     * Custom response returned to the visitor when an injection attempt is blocked.
+     */
+    customRefusalMessage?: string | undefined;
+    /**
+     * Additional regex patterns to classify as prompt injection attacks.
+     */
+    customInjectionPatterns?: RegExp[] | undefined;
+    /**
+     * Data Privacy & Sovereignty policy for masking sensitive PII (TCKN, Credit Cards, IBAN, etc.).
+     */
+    pii?: PIIPolicy | undefined;
+}

package/dist/core/sml-engine.d.ts

@@ -0,0 +1,65 @@
+/** Stable SDK boundary: transformer implementation types are kept private. */
+export interface EmbeddingExtractor {
+    (text: string | string[], options: {
+        pooling: 'mean';
+        normalize: boolean;
+    }): Promise<{
+        data: Float32Array;
+        dims: number[];
+    }>;
+}
+export declare class CircuitBreakerOpenError extends Error {
+    constructor(message: string);
+}
+export interface SMLEngineConfig {
+    cacheDir?: string | undefined;
+    localFilesOnly?: boolean | undefined;
+    modelName?: string | undefined;
+    cacheSize?: number | undefined;
+    dtype?: 'q8' | 'fp32' | 'fp16' | 'q4' | undefined;
+    quantized?: boolean | undefined;
+    circuitBreakerThreshold?: number | undefined;
+    circuitBreakerCooldownMs?: number | undefined;
+}
+export declare class SMLEngine {
+    private readonly modelName;
+    private readonly cacheDir;
+    private readonly localFilesOnly;
+    private readonly dtype;
+    private extractorPromise;
+    private embeddingCache;
+    private circuitState;
+    private failureCount;
+    private lastFailureTime;
+    private readonly failureThreshold;
+    private readonly cooldownMs;
+    constructor(config?: SMLEngineConfig);
+    /**
+     * Resets the circuit breaker state (useful for testing and recovery).
+     */
+    resetCircuit(): void;
+    getCircuitState(): 'CLOSED' | 'OPEN' | 'HALF_OPEN';
+    getFailureCount(): number;
+    /**
+     * Thread-safe pipeline initializer guarded by a Circuit Breaker.
+     */
+    init(): Promise<EmbeddingExtractor>;
+    /**
+     * Generates a normalized embedding vector for a single text input (with LRU caching).
+     */
+    getEmbedding(text: string): Promise<Float32Array>;
+    /**
+     * Generates embedding vectors for multiple texts in batch mode.
+     */
+    getBatchEmbeddings(texts: string[]): Promise<Float32Array[]>;
+    /**
+     * Compares the query against target embeddings and returns the highest scoring match above the threshold.
+     */
+    findBestMatch<T extends {
+        id: string;
+        embedding: number[] | Float32Array;
+    }>(query: string, targets: T[], threshold?: number): Promise<{
+        target: T;
+        score: number;
+    } | null>;
+}

package/dist/core/tool-registry.d.ts

@@ -0,0 +1,17 @@
+import type { IAgentTool } from '../interfaces/tool.interface.js';
+import { SMLEngine } from './sml-engine.js';
+export declare class ToolRegistry {
+    private smlEngine;
+    private registryPath;
+    private activeTools;
+    private toolEmbeddings;
+    constructor(smlEngine: SMLEngine, registryPath?: string);
+    initialize(tools: IAgentTool[]): Promise<void>;
+    registerTool(tool: IAgentTool): Promise<void>;
+    getTool(name: string): IAgentTool | undefined;
+    getAllTools(): IAgentTool[];
+    /**
+     * Routes a user query to the most semantically relevant tool.
+     */
+    routeQueryToTool(query: string, threshold?: number): Promise<IAgentTool | null>;
+}

package/dist/core/vector/bm25.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Zero-dependency BM25 (Best Matching 25) Sparse Keyword Retrieval.
+ * Provides lexical term matching (exact SKU codes, error messages, proper nouns, part numbers)
+ * to complement dense semantic vector search.
+ */
+export interface BM25Document {
+    id: string;
+    content: string;
+    metadata?: Record<string, any> | undefined;
+}
+export declare class BM25Index {
+    private docCount;
+    private avgDocLength;
+    private docLengths;
+    private docTermFreqs;
+    private termDocFreqs;
+    private documents;
+    private readonly k1;
+    private readonly b;
+    constructor(options?: {
+        k1?: number;
+        b?: number;
+    });
+    /**
+     * Tokenizes text into lowercase normalized alphanumeric terms.
+     */
+    private tokenize;
+    /**
+     * Indexes a collection of documents.
+     */
+    index(docs: BM25Document[]): void;
+    /**
+     * Computes the Robertson-Spärck Jones IDF for a query term.
+     */
+    private computeIDF;
+    /**
+     * Scores documents against a search query using the Okapi BM25 formula.
+     */
+    search(query: string, topK?: number, filter?: Record<string, any>): Array<{
+        document: BM25Document;
+        score: number;
+    }>;
+    clear(): void;
+}

package/dist/core/vector/hybrid-search.d.ts

@@ -0,0 +1,15 @@
+import type { VectorDocument } from '../../interfaces/vector.interface.js';
+export interface HybridScoredDocument extends VectorDocument {
+    denseRank?: number | undefined;
+    sparseRank?: number | undefined;
+    rrfScore: number;
+}
+/**
+ * Reciprocal Rank Fusion (RRF) algorithm.
+ * Combines dense vector semantic rankings and sparse BM25 lexical rankings into
+ * a single high-precision ranking without needing score normalization.
+ *
+ * Formula: RRF_score(d) = sum_{m in models} 1 / (k + rank_m(d))
+ * Standard constant k = 60
+ */
+export declare function reciprocalRankFusion(denseResults: VectorDocument[], sparseResults: VectorDocument[], topK?: number, rrfK?: number): HybridScoredDocument[];

package/dist/index.d.ts

@@ -0,0 +1,51 @@
+export * from './interfaces/engine-config.interface.js';
+export * from './interfaces/query-context.interface.js';
+export * from './interfaces/tool.interface.js';
+export * from './interfaces/vector.interface.js';
+export * from './interfaces/faq.interface.js';
+export * from './interfaces/rules.interface.js';
+export * from './interfaces/cache.interface.js';
+export * from './interfaces/llm.interface.js';
+export * from './core/security/security.types.js';
+export { RAGEngine } from './core/engine.js';
+export { SMLEngine, type SMLEngineConfig, type EmbeddingExtractor, CircuitBreakerOpenError } from './core/sml-engine.js';
+export { ToolRegistry } from './core/tool-registry.js';
+export { CacheManager } from './core/cache-manager.js';
+export { PromptBuilder, type PromptContext } from './core/prompt-builder.js';
+export { TokenTracker, type EngineMetricsSummary, type ResponseMetrics, type TokenTrackerOptions } from './core/metrics/token-tracker.js';
+export { PrometheusExporter, type PrometheusMetricOptions } from './core/metrics/prometheus-exporter.js';
+export { RAGEvaluator, type RetrievalEvalResult, type FaithfulnessEvalResult, type FullRAGEvaluationReport } from './core/metrics/rag-evaluator.js';
+export { BM25Index, type BM25Document } from './core/vector/bm25.js';
+export { reciprocalRankFusion, type HybridScoredDocument } from './core/vector/hybrid-search.js';
+export { SecurityGuard } from './core/security/security-guard.js';
+export { InputSanitizer } from './core/security/sanitizer.js';
+export { InjectionDetector, type DetectionResult } from './core/security/injection-detector.js';
+export { PIIGuard, type PIIPolicy, type PIISanitizationResult } from './core/security/pii-guard.js';
+export { ConfirmationService, type ConfirmationTicketPayload } from './core/security/confirmation.js';
+export * from './core/queue/job-queue.interface.js';
+export { InMemoryJobQueue, type InMemoryQueueOptions } from './core/queue/in-memory-queue.js';
+export { BullMQJobQueue, type BullMQQueueOptions } from './core/queue/bullmq-queue.js';
+export { createJobQueue, type QueueConfig } from './core/queue/queue-factory.js';
+export { DocumentIngestionService, type DocumentUploadPayload, type StreamIngestionOptions, type IngestionJobResult, type DocumentIngestionConfig } from './services/document-ingestion.service.js';
+export { RedisCacheAdapter, type IRedisClientLike, type RedisCacheOptions } from './adapters/cache/redis-cache.adapter.js';
+export { createRagMiddleware } from './api/middleware.js';
+export { createApiRoutes, type ApiRouteOptions } from './api/routes.js';
+export { createAuthToken, verifyAuthToken, createAuthMiddleware, requireRoles, type AuthTokenPayload, type AuthMiddlewareOptions } from './api/auth.js';
+export { OpenAIAdapter } from './adapters/llm/openai.adapter.js';
+export { GeminiAdapter } from './adapters/llm/gemini.adapter.js';
+export { OpenRouterAdapter } from './adapters/llm/openrouter.adapter.js';
+export { CustomLLMAdapter, type CustomLLMConfig } from './adapters/llm/custom.adapter.js';
+export { OllamaAdapter, type OllamaConfig } from './adapters/llm/ollama.adapter.js';
+export { FallbackLLMProvider, type FallbackProviderOptions } from './adapters/llm/fallback.adapter.js';
+export { TieredLLMProvider, type TieredLLMConfig } from './adapters/llm/tiered.adapter.js';
+export { MemoryVectorAdapter } from './adapters/vector/memory-vector.adapter.js';
+export { StaticRulesAdapter } from './adapters/rules/static-rules.adapter.js';
+export { UrlFaqAdapter } from './adapters/faq/url-faq.adapter.js';
+export { LocalFaqAdapter, type LocalFaqAdapterOptions } from './adapters/faq/local-faq.adapter.js';
+export { RedisConfirmationNonceStore, MemoryConfirmationNonceStore, type ConfirmationNonceStore } from './core/security/nonce-store.js';
+export { PostgresVectorAdapter, type PostgresVectorOptions, type EmbeddingProvider } from './adapters/vector/postgres-vector.adapter.js';
+export { PostgresIngestionStore, IngestionQuotaError, IngestionConflictError, type StoredUpload } from './core/ingestion/postgres-ingestion-store.js';
+export { QueuedIngestionService, type QueuedIngestionOptions, type IngestionReference } from './core/ingestion/queued-ingestion.service.js';
+export { IngestionWorker } from './core/ingestion/ingestion-worker.js';
+export { createTenantRateLimiter } from './api/rate-limit.js';
+export type { IIngestionBackend, IngestionSettings } from './interfaces/ingestion.interface.js';