@denizsvnc/ai-rag-engine 2.2.1 → 2.2.3

package/dist/interfaces/engine-config.interface.d.ts

@@ -9,65 +9,170 @@ import type { IAgentTool } from './tool.interface.js';
 import type { ILLMProvider } from './llm.interface.js';
 import type { SecurityConfig } from '../core/security/security.types.js';
 import type { ResponseMetrics, TokenTrackerOptions } from '../core/metrics/token-tracker.js';
+import type { SMLModelPreset } from '../core/sml-engine.js';
+/**
+ * Enterprise configuration options for initializing the `RAGEngine`.
+ *
+ * @example
+ * ```typescript
+ * const engine = new RAGEngine({
+ *   llmProvider: new GeminiAdapter({ apiKey: process.env.GEMINI_API_KEY, model: 'gemini-1.5-pro' }),
+ *   inputs: {
+ *     vectorStore: new PostgresVectorAdapter({ connectionString: process.env.DATABASE_URL })
+ *   },
+ *   smlConfig: {
+ *     enabled: true,
+ *     modelName: SMLModelPresets.MULTILINGUAL, // 'Xenova/paraphrase-multilingual-MiniLM-L12-v2'
+ *     matchThreshold: 0.65
+ *   }
+ * });
+ * ```
+ */
 export interface EngineConfig {
+    /**
+     * Primary LLM Provider used for synthesizing final RAG answers.
+     * Can be `GeminiAdapter`, `OpenAIAdapter`, `OllamaAdapter`, `TieredLLMProvider`, or `FallbackLLMProvider`.
+     */
     llmProvider: ILLMProvider;
+    /**
+     * Cryptographic mutation confirmation settings for sensitive/destructive tool executions.
+     */
     confirmation?: {
+        /** HMAC secret used to sign confirmation tickets. */
         secret?: string | undefined;
+        /** Single-use nonce store preventing replay attacks. */
         nonceStore?: ConfirmationNonceStore | undefined;
     } | undefined;
+    /**
+     * Enterprise Zero-Trust security guardrails: prompt injection detection, PII redaction (TCKN, IBAN, Credit Card), and boundary isolation.
+     */
     security?: SecurityConfig | undefined;
+    /**
+     * Local Small Machine Learning (SML) engine configuration for sub-millisecond semantic routing, FAQ answering, and tool dispatching.
+     * Runs locally in-process with zero external API calls.
+     */
     smlConfig?: {
+        /** Whether local semantic routing and embedding computation is enabled. Defaults to true. */
         enabled: boolean;
-        modelName?: string | undefined;
+        /**
+         * Hugging Face ONNX embedding model or preset.
+         * Presets: `SMLModelPresets.ENGLISH_FAST`, `SMLModelPresets.MULTILINGUAL`, or `SMLModelPresets.MULTILINGUAL_ACCURATE`.
+         * Can also be any custom HuggingFace model string (e.g. `'BAAI/bge-m3'`, `'intfloat/multilingual-e5-small'`).
+         * Defaults to `'Xenova/all-MiniLM-L6-v2'`.
+         */
+        modelName?: string | SMLModelPreset | undefined;
+        /** Minimum cosine similarity threshold (0.0 - 1.0) required to trigger FAQ matches or tool routing. Defaults to 0.65. */
         matchThreshold?: number | undefined;
+        /** Model tensor quantization: 'q8' (default), 'fp32', 'fp16', or 'q4'. */
+        dtype?: 'q8' | 'fp32' | 'fp16' | 'q4' | undefined;
+        /** Directory where downloaded ONNX models are stored. Defaults to process.env.RAG_MODEL_CACHE_DIR or './data/models'. */
+        cacheDir?: string | undefined;
+        /** When true, only loads models already downloaded to disk. Defaults to false. */
+        localFilesOnly?: boolean | undefined;
+        /** Maximum number of embeddings held in RAM LRU cache. Defaults to 1000. */
+        cacheSize?: number | undefined;
+        /** Consecutive failures before tripping the circuit breaker OPEN. Defaults to 3. */
+        circuitBreakerThreshold?: number | undefined;
+        /** Cooldown period in ms before attempting recovery from OPEN circuit. Defaults to 15000 (15s). */
+        circuitBreakerCooldownMs?: number | undefined;
     } | undefined;
+    /**
+     * Knowledge layers: vector store for unstructured documents, static business rules, and high-frequency FAQ entries.
+     */
     inputs: {
+        /** Vector store adapter (`MemoryVectorAdapter` or `PostgresVectorAdapter`). */
         vectorStore?: IVectorStoreAdapter | undefined;
+        /** Business rules adapter (`StaticRulesAdapter`). */
         rules?: IRulesAdapter | undefined;
+        /** Fast semantic FAQ adapter (`LocalFaqAdapter` or `UrlFaqAdapter`). */
         faq?: IFAQAdapter | undefined;
     };
+    /**
+     * Governed agent tools for live tool execution and dynamic function calling.
+     */
     tools?: {
+        /** Path to persist computed tool embeddings (e.g. './data/tool-registry.json'). */
         registryFilePath: string;
+        /** Array of governed agent tools conforming to `IAgentTool`. */
         items: IAgentTool[];
     } | undefined;
+    /**
+     * Multi-tiered distributed caching configuration to serve repeated queries with sub-millisecond response time.
+     */
     cache?: {
+        /** Enable or disable semantic query caching. */
         enabled: boolean;
+        /** Distributed cache adapter (`RedisCacheAdapter`). If omitted, uses fast in-memory LRU cache. */
         adapter?: ICacheAdapter | undefined;
+        /** File path for persisting cache to disk across server restarts. */
         filePath?: string | undefined;
+        /** Minimum frequency threshold before a response is persisted to disk. */
         minHitsToPersist?: number | undefined;
+        /** Time-to-live for cached queries in milliseconds. Defaults to 24 hours. */
         ttlMs?: number | undefined;
+        /** Time-to-live for cached tool execution results in milliseconds. */
         toolResultTtlMs?: number | undefined;
+        /** Whether to cache tool execution results. */
         cacheToolResults?: boolean | undefined;
     } | undefined;
+    /**
+     * Hybrid search and document retrieval parameters.
+     */
     retrieval?: {
+        /** Search strategy: 'dense' (embeddings only), 'sparse' (BM25 keyword only), or 'hybrid' (RRF fusion). Defaults to 'hybrid'. */
         strategy?: 'dense' | 'sparse' | 'hybrid' | undefined;
+        /** Maximum number of top relevant document chunks passed to the LLM prompt. Defaults to 5. */
         topK?: number | undefined;
+        /** Minimum relevance score filter. */
         minRelevanceScore?: number | undefined;
     } | undefined;
+    /**
+     * FinOps telemetry options for token usage estimation and latency tracking.
+     */
     metrics?: TokenTrackerOptions | undefined;
+    /**
+     * Durable background ingestion pipeline settings for PDF, Markdown, and text streams.
+     */
     ingestion?: {
+        /** Storage backend for uploaded documents (`PostgresIngestionStore`). */
         backend?: IIngestionBackend | undefined;
+        /** Target text chunk length in characters. Defaults to 500. */
         chunkSize?: number | undefined;
+        /** Sliding window overlap between chunks in characters. Defaults to 50. */
         chunkOverlap?: number | undefined;
+        /** Background queue engine (`BullMQQueue` or `InMemoryJobQueue`). */
         jobQueue?: IJobQueue | undefined;
     } | undefined;
 }
+/**
+ * Standardized response object returned by `engine.query()`.
+ */
 export interface RAGResponse {
+    /** The final synthesized answer from LLM, FAQ match, or cached response. */
     answer: string;
+    /** Resolved query intent: 'CACHE' | 'FAQ' | 'TOOL_CALL' | 'RAG_QUERY' | 'SECURITY_BLOCKED'. */
     intent: string;
+    /** The component that resolved the request (e.g. 'InMemoryCache', 'LocalFaqAdapter', 'GeminiAdapter'). */
     resolvedBy: string;
+    /** Document IDs and titles retrieved from the vector store and cited in the answer. */
     sources?: string[] | undefined;
+    /** Active business rules enforced during answer synthesis. */
     appliedRules?: string[] | undefined;
+    /** Tools that were executed during the query lifecycle. */
     executedTools?: Array<{
         name: string;
         args: any;
     }> | undefined;
+    /** Pending cryptographic confirmation tickets for mutating tool operations. */
     confirmationRequests?: Array<{
         toolName: string;
         proposedArgs: unknown;
         confirmationTicket: string;
     }> | undefined;
+    /** Security alerts (e.g. prompt injection warnings, masked PII types). */
     securityWarning?: string | undefined;
+    /** Total round-trip execution latency in milliseconds. */
     latencyMs: number;
+    /** Token consumption and FinOps cost metrics. */
     metrics?: ResponseMetrics | undefined;
 }

package/dist/scripts/migrate-vector.js

@@ -29,7 +29,19 @@ var CircuitBreakerOpenError = class extends Error {
     this.name = "CircuitBreakerOpenError";
   }
 };
-var SMLEngine = class {
+var SMLEngine = class _SMLEngine {
+  static initLockChain = Promise.resolve();
+  static async acquireInitLock() {
+    let releaseLock = () => {
+    };
+    const nextLock = new Promise((resolve) => {
+      releaseLock = resolve;
+    });
+    const currentLock = _SMLEngine.initLockChain;
+    _SMLEngine.initLockChain = currentLock.then(() => nextLock);
+    await currentLock;
+    return releaseLock;
+  }
   modelName;
   cacheDir;
   localFilesOnly;
@@ -89,48 +101,62 @@ var SMLEngine = class {
       console.log(`[SML CircuitBreaker] Cooldown expired. Transitioned to HALF_OPEN for probe.`);
     }
     if (!this.extractorPromise) {
-      console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
-      this.extractorPromise = import("@huggingface/transformers").then(async ({ pipeline, env }) => {
-        const previousCacheDir = env.cacheDir;
-        const previousAllowRemote = env.allowRemoteModels;
+      this.extractorPromise = (async () => {
+        const releaseLock = await _SMLEngine.acquireInitLock();
         try {
-          if (this.cacheDir) {
-            env.cacheDir = this.cacheDir;
+          console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
+          const { pipeline, env } = await import("@huggingface/transformers");
+          const previousCacheDir = env.cacheDir;
+          const previousAllowRemote = env.allowRemoteModels;
+          const previousLocalModelPath = env.localModelPath;
+          try {
+            if (this.cacheDir) {
+              env.cacheDir = this.cacheDir;
+            }
+            if (this.localFilesOnly !== void 0) {
+              env.allowRemoteModels = !this.localFilesOnly;
+            }
+            const pipelineOptions = {
+              dtype: this.dtype
+            };
+            if (this.cacheDir) {
+              pipelineOptions.cache_dir = this.cacheDir;
+            }
+            if (this.localFilesOnly !== void 0) {
+              pipelineOptions.local_files_only = this.localFilesOnly;
+            }
+            const pipelineInstance = await pipeline("feature-extraction", this.modelName, pipelineOptions);
+            this.circuitState = "CLOSED";
+            this.failureCount = 0;
+            const extract = async (text, options) => {
+              const output = await pipelineInstance(text, options);
+              const values = output.data;
+              const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
+              return { data, dims: output.dims };
+            };
+            return extract;
+          } finally {
+            env.cacheDir = previousCacheDir;
+            env.allowRemoteModels = previousAllowRemote;
+            env.localModelPath = previousLocalModelPath;
           }
-          if (this.localFilesOnly) {
-            env.allowRemoteModels = false;
+        } catch (err) {
+          this.failureCount++;
+          this.lastFailureTime = Date.now();
+          this.extractorPromise = null;
+          if (this.failureCount >= this.failureThreshold) {
+            this.circuitState = "OPEN";
+            console.error(
+              `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
+            );
+          } else {
+            console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
           }
-          return await pipeline("feature-extraction", this.modelName, {
-            dtype: this.dtype
-          });
+          throw err;
         } finally {
-          env.cacheDir = previousCacheDir;
-          env.allowRemoteModels = previousAllowRemote;
+          releaseLock();
         }
-      }).then((pipelineInstance) => {
-        this.circuitState = "CLOSED";
-        this.failureCount = 0;
-        const extract = async (text, options) => {
-          const output = await pipelineInstance(text, options);
-          const values = output.data;
-          const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
-          return { data, dims: output.dims };
-        };
-        return extract;
-      }).catch((err) => {
-        this.failureCount++;
-        this.lastFailureTime = Date.now();
-        this.extractorPromise = null;
-        if (this.failureCount >= this.failureThreshold) {
-          this.circuitState = "OPEN";
-          console.error(
-            `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
-          );
-        } else {
-          console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
-        }
-        throw err;
-      });
+      })();
     }
     return this.extractorPromise;
   }

package/dist/services/document-ingestion.service.d.ts

@@ -74,15 +74,22 @@ export declare class DocumentIngestionService {
      */
     static extractText(filename: string, buffer: Buffer): string;
     /**
-     * Enterprise High-Efficiency PDF Lexical Extractor.
-     * Parses PDF content streams without heavy external C++/WASM dependencies.
+     * Enterprise High-Efficiency PDF Lexical Extractor with native zlib.inflate decompression.
+     * Decodes and parses PDF content streams without heavy external C++/WASM dependencies.
+     *
+     * Supports:
+     * - Decompression of compressed stream objects (/Filter /FlateDecode) via Node.js native zlib.
+     * - BT (Begin Text) ... ET (End Text) matrices.
+     * - TJ array operators with kerning spacing preservation.
+     * - Literal (Tj) and hexadecimal (<hex> Tj) text operators with escape and octal decoding.
      */
-    static extractPdfText(rawContent: string): string;
+    static extractPdfText(rawContent: Buffer | string): string;
     static chunkText(text: string, chunkSize?: number, overlap?: number): string[];
     /**
      * Streaming chunk generator.
      * Ingests chunks of text or buffers incrementally with natural backpressure,
      * maintaining a constant O(1) memory footprint regardless of file size.
+     * Preserves UTF-8 multibyte character boundaries across stream buffer splits.
      */
     static chunkStream(textStream: AsyncIterable<string | Buffer>, chunkSize?: number, overlap?: number): AsyncGenerator<string, void, unknown>;
     /**

package/dist/worker.js

@@ -31,7 +31,19 @@ var CircuitBreakerOpenError = class extends Error {
     this.name = "CircuitBreakerOpenError";
   }
 };
-var SMLEngine = class {
+var SMLEngine = class _SMLEngine {
+  static initLockChain = Promise.resolve();
+  static async acquireInitLock() {
+    let releaseLock = () => {
+    };
+    const nextLock = new Promise((resolve) => {
+      releaseLock = resolve;
+    });
+    const currentLock = _SMLEngine.initLockChain;
+    _SMLEngine.initLockChain = currentLock.then(() => nextLock);
+    await currentLock;
+    return releaseLock;
+  }
   modelName;
   cacheDir;
   localFilesOnly;
@@ -91,48 +103,62 @@ var SMLEngine = class {
       console.log(`[SML CircuitBreaker] Cooldown expired. Transitioned to HALF_OPEN for probe.`);
     }
     if (!this.extractorPromise) {
-      console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
-      this.extractorPromise = import("@huggingface/transformers").then(async ({ pipeline: pipeline2, env }) => {
-        const previousCacheDir = env.cacheDir;
-        const previousAllowRemote = env.allowRemoteModels;
+      this.extractorPromise = (async () => {
+        const releaseLock = await _SMLEngine.acquireInitLock();
         try {
-          if (this.cacheDir) {
-            env.cacheDir = this.cacheDir;
+          console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
+          const { pipeline: pipeline2, env } = await import("@huggingface/transformers");
+          const previousCacheDir = env.cacheDir;
+          const previousAllowRemote = env.allowRemoteModels;
+          const previousLocalModelPath = env.localModelPath;
+          try {
+            if (this.cacheDir) {
+              env.cacheDir = this.cacheDir;
+            }
+            if (this.localFilesOnly !== void 0) {
+              env.allowRemoteModels = !this.localFilesOnly;
+            }
+            const pipelineOptions = {
+              dtype: this.dtype
+            };
+            if (this.cacheDir) {
+              pipelineOptions.cache_dir = this.cacheDir;
+            }
+            if (this.localFilesOnly !== void 0) {
+              pipelineOptions.local_files_only = this.localFilesOnly;
+            }
+            const pipelineInstance = await pipeline2("feature-extraction", this.modelName, pipelineOptions);
+            this.circuitState = "CLOSED";
+            this.failureCount = 0;
+            const extract = async (text, options) => {
+              const output = await pipelineInstance(text, options);
+              const values = output.data;
+              const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
+              return { data, dims: output.dims };
+            };
+            return extract;
+          } finally {
+            env.cacheDir = previousCacheDir;
+            env.allowRemoteModels = previousAllowRemote;
+            env.localModelPath = previousLocalModelPath;
           }
-          if (this.localFilesOnly) {
-            env.allowRemoteModels = false;
+        } catch (err) {
+          this.failureCount++;
+          this.lastFailureTime = Date.now();
+          this.extractorPromise = null;
+          if (this.failureCount >= this.failureThreshold) {
+            this.circuitState = "OPEN";
+            console.error(
+              `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
+            );
+          } else {
+            console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
           }
-          return await pipeline2("feature-extraction", this.modelName, {
-            dtype: this.dtype
-          });
+          throw err;
         } finally {
-          env.cacheDir = previousCacheDir;
-          env.allowRemoteModels = previousAllowRemote;
-        }
-      }).then((pipelineInstance) => {
-        this.circuitState = "CLOSED";
-        this.failureCount = 0;
-        const extract = async (text, options) => {
-          const output = await pipelineInstance(text, options);
-          const values = output.data;
-          const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
-          return { data, dims: output.dims };
-        };
-        return extract;
-      }).catch((err) => {
-        this.failureCount++;
-        this.lastFailureTime = Date.now();
-        this.extractorPromise = null;
-        if (this.failureCount >= this.failureThreshold) {
-          this.circuitState = "OPEN";
-          console.error(
-            `[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
-          );
-        } else {
-          console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
+          releaseLock();
         }
-        throw err;
-      });
+      })();
     }
     return this.extractorPromise;
   }
@@ -708,6 +734,8 @@ import fs from "node:fs";
 import fsp from "node:fs/promises";
 import path from "node:path";
 import os from "node:os";
+import zlib from "node:zlib";
+import { StringDecoder } from "node:string_decoder";
 
 // src/core/security/pii-guard.ts
 var PIIGuard = class {
@@ -1134,17 +1162,71 @@ var DocumentIngestionService = class _DocumentIngestionService {
    * - Performs binary artifact suppression and multi-lingual UTF-8 normalization (Turkish, European, and CJK diacritics).
    */
   static extractText(filename, buffer) {
-    const rawString = buffer.toString("utf-8");
-    if (buffer.length >= 4 && buffer.slice(0, 4).toString() === "%PDF") {
-      return _DocumentIngestionService.extractPdfText(rawString);
+    if (buffer.length >= 4 && buffer.subarray(0, 4).toString() === "%PDF") {
+      return _DocumentIngestionService.extractPdfText(buffer);
     }
-    return rawString.trim();
+    return buffer.toString("utf-8").trim();
   }
   /**
-   * Enterprise High-Efficiency PDF Lexical Extractor.
-   * Parses PDF content streams without heavy external C++/WASM dependencies.
+   * Enterprise High-Efficiency PDF Lexical Extractor with native zlib.inflate decompression.
+   * Decodes and parses PDF content streams without heavy external C++/WASM dependencies.
+   * 
+   * Supports:
+   * - Decompression of compressed stream objects (/Filter /FlateDecode) via Node.js native zlib.
+   * - BT (Begin Text) ... ET (End Text) matrices.
+   * - TJ array operators with kerning spacing preservation.
+   * - Literal (Tj) and hexadecimal (<hex> Tj) text operators with escape and octal decoding.
    */
   static extractPdfText(rawContent) {
+    const buffer = Buffer.isBuffer(rawContent) ? rawContent : Buffer.from(rawContent, "latin1");
+    const decompressedStreams = [];
+    const streamMarker = Buffer.from("stream");
+    const endStreamMarker = Buffer.from("endstream");
+    let pos = 0;
+    while (pos < buffer.length) {
+      const streamIdx = buffer.indexOf(streamMarker, pos);
+      if (streamIdx === -1) break;
+      const prevChar = streamIdx > 0 ? buffer[streamIdx - 1] : 32;
+      if (prevChar !== 10 && prevChar !== 13 && prevChar !== 32 && prevChar !== 62) {
+        pos = streamIdx + 6;
+        continue;
+      }
+      let dataStart = streamIdx + 6;
+      if (buffer[dataStart] === 13 && buffer[dataStart + 1] === 10) {
+        dataStart += 2;
+      } else if (buffer[dataStart] === 10 || buffer[dataStart] === 13) {
+        dataStart += 1;
+      }
+      const endIdx = buffer.indexOf(endStreamMarker, dataStart);
+      if (endIdx === -1) break;
+      let dataEnd = endIdx;
+      while (dataEnd > dataStart && (buffer[dataEnd - 1] === 10 || buffer[dataEnd - 1] === 13 || buffer[dataEnd - 1] === 32)) {
+        dataEnd--;
+      }
+      const streamBytes = buffer.subarray(dataStart, dataEnd);
+      const dictStart = Math.max(0, streamIdx - 500);
+      const dictSnippet = buffer.subarray(dictStart, streamIdx).toString("latin1");
+      const isFlate = /\/Filter\s*(?:\[\s*)?\/(?:FlateDecode|Fl)\b/.test(dictSnippet) || !/\/Filter\b/.test(dictSnippet);
+      let decoded = "";
+      if (isFlate && streamBytes.length > 0) {
+        try {
+          decoded = zlib.inflateSync(streamBytes).toString("utf-8");
+        } catch {
+          try {
+            decoded = zlib.inflateRawSync(streamBytes).toString("utf-8");
+          } catch {
+            decoded = streamBytes.toString("utf-8");
+          }
+        }
+      } else {
+        decoded = streamBytes.toString("utf-8");
+      }
+      if (decoded && decoded.trim().length > 0) {
+        decompressedStreams.push(decoded);
+      }
+      pos = endIdx + 9;
+    }
+    const contentToScan = [buffer.toString("latin1"), ...decompressedStreams].join("\n");
     const extractedSegments = [];
     const decodePdfLiteral = (encoded) => {
       return encoded.replace(/\\([0-7]{1,3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8))).replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, "	").replace(/\\b/g, "\b").replace(/\\f/g, "\f").replace(/\\\(/g, "(").replace(/\\\)/g, ")").replace(/\\\\/g, "\\");
@@ -1159,7 +1241,7 @@ var DocumentIngestionService = class _DocumentIngestionService {
     };
     const btEtRegex = /BT[\s\S]*?ET/g;
     let btMatch;
-    while ((btMatch = btEtRegex.exec(rawContent)) !== null) {
+    while ((btMatch = btEtRegex.exec(contentToScan)) !== null) {
       const blockContent = btMatch[0];
       const tjArrayRegex = /\[([\s\S]*?)\]\s*TJ/g;
       let tjArrMatch;
@@ -1204,7 +1286,7 @@ var DocumentIngestionService = class _DocumentIngestionService {
     if (extractedSegments.length > 0) {
       return extractedSegments.join(" ").replace(/\s+/g, " ").trim();
     }
-    const cleaned = rawContent.replace(/%PDF-[^\n]+/g, "").replace(/stream[\s\S]*?endstream/g, (streamObj) => {
+    const cleaned = contentToScan.replace(/%PDF-[^\n]+/g, "").replace(/stream[\s\S]*?endstream/g, (streamObj) => {
       const innerMatch = streamObj.match(/\((.*?)\)/g);
       return innerMatch ? innerMatch.map((s) => decodePdfLiteral(s.slice(1, -1))).join(" ") : "";
     }).replace(/obj[\s\S]*?endobj/g, (obj) => {
@@ -1214,8 +1296,17 @@ var DocumentIngestionService = class _DocumentIngestionService {
     return cleaned || "PDF document with no extractable plain text stream.";
   }
   static chunkText(text, chunkSize = 500, overlap = 50) {
-    if (!text || text.length <= chunkSize) {
-      return text ? [text] : [];
+    if (!Number.isFinite(chunkSize) || chunkSize <= 0) {
+      throw new RangeError(`[DocumentIngestionService] chunkSize must be a positive integer greater than 0. Received: ${chunkSize}`);
+    }
+    if (!Number.isFinite(overlap) || overlap < 0 || overlap >= chunkSize) {
+      throw new RangeError(`[DocumentIngestionService] overlap must be non-negative and strictly less than chunkSize (${chunkSize}). Received: ${overlap}`);
+    }
+    if (!text || text.trim().length === 0) {
+      return [];
+    }
+    if (text.length <= chunkSize) {
+      return [text.trim()];
     }
     const chunks = [];
     let startIndex = 0;
@@ -1237,6 +1328,9 @@ var DocumentIngestionService = class _DocumentIngestionService {
       if (chunk.length > 0) {
         chunks.push(chunk);
       }
+      if (endIndex >= text.length) {
+        break;
+      }
       startIndex = Math.max(startIndex + 1, endIndex - overlap);
     }
     return chunks;
@@ -1245,11 +1339,19 @@ var DocumentIngestionService = class _DocumentIngestionService {
    * Streaming chunk generator.
    * Ingests chunks of text or buffers incrementally with natural backpressure,
    * maintaining a constant O(1) memory footprint regardless of file size.
+   * Preserves UTF-8 multibyte character boundaries across stream buffer splits.
    */
   static async *chunkStream(textStream, chunkSize = 500, overlap = 50) {
+    if (!Number.isFinite(chunkSize) || chunkSize <= 0) {
+      throw new RangeError(`[DocumentIngestionService] chunkSize must be a positive integer greater than 0. Received: ${chunkSize}`);
+    }
+    if (!Number.isFinite(overlap) || overlap < 0 || overlap >= chunkSize) {
+      throw new RangeError(`[DocumentIngestionService] overlap must be non-negative and strictly less than chunkSize (${chunkSize}). Received: ${overlap}`);
+    }
+    const decoder = new StringDecoder("utf-8");
     let window = "";
     for await (const rawChunk of textStream) {
-      const chunkStr = Buffer.isBuffer(rawChunk) ? rawChunk.toString("utf-8") : String(rawChunk);
+      const chunkStr = Buffer.isBuffer(rawChunk) ? decoder.write(rawChunk) : String(rawChunk);
       window += chunkStr;
       while (window.length >= chunkSize) {
         let endIndex = chunkSize;
@@ -1267,9 +1369,13 @@ var DocumentIngestionService = class _DocumentIngestionService {
         if (emitted.length > 0) {
           yield emitted;
         }
-        window = window.slice(Math.max(1, endIndex - overlap));
+        window = window.slice(Math.max(0, endIndex - overlap));
       }
     }
+    const flushed = decoder.end();
+    if (flushed) {
+      window += flushed;
+    }
     const remaining = window.trim();
     if (remaining.length > 0) {
       yield remaining;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@denizsvnc/ai-rag-engine",
-  "version": "2.2.1",
+  "version": "2.2.3",
   "description": "TypeScript RAG engine with pluggable providers, local semantic routing, tenant-aware retrieval, and durable document ingestion.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -38,6 +38,7 @@
     "test:consumer": "node test/package/run.mjs",
     "clean": "node tools/clean.mjs",
     "test:consumer:integration": "node test/consumer/run.mjs",
+    "test:bun": "bun test test/bun/bun-compat.test.ts",
     "test:package": "npm run build && node tools/verify-package.mjs"
   },
   "keywords": [