@denizsvnc/ai-rag-engine 2.2.2 → 2.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/dist/adapters/llm/gemini.adapter.d.ts +10 -1
- package/dist/adapters/llm/openai.adapter.d.ts +12 -1
- package/dist/adapters/llm/openrouter.adapter.d.ts +4 -2
- package/dist/api/server.js +247 -67
- package/dist/core/engine.d.ts +54 -2
- package/dist/core/metrics/rag-evaluator.d.ts +5 -0
- package/dist/core/sml-engine.d.ts +37 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +306 -76
- package/dist/interfaces/engine-config.interface.d.ts +106 -1
- package/dist/scripts/migrate-vector.js +64 -38
- package/dist/services/document-ingestion.service.d.ts +10 -3
- package/dist/worker.js +153 -50
- package/package.json +2 -1
|
@@ -11,17 +11,53 @@ export interface EmbeddingExtractor {
|
|
|
11
11
|
export declare class CircuitBreakerOpenError extends Error {
|
|
12
12
|
constructor(message: string);
|
|
13
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Industry-standard ONNX embedding model presets for the Small Machine Learning (SML) engine.
|
|
16
|
+
* Powered by `@huggingface/transformers` without requiring Python or external API keys.
|
|
17
|
+
*/
|
|
18
|
+
export declare const SMLModelPresets: {
|
|
19
|
+
/**
|
|
20
|
+
* Ultra-fast English embedding model (~30MB q8 ONNX).
|
|
21
|
+
* Ideal for English-only workloads, low-latency semantic routing, and minimal RAM footprint.
|
|
22
|
+
*/
|
|
23
|
+
readonly ENGLISH_FAST: 'Xenova/all-MiniLM-L6-v2';
|
|
24
|
+
/**
|
|
25
|
+
* Highly recommended balanced multilingual model (~117MB q8 ONNX).
|
|
26
|
+
* Provides exceptional semantic similarity across 50+ languages including Turkish, German, English, French, and Spanish.
|
|
27
|
+
*/
|
|
28
|
+
readonly MULTILINGUAL: 'Xenova/paraphrase-multilingual-MiniLM-L12-v2';
|
|
29
|
+
/**
|
|
30
|
+
* Heavyweight high-accuracy multilingual foundation model (1024-dim, long context, 100+ languages).
|
|
31
|
+
* Highest precision across mixed-language corpora, but requires higher memory and CPU allocation.
|
|
32
|
+
*/
|
|
33
|
+
readonly MULTILINGUAL_ACCURATE: 'Xenova/bge-m3';
|
|
34
|
+
};
|
|
35
|
+
export type SMLModelPreset = typeof SMLModelPresets[keyof typeof SMLModelPresets];
|
|
14
36
|
export interface SMLEngineConfig {
|
|
37
|
+
/** Directory where downloaded ONNX models are cached. Defaults to process.env.RAG_MODEL_CACHE_DIR or './data/models'. */
|
|
15
38
|
cacheDir?: string | undefined;
|
|
39
|
+
/** When true, only loads models already present on local disk and prevents external network fetching. */
|
|
16
40
|
localFilesOnly?: boolean | undefined;
|
|
17
|
-
|
|
41
|
+
/**
|
|
42
|
+
* Hugging Face ONNX model identifier.
|
|
43
|
+
* Use a preset (e.g. `SMLModelPresets.MULTILINGUAL`) or pass any custom HuggingFace model string (e.g. `'my-org/my-custom-model'`).
|
|
44
|
+
* Defaults to `'Xenova/all-MiniLM-L6-v2'`.
|
|
45
|
+
*/
|
|
46
|
+
modelName?: string | SMLModelPreset | undefined;
|
|
47
|
+
/** Maximum number of precomputed query/document embeddings retained in RAM LRU cache. Defaults to 1000. */
|
|
18
48
|
cacheSize?: number | undefined;
|
|
49
|
+
/** Model quantization data type: 'q8' (default), 'fp32', 'fp16', or 'q4'. */
|
|
19
50
|
dtype?: 'q8' | 'fp32' | 'fp16' | 'q4' | undefined;
|
|
51
|
+
/** Legacy flag for quantization: false forces 'fp32'. */
|
|
20
52
|
quantized?: boolean | undefined;
|
|
53
|
+
/** Number of consecutive load failures before tripping the circuit breaker OPEN. Defaults to 3. */
|
|
21
54
|
circuitBreakerThreshold?: number | undefined;
|
|
55
|
+
/** Cooldown period in milliseconds before re-testing model recovery. Defaults to 15000 (15s). */
|
|
22
56
|
circuitBreakerCooldownMs?: number | undefined;
|
|
23
57
|
}
|
|
24
58
|
export declare class SMLEngine {
|
|
59
|
+
private static initLockChain;
|
|
60
|
+
private static acquireInitLock;
|
|
25
61
|
private readonly modelName;
|
|
26
62
|
private readonly cacheDir;
|
|
27
63
|
private readonly localFilesOnly;
|
package/dist/index.d.ts
CHANGED
|
@@ -8,7 +8,7 @@ export * from './interfaces/cache.interface.js';
|
|
|
8
8
|
export * from './interfaces/llm.interface.js';
|
|
9
9
|
export * from './core/security/security.types.js';
|
|
10
10
|
export { RAGEngine } from './core/engine.js';
|
|
11
|
-
export { SMLEngine, type SMLEngineConfig, type EmbeddingExtractor, CircuitBreakerOpenError } from './core/sml-engine.js';
|
|
11
|
+
export { SMLEngine, SMLModelPresets, type SMLModelPreset, type SMLEngineConfig, type EmbeddingExtractor, CircuitBreakerOpenError } from './core/sml-engine.js';
|
|
12
12
|
export { ToolRegistry } from './core/tool-registry.js';
|
|
13
13
|
export { CacheManager } from './core/cache-manager.js';
|
|
14
14
|
export { PromptBuilder, type PromptContext } from './core/prompt-builder.js';
|
|
@@ -31,9 +31,9 @@ export { RedisCacheAdapter, type IRedisClientLike, type RedisCacheOptions } from
|
|
|
31
31
|
export { createRagMiddleware } from './api/middleware.js';
|
|
32
32
|
export { createApiRoutes, type ApiRouteOptions } from './api/routes.js';
|
|
33
33
|
export { createAuthToken, verifyAuthToken, createAuthMiddleware, requireRoles, type AuthTokenPayload, type AuthMiddlewareOptions } from './api/auth.js';
|
|
34
|
-
export { OpenAIAdapter } from './adapters/llm/openai.adapter.js';
|
|
35
|
-
export { GeminiAdapter } from './adapters/llm/gemini.adapter.js';
|
|
36
|
-
export { OpenRouterAdapter } from './adapters/llm/openrouter.adapter.js';
|
|
34
|
+
export { OpenAIAdapter, type OpenAIConfig } from './adapters/llm/openai.adapter.js';
|
|
35
|
+
export { GeminiAdapter, type GeminiConfig } from './adapters/llm/gemini.adapter.js';
|
|
36
|
+
export { OpenRouterAdapter, type OpenRouterConfig } from './adapters/llm/openrouter.adapter.js';
|
|
37
37
|
export { CustomLLMAdapter, type CustomLLMConfig } from './adapters/llm/custom.adapter.js';
|
|
38
38
|
export { OllamaAdapter, type OllamaConfig } from './adapters/llm/ollama.adapter.js';
|
|
39
39
|
export { FallbackLLMProvider, type FallbackProviderOptions } from './adapters/llm/fallback.adapter.js';
|
package/dist/index.js
CHANGED
|
@@ -164,7 +164,36 @@ var CircuitBreakerOpenError = class extends Error {
|
|
|
164
164
|
this.name = "CircuitBreakerOpenError";
|
|
165
165
|
}
|
|
166
166
|
};
|
|
167
|
-
var
|
|
167
|
+
var SMLModelPresets = {
|
|
168
|
+
/**
|
|
169
|
+
* Ultra-fast English embedding model (~30MB q8 ONNX).
|
|
170
|
+
* Ideal for English-only workloads, low-latency semantic routing, and minimal RAM footprint.
|
|
171
|
+
*/
|
|
172
|
+
ENGLISH_FAST: "Xenova/all-MiniLM-L6-v2",
|
|
173
|
+
/**
|
|
174
|
+
* Highly recommended balanced multilingual model (~117MB q8 ONNX).
|
|
175
|
+
* Provides exceptional semantic similarity across 50+ languages including Turkish, German, English, French, and Spanish.
|
|
176
|
+
*/
|
|
177
|
+
MULTILINGUAL: "Xenova/paraphrase-multilingual-MiniLM-L12-v2",
|
|
178
|
+
/**
|
|
179
|
+
* Heavyweight high-accuracy multilingual foundation model (1024-dim, long context, 100+ languages).
|
|
180
|
+
* Highest precision across mixed-language corpora, but requires higher memory and CPU allocation.
|
|
181
|
+
*/
|
|
182
|
+
MULTILINGUAL_ACCURATE: "Xenova/bge-m3"
|
|
183
|
+
};
|
|
184
|
+
var SMLEngine = class _SMLEngine {
|
|
185
|
+
static initLockChain = Promise.resolve();
|
|
186
|
+
static async acquireInitLock() {
|
|
187
|
+
let releaseLock = () => {
|
|
188
|
+
};
|
|
189
|
+
const nextLock = new Promise((resolve) => {
|
|
190
|
+
releaseLock = resolve;
|
|
191
|
+
});
|
|
192
|
+
const currentLock = _SMLEngine.initLockChain;
|
|
193
|
+
_SMLEngine.initLockChain = currentLock.then(() => nextLock);
|
|
194
|
+
await currentLock;
|
|
195
|
+
return releaseLock;
|
|
196
|
+
}
|
|
168
197
|
modelName;
|
|
169
198
|
cacheDir;
|
|
170
199
|
localFilesOnly;
|
|
@@ -224,48 +253,62 @@ var SMLEngine = class {
|
|
|
224
253
|
console.log(`[SML CircuitBreaker] Cooldown expired. Transitioned to HALF_OPEN for probe.`);
|
|
225
254
|
}
|
|
226
255
|
if (!this.extractorPromise) {
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
const previousCacheDir = env.cacheDir;
|
|
230
|
-
const previousAllowRemote = env.allowRemoteModels;
|
|
256
|
+
this.extractorPromise = (async () => {
|
|
257
|
+
const releaseLock = await _SMLEngine.acquireInitLock();
|
|
231
258
|
try {
|
|
232
|
-
|
|
233
|
-
|
|
259
|
+
console.log(`[SML] Loading ${this.modelName} (dtype: ${this.dtype})...`);
|
|
260
|
+
const { pipeline: pipeline2, env } = await import("@huggingface/transformers");
|
|
261
|
+
const previousCacheDir = env.cacheDir;
|
|
262
|
+
const previousAllowRemote = env.allowRemoteModels;
|
|
263
|
+
const previousLocalModelPath = env.localModelPath;
|
|
264
|
+
try {
|
|
265
|
+
if (this.cacheDir) {
|
|
266
|
+
env.cacheDir = this.cacheDir;
|
|
267
|
+
}
|
|
268
|
+
if (this.localFilesOnly !== void 0) {
|
|
269
|
+
env.allowRemoteModels = !this.localFilesOnly;
|
|
270
|
+
}
|
|
271
|
+
const pipelineOptions = {
|
|
272
|
+
dtype: this.dtype
|
|
273
|
+
};
|
|
274
|
+
if (this.cacheDir) {
|
|
275
|
+
pipelineOptions.cache_dir = this.cacheDir;
|
|
276
|
+
}
|
|
277
|
+
if (this.localFilesOnly !== void 0) {
|
|
278
|
+
pipelineOptions.local_files_only = this.localFilesOnly;
|
|
279
|
+
}
|
|
280
|
+
const pipelineInstance = await pipeline2("feature-extraction", this.modelName, pipelineOptions);
|
|
281
|
+
this.circuitState = "CLOSED";
|
|
282
|
+
this.failureCount = 0;
|
|
283
|
+
const extract = async (text, options) => {
|
|
284
|
+
const output = await pipelineInstance(text, options);
|
|
285
|
+
const values = output.data;
|
|
286
|
+
const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
|
|
287
|
+
return { data, dims: output.dims };
|
|
288
|
+
};
|
|
289
|
+
return extract;
|
|
290
|
+
} finally {
|
|
291
|
+
env.cacheDir = previousCacheDir;
|
|
292
|
+
env.allowRemoteModels = previousAllowRemote;
|
|
293
|
+
env.localModelPath = previousLocalModelPath;
|
|
234
294
|
}
|
|
235
|
-
|
|
236
|
-
|
|
295
|
+
} catch (err) {
|
|
296
|
+
this.failureCount++;
|
|
297
|
+
this.lastFailureTime = Date.now();
|
|
298
|
+
this.extractorPromise = null;
|
|
299
|
+
if (this.failureCount >= this.failureThreshold) {
|
|
300
|
+
this.circuitState = "OPEN";
|
|
301
|
+
console.error(
|
|
302
|
+
`[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
|
|
303
|
+
);
|
|
304
|
+
} else {
|
|
305
|
+
console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
|
|
237
306
|
}
|
|
238
|
-
|
|
239
|
-
dtype: this.dtype
|
|
240
|
-
});
|
|
307
|
+
throw err;
|
|
241
308
|
} finally {
|
|
242
|
-
|
|
243
|
-
env.allowRemoteModels = previousAllowRemote;
|
|
309
|
+
releaseLock();
|
|
244
310
|
}
|
|
245
|
-
})
|
|
246
|
-
this.circuitState = "CLOSED";
|
|
247
|
-
this.failureCount = 0;
|
|
248
|
-
const extract = async (text, options) => {
|
|
249
|
-
const output = await pipelineInstance(text, options);
|
|
250
|
-
const values = output.data;
|
|
251
|
-
const data = output.data instanceof Float32Array ? output.data : Float32Array.from({ length: values.length }, (_, index) => Number(values[index]));
|
|
252
|
-
return { data, dims: output.dims };
|
|
253
|
-
};
|
|
254
|
-
return extract;
|
|
255
|
-
}).catch((err) => {
|
|
256
|
-
this.failureCount++;
|
|
257
|
-
this.lastFailureTime = Date.now();
|
|
258
|
-
this.extractorPromise = null;
|
|
259
|
-
if (this.failureCount >= this.failureThreshold) {
|
|
260
|
-
this.circuitState = "OPEN";
|
|
261
|
-
console.error(
|
|
262
|
-
`[SML CircuitBreaker] \u26A0\uFE0F Tripped to OPEN state! Model failed ${this.failureCount} times. Pausing requests for ${this.cooldownMs / 1e3}s.`
|
|
263
|
-
);
|
|
264
|
-
} else {
|
|
265
|
-
console.error(`[SML] Model loading error (attempt ${this.failureCount}/${this.failureThreshold}):`, err);
|
|
266
|
-
}
|
|
267
|
-
throw err;
|
|
268
|
-
});
|
|
311
|
+
})();
|
|
269
312
|
}
|
|
270
313
|
return this.extractorPromise;
|
|
271
314
|
}
|
|
@@ -1207,6 +1250,8 @@ import fs3 from "node:fs";
|
|
|
1207
1250
|
import fsp from "node:fs/promises";
|
|
1208
1251
|
import path3 from "node:path";
|
|
1209
1252
|
import os from "node:os";
|
|
1253
|
+
import zlib from "node:zlib";
|
|
1254
|
+
import { StringDecoder } from "node:string_decoder";
|
|
1210
1255
|
|
|
1211
1256
|
// src/core/queue/in-memory-queue.ts
|
|
1212
1257
|
import { randomUUID } from "node:crypto";
|
|
@@ -1458,17 +1503,71 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1458
1503
|
* - Performs binary artifact suppression and multi-lingual UTF-8 normalization (Turkish, European, and CJK diacritics).
|
|
1459
1504
|
*/
|
|
1460
1505
|
static extractText(filename, buffer) {
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
return _DocumentIngestionService.extractPdfText(rawString);
|
|
1506
|
+
if (buffer.length >= 4 && buffer.subarray(0, 4).toString() === "%PDF") {
|
|
1507
|
+
return _DocumentIngestionService.extractPdfText(buffer);
|
|
1464
1508
|
}
|
|
1465
|
-
return
|
|
1509
|
+
return buffer.toString("utf-8").trim();
|
|
1466
1510
|
}
|
|
1467
1511
|
/**
|
|
1468
|
-
* Enterprise High-Efficiency PDF Lexical Extractor.
|
|
1469
|
-
*
|
|
1512
|
+
* Enterprise High-Efficiency PDF Lexical Extractor with native zlib.inflate decompression.
|
|
1513
|
+
* Decodes and parses PDF content streams without heavy external C++/WASM dependencies.
|
|
1514
|
+
*
|
|
1515
|
+
* Supports:
|
|
1516
|
+
* - Decompression of compressed stream objects (/Filter /FlateDecode) via Node.js native zlib.
|
|
1517
|
+
* - BT (Begin Text) ... ET (End Text) matrices.
|
|
1518
|
+
* - TJ array operators with kerning spacing preservation.
|
|
1519
|
+
* - Literal (Tj) and hexadecimal (<hex> Tj) text operators with escape and octal decoding.
|
|
1470
1520
|
*/
|
|
1471
1521
|
static extractPdfText(rawContent) {
|
|
1522
|
+
const buffer = Buffer.isBuffer(rawContent) ? rawContent : Buffer.from(rawContent, "latin1");
|
|
1523
|
+
const decompressedStreams = [];
|
|
1524
|
+
const streamMarker = Buffer.from("stream");
|
|
1525
|
+
const endStreamMarker = Buffer.from("endstream");
|
|
1526
|
+
let pos = 0;
|
|
1527
|
+
while (pos < buffer.length) {
|
|
1528
|
+
const streamIdx = buffer.indexOf(streamMarker, pos);
|
|
1529
|
+
if (streamIdx === -1) break;
|
|
1530
|
+
const prevChar = streamIdx > 0 ? buffer[streamIdx - 1] : 32;
|
|
1531
|
+
if (prevChar !== 10 && prevChar !== 13 && prevChar !== 32 && prevChar !== 62) {
|
|
1532
|
+
pos = streamIdx + 6;
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
let dataStart = streamIdx + 6;
|
|
1536
|
+
if (buffer[dataStart] === 13 && buffer[dataStart + 1] === 10) {
|
|
1537
|
+
dataStart += 2;
|
|
1538
|
+
} else if (buffer[dataStart] === 10 || buffer[dataStart] === 13) {
|
|
1539
|
+
dataStart += 1;
|
|
1540
|
+
}
|
|
1541
|
+
const endIdx = buffer.indexOf(endStreamMarker, dataStart);
|
|
1542
|
+
if (endIdx === -1) break;
|
|
1543
|
+
let dataEnd = endIdx;
|
|
1544
|
+
while (dataEnd > dataStart && (buffer[dataEnd - 1] === 10 || buffer[dataEnd - 1] === 13 || buffer[dataEnd - 1] === 32)) {
|
|
1545
|
+
dataEnd--;
|
|
1546
|
+
}
|
|
1547
|
+
const streamBytes = buffer.subarray(dataStart, dataEnd);
|
|
1548
|
+
const dictStart = Math.max(0, streamIdx - 500);
|
|
1549
|
+
const dictSnippet = buffer.subarray(dictStart, streamIdx).toString("latin1");
|
|
1550
|
+
const isFlate = /\/Filter\s*(?:\[\s*)?\/(?:FlateDecode|Fl)\b/.test(dictSnippet) || !/\/Filter\b/.test(dictSnippet);
|
|
1551
|
+
let decoded = "";
|
|
1552
|
+
if (isFlate && streamBytes.length > 0) {
|
|
1553
|
+
try {
|
|
1554
|
+
decoded = zlib.inflateSync(streamBytes).toString("utf-8");
|
|
1555
|
+
} catch {
|
|
1556
|
+
try {
|
|
1557
|
+
decoded = zlib.inflateRawSync(streamBytes).toString("utf-8");
|
|
1558
|
+
} catch {
|
|
1559
|
+
decoded = streamBytes.toString("utf-8");
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
} else {
|
|
1563
|
+
decoded = streamBytes.toString("utf-8");
|
|
1564
|
+
}
|
|
1565
|
+
if (decoded && decoded.trim().length > 0) {
|
|
1566
|
+
decompressedStreams.push(decoded);
|
|
1567
|
+
}
|
|
1568
|
+
pos = endIdx + 9;
|
|
1569
|
+
}
|
|
1570
|
+
const contentToScan = [buffer.toString("latin1"), ...decompressedStreams].join("\n");
|
|
1472
1571
|
const extractedSegments = [];
|
|
1473
1572
|
const decodePdfLiteral = (encoded) => {
|
|
1474
1573
|
return encoded.replace(/\\([0-7]{1,3})/g, (_, oct) => String.fromCharCode(parseInt(oct, 8))).replace(/\\n/g, "\n").replace(/\\r/g, "\r").replace(/\\t/g, " ").replace(/\\b/g, "\b").replace(/\\f/g, "\f").replace(/\\\(/g, "(").replace(/\\\)/g, ")").replace(/\\\\/g, "\\");
|
|
@@ -1483,7 +1582,7 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1483
1582
|
};
|
|
1484
1583
|
const btEtRegex = /BT[\s\S]*?ET/g;
|
|
1485
1584
|
let btMatch;
|
|
1486
|
-
while ((btMatch = btEtRegex.exec(
|
|
1585
|
+
while ((btMatch = btEtRegex.exec(contentToScan)) !== null) {
|
|
1487
1586
|
const blockContent = btMatch[0];
|
|
1488
1587
|
const tjArrayRegex = /\[([\s\S]*?)\]\s*TJ/g;
|
|
1489
1588
|
let tjArrMatch;
|
|
@@ -1528,7 +1627,7 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1528
1627
|
if (extractedSegments.length > 0) {
|
|
1529
1628
|
return extractedSegments.join(" ").replace(/\s+/g, " ").trim();
|
|
1530
1629
|
}
|
|
1531
|
-
const cleaned =
|
|
1630
|
+
const cleaned = contentToScan.replace(/%PDF-[^\n]+/g, "").replace(/stream[\s\S]*?endstream/g, (streamObj) => {
|
|
1532
1631
|
const innerMatch = streamObj.match(/\((.*?)\)/g);
|
|
1533
1632
|
return innerMatch ? innerMatch.map((s) => decodePdfLiteral(s.slice(1, -1))).join(" ") : "";
|
|
1534
1633
|
}).replace(/obj[\s\S]*?endobj/g, (obj) => {
|
|
@@ -1538,8 +1637,17 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1538
1637
|
return cleaned || "PDF document with no extractable plain text stream.";
|
|
1539
1638
|
}
|
|
1540
1639
|
static chunkText(text, chunkSize = 500, overlap = 50) {
|
|
1541
|
-
if (!
|
|
1542
|
-
|
|
1640
|
+
if (!Number.isFinite(chunkSize) || chunkSize <= 0) {
|
|
1641
|
+
throw new RangeError(`[DocumentIngestionService] chunkSize must be a positive integer greater than 0. Received: ${chunkSize}`);
|
|
1642
|
+
}
|
|
1643
|
+
if (!Number.isFinite(overlap) || overlap < 0 || overlap >= chunkSize) {
|
|
1644
|
+
throw new RangeError(`[DocumentIngestionService] overlap must be non-negative and strictly less than chunkSize (${chunkSize}). Received: ${overlap}`);
|
|
1645
|
+
}
|
|
1646
|
+
if (!text || text.trim().length === 0) {
|
|
1647
|
+
return [];
|
|
1648
|
+
}
|
|
1649
|
+
if (text.length <= chunkSize) {
|
|
1650
|
+
return [text.trim()];
|
|
1543
1651
|
}
|
|
1544
1652
|
const chunks = [];
|
|
1545
1653
|
let startIndex = 0;
|
|
@@ -1572,11 +1680,19 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1572
1680
|
* Streaming chunk generator.
|
|
1573
1681
|
* Ingests chunks of text or buffers incrementally with natural backpressure,
|
|
1574
1682
|
* maintaining a constant O(1) memory footprint regardless of file size.
|
|
1683
|
+
* Preserves UTF-8 multibyte character boundaries across stream buffer splits.
|
|
1575
1684
|
*/
|
|
1576
1685
|
static async *chunkStream(textStream, chunkSize = 500, overlap = 50) {
|
|
1686
|
+
if (!Number.isFinite(chunkSize) || chunkSize <= 0) {
|
|
1687
|
+
throw new RangeError(`[DocumentIngestionService] chunkSize must be a positive integer greater than 0. Received: ${chunkSize}`);
|
|
1688
|
+
}
|
|
1689
|
+
if (!Number.isFinite(overlap) || overlap < 0 || overlap >= chunkSize) {
|
|
1690
|
+
throw new RangeError(`[DocumentIngestionService] overlap must be non-negative and strictly less than chunkSize (${chunkSize}). Received: ${overlap}`);
|
|
1691
|
+
}
|
|
1692
|
+
const decoder = new StringDecoder("utf-8");
|
|
1577
1693
|
let window = "";
|
|
1578
1694
|
for await (const rawChunk of textStream) {
|
|
1579
|
-
const chunkStr = Buffer.isBuffer(rawChunk) ?
|
|
1695
|
+
const chunkStr = Buffer.isBuffer(rawChunk) ? decoder.write(rawChunk) : String(rawChunk);
|
|
1580
1696
|
window += chunkStr;
|
|
1581
1697
|
while (window.length >= chunkSize) {
|
|
1582
1698
|
let endIndex = chunkSize;
|
|
@@ -1594,9 +1710,13 @@ var DocumentIngestionService = class _DocumentIngestionService {
|
|
|
1594
1710
|
if (emitted.length > 0) {
|
|
1595
1711
|
yield emitted;
|
|
1596
1712
|
}
|
|
1597
|
-
window = window.slice(Math.max(
|
|
1713
|
+
window = window.slice(Math.max(0, endIndex - overlap));
|
|
1598
1714
|
}
|
|
1599
1715
|
}
|
|
1716
|
+
const flushed = decoder.end();
|
|
1717
|
+
if (flushed) {
|
|
1718
|
+
window += flushed;
|
|
1719
|
+
}
|
|
1600
1720
|
const remaining = window.trim();
|
|
1601
1721
|
if (remaining.length > 0) {
|
|
1602
1722
|
yield remaining;
|
|
@@ -2034,6 +2154,11 @@ var MemoryVectorAdapter = class {
|
|
|
2034
2154
|
|
|
2035
2155
|
// src/core/engine.ts
|
|
2036
2156
|
var RAGEngine = class {
|
|
2157
|
+
/**
|
|
2158
|
+
* Initializes a new instance of `RAGEngine` with the provided configuration.
|
|
2159
|
+
*
|
|
2160
|
+
* @param config - The engine configuration options (`EngineConfig`).
|
|
2161
|
+
*/
|
|
2037
2162
|
constructor(config) {
|
|
2038
2163
|
this.config = config;
|
|
2039
2164
|
if (config.tools?.items.some((tool) => tool.isMutating || tool.requiresConfirmation)) {
|
|
@@ -2049,7 +2174,15 @@ var RAGEngine = class {
|
|
|
2049
2174
|
config.cache?.ttlMs || 1e3 * 60 * 60 * 24
|
|
2050
2175
|
);
|
|
2051
2176
|
}
|
|
2052
|
-
this.smlEngine = new SMLEngine({
|
|
2177
|
+
this.smlEngine = new SMLEngine({
|
|
2178
|
+
modelName: config.smlConfig?.modelName,
|
|
2179
|
+
dtype: config.smlConfig?.dtype,
|
|
2180
|
+
cacheDir: config.smlConfig?.cacheDir,
|
|
2181
|
+
localFilesOnly: config.smlConfig?.localFilesOnly,
|
|
2182
|
+
cacheSize: config.smlConfig?.cacheSize,
|
|
2183
|
+
circuitBreakerThreshold: config.smlConfig?.circuitBreakerThreshold,
|
|
2184
|
+
circuitBreakerCooldownMs: config.smlConfig?.circuitBreakerCooldownMs
|
|
2185
|
+
});
|
|
2053
2186
|
const vectorStore = config.inputs?.vectorStore || new MemoryVectorAdapter(this.smlEngine);
|
|
2054
2187
|
if (typeof vectorStore.setSMLEngine === "function") {
|
|
2055
2188
|
vectorStore.setSMLEngine(this.smlEngine);
|
|
@@ -2154,8 +2287,27 @@ var RAGEngine = class {
|
|
|
2154
2287
|
await this.toolRegistry.registerTool(tool);
|
|
2155
2288
|
}
|
|
2156
2289
|
/**
|
|
2157
|
-
* Executes the query lifecycle:
|
|
2158
|
-
* Security
|
|
2290
|
+
* Executes the complete RAG query lifecycle:
|
|
2291
|
+
* 1. Security Inspection (Prompt Injection Defense, PII Masking)
|
|
2292
|
+
* 2. Cache Lookup (Role & Tenant isolated caching)
|
|
2293
|
+
* 3. SML Semantic Routing (FAQ matching and Tool execution)
|
|
2294
|
+
* 4. Hybrid Dense/Sparse Retrieval (Vector search + BM25 keyword search)
|
|
2295
|
+
* 5. Untrusted Boundary Framing (Anti-indirect injection prompt wrapping)
|
|
2296
|
+
* 6. LLM Answer Synthesis & Citation Tagging
|
|
2297
|
+
*
|
|
2298
|
+
* @param userPrompt - The natural language query or question from the user.
|
|
2299
|
+
* @param context - Optional multi-tenant security context including `tenantId`, `userId`, and `roles`.
|
|
2300
|
+
* @returns Promise resolving to `RAGResponse` containing synthesized answer, citations, intent, and metrics.
|
|
2301
|
+
*
|
|
2302
|
+
* @example
|
|
2303
|
+
* ```typescript
|
|
2304
|
+
* const response = await engine.query('What is our return policy?', {
|
|
2305
|
+
* tenantId: 'tenant_123',
|
|
2306
|
+
* userId: 'usr_456',
|
|
2307
|
+
* roles: ['customer']
|
|
2308
|
+
* });
|
|
2309
|
+
* console.log(response.answer);
|
|
2310
|
+
* ```
|
|
2159
2311
|
*/
|
|
2160
2312
|
async query(userPrompt, context) {
|
|
2161
2313
|
const startTime = Date.now();
|
|
@@ -2592,7 +2744,9 @@ var RAGEvaluator = class {
|
|
|
2592
2744
|
* Proportion of retrieved documents in the top-K that are genuinely relevant.
|
|
2593
2745
|
*/
|
|
2594
2746
|
static precisionAtK(retrievedIds, relevantIds, k = 3) {
|
|
2595
|
-
|
|
2747
|
+
if (!Number.isFinite(k) || k <= 0) return 0;
|
|
2748
|
+
const uniqueRetrieved = Array.from(new Set(retrievedIds));
|
|
2749
|
+
const topK = uniqueRetrieved.slice(0, k);
|
|
2596
2750
|
if (topK.length === 0) return 0;
|
|
2597
2751
|
const relevantSet = new Set(relevantIds);
|
|
2598
2752
|
const hits = topK.filter((id) => relevantSet.has(id)).length;
|
|
@@ -2604,7 +2758,9 @@ var RAGEvaluator = class {
|
|
|
2604
2758
|
*/
|
|
2605
2759
|
static recallAtK(retrievedIds, relevantIds, k = 3) {
|
|
2606
2760
|
if (relevantIds.length === 0) return 1;
|
|
2607
|
-
|
|
2761
|
+
if (!Number.isFinite(k) || k <= 0) return 0;
|
|
2762
|
+
const uniqueRetrieved = Array.from(new Set(retrievedIds));
|
|
2763
|
+
const topK = uniqueRetrieved.slice(0, k);
|
|
2608
2764
|
const relevantSet = new Set(relevantIds);
|
|
2609
2765
|
const hits = topK.filter((id) => relevantSet.has(id)).length;
|
|
2610
2766
|
return Number((hits / relevantIds.length).toFixed(4));
|
|
@@ -2638,28 +2794,44 @@ var RAGEvaluator = class {
|
|
|
2638
2794
|
groundingScore: res.score
|
|
2639
2795
|
};
|
|
2640
2796
|
}
|
|
2797
|
+
/**
|
|
2798
|
+
* Evaluates Answer Faithfulness / Groundedness using deterministic heuristics.
|
|
2799
|
+
* Note: This measures lexical consistency, numerical overlap, and negation polarity.
|
|
2800
|
+
* It serves as a fast in-process heuristic filter and is not an external semantic LLM judge.
|
|
2801
|
+
*/
|
|
2641
2802
|
static evaluateFaithfulness(answer, contextTexts) {
|
|
2803
|
+
if (!answer || answer.trim().length === 0) {
|
|
2804
|
+
return {
|
|
2805
|
+
score: 0,
|
|
2806
|
+
lexicalGroundingScore: 0,
|
|
2807
|
+
totalSentences: 0,
|
|
2808
|
+
groundedSentences: 0,
|
|
2809
|
+
contradictionsDetected: [],
|
|
2810
|
+
ungroundedClaims: ["Empty answer provided"]
|
|
2811
|
+
};
|
|
2812
|
+
}
|
|
2642
2813
|
const combinedContext = contextTexts.join(" ").toLowerCase();
|
|
2643
2814
|
const contextClean = combinedContext.replace(/[^\p{L}\p{N}\s]/gu, " ");
|
|
2644
2815
|
const contextWordTokens = new Set(contextClean.split(/\s+/).filter((w) => w.length > 2));
|
|
2645
2816
|
const contextNumbers = new Set(
|
|
2646
2817
|
(combinedContext.match(/\b\d+(?:[\.,]\d+)?\b/g) || []).map((n) => n.replace(",", "."))
|
|
2647
2818
|
);
|
|
2648
|
-
const sentences = answer.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter((s) => s.length >
|
|
2819
|
+
const sentences = answer.split(/(?<=[.!?])\s+/).map((s) => s.trim()).filter((s) => s.length > 0);
|
|
2649
2820
|
if (sentences.length === 0) {
|
|
2650
2821
|
return {
|
|
2651
|
-
score:
|
|
2652
|
-
lexicalGroundingScore:
|
|
2822
|
+
score: 0,
|
|
2823
|
+
lexicalGroundingScore: 0,
|
|
2653
2824
|
totalSentences: 0,
|
|
2654
2825
|
groundedSentences: 0,
|
|
2655
2826
|
contradictionsDetected: [],
|
|
2656
|
-
ungroundedClaims: []
|
|
2827
|
+
ungroundedClaims: ["No meaningful sentences detected"]
|
|
2657
2828
|
};
|
|
2658
2829
|
}
|
|
2659
2830
|
const ungroundedClaims = [];
|
|
2660
2831
|
const contradictionsDetected = [];
|
|
2661
2832
|
let groundedCount = 0;
|
|
2662
2833
|
let totalLexicalOverlapSum = 0;
|
|
2834
|
+
const negationTokens = /* @__PURE__ */ new Set(["not", "never", "cannot", "de\u011Fil", "yok", "olumsuz", "asla", "hay\u0131r"]);
|
|
2663
2835
|
for (const sentence of sentences) {
|
|
2664
2836
|
const lowerSentence = sentence.toLowerCase();
|
|
2665
2837
|
const sentenceNumbers = (lowerSentence.match(/\b\d+(?:[\.,]\d+)?\b/g) || []).map((n) => n.replace(",", "."));
|
|
@@ -2673,16 +2845,28 @@ var RAGEvaluator = class {
|
|
|
2673
2845
|
break;
|
|
2674
2846
|
}
|
|
2675
2847
|
}
|
|
2676
|
-
const
|
|
2848
|
+
const rawWords = lowerSentence.replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((w) => w.length > 0);
|
|
2849
|
+
const sentenceNegations = rawWords.filter((w) => negationTokens.has(w));
|
|
2850
|
+
let hasNegationMismatch = false;
|
|
2851
|
+
if (sentenceNegations.length > 0) {
|
|
2852
|
+
const contextHasNegation = sentenceNegations.some((neg) => contextWordTokens.has(neg) || combinedContext.includes(neg));
|
|
2853
|
+
if (!contextHasNegation) {
|
|
2854
|
+
hasNegationMismatch = true;
|
|
2855
|
+
contradictionsDetected.push(
|
|
2856
|
+
`Polarity Warning: Negation '${sentenceNegations.join(", ")}' in "${sentence}" absent from retrieved context.`
|
|
2857
|
+
);
|
|
2858
|
+
}
|
|
2859
|
+
}
|
|
2860
|
+
const words = rawWords.filter((w) => w.length > 2);
|
|
2677
2861
|
if (words.length === 0) {
|
|
2678
|
-
if (!hasNumberContradiction) groundedCount++;
|
|
2862
|
+
if (!hasNumberContradiction && !hasNegationMismatch) groundedCount++;
|
|
2679
2863
|
totalLexicalOverlapSum += 1;
|
|
2680
2864
|
continue;
|
|
2681
2865
|
}
|
|
2682
2866
|
const groundedWords = words.filter((word) => contextWordTokens.has(word) || combinedContext.includes(word));
|
|
2683
2867
|
const overlapRatio = groundedWords.length / words.length;
|
|
2684
2868
|
totalLexicalOverlapSum += overlapRatio;
|
|
2685
|
-
if (overlapRatio >= 0.4 && !hasNumberContradiction) {
|
|
2869
|
+
if (overlapRatio >= 0.4 && !hasNumberContradiction && !hasNegationMismatch) {
|
|
2686
2870
|
groundedCount++;
|
|
2687
2871
|
} else {
|
|
2688
2872
|
ungroundedClaims.push(sentence);
|
|
@@ -3413,13 +3597,29 @@ var OpenAIAdapter = class _OpenAIAdapter {
|
|
|
3413
3597
|
apiKey;
|
|
3414
3598
|
model;
|
|
3415
3599
|
endpoint;
|
|
3416
|
-
|
|
3417
|
-
|
|
3600
|
+
temperature;
|
|
3601
|
+
constructor(configOrApiKey, model, endpoint) {
|
|
3602
|
+
let key = "";
|
|
3603
|
+
let resolvedModel = "";
|
|
3604
|
+
let resolvedEndpoint = "";
|
|
3605
|
+
let temp = 0.1;
|
|
3606
|
+
if (typeof configOrApiKey === "object" && configOrApiKey !== null) {
|
|
3607
|
+
key = configOrApiKey.apiKey || process.env.OPENAI_API_KEY || "";
|
|
3608
|
+
resolvedModel = configOrApiKey.model || process.env.OPENAI_MODEL || process.env.OPENAI_MODEL_FAST || "gpt-4o-mini";
|
|
3609
|
+
resolvedEndpoint = configOrApiKey.endpoint || process.env.OPENAI_ENDPOINT || "https://api.openai.com/v1/chat/completions";
|
|
3610
|
+
if (configOrApiKey.temperature !== void 0) temp = configOrApiKey.temperature;
|
|
3611
|
+
} else {
|
|
3612
|
+
key = configOrApiKey || process.env.OPENAI_API_KEY || "";
|
|
3613
|
+
resolvedModel = model || process.env.OPENAI_MODEL || process.env.OPENAI_MODEL_FAST || "gpt-4o-mini";
|
|
3614
|
+
resolvedEndpoint = endpoint || process.env.OPENAI_ENDPOINT || "https://api.openai.com/v1/chat/completions";
|
|
3615
|
+
}
|
|
3616
|
+
this.apiKey = key;
|
|
3418
3617
|
if (!this.apiKey) {
|
|
3419
3618
|
throw new Error("[OpenAIAdapter] API Key not found. Provide it in constructor or set OPENAI_API_KEY in .env");
|
|
3420
3619
|
}
|
|
3421
|
-
this.model =
|
|
3422
|
-
this.endpoint =
|
|
3620
|
+
this.model = resolvedModel;
|
|
3621
|
+
this.endpoint = resolvedEndpoint;
|
|
3622
|
+
this.temperature = temp;
|
|
3423
3623
|
}
|
|
3424
3624
|
static createFast(apiKey) {
|
|
3425
3625
|
return new _OpenAIAdapter(apiKey, process.env.OPENAI_MODEL_FAST || "gpt-4o-mini");
|
|
@@ -3435,7 +3635,7 @@ var OpenAIAdapter = class _OpenAIAdapter {
|
|
|
3435
3635
|
const body = {
|
|
3436
3636
|
model: this.model,
|
|
3437
3637
|
messages,
|
|
3438
|
-
temperature:
|
|
3638
|
+
temperature: this.temperature
|
|
3439
3639
|
// Low temperature to reduce hallucination in RAG pipelines
|
|
3440
3640
|
};
|
|
3441
3641
|
if (request.tools && request.tools.length > 0) {
|
|
@@ -3485,15 +3685,28 @@ var OpenAIAdapter = class _OpenAIAdapter {
|
|
|
3485
3685
|
var GeminiAdapter = class _GeminiAdapter {
|
|
3486
3686
|
apiKey;
|
|
3487
3687
|
model;
|
|
3488
|
-
|
|
3489
|
-
|
|
3688
|
+
temperature;
|
|
3689
|
+
constructor(configOrApiKey, model) {
|
|
3690
|
+
let key = "";
|
|
3691
|
+
let resolvedModel = "";
|
|
3692
|
+
let temp = 0.1;
|
|
3693
|
+
if (typeof configOrApiKey === "object" && configOrApiKey !== null) {
|
|
3694
|
+
key = configOrApiKey.apiKey || process.env.GEMINI_API_KEY || "";
|
|
3695
|
+
resolvedModel = configOrApiKey.model || process.env.GEMINI_MODEL || process.env.GEMINI_MODEL_FAST || "gemini-2.0-flash";
|
|
3696
|
+
if (configOrApiKey.temperature !== void 0) temp = configOrApiKey.temperature;
|
|
3697
|
+
} else {
|
|
3698
|
+
key = configOrApiKey || process.env.GEMINI_API_KEY || "";
|
|
3699
|
+
resolvedModel = model || process.env.GEMINI_MODEL || process.env.GEMINI_MODEL_FAST || "gemini-2.0-flash";
|
|
3700
|
+
}
|
|
3701
|
+
this.apiKey = key;
|
|
3490
3702
|
if (!this.apiKey) {
|
|
3491
3703
|
throw new Error("[GeminiAdapter] API Key not found. Provide it in constructor or set GEMINI_API_KEY in .env");
|
|
3492
3704
|
}
|
|
3493
|
-
this.model =
|
|
3705
|
+
this.model = resolvedModel;
|
|
3706
|
+
this.temperature = temp;
|
|
3494
3707
|
}
|
|
3495
3708
|
static createFast(apiKey) {
|
|
3496
|
-
return new _GeminiAdapter(apiKey, process.env.GEMINI_MODEL_FAST || "gemini-
|
|
3709
|
+
return new _GeminiAdapter(apiKey, process.env.GEMINI_MODEL_FAST || "gemini-2.0-flash");
|
|
3497
3710
|
}
|
|
3498
3711
|
static createSmart(apiKey) {
|
|
3499
3712
|
return new _GeminiAdapter(apiKey, process.env.GEMINI_MODEL_SMART || "gemini-1.5-pro");
|
|
@@ -3508,7 +3721,7 @@ var GeminiAdapter = class _GeminiAdapter {
|
|
|
3508
3721
|
}
|
|
3509
3722
|
],
|
|
3510
3723
|
generationConfig: {
|
|
3511
|
-
temperature:
|
|
3724
|
+
temperature: this.temperature
|
|
3512
3725
|
// Low temperature to reduce hallucination in RAG pipelines
|
|
3513
3726
|
}
|
|
3514
3727
|
};
|
|
@@ -3572,18 +3785,34 @@ var GeminiAdapter = class _GeminiAdapter {
|
|
|
3572
3785
|
|
|
3573
3786
|
// src/adapters/llm/openrouter.adapter.ts
|
|
3574
3787
|
var OpenRouterAdapter = class _OpenRouterAdapter extends OpenAIAdapter {
|
|
3575
|
-
constructor(
|
|
3788
|
+
constructor(configOrApiKey, model, endpoint = "https://openrouter.ai/api/v1/chat/completions") {
|
|
3576
3789
|
if (process.env.ALLOW_PUBLIC_OPENSOURCE_LLM === "false") {
|
|
3577
3790
|
throw new Error(
|
|
3578
3791
|
"[OpenRouterAdapter] Blocked by security policy: ALLOW_PUBLIC_OPENSOURCE_LLM is set to false in .env. Corporate prompts cannot be routed to public external open-source models."
|
|
3579
3792
|
);
|
|
3580
3793
|
}
|
|
3581
|
-
|
|
3794
|
+
let key = "";
|
|
3795
|
+
let resolvedModel = "";
|
|
3796
|
+
let resolvedEndpoint = endpoint;
|
|
3797
|
+
let temp = 0.1;
|
|
3798
|
+
if (typeof configOrApiKey === "object" && configOrApiKey !== null) {
|
|
3799
|
+
key = configOrApiKey.apiKey || process.env.OPENROUTER_API_KEY || "";
|
|
3800
|
+
resolvedModel = configOrApiKey.model || process.env.OPENROUTER_MODEL || process.env.OPENROUTER_MODEL_FAST || "meta-llama/llama-3.2-3b-instruct:free";
|
|
3801
|
+
if (configOrApiKey.endpoint) resolvedEndpoint = configOrApiKey.endpoint;
|
|
3802
|
+
if (configOrApiKey.temperature !== void 0) temp = configOrApiKey.temperature;
|
|
3803
|
+
} else {
|
|
3804
|
+
key = configOrApiKey || process.env.OPENROUTER_API_KEY || "";
|
|
3805
|
+
resolvedModel = model || process.env.OPENROUTER_MODEL || process.env.OPENROUTER_MODEL_FAST || "meta-llama/llama-3.2-3b-instruct:free";
|
|
3806
|
+
}
|
|
3582
3807
|
if (!key) {
|
|
3583
3808
|
throw new Error("[OpenRouterAdapter] API Key not found. Provide it in constructor or set OPENROUTER_API_KEY in .env");
|
|
3584
3809
|
}
|
|
3585
|
-
|
|
3586
|
-
|
|
3810
|
+
super({
|
|
3811
|
+
apiKey: key,
|
|
3812
|
+
model: resolvedModel,
|
|
3813
|
+
endpoint: resolvedEndpoint,
|
|
3814
|
+
temperature: temp
|
|
3815
|
+
});
|
|
3587
3816
|
}
|
|
3588
3817
|
static createFast(apiKey) {
|
|
3589
3818
|
return new _OpenRouterAdapter(apiKey, process.env.OPENROUTER_MODEL_FAST || "meta-llama/llama-3.2-3b-instruct:free");
|
|
@@ -4539,6 +4768,7 @@ export {
|
|
|
4539
4768
|
RedisCacheAdapter,
|
|
4540
4769
|
RedisConfirmationNonceStore,
|
|
4541
4770
|
SMLEngine,
|
|
4771
|
+
SMLModelPresets,
|
|
4542
4772
|
SecurityGuard,
|
|
4543
4773
|
StaticRulesAdapter,
|
|
4544
4774
|
TieredLLMProvider,
|