@retrivora-ai/rag-engine 0.4.5 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -57
- package/dist/{ChromaDBProvider-GI7TB7GJ.mjs → ChromaDBProvider-MIDOR4FW.mjs} +2 -2
- package/dist/{DocumentChunker-3yElxTO3.d.mts → DocumentChunker-C-sCZPhi.d.mts} +6 -6
- package/dist/{DocumentChunker-3yElxTO3.d.ts → DocumentChunker-C-sCZPhi.d.ts} +6 -6
- package/dist/{MilvusProvider-WDVTFB7D.mjs → MilvusProvider-U7SKC27V.mjs} +2 -2
- package/dist/{MongoDBProvider-RE3Q5S5B.mjs → MongoDBProvider-Z6ALOVDN.mjs} +2 -2
- package/dist/{PineconeProvider-BE2JWSPD.mjs → PineconeProvider-QZNRKTN2.mjs} +2 -2
- package/dist/{PostgreSQLProvider-5HHTK4SU.mjs → PostgreSQLProvider-BMOETDZA.mjs} +2 -2
- package/dist/{QdrantProvider-XVDVBNIG.mjs → QdrantProvider-YNUNEOZH.mjs} +2 -2
- package/dist/{RagConfig-BgRDL9Vy.d.mts → RagConfig-DRJO4hGU.d.mts} +12 -1
- package/dist/{RagConfig-BgRDL9Vy.d.ts → RagConfig-DRJO4hGU.d.ts} +12 -1
- package/dist/{RedisProvider-EK2R2PQH.mjs → RedisProvider-SR65SCKV.mjs} +2 -2
- package/dist/{SimpleGraphProvider-M6T7SE7D.mjs → SimpleGraphProvider-SLOXO4M7.mjs} +1 -1
- package/dist/{UniversalVectorProvider-YIDRX6VT.mjs → UniversalVectorProvider-IN67OS56.mjs} +3 -3
- package/dist/{WeaviateProvider-4CAPQ7UY.mjs → WeaviateProvider-5FWDFITI.mjs} +2 -2
- package/dist/{chunk-5KNBWQM6.mjs → chunk-3DSHW676.mjs} +5 -1
- package/dist/{chunk-PRC5CZIZ.mjs → chunk-5W2YWFT3.mjs} +1248 -1361
- package/dist/{chunk-H6RKMU7W.mjs → chunk-5YGUXK7Z.mjs} +1 -1
- package/dist/{chunk-PQKTC73Y.mjs → chunk-CD6TSNL4.mjs} +67 -6
- package/dist/{chunk-RK2UDJA2.mjs → chunk-CFVEZTBJ.mjs} +1 -1
- package/dist/{chunk-LJWWPTWE.mjs → chunk-FLOSGE6A.mjs} +76 -14
- package/dist/{chunk-GQT5LF4G.mjs → chunk-LR3VMDVK.mjs} +2 -2
- package/dist/{chunk-KTS3LLHY.mjs → chunk-M6JSPGAR.mjs} +5 -5
- package/dist/{chunk-3QWAK3RZ.mjs → chunk-U55XRW3U.mjs} +6 -2
- package/dist/{chunk-XCNXPECE.mjs → chunk-VUQJVIJT.mjs} +55 -1
- package/dist/chunk-X4TOT24V.mjs +89 -0
- package/dist/{chunk-EDLTMSNY.mjs → chunk-YLTMFW4M.mjs} +1 -1
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +1488 -1371
- package/dist/handlers/index.mjs +5 -3
- package/dist/index-B2mutkgp.d.ts +116 -0
- package/dist/index-Bjy0es5a.d.mts +116 -0
- package/dist/index.d.mts +17 -11
- package/dist/index.d.ts +17 -11
- package/dist/index.js +253 -363
- package/dist/index.mjs +243 -353
- package/dist/server.d.mts +108 -158
- package/dist/server.d.ts +108 -158
- package/dist/server.js +1452 -1386
- package/dist/server.mjs +12 -12
- package/package.json +5 -1
- package/src/config/RagConfig.ts +7 -0
- package/src/core/ConfigValidator.ts +66 -492
- package/src/core/LangChainAgent.ts +78 -0
- package/src/core/Pipeline.ts +210 -240
- package/src/core/ProviderHealthCheck.ts +35 -406
- package/src/core/ProviderInterfaces.ts +37 -0
- package/src/core/ProviderRegistry.ts +70 -55
- package/src/core/QueryProcessor.ts +173 -0
- package/src/core/VectorPlugin.ts +7 -0
- package/src/handlers/index.ts +45 -0
- package/src/llm/ILLMProvider.ts +10 -0
- package/src/llm/LLMFactory.ts +33 -13
- package/src/llm/providers/AnthropicProvider.ts +55 -15
- package/src/llm/providers/GeminiProvider.ts +51 -0
- package/src/llm/providers/OllamaProvider.ts +100 -15
- package/src/llm/providers/OpenAIProvider.ts +60 -11
- package/src/providers/vectordb/BaseVectorProvider.ts +11 -0
- package/src/providers/vectordb/MilvusProvider.ts +4 -0
- package/src/providers/vectordb/MongoDBProvider.ts +72 -8
- package/src/providers/vectordb/PineconeProvider.ts +60 -5
- package/src/providers/vectordb/PostgreSQLProvider.ts +84 -14
- package/src/providers/vectordb/QdrantProvider.ts +4 -0
- package/src/providers/vectordb/WeaviateProvider.ts +8 -4
- package/src/rag/DocumentChunker.ts +15 -19
- package/src/rag/EntityExtractor.ts +3 -0
- package/src/rag/LlamaIndexIngestor.ts +61 -0
- package/src/rag/Reranker.ts +20 -0
- package/src/server.ts +1 -1
- package/src/types/index.ts +9 -0
- package/dist/chunk-FWCSY2DS.mjs +0 -37
- package/dist/index-7qeLTPBL.d.mts +0 -114
- package/dist/index-DowY4_K0.d.ts +0 -114
|
@@ -2,11 +2,15 @@ import {
|
|
|
2
2
|
buildPayload,
|
|
3
3
|
mergeDefined,
|
|
4
4
|
resolvePath
|
|
5
|
-
} from "./chunk-
|
|
5
|
+
} from "./chunk-YLTMFW4M.mjs";
|
|
6
6
|
import {
|
|
7
|
+
__asyncGenerator,
|
|
8
|
+
__await,
|
|
9
|
+
__forAwait,
|
|
7
10
|
__spreadProps,
|
|
8
|
-
__spreadValues
|
|
9
|
-
|
|
11
|
+
__spreadValues,
|
|
12
|
+
__yieldStar
|
|
13
|
+
} from "./chunk-X4TOT24V.mjs";
|
|
10
14
|
|
|
11
15
|
// src/handlers/index.ts
|
|
12
16
|
import { NextResponse } from "next/server";
|
|
@@ -50,7 +54,6 @@ var PROVIDERS_WITH_EMBEDDINGS = [
|
|
|
50
54
|
"rest",
|
|
51
55
|
"universal_rest"
|
|
52
56
|
];
|
|
53
|
-
var UI_VISUAL_STYLES = ["glass", "solid"];
|
|
54
57
|
var UI_BORDER_RADIUS_OPTIONS = ["none", "sm", "md", "lg", "xl", "full"];
|
|
55
58
|
|
|
56
59
|
// src/config/serverConfig.ts
|
|
@@ -228,657 +231,68 @@ var ConfigResolver = class {
|
|
|
228
231
|
}
|
|
229
232
|
};
|
|
230
233
|
|
|
231
|
-
// src/
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
if (!config.projectId) {
|
|
240
|
-
errors.push({
|
|
241
|
-
field: "projectId",
|
|
242
|
-
message: "projectId is required",
|
|
243
|
-
severity: "error"
|
|
244
|
-
});
|
|
245
|
-
}
|
|
246
|
-
errors.push(...this.validateVectorDbConfig(config.vectorDb));
|
|
247
|
-
errors.push(...this.validateLLMConfig(config.llm));
|
|
248
|
-
if (config.embedding) {
|
|
249
|
-
errors.push(...this.validateEmbeddingConfig(config.embedding));
|
|
250
|
-
} else if (config.llm.provider === "anthropic") {
|
|
251
|
-
errors.push({
|
|
252
|
-
field: "embedding",
|
|
253
|
-
message: "Embedding config is required when using Anthropic LLM",
|
|
254
|
-
suggestion: 'Provide embedding config with provider (e.g., "openai" or "ollama")',
|
|
255
|
-
severity: "error"
|
|
256
|
-
});
|
|
257
|
-
}
|
|
258
|
-
if (config.ui) {
|
|
259
|
-
errors.push(...this.validateUIConfig(config.ui));
|
|
260
|
-
}
|
|
261
|
-
if (config.rag) {
|
|
262
|
-
errors.push(...this.validateRAGConfig(config.rag));
|
|
263
|
-
}
|
|
264
|
-
return errors;
|
|
265
|
-
}
|
|
266
|
-
static validateVectorDbConfig(config) {
|
|
267
|
-
const errors = [];
|
|
268
|
-
if (!config.provider) {
|
|
269
|
-
errors.push({
|
|
270
|
-
field: "vectorDb.provider",
|
|
271
|
-
message: "Vector database provider is required",
|
|
272
|
-
severity: "error"
|
|
273
|
-
});
|
|
274
|
-
return errors;
|
|
275
|
-
}
|
|
276
|
-
if (!config.indexName) {
|
|
277
|
-
errors.push({
|
|
278
|
-
field: "vectorDb.indexName",
|
|
279
|
-
message: "Vector database index name is required",
|
|
280
|
-
severity: "error"
|
|
281
|
-
});
|
|
282
|
-
}
|
|
283
|
-
switch (config.provider) {
|
|
284
|
-
case "pinecone":
|
|
285
|
-
errors.push(...this.validatePineconeConfig(config));
|
|
286
|
-
break;
|
|
287
|
-
case "pgvector":
|
|
288
|
-
case "postgresql":
|
|
289
|
-
errors.push(...this.validatePostgresConfig(config));
|
|
290
|
-
break;
|
|
291
|
-
case "mongodb":
|
|
292
|
-
errors.push(...this.validateMongoDBConfig(config));
|
|
293
|
-
break;
|
|
294
|
-
case "milvus":
|
|
295
|
-
errors.push(...this.validateMilvusConfig(config));
|
|
296
|
-
break;
|
|
297
|
-
case "qdrant":
|
|
298
|
-
errors.push(...this.validateQdrantConfig(config));
|
|
299
|
-
break;
|
|
300
|
-
case "chromadb":
|
|
301
|
-
errors.push(...this.validateChromaDBConfig(config));
|
|
302
|
-
break;
|
|
303
|
-
case "redis":
|
|
304
|
-
errors.push(...this.validateRedisConfig(config));
|
|
305
|
-
break;
|
|
306
|
-
case "weaviate":
|
|
307
|
-
errors.push(...this.validateWeaviateConfig(config));
|
|
308
|
-
break;
|
|
309
|
-
case "universal_rest":
|
|
310
|
-
case "rest":
|
|
311
|
-
errors.push(...this.validateRestConfig(config));
|
|
312
|
-
break;
|
|
313
|
-
}
|
|
314
|
-
return errors;
|
|
315
|
-
}
|
|
316
|
-
/**
|
|
317
|
-
* Pinecone — only needs apiKey (environment was removed in SDK v7+).
|
|
318
|
-
* Options key: { apiKey: string }
|
|
319
|
-
*/
|
|
320
|
-
static validatePineconeConfig(config) {
|
|
321
|
-
const errors = [];
|
|
322
|
-
const opts = config.options;
|
|
323
|
-
if (!opts.apiKey) {
|
|
324
|
-
errors.push({
|
|
325
|
-
field: "vectorDb.options.apiKey",
|
|
326
|
-
message: "Pinecone API key is required",
|
|
327
|
-
suggestion: "Set PINECONE_API_KEY environment variable",
|
|
328
|
-
severity: "error"
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
return errors;
|
|
332
|
-
}
|
|
333
|
-
/**
|
|
334
|
-
* PostgreSQL / pgvector — needs a connection string.
|
|
335
|
-
* Options key: { connectionString: string }
|
|
336
|
-
*/
|
|
337
|
-
static validatePostgresConfig(config) {
|
|
338
|
-
const errors = [];
|
|
339
|
-
const opts = config.options;
|
|
340
|
-
if (!opts.connectionString) {
|
|
341
|
-
errors.push({
|
|
342
|
-
field: "vectorDb.options.connectionString",
|
|
343
|
-
message: "PostgreSQL connection string is required",
|
|
344
|
-
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
345
|
-
severity: "error"
|
|
346
|
-
});
|
|
347
|
-
}
|
|
348
|
-
if (opts.tables && typeof opts.tables !== "string" && !Array.isArray(opts.tables)) {
|
|
349
|
-
errors.push({
|
|
350
|
-
field: "vectorDb.options.tables",
|
|
351
|
-
message: "PostgreSQL tables must be a string or a string array",
|
|
352
|
-
severity: "error"
|
|
353
|
-
});
|
|
354
|
-
}
|
|
355
|
-
if (opts.searchFields && typeof opts.searchFields !== "string" && !Array.isArray(opts.searchFields)) {
|
|
356
|
-
errors.push({
|
|
357
|
-
field: "vectorDb.options.searchFields",
|
|
358
|
-
message: "PostgreSQL searchFields must be a string or a string array",
|
|
359
|
-
severity: "error"
|
|
360
|
-
});
|
|
361
|
-
}
|
|
362
|
-
return errors;
|
|
363
|
-
}
|
|
364
|
-
/**
|
|
365
|
-
* MongoDB — needs uri, database, and collection.
|
|
366
|
-
* Options keys: { uri: string, database: string, collection: string }
|
|
367
|
-
*/
|
|
368
|
-
static validateMongoDBConfig(config) {
|
|
369
|
-
const errors = [];
|
|
370
|
-
const opts = config.options;
|
|
371
|
-
if (!opts.uri) {
|
|
372
|
-
errors.push({
|
|
373
|
-
field: "vectorDb.options.uri",
|
|
374
|
-
message: "MongoDB connection URI is required",
|
|
375
|
-
suggestion: "Set MONGODB_URI environment variable",
|
|
376
|
-
severity: "error"
|
|
377
|
-
});
|
|
378
|
-
}
|
|
379
|
-
if (!opts.database) {
|
|
380
|
-
errors.push({
|
|
381
|
-
field: "vectorDb.options.database",
|
|
382
|
-
message: "MongoDB database name is required",
|
|
383
|
-
suggestion: "Set MONGODB_DB environment variable",
|
|
384
|
-
severity: "error"
|
|
385
|
-
});
|
|
386
|
-
}
|
|
387
|
-
if (!opts.collection) {
|
|
388
|
-
errors.push({
|
|
389
|
-
field: "vectorDb.options.collection",
|
|
390
|
-
message: "MongoDB collection name is required",
|
|
391
|
-
suggestion: "Set MONGODB_COLLECTION environment variable",
|
|
392
|
-
severity: "error"
|
|
393
|
-
});
|
|
394
|
-
}
|
|
395
|
-
return errors;
|
|
396
|
-
}
|
|
397
|
-
/**
|
|
398
|
-
* Milvus — accepts baseUrl OR uri (preferred) OR host+port.
|
|
399
|
-
* MilvusProvider reads opts.baseUrl ?? opts.uri.
|
|
400
|
-
* Options keys: { baseUrl?: string, uri?: string, host?: string, port?: number }
|
|
401
|
-
*/
|
|
402
|
-
static validateMilvusConfig(config) {
|
|
403
|
-
const errors = [];
|
|
404
|
-
const opts = config.options;
|
|
405
|
-
const hasBaseUrl = Boolean(opts.baseUrl || opts.uri);
|
|
406
|
-
const hasHostPort = Boolean(opts.host && opts.port);
|
|
407
|
-
if (!hasBaseUrl && !hasHostPort) {
|
|
408
|
-
errors.push({
|
|
409
|
-
field: "vectorDb.options.baseUrl",
|
|
410
|
-
message: 'Milvus connection is required: provide baseUrl (e.g., "http://localhost:19530") or host + port',
|
|
411
|
-
suggestion: "Set MILVUS_URL environment variable",
|
|
412
|
-
severity: "error"
|
|
413
|
-
});
|
|
414
|
-
}
|
|
415
|
-
return errors;
|
|
416
|
-
}
|
|
417
|
-
/**
|
|
418
|
-
* Qdrant — needs baseUrl (or url alias).
|
|
419
|
-
* Options keys: { baseUrl: string, apiKey?: string }
|
|
420
|
-
*/
|
|
421
|
-
static validateQdrantConfig(config) {
|
|
422
|
-
const errors = [];
|
|
423
|
-
const opts = config.options;
|
|
424
|
-
if (!opts.baseUrl && !opts.url) {
|
|
425
|
-
errors.push({
|
|
426
|
-
field: "vectorDb.options.baseUrl",
|
|
427
|
-
message: "Qdrant base URL is required",
|
|
428
|
-
suggestion: 'Set QDRANT_URL environment variable (e.g., "http://localhost:6333")',
|
|
429
|
-
severity: "error"
|
|
430
|
-
});
|
|
431
|
-
}
|
|
432
|
-
return errors;
|
|
433
|
-
}
|
|
434
|
-
/**
|
|
435
|
-
* ChromaDB — accepts baseUrl OR host.
|
|
436
|
-
* ChromaDBProvider reads opts.baseUrl.
|
|
437
|
-
* Options keys: { baseUrl?: string, host?: string, port?: number }
|
|
438
|
-
*/
|
|
439
|
-
static validateChromaDBConfig(config) {
|
|
440
|
-
const errors = [];
|
|
441
|
-
const opts = config.options;
|
|
442
|
-
if (!opts.baseUrl && !opts.host) {
|
|
443
|
-
errors.push({
|
|
444
|
-
field: "vectorDb.options.baseUrl",
|
|
445
|
-
message: 'ChromaDB connection is required: provide baseUrl (e.g., "http://localhost:8000") or host',
|
|
446
|
-
suggestion: "Set CHROMADB_URL environment variable",
|
|
447
|
-
severity: "error"
|
|
448
|
-
});
|
|
449
|
-
}
|
|
450
|
-
return errors;
|
|
451
|
-
}
|
|
452
|
-
/**
|
|
453
|
-
* Redis — accepts baseUrl OR url OR host+port.
|
|
454
|
-
* RedisProvider reads opts.baseUrl.
|
|
455
|
-
* Options keys: { baseUrl?: string, url?: string, host?: string, port?: number, apiKey?: string }
|
|
456
|
-
*/
|
|
457
|
-
static validateRedisConfig(config) {
|
|
458
|
-
const errors = [];
|
|
459
|
-
const opts = config.options;
|
|
460
|
-
const hasUrl = Boolean(opts.baseUrl || opts.url);
|
|
461
|
-
const hasHostPort = Boolean(opts.host && opts.port);
|
|
462
|
-
if (!hasUrl && !hasHostPort) {
|
|
463
|
-
errors.push({
|
|
464
|
-
field: "vectorDb.options.baseUrl",
|
|
465
|
-
message: "Redis connection is required: provide baseUrl/url or host + port",
|
|
466
|
-
suggestion: "Set REDIS_URL environment variable",
|
|
467
|
-
severity: "error"
|
|
468
|
-
});
|
|
469
|
-
}
|
|
470
|
-
return errors;
|
|
471
|
-
}
|
|
472
|
-
/**
|
|
473
|
-
* Weaviate — accepts baseUrl OR url.
|
|
474
|
-
* WeaviateProvider reads opts.baseUrl.
|
|
475
|
-
* Options keys: { baseUrl?: string, url?: string, apiKey?: string }
|
|
476
|
-
*/
|
|
477
|
-
static validateWeaviateConfig(config) {
|
|
478
|
-
const errors = [];
|
|
479
|
-
const opts = config.options;
|
|
480
|
-
if (!opts.baseUrl && !opts.url) {
|
|
481
|
-
errors.push({
|
|
482
|
-
field: "vectorDb.options.baseUrl",
|
|
483
|
-
message: "Weaviate instance URL is required",
|
|
484
|
-
suggestion: "Set WEAVIATE_URL environment variable",
|
|
485
|
-
severity: "error"
|
|
486
|
-
});
|
|
487
|
-
}
|
|
488
|
-
return errors;
|
|
489
|
-
}
|
|
490
|
-
/**
|
|
491
|
-
* Universal REST / custom REST adapter.
|
|
492
|
-
* Options key: { baseUrl: string }
|
|
493
|
-
*/
|
|
494
|
-
static validateRestConfig(config) {
|
|
495
|
-
const errors = [];
|
|
496
|
-
const opts = config.options;
|
|
497
|
-
if (!opts.baseUrl) {
|
|
498
|
-
errors.push({
|
|
499
|
-
field: "vectorDb.options.baseUrl",
|
|
500
|
-
message: "REST API base URL is required",
|
|
501
|
-
suggestion: "Set VECTOR_BASE_URL environment variable",
|
|
502
|
-
severity: "error"
|
|
503
|
-
});
|
|
504
|
-
}
|
|
505
|
-
return errors;
|
|
234
|
+
// src/llm/providers/OpenAIProvider.ts
|
|
235
|
+
import OpenAI from "openai";
|
|
236
|
+
var OpenAIProvider = class {
|
|
237
|
+
constructor(llmConfig, embeddingConfig) {
|
|
238
|
+
if (!llmConfig.apiKey) throw new Error("[OpenAIProvider] llmConfig.apiKey is required");
|
|
239
|
+
this.client = new OpenAI({ apiKey: llmConfig.apiKey });
|
|
240
|
+
this.llmConfig = llmConfig;
|
|
241
|
+
this.embeddingConfig = embeddingConfig;
|
|
506
242
|
}
|
|
507
|
-
static
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
message: "LLM provider is required",
|
|
514
|
-
severity: "error"
|
|
515
|
-
});
|
|
516
|
-
return errors;
|
|
517
|
-
}
|
|
518
|
-
if (!config.model) {
|
|
519
|
-
errors.push({
|
|
520
|
-
field: "llm.model",
|
|
521
|
-
message: "LLM model name is required",
|
|
522
|
-
suggestion: `e.g., "gpt-4o" for OpenAI, "claude-3-opus" for Anthropic`,
|
|
523
|
-
severity: "error"
|
|
524
|
-
});
|
|
525
|
-
}
|
|
526
|
-
switch (config.provider) {
|
|
527
|
-
case "openai":
|
|
243
|
+
static getValidator() {
|
|
244
|
+
return {
|
|
245
|
+
validate(config) {
|
|
246
|
+
const errors = [];
|
|
247
|
+
const isEmbedding = config.provider === "openai" && "dimensions" in config;
|
|
248
|
+
const prefix = isEmbedding ? "embedding" : "llm";
|
|
528
249
|
if (!config.apiKey) {
|
|
529
250
|
errors.push({
|
|
530
|
-
field:
|
|
251
|
+
field: `${prefix}.apiKey`,
|
|
531
252
|
message: "OpenAI API key is required",
|
|
532
253
|
suggestion: "Set OPENAI_API_KEY environment variable",
|
|
533
254
|
severity: "error"
|
|
534
255
|
});
|
|
535
256
|
}
|
|
536
|
-
|
|
537
|
-
case "anthropic":
|
|
538
|
-
if (!config.apiKey) {
|
|
539
|
-
errors.push({
|
|
540
|
-
field: "llm.apiKey",
|
|
541
|
-
message: "Anthropic API key is required",
|
|
542
|
-
suggestion: "Set ANTHROPIC_API_KEY environment variable",
|
|
543
|
-
severity: "error"
|
|
544
|
-
});
|
|
545
|
-
}
|
|
546
|
-
break;
|
|
547
|
-
case "gemini":
|
|
548
|
-
if (!config.apiKey) {
|
|
549
|
-
errors.push({
|
|
550
|
-
field: "llm.apiKey",
|
|
551
|
-
message: "Gemini API key is required",
|
|
552
|
-
suggestion: "Set GEMINI_API_KEY environment variable",
|
|
553
|
-
severity: "error"
|
|
554
|
-
});
|
|
555
|
-
}
|
|
556
|
-
break;
|
|
557
|
-
case "ollama":
|
|
558
|
-
if (!config.baseUrl) {
|
|
257
|
+
if (!config.model) {
|
|
559
258
|
errors.push({
|
|
560
|
-
field:
|
|
561
|
-
message: "
|
|
562
|
-
suggestion: '
|
|
259
|
+
field: `${prefix}.model`,
|
|
260
|
+
message: "OpenAI model name is required",
|
|
261
|
+
suggestion: isEmbedding ? 'e.g., "text-embedding-3-small"' : 'e.g., "gpt-4o"',
|
|
563
262
|
severity: "error"
|
|
564
263
|
});
|
|
565
264
|
}
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
265
|
+
return errors;
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
static getHealthChecker() {
|
|
270
|
+
return {
|
|
271
|
+
async check(config) {
|
|
272
|
+
const timestamp = Date.now();
|
|
273
|
+
const apiKey = config.apiKey;
|
|
274
|
+
const modelName = config.model;
|
|
275
|
+
try {
|
|
276
|
+
const OpenAI2 = await import("openai");
|
|
277
|
+
const client = new OpenAI2.default({ apiKey });
|
|
278
|
+
const models = await client.models.list();
|
|
279
|
+
const hasModel = models.data.some((m) => m.id === modelName);
|
|
280
|
+
return {
|
|
281
|
+
healthy: true,
|
|
282
|
+
provider: "openai",
|
|
283
|
+
capabilities: { model: modelName, available: hasModel, totalModels: models.data.length },
|
|
284
|
+
timestamp
|
|
285
|
+
};
|
|
286
|
+
} catch (error) {
|
|
287
|
+
return {
|
|
288
|
+
healthy: false,
|
|
289
|
+
provider: "openai",
|
|
290
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
291
|
+
timestamp
|
|
292
|
+
};
|
|
576
293
|
}
|
|
577
|
-
break;
|
|
578
|
-
}
|
|
579
|
-
if (config.temperature !== void 0) {
|
|
580
|
-
if (config.temperature < 0 || config.temperature > 2) {
|
|
581
|
-
errors.push({
|
|
582
|
-
field: "llm.temperature",
|
|
583
|
-
message: "Temperature must be between 0 and 2",
|
|
584
|
-
severity: "error"
|
|
585
|
-
});
|
|
586
294
|
}
|
|
587
|
-
}
|
|
588
|
-
if (config.maxTokens !== void 0 && config.maxTokens <= 0) {
|
|
589
|
-
errors.push({
|
|
590
|
-
field: "llm.maxTokens",
|
|
591
|
-
message: "maxTokens must be greater than 0",
|
|
592
|
-
severity: "error"
|
|
593
|
-
});
|
|
594
|
-
}
|
|
595
|
-
return errors;
|
|
596
|
-
}
|
|
597
|
-
static validateEmbeddingConfig(config) {
|
|
598
|
-
const errors = [];
|
|
599
|
-
if (!config.provider) {
|
|
600
|
-
errors.push({
|
|
601
|
-
field: "embedding.provider",
|
|
602
|
-
message: "Embedding provider is required",
|
|
603
|
-
severity: "error"
|
|
604
|
-
});
|
|
605
|
-
return errors;
|
|
606
|
-
}
|
|
607
|
-
if (!config.model) {
|
|
608
|
-
errors.push({
|
|
609
|
-
field: "embedding.model",
|
|
610
|
-
message: "Embedding model name is required",
|
|
611
|
-
suggestion: 'e.g., "text-embedding-3-small" for OpenAI, "nomic-embed-text" for Ollama',
|
|
612
|
-
severity: "error"
|
|
613
|
-
});
|
|
614
|
-
}
|
|
615
|
-
if (config.provider === "openai" && !config.apiKey) {
|
|
616
|
-
errors.push({
|
|
617
|
-
field: "embedding.apiKey",
|
|
618
|
-
message: "OpenAI API key is required for embedding",
|
|
619
|
-
suggestion: "Set OPENAI_API_KEY environment variable",
|
|
620
|
-
severity: "error"
|
|
621
|
-
});
|
|
622
|
-
}
|
|
623
|
-
if (config.provider === "gemini" && !config.apiKey) {
|
|
624
|
-
errors.push({
|
|
625
|
-
field: "embedding.apiKey",
|
|
626
|
-
message: "Gemini API key is required for embedding",
|
|
627
|
-
suggestion: "Set GEMINI_API_KEY environment variable",
|
|
628
|
-
severity: "error"
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
if (config.provider === "ollama" && !config.baseUrl) {
|
|
632
|
-
errors.push({
|
|
633
|
-
field: "embedding.baseUrl",
|
|
634
|
-
message: "Ollama base URL is required for embedding",
|
|
635
|
-
suggestion: "Set EMBEDDING_BASE_URL environment variable",
|
|
636
|
-
severity: "error"
|
|
637
|
-
});
|
|
638
|
-
}
|
|
639
|
-
if (config.dimensions !== void 0 && config.dimensions <= 0) {
|
|
640
|
-
errors.push({
|
|
641
|
-
field: "embedding.dimensions",
|
|
642
|
-
message: "Embedding dimensions must be greater than 0",
|
|
643
|
-
severity: "error"
|
|
644
|
-
});
|
|
645
|
-
}
|
|
646
|
-
return errors;
|
|
647
|
-
}
|
|
648
|
-
static validateUIConfig(config) {
|
|
649
|
-
const errors = [];
|
|
650
|
-
if (config.primaryColor) {
|
|
651
|
-
if (!this.isValidCSSColor(config.primaryColor)) {
|
|
652
|
-
errors.push({
|
|
653
|
-
field: "ui.primaryColor",
|
|
654
|
-
message: "Invalid CSS color format",
|
|
655
|
-
suggestion: "Use valid CSS colors: hex (#FF0000), rgb, hsl, or named colors",
|
|
656
|
-
severity: "warning"
|
|
657
|
-
});
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
if (config.borderRadius) {
|
|
661
|
-
if (!UI_BORDER_RADIUS_OPTIONS.includes(config.borderRadius)) {
|
|
662
|
-
errors.push({
|
|
663
|
-
field: "ui.borderRadius",
|
|
664
|
-
message: `borderRadius must be one of: ${UI_BORDER_RADIUS_OPTIONS.join(", ")}`,
|
|
665
|
-
severity: "warning"
|
|
666
|
-
});
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
if (config.visualStyle) {
|
|
670
|
-
if (!UI_VISUAL_STYLES.includes(config.visualStyle)) {
|
|
671
|
-
errors.push({
|
|
672
|
-
field: "ui.visualStyle",
|
|
673
|
-
message: `visualStyle must be one of: ${UI_VISUAL_STYLES.join(", ")}`,
|
|
674
|
-
severity: "warning"
|
|
675
|
-
});
|
|
676
|
-
}
|
|
677
|
-
}
|
|
678
|
-
return errors;
|
|
679
|
-
}
|
|
680
|
-
static validateRAGConfig(config) {
|
|
681
|
-
const errors = [];
|
|
682
|
-
if (config.topK !== void 0) {
|
|
683
|
-
if (typeof config.topK !== "number" || config.topK <= 0) {
|
|
684
|
-
errors.push({
|
|
685
|
-
field: "rag.topK",
|
|
686
|
-
message: "topK must be a positive integer",
|
|
687
|
-
severity: "error"
|
|
688
|
-
});
|
|
689
|
-
}
|
|
690
|
-
}
|
|
691
|
-
if (config.scoreThreshold !== void 0) {
|
|
692
|
-
if (typeof config.scoreThreshold !== "number" || config.scoreThreshold < 0 || config.scoreThreshold > 1) {
|
|
693
|
-
errors.push({
|
|
694
|
-
field: "rag.scoreThreshold",
|
|
695
|
-
message: "scoreThreshold must be between 0 and 1",
|
|
696
|
-
severity: "error"
|
|
697
|
-
});
|
|
698
|
-
}
|
|
699
|
-
}
|
|
700
|
-
if (config.chunkSize !== void 0) {
|
|
701
|
-
if (typeof config.chunkSize !== "number" || config.chunkSize <= 0) {
|
|
702
|
-
errors.push({
|
|
703
|
-
field: "rag.chunkSize",
|
|
704
|
-
message: "chunkSize must be a positive integer",
|
|
705
|
-
severity: "error"
|
|
706
|
-
});
|
|
707
|
-
}
|
|
708
|
-
}
|
|
709
|
-
if (config.chunkOverlap !== void 0) {
|
|
710
|
-
if (typeof config.chunkOverlap !== "number" || config.chunkOverlap < 0) {
|
|
711
|
-
errors.push({
|
|
712
|
-
field: "rag.chunkOverlap",
|
|
713
|
-
message: "chunkOverlap must be a non-negative integer",
|
|
714
|
-
severity: "error"
|
|
715
|
-
});
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
return errors;
|
|
719
|
-
}
|
|
720
|
-
static isValidCSSColor(color) {
|
|
721
|
-
const hexRegex = /^#([0-9a-f]{3}){1,2}$/i;
|
|
722
|
-
const rgbRegex = /^rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*(\d+(?:\.\d+)?))?\)$/i;
|
|
723
|
-
const hslRegex = /^hsla?\((\d+),\s*(\d+)%,\s*(\d+)%(?:,\s*(\d+(?:\.\d+)?))?\)$/i;
|
|
724
|
-
const namedColors = ["red", "blue", "green", "black", "white", "transparent"];
|
|
725
|
-
return hexRegex.test(color) || rgbRegex.test(color) || hslRegex.test(color) || namedColors.includes(color.toLowerCase());
|
|
726
|
-
}
|
|
727
|
-
/**
|
|
728
|
-
* Throws if there are error-level validation issues.
|
|
729
|
-
* Logs warnings to console.
|
|
730
|
-
*/
|
|
731
|
-
static validateAndThrow(config) {
|
|
732
|
-
const errors = this.validate(config);
|
|
733
|
-
const errorItems = errors.filter((e) => e.severity === "error");
|
|
734
|
-
const warnings = errors.filter((e) => e.severity === "warning");
|
|
735
|
-
if (warnings.length > 0) {
|
|
736
|
-
console.warn("[ConfigValidator] Configuration warnings:");
|
|
737
|
-
warnings.forEach((w) => {
|
|
738
|
-
console.warn(` ${w.field}: ${w.message}`);
|
|
739
|
-
if (w.suggestion) console.warn(` Suggestion: ${w.suggestion}`);
|
|
740
|
-
});
|
|
741
|
-
}
|
|
742
|
-
if (errorItems.length > 0) {
|
|
743
|
-
const message = errorItems.map((e) => `${e.field}: ${e.message}${e.suggestion ? ` (${e.suggestion})` : ""}`).join("\n ");
|
|
744
|
-
throw new Error(`[ConfigValidator] Configuration validation failed:
|
|
745
|
-
${message}`);
|
|
746
|
-
}
|
|
747
|
-
}
|
|
748
|
-
};
|
|
749
|
-
|
|
750
|
-
// src/rag/DocumentChunker.ts
|
|
751
|
-
var DocumentChunker = class {
|
|
752
|
-
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n\n", "\n", " ", ""]) {
|
|
753
|
-
this.chunkSize = chunkSize;
|
|
754
|
-
this.chunkOverlap = chunkOverlap;
|
|
755
|
-
this.separators = separators;
|
|
756
|
-
}
|
|
757
|
-
/**
|
|
758
|
-
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
759
|
-
*/
|
|
760
|
-
chunk(text, options = {}) {
|
|
761
|
-
const {
|
|
762
|
-
chunkSize = this.chunkSize,
|
|
763
|
-
chunkOverlap = this.chunkOverlap,
|
|
764
|
-
docId = `doc_${Date.now()}`,
|
|
765
|
-
metadata = {},
|
|
766
|
-
separators = this.separators
|
|
767
|
-
} = options;
|
|
768
|
-
const finalChunks = [];
|
|
769
|
-
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
770
|
-
let currentChunk = [];
|
|
771
|
-
let currentLength = 0;
|
|
772
|
-
let chunkIndex = 0;
|
|
773
|
-
for (const split of splits) {
|
|
774
|
-
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
775
|
-
finalChunks.push({
|
|
776
|
-
id: `${docId}_chunk_${chunkIndex++}`,
|
|
777
|
-
content: currentChunk.join("").trim(),
|
|
778
|
-
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
779
|
-
});
|
|
780
|
-
const overlapItems = [];
|
|
781
|
-
let overlapLen = 0;
|
|
782
|
-
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
783
|
-
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
784
|
-
overlapItems.unshift(currentChunk[i]);
|
|
785
|
-
overlapLen += currentChunk[i].length;
|
|
786
|
-
} else {
|
|
787
|
-
break;
|
|
788
|
-
}
|
|
789
|
-
}
|
|
790
|
-
currentChunk = overlapItems;
|
|
791
|
-
currentLength = overlapLen;
|
|
792
|
-
}
|
|
793
|
-
currentChunk.push(split);
|
|
794
|
-
currentLength += split.length;
|
|
795
|
-
}
|
|
796
|
-
if (currentChunk.length > 0) {
|
|
797
|
-
finalChunks.push({
|
|
798
|
-
id: `${docId}_chunk_${chunkIndex}`,
|
|
799
|
-
content: currentChunk.join("").trim(),
|
|
800
|
-
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
801
|
-
});
|
|
802
|
-
}
|
|
803
|
-
return finalChunks;
|
|
804
|
-
}
|
|
805
|
-
/**
|
|
806
|
-
* Recursively split text based on separators.
|
|
807
|
-
*/
|
|
808
|
-
recursiveSplit(text, separators, chunkSize) {
|
|
809
|
-
const finalSplits = [];
|
|
810
|
-
let separator = separators[separators.length - 1];
|
|
811
|
-
let nextSeparators = [];
|
|
812
|
-
for (let i = 0; i < separators.length; i++) {
|
|
813
|
-
if (text.includes(separators[i])) {
|
|
814
|
-
separator = separators[i];
|
|
815
|
-
nextSeparators = separators.slice(i + 1);
|
|
816
|
-
break;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
const parts = text.split(separator);
|
|
820
|
-
for (const part of parts) {
|
|
821
|
-
if (part.length <= chunkSize) {
|
|
822
|
-
finalSplits.push(part + separator);
|
|
823
|
-
} else if (nextSeparators.length > 0) {
|
|
824
|
-
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
825
|
-
} else {
|
|
826
|
-
finalSplits.push(part);
|
|
827
|
-
}
|
|
828
|
-
}
|
|
829
|
-
return finalSplits;
|
|
830
|
-
}
|
|
831
|
-
/**
|
|
832
|
-
* Chunk multiple documents at once.
|
|
833
|
-
*/
|
|
834
|
-
chunkMany(documents) {
|
|
835
|
-
return documents.flatMap(
|
|
836
|
-
(doc) => this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
|
|
837
|
-
);
|
|
838
|
-
}
|
|
839
|
-
};
|
|
840
|
-
|
|
841
|
-
// src/rag/EntityExtractor.ts
|
|
842
|
-
var EntityExtractor = class {
|
|
843
|
-
constructor(llm) {
|
|
844
|
-
this.llm = llm;
|
|
845
|
-
}
|
|
846
|
-
/**
|
|
847
|
-
* Extract nodes and edges from a text chunk.
|
|
848
|
-
*/
|
|
849
|
-
async extract(text) {
|
|
850
|
-
const prompt = `
|
|
851
|
-
Extract entities and relationships from the following text.
|
|
852
|
-
Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
|
|
853
|
-
Use the same ID for the same entity.
|
|
854
|
-
|
|
855
|
-
Text:
|
|
856
|
-
"${text}"
|
|
857
|
-
|
|
858
|
-
Output JSON:
|
|
859
|
-
`;
|
|
860
|
-
const response = await this.llm.chat([
|
|
861
|
-
{ role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
|
|
862
|
-
{ role: "user", content: prompt }
|
|
863
|
-
], "");
|
|
864
|
-
try {
|
|
865
|
-
const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
|
|
866
|
-
return JSON.parse(cleanJson);
|
|
867
|
-
} catch (e) {
|
|
868
|
-
console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
|
|
869
|
-
return { nodes: [], edges: [] };
|
|
870
|
-
}
|
|
871
|
-
}
|
|
872
|
-
};
|
|
873
|
-
|
|
874
|
-
// src/llm/providers/OpenAIProvider.ts
|
|
875
|
-
import OpenAI from "openai";
|
|
876
|
-
var OpenAIProvider = class {
|
|
877
|
-
constructor(llmConfig, embeddingConfig) {
|
|
878
|
-
if (!llmConfig.apiKey) throw new Error("[OpenAIProvider] llmConfig.apiKey is required");
|
|
879
|
-
this.client = new OpenAI({ apiKey: llmConfig.apiKey });
|
|
880
|
-
this.llmConfig = llmConfig;
|
|
881
|
-
this.embeddingConfig = embeddingConfig;
|
|
295
|
+
};
|
|
882
296
|
}
|
|
883
297
|
async chat(messages, context, options) {
|
|
884
298
|
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
@@ -941,6 +355,56 @@ var AnthropicProvider = class {
|
|
|
941
355
|
this.llmConfig = llmConfig;
|
|
942
356
|
this.embeddingConfig = embeddingConfig;
|
|
943
357
|
}
|
|
358
|
+
static getValidator() {
|
|
359
|
+
return {
|
|
360
|
+
validate(config) {
|
|
361
|
+
const errors = [];
|
|
362
|
+
if (!config.apiKey) {
|
|
363
|
+
errors.push({
|
|
364
|
+
field: "llm.apiKey",
|
|
365
|
+
message: "Anthropic API key is required",
|
|
366
|
+
suggestion: "Set ANTHROPIC_API_KEY environment variable",
|
|
367
|
+
severity: "error"
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
if (!config.model) {
|
|
371
|
+
errors.push({
|
|
372
|
+
field: "llm.model",
|
|
373
|
+
message: "Anthropic model name is required",
|
|
374
|
+
suggestion: 'e.g., "claude-3-5-sonnet-20241022"',
|
|
375
|
+
severity: "error"
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
return errors;
|
|
379
|
+
}
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
static getHealthChecker() {
|
|
383
|
+
return {
|
|
384
|
+
async check(config) {
|
|
385
|
+
const timestamp = Date.now();
|
|
386
|
+
const apiKey = config.apiKey;
|
|
387
|
+
const modelName = config.model;
|
|
388
|
+
try {
|
|
389
|
+
const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
|
|
390
|
+
const client = new Anthropic2({ apiKey });
|
|
391
|
+
await client.messages.create({
|
|
392
|
+
model: modelName,
|
|
393
|
+
max_tokens: 10,
|
|
394
|
+
messages: [{ role: "user", content: "ping" }]
|
|
395
|
+
});
|
|
396
|
+
return { healthy: true, provider: "anthropic", capabilities: { model: modelName }, timestamp };
|
|
397
|
+
} catch (error) {
|
|
398
|
+
return {
|
|
399
|
+
healthy: false,
|
|
400
|
+
provider: "anthropic",
|
|
401
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
402
|
+
timestamp
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
};
|
|
407
|
+
}
|
|
944
408
|
async chat(messages, context, options) {
|
|
945
409
|
var _a, _b, _c;
|
|
946
410
|
const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the context below to answer the user's question accurately.
|
|
@@ -964,11 +428,6 @@ ${context}`;
|
|
|
964
428
|
const block = response.content[0];
|
|
965
429
|
return block.type === "text" ? block.text : "";
|
|
966
430
|
}
|
|
967
|
-
/**
|
|
968
|
-
* Anthropic does not offer an embedding API.
|
|
969
|
-
* This method throws with a clear error so developers know to configure
|
|
970
|
-
* a separate embedding provider (OpenAI or Ollama).
|
|
971
|
-
*/
|
|
972
431
|
async embed(text, options) {
|
|
973
432
|
void text;
|
|
974
433
|
void options;
|
|
@@ -1004,22 +463,66 @@ var OllamaProvider = class {
|
|
|
1004
463
|
this.llmConfig = llmConfig;
|
|
1005
464
|
this.embeddingConfig = embeddingConfig;
|
|
1006
465
|
}
|
|
466
|
+
static getValidator() {
|
|
467
|
+
return {
|
|
468
|
+
validate(config) {
|
|
469
|
+
const errors = [];
|
|
470
|
+
if (!config.model) {
|
|
471
|
+
const isEmbedding = config.provider === "ollama" && "dimensions" in config;
|
|
472
|
+
const prefix = isEmbedding ? "embedding" : "llm";
|
|
473
|
+
errors.push({
|
|
474
|
+
field: `${prefix}.model`,
|
|
475
|
+
message: "Ollama model name is required",
|
|
476
|
+
suggestion: 'e.g., "llama3" or "mistral"',
|
|
477
|
+
severity: "error"
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
return errors;
|
|
481
|
+
}
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
static getHealthChecker() {
|
|
485
|
+
return {
|
|
486
|
+
async check(config) {
|
|
487
|
+
const timestamp = Date.now();
|
|
488
|
+
const baseUrl = config.baseUrl || "http://localhost:11434";
|
|
489
|
+
const modelName = config.model;
|
|
490
|
+
try {
|
|
491
|
+
const axios3 = (await import("axios")).default;
|
|
492
|
+
const { data } = await axios3.get(`${baseUrl}/api/tags`);
|
|
493
|
+
const models = data.models || [];
|
|
494
|
+
const hasModel = models.some((m) => m.name === modelName || m.name.startsWith(`${modelName}:`));
|
|
495
|
+
return {
|
|
496
|
+
healthy: true,
|
|
497
|
+
provider: "ollama",
|
|
498
|
+
capabilities: {
|
|
499
|
+
baseUrl,
|
|
500
|
+
model: modelName,
|
|
501
|
+
available: hasModel,
|
|
502
|
+
totalModels: models.length
|
|
503
|
+
},
|
|
504
|
+
timestamp
|
|
505
|
+
};
|
|
506
|
+
} catch (e) {
|
|
507
|
+
return {
|
|
508
|
+
healthy: false,
|
|
509
|
+
provider: "ollama",
|
|
510
|
+
error: `Ollama server not reachable at ${baseUrl}. Is it running?`,
|
|
511
|
+
timestamp
|
|
512
|
+
};
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
};
|
|
516
|
+
}
|
|
1007
517
|
async chat(messages, context, options) {
|
|
1008
|
-
var _a, _b, _c, _d
|
|
1009
|
-
const
|
|
1010
|
-
|
|
1011
|
-
Context:
|
|
1012
|
-
${context}`;
|
|
1013
|
-
const system = systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
|
|
1014
|
-
|
|
1015
|
-
Context:
|
|
1016
|
-
${context}`;
|
|
518
|
+
var _a, _b, _c, _d;
|
|
519
|
+
const system = this.buildSystemPrompt(context);
|
|
1017
520
|
const { data } = await this.http.post("/api/chat", {
|
|
1018
521
|
model: this.llmConfig.model,
|
|
1019
522
|
stream: false,
|
|
1020
523
|
options: {
|
|
1021
|
-
temperature: (
|
|
1022
|
-
num_predict: (
|
|
524
|
+
temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : 0.7,
|
|
525
|
+
num_predict: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024
|
|
1023
526
|
},
|
|
1024
527
|
messages: [
|
|
1025
528
|
{ role: "system", content: system },
|
|
@@ -1028,13 +531,67 @@ ${context}`;
|
|
|
1028
531
|
});
|
|
1029
532
|
return data.message.content;
|
|
1030
533
|
}
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
534
|
+
chatStream(messages, context, options) {
|
|
535
|
+
return __asyncGenerator(this, null, function* () {
|
|
536
|
+
var _a, _b, _c, _d, _e;
|
|
537
|
+
const system = this.buildSystemPrompt(context);
|
|
538
|
+
const response = yield new __await(this.http.post("/api/chat", {
|
|
539
|
+
model: this.llmConfig.model,
|
|
540
|
+
stream: true,
|
|
541
|
+
options: {
|
|
542
|
+
temperature: (_b = (_a = options == null ? void 0 : options.temperature) != null ? _a : this.llmConfig.temperature) != null ? _b : 0.7,
|
|
543
|
+
num_predict: (_d = (_c = options == null ? void 0 : options.maxTokens) != null ? _c : this.llmConfig.maxTokens) != null ? _d : 1024
|
|
544
|
+
},
|
|
545
|
+
messages: [
|
|
546
|
+
{ role: "system", content: system },
|
|
547
|
+
...messages.map((m) => ({ role: m.role, content: m.content }))
|
|
548
|
+
]
|
|
549
|
+
}, { responseType: "stream" }));
|
|
550
|
+
try {
|
|
551
|
+
for (var iter = __forAwait(response.data), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
552
|
+
const chunk = temp.value;
|
|
553
|
+
const lines = chunk.toString().split("\n").filter(Boolean);
|
|
554
|
+
for (const line of lines) {
|
|
555
|
+
try {
|
|
556
|
+
const json = JSON.parse(line);
|
|
557
|
+
if ((_e = json.message) == null ? void 0 : _e.content) {
|
|
558
|
+
yield json.message.content;
|
|
559
|
+
}
|
|
560
|
+
if (json.done) return;
|
|
561
|
+
} catch (e) {
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
} catch (temp) {
|
|
566
|
+
error = [temp];
|
|
567
|
+
} finally {
|
|
568
|
+
try {
|
|
569
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
570
|
+
} finally {
|
|
571
|
+
if (error)
|
|
572
|
+
throw error[0];
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
buildSystemPrompt(context) {
|
|
578
|
+
var _a;
|
|
579
|
+
const systemPrompt = (_a = this.llmConfig.systemPrompt) != null ? _a : `You are a helpful assistant. Use the provided context to answer the user's question.
|
|
580
|
+
|
|
581
|
+
Context:
|
|
582
|
+
${context}`;
|
|
583
|
+
return systemPrompt.includes("{{context}}") ? systemPrompt.replace("{{context}}", context) : `${systemPrompt}
|
|
584
|
+
|
|
585
|
+
Context:
|
|
586
|
+
${context}`;
|
|
587
|
+
}
|
|
588
|
+
async embed(text, options) {
|
|
589
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k;
|
|
590
|
+
const model = (_c = (_b = options == null ? void 0 : options.model) != null ? _b : (_a = this.embeddingConfig) == null ? void 0 : _a.model) != null ? _c : "nomic-embed-text";
|
|
591
|
+
const baseURL = (_f = (_e = (_d = this.embeddingConfig) == null ? void 0 : _d.baseUrl) != null ? _e : this.llmConfig.baseUrl) != null ? _f : "http://localhost:11434";
|
|
592
|
+
const client = baseURL !== ((_g = this.llmConfig.baseUrl) != null ? _g : "http://localhost:11434") ? axios.create({ baseURL, timeout: 6e4 }) : this.http;
|
|
593
|
+
let prompt = text;
|
|
594
|
+
const queryPrefix = (_i = (_h = this.embeddingConfig) == null ? void 0 : _h.queryPrefix) != null ? _i : model.includes("nomic") ? "search_query: " : "";
|
|
1038
595
|
const docPrefix = (_k = (_j = this.embeddingConfig) == null ? void 0 : _j.documentPrefix) != null ? _k : model.includes("nomic") ? "search_document: " : "";
|
|
1039
596
|
if ((options == null ? void 0 : options.taskType) === "query" && queryPrefix) {
|
|
1040
597
|
if (!prompt.startsWith(queryPrefix)) {
|
|
@@ -1084,6 +641,52 @@ var GeminiProvider = class {
|
|
|
1084
641
|
});
|
|
1085
642
|
}
|
|
1086
643
|
}
|
|
644
|
+
static getValidator() {
|
|
645
|
+
return {
|
|
646
|
+
validate(config) {
|
|
647
|
+
const errors = [];
|
|
648
|
+
if (!config.apiKey && !process.env.GOOGLE_GENAI_API_KEY) {
|
|
649
|
+
errors.push({
|
|
650
|
+
field: "llm.apiKey",
|
|
651
|
+
message: "Gemini API key is required",
|
|
652
|
+
suggestion: "Set GOOGLE_GENAI_API_KEY environment variable or provide in config",
|
|
653
|
+
severity: "error"
|
|
654
|
+
});
|
|
655
|
+
}
|
|
656
|
+
if (!config.model) {
|
|
657
|
+
errors.push({ field: "llm.model", message: "Gemini model name is required", severity: "error" });
|
|
658
|
+
}
|
|
659
|
+
return errors;
|
|
660
|
+
}
|
|
661
|
+
};
|
|
662
|
+
}
|
|
663
|
+
static getHealthChecker() {
|
|
664
|
+
return {
|
|
665
|
+
async check(config) {
|
|
666
|
+
const timestamp = Date.now();
|
|
667
|
+
const apiKey = config.apiKey || process.env.GOOGLE_GENAI_API_KEY || "";
|
|
668
|
+
const modelName = config.model;
|
|
669
|
+
try {
|
|
670
|
+
const { GoogleGenAI: GoogleGenAI2 } = await import("@google/genai");
|
|
671
|
+
const genAI = new GoogleGenAI2({ apiKey });
|
|
672
|
+
await genAI.models.get({ model: modelName });
|
|
673
|
+
return {
|
|
674
|
+
healthy: true,
|
|
675
|
+
provider: "gemini",
|
|
676
|
+
capabilities: { model: modelName },
|
|
677
|
+
timestamp
|
|
678
|
+
};
|
|
679
|
+
} catch (error) {
|
|
680
|
+
return {
|
|
681
|
+
healthy: false,
|
|
682
|
+
provider: "gemini",
|
|
683
|
+
error: error instanceof Error ? error.message : String(error),
|
|
684
|
+
timestamp
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
};
|
|
689
|
+
}
|
|
1087
690
|
sanitizeModel(model) {
|
|
1088
691
|
if (!model) return model;
|
|
1089
692
|
return model.split(":")[0];
|
|
@@ -1325,189 +928,610 @@ ${context != null ? context : "None"}` },
|
|
|
1325
928
|
if (result === void 0) {
|
|
1326
929
|
throw new Error(`[UniversalLLMAdapter] Could not extract text from path '${extractPath}' in response.`);
|
|
1327
930
|
}
|
|
1328
|
-
return String(result);
|
|
1329
|
-
}
|
|
1330
|
-
async embed(text) {
|
|
1331
|
-
var _a, _b;
|
|
1332
|
-
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
1333
|
-
let payload;
|
|
1334
|
-
if (this.opts.embedPayloadTemplate) {
|
|
1335
|
-
payload = buildPayload(this.opts.embedPayloadTemplate, {
|
|
1336
|
-
model: this.model,
|
|
1337
|
-
input: text
|
|
931
|
+
return String(result);
|
|
932
|
+
}
|
|
933
|
+
async embed(text) {
|
|
934
|
+
var _a, _b;
|
|
935
|
+
const path = (_a = this.opts.embedPath) != null ? _a : "/embeddings";
|
|
936
|
+
let payload;
|
|
937
|
+
if (this.opts.embedPayloadTemplate) {
|
|
938
|
+
payload = buildPayload(this.opts.embedPayloadTemplate, {
|
|
939
|
+
model: this.model,
|
|
940
|
+
input: text
|
|
941
|
+
});
|
|
942
|
+
} else {
|
|
943
|
+
payload = {
|
|
944
|
+
model: this.model,
|
|
945
|
+
input: text
|
|
946
|
+
};
|
|
947
|
+
}
|
|
948
|
+
const { data } = await this.http.post(path, payload);
|
|
949
|
+
const extractPath = (_b = this.opts.embedExtractPath) != null ? _b : "data[0].embedding";
|
|
950
|
+
const vector = resolvePath(data, extractPath);
|
|
951
|
+
if (!Array.isArray(vector)) {
|
|
952
|
+
throw new Error(`[UniversalLLMAdapter] Expected a number array at '${extractPath}' for embeddings.`);
|
|
953
|
+
}
|
|
954
|
+
return vector;
|
|
955
|
+
}
|
|
956
|
+
async batchEmbed(texts) {
|
|
957
|
+
const vectors = [];
|
|
958
|
+
for (const text of texts) {
|
|
959
|
+
vectors.push(await this.embed(text));
|
|
960
|
+
}
|
|
961
|
+
return vectors;
|
|
962
|
+
}
|
|
963
|
+
async ping() {
|
|
964
|
+
try {
|
|
965
|
+
if (this.opts.pingPath) {
|
|
966
|
+
await this.http.get(this.opts.pingPath);
|
|
967
|
+
}
|
|
968
|
+
return true;
|
|
969
|
+
} catch (err) {
|
|
970
|
+
console.error("[UniversalLLMAdapter] Ping failed:", err);
|
|
971
|
+
return false;
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
};
|
|
975
|
+
|
|
976
|
+
// src/llm/LLMFactory.ts
|
|
977
|
+
var LLMFactory = class _LLMFactory {
|
|
978
|
+
static create(llmConfig, embeddingConfig) {
|
|
979
|
+
var _a;
|
|
980
|
+
switch (llmConfig.provider) {
|
|
981
|
+
case "openai":
|
|
982
|
+
return new OpenAIProvider(llmConfig, embeddingConfig);
|
|
983
|
+
case "anthropic":
|
|
984
|
+
return new AnthropicProvider(llmConfig, embeddingConfig);
|
|
985
|
+
case "ollama":
|
|
986
|
+
return new OllamaProvider(llmConfig, embeddingConfig);
|
|
987
|
+
case "gemini":
|
|
988
|
+
return new GeminiProvider(llmConfig, embeddingConfig);
|
|
989
|
+
case "rest":
|
|
990
|
+
case "universal_rest":
|
|
991
|
+
case "custom":
|
|
992
|
+
return new UniversalLLMAdapter(llmConfig);
|
|
993
|
+
default:
|
|
994
|
+
if (llmConfig.baseUrl || ((_a = llmConfig.options) == null ? void 0 : _a.baseUrl)) {
|
|
995
|
+
return new UniversalLLMAdapter(llmConfig);
|
|
996
|
+
}
|
|
997
|
+
throw new Error(`[LLMFactory] Unknown provider "${llmConfig.provider}"`);
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
static getValidator(provider) {
|
|
1001
|
+
const providerClass = this.getProviderClass(provider);
|
|
1002
|
+
return providerClass && providerClass.getValidator ? providerClass.getValidator() : null;
|
|
1003
|
+
}
|
|
1004
|
+
static getHealthChecker(provider) {
|
|
1005
|
+
const providerClass = this.getProviderClass(provider);
|
|
1006
|
+
return providerClass && providerClass.getHealthChecker ? providerClass.getHealthChecker() : null;
|
|
1007
|
+
}
|
|
1008
|
+
static getProviderClass(provider) {
|
|
1009
|
+
switch (provider) {
|
|
1010
|
+
case "openai":
|
|
1011
|
+
return OpenAIProvider;
|
|
1012
|
+
case "anthropic":
|
|
1013
|
+
return AnthropicProvider;
|
|
1014
|
+
case "ollama":
|
|
1015
|
+
return OllamaProvider;
|
|
1016
|
+
case "gemini":
|
|
1017
|
+
return GeminiProvider;
|
|
1018
|
+
case "rest":
|
|
1019
|
+
case "universal_rest":
|
|
1020
|
+
case "custom":
|
|
1021
|
+
return UniversalLLMAdapter;
|
|
1022
|
+
default:
|
|
1023
|
+
return null;
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
/**
|
|
1027
|
+
* Creates a dedicated embedding-only provider.
|
|
1028
|
+
*/
|
|
1029
|
+
static createEmbeddingProvider(embeddingConfig) {
|
|
1030
|
+
const fakeLLMConfig = {
|
|
1031
|
+
provider: embeddingConfig.provider,
|
|
1032
|
+
model: embeddingConfig.model,
|
|
1033
|
+
apiKey: embeddingConfig.apiKey,
|
|
1034
|
+
baseUrl: embeddingConfig.baseUrl,
|
|
1035
|
+
options: embeddingConfig.options
|
|
1036
|
+
};
|
|
1037
|
+
return _LLMFactory.create(fakeLLMConfig, embeddingConfig);
|
|
1038
|
+
}
|
|
1039
|
+
};
|
|
1040
|
+
|
|
1041
|
+
// src/core/ProviderRegistry.ts
|
|
1042
|
+
var ProviderRegistry = class {
|
|
1043
|
+
static registerVectorProvider(name, providerClass) {
|
|
1044
|
+
this.vectorProviders[name] = providerClass;
|
|
1045
|
+
if (providerClass.getValidator) {
|
|
1046
|
+
this.vectorValidators[name] = providerClass.getValidator();
|
|
1047
|
+
}
|
|
1048
|
+
if (providerClass.getHealthChecker) {
|
|
1049
|
+
this.vectorHealthCheckers[name] = providerClass.getHealthChecker();
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
static async getVectorValidator(provider) {
|
|
1053
|
+
if (this.vectorValidators[provider]) return this.vectorValidators[provider];
|
|
1054
|
+
try {
|
|
1055
|
+
const providerClass = await this.loadVectorProviderClass(provider);
|
|
1056
|
+
if (providerClass.getValidator) {
|
|
1057
|
+
this.vectorValidators[provider] = providerClass.getValidator();
|
|
1058
|
+
return this.vectorValidators[provider];
|
|
1059
|
+
}
|
|
1060
|
+
} catch (e) {
|
|
1061
|
+
console.warn(`[ProviderRegistry] Failed to load validator for ${provider}:`, e);
|
|
1062
|
+
}
|
|
1063
|
+
return null;
|
|
1064
|
+
}
|
|
1065
|
+
static async getVectorHealthChecker(provider) {
|
|
1066
|
+
if (this.vectorHealthCheckers[provider]) return this.vectorHealthCheckers[provider];
|
|
1067
|
+
try {
|
|
1068
|
+
const providerClass = await this.loadVectorProviderClass(provider);
|
|
1069
|
+
if (providerClass.getHealthChecker) {
|
|
1070
|
+
this.vectorHealthCheckers[provider] = providerClass.getHealthChecker();
|
|
1071
|
+
return this.vectorHealthCheckers[provider];
|
|
1072
|
+
}
|
|
1073
|
+
} catch (e) {
|
|
1074
|
+
console.warn(`[ProviderRegistry] Failed to load health checker for ${provider}:`, e);
|
|
1075
|
+
}
|
|
1076
|
+
return null;
|
|
1077
|
+
}
|
|
1078
|
+
static async loadVectorProviderClass(provider) {
|
|
1079
|
+
if (this.vectorProviders[provider]) return this.vectorProviders[provider];
|
|
1080
|
+
switch (provider) {
|
|
1081
|
+
case "pinecone": {
|
|
1082
|
+
const { PineconeProvider } = await import("./PineconeProvider-QZNRKTN2.mjs");
|
|
1083
|
+
return PineconeProvider;
|
|
1084
|
+
}
|
|
1085
|
+
case "pgvector":
|
|
1086
|
+
case "postgresql": {
|
|
1087
|
+
const { PostgreSQLProvider } = await import("./PostgreSQLProvider-BMOETDZA.mjs");
|
|
1088
|
+
return PostgreSQLProvider;
|
|
1089
|
+
}
|
|
1090
|
+
case "mongodb": {
|
|
1091
|
+
const { MongoDBProvider } = await import("./MongoDBProvider-Z6ALOVDN.mjs");
|
|
1092
|
+
return MongoDBProvider;
|
|
1093
|
+
}
|
|
1094
|
+
case "milvus": {
|
|
1095
|
+
const { MilvusProvider } = await import("./MilvusProvider-U7SKC27V.mjs");
|
|
1096
|
+
return MilvusProvider;
|
|
1097
|
+
}
|
|
1098
|
+
case "qdrant": {
|
|
1099
|
+
const { QdrantProvider } = await import("./QdrantProvider-YNUNEOZH.mjs");
|
|
1100
|
+
return QdrantProvider;
|
|
1101
|
+
}
|
|
1102
|
+
case "chromadb": {
|
|
1103
|
+
const { ChromaDBProvider } = await import("./ChromaDBProvider-MIDOR4FW.mjs");
|
|
1104
|
+
return ChromaDBProvider;
|
|
1105
|
+
}
|
|
1106
|
+
case "redis": {
|
|
1107
|
+
const { RedisProvider } = await import("./RedisProvider-SR65SCKV.mjs");
|
|
1108
|
+
return RedisProvider;
|
|
1109
|
+
}
|
|
1110
|
+
case "weaviate": {
|
|
1111
|
+
const { WeaviateProvider } = await import("./WeaviateProvider-5FWDFITI.mjs");
|
|
1112
|
+
return WeaviateProvider;
|
|
1113
|
+
}
|
|
1114
|
+
case "universal_rest":
|
|
1115
|
+
case "rest": {
|
|
1116
|
+
const { UniversalVectorProvider } = await import("./UniversalVectorProvider-IN67OS56.mjs");
|
|
1117
|
+
return UniversalVectorProvider;
|
|
1118
|
+
}
|
|
1119
|
+
default:
|
|
1120
|
+
throw new Error(`Unsupported vector provider: ${provider}`);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
static async createVectorProvider(config) {
|
|
1124
|
+
const providerClass = await this.loadVectorProviderClass(config.provider);
|
|
1125
|
+
return new providerClass(config);
|
|
1126
|
+
}
|
|
1127
|
+
static async createGraphProvider(config) {
|
|
1128
|
+
const { provider } = config;
|
|
1129
|
+
if (this.graphProviders[provider]) {
|
|
1130
|
+
return new this.graphProviders[provider](config);
|
|
1131
|
+
}
|
|
1132
|
+
switch (provider) {
|
|
1133
|
+
case "simple": {
|
|
1134
|
+
const { SimpleGraphProvider } = await import("./SimpleGraphProvider-SLOXO4M7.mjs");
|
|
1135
|
+
return new SimpleGraphProvider(config);
|
|
1136
|
+
}
|
|
1137
|
+
default:
|
|
1138
|
+
throw new Error(`Unsupported graph provider: ${provider}`);
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
static createLLMProvider(llmConfig, embeddingConfig) {
|
|
1142
|
+
return LLMFactory.create(llmConfig, embeddingConfig);
|
|
1143
|
+
}
|
|
1144
|
+
};
|
|
1145
|
+
ProviderRegistry.vectorProviders = {};
|
|
1146
|
+
ProviderRegistry.graphProviders = {};
|
|
1147
|
+
ProviderRegistry.vectorValidators = {};
|
|
1148
|
+
ProviderRegistry.vectorHealthCheckers = {};
|
|
1149
|
+
ProviderRegistry.llmValidators = {};
|
|
1150
|
+
ProviderRegistry.llmHealthCheckers = {};
|
|
1151
|
+
|
|
1152
|
+
// src/core/ConfigValidator.ts
|
|
1153
|
+
var ConfigValidator = class {
|
|
1154
|
+
/**
|
|
1155
|
+
* Validates the entire RagConfig object.
|
|
1156
|
+
*/
|
|
1157
|
+
static async validate(config) {
|
|
1158
|
+
const errors = [];
|
|
1159
|
+
if (!config.projectId) {
|
|
1160
|
+
errors.push({
|
|
1161
|
+
field: "projectId",
|
|
1162
|
+
message: "projectId is required",
|
|
1163
|
+
severity: "error"
|
|
1164
|
+
});
|
|
1165
|
+
}
|
|
1166
|
+
errors.push(...await this.validateVectorDbConfig(config.vectorDb));
|
|
1167
|
+
errors.push(...await this.validateLLMConfig(config.llm));
|
|
1168
|
+
if (config.embedding) {
|
|
1169
|
+
errors.push(...await this.validateEmbeddingConfig(config.embedding));
|
|
1170
|
+
} else if (config.llm.provider === "anthropic") {
|
|
1171
|
+
errors.push({
|
|
1172
|
+
field: "embedding",
|
|
1173
|
+
message: "Embedding config is required when using Anthropic LLM",
|
|
1174
|
+
suggestion: 'Provide embedding config with provider (e.g., "openai" or "ollama")',
|
|
1175
|
+
severity: "error"
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
if (config.ui) {
|
|
1179
|
+
errors.push(...this.validateUIConfig(config.ui));
|
|
1180
|
+
}
|
|
1181
|
+
if (config.rag) {
|
|
1182
|
+
errors.push(...this.validateRAGConfig(config.rag));
|
|
1183
|
+
}
|
|
1184
|
+
return errors;
|
|
1185
|
+
}
|
|
1186
|
+
static async validateVectorDbConfig(config) {
|
|
1187
|
+
const errors = [];
|
|
1188
|
+
if (!config.provider) {
|
|
1189
|
+
errors.push({ field: "vectorDb.provider", message: "Vector database provider is required", severity: "error" });
|
|
1190
|
+
return errors;
|
|
1191
|
+
}
|
|
1192
|
+
if (!config.indexName) {
|
|
1193
|
+
errors.push({ field: "vectorDb.indexName", message: "Vector database index name is required", severity: "error" });
|
|
1194
|
+
}
|
|
1195
|
+
const validator = await ProviderRegistry.getVectorValidator(config.provider);
|
|
1196
|
+
if (validator) {
|
|
1197
|
+
errors.push(...validator.validate(config));
|
|
1198
|
+
} else {
|
|
1199
|
+
this.fallbackVectorValidation(config, errors);
|
|
1200
|
+
}
|
|
1201
|
+
return errors;
|
|
1202
|
+
}
|
|
1203
|
+
static async validateLLMConfig(config) {
|
|
1204
|
+
const errors = [];
|
|
1205
|
+
if (!config.provider) {
|
|
1206
|
+
errors.push({ field: "llm.provider", message: "LLM provider is required", severity: "error" });
|
|
1207
|
+
return errors;
|
|
1208
|
+
}
|
|
1209
|
+
const validator = LLMFactory.getValidator(config.provider);
|
|
1210
|
+
if (validator) {
|
|
1211
|
+
errors.push(...validator.validate(config));
|
|
1212
|
+
} else {
|
|
1213
|
+
this.fallbackLLMValidation(config, errors);
|
|
1214
|
+
}
|
|
1215
|
+
if (config.temperature !== void 0 && (config.temperature < 0 || config.temperature > 2)) {
|
|
1216
|
+
errors.push({ field: "llm.temperature", message: "Temperature must be between 0 and 2", severity: "error" });
|
|
1217
|
+
}
|
|
1218
|
+
return errors;
|
|
1219
|
+
}
|
|
1220
|
+
static async validateEmbeddingConfig(config) {
|
|
1221
|
+
const errors = [];
|
|
1222
|
+
if (!config.provider) {
|
|
1223
|
+
errors.push({ field: "embedding.provider", message: "Embedding provider is required", severity: "error" });
|
|
1224
|
+
return errors;
|
|
1225
|
+
}
|
|
1226
|
+
const validator = LLMFactory.getValidator(config.provider);
|
|
1227
|
+
if (validator) {
|
|
1228
|
+
errors.push(...validator.validate(config));
|
|
1229
|
+
} else {
|
|
1230
|
+
this.fallbackEmbeddingValidation(config, errors);
|
|
1231
|
+
}
|
|
1232
|
+
return errors;
|
|
1233
|
+
}
|
|
1234
|
+
/**
|
|
1235
|
+
* Temporary fallbacks for providers not yet migrated to the pluggable architecture.
|
|
1236
|
+
*/
|
|
1237
|
+
static fallbackVectorValidation(config, errors) {
|
|
1238
|
+
const opts = config.options || {};
|
|
1239
|
+
switch (config.provider) {
|
|
1240
|
+
case "milvus":
|
|
1241
|
+
if (!opts.baseUrl && !opts.uri && !(opts.host && opts.port)) {
|
|
1242
|
+
errors.push({ field: "vectorDb.options.baseUrl", message: "Milvus connection info required", severity: "error" });
|
|
1243
|
+
}
|
|
1244
|
+
break;
|
|
1245
|
+
case "qdrant":
|
|
1246
|
+
if (!opts.baseUrl && !opts.url) {
|
|
1247
|
+
errors.push({ field: "vectorDb.options.baseUrl", message: "Qdrant URL required", severity: "error" });
|
|
1248
|
+
}
|
|
1249
|
+
break;
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
static fallbackLLMValidation(config, errors) {
|
|
1253
|
+
if (!config.model) {
|
|
1254
|
+
errors.push({ field: "llm.model", message: "LLM model name is required", severity: "error" });
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
static fallbackEmbeddingValidation(config, errors) {
|
|
1258
|
+
if (!config.model) {
|
|
1259
|
+
errors.push({ field: "embedding.model", message: "Embedding model name is required", severity: "error" });
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
static validateUIConfig(config) {
|
|
1263
|
+
const errors = [];
|
|
1264
|
+
if (config.primaryColor && !this.isValidCSSColor(config.primaryColor)) {
|
|
1265
|
+
errors.push({ field: "ui.primaryColor", message: "Invalid CSS color format", severity: "warning" });
|
|
1266
|
+
}
|
|
1267
|
+
if (config.borderRadius && !UI_BORDER_RADIUS_OPTIONS.includes(config.borderRadius)) {
|
|
1268
|
+
errors.push({ field: "ui.borderRadius", message: `borderRadius must be one of: ${UI_BORDER_RADIUS_OPTIONS.join(", ")}`, severity: "warning" });
|
|
1269
|
+
}
|
|
1270
|
+
return errors;
|
|
1271
|
+
}
|
|
1272
|
+
static validateRAGConfig(config) {
|
|
1273
|
+
const errors = [];
|
|
1274
|
+
if (config.topK !== void 0 && (typeof config.topK !== "number" || config.topK <= 0)) {
|
|
1275
|
+
errors.push({ field: "rag.topK", message: "topK must be a positive integer", severity: "error" });
|
|
1276
|
+
}
|
|
1277
|
+
return errors;
|
|
1278
|
+
}
|
|
1279
|
+
static isValidCSSColor(color) {
|
|
1280
|
+
const hexRegex = /^#([0-9a-f]{3}){1,2}$/i;
|
|
1281
|
+
const rgbRegex = /^rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*(\d+(?:\.\d+)?))?\)$/i;
|
|
1282
|
+
const namedColors = ["red", "blue", "green", "black", "white", "transparent"];
|
|
1283
|
+
return hexRegex.test(color) || rgbRegex.test(color) || namedColors.includes(color.toLowerCase());
|
|
1284
|
+
}
|
|
1285
|
+
static async validateAndThrow(config) {
|
|
1286
|
+
const errors = await this.validate(config);
|
|
1287
|
+
const errorItems = errors.filter((e) => e.severity === "error");
|
|
1288
|
+
if (errorItems.length > 0) {
|
|
1289
|
+
const message = errorItems.map((e) => `${e.field}: ${e.message}${e.suggestion ? ` (${e.suggestion})` : ""}`).join("\n ");
|
|
1290
|
+
throw new Error(`[ConfigValidator] Configuration validation failed:
|
|
1291
|
+
${message}`);
|
|
1292
|
+
}
|
|
1293
|
+
}
|
|
1294
|
+
};
|
|
1295
|
+
|
|
1296
|
+
// src/rag/DocumentChunker.ts
|
|
1297
|
+
var DocumentChunker = class {
|
|
1298
|
+
constructor(chunkSize = 1e3, chunkOverlap = 200, separators = ["\n# ", "\n## ", "\n### ", "\n#### ", "\n\n", "\n", " ", ""]) {
|
|
1299
|
+
this.chunkSize = chunkSize;
|
|
1300
|
+
this.chunkOverlap = chunkOverlap;
|
|
1301
|
+
this.separators = separators;
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Split a single text string into overlapping chunks using a recursive strategy.
|
|
1305
|
+
* Preserves structural boundaries (Markdown headers) where possible.
|
|
1306
|
+
*/
|
|
1307
|
+
chunk(text, options = {}) {
|
|
1308
|
+
const {
|
|
1309
|
+
chunkSize = this.chunkSize,
|
|
1310
|
+
chunkOverlap = this.chunkOverlap,
|
|
1311
|
+
docId = `doc_${Date.now()}`,
|
|
1312
|
+
metadata = {},
|
|
1313
|
+
separators = this.separators
|
|
1314
|
+
} = options;
|
|
1315
|
+
const finalChunks = [];
|
|
1316
|
+
const splits = this.recursiveSplit(text, separators, chunkSize);
|
|
1317
|
+
let currentChunk = [];
|
|
1318
|
+
let currentLength = 0;
|
|
1319
|
+
let chunkIndex = 0;
|
|
1320
|
+
for (const split of splits) {
|
|
1321
|
+
if (currentLength + split.length > chunkSize && currentChunk.length > 0) {
|
|
1322
|
+
finalChunks.push({
|
|
1323
|
+
id: `${docId}_chunk_${chunkIndex++}`,
|
|
1324
|
+
content: currentChunk.join("").trim(),
|
|
1325
|
+
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex: chunkIndex - 1 })
|
|
1326
|
+
});
|
|
1327
|
+
const overlapItems = [];
|
|
1328
|
+
let overlapLen = 0;
|
|
1329
|
+
for (let i = currentChunk.length - 1; i >= 0; i--) {
|
|
1330
|
+
if (overlapLen + currentChunk[i].length <= chunkOverlap) {
|
|
1331
|
+
overlapItems.unshift(currentChunk[i]);
|
|
1332
|
+
overlapLen += currentChunk[i].length;
|
|
1333
|
+
} else {
|
|
1334
|
+
break;
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
currentChunk = overlapItems;
|
|
1338
|
+
currentLength = overlapLen;
|
|
1339
|
+
}
|
|
1340
|
+
currentChunk.push(split);
|
|
1341
|
+
currentLength += split.length;
|
|
1342
|
+
}
|
|
1343
|
+
if (currentChunk.length > 0) {
|
|
1344
|
+
finalChunks.push({
|
|
1345
|
+
id: `${docId}_chunk_${chunkIndex}`,
|
|
1346
|
+
content: currentChunk.join("").trim(),
|
|
1347
|
+
metadata: __spreadProps(__spreadValues({}, metadata), { docId, chunkIndex })
|
|
1338
1348
|
});
|
|
1339
|
-
} else {
|
|
1340
|
-
payload = {
|
|
1341
|
-
model: this.model,
|
|
1342
|
-
input: text
|
|
1343
|
-
};
|
|
1344
|
-
}
|
|
1345
|
-
const { data } = await this.http.post(path, payload);
|
|
1346
|
-
const extractPath = (_b = this.opts.embedExtractPath) != null ? _b : "data[0].embedding";
|
|
1347
|
-
const vector = resolvePath(data, extractPath);
|
|
1348
|
-
if (!Array.isArray(vector)) {
|
|
1349
|
-
throw new Error(`[UniversalLLMAdapter] Expected a number array at '${extractPath}' for embeddings.`);
|
|
1350
1349
|
}
|
|
1351
|
-
return
|
|
1350
|
+
return finalChunks;
|
|
1352
1351
|
}
|
|
1353
|
-
|
|
1354
|
-
const
|
|
1355
|
-
|
|
1356
|
-
|
|
1352
|
+
recursiveSplit(text, separators, chunkSize) {
|
|
1353
|
+
const finalSplits = [];
|
|
1354
|
+
let separator = separators[separators.length - 1];
|
|
1355
|
+
let nextSeparators = [];
|
|
1356
|
+
for (let i = 0; i < separators.length; i++) {
|
|
1357
|
+
const sep = separators[i];
|
|
1358
|
+
if (text.includes(sep)) {
|
|
1359
|
+
separator = sep;
|
|
1360
|
+
nextSeparators = separators.slice(i + 1);
|
|
1361
|
+
break;
|
|
1362
|
+
}
|
|
1357
1363
|
}
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
if (
|
|
1363
|
-
|
|
1364
|
+
const parts = text.split(separator);
|
|
1365
|
+
for (const part of parts) {
|
|
1366
|
+
if (part.length <= chunkSize) {
|
|
1367
|
+
finalSplits.push(part + (part === parts[parts.length - 1] ? "" : separator));
|
|
1368
|
+
} else if (nextSeparators.length > 0) {
|
|
1369
|
+
finalSplits.push(...this.recursiveSplit(part, nextSeparators, chunkSize));
|
|
1370
|
+
} else {
|
|
1371
|
+
finalSplits.push(part);
|
|
1364
1372
|
}
|
|
1365
|
-
return true;
|
|
1366
|
-
} catch (err) {
|
|
1367
|
-
console.error("[UniversalLLMAdapter] Ping failed:", err);
|
|
1368
|
-
return false;
|
|
1369
1373
|
}
|
|
1374
|
+
return finalSplits;
|
|
1375
|
+
}
|
|
1376
|
+
chunkMany(documents) {
|
|
1377
|
+
return documents.flatMap(
|
|
1378
|
+
(doc) => this.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata })
|
|
1379
|
+
);
|
|
1370
1380
|
}
|
|
1371
1381
|
};
|
|
1372
1382
|
|
|
1373
|
-
// src/
|
|
1374
|
-
var
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
switch (llmConfig.provider) {
|
|
1378
|
-
case "openai":
|
|
1379
|
-
return new OpenAIProvider(llmConfig, embeddingConfig);
|
|
1380
|
-
case "anthropic":
|
|
1381
|
-
return new AnthropicProvider(llmConfig, embeddingConfig);
|
|
1382
|
-
case "ollama":
|
|
1383
|
-
return new OllamaProvider(llmConfig, embeddingConfig);
|
|
1384
|
-
case "gemini":
|
|
1385
|
-
return new GeminiProvider(llmConfig, embeddingConfig);
|
|
1386
|
-
case "rest":
|
|
1387
|
-
case "universal_rest":
|
|
1388
|
-
case "custom":
|
|
1389
|
-
return new UniversalLLMAdapter(llmConfig);
|
|
1390
|
-
default:
|
|
1391
|
-
if (llmConfig.baseUrl || ((_a = llmConfig.options) == null ? void 0 : _a.baseUrl)) {
|
|
1392
|
-
return new UniversalLLMAdapter(llmConfig);
|
|
1393
|
-
}
|
|
1394
|
-
throw new Error(
|
|
1395
|
-
`[LLMFactory] Unknown provider "${llmConfig.provider}". Supported: openai | anthropic | ollama | rest | custom`
|
|
1396
|
-
);
|
|
1397
|
-
}
|
|
1383
|
+
// src/rag/EntityExtractor.ts
|
|
1384
|
+
var EntityExtractor = class {
|
|
1385
|
+
constructor(llm) {
|
|
1386
|
+
this.llm = llm;
|
|
1398
1387
|
}
|
|
1399
1388
|
/**
|
|
1400
|
-
*
|
|
1401
|
-
* Useful when the LLM provider (e.g. Anthropic) doesn't support embeddings.
|
|
1389
|
+
* Extract nodes and edges from a text chunk.
|
|
1402
1390
|
*/
|
|
1403
|
-
|
|
1404
|
-
const
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1391
|
+
async extract(text) {
|
|
1392
|
+
const prompt = `
|
|
1393
|
+
Extract entities and relationships from the following text.
|
|
1394
|
+
Format the output as JSON with two keys: "nodes" (array of {id, label, properties}) and "edges" (array of {source, target, type, properties}).
|
|
1395
|
+
Use the same ID for the same entity.
|
|
1396
|
+
|
|
1397
|
+
IMPORTANT: Ensure all property values are valid JSON types (strings, numbers, booleans, or null).
|
|
1398
|
+
DO NOT include mathematical expressions like "4.5/5" as numbers; use strings instead.
|
|
1399
|
+
|
|
1400
|
+
Text:
|
|
1401
|
+
"${text}"
|
|
1402
|
+
|
|
1403
|
+
Output JSON:
|
|
1404
|
+
`;
|
|
1405
|
+
const response = await this.llm.chat([
|
|
1406
|
+
{ role: "system", content: "You are an expert at knowledge graph extraction. Respond only with valid JSON." },
|
|
1407
|
+
{ role: "user", content: prompt }
|
|
1408
|
+
], "");
|
|
1409
|
+
try {
|
|
1410
|
+
const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
|
|
1411
|
+
return JSON.parse(cleanJson);
|
|
1412
|
+
} catch (e) {
|
|
1413
|
+
console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
|
|
1414
|
+
return { nodes: [], edges: [] };
|
|
1415
|
+
}
|
|
1412
1416
|
}
|
|
1413
1417
|
};
|
|
1414
1418
|
|
|
1415
|
-
// src/
|
|
1416
|
-
var
|
|
1417
|
-
static registerVectorProvider(name, providerClass) {
|
|
1418
|
-
this.vectorProviders[name] = providerClass;
|
|
1419
|
-
}
|
|
1419
|
+
// src/rag/Reranker.ts
|
|
1420
|
+
var Reranker = class {
|
|
1420
1421
|
/**
|
|
1421
|
-
*
|
|
1422
|
+
* Re-ranks matches based on a secondary relevance score.
|
|
1423
|
+
* In a production environment, this would call a Cross-Encoder model.
|
|
1424
|
+
* Here we implement a placeholder that filters by score and limits count.
|
|
1422
1425
|
*/
|
|
1423
|
-
|
|
1424
|
-
|
|
1426
|
+
async rerank(matches, query, limit = 5) {
|
|
1427
|
+
return matches.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
1425
1428
|
}
|
|
1429
|
+
};
|
|
1430
|
+
|
|
1431
|
+
// src/rag/LlamaIndexIngestor.ts
|
|
1432
|
+
var LlamaIndexIngestor = class {
|
|
1426
1433
|
/**
|
|
1427
|
-
*
|
|
1428
|
-
*
|
|
1434
|
+
* Chunks document content using LlamaIndex SentenceSplitter.
|
|
1435
|
+
* This respects sentence and paragraph boundaries much more effectively
|
|
1436
|
+
* than standard character-count splitting.
|
|
1429
1437
|
*/
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
return new MilvusProvider(config);
|
|
1452
|
-
}
|
|
1453
|
-
case "qdrant": {
|
|
1454
|
-
const { QdrantProvider } = await import("./QdrantProvider-XVDVBNIG.mjs");
|
|
1455
|
-
return new QdrantProvider(config);
|
|
1456
|
-
}
|
|
1457
|
-
case "chromadb": {
|
|
1458
|
-
const { ChromaDBProvider } = await import("./ChromaDBProvider-GI7TB7GJ.mjs");
|
|
1459
|
-
return new ChromaDBProvider(config);
|
|
1460
|
-
}
|
|
1461
|
-
case "redis": {
|
|
1462
|
-
const { RedisProvider } = await import("./RedisProvider-EK2R2PQH.mjs");
|
|
1463
|
-
return new RedisProvider(config);
|
|
1464
|
-
}
|
|
1465
|
-
case "weaviate": {
|
|
1466
|
-
const { WeaviateProvider } = await import("./WeaviateProvider-4CAPQ7UY.mjs");
|
|
1467
|
-
return new WeaviateProvider(config);
|
|
1468
|
-
}
|
|
1469
|
-
case "universal_rest":
|
|
1470
|
-
case "rest": {
|
|
1471
|
-
const { UniversalVectorProvider } = await import("./UniversalVectorProvider-YIDRX6VT.mjs");
|
|
1472
|
-
return new UniversalVectorProvider(config);
|
|
1473
|
-
}
|
|
1474
|
-
default:
|
|
1475
|
-
throw new Error(
|
|
1476
|
-
`[ProviderRegistry] Unsupported vector provider: "${provider}". Built-in providers: pinecone | pgvector | postgresql | mongodb | milvus | qdrant | chromadb | redis | weaviate | universal_rest. For custom providers, call ProviderRegistry.registerVectorProvider("${provider}", YourClass).`
|
|
1477
|
-
);
|
|
1438
|
+
async chunk(text, options = {}) {
|
|
1439
|
+
var _a, _b;
|
|
1440
|
+
try {
|
|
1441
|
+
const { Document, SentenceSplitter, MetadataMode } = await import(`${"llamaindex"}`);
|
|
1442
|
+
const splitter = new SentenceSplitter({
|
|
1443
|
+
chunkSize: (_a = options.chunkSize) != null ? _a : 1e3,
|
|
1444
|
+
chunkOverlap: (_b = options.chunkOverlap) != null ? _b : 200
|
|
1445
|
+
});
|
|
1446
|
+
const doc = new Document({ text, metadata: options.metadata || {} });
|
|
1447
|
+
const nodes = splitter.getNodesFromDocuments([doc]);
|
|
1448
|
+
return nodes.map((node, index) => ({
|
|
1449
|
+
id: `${options.docId || "doc"}_node_${index}`,
|
|
1450
|
+
content: node.getContent(MetadataMode.ALL),
|
|
1451
|
+
metadata: __spreadProps(__spreadValues(__spreadValues({}, options.metadata), node.metadata), {
|
|
1452
|
+
nodeId: node.id_,
|
|
1453
|
+
chunkIndex: index
|
|
1454
|
+
})
|
|
1455
|
+
}));
|
|
1456
|
+
} catch (error) {
|
|
1457
|
+
console.warn("[LlamaIndexIngestor] LlamaIndex package not found or failed. Falling back to default chunker.");
|
|
1458
|
+
throw error;
|
|
1478
1459
|
}
|
|
1479
1460
|
}
|
|
1480
1461
|
/**
|
|
1481
|
-
*
|
|
1462
|
+
* Batch processing for multiple documents.
|
|
1482
1463
|
*/
|
|
1483
|
-
|
|
1484
|
-
const
|
|
1485
|
-
|
|
1486
|
-
|
|
1464
|
+
async chunkMany(documents, options = {}) {
|
|
1465
|
+
const allChunks = [];
|
|
1466
|
+
for (const doc of documents) {
|
|
1467
|
+
const chunks = await this.chunk(doc.content, __spreadProps(__spreadValues({}, options), { docId: doc.docId, metadata: doc.metadata }));
|
|
1468
|
+
allChunks.push(...chunks);
|
|
1487
1469
|
}
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1470
|
+
return allChunks;
|
|
1471
|
+
}
|
|
1472
|
+
};
|
|
1473
|
+
|
|
1474
|
+
// src/core/LangChainAgent.ts
|
|
1475
|
+
var LangChainAgent = class {
|
|
1476
|
+
constructor(pipeline, config) {
|
|
1477
|
+
this.pipeline = pipeline;
|
|
1478
|
+
this.config = config;
|
|
1479
|
+
}
|
|
1480
|
+
/**
|
|
1481
|
+
* Initializes the agent with the RAG pipeline as a primary tool.
|
|
1482
|
+
* Dynamically imports LangChain dependencies to avoid build errors if missing.
|
|
1483
|
+
*/
|
|
1484
|
+
async initialize(chatModel) {
|
|
1485
|
+
try {
|
|
1486
|
+
const { DynamicTool } = await import(`${"@langchain/core/tools"}`);
|
|
1487
|
+
const { ChatPromptTemplate, MessagesPlaceholder } = await import(`${"@langchain/core/prompts"}`);
|
|
1488
|
+
const { AgentExecutor, createOpenAIFunctionsAgent } = await import(`${"langchain/agents"}`);
|
|
1489
|
+
const searchTool = new DynamicTool({
|
|
1490
|
+
name: "document_search",
|
|
1491
|
+
description: "Use this tool to search through the knowledge base and document repository. Input should be a specific search query.",
|
|
1492
|
+
func: async (query) => {
|
|
1493
|
+
const response = await this.pipeline.ask(query);
|
|
1494
|
+
return `Search Results:
|
|
1495
|
+
${response.reply}
|
|
1496
|
+
|
|
1497
|
+
Sources Used: ${JSON.stringify(response.sources.map((s) => s.id))}`;
|
|
1498
|
+
}
|
|
1499
|
+
});
|
|
1500
|
+
const tools = [searchTool];
|
|
1501
|
+
const prompt = ChatPromptTemplate.fromMessages([
|
|
1502
|
+
["system", this.config.llm.systemPrompt || "You are a helpful AI assistant with access to a document search tool."],
|
|
1503
|
+
new MessagesPlaceholder("chat_history"),
|
|
1504
|
+
["human", "{input}"],
|
|
1505
|
+
new MessagesPlaceholder("agent_scratchpad")
|
|
1506
|
+
]);
|
|
1507
|
+
const agent = await createOpenAIFunctionsAgent({
|
|
1508
|
+
llm: chatModel,
|
|
1509
|
+
tools,
|
|
1510
|
+
prompt
|
|
1511
|
+
});
|
|
1512
|
+
this.executor = new AgentExecutor({
|
|
1513
|
+
agent,
|
|
1514
|
+
tools
|
|
1515
|
+
});
|
|
1516
|
+
} catch (error) {
|
|
1517
|
+
console.error("[LangChainAgent] Failed to initialize. Ensure 'langchain' and '@langchain/core' are installed.");
|
|
1518
|
+
throw error;
|
|
1500
1519
|
}
|
|
1501
1520
|
}
|
|
1502
1521
|
/**
|
|
1503
|
-
*
|
|
1522
|
+
* Run the agentic flow.
|
|
1504
1523
|
*/
|
|
1505
|
-
|
|
1506
|
-
|
|
1524
|
+
async run(input, chatHistory = []) {
|
|
1525
|
+
if (!this.executor) {
|
|
1526
|
+
throw new Error("[LangChainAgent] Agent not initialized. Call initialize() first.");
|
|
1527
|
+
}
|
|
1528
|
+
const response = await this.executor.invoke({
|
|
1529
|
+
input,
|
|
1530
|
+
chat_history: chatHistory
|
|
1531
|
+
});
|
|
1532
|
+
return response.output;
|
|
1507
1533
|
}
|
|
1508
1534
|
};
|
|
1509
|
-
ProviderRegistry.vectorProviders = {};
|
|
1510
|
-
ProviderRegistry.graphProviders = {};
|
|
1511
1535
|
|
|
1512
1536
|
// src/core/BatchProcessor.ts
|
|
1513
1537
|
function isTransientError(error) {
|
|
@@ -1815,122 +1839,143 @@ var EmbeddingStrategyResolver = class {
|
|
|
1815
1839
|
}
|
|
1816
1840
|
};
|
|
1817
1841
|
|
|
1818
|
-
// src/core/
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1842
|
+
// src/core/QueryProcessor.ts
|
|
1843
|
+
var QueryProcessor = class {
|
|
1844
|
+
/**
|
|
1845
|
+
* Normalizes a string value by collapsing whitespace and trimming.
|
|
1846
|
+
*/
|
|
1847
|
+
static normalizeHintValue(value) {
|
|
1848
|
+
return value.replace(/\s+/g, " ").trim();
|
|
1849
|
+
}
|
|
1850
|
+
/**
|
|
1851
|
+
* Checks if a string is likely a question word or common prompt phrase.
|
|
1852
|
+
*/
|
|
1853
|
+
static isLikelyPromptPhrase(value) {
|
|
1854
|
+
return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
|
|
1855
|
+
}
|
|
1856
|
+
/**
|
|
1857
|
+
* Scans a natural language question for potential metadata hints and keywords.
|
|
1858
|
+
*/
|
|
1859
|
+
static extractQueryFieldHints(question) {
|
|
1860
|
+
var _a, _b, _c, _d;
|
|
1861
|
+
if (!question.trim()) return [];
|
|
1862
|
+
const hints = /* @__PURE__ */ new Map();
|
|
1863
|
+
const addHint = (value, field) => {
|
|
1864
|
+
const normalizedValue = this.normalizeHintValue(value);
|
|
1865
|
+
if (!normalizedValue) return;
|
|
1866
|
+
const normalizedField = field ? field.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim() : void 0;
|
|
1867
|
+
const key = `${normalizedField != null ? normalizedField : "*"}::${normalizedValue.toLowerCase()}`;
|
|
1868
|
+
if (!hints.has(key)) {
|
|
1869
|
+
hints.set(key, __spreadValues({
|
|
1870
|
+
value: normalizedValue
|
|
1871
|
+
}, normalizedField ? { field: normalizedField } : {}));
|
|
1872
|
+
}
|
|
1873
|
+
};
|
|
1874
|
+
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
1875
|
+
addHint(match[1]);
|
|
1838
1876
|
}
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
for (const pattern of personCompanyPatterns) {
|
|
1854
|
-
for (const match of question.matchAll(pattern)) {
|
|
1855
|
-
const name = match[1];
|
|
1856
|
-
if (name) addHint(name, "name");
|
|
1857
|
-
}
|
|
1858
|
-
}
|
|
1859
|
-
const universalPatterns = [
|
|
1860
|
-
{ regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: "email", group: 1 },
|
|
1861
|
-
{ regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: "phone", group: 1 },
|
|
1862
|
-
{ regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: "date", group: 1 },
|
|
1863
|
-
{ regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: "amount", group: 1 },
|
|
1864
|
-
{ regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: "id", group: 2 },
|
|
1865
|
-
// Generic quoted phrase / proper-noun sequences as keywords (already partially handled above)
|
|
1866
|
-
{ regex: /"([^"]{2,120})"/g, group: 1 },
|
|
1867
|
-
{ regex: /'([^']{2,120})'/g, group: 1 }
|
|
1868
|
-
];
|
|
1869
|
-
for (const p of universalPatterns) {
|
|
1870
|
-
for (const match of question.matchAll(p.regex)) {
|
|
1871
|
-
const val = p.group ? (_a = match[p.group]) != null ? _a : match[0] : match[0];
|
|
1872
|
-
if (!val) continue;
|
|
1873
|
-
if (p.field) addHint(val, p.field);
|
|
1874
|
-
else addHint(val);
|
|
1875
|
-
}
|
|
1876
|
-
}
|
|
1877
|
-
for (const pattern of naturalQuestionPatterns) {
|
|
1878
|
-
for (const match of question.matchAll(pattern)) {
|
|
1879
|
-
const value = (_b = match[2]) != null ? _b : match[1];
|
|
1880
|
-
if (value) addHint(value);
|
|
1881
|
-
}
|
|
1882
|
-
}
|
|
1883
|
-
const fieldPattern = `([^\\n:=?.!,]{1,60}?)`;
|
|
1884
|
-
const valuePattern = `([^\\n?.!,]{1,120}?)`;
|
|
1885
|
-
const fieldValuePatterns = [
|
|
1886
|
-
new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
1887
|
-
new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
1888
|
-
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
1889
|
-
];
|
|
1890
|
-
for (const pattern of fieldValuePatterns) {
|
|
1891
|
-
for (const match of question.matchAll(pattern)) {
|
|
1892
|
-
const field = normalizeHintValue((_c = match[1]) != null ? _c : "");
|
|
1893
|
-
const value = (_d = match[2]) != null ? _d : "";
|
|
1894
|
-
if (field && !isLikelyPromptPhrase(field)) {
|
|
1895
|
-
addHint(value, field);
|
|
1896
|
-
} else {
|
|
1897
|
-
addHint(value);
|
|
1877
|
+
const naturalQuestionPatterns = [
|
|
1878
|
+
/\b(?:what|which)\s+(?:is|are|was|were)\s+(?:the\s+)?([^?.!,]{1,60}?)\s+of\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
1879
|
+
/\b(?:who|what)\s+(?:is|are|was|were)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
1880
|
+
/\b(?:about|for|regarding)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi
|
|
1881
|
+
];
|
|
1882
|
+
const personCompanyPatterns = [
|
|
1883
|
+
/\bcompany(?:\s+name)?\s+(?:of|for)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
|
|
1884
|
+
/\b(?:which|what)\s+company\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
|
|
1885
|
+
/\bwhere\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi
|
|
1886
|
+
];
|
|
1887
|
+
for (const pattern of personCompanyPatterns) {
|
|
1888
|
+
for (const match of question.matchAll(pattern)) {
|
|
1889
|
+
const name = match[1];
|
|
1890
|
+
if (name) addHint(name, "name");
|
|
1898
1891
|
}
|
|
1899
1892
|
}
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
}
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1893
|
+
const universalPatterns = [
|
|
1894
|
+
{ regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: "email", group: 1 },
|
|
1895
|
+
{ regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: "phone", group: 1 },
|
|
1896
|
+
{ regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: "date", group: 1 },
|
|
1897
|
+
{ regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: "amount", group: 1 },
|
|
1898
|
+
{ regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: "id", group: 2 },
|
|
1899
|
+
{ regex: /"([^"]{2,120})"/g, group: 1 },
|
|
1900
|
+
{ regex: /'([^']{2,120})'/g, group: 1 }
|
|
1901
|
+
];
|
|
1902
|
+
for (const p of universalPatterns) {
|
|
1903
|
+
for (const match of question.matchAll(p.regex)) {
|
|
1904
|
+
const val = p.group ? (_a = match[p.group]) != null ? _a : match[0] : match[0];
|
|
1905
|
+
if (!val) continue;
|
|
1906
|
+
if (p.field) addHint(val, p.field);
|
|
1907
|
+
else addHint(val);
|
|
1908
|
+
}
|
|
1913
1909
|
}
|
|
1910
|
+
for (const pattern of naturalQuestionPatterns) {
|
|
1911
|
+
for (const match of question.matchAll(pattern)) {
|
|
1912
|
+
const value = (_b = match[2]) != null ? _b : match[1];
|
|
1913
|
+
if (value) addHint(value);
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
const fieldPattern = `([^\\n:=?.!,]{1,60}?)`;
|
|
1917
|
+
const valuePattern = `([^\\n?.!,]{1,120}?)`;
|
|
1918
|
+
const fieldValuePatterns = [
|
|
1919
|
+
new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
1920
|
+
new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
|
|
1921
|
+
new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
|
|
1922
|
+
];
|
|
1923
|
+
for (const pattern of fieldValuePatterns) {
|
|
1924
|
+
for (const match of question.matchAll(pattern)) {
|
|
1925
|
+
const field = this.normalizeHintValue((_c = match[1]) != null ? _c : "");
|
|
1926
|
+
const value = (_d = match[2]) != null ? _d : "";
|
|
1927
|
+
if (field && !this.isLikelyPromptPhrase(field)) {
|
|
1928
|
+
addHint(value, field);
|
|
1929
|
+
} else {
|
|
1930
|
+
addHint(value);
|
|
1931
|
+
}
|
|
1932
|
+
}
|
|
1933
|
+
}
|
|
1934
|
+
for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
|
|
1935
|
+
addHint(match[0]);
|
|
1936
|
+
}
|
|
1937
|
+
return [...hints.values()];
|
|
1914
1938
|
}
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1939
|
+
/**
|
|
1940
|
+
* Constructs a QueryFilter object from extracted hints.
|
|
1941
|
+
*/
|
|
1942
|
+
static buildQueryFilter(question, hints) {
|
|
1943
|
+
const filter = { metadata: {}, keywords: [], queryText: question };
|
|
1944
|
+
for (const hint of hints) {
|
|
1945
|
+
if (hint.field) {
|
|
1946
|
+
filter.metadata[hint.field] = hint.value;
|
|
1947
|
+
} else {
|
|
1948
|
+
filter.keywords.push(hint.value);
|
|
1949
|
+
}
|
|
1950
|
+
}
|
|
1951
|
+
for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
|
|
1952
|
+
const term = this.normalizeHintValue(match[0]);
|
|
1953
|
+
if (term && !filter.keywords.includes(term)) filter.keywords.push(term);
|
|
1954
|
+
}
|
|
1955
|
+
if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
|
|
1956
|
+
if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
|
|
1957
|
+
return filter;
|
|
1918
1958
|
}
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
}
|
|
1959
|
+
};
|
|
1960
|
+
|
|
1961
|
+
// src/core/Pipeline.ts
|
|
1923
1962
|
var Pipeline = class {
|
|
1924
1963
|
constructor(config) {
|
|
1925
|
-
this.initialised = false;
|
|
1926
|
-
var _a, _b, _c, _d;
|
|
1927
1964
|
this.config = config;
|
|
1965
|
+
this.embeddingCache = /* @__PURE__ */ new Map();
|
|
1966
|
+
this.initialised = false;
|
|
1967
|
+
var _a, _b, _c, _d, _e;
|
|
1928
1968
|
this.chunker = new DocumentChunker(
|
|
1929
1969
|
(_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
|
|
1930
1970
|
(_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
|
|
1931
1971
|
);
|
|
1972
|
+
if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
|
|
1973
|
+
this.llamaIngestor = new LlamaIndexIngestor();
|
|
1974
|
+
}
|
|
1975
|
+
this.reranker = new Reranker();
|
|
1932
1976
|
}
|
|
1933
1977
|
async initialize() {
|
|
1978
|
+
var _a;
|
|
1934
1979
|
if (this.initialised) return;
|
|
1935
1980
|
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
1936
1981
|
const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
@@ -1945,6 +1990,10 @@ var Pipeline = class {
|
|
|
1945
1990
|
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
1946
1991
|
}
|
|
1947
1992
|
await this.vectorDB.initialize();
|
|
1993
|
+
if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic") {
|
|
1994
|
+
this.agent = new LangChainAgent(this, this.config);
|
|
1995
|
+
await this.agent.initialize(this.llmProvider);
|
|
1996
|
+
}
|
|
1948
1997
|
this.initialised = true;
|
|
1949
1998
|
}
|
|
1950
1999
|
/**
|
|
@@ -1957,69 +2006,21 @@ var Pipeline = class {
|
|
|
1957
2006
|
const results = [];
|
|
1958
2007
|
for (const doc of documents) {
|
|
1959
2008
|
try {
|
|
1960
|
-
const chunks = this.
|
|
1961
|
-
|
|
1962
|
-
metadata: doc.metadata
|
|
1963
|
-
});
|
|
1964
|
-
const embedBatchOptions = {
|
|
1965
|
-
batchSize: 50,
|
|
1966
|
-
maxRetries: 3,
|
|
1967
|
-
initialDelayMs: 100
|
|
1968
|
-
};
|
|
1969
|
-
const vectors = await BatchProcessor.mapWithRetry(
|
|
1970
|
-
chunks.map((c) => c.content),
|
|
1971
|
-
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
1972
|
-
embedBatchOptions
|
|
1973
|
-
);
|
|
1974
|
-
if (vectors.length !== chunks.length) {
|
|
1975
|
-
throw new Error(`Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
|
|
1976
|
-
}
|
|
2009
|
+
const chunks = await this.prepareChunks(doc);
|
|
2010
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
1977
2011
|
const upsertDocs = chunks.map((chunk, i) => ({
|
|
1978
2012
|
id: chunk.id,
|
|
1979
2013
|
vector: vectors[i],
|
|
1980
2014
|
content: chunk.content,
|
|
1981
2015
|
metadata: chunk.metadata
|
|
1982
|
-
}));
|
|
1983
|
-
const
|
|
1984
|
-
batchSize: 100,
|
|
1985
|
-
maxRetries: 3,
|
|
1986
|
-
initialDelayMs: 100
|
|
1987
|
-
};
|
|
1988
|
-
const upsertResult = await BatchProcessor.processBatch(
|
|
1989
|
-
upsertDocs,
|
|
1990
|
-
(batch) => this.vectorDB.batchUpsert(batch, ns),
|
|
1991
|
-
upsertBatchOptions
|
|
1992
|
-
);
|
|
1993
|
-
if (upsertResult.errors.length > 0) {
|
|
1994
|
-
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed for doc ${doc.docId}`);
|
|
1995
|
-
}
|
|
2016
|
+
}));
|
|
2017
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
1996
2018
|
results.push({
|
|
1997
2019
|
docId: doc.docId,
|
|
1998
|
-
chunksIngested:
|
|
2020
|
+
chunksIngested: totalProcessed
|
|
1999
2021
|
});
|
|
2000
2022
|
if (this.graphDB && this.entityExtractor) {
|
|
2001
|
-
|
|
2002
|
-
const extractionOptions = {
|
|
2003
|
-
batchSize: 2,
|
|
2004
|
-
// Low concurrency for LLM extraction
|
|
2005
|
-
maxRetries: 1,
|
|
2006
|
-
initialDelayMs: 500
|
|
2007
|
-
};
|
|
2008
|
-
await BatchProcessor.processBatch(
|
|
2009
|
-
chunks,
|
|
2010
|
-
async (batch) => {
|
|
2011
|
-
for (const chunk of batch) {
|
|
2012
|
-
try {
|
|
2013
|
-
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
2014
|
-
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
2015
|
-
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
2016
|
-
} catch (err) {
|
|
2017
|
-
console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
|
|
2018
|
-
}
|
|
2019
|
-
}
|
|
2020
|
-
},
|
|
2021
|
-
extractionOptions
|
|
2022
|
-
);
|
|
2023
|
+
await this.processGraphIngestion(doc.docId, chunks);
|
|
2023
2024
|
}
|
|
2024
2025
|
} catch (error) {
|
|
2025
2026
|
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
@@ -2028,45 +2029,210 @@ var Pipeline = class {
|
|
|
2028
2029
|
}
|
|
2029
2030
|
return results;
|
|
2030
2031
|
}
|
|
2032
|
+
/**
|
|
2033
|
+
* Step 1: Chunk the document content.
|
|
2034
|
+
*/
|
|
2035
|
+
async prepareChunks(doc) {
|
|
2036
|
+
var _a, _b;
|
|
2037
|
+
if (this.llamaIngestor) {
|
|
2038
|
+
return await this.llamaIngestor.chunk(doc.content, {
|
|
2039
|
+
docId: doc.docId,
|
|
2040
|
+
metadata: doc.metadata,
|
|
2041
|
+
chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
|
|
2042
|
+
chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
|
|
2043
|
+
});
|
|
2044
|
+
}
|
|
2045
|
+
return this.chunker.chunk(doc.content, {
|
|
2046
|
+
docId: doc.docId,
|
|
2047
|
+
metadata: doc.metadata
|
|
2048
|
+
});
|
|
2049
|
+
}
|
|
2050
|
+
/**
|
|
2051
|
+
* Step 2: Generate embeddings for chunks with retry logic.
|
|
2052
|
+
*/
|
|
2053
|
+
async processEmbeddings(chunks) {
|
|
2054
|
+
const embedBatchOptions = {
|
|
2055
|
+
batchSize: 50,
|
|
2056
|
+
maxRetries: 3,
|
|
2057
|
+
initialDelayMs: 100
|
|
2058
|
+
};
|
|
2059
|
+
const vectors = await BatchProcessor.mapWithRetry(
|
|
2060
|
+
chunks.map((c) => c.content),
|
|
2061
|
+
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
2062
|
+
embedBatchOptions
|
|
2063
|
+
);
|
|
2064
|
+
if (vectors.length !== chunks.length) {
|
|
2065
|
+
throw new Error(`Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
|
|
2066
|
+
}
|
|
2067
|
+
return vectors;
|
|
2068
|
+
}
|
|
2069
|
+
/**
|
|
2070
|
+
* Step 3: Upsert chunks to vector database with retry logic.
|
|
2071
|
+
*/
|
|
2072
|
+
async processUpserts(upsertDocs, namespace) {
|
|
2073
|
+
const upsertBatchOptions = {
|
|
2074
|
+
batchSize: 100,
|
|
2075
|
+
maxRetries: 3,
|
|
2076
|
+
initialDelayMs: 100
|
|
2077
|
+
};
|
|
2078
|
+
const upsertResult = await BatchProcessor.processBatch(
|
|
2079
|
+
upsertDocs,
|
|
2080
|
+
(batch) => this.vectorDB.batchUpsert(batch, namespace),
|
|
2081
|
+
upsertBatchOptions
|
|
2082
|
+
);
|
|
2083
|
+
if (upsertResult.errors.length > 0) {
|
|
2084
|
+
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
|
|
2085
|
+
}
|
|
2086
|
+
return upsertResult.totalProcessed;
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Step 4: Optional graph-based entity extraction and ingestion.
|
|
2090
|
+
*/
|
|
2091
|
+
async processGraphIngestion(docId, chunks) {
|
|
2092
|
+
console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
|
|
2093
|
+
const extractionOptions = {
|
|
2094
|
+
batchSize: 2,
|
|
2095
|
+
// Low concurrency for LLM extraction
|
|
2096
|
+
maxRetries: 1,
|
|
2097
|
+
initialDelayMs: 500
|
|
2098
|
+
};
|
|
2099
|
+
await BatchProcessor.processBatch(
|
|
2100
|
+
chunks,
|
|
2101
|
+
async (batch) => {
|
|
2102
|
+
for (const chunk of batch) {
|
|
2103
|
+
try {
|
|
2104
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
2105
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
2106
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
2107
|
+
} catch (err) {
|
|
2108
|
+
console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
|
|
2109
|
+
}
|
|
2110
|
+
}
|
|
2111
|
+
},
|
|
2112
|
+
extractionOptions
|
|
2113
|
+
);
|
|
2114
|
+
}
|
|
2031
2115
|
async ask(question, history = [], namespace) {
|
|
2032
|
-
var _a
|
|
2116
|
+
var _a;
|
|
2033
2117
|
await this.initialize();
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2118
|
+
if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
|
|
2119
|
+
console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
|
|
2120
|
+
const agentReply = await this.agent.run(question, history);
|
|
2121
|
+
return { reply: agentReply, sources: [] };
|
|
2122
|
+
}
|
|
2123
|
+
const stream = this.askStream(question, history, namespace);
|
|
2124
|
+
let reply = "";
|
|
2125
|
+
let sources = [];
|
|
2126
|
+
let graphData;
|
|
2037
2127
|
try {
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2128
|
+
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
2129
|
+
const chunk = temp.value;
|
|
2130
|
+
if (typeof chunk === "string") {
|
|
2131
|
+
reply += chunk;
|
|
2132
|
+
} else if ("sources" in chunk) {
|
|
2133
|
+
sources = chunk.sources;
|
|
2134
|
+
graphData = chunk.graphData;
|
|
2135
|
+
}
|
|
2041
2136
|
}
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
graphData = await this.graphDB.query(searchQuery);
|
|
2137
|
+
} catch (temp) {
|
|
2138
|
+
error = [temp];
|
|
2139
|
+
} finally {
|
|
2140
|
+
try {
|
|
2141
|
+
more && (temp = iter.return) && await temp.call(iter);
|
|
2142
|
+
} finally {
|
|
2143
|
+
if (error)
|
|
2144
|
+
throw error[0];
|
|
2051
2145
|
}
|
|
2052
|
-
|
|
2146
|
+
}
|
|
2147
|
+
return { reply, sources, graphData };
|
|
2148
|
+
}
|
|
2149
|
+
/**
|
|
2150
|
+
* High-performance streaming RAG flow.
|
|
2151
|
+
* Yields text chunks first, then the retrieval metadata at the end.
|
|
2152
|
+
*/
|
|
2153
|
+
askStream(_0) {
|
|
2154
|
+
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
2155
|
+
var _a, _b, _c, _d, _e, _f;
|
|
2156
|
+
yield new __await(this.initialize());
|
|
2157
|
+
const ns = namespace != null ? namespace : this.config.projectId;
|
|
2158
|
+
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
2159
|
+
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
2160
|
+
try {
|
|
2161
|
+
let searchQuery = question;
|
|
2162
|
+
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
2163
|
+
searchQuery = yield new __await(this.rewriteQuery(question, history));
|
|
2164
|
+
}
|
|
2165
|
+
const hints = QueryProcessor.extractQueryFieldHints(question);
|
|
2166
|
+
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
2167
|
+
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
2168
|
+
namespace: ns,
|
|
2169
|
+
topK: topK * 2,
|
|
2170
|
+
filter
|
|
2171
|
+
}));
|
|
2172
|
+
let sources = rawSources.filter((m) => m.score >= scoreThreshold);
|
|
2173
|
+
if ((_f = this.config.rag) == null ? void 0 : _f.useReranking) {
|
|
2174
|
+
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
2175
|
+
} else {
|
|
2176
|
+
sources = sources.slice(0, topK);
|
|
2177
|
+
}
|
|
2178
|
+
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
2053
2179
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2180
|
+
if (graphData && graphData.nodes.length > 0) {
|
|
2181
|
+
const graphContext = graphData.nodes.map(
|
|
2182
|
+
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
2183
|
+
).join("\n");
|
|
2184
|
+
context = `GRAPH KNOWLEDGE:
|
|
2059
2185
|
${graphContext}
|
|
2060
2186
|
|
|
2061
2187
|
VECTOR CONTEXT:
|
|
2062
2188
|
${context}`;
|
|
2189
|
+
}
|
|
2190
|
+
const messages = [...history, { role: "user", content: question }];
|
|
2191
|
+
if (this.llmProvider.chatStream) {
|
|
2192
|
+
try {
|
|
2193
|
+
for (var iter = __forAwait(this.llmProvider.chatStream(messages, context)), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
2194
|
+
const chunk = temp.value;
|
|
2195
|
+
yield chunk;
|
|
2196
|
+
}
|
|
2197
|
+
} catch (temp) {
|
|
2198
|
+
error = [temp];
|
|
2199
|
+
} finally {
|
|
2200
|
+
try {
|
|
2201
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
2202
|
+
} finally {
|
|
2203
|
+
if (error)
|
|
2204
|
+
throw error[0];
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
} else {
|
|
2208
|
+
const reply = yield new __await(this.llmProvider.chat(messages, context));
|
|
2209
|
+
yield reply;
|
|
2210
|
+
}
|
|
2211
|
+
yield { reply: "", sources, graphData };
|
|
2212
|
+
} catch (error2) {
|
|
2213
|
+
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
2063
2214
|
}
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2215
|
+
});
|
|
2216
|
+
}
|
|
2217
|
+
/**
|
|
2218
|
+
* Universal retrieval method combining all enabled providers.
|
|
2219
|
+
*/
|
|
2220
|
+
async retrieve(query, options) {
|
|
2221
|
+
var _a, _b, _c;
|
|
2222
|
+
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
2223
|
+
const topK = (_b = options.topK) != null ? _b : 5;
|
|
2224
|
+
const cacheKey = `${ns}::${query}`;
|
|
2225
|
+
let queryVector = this.embeddingCache.get(cacheKey);
|
|
2226
|
+
const [retrievedVector, graphData] = await Promise.all([
|
|
2227
|
+
queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
|
|
2228
|
+
this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
|
|
2229
|
+
]);
|
|
2230
|
+
if (!queryVector) {
|
|
2231
|
+
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
2232
|
+
queryVector = retrievedVector;
|
|
2069
2233
|
}
|
|
2234
|
+
const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
|
|
2235
|
+
return { sources, graphData };
|
|
2070
2236
|
}
|
|
2071
2237
|
/**
|
|
2072
2238
|
* Rewrite the user query for better retrieval performance.
|
|
@@ -2097,11 +2263,11 @@ var ProviderHealthCheck = class {
|
|
|
2097
2263
|
*/
|
|
2098
2264
|
static async checkVectorProvider(config) {
|
|
2099
2265
|
const timestamp = Date.now();
|
|
2100
|
-
const configErrors = ConfigValidator.validate({
|
|
2266
|
+
const configErrors = await ConfigValidator.validate({
|
|
2101
2267
|
projectId: "health-check",
|
|
2102
2268
|
vectorDb: config,
|
|
2103
|
-
llm: { provider: "openai", model: "gpt-4o" },
|
|
2104
|
-
embedding: { provider: "openai", model: "text-embedding-3-small" }
|
|
2269
|
+
llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
|
|
2270
|
+
embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
|
|
2105
2271
|
});
|
|
2106
2272
|
const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
|
|
2107
2273
|
if (vectorDbErrors.length > 0) {
|
|
@@ -2113,35 +2279,11 @@ var ProviderHealthCheck = class {
|
|
|
2113
2279
|
};
|
|
2114
2280
|
}
|
|
2115
2281
|
try {
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
case "pgvector":
|
|
2120
|
-
case "postgresql":
|
|
2121
|
-
return await this.checkPostgres(config, timestamp);
|
|
2122
|
-
case "mongodb":
|
|
2123
|
-
return await this.checkMongoDB(config, timestamp);
|
|
2124
|
-
case "milvus":
|
|
2125
|
-
return await this.checkMilvus(config, timestamp);
|
|
2126
|
-
case "qdrant":
|
|
2127
|
-
return await this.checkQdrant(config, timestamp);
|
|
2128
|
-
case "chromadb":
|
|
2129
|
-
return await this.checkChromaDB(config, timestamp);
|
|
2130
|
-
case "redis":
|
|
2131
|
-
return await this.checkRedis(config, timestamp);
|
|
2132
|
-
case "weaviate":
|
|
2133
|
-
return await this.checkWeaviate(config, timestamp);
|
|
2134
|
-
case "rest":
|
|
2135
|
-
case "universal_rest":
|
|
2136
|
-
return await this.checkRestAPI(config, timestamp);
|
|
2137
|
-
default:
|
|
2138
|
-
return {
|
|
2139
|
-
healthy: false,
|
|
2140
|
-
provider: config.provider,
|
|
2141
|
-
error: `Unsupported provider: ${config.provider}`,
|
|
2142
|
-
timestamp
|
|
2143
|
-
};
|
|
2282
|
+
const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
|
|
2283
|
+
if (checker) {
|
|
2284
|
+
return await checker.check(config);
|
|
2144
2285
|
}
|
|
2286
|
+
return await this.fallbackVectorHealthCheck(config, timestamp);
|
|
2145
2287
|
} catch (error) {
|
|
2146
2288
|
return {
|
|
2147
2289
|
healthy: false,
|
|
@@ -2151,363 +2293,50 @@ var ProviderHealthCheck = class {
|
|
|
2151
2293
|
};
|
|
2152
2294
|
}
|
|
2153
2295
|
}
|
|
2154
|
-
static async
|
|
2155
|
-
|
|
2156
|
-
const
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
const indexes = await client.listIndexes();
|
|
2161
|
-
const indexNames = (_b = (_a = indexes.indexes) == null ? void 0 : _a.map((i) => i.name)) != null ? _b : [];
|
|
2162
|
-
if (!indexNames.includes(config.indexName)) {
|
|
2296
|
+
static async fallbackVectorHealthCheck(config, timestamp) {
|
|
2297
|
+
const opts = config.options || {};
|
|
2298
|
+
const baseUrl = opts.baseUrl || opts.uri;
|
|
2299
|
+
if (baseUrl && baseUrl.startsWith("http")) {
|
|
2300
|
+
try {
|
|
2301
|
+
const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
|
|
2163
2302
|
return {
|
|
2164
|
-
healthy:
|
|
2165
|
-
provider:
|
|
2166
|
-
error:
|
|
2303
|
+
healthy: response.ok,
|
|
2304
|
+
provider: config.provider,
|
|
2305
|
+
error: response.ok ? void 0 : `Health check failed: ${response.status}`,
|
|
2167
2306
|
timestamp
|
|
2168
2307
|
};
|
|
2169
|
-
}
|
|
2170
|
-
return {
|
|
2171
|
-
healthy: true,
|
|
2172
|
-
provider: "pinecone",
|
|
2173
|
-
capabilities: { indexes: indexNames.length, targetIndex: config.indexName },
|
|
2174
|
-
timestamp
|
|
2175
|
-
};
|
|
2176
|
-
} catch (error) {
|
|
2177
|
-
return {
|
|
2178
|
-
healthy: false,
|
|
2179
|
-
provider: "pinecone",
|
|
2180
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2181
|
-
timestamp
|
|
2182
|
-
};
|
|
2183
|
-
}
|
|
2184
|
-
}
|
|
2185
|
-
static async checkPostgres(config, timestamp) {
|
|
2186
|
-
const opts = config.options;
|
|
2187
|
-
try {
|
|
2188
|
-
const { Client } = await import("pg");
|
|
2189
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
2190
|
-
await client.connect();
|
|
2191
|
-
const result = await client.query(`
|
|
2192
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
2193
|
-
`);
|
|
2194
|
-
const hasVector = result.rows[0].exists;
|
|
2195
|
-
await client.end();
|
|
2196
|
-
return {
|
|
2197
|
-
healthy: true,
|
|
2198
|
-
provider: "postgresql",
|
|
2199
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
2200
|
-
timestamp
|
|
2201
|
-
};
|
|
2202
|
-
} catch (error) {
|
|
2203
|
-
return {
|
|
2204
|
-
healthy: false,
|
|
2205
|
-
provider: "postgresql",
|
|
2206
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2207
|
-
timestamp
|
|
2208
|
-
};
|
|
2209
|
-
}
|
|
2210
|
-
}
|
|
2211
|
-
static async checkMongoDB(config, timestamp) {
|
|
2212
|
-
const opts = config.options;
|
|
2213
|
-
try {
|
|
2214
|
-
const { MongoClient } = await import("mongodb");
|
|
2215
|
-
const client = new MongoClient(opts.uri);
|
|
2216
|
-
await client.connect();
|
|
2217
|
-
const db = client.db(opts.database);
|
|
2218
|
-
const collections = await db.listCollections().toArray();
|
|
2219
|
-
const collectionNames = collections.map((c) => c.name);
|
|
2220
|
-
const hasCollection = collectionNames.includes(opts.collection);
|
|
2221
|
-
await client.close();
|
|
2222
|
-
return {
|
|
2223
|
-
healthy: true,
|
|
2224
|
-
provider: "mongodb",
|
|
2225
|
-
capabilities: {
|
|
2226
|
-
collections: collectionNames.length,
|
|
2227
|
-
targetCollection: hasCollection ? opts.collection : "NOT FOUND (will be created on first insert)"
|
|
2228
|
-
},
|
|
2229
|
-
timestamp
|
|
2230
|
-
};
|
|
2231
|
-
} catch (error) {
|
|
2232
|
-
return {
|
|
2233
|
-
healthy: false,
|
|
2234
|
-
provider: "mongodb",
|
|
2235
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2236
|
-
timestamp
|
|
2237
|
-
};
|
|
2238
|
-
}
|
|
2239
|
-
}
|
|
2240
|
-
/**
|
|
2241
|
-
* Milvus health check — uses baseUrl/uri (matching MilvusProvider constructor).
|
|
2242
|
-
*/
|
|
2243
|
-
static async checkMilvus(config, timestamp) {
|
|
2244
|
-
const opts = config.options;
|
|
2245
|
-
const baseUrl = opts.baseUrl || opts.uri || (opts.host ? `http://${opts.host}:${opts.port || 19530}` : "http://localhost:19530");
|
|
2246
|
-
try {
|
|
2247
|
-
const controller = new AbortController();
|
|
2248
|
-
const timeoutId = setTimeout(() => controller.abort(), 5e3);
|
|
2249
|
-
const response = await fetch(`${baseUrl}/v1/health`, { signal: controller.signal });
|
|
2250
|
-
clearTimeout(timeoutId);
|
|
2251
|
-
if (!response.ok) {
|
|
2252
|
-
throw new Error(`Health check returned ${response.status}`);
|
|
2253
|
-
}
|
|
2254
|
-
return {
|
|
2255
|
-
healthy: true,
|
|
2256
|
-
provider: "milvus",
|
|
2257
|
-
capabilities: { endpoint: baseUrl },
|
|
2258
|
-
timestamp
|
|
2259
|
-
};
|
|
2260
|
-
} catch (error) {
|
|
2261
|
-
return {
|
|
2262
|
-
healthy: false,
|
|
2263
|
-
provider: "milvus",
|
|
2264
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2265
|
-
timestamp
|
|
2266
|
-
};
|
|
2267
|
-
}
|
|
2268
|
-
}
|
|
2269
|
-
/**
|
|
2270
|
-
* Qdrant health check — uses baseUrl (matching QdrantProvider constructor).
|
|
2271
|
-
*/
|
|
2272
|
-
static async checkQdrant(config, timestamp) {
|
|
2273
|
-
const opts = config.options;
|
|
2274
|
-
const baseUrl = (opts.baseUrl || opts.url || "http://localhost:6333").replace(/\/$/, "");
|
|
2275
|
-
try {
|
|
2276
|
-
const apiKey = opts.apiKey;
|
|
2277
|
-
const response = await fetch(`${baseUrl}/`, {
|
|
2278
|
-
headers: apiKey ? { "api-key": apiKey } : {}
|
|
2279
|
-
});
|
|
2280
|
-
if (!response.ok) throw new Error(`Health check returned ${response.status}`);
|
|
2281
|
-
const health = await response.json();
|
|
2282
|
-
return {
|
|
2283
|
-
healthy: true,
|
|
2284
|
-
provider: "qdrant",
|
|
2285
|
-
capabilities: health,
|
|
2286
|
-
timestamp
|
|
2287
|
-
};
|
|
2288
|
-
} catch (error) {
|
|
2289
|
-
return {
|
|
2290
|
-
healthy: false,
|
|
2291
|
-
provider: "qdrant",
|
|
2292
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2293
|
-
timestamp
|
|
2294
|
-
};
|
|
2295
|
-
}
|
|
2296
|
-
}
|
|
2297
|
-
/**
|
|
2298
|
-
* ChromaDB health check — uses baseUrl/host (matching ChromaDBProvider constructor).
|
|
2299
|
-
*/
|
|
2300
|
-
static async checkChromaDB(config, timestamp) {
|
|
2301
|
-
const opts = config.options;
|
|
2302
|
-
const baseUrl = opts.baseUrl || (opts.host ? `http://${opts.host}:${opts.port || 8e3}` : "http://localhost:8000");
|
|
2303
|
-
try {
|
|
2304
|
-
const response = await fetch(`${baseUrl}/api/v1/heartbeat`);
|
|
2305
|
-
if (!response.ok) throw new Error(`Health check returned ${response.status}`);
|
|
2306
|
-
return {
|
|
2307
|
-
healthy: true,
|
|
2308
|
-
provider: "chromadb",
|
|
2309
|
-
capabilities: { endpoint: baseUrl },
|
|
2310
|
-
timestamp
|
|
2311
|
-
};
|
|
2312
|
-
} catch (error) {
|
|
2313
|
-
return {
|
|
2314
|
-
healthy: false,
|
|
2315
|
-
provider: "chromadb",
|
|
2316
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2317
|
-
timestamp
|
|
2318
|
-
};
|
|
2319
|
-
}
|
|
2320
|
-
}
|
|
2321
|
-
/**
|
|
2322
|
-
* Redis health check — Redis is TCP-only (no HTTP endpoint).
|
|
2323
|
-
* We report healthy=true with a note; actual connectivity is validated at first operation.
|
|
2324
|
-
*/
|
|
2325
|
-
static async checkRedis(config, timestamp) {
|
|
2326
|
-
const opts = config.options;
|
|
2327
|
-
const url = opts.baseUrl || opts.url || (opts.host && opts.port ? `redis://${opts.host}:${opts.port}` : "redis://localhost:6379");
|
|
2328
|
-
if (url.startsWith("http")) {
|
|
2329
|
-
try {
|
|
2330
|
-
await fetch(url);
|
|
2331
|
-
return { healthy: true, provider: "redis", capabilities: { endpoint: url }, timestamp };
|
|
2332
2308
|
} catch (e) {
|
|
2309
|
+
return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
|
|
2333
2310
|
}
|
|
2334
2311
|
}
|
|
2335
2312
|
return {
|
|
2336
2313
|
healthy: true,
|
|
2337
|
-
provider:
|
|
2338
|
-
|
|
2339
|
-
endpoint: url,
|
|
2340
|
-
note: "Redis uses TCP; connectivity is validated on first operation."
|
|
2341
|
-
},
|
|
2314
|
+
provider: config.provider,
|
|
2315
|
+
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
2342
2316
|
timestamp
|
|
2343
2317
|
};
|
|
2344
2318
|
}
|
|
2345
|
-
/**
|
|
2346
|
-
* Weaviate health check — uses baseUrl/url (matching WeaviateProvider constructor).
|
|
2347
|
-
*/
|
|
2348
|
-
static async checkWeaviate(config, timestamp) {
|
|
2349
|
-
const opts = config.options;
|
|
2350
|
-
const baseUrl = (opts.baseUrl || opts.url || "http://localhost:8080").replace(/\/$/, "");
|
|
2351
|
-
try {
|
|
2352
|
-
const response = await fetch(`${baseUrl}/v1/.well-known/ready`);
|
|
2353
|
-
if (!response.ok) throw new Error(`Health check returned ${response.status}`);
|
|
2354
|
-
return {
|
|
2355
|
-
healthy: true,
|
|
2356
|
-
provider: "weaviate",
|
|
2357
|
-
capabilities: { endpoint: baseUrl },
|
|
2358
|
-
timestamp
|
|
2359
|
-
};
|
|
2360
|
-
} catch (error) {
|
|
2361
|
-
return {
|
|
2362
|
-
healthy: false,
|
|
2363
|
-
provider: "weaviate",
|
|
2364
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2365
|
-
timestamp
|
|
2366
|
-
};
|
|
2367
|
-
}
|
|
2368
|
-
}
|
|
2369
|
-
static async checkRestAPI(config, timestamp) {
|
|
2370
|
-
const opts = config.options;
|
|
2371
|
-
const baseUrl = (opts.baseUrl || "").replace(/\/$/, "");
|
|
2372
|
-
if (!baseUrl) {
|
|
2373
|
-
return { healthy: false, provider: "rest", error: "baseUrl is required", timestamp };
|
|
2374
|
-
}
|
|
2375
|
-
try {
|
|
2376
|
-
const response = await fetch(`${baseUrl}/health`, {
|
|
2377
|
-
headers: opts.headers ? JSON.parse(opts.headers) : {}
|
|
2378
|
-
});
|
|
2379
|
-
return {
|
|
2380
|
-
healthy: response.ok,
|
|
2381
|
-
provider: "rest",
|
|
2382
|
-
error: response.ok ? void 0 : `Health check returned ${response.status}`,
|
|
2383
|
-
timestamp
|
|
2384
|
-
};
|
|
2385
|
-
} catch (error) {
|
|
2386
|
-
return {
|
|
2387
|
-
healthy: false,
|
|
2388
|
-
provider: "rest",
|
|
2389
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2390
|
-
timestamp
|
|
2391
|
-
};
|
|
2392
|
-
}
|
|
2393
|
-
}
|
|
2394
2319
|
/**
|
|
2395
2320
|
* Validates LLM provider configuration.
|
|
2396
2321
|
*/
|
|
2397
2322
|
static async checkLLMProvider(config) {
|
|
2398
2323
|
const timestamp = Date.now();
|
|
2399
2324
|
try {
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
case "anthropic":
|
|
2404
|
-
return await this.checkAnthropic(config, timestamp);
|
|
2405
|
-
case "ollama":
|
|
2406
|
-
return await this.checkOllama(config, timestamp);
|
|
2407
|
-
case "rest":
|
|
2408
|
-
case "universal_rest":
|
|
2409
|
-
return await this.checkLLMRestAPI(config, timestamp);
|
|
2410
|
-
default:
|
|
2411
|
-
return {
|
|
2412
|
-
healthy: false,
|
|
2413
|
-
provider: config.provider,
|
|
2414
|
-
error: `Unsupported provider: ${config.provider}`,
|
|
2415
|
-
timestamp
|
|
2416
|
-
};
|
|
2325
|
+
const checker = LLMFactory.getHealthChecker(config.provider);
|
|
2326
|
+
if (checker) {
|
|
2327
|
+
return await checker.check(config);
|
|
2417
2328
|
}
|
|
2418
|
-
} catch (error) {
|
|
2419
|
-
return {
|
|
2420
|
-
healthy: false,
|
|
2421
|
-
provider: config.provider,
|
|
2422
|
-
error: error instanceof Error ? error.message : String(error),
|
|
2423
|
-
timestamp
|
|
2424
|
-
};
|
|
2425
|
-
}
|
|
2426
|
-
}
|
|
2427
|
-
static async checkOpenAI(config, timestamp) {
|
|
2428
|
-
try {
|
|
2429
|
-
const OpenAI2 = await import("openai");
|
|
2430
|
-
const client = new OpenAI2.default({ apiKey: config.apiKey });
|
|
2431
|
-
const models = await client.models.list();
|
|
2432
|
-
const hasModel = models.data.some((m) => m.id === config.model);
|
|
2433
|
-
return {
|
|
2434
|
-
healthy: true,
|
|
2435
|
-
provider: "openai",
|
|
2436
|
-
capabilities: { model: config.model, available: hasModel, totalModels: models.data.length },
|
|
2437
|
-
timestamp
|
|
2438
|
-
};
|
|
2439
|
-
} catch (error) {
|
|
2440
|
-
return {
|
|
2441
|
-
healthy: false,
|
|
2442
|
-
provider: "openai",
|
|
2443
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2444
|
-
timestamp
|
|
2445
|
-
};
|
|
2446
|
-
}
|
|
2447
|
-
}
|
|
2448
|
-
static async checkAnthropic(config, timestamp) {
|
|
2449
|
-
try {
|
|
2450
|
-
const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
|
|
2451
|
-
const client = new Anthropic2({ apiKey: config.apiKey });
|
|
2452
|
-
await client.messages.create({
|
|
2453
|
-
model: config.model,
|
|
2454
|
-
max_tokens: 10,
|
|
2455
|
-
messages: [{ role: "user", content: "ping" }]
|
|
2456
|
-
});
|
|
2457
|
-
return { healthy: true, provider: "anthropic", capabilities: { model: config.model }, timestamp };
|
|
2458
|
-
} catch (error) {
|
|
2459
|
-
return {
|
|
2460
|
-
healthy: false,
|
|
2461
|
-
provider: "anthropic",
|
|
2462
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2463
|
-
timestamp
|
|
2464
|
-
};
|
|
2465
|
-
}
|
|
2466
|
-
}
|
|
2467
|
-
static async checkOllama(config, timestamp) {
|
|
2468
|
-
const baseUrl = (config.baseUrl || "http://localhost:11434").replace(/\/$/, "");
|
|
2469
|
-
try {
|
|
2470
|
-
const response = await fetch(`${baseUrl}/api/tags`);
|
|
2471
|
-
if (!response.ok) throw new Error(`Health check returned ${response.status}`);
|
|
2472
|
-
const data = await response.json();
|
|
2473
|
-
const models = data.models || [];
|
|
2474
|
-
const hasModel = models.some((m) => m.name === config.model);
|
|
2475
2329
|
return {
|
|
2476
2330
|
healthy: true,
|
|
2477
|
-
provider: "ollama",
|
|
2478
|
-
capabilities: { model: config.model, available: hasModel, totalModels: models.length },
|
|
2479
|
-
timestamp
|
|
2480
|
-
};
|
|
2481
|
-
} catch (error) {
|
|
2482
|
-
return {
|
|
2483
|
-
healthy: false,
|
|
2484
|
-
provider: "ollama",
|
|
2485
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2486
|
-
timestamp
|
|
2487
|
-
};
|
|
2488
|
-
}
|
|
2489
|
-
}
|
|
2490
|
-
static async checkLLMRestAPI(config, timestamp) {
|
|
2491
|
-
var _a, _b;
|
|
2492
|
-
const baseUrlRaw = config.baseUrl || ((_a = config.options) == null ? void 0 : _a.baseUrl);
|
|
2493
|
-
const baseUrl = (typeof baseUrlRaw === "string" ? baseUrlRaw : "").replace(/\/$/, "");
|
|
2494
|
-
if (!baseUrl) {
|
|
2495
|
-
return { healthy: false, provider: config.provider, error: "baseUrl is required", timestamp };
|
|
2496
|
-
}
|
|
2497
|
-
try {
|
|
2498
|
-
const headers = (_b = config.options) == null ? void 0 : _b.headers;
|
|
2499
|
-
const response = await fetch(`${baseUrl}/health`, { headers: headers || void 0 });
|
|
2500
|
-
return {
|
|
2501
|
-
healthy: response.ok,
|
|
2502
2331
|
provider: config.provider,
|
|
2503
|
-
error:
|
|
2332
|
+
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
2504
2333
|
timestamp
|
|
2505
2334
|
};
|
|
2506
2335
|
} catch (error) {
|
|
2507
2336
|
return {
|
|
2508
2337
|
healthy: false,
|
|
2509
2338
|
provider: config.provider,
|
|
2510
|
-
error:
|
|
2339
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2511
2340
|
timestamp
|
|
2512
2341
|
};
|
|
2513
2342
|
}
|
|
@@ -2582,6 +2411,14 @@ var VectorPlugin = class {
|
|
|
2582
2411
|
async chat(message, history = [], namespace) {
|
|
2583
2412
|
return this.pipeline.ask(message, history, namespace);
|
|
2584
2413
|
}
|
|
2414
|
+
/**
|
|
2415
|
+
* Run a streaming chat query.
|
|
2416
|
+
*/
|
|
2417
|
+
chatStream(_0) {
|
|
2418
|
+
return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
|
|
2419
|
+
yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
|
|
2420
|
+
});
|
|
2421
|
+
}
|
|
2585
2422
|
/**
|
|
2586
2423
|
* Ingest documents into the vector database.
|
|
2587
2424
|
*/
|
|
@@ -2667,6 +2504,55 @@ function createChatHandler(config) {
|
|
|
2667
2504
|
}
|
|
2668
2505
|
};
|
|
2669
2506
|
}
|
|
2507
|
+
function createStreamHandler(config) {
|
|
2508
|
+
const plugin = new VectorPlugin(config);
|
|
2509
|
+
return async function POST(req) {
|
|
2510
|
+
try {
|
|
2511
|
+
const { message, history = [], namespace } = await req.json();
|
|
2512
|
+
const encoder = new TextEncoder();
|
|
2513
|
+
const stream = new ReadableStream({
|
|
2514
|
+
async start(controller) {
|
|
2515
|
+
const pipelineStream = plugin.chatStream(message, history, namespace);
|
|
2516
|
+
try {
|
|
2517
|
+
for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
2518
|
+
const chunk = temp.value;
|
|
2519
|
+
if (typeof chunk === "string") {
|
|
2520
|
+
controller.enqueue(encoder.encode(chunk));
|
|
2521
|
+
} else {
|
|
2522
|
+
controller.enqueue(encoder.encode(`
|
|
2523
|
+
|
|
2524
|
+
__METADATA__${JSON.stringify(chunk)}`));
|
|
2525
|
+
}
|
|
2526
|
+
}
|
|
2527
|
+
} catch (temp) {
|
|
2528
|
+
error = [temp];
|
|
2529
|
+
} finally {
|
|
2530
|
+
try {
|
|
2531
|
+
more && (temp = iter.return) && await temp.call(iter);
|
|
2532
|
+
} finally {
|
|
2533
|
+
if (error)
|
|
2534
|
+
throw error[0];
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
controller.close();
|
|
2538
|
+
}
|
|
2539
|
+
});
|
|
2540
|
+
return new Response(stream, {
|
|
2541
|
+
headers: {
|
|
2542
|
+
"Content-Type": "text/event-stream",
|
|
2543
|
+
"Cache-Control": "no-cache",
|
|
2544
|
+
"Connection": "keep-alive"
|
|
2545
|
+
}
|
|
2546
|
+
});
|
|
2547
|
+
} catch (err) {
|
|
2548
|
+
const message = err instanceof Error ? err.message : "Internal server error";
|
|
2549
|
+
return new Response(JSON.stringify({ error: message }), {
|
|
2550
|
+
status: 500,
|
|
2551
|
+
headers: { "Content-Type": "application/json" }
|
|
2552
|
+
});
|
|
2553
|
+
}
|
|
2554
|
+
};
|
|
2555
|
+
}
|
|
2670
2556
|
function createIngestHandler(config) {
|
|
2671
2557
|
const plugin = new VectorPlugin(config);
|
|
2672
2558
|
return async function POST(req) {
|
|
@@ -2738,8 +2624,6 @@ function createUploadHandler(config) {
|
|
|
2738
2624
|
export {
|
|
2739
2625
|
getRagConfig,
|
|
2740
2626
|
ConfigResolver,
|
|
2741
|
-
ConfigValidator,
|
|
2742
|
-
DocumentChunker,
|
|
2743
2627
|
OpenAIProvider,
|
|
2744
2628
|
AnthropicProvider,
|
|
2745
2629
|
OllamaProvider,
|
|
@@ -2748,6 +2632,8 @@ export {
|
|
|
2748
2632
|
UniversalLLMAdapter,
|
|
2749
2633
|
LLMFactory,
|
|
2750
2634
|
ProviderRegistry,
|
|
2635
|
+
ConfigValidator,
|
|
2636
|
+
DocumentChunker,
|
|
2751
2637
|
BatchProcessor,
|
|
2752
2638
|
EmbeddingStrategy,
|
|
2753
2639
|
EmbeddingStrategyResolver,
|
|
@@ -2756,6 +2642,7 @@ export {
|
|
|
2756
2642
|
VectorPlugin,
|
|
2757
2643
|
DocumentParser,
|
|
2758
2644
|
createChatHandler,
|
|
2645
|
+
createStreamHandler,
|
|
2759
2646
|
createIngestHandler,
|
|
2760
2647
|
createHealthHandler,
|
|
2761
2648
|
createUploadHandler
|