@retrivora-ai/rag-engine 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-cfaMidtA.d.mts → DocumentChunker-BEyzadsv.d.mts} +2 -2
- package/dist/{DocumentChunker-cfaMidtA.d.ts → DocumentChunker-BEyzadsv.d.ts} +2 -2
- package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
- package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
- package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
- package/dist/{RagConfig-DG_0f8ka.d.mts → RagConfig-hBGXJmSx.d.mts} +3 -3
- package/dist/{RagConfig-DG_0f8ka.d.ts → RagConfig-hBGXJmSx.d.ts} +3 -3
- package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
- package/dist/chunk-HSBXE2WV.mjs +1758 -0
- package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
- package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
- package/dist/handlers/index.d.mts +3 -44
- package/dist/handlers/index.d.ts +3 -44
- package/dist/handlers/index.js +1322 -57
- package/dist/handlers/index.mjs +1 -1
- package/dist/index-Bx182KKn.d.ts +64 -0
- package/dist/index-Ck2pt7-8.d.mts +64 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/server.d.mts +65 -18
- package/dist/server.d.ts +65 -18
- package/dist/server.js +1322 -57
- package/dist/server.mjs +4 -4
- package/package.json +2 -1
- package/src/config/serverConfig.ts +3 -0
- package/src/core/BatchProcessor.ts +347 -0
- package/src/core/ConfigValidator.ts +560 -0
- package/src/core/Pipeline.ts +89 -39
- package/src/core/ProviderHealthCheck.ts +565 -0
- package/src/core/VectorPlugin.ts +49 -8
- package/src/handlers/index.ts +2 -2
- package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
- package/src/providers/vectordb/MilvusProvider.ts +1 -1
- package/src/providers/vectordb/MongoDBProvider.ts +1 -1
- package/src/providers/vectordb/PineconeProvider.ts +4 -4
- package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
- package/src/providers/vectordb/QdrantProvider.ts +32 -2
- package/src/rag/DocumentChunker.ts +2 -2
- package/src/types/index.ts +3 -3
- package/dist/chunk-BP4U4TT5.mjs +0 -548
package/dist/handlers/index.js
CHANGED
|
@@ -557,7 +557,7 @@ var init_PineconeProvider = __esm({
|
|
|
557
557
|
var _a;
|
|
558
558
|
await this.index(namespace).upsert({
|
|
559
559
|
records: [{
|
|
560
|
-
id: doc.id,
|
|
560
|
+
id: String(doc.id),
|
|
561
561
|
values: doc.vector,
|
|
562
562
|
metadata: __spreadValues({ content: doc.content }, (_a = doc.metadata) != null ? _a : {})
|
|
563
563
|
}]
|
|
@@ -569,7 +569,7 @@ var init_PineconeProvider = __esm({
|
|
|
569
569
|
const records = docs.slice(i, i + BATCH).map((d) => {
|
|
570
570
|
var _a;
|
|
571
571
|
return {
|
|
572
|
-
id: d.id,
|
|
572
|
+
id: String(d.id),
|
|
573
573
|
values: d.vector,
|
|
574
574
|
metadata: __spreadValues({ content: d.content }, (_a = d.metadata) != null ? _a : {})
|
|
575
575
|
};
|
|
@@ -595,7 +595,7 @@ var init_PineconeProvider = __esm({
|
|
|
595
595
|
});
|
|
596
596
|
}
|
|
597
597
|
async delete(id, namespace) {
|
|
598
|
-
await this.index(namespace).deleteOne({ id });
|
|
598
|
+
await this.index(namespace).deleteOne({ id: String(id) });
|
|
599
599
|
}
|
|
600
600
|
async deleteNamespace(namespace) {
|
|
601
601
|
await this.client.index(this.indexName).namespace(namespace).deleteAll();
|
|
@@ -673,8 +673,37 @@ var init_PostgreSQLProvider = __esm({
|
|
|
673
673
|
);
|
|
674
674
|
}
|
|
675
675
|
async batchUpsert(docs, namespace = "") {
|
|
676
|
-
|
|
677
|
-
|
|
676
|
+
if (docs.length === 0) return;
|
|
677
|
+
const client = await this.pool.connect();
|
|
678
|
+
try {
|
|
679
|
+
await client.query("BEGIN");
|
|
680
|
+
const BATCH_SIZE = 50;
|
|
681
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
682
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
683
|
+
const values = [];
|
|
684
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
685
|
+
var _a;
|
|
686
|
+
const offset = idx * 5;
|
|
687
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
688
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
689
|
+
}).join(", ");
|
|
690
|
+
const query = `
|
|
691
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
692
|
+
VALUES ${valuePlaceholders}
|
|
693
|
+
ON CONFLICT (id) DO UPDATE
|
|
694
|
+
SET namespace = EXCLUDED.namespace,
|
|
695
|
+
content = EXCLUDED.content,
|
|
696
|
+
metadata = EXCLUDED.metadata,
|
|
697
|
+
embedding = EXCLUDED.embedding
|
|
698
|
+
`;
|
|
699
|
+
await client.query(query, values);
|
|
700
|
+
}
|
|
701
|
+
await client.query("COMMIT");
|
|
702
|
+
} catch (error) {
|
|
703
|
+
await client.query("ROLLBACK");
|
|
704
|
+
throw error;
|
|
705
|
+
} finally {
|
|
706
|
+
client.release();
|
|
678
707
|
}
|
|
679
708
|
}
|
|
680
709
|
async query(vector, topK, namespace, filter) {
|
|
@@ -947,6 +976,33 @@ var init_QdrantProvider = __esm({
|
|
|
947
976
|
}
|
|
948
977
|
async initialize() {
|
|
949
978
|
await this.ping();
|
|
979
|
+
await this.ensureIndex();
|
|
980
|
+
}
|
|
981
|
+
/**
|
|
982
|
+
* Ensures that the 'namespace' field has a keyword index for efficient filtering.
|
|
983
|
+
* Qdrant requires this for search filters to work in many configurations.
|
|
984
|
+
*/
|
|
985
|
+
async ensureIndex() {
|
|
986
|
+
var _a, _b;
|
|
987
|
+
try {
|
|
988
|
+
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
989
|
+
field_name: "namespace",
|
|
990
|
+
field_schema: "keyword"
|
|
991
|
+
});
|
|
992
|
+
console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
|
|
993
|
+
} catch (err) {
|
|
994
|
+
let status;
|
|
995
|
+
let data;
|
|
996
|
+
if (import_axios4.default.isAxiosError(err)) {
|
|
997
|
+
status = (_a = err.response) == null ? void 0 : _a.status;
|
|
998
|
+
data = (_b = err.response) == null ? void 0 : _b.data;
|
|
999
|
+
}
|
|
1000
|
+
if (status === 409) {
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
1004
|
+
console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
|
|
1005
|
+
}
|
|
950
1006
|
}
|
|
951
1007
|
async upsert(doc, namespace) {
|
|
952
1008
|
await this.batchUpsert([doc], namespace);
|
|
@@ -991,7 +1047,6 @@ var init_QdrantProvider = __esm({
|
|
|
991
1047
|
points: [id]
|
|
992
1048
|
});
|
|
993
1049
|
}
|
|
994
|
-
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
995
1050
|
async deleteNamespace(_namespace) {
|
|
996
1051
|
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
997
1052
|
filter: {
|
|
@@ -1335,7 +1390,7 @@ function readEnum(env, name, fallback, allowed) {
|
|
|
1335
1390
|
throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
|
|
1336
1391
|
}
|
|
1337
1392
|
function getRagConfig(env = process.env) {
|
|
1338
|
-
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J;
|
|
1393
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K;
|
|
1339
1394
|
const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
|
|
1340
1395
|
const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
|
|
1341
1396
|
const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
|
|
@@ -1360,6 +1415,9 @@ function getRagConfig(env = process.env) {
|
|
|
1360
1415
|
vectorDbOptions.database = (_j = readString(env, "MONGODB_DB")) != null ? _j : "";
|
|
1361
1416
|
vectorDbOptions.collection = (_k = readString(env, "MONGODB_COLLECTION")) != null ? _k : "";
|
|
1362
1417
|
vectorDbOptions.indexName = (_l = readString(env, "MONGODB_INDEX_NAME")) != null ? _l : "vector_index";
|
|
1418
|
+
} else if (vectorProvider === "qdrant") {
|
|
1419
|
+
vectorDbOptions.baseUrl = (_m = readString(env, "QDRANT_URL")) != null ? _m : "http://localhost:6333";
|
|
1420
|
+
vectorDbOptions.apiKey = readString(env, "QDRANT_API_KEY");
|
|
1363
1421
|
}
|
|
1364
1422
|
const llmApiKeyByProvider = {
|
|
1365
1423
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
@@ -1370,19 +1428,19 @@ function getRagConfig(env = process.env) {
|
|
|
1370
1428
|
const embeddingApiKeyByProvider = {
|
|
1371
1429
|
openai: readString(env, "OPENAI_API_KEY"),
|
|
1372
1430
|
ollama: void 0,
|
|
1373
|
-
custom: (
|
|
1431
|
+
custom: (_n = readString(env, "EMBEDDING_API_KEY")) != null ? _n : readString(env, "OPENAI_API_KEY")
|
|
1374
1432
|
};
|
|
1375
1433
|
return {
|
|
1376
1434
|
projectId,
|
|
1377
1435
|
vectorDb: {
|
|
1378
1436
|
provider: vectorProvider,
|
|
1379
|
-
indexName: (
|
|
1437
|
+
indexName: (_o = readString(env, "VECTOR_DB_INDEX")) != null ? _o : "rag-index",
|
|
1380
1438
|
options: vectorDbOptions
|
|
1381
1439
|
},
|
|
1382
1440
|
llm: {
|
|
1383
1441
|
provider: llmProvider,
|
|
1384
|
-
model: (
|
|
1385
|
-
apiKey: (
|
|
1442
|
+
model: (_p = readString(env, "LLM_MODEL")) != null ? _p : "gpt-4o",
|
|
1443
|
+
apiKey: (_q = llmApiKeyByProvider[llmProvider]) != null ? _q : "",
|
|
1386
1444
|
baseUrl: readString(env, "LLM_BASE_URL"),
|
|
1387
1445
|
systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
|
|
1388
1446
|
maxTokens: readNumber(env, "LLM_MAX_TOKENS", 1024),
|
|
@@ -1393,7 +1451,7 @@ function getRagConfig(env = process.env) {
|
|
|
1393
1451
|
},
|
|
1394
1452
|
embedding: {
|
|
1395
1453
|
provider: embeddingProvider,
|
|
1396
|
-
model: (
|
|
1454
|
+
model: (_r = readString(env, "EMBEDDING_MODEL")) != null ? _r : "text-embedding-3-small",
|
|
1397
1455
|
apiKey: embeddingApiKeyByProvider[embeddingProvider],
|
|
1398
1456
|
baseUrl: readString(env, "EMBEDDING_BASE_URL"),
|
|
1399
1457
|
dimensions: embeddingDimensions,
|
|
@@ -1402,16 +1460,16 @@ function getRagConfig(env = process.env) {
|
|
|
1402
1460
|
}
|
|
1403
1461
|
},
|
|
1404
1462
|
ui: {
|
|
1405
|
-
title: (
|
|
1406
|
-
subtitle: (
|
|
1407
|
-
primaryColor: (
|
|
1408
|
-
accentColor: (
|
|
1409
|
-
logoUrl: (
|
|
1410
|
-
placeholder: (
|
|
1411
|
-
showSources: ((
|
|
1412
|
-
welcomeMessage: (
|
|
1413
|
-
visualStyle: (
|
|
1414
|
-
borderRadius: (
|
|
1463
|
+
title: (_t = (_s = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _s : readString(env, "UI_TITLE")) != null ? _t : "AI Assistant",
|
|
1464
|
+
subtitle: (_v = (_u = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _u : readString(env, "UI_SUBTITLE")) != null ? _v : "Powered by RAG",
|
|
1465
|
+
primaryColor: (_x = (_w = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _w : readString(env, "UI_PRIMARY_COLOR")) != null ? _x : "#10b981",
|
|
1466
|
+
accentColor: (_z = (_y = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _y : readString(env, "UI_ACCENT_COLOR")) != null ? _z : "#3b82f6",
|
|
1467
|
+
logoUrl: (_A = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _A : readString(env, "UI_LOGO_URL"),
|
|
1468
|
+
placeholder: (_C = (_B = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _B : readString(env, "UI_PLACEHOLDER")) != null ? _C : "Ask me anything\u2026",
|
|
1469
|
+
showSources: ((_E = (_D = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _D : readString(env, "UI_SHOW_SOURCES")) != null ? _E : "true") !== "false",
|
|
1470
|
+
welcomeMessage: (_G = (_F = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _F : readString(env, "UI_WELCOME_MESSAGE")) != null ? _G : "Hello! I'm your AI assistant. Ask me anything about your documents.",
|
|
1471
|
+
visualStyle: (_I = (_H = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _H : readString(env, "UI_VISUAL_STYLE")) != null ? _I : "glass",
|
|
1472
|
+
borderRadius: (_K = (_J = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _J : readString(env, "UI_BORDER_RADIUS")) != null ? _K : "xl"
|
|
1415
1473
|
},
|
|
1416
1474
|
rag: {
|
|
1417
1475
|
topK: readNumber(env, "RAG_TOP_K", 5),
|
|
@@ -1464,6 +1522,465 @@ var ConfigResolver = class {
|
|
|
1464
1522
|
}
|
|
1465
1523
|
};
|
|
1466
1524
|
|
|
1525
|
+
// src/core/ConfigValidator.ts
|
|
1526
|
+
var ConfigValidator = class {
|
|
1527
|
+
/**
|
|
1528
|
+
* Validates the entire RagConfig object.
|
|
1529
|
+
* Returns an array of validation errors. Empty array = valid config.
|
|
1530
|
+
*/
|
|
1531
|
+
static validate(config) {
|
|
1532
|
+
const errors = [];
|
|
1533
|
+
if (!config.projectId) {
|
|
1534
|
+
errors.push({
|
|
1535
|
+
field: "projectId",
|
|
1536
|
+
message: "projectId is required",
|
|
1537
|
+
severity: "error"
|
|
1538
|
+
});
|
|
1539
|
+
}
|
|
1540
|
+
errors.push(...this.validateVectorDbConfig(config.vectorDb));
|
|
1541
|
+
errors.push(...this.validateLLMConfig(config.llm));
|
|
1542
|
+
if (config.embedding) {
|
|
1543
|
+
errors.push(...this.validateEmbeddingConfig(config.embedding));
|
|
1544
|
+
} else if (config.llm.provider === "anthropic") {
|
|
1545
|
+
errors.push({
|
|
1546
|
+
field: "embedding",
|
|
1547
|
+
message: "Embedding config is required when using Anthropic LLM",
|
|
1548
|
+
suggestion: 'Provide embedding config with provider (e.g., "openai")',
|
|
1549
|
+
severity: "error"
|
|
1550
|
+
});
|
|
1551
|
+
}
|
|
1552
|
+
if (config.ui) {
|
|
1553
|
+
errors.push(...this.validateUIConfig(config.ui));
|
|
1554
|
+
}
|
|
1555
|
+
if (config.rag) {
|
|
1556
|
+
errors.push(...this.validateRAGConfig(config.rag));
|
|
1557
|
+
}
|
|
1558
|
+
return errors;
|
|
1559
|
+
}
|
|
1560
|
+
static validateVectorDbConfig(config) {
|
|
1561
|
+
const errors = [];
|
|
1562
|
+
if (!config.provider) {
|
|
1563
|
+
errors.push({
|
|
1564
|
+
field: "vectorDb.provider",
|
|
1565
|
+
message: "Vector database provider is required",
|
|
1566
|
+
severity: "error"
|
|
1567
|
+
});
|
|
1568
|
+
return errors;
|
|
1569
|
+
}
|
|
1570
|
+
if (!config.indexName) {
|
|
1571
|
+
errors.push({
|
|
1572
|
+
field: "vectorDb.indexName",
|
|
1573
|
+
message: "Vector database index name is required",
|
|
1574
|
+
severity: "error"
|
|
1575
|
+
});
|
|
1576
|
+
}
|
|
1577
|
+
switch (config.provider) {
|
|
1578
|
+
case "pinecone":
|
|
1579
|
+
errors.push(...this.validatePineconeConfig(config));
|
|
1580
|
+
break;
|
|
1581
|
+
case "pgvector":
|
|
1582
|
+
case "postgresql":
|
|
1583
|
+
errors.push(...this.validatePostgresConfig(config));
|
|
1584
|
+
break;
|
|
1585
|
+
case "mongodb":
|
|
1586
|
+
errors.push(...this.validateMongoDBConfig(config));
|
|
1587
|
+
break;
|
|
1588
|
+
case "milvus":
|
|
1589
|
+
errors.push(...this.validateMilvusConfig(config));
|
|
1590
|
+
break;
|
|
1591
|
+
case "qdrant":
|
|
1592
|
+
errors.push(...this.validateQdrantConfig(config));
|
|
1593
|
+
break;
|
|
1594
|
+
case "chromadb":
|
|
1595
|
+
errors.push(...this.validateChromaDBConfig(config));
|
|
1596
|
+
break;
|
|
1597
|
+
case "redis":
|
|
1598
|
+
errors.push(...this.validateRedisConfig(config));
|
|
1599
|
+
break;
|
|
1600
|
+
case "weaviate":
|
|
1601
|
+
errors.push(...this.validateWeaviateConfig(config));
|
|
1602
|
+
break;
|
|
1603
|
+
case "universal_rest":
|
|
1604
|
+
case "rest":
|
|
1605
|
+
errors.push(...this.validateRestConfig(config));
|
|
1606
|
+
break;
|
|
1607
|
+
}
|
|
1608
|
+
return errors;
|
|
1609
|
+
}
|
|
1610
|
+
static validatePineconeConfig(config) {
|
|
1611
|
+
const errors = [];
|
|
1612
|
+
const opts = config.options;
|
|
1613
|
+
if (!opts.apiKey) {
|
|
1614
|
+
errors.push({
|
|
1615
|
+
field: "vectorDb.options.apiKey",
|
|
1616
|
+
message: "Pinecone API key is required",
|
|
1617
|
+
suggestion: "Set PINECONE_API_KEY environment variable",
|
|
1618
|
+
severity: "error"
|
|
1619
|
+
});
|
|
1620
|
+
}
|
|
1621
|
+
if (!opts.environment) {
|
|
1622
|
+
errors.push({
|
|
1623
|
+
field: "vectorDb.options.environment",
|
|
1624
|
+
message: "Pinecone environment is required",
|
|
1625
|
+
suggestion: 'Set PINECONE_ENVIRONMENT environment variable (e.g., "gcp-starter")',
|
|
1626
|
+
severity: "error"
|
|
1627
|
+
});
|
|
1628
|
+
}
|
|
1629
|
+
return errors;
|
|
1630
|
+
}
|
|
1631
|
+
static validatePostgresConfig(config) {
|
|
1632
|
+
const errors = [];
|
|
1633
|
+
const opts = config.options;
|
|
1634
|
+
if (!opts.connectionString) {
|
|
1635
|
+
errors.push({
|
|
1636
|
+
field: "vectorDb.options.connectionString",
|
|
1637
|
+
message: "PostgreSQL connection string is required",
|
|
1638
|
+
suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
|
|
1639
|
+
severity: "error"
|
|
1640
|
+
});
|
|
1641
|
+
}
|
|
1642
|
+
return errors;
|
|
1643
|
+
}
|
|
1644
|
+
static validateMongoDBConfig(config) {
|
|
1645
|
+
const errors = [];
|
|
1646
|
+
const opts = config.options;
|
|
1647
|
+
if (!opts.uri) {
|
|
1648
|
+
errors.push({
|
|
1649
|
+
field: "vectorDb.options.uri",
|
|
1650
|
+
message: "MongoDB connection URI is required",
|
|
1651
|
+
suggestion: "Set MONGODB_URI environment variable",
|
|
1652
|
+
severity: "error"
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
if (!opts.database) {
|
|
1656
|
+
errors.push({
|
|
1657
|
+
field: "vectorDb.options.database",
|
|
1658
|
+
message: "MongoDB database name is required",
|
|
1659
|
+
suggestion: "Set MONGODB_DB environment variable",
|
|
1660
|
+
severity: "error"
|
|
1661
|
+
});
|
|
1662
|
+
}
|
|
1663
|
+
if (!opts.collection) {
|
|
1664
|
+
errors.push({
|
|
1665
|
+
field: "vectorDb.options.collection",
|
|
1666
|
+
message: "MongoDB collection name is required",
|
|
1667
|
+
suggestion: "Set MONGODB_COLLECTION environment variable",
|
|
1668
|
+
severity: "error"
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
return errors;
|
|
1672
|
+
}
|
|
1673
|
+
static validateMilvusConfig(config) {
|
|
1674
|
+
const errors = [];
|
|
1675
|
+
const opts = config.options;
|
|
1676
|
+
if (!opts.host) {
|
|
1677
|
+
errors.push({
|
|
1678
|
+
field: "vectorDb.options.host",
|
|
1679
|
+
message: "Milvus host is required",
|
|
1680
|
+
suggestion: "Set MILVUS_HOST environment variable",
|
|
1681
|
+
severity: "error"
|
|
1682
|
+
});
|
|
1683
|
+
}
|
|
1684
|
+
if (!opts.port) {
|
|
1685
|
+
errors.push({
|
|
1686
|
+
field: "vectorDb.options.port",
|
|
1687
|
+
message: "Milvus port is required",
|
|
1688
|
+
suggestion: "Set MILVUS_PORT environment variable (default: 19530)",
|
|
1689
|
+
severity: "error"
|
|
1690
|
+
});
|
|
1691
|
+
}
|
|
1692
|
+
return errors;
|
|
1693
|
+
}
|
|
1694
|
+
static validateQdrantConfig(config) {
|
|
1695
|
+
const errors = [];
|
|
1696
|
+
const opts = config.options;
|
|
1697
|
+
if (!opts.baseUrl && !opts.url) {
|
|
1698
|
+
errors.push({
|
|
1699
|
+
field: "vectorDb.options.baseUrl",
|
|
1700
|
+
message: "Qdrant base URL is required",
|
|
1701
|
+
suggestion: 'Set QDRANT_URL environment variable (e.g., "http://localhost:6333")',
|
|
1702
|
+
severity: "error"
|
|
1703
|
+
});
|
|
1704
|
+
}
|
|
1705
|
+
return errors;
|
|
1706
|
+
}
|
|
1707
|
+
static validateChromaDBConfig(config) {
|
|
1708
|
+
const errors = [];
|
|
1709
|
+
const opts = config.options;
|
|
1710
|
+
if (!opts.host && !opts.path) {
|
|
1711
|
+
errors.push({
|
|
1712
|
+
field: "vectorDb.options.host",
|
|
1713
|
+
message: "ChromaDB host or path is required",
|
|
1714
|
+
suggestion: "Set CHROMADB_HOST or CHROMADB_PATH environment variable",
|
|
1715
|
+
severity: "error"
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
return errors;
|
|
1719
|
+
}
|
|
1720
|
+
static validateRedisConfig(config) {
|
|
1721
|
+
const errors = [];
|
|
1722
|
+
const opts = config.options;
|
|
1723
|
+
if (!opts.url && (!opts.host || !opts.port)) {
|
|
1724
|
+
errors.push({
|
|
1725
|
+
field: "vectorDb.options.url",
|
|
1726
|
+
message: "Redis connection URL or host:port is required",
|
|
1727
|
+
suggestion: "Set REDIS_URL or REDIS_HOST/REDIS_PORT environment variables",
|
|
1728
|
+
severity: "error"
|
|
1729
|
+
});
|
|
1730
|
+
}
|
|
1731
|
+
return errors;
|
|
1732
|
+
}
|
|
1733
|
+
static validateWeaviateConfig(config) {
|
|
1734
|
+
const errors = [];
|
|
1735
|
+
const opts = config.options;
|
|
1736
|
+
if (!opts.url) {
|
|
1737
|
+
errors.push({
|
|
1738
|
+
field: "vectorDb.options.url",
|
|
1739
|
+
message: "Weaviate instance URL is required",
|
|
1740
|
+
suggestion: "Set WEAVIATE_URL environment variable",
|
|
1741
|
+
severity: "error"
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
return errors;
|
|
1745
|
+
}
|
|
1746
|
+
static validateRestConfig(config) {
|
|
1747
|
+
const errors = [];
|
|
1748
|
+
const opts = config.options;
|
|
1749
|
+
if (!opts.baseUrl) {
|
|
1750
|
+
errors.push({
|
|
1751
|
+
field: "vectorDb.options.baseUrl",
|
|
1752
|
+
message: "REST API base URL is required",
|
|
1753
|
+
suggestion: "Set VECTOR_BASE_URL environment variable",
|
|
1754
|
+
severity: "error"
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
return errors;
|
|
1758
|
+
}
|
|
1759
|
+
static validateLLMConfig(config) {
|
|
1760
|
+
var _a;
|
|
1761
|
+
const errors = [];
|
|
1762
|
+
if (!config.provider) {
|
|
1763
|
+
errors.push({
|
|
1764
|
+
field: "llm.provider",
|
|
1765
|
+
message: "LLM provider is required",
|
|
1766
|
+
severity: "error"
|
|
1767
|
+
});
|
|
1768
|
+
return errors;
|
|
1769
|
+
}
|
|
1770
|
+
if (!config.model) {
|
|
1771
|
+
errors.push({
|
|
1772
|
+
field: "llm.model",
|
|
1773
|
+
message: "LLM model name is required",
|
|
1774
|
+
suggestion: `e.g., "gpt-4o" for OpenAI, "claude-3-opus" for Anthropic`,
|
|
1775
|
+
severity: "error"
|
|
1776
|
+
});
|
|
1777
|
+
}
|
|
1778
|
+
switch (config.provider) {
|
|
1779
|
+
case "openai":
|
|
1780
|
+
if (!config.apiKey) {
|
|
1781
|
+
errors.push({
|
|
1782
|
+
field: "llm.apiKey",
|
|
1783
|
+
message: "OpenAI API key is required",
|
|
1784
|
+
suggestion: "Set OPENAI_API_KEY environment variable",
|
|
1785
|
+
severity: "error"
|
|
1786
|
+
});
|
|
1787
|
+
}
|
|
1788
|
+
break;
|
|
1789
|
+
case "anthropic":
|
|
1790
|
+
if (!config.apiKey) {
|
|
1791
|
+
errors.push({
|
|
1792
|
+
field: "llm.apiKey",
|
|
1793
|
+
message: "Anthropic API key is required",
|
|
1794
|
+
suggestion: "Set ANTHROPIC_API_KEY environment variable",
|
|
1795
|
+
severity: "error"
|
|
1796
|
+
});
|
|
1797
|
+
}
|
|
1798
|
+
break;
|
|
1799
|
+
case "ollama":
|
|
1800
|
+
if (!config.baseUrl) {
|
|
1801
|
+
errors.push({
|
|
1802
|
+
field: "llm.baseUrl",
|
|
1803
|
+
message: "Ollama base URL is required",
|
|
1804
|
+
suggestion: 'Set LLM_BASE_URL environment variable (e.g., "http://localhost:11434")',
|
|
1805
|
+
severity: "error"
|
|
1806
|
+
});
|
|
1807
|
+
}
|
|
1808
|
+
break;
|
|
1809
|
+
case "rest":
|
|
1810
|
+
case "universal_rest":
|
|
1811
|
+
if (!config.baseUrl && !((_a = config.options) == null ? void 0 : _a.baseUrl)) {
|
|
1812
|
+
errors.push({
|
|
1813
|
+
field: "llm.baseUrl",
|
|
1814
|
+
message: "REST API base URL is required",
|
|
1815
|
+
suggestion: "Set LLM_BASE_URL environment variable",
|
|
1816
|
+
severity: "error"
|
|
1817
|
+
});
|
|
1818
|
+
}
|
|
1819
|
+
break;
|
|
1820
|
+
}
|
|
1821
|
+
if (config.temperature !== void 0) {
|
|
1822
|
+
if (config.temperature < 0 || config.temperature > 2) {
|
|
1823
|
+
errors.push({
|
|
1824
|
+
field: "llm.temperature",
|
|
1825
|
+
message: "Temperature must be between 0 and 2",
|
|
1826
|
+
severity: "error"
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
if (config.maxTokens !== void 0 && config.maxTokens <= 0) {
|
|
1831
|
+
errors.push({
|
|
1832
|
+
field: "llm.maxTokens",
|
|
1833
|
+
message: "maxTokens must be greater than 0",
|
|
1834
|
+
severity: "error"
|
|
1835
|
+
});
|
|
1836
|
+
}
|
|
1837
|
+
return errors;
|
|
1838
|
+
}
|
|
1839
|
+
static validateEmbeddingConfig(config) {
|
|
1840
|
+
const errors = [];
|
|
1841
|
+
if (!config.provider) {
|
|
1842
|
+
errors.push({
|
|
1843
|
+
field: "embedding.provider",
|
|
1844
|
+
message: "Embedding provider is required",
|
|
1845
|
+
severity: "error"
|
|
1846
|
+
});
|
|
1847
|
+
return errors;
|
|
1848
|
+
}
|
|
1849
|
+
if (!config.model) {
|
|
1850
|
+
errors.push({
|
|
1851
|
+
field: "embedding.model",
|
|
1852
|
+
message: "Embedding model name is required",
|
|
1853
|
+
suggestion: 'e.g., "text-embedding-3-small" for OpenAI',
|
|
1854
|
+
severity: "error"
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
if (config.provider === "openai" && !config.apiKey) {
|
|
1858
|
+
errors.push({
|
|
1859
|
+
field: "embedding.apiKey",
|
|
1860
|
+
message: "OpenAI API key is required for embedding",
|
|
1861
|
+
suggestion: "Set OPENAI_API_KEY environment variable",
|
|
1862
|
+
severity: "error"
|
|
1863
|
+
});
|
|
1864
|
+
}
|
|
1865
|
+
if (config.provider === "ollama" && !config.baseUrl) {
|
|
1866
|
+
errors.push({
|
|
1867
|
+
field: "embedding.baseUrl",
|
|
1868
|
+
message: "Ollama base URL is required",
|
|
1869
|
+
suggestion: "Set EMBEDDING_BASE_URL environment variable",
|
|
1870
|
+
severity: "error"
|
|
1871
|
+
});
|
|
1872
|
+
}
|
|
1873
|
+
if (config.dimensions !== void 0 && config.dimensions <= 0) {
|
|
1874
|
+
errors.push({
|
|
1875
|
+
field: "embedding.dimensions",
|
|
1876
|
+
message: "Embedding dimensions must be greater than 0",
|
|
1877
|
+
severity: "error"
|
|
1878
|
+
});
|
|
1879
|
+
}
|
|
1880
|
+
return errors;
|
|
1881
|
+
}
|
|
1882
|
+
static validateUIConfig(config) {
|
|
1883
|
+
const errors = [];
|
|
1884
|
+
if (config.primaryColor) {
|
|
1885
|
+
if (!this.isValidCSSColor(config.primaryColor)) {
|
|
1886
|
+
errors.push({
|
|
1887
|
+
field: "ui.primaryColor",
|
|
1888
|
+
message: "Invalid CSS color format",
|
|
1889
|
+
suggestion: "Use valid CSS colors: hex (#FF0000), rgb, hsl, or named colors",
|
|
1890
|
+
severity: "warning"
|
|
1891
|
+
});
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
if (config.borderRadius) {
|
|
1895
|
+
const validValues = ["none", "sm", "md", "lg", "xl", "full"];
|
|
1896
|
+
if (!validValues.includes(config.borderRadius)) {
|
|
1897
|
+
errors.push({
|
|
1898
|
+
field: "ui.borderRadius",
|
|
1899
|
+
message: `borderRadius must be one of: ${validValues.join(", ")}`,
|
|
1900
|
+
severity: "warning"
|
|
1901
|
+
});
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
if (config.visualStyle) {
|
|
1905
|
+
const validValues = ["glass", "solid"];
|
|
1906
|
+
if (!validValues.includes(config.visualStyle)) {
|
|
1907
|
+
errors.push({
|
|
1908
|
+
field: "ui.visualStyle",
|
|
1909
|
+
message: `visualStyle must be one of: ${validValues.join(", ")}`,
|
|
1910
|
+
severity: "warning"
|
|
1911
|
+
});
|
|
1912
|
+
}
|
|
1913
|
+
}
|
|
1914
|
+
return errors;
|
|
1915
|
+
}
|
|
1916
|
+
static validateRAGConfig(config) {
|
|
1917
|
+
const errors = [];
|
|
1918
|
+
if (config.topK !== void 0) {
|
|
1919
|
+
if (typeof config.topK !== "number" || config.topK <= 0) {
|
|
1920
|
+
errors.push({
|
|
1921
|
+
field: "rag.topK",
|
|
1922
|
+
message: "topK must be a positive integer",
|
|
1923
|
+
severity: "error"
|
|
1924
|
+
});
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
if (config.scoreThreshold !== void 0) {
|
|
1928
|
+
if (typeof config.scoreThreshold !== "number" || config.scoreThreshold < 0 || config.scoreThreshold > 1) {
|
|
1929
|
+
errors.push({
|
|
1930
|
+
field: "rag.scoreThreshold",
|
|
1931
|
+
message: "scoreThreshold must be between 0 and 1",
|
|
1932
|
+
severity: "error"
|
|
1933
|
+
});
|
|
1934
|
+
}
|
|
1935
|
+
}
|
|
1936
|
+
if (config.chunkSize !== void 0) {
|
|
1937
|
+
if (typeof config.chunkSize !== "number" || config.chunkSize <= 0) {
|
|
1938
|
+
errors.push({
|
|
1939
|
+
field: "rag.chunkSize",
|
|
1940
|
+
message: "chunkSize must be a positive integer",
|
|
1941
|
+
severity: "error"
|
|
1942
|
+
});
|
|
1943
|
+
}
|
|
1944
|
+
}
|
|
1945
|
+
if (config.chunkOverlap !== void 0) {
|
|
1946
|
+
if (typeof config.chunkOverlap !== "number" || config.chunkOverlap < 0) {
|
|
1947
|
+
errors.push({
|
|
1948
|
+
field: "rag.chunkOverlap",
|
|
1949
|
+
message: "chunkOverlap must be a non-negative integer",
|
|
1950
|
+
severity: "error"
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
return errors;
|
|
1955
|
+
}
|
|
1956
|
+
static isValidCSSColor(color) {
|
|
1957
|
+
const s = new Option().style;
|
|
1958
|
+
s.color = color;
|
|
1959
|
+
return s.color !== "";
|
|
1960
|
+
}
|
|
1961
|
+
/**
|
|
1962
|
+
* Throws if there are error-level validation issues.
|
|
1963
|
+
* Logs warnings to console.
|
|
1964
|
+
*/
|
|
1965
|
+
static validateAndThrow(config) {
|
|
1966
|
+
const errors = this.validate(config);
|
|
1967
|
+
const errorItems = errors.filter((e) => e.severity === "error");
|
|
1968
|
+
const warnings = errors.filter((e) => e.severity === "warning");
|
|
1969
|
+
if (warnings.length > 0) {
|
|
1970
|
+
console.warn("[ConfigValidator] Configuration warnings:");
|
|
1971
|
+
warnings.forEach((w) => {
|
|
1972
|
+
console.warn(` ${w.field}: ${w.message}`);
|
|
1973
|
+
if (w.suggestion) console.warn(` Suggestion: ${w.suggestion}`);
|
|
1974
|
+
});
|
|
1975
|
+
}
|
|
1976
|
+
if (errorItems.length > 0) {
|
|
1977
|
+
const message = errorItems.map((e) => `${e.field}: ${e.message}${e.suggestion ? ` (${e.suggestion})` : ""}`).join("\n ");
|
|
1978
|
+
throw new Error(`[ConfigValidator] Configuration validation failed:
|
|
1979
|
+
${message}`);
|
|
1980
|
+
}
|
|
1981
|
+
}
|
|
1982
|
+
};
|
|
1983
|
+
|
|
1467
1984
|
// src/rag/DocumentChunker.ts
|
|
1468
1985
|
var DocumentChunker = class {
|
|
1469
1986
|
constructor(chunkSize = 1e3, chunkOverlap = 200) {
|
|
@@ -1579,6 +2096,230 @@ var ProviderRegistry = class {
|
|
|
1579
2096
|
};
|
|
1580
2097
|
ProviderRegistry.vectorProviders = {};
|
|
1581
2098
|
|
|
2099
|
+
// src/core/BatchProcessor.ts
|
|
2100
|
+
function isTransientError(error) {
|
|
2101
|
+
if (!(error instanceof Error)) return false;
|
|
2102
|
+
const message = error.message.toLowerCase();
|
|
2103
|
+
if (message.includes("econnrefused") || message.includes("econnreset") || message.includes("timeout") || message.includes("network") || message.includes("socket")) {
|
|
2104
|
+
return true;
|
|
2105
|
+
}
|
|
2106
|
+
if (message.includes("429") || message.includes("503") || message.includes("502") || message.includes("rate limit") || message.includes("too many requests")) {
|
|
2107
|
+
return true;
|
|
2108
|
+
}
|
|
2109
|
+
return false;
|
|
2110
|
+
}
|
|
2111
|
+
function calculateBackoffDelay(attempt, initialDelayMs, maxDelayMs, multiplier) {
|
|
2112
|
+
const exponentialDelay = Math.min(
|
|
2113
|
+
initialDelayMs * Math.pow(multiplier, attempt),
|
|
2114
|
+
maxDelayMs
|
|
2115
|
+
);
|
|
2116
|
+
const jitter = exponentialDelay * 0.1 * (Math.random() * 2 - 1);
|
|
2117
|
+
return Math.max(0, exponentialDelay + jitter);
|
|
2118
|
+
}
|
|
2119
|
+
function sleep(ms) {
|
|
2120
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
2121
|
+
}
|
|
2122
|
+
var BatchProcessor = class {
|
|
2123
|
+
/**
|
|
2124
|
+
* Processes an array of items in configurable batches with retry logic.
|
|
2125
|
+
*
|
|
2126
|
+
* @param items - Items to process
|
|
2127
|
+
* @param processor - Async function that processes a batch of items
|
|
2128
|
+
* @param options - Configuration for batch size, retries, etc.
|
|
2129
|
+
* @returns Object with results, errors, and statistics
|
|
2130
|
+
*
|
|
2131
|
+
* @example
|
|
2132
|
+
* const docs = [...];
|
|
2133
|
+
* const result = await BatchProcessor.processBatch(
|
|
2134
|
+
* docs,
|
|
2135
|
+
* (batch) => vectorDB.batchUpsert(batch),
|
|
2136
|
+
* { batchSize: 100, maxRetries: 3 }
|
|
2137
|
+
* );
|
|
2138
|
+
*/
|
|
2139
|
+
static async processBatch(items, processor, options) {
|
|
2140
|
+
const {
|
|
2141
|
+
batchSize = 100,
|
|
2142
|
+
maxRetries = 3,
|
|
2143
|
+
initialDelayMs = 100,
|
|
2144
|
+
maxDelayMs = 1e4,
|
|
2145
|
+
backoffMultiplier = 2,
|
|
2146
|
+
throwOnPartialFailure = false
|
|
2147
|
+
} = options != null ? options : {};
|
|
2148
|
+
const results = [];
|
|
2149
|
+
const errors = [];
|
|
2150
|
+
const batches = [];
|
|
2151
|
+
for (let i = 0; i < items.length; i += batchSize) {
|
|
2152
|
+
batches.push(items.slice(i, i + batchSize));
|
|
2153
|
+
}
|
|
2154
|
+
if (batches.length === 0) {
|
|
2155
|
+
return { results: [], errors: [], totalProcessed: 0, totalFailed: 0 };
|
|
2156
|
+
}
|
|
2157
|
+
for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
|
|
2158
|
+
const batch = batches[batchIndex];
|
|
2159
|
+
let success = false;
|
|
2160
|
+
let lastError;
|
|
2161
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
2162
|
+
try {
|
|
2163
|
+
const result = await processor(batch);
|
|
2164
|
+
results.push(result);
|
|
2165
|
+
success = true;
|
|
2166
|
+
break;
|
|
2167
|
+
} catch (err) {
|
|
2168
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
2169
|
+
if (!isTransientError(err) || attempt === maxRetries) {
|
|
2170
|
+
break;
|
|
2171
|
+
}
|
|
2172
|
+
const delay = calculateBackoffDelay(
|
|
2173
|
+
attempt,
|
|
2174
|
+
initialDelayMs,
|
|
2175
|
+
maxDelayMs,
|
|
2176
|
+
backoffMultiplier
|
|
2177
|
+
);
|
|
2178
|
+
await sleep(delay);
|
|
2179
|
+
}
|
|
2180
|
+
}
|
|
2181
|
+
if (!success && lastError) {
|
|
2182
|
+
errors.push({
|
|
2183
|
+
index: batchIndex,
|
|
2184
|
+
error: lastError,
|
|
2185
|
+
itemCount: batch.length
|
|
2186
|
+
});
|
|
2187
|
+
}
|
|
2188
|
+
}
|
|
2189
|
+
const totalProcessed = items.length - (errors.length > 0 ? errors.reduce((sum, e) => sum + (e.itemCount || 0), 0) : 0);
|
|
2190
|
+
const totalFailed = errors.reduce((sum, e) => sum + (e.itemCount || 0), 0);
|
|
2191
|
+
if (throwOnPartialFailure && errors.length > 0) {
|
|
2192
|
+
const errorMessages = errors.map((e) => `Batch ${e.index} (${e.itemCount} items): ${e.error.message}`).join("\n");
|
|
2193
|
+
throw new Error(
|
|
2194
|
+
`[BatchProcessor] Batch processing failed:
|
|
2195
|
+
${errorMessages}`
|
|
2196
|
+
);
|
|
2197
|
+
}
|
|
2198
|
+
return { results, errors, totalProcessed, totalFailed };
|
|
2199
|
+
}
|
|
2200
|
+
/**
|
|
2201
|
+
* Processes items sequentially (one at a time) with retry logic.
|
|
2202
|
+
* Useful for operations that don't support batching or when granular
|
|
2203
|
+
* error tracking is needed.
|
|
2204
|
+
*/
|
|
2205
|
+
static async processSequential(items, processor, options) {
|
|
2206
|
+
const {
|
|
2207
|
+
maxRetries = 3,
|
|
2208
|
+
initialDelayMs = 100,
|
|
2209
|
+
maxDelayMs = 1e4,
|
|
2210
|
+
backoffMultiplier = 2,
|
|
2211
|
+
throwOnPartialFailure = false
|
|
2212
|
+
} = options != null ? options : {};
|
|
2213
|
+
const results = [];
|
|
2214
|
+
const errors = [];
|
|
2215
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
2216
|
+
const item = items[itemIndex];
|
|
2217
|
+
let success = false;
|
|
2218
|
+
let lastError;
|
|
2219
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
2220
|
+
try {
|
|
2221
|
+
const result = await processor(item);
|
|
2222
|
+
results.push(result);
|
|
2223
|
+
success = true;
|
|
2224
|
+
break;
|
|
2225
|
+
} catch (err) {
|
|
2226
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
2227
|
+
if (!isTransientError(err) || attempt === maxRetries) {
|
|
2228
|
+
break;
|
|
2229
|
+
}
|
|
2230
|
+
const delay = calculateBackoffDelay(
|
|
2231
|
+
attempt,
|
|
2232
|
+
initialDelayMs,
|
|
2233
|
+
maxDelayMs,
|
|
2234
|
+
backoffMultiplier
|
|
2235
|
+
);
|
|
2236
|
+
await sleep(delay);
|
|
2237
|
+
}
|
|
2238
|
+
}
|
|
2239
|
+
if (!success && lastError) {
|
|
2240
|
+
errors.push({ index: itemIndex, error: lastError, itemCount: 1 });
|
|
2241
|
+
}
|
|
2242
|
+
}
|
|
2243
|
+
const totalProcessed = items.length - errors.length;
|
|
2244
|
+
const totalFailed = errors.length;
|
|
2245
|
+
if (throwOnPartialFailure && errors.length > 0) {
|
|
2246
|
+
const errorMessages = errors.map((e) => `Item ${e.index}: ${e.error.message}`).join("\n");
|
|
2247
|
+
throw new Error(
|
|
2248
|
+
`[BatchProcessor] Sequential processing failed:
|
|
2249
|
+
${errorMessages}`
|
|
2250
|
+
);
|
|
2251
|
+
}
|
|
2252
|
+
return { results, errors, totalProcessed, totalFailed };
|
|
2253
|
+
}
|
|
2254
|
+
/**
|
|
2255
|
+
* Maps over items with retry logic, returning results in order.
|
|
2256
|
+
* Like Array.map() but with async processing and automatic retries.
|
|
2257
|
+
*/
|
|
2258
|
+
static async mapWithRetry(items, mapper, options) {
|
|
2259
|
+
const result = await this.processSequential(items, mapper, options);
|
|
2260
|
+
if (result.errors.length > 0) {
|
|
2261
|
+
console.warn(
|
|
2262
|
+
`[BatchProcessor] mapWithRetry: ${result.errors.length} items failed`
|
|
2263
|
+
);
|
|
2264
|
+
}
|
|
2265
|
+
const orderedResults = new Array(items.length);
|
|
2266
|
+
let resultIndex = 0;
|
|
2267
|
+
let errorIndex = 0;
|
|
2268
|
+
for (let i = 0; i < items.length; i++) {
|
|
2269
|
+
if (errorIndex < result.errors.length && result.errors[errorIndex].index === i) {
|
|
2270
|
+
orderedResults[i] = void 0;
|
|
2271
|
+
errorIndex++;
|
|
2272
|
+
} else {
|
|
2273
|
+
orderedResults[i] = result.results[resultIndex];
|
|
2274
|
+
resultIndex++;
|
|
2275
|
+
}
|
|
2276
|
+
}
|
|
2277
|
+
return orderedResults.filter((r) => r !== void 0);
|
|
2278
|
+
}
|
|
2279
|
+
/**
|
|
2280
|
+
* Parallel processor with concurrency limit.
|
|
2281
|
+
* Processes up to `concurrency` items at the same time.
|
|
2282
|
+
*/
|
|
2283
|
+
static async processConcurrent(items, processor, concurrency = 5, options) {
|
|
2284
|
+
const { throwOnPartialFailure = false } = options != null ? options : {};
|
|
2285
|
+
const results = [];
|
|
2286
|
+
const errors = [];
|
|
2287
|
+
for (let i = 0; i < items.length; i += concurrency) {
|
|
2288
|
+
const chunk = items.slice(i, i + concurrency).map((item, idx) => ({
|
|
2289
|
+
item,
|
|
2290
|
+
originalIndex: i + idx
|
|
2291
|
+
}));
|
|
2292
|
+
const chunkPromises = chunk.map(async ({ item, originalIndex }) => {
|
|
2293
|
+
try {
|
|
2294
|
+
const result = await processor(item);
|
|
2295
|
+
return { success: true, result, index: originalIndex };
|
|
2296
|
+
} catch (err) {
|
|
2297
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
2298
|
+
return { success: false, error, index: originalIndex };
|
|
2299
|
+
}
|
|
2300
|
+
});
|
|
2301
|
+
const chunkResults = await Promise.all(chunkPromises);
|
|
2302
|
+
for (const { success, result, error, index } of chunkResults) {
|
|
2303
|
+
if (success) {
|
|
2304
|
+
results.push(result);
|
|
2305
|
+
} else {
|
|
2306
|
+
errors.push({ index, error, itemCount: 1 });
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
}
|
|
2310
|
+
const totalProcessed = items.length - errors.length;
|
|
2311
|
+
const totalFailed = errors.length;
|
|
2312
|
+
if (throwOnPartialFailure && errors.length > 0) {
|
|
2313
|
+
const errorMessages = errors.map((e) => `Item ${e.index}: ${e.error.message}`).join("\n");
|
|
2314
|
+
throw new Error(
|
|
2315
|
+
`[BatchProcessor] Concurrent processing failed:
|
|
2316
|
+
${errorMessages}`
|
|
2317
|
+
);
|
|
2318
|
+
}
|
|
2319
|
+
return { results, errors, totalProcessed, totalFailed };
|
|
2320
|
+
}
|
|
2321
|
+
};
|
|
2322
|
+
|
|
1582
2323
|
// src/core/Pipeline.ts
|
|
1583
2324
|
var Pipeline = class {
|
|
1584
2325
|
constructor(config) {
|
|
@@ -1603,24 +2344,64 @@ var Pipeline = class {
|
|
|
1603
2344
|
await this.vectorDB.initialize();
|
|
1604
2345
|
this.initialised = true;
|
|
1605
2346
|
}
|
|
2347
|
+
/**
|
|
2348
|
+
* Ingest documents with automatic chunking, embedding, and batch processing.
|
|
2349
|
+
* Handles retries for transient failures.
|
|
2350
|
+
*/
|
|
1606
2351
|
async ingest(documents, namespace) {
|
|
1607
2352
|
await this.initialize();
|
|
1608
2353
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
1609
2354
|
const results = [];
|
|
1610
2355
|
for (const doc of documents) {
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
2356
|
+
try {
|
|
2357
|
+
const chunks = this.chunker.chunk(doc.content, {
|
|
2358
|
+
docId: doc.docId,
|
|
2359
|
+
metadata: doc.metadata
|
|
2360
|
+
});
|
|
2361
|
+
const batchOptions = {
|
|
2362
|
+
batchSize: 50,
|
|
2363
|
+
// Embedding batch size
|
|
2364
|
+
maxRetries: 3,
|
|
2365
|
+
initialDelayMs: 100
|
|
2366
|
+
};
|
|
2367
|
+
const vectors = await BatchProcessor.mapWithRetry(
|
|
2368
|
+
chunks.map((c) => c.content),
|
|
2369
|
+
(text) => this.embeddingProvider.embed(text),
|
|
2370
|
+
batchOptions
|
|
2371
|
+
);
|
|
2372
|
+
if (vectors.length !== chunks.length) {
|
|
2373
|
+
throw new Error(`Embedding failed: got ${vectors.length} vectors for ${chunks.length} chunks`);
|
|
2374
|
+
}
|
|
2375
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
2376
|
+
id: chunk.id,
|
|
2377
|
+
vector: vectors[i],
|
|
2378
|
+
content: chunk.content,
|
|
2379
|
+
metadata: chunk.metadata
|
|
2380
|
+
}));
|
|
2381
|
+
const upsertBatchOptions = {
|
|
2382
|
+
batchSize: 100,
|
|
2383
|
+
maxRetries: 3,
|
|
2384
|
+
initialDelayMs: 100
|
|
2385
|
+
};
|
|
2386
|
+
const upsertResult = await BatchProcessor.processBatch(
|
|
2387
|
+
upsertDocs,
|
|
2388
|
+
(batch) => this.vectorDB.batchUpsert(batch, ns),
|
|
2389
|
+
upsertBatchOptions
|
|
2390
|
+
);
|
|
2391
|
+
if (upsertResult.errors.length > 0) {
|
|
2392
|
+
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed for doc ${doc.docId}`);
|
|
2393
|
+
}
|
|
2394
|
+
results.push({
|
|
2395
|
+
docId: doc.docId,
|
|
2396
|
+
chunksIngested: upsertResult.totalProcessed
|
|
2397
|
+
});
|
|
2398
|
+
} catch (error) {
|
|
2399
|
+
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
2400
|
+
results.push({
|
|
2401
|
+
docId: doc.docId,
|
|
2402
|
+
chunksIngested: 0
|
|
2403
|
+
});
|
|
2404
|
+
}
|
|
1624
2405
|
}
|
|
1625
2406
|
return results;
|
|
1626
2407
|
}
|
|
@@ -1630,22 +2411,483 @@ var Pipeline = class {
|
|
|
1630
2411
|
const ns = namespace != null ? namespace : this.config.projectId;
|
|
1631
2412
|
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
1632
2413
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
2414
|
+
try {
|
|
2415
|
+
const queryVector = await this.embeddingProvider.embed(question);
|
|
2416
|
+
const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
|
|
2417
|
+
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
2418
|
+
const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
1637
2419
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
2420
|
+
const messages = [...history, { role: "user", content: question }];
|
|
2421
|
+
const reply = await this.llmProvider.chat(messages, context);
|
|
2422
|
+
return { reply, sources };
|
|
2423
|
+
} catch (error) {
|
|
2424
|
+
throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2425
|
+
}
|
|
1641
2426
|
}
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
2427
|
+
};
|
|
2428
|
+
|
|
2429
|
+
// src/core/ProviderHealthCheck.ts
|
|
2430
|
+
var ProviderHealthCheck = class {
|
|
2431
|
+
/**
|
|
2432
|
+
* Validates vector database configuration before initialization.
|
|
2433
|
+
* Performs connectivity checks and verifies required resources exist.
|
|
2434
|
+
*/
|
|
2435
|
+
static async checkVectorProvider(config) {
|
|
2436
|
+
const timestamp = Date.now();
|
|
2437
|
+
const configErrors = ConfigValidator.validate({
|
|
2438
|
+
projectId: "health-check",
|
|
2439
|
+
vectorDb: config,
|
|
2440
|
+
llm: { provider: "openai", model: "gpt-4o" },
|
|
2441
|
+
// dummy
|
|
2442
|
+
embedding: { provider: "openai", model: "text-embedding-3-small" }
|
|
2443
|
+
});
|
|
2444
|
+
const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
|
|
2445
|
+
if (vectorDbErrors.length > 0) {
|
|
2446
|
+
return {
|
|
2447
|
+
healthy: false,
|
|
2448
|
+
provider: config.provider,
|
|
2449
|
+
error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
|
|
2450
|
+
timestamp
|
|
2451
|
+
};
|
|
2452
|
+
}
|
|
2453
|
+
try {
|
|
2454
|
+
switch (config.provider) {
|
|
2455
|
+
case "pinecone":
|
|
2456
|
+
return await this.checkPinecone(config, timestamp);
|
|
2457
|
+
case "pgvector":
|
|
2458
|
+
case "postgresql":
|
|
2459
|
+
return await this.checkPostgres(config, timestamp);
|
|
2460
|
+
case "mongodb":
|
|
2461
|
+
return await this.checkMongoDB(config, timestamp);
|
|
2462
|
+
case "milvus":
|
|
2463
|
+
return await this.checkMilvus(config, timestamp);
|
|
2464
|
+
case "qdrant":
|
|
2465
|
+
return await this.checkQdrant(config, timestamp);
|
|
2466
|
+
case "chromadb":
|
|
2467
|
+
return await this.checkChromaDB(config, timestamp);
|
|
2468
|
+
case "redis":
|
|
2469
|
+
return await this.checkRedis(config, timestamp);
|
|
2470
|
+
case "weaviate":
|
|
2471
|
+
return await this.checkWeaviate(config, timestamp);
|
|
2472
|
+
case "rest":
|
|
2473
|
+
case "universal_rest":
|
|
2474
|
+
return await this.checkRestAPI(config, timestamp);
|
|
2475
|
+
default:
|
|
2476
|
+
return {
|
|
2477
|
+
healthy: false,
|
|
2478
|
+
provider: config.provider,
|
|
2479
|
+
error: `Unsupported provider: ${config.provider}`,
|
|
2480
|
+
timestamp
|
|
2481
|
+
};
|
|
2482
|
+
}
|
|
2483
|
+
} catch (error) {
|
|
2484
|
+
return {
|
|
2485
|
+
healthy: false,
|
|
2486
|
+
provider: config.provider,
|
|
2487
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2488
|
+
timestamp
|
|
2489
|
+
};
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
static async checkPinecone(config, timestamp) {
|
|
2493
|
+
var _a, _b;
|
|
2494
|
+
const opts = config.options;
|
|
2495
|
+
try {
|
|
2496
|
+
const { Pinecone: Pinecone2 } = await import("@pinecone-database/pinecone");
|
|
2497
|
+
const client = new Pinecone2({ apiKey: opts.apiKey });
|
|
2498
|
+
const indexes = await client.listIndexes();
|
|
2499
|
+
const indexNames = (_b = (_a = indexes.indexes) == null ? void 0 : _a.map((i) => i.name)) != null ? _b : [];
|
|
2500
|
+
if (!indexNames.includes(config.indexName)) {
|
|
2501
|
+
return {
|
|
2502
|
+
healthy: false,
|
|
2503
|
+
provider: "pinecone",
|
|
2504
|
+
error: `Index "${config.indexName}" not found. Available: ${indexNames.join(", ")}`,
|
|
2505
|
+
timestamp
|
|
2506
|
+
};
|
|
2507
|
+
}
|
|
2508
|
+
return {
|
|
2509
|
+
healthy: true,
|
|
2510
|
+
provider: "pinecone",
|
|
2511
|
+
capabilities: {
|
|
2512
|
+
indexes: indexNames.length,
|
|
2513
|
+
targetIndex: config.indexName
|
|
2514
|
+
},
|
|
2515
|
+
timestamp
|
|
2516
|
+
};
|
|
2517
|
+
} catch (error) {
|
|
2518
|
+
return {
|
|
2519
|
+
healthy: false,
|
|
2520
|
+
provider: "pinecone",
|
|
2521
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2522
|
+
timestamp
|
|
2523
|
+
};
|
|
2524
|
+
}
|
|
2525
|
+
}
|
|
2526
|
+
static async checkPostgres(config, timestamp) {
|
|
2527
|
+
const opts = config.options;
|
|
2528
|
+
try {
|
|
2529
|
+
const { Client } = await import("pg");
|
|
2530
|
+
const client = new Client({ connectionString: opts.connectionString });
|
|
2531
|
+
await client.connect();
|
|
2532
|
+
const result = await client.query(`
|
|
2533
|
+
SELECT EXISTS(
|
|
2534
|
+
SELECT 1 FROM pg_extension WHERE extname = 'vector'
|
|
2535
|
+
);
|
|
2536
|
+
`);
|
|
2537
|
+
const hasVector = result.rows[0].exists;
|
|
2538
|
+
await client.end();
|
|
2539
|
+
return {
|
|
2540
|
+
healthy: true,
|
|
2541
|
+
provider: "postgresql",
|
|
2542
|
+
capabilities: { pgvectorInstalled: hasVector },
|
|
2543
|
+
timestamp
|
|
2544
|
+
};
|
|
2545
|
+
} catch (error) {
|
|
2546
|
+
return {
|
|
2547
|
+
healthy: false,
|
|
2548
|
+
provider: "postgresql",
|
|
2549
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2550
|
+
timestamp
|
|
2551
|
+
};
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2554
|
+
static async checkMongoDB(config, timestamp) {
|
|
2555
|
+
const opts = config.options;
|
|
2556
|
+
try {
|
|
2557
|
+
const { MongoClient: MongoClient2 } = await import("mongodb");
|
|
2558
|
+
const client = new MongoClient2(opts.uri);
|
|
2559
|
+
await client.connect();
|
|
2560
|
+
const db = client.db(opts.database);
|
|
2561
|
+
const collections = await db.listCollections().toArray();
|
|
2562
|
+
const collectionNames = collections.map((c) => c.name);
|
|
2563
|
+
const hasCollection = collectionNames.includes(opts.collection);
|
|
2564
|
+
await client.close();
|
|
2565
|
+
return {
|
|
2566
|
+
healthy: true,
|
|
2567
|
+
provider: "mongodb",
|
|
2568
|
+
capabilities: {
|
|
2569
|
+
collections: collectionNames.length,
|
|
2570
|
+
targetCollection: hasCollection ? opts.collection : "NOT FOUND"
|
|
2571
|
+
},
|
|
2572
|
+
timestamp
|
|
2573
|
+
};
|
|
2574
|
+
} catch (error) {
|
|
2575
|
+
return {
|
|
2576
|
+
healthy: false,
|
|
2577
|
+
provider: "mongodb",
|
|
2578
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2579
|
+
timestamp
|
|
2580
|
+
};
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
static async checkMilvus(config, timestamp) {
|
|
2584
|
+
const opts = config.options;
|
|
2585
|
+
const host = opts.host || "localhost";
|
|
2586
|
+
const port = opts.port || 19530;
|
|
2587
|
+
try {
|
|
2588
|
+
const controller = new AbortController();
|
|
2589
|
+
const timeoutId = setTimeout(() => controller.abort(), 5e3);
|
|
2590
|
+
const response = await fetch(`http://${host}:${port}/healthz`, {
|
|
2591
|
+
signal: controller.signal
|
|
2592
|
+
});
|
|
2593
|
+
clearTimeout(timeoutId);
|
|
2594
|
+
if (!response.ok) {
|
|
2595
|
+
throw new Error(`Health check returned ${response.status}`);
|
|
2596
|
+
}
|
|
2597
|
+
return {
|
|
2598
|
+
healthy: true,
|
|
2599
|
+
provider: "milvus",
|
|
2600
|
+
capabilities: { endpoint: `${host}:${port}` },
|
|
2601
|
+
timestamp
|
|
2602
|
+
};
|
|
2603
|
+
} catch (error) {
|
|
2604
|
+
return {
|
|
2605
|
+
healthy: false,
|
|
2606
|
+
provider: "milvus",
|
|
2607
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2608
|
+
timestamp
|
|
2609
|
+
};
|
|
2610
|
+
}
|
|
2611
|
+
}
|
|
2612
|
+
static async checkQdrant(config, timestamp) {
|
|
2613
|
+
const opts = config.options;
|
|
2614
|
+
const baseUrl = opts.baseUrl || opts.url || "http://localhost:6333";
|
|
2615
|
+
try {
|
|
2616
|
+
const response = await fetch(`${baseUrl}/health`);
|
|
2617
|
+
if (!response.ok) {
|
|
2618
|
+
throw new Error(`Health check returned ${response.status}`);
|
|
2619
|
+
}
|
|
2620
|
+
const health = await response.json();
|
|
2621
|
+
return {
|
|
2622
|
+
healthy: true,
|
|
2623
|
+
provider: "qdrant",
|
|
2624
|
+
capabilities: health,
|
|
2625
|
+
timestamp
|
|
2626
|
+
};
|
|
2627
|
+
} catch (error) {
|
|
2628
|
+
return {
|
|
2629
|
+
healthy: false,
|
|
2630
|
+
provider: "qdrant",
|
|
2631
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2632
|
+
timestamp
|
|
2633
|
+
};
|
|
2634
|
+
}
|
|
2635
|
+
}
|
|
2636
|
+
static async checkChromaDB(config, timestamp) {
|
|
2637
|
+
const opts = config.options;
|
|
2638
|
+
const host = opts.host || "localhost";
|
|
2639
|
+
const port = opts.port || 8e3;
|
|
2640
|
+
try {
|
|
2641
|
+
const response = await fetch(`http://${host}:${port}/api/v1/heartbeat`);
|
|
2642
|
+
if (!response.ok) {
|
|
2643
|
+
throw new Error(`Health check returned ${response.status}`);
|
|
2644
|
+
}
|
|
2645
|
+
return {
|
|
2646
|
+
healthy: true,
|
|
2647
|
+
provider: "chromadb",
|
|
2648
|
+
capabilities: { endpoint: `${host}:${port}` },
|
|
2649
|
+
timestamp
|
|
2650
|
+
};
|
|
2651
|
+
} catch (error) {
|
|
2652
|
+
return {
|
|
2653
|
+
healthy: false,
|
|
2654
|
+
provider: "chromadb",
|
|
2655
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2656
|
+
timestamp
|
|
2657
|
+
};
|
|
2658
|
+
}
|
|
2659
|
+
}
|
|
2660
|
+
static async checkRedis(config, timestamp) {
|
|
2661
|
+
const opts = config.options;
|
|
2662
|
+
const url = opts.url || `redis://${opts.host}:${opts.port}`;
|
|
2663
|
+
try {
|
|
2664
|
+
await fetch(url);
|
|
2665
|
+
return {
|
|
2666
|
+
healthy: true,
|
|
2667
|
+
provider: "redis",
|
|
2668
|
+
capabilities: { endpoint: url },
|
|
2669
|
+
timestamp
|
|
2670
|
+
};
|
|
2671
|
+
} catch (error) {
|
|
2672
|
+
return {
|
|
2673
|
+
healthy: false,
|
|
2674
|
+
provider: "redis",
|
|
2675
|
+
error: `Connection check failed: ${error instanceof Error ? error.message : String(error)}. Ensure Redis is running at ${url}`,
|
|
2676
|
+
timestamp
|
|
2677
|
+
};
|
|
2678
|
+
}
|
|
2679
|
+
}
|
|
2680
|
+
static async checkWeaviate(config, timestamp) {
|
|
2681
|
+
const opts = config.options;
|
|
2682
|
+
const url = (opts.url || "http://localhost:8080").replace(/\/$/, "");
|
|
2683
|
+
try {
|
|
2684
|
+
const response = await fetch(`${url}/.well-known/ready`);
|
|
2685
|
+
if (!response.ok) {
|
|
2686
|
+
throw new Error(`Health check returned ${response.status}`);
|
|
2687
|
+
}
|
|
2688
|
+
return {
|
|
2689
|
+
healthy: true,
|
|
2690
|
+
provider: "weaviate",
|
|
2691
|
+
capabilities: { endpoint: url },
|
|
2692
|
+
timestamp
|
|
2693
|
+
};
|
|
2694
|
+
} catch (error) {
|
|
2695
|
+
return {
|
|
2696
|
+
healthy: false,
|
|
2697
|
+
provider: "weaviate",
|
|
2698
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2699
|
+
timestamp
|
|
2700
|
+
};
|
|
2701
|
+
}
|
|
2702
|
+
}
|
|
2703
|
+
static async checkRestAPI(config, timestamp) {
|
|
2704
|
+
const opts = config.options;
|
|
2705
|
+
const baseUrl = (opts.baseUrl || "").replace(/\/$/, "");
|
|
2706
|
+
if (!baseUrl) {
|
|
2707
|
+
return {
|
|
2708
|
+
healthy: false,
|
|
2709
|
+
provider: "rest",
|
|
2710
|
+
error: "baseUrl is required",
|
|
2711
|
+
timestamp
|
|
2712
|
+
};
|
|
2713
|
+
}
|
|
2714
|
+
try {
|
|
2715
|
+
const response = await fetch(`${baseUrl}/health`, {
|
|
2716
|
+
headers: opts.headers ? JSON.parse(opts.headers) : {}
|
|
2717
|
+
});
|
|
2718
|
+
return {
|
|
2719
|
+
healthy: response.ok,
|
|
2720
|
+
provider: "rest",
|
|
2721
|
+
error: response.ok ? void 0 : `Health check returned ${response.status}`,
|
|
2722
|
+
timestamp
|
|
2723
|
+
};
|
|
2724
|
+
} catch (error) {
|
|
2725
|
+
return {
|
|
2726
|
+
healthy: false,
|
|
2727
|
+
provider: "rest",
|
|
2728
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2729
|
+
timestamp
|
|
2730
|
+
};
|
|
2731
|
+
}
|
|
2732
|
+
}
|
|
2733
|
+
/**
|
|
2734
|
+
* Validates LLM provider configuration.
|
|
2735
|
+
*/
|
|
2736
|
+
static async checkLLMProvider(config) {
|
|
2737
|
+
const timestamp = Date.now();
|
|
2738
|
+
try {
|
|
2739
|
+
switch (config.provider) {
|
|
2740
|
+
case "openai":
|
|
2741
|
+
return await this.checkOpenAI(config, timestamp);
|
|
2742
|
+
case "anthropic":
|
|
2743
|
+
return await this.checkAnthropic(config, timestamp);
|
|
2744
|
+
case "ollama":
|
|
2745
|
+
return await this.checkOllama(config, timestamp);
|
|
2746
|
+
case "rest":
|
|
2747
|
+
case "universal_rest":
|
|
2748
|
+
return await this.checkLLMRestAPI(config, timestamp);
|
|
2749
|
+
default:
|
|
2750
|
+
return {
|
|
2751
|
+
healthy: false,
|
|
2752
|
+
provider: config.provider,
|
|
2753
|
+
error: `Unsupported provider: ${config.provider}`,
|
|
2754
|
+
timestamp
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
} catch (error) {
|
|
2758
|
+
return {
|
|
2759
|
+
healthy: false,
|
|
2760
|
+
provider: config.provider,
|
|
2761
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2762
|
+
timestamp
|
|
2763
|
+
};
|
|
2764
|
+
}
|
|
2765
|
+
}
|
|
2766
|
+
static async checkOpenAI(config, timestamp) {
|
|
2767
|
+
try {
|
|
2768
|
+
const OpenAI2 = await import("openai");
|
|
2769
|
+
const client = new OpenAI2.default({ apiKey: config.apiKey });
|
|
2770
|
+
const models = await client.models.list();
|
|
2771
|
+
const hasModel = models.data.some((m) => m.id === config.model);
|
|
2772
|
+
return {
|
|
2773
|
+
healthy: true,
|
|
2774
|
+
provider: "openai",
|
|
2775
|
+
capabilities: {
|
|
2776
|
+
model: config.model,
|
|
2777
|
+
available: hasModel,
|
|
2778
|
+
totalModels: models.data.length
|
|
2779
|
+
},
|
|
2780
|
+
timestamp
|
|
2781
|
+
};
|
|
2782
|
+
} catch (error) {
|
|
2783
|
+
return {
|
|
2784
|
+
healthy: false,
|
|
2785
|
+
provider: "openai",
|
|
2786
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2787
|
+
timestamp
|
|
2788
|
+
};
|
|
2789
|
+
}
|
|
2790
|
+
}
|
|
2791
|
+
static async checkAnthropic(config, timestamp) {
|
|
2792
|
+
try {
|
|
2793
|
+
const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
|
|
2794
|
+
const client = new Anthropic2({ apiKey: config.apiKey });
|
|
2795
|
+
await client.messages.create({
|
|
2796
|
+
model: config.model,
|
|
2797
|
+
max_tokens: 10,
|
|
2798
|
+
messages: [{ role: "user", content: "ping" }]
|
|
2799
|
+
});
|
|
2800
|
+
return {
|
|
2801
|
+
healthy: true,
|
|
2802
|
+
provider: "anthropic",
|
|
2803
|
+
capabilities: { model: config.model },
|
|
2804
|
+
timestamp
|
|
2805
|
+
};
|
|
2806
|
+
} catch (error) {
|
|
2807
|
+
return {
|
|
2808
|
+
healthy: false,
|
|
2809
|
+
provider: "anthropic",
|
|
2810
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2811
|
+
timestamp
|
|
2812
|
+
};
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
static async checkOllama(config, timestamp) {
|
|
2816
|
+
const baseUrl = (config.baseUrl || "http://localhost:11434").replace(/\/$/, "");
|
|
2817
|
+
try {
|
|
2818
|
+
const response = await fetch(`${baseUrl}/api/tags`);
|
|
2819
|
+
if (!response.ok) {
|
|
2820
|
+
throw new Error(`Health check returned ${response.status}`);
|
|
2821
|
+
}
|
|
2822
|
+
const data = await response.json();
|
|
2823
|
+
const models = data.models || [];
|
|
2824
|
+
const hasModel = models.some((m) => m.name === config.model);
|
|
2825
|
+
return {
|
|
2826
|
+
healthy: true,
|
|
2827
|
+
provider: "ollama",
|
|
2828
|
+
capabilities: {
|
|
2829
|
+
model: config.model,
|
|
2830
|
+
available: hasModel,
|
|
2831
|
+
totalModels: models.length
|
|
2832
|
+
},
|
|
2833
|
+
timestamp
|
|
2834
|
+
};
|
|
2835
|
+
} catch (error) {
|
|
2836
|
+
return {
|
|
2837
|
+
healthy: false,
|
|
2838
|
+
provider: "ollama",
|
|
2839
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2840
|
+
timestamp
|
|
2841
|
+
};
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
static async checkLLMRestAPI(config, timestamp) {
|
|
2845
|
+
var _a, _b;
|
|
2846
|
+
const baseUrlRaw = config.baseUrl || ((_a = config.options) == null ? void 0 : _a.baseUrl);
|
|
2847
|
+
const baseUrl = (typeof baseUrlRaw === "string" ? baseUrlRaw : "").replace(/\/$/, "");
|
|
2848
|
+
if (!baseUrl) {
|
|
2849
|
+
return {
|
|
2850
|
+
healthy: false,
|
|
2851
|
+
provider: config.provider,
|
|
2852
|
+
error: "baseUrl is required",
|
|
2853
|
+
timestamp
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
try {
|
|
2857
|
+
const headers = (_b = config.options) == null ? void 0 : _b.headers;
|
|
2858
|
+
const response = await fetch(`${baseUrl}/health`, {
|
|
2859
|
+
headers: headers || void 0
|
|
2860
|
+
});
|
|
2861
|
+
return {
|
|
2862
|
+
healthy: response.ok,
|
|
2863
|
+
provider: config.provider,
|
|
2864
|
+
error: response.ok ? void 0 : `Health check returned ${response.status}`,
|
|
2865
|
+
timestamp
|
|
2866
|
+
};
|
|
2867
|
+
} catch (error) {
|
|
2868
|
+
return {
|
|
2869
|
+
healthy: false,
|
|
2870
|
+
provider: config.provider,
|
|
2871
|
+
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2872
|
+
timestamp
|
|
2873
|
+
};
|
|
2874
|
+
}
|
|
2875
|
+
}
|
|
2876
|
+
/**
|
|
2877
|
+
* Runs comprehensive health checks on all configured providers.
|
|
2878
|
+
*/
|
|
2879
|
+
static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
|
|
2880
|
+
const [vectorDb, llm, embedding] = await Promise.all([
|
|
2881
|
+
this.checkVectorProvider(vectorDbConfig),
|
|
2882
|
+
this.checkLLMProvider(llmConfig),
|
|
2883
|
+
embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
|
|
1647
2884
|
]);
|
|
1648
|
-
return {
|
|
2885
|
+
return {
|
|
2886
|
+
vectorDb,
|
|
2887
|
+
llm,
|
|
2888
|
+
embedding,
|
|
2889
|
+
allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
|
|
2890
|
+
};
|
|
1649
2891
|
}
|
|
1650
2892
|
};
|
|
1651
2893
|
|
|
@@ -1654,10 +2896,26 @@ var VectorPlugin = class {
|
|
|
1654
2896
|
/**
|
|
1655
2897
|
* Initializes the plugin with the host configuration.
|
|
1656
2898
|
* @param hostConfig - Configuration object passed from the host application.
|
|
2899
|
+
* @throws Error if configuration is invalid or providers are unhealthy
|
|
2900
|
+
*
|
|
2901
|
+
* @example
|
|
2902
|
+
* const plugin = new VectorPlugin({
|
|
2903
|
+
* projectId: 'my-app',
|
|
2904
|
+
* vectorDb: {
|
|
2905
|
+
* provider: 'pinecone',
|
|
2906
|
+
* indexName: 'my-index',
|
|
2907
|
+
* options: { apiKey: process.env.PINECONE_API_KEY }
|
|
2908
|
+
* },
|
|
2909
|
+
* llm: {
|
|
2910
|
+
* provider: 'openai',
|
|
2911
|
+
* model: 'gpt-4o',
|
|
2912
|
+
* apiKey: process.env.OPENAI_API_KEY
|
|
2913
|
+
* }
|
|
2914
|
+
* });
|
|
1657
2915
|
*/
|
|
1658
2916
|
constructor(hostConfig) {
|
|
1659
2917
|
this.config = ConfigResolver.resolve(hostConfig);
|
|
1660
|
-
|
|
2918
|
+
ConfigValidator.validateAndThrow(this.config);
|
|
1661
2919
|
this.pipeline = new Pipeline(this.config);
|
|
1662
2920
|
}
|
|
1663
2921
|
/**
|
|
@@ -1666,6 +2924,19 @@ var VectorPlugin = class {
|
|
|
1666
2924
|
getConfig() {
|
|
1667
2925
|
return this.config;
|
|
1668
2926
|
}
|
|
2927
|
+
/**
|
|
2928
|
+
* Perform pre-flight health checks on all configured providers.
|
|
2929
|
+
* Useful to verify connectivity before running operations.
|
|
2930
|
+
*
|
|
2931
|
+
* @returns Health status for vector DB, LLM, and embedding providers
|
|
2932
|
+
*/
|
|
2933
|
+
async checkHealth() {
|
|
2934
|
+
return ProviderHealthCheck.checkAll(
|
|
2935
|
+
this.config.vectorDb,
|
|
2936
|
+
this.config.llm,
|
|
2937
|
+
this.config.embedding
|
|
2938
|
+
);
|
|
2939
|
+
}
|
|
1669
2940
|
/**
|
|
1670
2941
|
* Run a chat query.
|
|
1671
2942
|
*/
|
|
@@ -1678,12 +2949,6 @@ var VectorPlugin = class {
|
|
|
1678
2949
|
async ingest(documents, namespace) {
|
|
1679
2950
|
return this.pipeline.ingest(documents, namespace);
|
|
1680
2951
|
}
|
|
1681
|
-
/**
|
|
1682
|
-
* Check the health of the connected providers.
|
|
1683
|
-
*/
|
|
1684
|
-
async health() {
|
|
1685
|
-
return this.pipeline.health();
|
|
1686
|
-
}
|
|
1687
2952
|
};
|
|
1688
2953
|
|
|
1689
2954
|
// src/utils/DocumentParser.ts
|
|
@@ -1784,8 +3049,8 @@ function createHealthHandler(config) {
|
|
|
1784
3049
|
const plugin = new VectorPlugin(config);
|
|
1785
3050
|
return async function GET() {
|
|
1786
3051
|
try {
|
|
1787
|
-
const health = await plugin.
|
|
1788
|
-
const status = health.
|
|
3052
|
+
const health = await plugin.checkHealth();
|
|
3053
|
+
const status = health.allHealthy ? "ok" : "degraded";
|
|
1789
3054
|
return import_server.NextResponse.json(__spreadProps(__spreadValues({
|
|
1790
3055
|
status
|
|
1791
3056
|
}, health), {
|