@retrivora-ai/rag-engine 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/dist/{DocumentChunker-cfaMidtA.d.mts → DocumentChunker-BEyzadsv.d.mts} +2 -2
  2. package/dist/{DocumentChunker-cfaMidtA.d.ts → DocumentChunker-BEyzadsv.d.ts} +2 -2
  3. package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
  4. package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
  5. package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
  6. package/dist/{RagConfig-DG_0f8ka.d.mts → RagConfig-hBGXJmSx.d.mts} +3 -3
  7. package/dist/{RagConfig-DG_0f8ka.d.ts → RagConfig-hBGXJmSx.d.ts} +3 -3
  8. package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
  9. package/dist/chunk-HSBXE2WV.mjs +1758 -0
  10. package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
  11. package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
  12. package/dist/handlers/index.d.mts +3 -44
  13. package/dist/handlers/index.d.ts +3 -44
  14. package/dist/handlers/index.js +1322 -57
  15. package/dist/handlers/index.mjs +1 -1
  16. package/dist/index-Bx182KKn.d.ts +64 -0
  17. package/dist/index-Ck2pt7-8.d.mts +64 -0
  18. package/dist/index.d.mts +4 -4
  19. package/dist/index.d.ts +4 -4
  20. package/dist/server.d.mts +65 -18
  21. package/dist/server.d.ts +65 -18
  22. package/dist/server.js +1322 -57
  23. package/dist/server.mjs +4 -4
  24. package/package.json +2 -1
  25. package/src/config/serverConfig.ts +3 -0
  26. package/src/core/BatchProcessor.ts +347 -0
  27. package/src/core/ConfigValidator.ts +560 -0
  28. package/src/core/Pipeline.ts +89 -39
  29. package/src/core/ProviderHealthCheck.ts +565 -0
  30. package/src/core/VectorPlugin.ts +49 -8
  31. package/src/handlers/index.ts +2 -2
  32. package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
  33. package/src/providers/vectordb/MilvusProvider.ts +1 -1
  34. package/src/providers/vectordb/MongoDBProvider.ts +1 -1
  35. package/src/providers/vectordb/PineconeProvider.ts +4 -4
  36. package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
  37. package/src/providers/vectordb/QdrantProvider.ts +32 -2
  38. package/src/rag/DocumentChunker.ts +2 -2
  39. package/src/types/index.ts +3 -3
  40. package/dist/chunk-BP4U4TT5.mjs +0 -548
package/dist/server.js CHANGED
@@ -557,7 +557,7 @@ var init_PineconeProvider = __esm({
557
557
  var _a;
558
558
  await this.index(namespace).upsert({
559
559
  records: [{
560
- id: doc.id,
560
+ id: String(doc.id),
561
561
  values: doc.vector,
562
562
  metadata: __spreadValues({ content: doc.content }, (_a = doc.metadata) != null ? _a : {})
563
563
  }]
@@ -569,7 +569,7 @@ var init_PineconeProvider = __esm({
569
569
  const records = docs.slice(i, i + BATCH).map((d) => {
570
570
  var _a;
571
571
  return {
572
- id: d.id,
572
+ id: String(d.id),
573
573
  values: d.vector,
574
574
  metadata: __spreadValues({ content: d.content }, (_a = d.metadata) != null ? _a : {})
575
575
  };
@@ -595,7 +595,7 @@ var init_PineconeProvider = __esm({
595
595
  });
596
596
  }
597
597
  async delete(id, namespace) {
598
- await this.index(namespace).deleteOne({ id });
598
+ await this.index(namespace).deleteOne({ id: String(id) });
599
599
  }
600
600
  async deleteNamespace(namespace) {
601
601
  await this.client.index(this.indexName).namespace(namespace).deleteAll();
@@ -673,8 +673,37 @@ var init_PostgreSQLProvider = __esm({
673
673
  );
674
674
  }
675
675
  async batchUpsert(docs, namespace = "") {
676
- for (const doc of docs) {
677
- await this.upsert(doc, namespace);
676
+ if (docs.length === 0) return;
677
+ const client = await this.pool.connect();
678
+ try {
679
+ await client.query("BEGIN");
680
+ const BATCH_SIZE = 50;
681
+ for (let i = 0; i < docs.length; i += BATCH_SIZE) {
682
+ const batch = docs.slice(i, i + BATCH_SIZE);
683
+ const values = [];
684
+ const valuePlaceholders = batch.map((doc, idx) => {
685
+ var _a;
686
+ const offset = idx * 5;
687
+ values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
688
+ return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
689
+ }).join(", ");
690
+ const query = `
691
+ INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
692
+ VALUES ${valuePlaceholders}
693
+ ON CONFLICT (id) DO UPDATE
694
+ SET namespace = EXCLUDED.namespace,
695
+ content = EXCLUDED.content,
696
+ metadata = EXCLUDED.metadata,
697
+ embedding = EXCLUDED.embedding
698
+ `;
699
+ await client.query(query, values);
700
+ }
701
+ await client.query("COMMIT");
702
+ } catch (error) {
703
+ await client.query("ROLLBACK");
704
+ throw error;
705
+ } finally {
706
+ client.release();
678
707
  }
679
708
  }
680
709
  async query(vector, topK, namespace, filter) {
@@ -947,6 +976,33 @@ var init_QdrantProvider = __esm({
947
976
  }
948
977
  async initialize() {
949
978
  await this.ping();
979
+ await this.ensureIndex();
980
+ }
981
+ /**
982
+ * Ensures that the 'namespace' field has a keyword index for efficient filtering.
983
+ * Qdrant requires this for search filters to work in many configurations.
984
+ */
985
+ async ensureIndex() {
986
+ var _a, _b;
987
+ try {
988
+ await this.http.put(`/collections/${this.indexName}/index`, {
989
+ field_name: "namespace",
990
+ field_schema: "keyword"
991
+ });
992
+ console.log(`[QdrantProvider] \u2705 Ensured keyword index for "namespace" on collection "${this.indexName}"`);
993
+ } catch (err) {
994
+ let status;
995
+ let data;
996
+ if (import_axios4.default.isAxiosError(err)) {
997
+ status = (_a = err.response) == null ? void 0 : _a.status;
998
+ data = (_b = err.response) == null ? void 0 : _b.data;
999
+ }
1000
+ if (status === 409) {
1001
+ return;
1002
+ }
1003
+ const errorMessage = err instanceof Error ? err.message : String(err);
1004
+ console.warn(`[QdrantProvider] \u26A0\uFE0F Could not ensure namespace index (Status: ${status}):`, JSON.stringify(data || errorMessage));
1005
+ }
950
1006
  }
951
1007
  async upsert(doc, namespace) {
952
1008
  await this.batchUpsert([doc], namespace);
@@ -991,7 +1047,6 @@ var init_QdrantProvider = __esm({
991
1047
  points: [id]
992
1048
  });
993
1049
  }
994
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
995
1050
  async deleteNamespace(_namespace) {
996
1051
  await this.http.post(`/collections/${this.indexName}/points/delete`, {
997
1052
  filter: {
@@ -1353,7 +1408,7 @@ function readEnum(env, name, fallback, allowed) {
1353
1408
  throw new Error(`[getRagConfig] ${name} must be one of: ${allowed.join(", ")}`);
1354
1409
  }
1355
1410
  function getRagConfig(env = process.env) {
1356
- var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J;
1411
+ var _a, _b, _c, _d, _e, _f, _g, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B, _C, _D, _E, _F, _G, _H, _I, _J, _K;
1357
1412
  const projectId = (_b = (_a = readString(env, "RAG_PROJECT_ID")) != null ? _a : readString(env, "NEXT_PUBLIC_PROJECT_ID")) != null ? _b : "__default__";
1358
1413
  const vectorProvider = readEnum(env, "VECTOR_DB_PROVIDER", "pinecone", VECTOR_DB_PROVIDERS);
1359
1414
  const llmProvider = readEnum(env, "LLM_PROVIDER", "openai", LLM_PROVIDERS);
@@ -1378,6 +1433,9 @@ function getRagConfig(env = process.env) {
1378
1433
  vectorDbOptions.database = (_j = readString(env, "MONGODB_DB")) != null ? _j : "";
1379
1434
  vectorDbOptions.collection = (_k = readString(env, "MONGODB_COLLECTION")) != null ? _k : "";
1380
1435
  vectorDbOptions.indexName = (_l = readString(env, "MONGODB_INDEX_NAME")) != null ? _l : "vector_index";
1436
+ } else if (vectorProvider === "qdrant") {
1437
+ vectorDbOptions.baseUrl = (_m = readString(env, "QDRANT_URL")) != null ? _m : "http://localhost:6333";
1438
+ vectorDbOptions.apiKey = readString(env, "QDRANT_API_KEY");
1381
1439
  }
1382
1440
  const llmApiKeyByProvider = {
1383
1441
  openai: readString(env, "OPENAI_API_KEY"),
@@ -1388,19 +1446,19 @@ function getRagConfig(env = process.env) {
1388
1446
  const embeddingApiKeyByProvider = {
1389
1447
  openai: readString(env, "OPENAI_API_KEY"),
1390
1448
  ollama: void 0,
1391
- custom: (_m = readString(env, "EMBEDDING_API_KEY")) != null ? _m : readString(env, "OPENAI_API_KEY")
1449
+ custom: (_n = readString(env, "EMBEDDING_API_KEY")) != null ? _n : readString(env, "OPENAI_API_KEY")
1392
1450
  };
1393
1451
  return {
1394
1452
  projectId,
1395
1453
  vectorDb: {
1396
1454
  provider: vectorProvider,
1397
- indexName: (_n = readString(env, "VECTOR_DB_INDEX")) != null ? _n : "rag-index",
1455
+ indexName: (_o = readString(env, "VECTOR_DB_INDEX")) != null ? _o : "rag-index",
1398
1456
  options: vectorDbOptions
1399
1457
  },
1400
1458
  llm: {
1401
1459
  provider: llmProvider,
1402
- model: (_o = readString(env, "LLM_MODEL")) != null ? _o : "gpt-4o",
1403
- apiKey: (_p = llmApiKeyByProvider[llmProvider]) != null ? _p : "",
1460
+ model: (_p = readString(env, "LLM_MODEL")) != null ? _p : "gpt-4o",
1461
+ apiKey: (_q = llmApiKeyByProvider[llmProvider]) != null ? _q : "",
1404
1462
  baseUrl: readString(env, "LLM_BASE_URL"),
1405
1463
  systemPrompt: readString(env, "LLM_SYSTEM_PROMPT"),
1406
1464
  maxTokens: readNumber(env, "LLM_MAX_TOKENS", 1024),
@@ -1411,7 +1469,7 @@ function getRagConfig(env = process.env) {
1411
1469
  },
1412
1470
  embedding: {
1413
1471
  provider: embeddingProvider,
1414
- model: (_q = readString(env, "EMBEDDING_MODEL")) != null ? _q : "text-embedding-3-small",
1472
+ model: (_r = readString(env, "EMBEDDING_MODEL")) != null ? _r : "text-embedding-3-small",
1415
1473
  apiKey: embeddingApiKeyByProvider[embeddingProvider],
1416
1474
  baseUrl: readString(env, "EMBEDDING_BASE_URL"),
1417
1475
  dimensions: embeddingDimensions,
@@ -1420,16 +1478,16 @@ function getRagConfig(env = process.env) {
1420
1478
  }
1421
1479
  },
1422
1480
  ui: {
1423
- title: (_s = (_r = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _r : readString(env, "UI_TITLE")) != null ? _s : "AI Assistant",
1424
- subtitle: (_u = (_t = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _t : readString(env, "UI_SUBTITLE")) != null ? _u : "Powered by RAG",
1425
- primaryColor: (_w = (_v = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _v : readString(env, "UI_PRIMARY_COLOR")) != null ? _w : "#10b981",
1426
- accentColor: (_y = (_x = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _x : readString(env, "UI_ACCENT_COLOR")) != null ? _y : "#3b82f6",
1427
- logoUrl: (_z = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _z : readString(env, "UI_LOGO_URL"),
1428
- placeholder: (_B = (_A = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _A : readString(env, "UI_PLACEHOLDER")) != null ? _B : "Ask me anything\u2026",
1429
- showSources: ((_D = (_C = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _C : readString(env, "UI_SHOW_SOURCES")) != null ? _D : "true") !== "false",
1430
- welcomeMessage: (_F = (_E = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _E : readString(env, "UI_WELCOME_MESSAGE")) != null ? _F : "Hello! I'm your AI assistant. Ask me anything about your documents.",
1431
- visualStyle: (_H = (_G = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _G : readString(env, "UI_VISUAL_STYLE")) != null ? _H : "glass",
1432
- borderRadius: (_J = (_I = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _I : readString(env, "UI_BORDER_RADIUS")) != null ? _J : "xl"
1481
+ title: (_t = (_s = readString(env, "NEXT_PUBLIC_UI_TITLE")) != null ? _s : readString(env, "UI_TITLE")) != null ? _t : "AI Assistant",
1482
+ subtitle: (_v = (_u = readString(env, "NEXT_PUBLIC_UI_SUBTITLE")) != null ? _u : readString(env, "UI_SUBTITLE")) != null ? _v : "Powered by RAG",
1483
+ primaryColor: (_x = (_w = readString(env, "NEXT_PUBLIC_PRIMARY_COLOR")) != null ? _w : readString(env, "UI_PRIMARY_COLOR")) != null ? _x : "#10b981",
1484
+ accentColor: (_z = (_y = readString(env, "NEXT_PUBLIC_ACCENT_COLOR")) != null ? _y : readString(env, "UI_ACCENT_COLOR")) != null ? _z : "#3b82f6",
1485
+ logoUrl: (_A = readString(env, "NEXT_PUBLIC_LOGO_URL")) != null ? _A : readString(env, "UI_LOGO_URL"),
1486
+ placeholder: (_C = (_B = readString(env, "NEXT_PUBLIC_PLACEHOLDER")) != null ? _B : readString(env, "UI_PLACEHOLDER")) != null ? _C : "Ask me anything\u2026",
1487
+ showSources: ((_E = (_D = readString(env, "NEXT_PUBLIC_SHOW_SOURCES")) != null ? _D : readString(env, "UI_SHOW_SOURCES")) != null ? _E : "true") !== "false",
1488
+ welcomeMessage: (_G = (_F = readString(env, "NEXT_PUBLIC_WELCOME_MESSAGE")) != null ? _F : readString(env, "UI_WELCOME_MESSAGE")) != null ? _G : "Hello! I'm your AI assistant. Ask me anything about your documents.",
1489
+ visualStyle: (_I = (_H = readString(env, "NEXT_PUBLIC_UI_VISUAL_STYLE")) != null ? _H : readString(env, "UI_VISUAL_STYLE")) != null ? _I : "glass",
1490
+ borderRadius: (_K = (_J = readString(env, "NEXT_PUBLIC_UI_BORDER_RADIUS")) != null ? _J : readString(env, "UI_BORDER_RADIUS")) != null ? _K : "xl"
1433
1491
  },
1434
1492
  rag: {
1435
1493
  topK: readNumber(env, "RAG_TOP_K", 5),
@@ -1482,6 +1540,465 @@ var ConfigResolver = class {
1482
1540
  }
1483
1541
  };
1484
1542
 
1543
+ // src/core/ConfigValidator.ts
1544
+ var ConfigValidator = class {
1545
+ /**
1546
+ * Validates the entire RagConfig object.
1547
+ * Returns an array of validation errors. Empty array = valid config.
1548
+ */
1549
+ static validate(config) {
1550
+ const errors = [];
1551
+ if (!config.projectId) {
1552
+ errors.push({
1553
+ field: "projectId",
1554
+ message: "projectId is required",
1555
+ severity: "error"
1556
+ });
1557
+ }
1558
+ errors.push(...this.validateVectorDbConfig(config.vectorDb));
1559
+ errors.push(...this.validateLLMConfig(config.llm));
1560
+ if (config.embedding) {
1561
+ errors.push(...this.validateEmbeddingConfig(config.embedding));
1562
+ } else if (config.llm.provider === "anthropic") {
1563
+ errors.push({
1564
+ field: "embedding",
1565
+ message: "Embedding config is required when using Anthropic LLM",
1566
+ suggestion: 'Provide embedding config with provider (e.g., "openai")',
1567
+ severity: "error"
1568
+ });
1569
+ }
1570
+ if (config.ui) {
1571
+ errors.push(...this.validateUIConfig(config.ui));
1572
+ }
1573
+ if (config.rag) {
1574
+ errors.push(...this.validateRAGConfig(config.rag));
1575
+ }
1576
+ return errors;
1577
+ }
1578
+ static validateVectorDbConfig(config) {
1579
+ const errors = [];
1580
+ if (!config.provider) {
1581
+ errors.push({
1582
+ field: "vectorDb.provider",
1583
+ message: "Vector database provider is required",
1584
+ severity: "error"
1585
+ });
1586
+ return errors;
1587
+ }
1588
+ if (!config.indexName) {
1589
+ errors.push({
1590
+ field: "vectorDb.indexName",
1591
+ message: "Vector database index name is required",
1592
+ severity: "error"
1593
+ });
1594
+ }
1595
+ switch (config.provider) {
1596
+ case "pinecone":
1597
+ errors.push(...this.validatePineconeConfig(config));
1598
+ break;
1599
+ case "pgvector":
1600
+ case "postgresql":
1601
+ errors.push(...this.validatePostgresConfig(config));
1602
+ break;
1603
+ case "mongodb":
1604
+ errors.push(...this.validateMongoDBConfig(config));
1605
+ break;
1606
+ case "milvus":
1607
+ errors.push(...this.validateMilvusConfig(config));
1608
+ break;
1609
+ case "qdrant":
1610
+ errors.push(...this.validateQdrantConfig(config));
1611
+ break;
1612
+ case "chromadb":
1613
+ errors.push(...this.validateChromaDBConfig(config));
1614
+ break;
1615
+ case "redis":
1616
+ errors.push(...this.validateRedisConfig(config));
1617
+ break;
1618
+ case "weaviate":
1619
+ errors.push(...this.validateWeaviateConfig(config));
1620
+ break;
1621
+ case "universal_rest":
1622
+ case "rest":
1623
+ errors.push(...this.validateRestConfig(config));
1624
+ break;
1625
+ }
1626
+ return errors;
1627
+ }
1628
+ static validatePineconeConfig(config) {
1629
+ const errors = [];
1630
+ const opts = config.options;
1631
+ if (!opts.apiKey) {
1632
+ errors.push({
1633
+ field: "vectorDb.options.apiKey",
1634
+ message: "Pinecone API key is required",
1635
+ suggestion: "Set PINECONE_API_KEY environment variable",
1636
+ severity: "error"
1637
+ });
1638
+ }
1639
+ if (!opts.environment) {
1640
+ errors.push({
1641
+ field: "vectorDb.options.environment",
1642
+ message: "Pinecone environment is required",
1643
+ suggestion: 'Set PINECONE_ENVIRONMENT environment variable (e.g., "gcp-starter")',
1644
+ severity: "error"
1645
+ });
1646
+ }
1647
+ return errors;
1648
+ }
1649
+ static validatePostgresConfig(config) {
1650
+ const errors = [];
1651
+ const opts = config.options;
1652
+ if (!opts.connectionString) {
1653
+ errors.push({
1654
+ field: "vectorDb.options.connectionString",
1655
+ message: "PostgreSQL connection string is required",
1656
+ suggestion: "Set PGVECTOR_CONNECTION_STRING environment variable",
1657
+ severity: "error"
1658
+ });
1659
+ }
1660
+ return errors;
1661
+ }
1662
+ static validateMongoDBConfig(config) {
1663
+ const errors = [];
1664
+ const opts = config.options;
1665
+ if (!opts.uri) {
1666
+ errors.push({
1667
+ field: "vectorDb.options.uri",
1668
+ message: "MongoDB connection URI is required",
1669
+ suggestion: "Set MONGODB_URI environment variable",
1670
+ severity: "error"
1671
+ });
1672
+ }
1673
+ if (!opts.database) {
1674
+ errors.push({
1675
+ field: "vectorDb.options.database",
1676
+ message: "MongoDB database name is required",
1677
+ suggestion: "Set MONGODB_DB environment variable",
1678
+ severity: "error"
1679
+ });
1680
+ }
1681
+ if (!opts.collection) {
1682
+ errors.push({
1683
+ field: "vectorDb.options.collection",
1684
+ message: "MongoDB collection name is required",
1685
+ suggestion: "Set MONGODB_COLLECTION environment variable",
1686
+ severity: "error"
1687
+ });
1688
+ }
1689
+ return errors;
1690
+ }
1691
+ static validateMilvusConfig(config) {
1692
+ const errors = [];
1693
+ const opts = config.options;
1694
+ if (!opts.host) {
1695
+ errors.push({
1696
+ field: "vectorDb.options.host",
1697
+ message: "Milvus host is required",
1698
+ suggestion: "Set MILVUS_HOST environment variable",
1699
+ severity: "error"
1700
+ });
1701
+ }
1702
+ if (!opts.port) {
1703
+ errors.push({
1704
+ field: "vectorDb.options.port",
1705
+ message: "Milvus port is required",
1706
+ suggestion: "Set MILVUS_PORT environment variable (default: 19530)",
1707
+ severity: "error"
1708
+ });
1709
+ }
1710
+ return errors;
1711
+ }
1712
+ static validateQdrantConfig(config) {
1713
+ const errors = [];
1714
+ const opts = config.options;
1715
+ if (!opts.baseUrl && !opts.url) {
1716
+ errors.push({
1717
+ field: "vectorDb.options.baseUrl",
1718
+ message: "Qdrant base URL is required",
1719
+ suggestion: 'Set QDRANT_URL environment variable (e.g., "http://localhost:6333")',
1720
+ severity: "error"
1721
+ });
1722
+ }
1723
+ return errors;
1724
+ }
1725
+ static validateChromaDBConfig(config) {
1726
+ const errors = [];
1727
+ const opts = config.options;
1728
+ if (!opts.host && !opts.path) {
1729
+ errors.push({
1730
+ field: "vectorDb.options.host",
1731
+ message: "ChromaDB host or path is required",
1732
+ suggestion: "Set CHROMADB_HOST or CHROMADB_PATH environment variable",
1733
+ severity: "error"
1734
+ });
1735
+ }
1736
+ return errors;
1737
+ }
1738
+ static validateRedisConfig(config) {
1739
+ const errors = [];
1740
+ const opts = config.options;
1741
+ if (!opts.url && (!opts.host || !opts.port)) {
1742
+ errors.push({
1743
+ field: "vectorDb.options.url",
1744
+ message: "Redis connection URL or host:port is required",
1745
+ suggestion: "Set REDIS_URL or REDIS_HOST/REDIS_PORT environment variables",
1746
+ severity: "error"
1747
+ });
1748
+ }
1749
+ return errors;
1750
+ }
1751
+ static validateWeaviateConfig(config) {
1752
+ const errors = [];
1753
+ const opts = config.options;
1754
+ if (!opts.url) {
1755
+ errors.push({
1756
+ field: "vectorDb.options.url",
1757
+ message: "Weaviate instance URL is required",
1758
+ suggestion: "Set WEAVIATE_URL environment variable",
1759
+ severity: "error"
1760
+ });
1761
+ }
1762
+ return errors;
1763
+ }
1764
+ static validateRestConfig(config) {
1765
+ const errors = [];
1766
+ const opts = config.options;
1767
+ if (!opts.baseUrl) {
1768
+ errors.push({
1769
+ field: "vectorDb.options.baseUrl",
1770
+ message: "REST API base URL is required",
1771
+ suggestion: "Set VECTOR_BASE_URL environment variable",
1772
+ severity: "error"
1773
+ });
1774
+ }
1775
+ return errors;
1776
+ }
1777
+ static validateLLMConfig(config) {
1778
+ var _a;
1779
+ const errors = [];
1780
+ if (!config.provider) {
1781
+ errors.push({
1782
+ field: "llm.provider",
1783
+ message: "LLM provider is required",
1784
+ severity: "error"
1785
+ });
1786
+ return errors;
1787
+ }
1788
+ if (!config.model) {
1789
+ errors.push({
1790
+ field: "llm.model",
1791
+ message: "LLM model name is required",
1792
+ suggestion: `e.g., "gpt-4o" for OpenAI, "claude-3-opus" for Anthropic`,
1793
+ severity: "error"
1794
+ });
1795
+ }
1796
+ switch (config.provider) {
1797
+ case "openai":
1798
+ if (!config.apiKey) {
1799
+ errors.push({
1800
+ field: "llm.apiKey",
1801
+ message: "OpenAI API key is required",
1802
+ suggestion: "Set OPENAI_API_KEY environment variable",
1803
+ severity: "error"
1804
+ });
1805
+ }
1806
+ break;
1807
+ case "anthropic":
1808
+ if (!config.apiKey) {
1809
+ errors.push({
1810
+ field: "llm.apiKey",
1811
+ message: "Anthropic API key is required",
1812
+ suggestion: "Set ANTHROPIC_API_KEY environment variable",
1813
+ severity: "error"
1814
+ });
1815
+ }
1816
+ break;
1817
+ case "ollama":
1818
+ if (!config.baseUrl) {
1819
+ errors.push({
1820
+ field: "llm.baseUrl",
1821
+ message: "Ollama base URL is required",
1822
+ suggestion: 'Set LLM_BASE_URL environment variable (e.g., "http://localhost:11434")',
1823
+ severity: "error"
1824
+ });
1825
+ }
1826
+ break;
1827
+ case "rest":
1828
+ case "universal_rest":
1829
+ if (!config.baseUrl && !((_a = config.options) == null ? void 0 : _a.baseUrl)) {
1830
+ errors.push({
1831
+ field: "llm.baseUrl",
1832
+ message: "REST API base URL is required",
1833
+ suggestion: "Set LLM_BASE_URL environment variable",
1834
+ severity: "error"
1835
+ });
1836
+ }
1837
+ break;
1838
+ }
1839
+ if (config.temperature !== void 0) {
1840
+ if (config.temperature < 0 || config.temperature > 2) {
1841
+ errors.push({
1842
+ field: "llm.temperature",
1843
+ message: "Temperature must be between 0 and 2",
1844
+ severity: "error"
1845
+ });
1846
+ }
1847
+ }
1848
+ if (config.maxTokens !== void 0 && config.maxTokens <= 0) {
1849
+ errors.push({
1850
+ field: "llm.maxTokens",
1851
+ message: "maxTokens must be greater than 0",
1852
+ severity: "error"
1853
+ });
1854
+ }
1855
+ return errors;
1856
+ }
1857
+ static validateEmbeddingConfig(config) {
1858
+ const errors = [];
1859
+ if (!config.provider) {
1860
+ errors.push({
1861
+ field: "embedding.provider",
1862
+ message: "Embedding provider is required",
1863
+ severity: "error"
1864
+ });
1865
+ return errors;
1866
+ }
1867
+ if (!config.model) {
1868
+ errors.push({
1869
+ field: "embedding.model",
1870
+ message: "Embedding model name is required",
1871
+ suggestion: 'e.g., "text-embedding-3-small" for OpenAI',
1872
+ severity: "error"
1873
+ });
1874
+ }
1875
+ if (config.provider === "openai" && !config.apiKey) {
1876
+ errors.push({
1877
+ field: "embedding.apiKey",
1878
+ message: "OpenAI API key is required for embedding",
1879
+ suggestion: "Set OPENAI_API_KEY environment variable",
1880
+ severity: "error"
1881
+ });
1882
+ }
1883
+ if (config.provider === "ollama" && !config.baseUrl) {
1884
+ errors.push({
1885
+ field: "embedding.baseUrl",
1886
+ message: "Ollama base URL is required",
1887
+ suggestion: "Set EMBEDDING_BASE_URL environment variable",
1888
+ severity: "error"
1889
+ });
1890
+ }
1891
+ if (config.dimensions !== void 0 && config.dimensions <= 0) {
1892
+ errors.push({
1893
+ field: "embedding.dimensions",
1894
+ message: "Embedding dimensions must be greater than 0",
1895
+ severity: "error"
1896
+ });
1897
+ }
1898
+ return errors;
1899
+ }
1900
+ static validateUIConfig(config) {
1901
+ const errors = [];
1902
+ if (config.primaryColor) {
1903
+ if (!this.isValidCSSColor(config.primaryColor)) {
1904
+ errors.push({
1905
+ field: "ui.primaryColor",
1906
+ message: "Invalid CSS color format",
1907
+ suggestion: "Use valid CSS colors: hex (#FF0000), rgb, hsl, or named colors",
1908
+ severity: "warning"
1909
+ });
1910
+ }
1911
+ }
1912
+ if (config.borderRadius) {
1913
+ const validValues = ["none", "sm", "md", "lg", "xl", "full"];
1914
+ if (!validValues.includes(config.borderRadius)) {
1915
+ errors.push({
1916
+ field: "ui.borderRadius",
1917
+ message: `borderRadius must be one of: ${validValues.join(", ")}`,
1918
+ severity: "warning"
1919
+ });
1920
+ }
1921
+ }
1922
+ if (config.visualStyle) {
1923
+ const validValues = ["glass", "solid"];
1924
+ if (!validValues.includes(config.visualStyle)) {
1925
+ errors.push({
1926
+ field: "ui.visualStyle",
1927
+ message: `visualStyle must be one of: ${validValues.join(", ")}`,
1928
+ severity: "warning"
1929
+ });
1930
+ }
1931
+ }
1932
+ return errors;
1933
+ }
1934
+ static validateRAGConfig(config) {
1935
+ const errors = [];
1936
+ if (config.topK !== void 0) {
1937
+ if (typeof config.topK !== "number" || config.topK <= 0) {
1938
+ errors.push({
1939
+ field: "rag.topK",
1940
+ message: "topK must be a positive integer",
1941
+ severity: "error"
1942
+ });
1943
+ }
1944
+ }
1945
+ if (config.scoreThreshold !== void 0) {
1946
+ if (typeof config.scoreThreshold !== "number" || config.scoreThreshold < 0 || config.scoreThreshold > 1) {
1947
+ errors.push({
1948
+ field: "rag.scoreThreshold",
1949
+ message: "scoreThreshold must be between 0 and 1",
1950
+ severity: "error"
1951
+ });
1952
+ }
1953
+ }
1954
+ if (config.chunkSize !== void 0) {
1955
+ if (typeof config.chunkSize !== "number" || config.chunkSize <= 0) {
1956
+ errors.push({
1957
+ field: "rag.chunkSize",
1958
+ message: "chunkSize must be a positive integer",
1959
+ severity: "error"
1960
+ });
1961
+ }
1962
+ }
1963
+ if (config.chunkOverlap !== void 0) {
1964
+ if (typeof config.chunkOverlap !== "number" || config.chunkOverlap < 0) {
1965
+ errors.push({
1966
+ field: "rag.chunkOverlap",
1967
+ message: "chunkOverlap must be a non-negative integer",
1968
+ severity: "error"
1969
+ });
1970
+ }
1971
+ }
1972
+ return errors;
1973
+ }
1974
+ static isValidCSSColor(color) {
1975
+ const s = new Option().style;
1976
+ s.color = color;
1977
+ return s.color !== "";
1978
+ }
1979
+ /**
1980
+ * Throws if there are error-level validation issues.
1981
+ * Logs warnings to console.
1982
+ */
1983
+ static validateAndThrow(config) {
1984
+ const errors = this.validate(config);
1985
+ const errorItems = errors.filter((e) => e.severity === "error");
1986
+ const warnings = errors.filter((e) => e.severity === "warning");
1987
+ if (warnings.length > 0) {
1988
+ console.warn("[ConfigValidator] Configuration warnings:");
1989
+ warnings.forEach((w) => {
1990
+ console.warn(` ${w.field}: ${w.message}`);
1991
+ if (w.suggestion) console.warn(` Suggestion: ${w.suggestion}`);
1992
+ });
1993
+ }
1994
+ if (errorItems.length > 0) {
1995
+ const message = errorItems.map((e) => `${e.field}: ${e.message}${e.suggestion ? ` (${e.suggestion})` : ""}`).join("\n ");
1996
+ throw new Error(`[ConfigValidator] Configuration validation failed:
1997
+ ${message}`);
1998
+ }
1999
+ }
2000
+ };
2001
+
1485
2002
  // src/rag/DocumentChunker.ts
1486
2003
  var DocumentChunker = class {
1487
2004
  constructor(chunkSize = 1e3, chunkOverlap = 200) {
@@ -1597,6 +2114,230 @@ var ProviderRegistry = class {
1597
2114
  };
1598
2115
  ProviderRegistry.vectorProviders = {};
1599
2116
 
2117
+ // src/core/BatchProcessor.ts
2118
+ function isTransientError(error) {
2119
+ if (!(error instanceof Error)) return false;
2120
+ const message = error.message.toLowerCase();
2121
+ if (message.includes("econnrefused") || message.includes("econnreset") || message.includes("timeout") || message.includes("network") || message.includes("socket")) {
2122
+ return true;
2123
+ }
2124
+ if (message.includes("429") || message.includes("503") || message.includes("502") || message.includes("rate limit") || message.includes("too many requests")) {
2125
+ return true;
2126
+ }
2127
+ return false;
2128
+ }
2129
+ function calculateBackoffDelay(attempt, initialDelayMs, maxDelayMs, multiplier) {
2130
+ const exponentialDelay = Math.min(
2131
+ initialDelayMs * Math.pow(multiplier, attempt),
2132
+ maxDelayMs
2133
+ );
2134
+ const jitter = exponentialDelay * 0.1 * (Math.random() * 2 - 1);
2135
+ return Math.max(0, exponentialDelay + jitter);
2136
+ }
2137
+ function sleep(ms) {
2138
+ return new Promise((resolve) => setTimeout(resolve, ms));
2139
+ }
2140
+ var BatchProcessor = class {
2141
+ /**
2142
+ * Processes an array of items in configurable batches with retry logic.
2143
+ *
2144
+ * @param items - Items to process
2145
+ * @param processor - Async function that processes a batch of items
2146
+ * @param options - Configuration for batch size, retries, etc.
2147
+ * @returns Object with results, errors, and statistics
2148
+ *
2149
+ * @example
2150
+ * const docs = [...];
2151
+ * const result = await BatchProcessor.processBatch(
2152
+ * docs,
2153
+ * (batch) => vectorDB.batchUpsert(batch),
2154
+ * { batchSize: 100, maxRetries: 3 }
2155
+ * );
2156
+ */
2157
+ static async processBatch(items, processor, options) {
2158
+ const {
2159
+ batchSize = 100,
2160
+ maxRetries = 3,
2161
+ initialDelayMs = 100,
2162
+ maxDelayMs = 1e4,
2163
+ backoffMultiplier = 2,
2164
+ throwOnPartialFailure = false
2165
+ } = options != null ? options : {};
2166
+ const results = [];
2167
+ const errors = [];
2168
+ const batches = [];
2169
+ for (let i = 0; i < items.length; i += batchSize) {
2170
+ batches.push(items.slice(i, i + batchSize));
2171
+ }
2172
+ if (batches.length === 0) {
2173
+ return { results: [], errors: [], totalProcessed: 0, totalFailed: 0 };
2174
+ }
2175
+ for (let batchIndex = 0; batchIndex < batches.length; batchIndex++) {
2176
+ const batch = batches[batchIndex];
2177
+ let success = false;
2178
+ let lastError;
2179
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
2180
+ try {
2181
+ const result = await processor(batch);
2182
+ results.push(result);
2183
+ success = true;
2184
+ break;
2185
+ } catch (err) {
2186
+ lastError = err instanceof Error ? err : new Error(String(err));
2187
+ if (!isTransientError(err) || attempt === maxRetries) {
2188
+ break;
2189
+ }
2190
+ const delay = calculateBackoffDelay(
2191
+ attempt,
2192
+ initialDelayMs,
2193
+ maxDelayMs,
2194
+ backoffMultiplier
2195
+ );
2196
+ await sleep(delay);
2197
+ }
2198
+ }
2199
+ if (!success && lastError) {
2200
+ errors.push({
2201
+ index: batchIndex,
2202
+ error: lastError,
2203
+ itemCount: batch.length
2204
+ });
2205
+ }
2206
+ }
2207
+ const totalProcessed = items.length - (errors.length > 0 ? errors.reduce((sum, e) => sum + (e.itemCount || 0), 0) : 0);
2208
+ const totalFailed = errors.reduce((sum, e) => sum + (e.itemCount || 0), 0);
2209
+ if (throwOnPartialFailure && errors.length > 0) {
2210
+ const errorMessages = errors.map((e) => `Batch ${e.index} (${e.itemCount} items): ${e.error.message}`).join("\n");
2211
+ throw new Error(
2212
+ `[BatchProcessor] Batch processing failed:
2213
+ ${errorMessages}`
2214
+ );
2215
+ }
2216
+ return { results, errors, totalProcessed, totalFailed };
2217
+ }
2218
+ /**
2219
+ * Processes items sequentially (one at a time) with retry logic.
2220
+ * Useful for operations that don't support batching or when granular
2221
+ * error tracking is needed.
2222
+ */
2223
+ static async processSequential(items, processor, options) {
2224
+ const {
2225
+ maxRetries = 3,
2226
+ initialDelayMs = 100,
2227
+ maxDelayMs = 1e4,
2228
+ backoffMultiplier = 2,
2229
+ throwOnPartialFailure = false
2230
+ } = options != null ? options : {};
2231
+ const results = [];
2232
+ const errors = [];
2233
+ for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
2234
+ const item = items[itemIndex];
2235
+ let success = false;
2236
+ let lastError;
2237
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
2238
+ try {
2239
+ const result = await processor(item);
2240
+ results.push(result);
2241
+ success = true;
2242
+ break;
2243
+ } catch (err) {
2244
+ lastError = err instanceof Error ? err : new Error(String(err));
2245
+ if (!isTransientError(err) || attempt === maxRetries) {
2246
+ break;
2247
+ }
2248
+ const delay = calculateBackoffDelay(
2249
+ attempt,
2250
+ initialDelayMs,
2251
+ maxDelayMs,
2252
+ backoffMultiplier
2253
+ );
2254
+ await sleep(delay);
2255
+ }
2256
+ }
2257
+ if (!success && lastError) {
2258
+ errors.push({ index: itemIndex, error: lastError, itemCount: 1 });
2259
+ }
2260
+ }
2261
+ const totalProcessed = items.length - errors.length;
2262
+ const totalFailed = errors.length;
2263
+ if (throwOnPartialFailure && errors.length > 0) {
2264
+ const errorMessages = errors.map((e) => `Item ${e.index}: ${e.error.message}`).join("\n");
2265
+ throw new Error(
2266
+ `[BatchProcessor] Sequential processing failed:
2267
+ ${errorMessages}`
2268
+ );
2269
+ }
2270
+ return { results, errors, totalProcessed, totalFailed };
2271
+ }
2272
+ /**
2273
+ * Maps over items with retry logic, returning results in order.
2274
+ * Like Array.map() but with async processing and automatic retries.
2275
+ */
2276
+ static async mapWithRetry(items, mapper, options) {
2277
+ const result = await this.processSequential(items, mapper, options);
2278
+ if (result.errors.length > 0) {
2279
+ console.warn(
2280
+ `[BatchProcessor] mapWithRetry: ${result.errors.length} items failed`
2281
+ );
2282
+ }
2283
+ const orderedResults = new Array(items.length);
2284
+ let resultIndex = 0;
2285
+ let errorIndex = 0;
2286
+ for (let i = 0; i < items.length; i++) {
2287
+ if (errorIndex < result.errors.length && result.errors[errorIndex].index === i) {
2288
+ orderedResults[i] = void 0;
2289
+ errorIndex++;
2290
+ } else {
2291
+ orderedResults[i] = result.results[resultIndex];
2292
+ resultIndex++;
2293
+ }
2294
+ }
2295
+ return orderedResults.filter((r) => r !== void 0);
2296
+ }
2297
+ /**
2298
+ * Parallel processor with concurrency limit.
2299
+ * Processes up to `concurrency` items at the same time.
2300
+ */
2301
+ static async processConcurrent(items, processor, concurrency = 5, options) {
2302
+ const { throwOnPartialFailure = false } = options != null ? options : {};
2303
+ const results = [];
2304
+ const errors = [];
2305
+ for (let i = 0; i < items.length; i += concurrency) {
2306
+ const chunk = items.slice(i, i + concurrency).map((item, idx) => ({
2307
+ item,
2308
+ originalIndex: i + idx
2309
+ }));
2310
+ const chunkPromises = chunk.map(async ({ item, originalIndex }) => {
2311
+ try {
2312
+ const result = await processor(item);
2313
+ return { success: true, result, index: originalIndex };
2314
+ } catch (err) {
2315
+ const error = err instanceof Error ? err : new Error(String(err));
2316
+ return { success: false, error, index: originalIndex };
2317
+ }
2318
+ });
2319
+ const chunkResults = await Promise.all(chunkPromises);
2320
+ for (const { success, result, error, index } of chunkResults) {
2321
+ if (success) {
2322
+ results.push(result);
2323
+ } else {
2324
+ errors.push({ index, error, itemCount: 1 });
2325
+ }
2326
+ }
2327
+ }
2328
+ const totalProcessed = items.length - errors.length;
2329
+ const totalFailed = errors.length;
2330
+ if (throwOnPartialFailure && errors.length > 0) {
2331
+ const errorMessages = errors.map((e) => `Item ${e.index}: ${e.error.message}`).join("\n");
2332
+ throw new Error(
2333
+ `[BatchProcessor] Concurrent processing failed:
2334
+ ${errorMessages}`
2335
+ );
2336
+ }
2337
+ return { results, errors, totalProcessed, totalFailed };
2338
+ }
2339
+ };
2340
+
1600
2341
  // src/core/Pipeline.ts
1601
2342
  var Pipeline = class {
1602
2343
  constructor(config) {
@@ -1621,24 +2362,64 @@ var Pipeline = class {
1621
2362
  await this.vectorDB.initialize();
1622
2363
  this.initialised = true;
1623
2364
  }
2365
+ /**
2366
+ * Ingest documents with automatic chunking, embedding, and batch processing.
2367
+ * Handles retries for transient failures.
2368
+ */
1624
2369
  async ingest(documents, namespace) {
1625
2370
  await this.initialize();
1626
2371
  const ns = namespace != null ? namespace : this.config.projectId;
1627
2372
  const results = [];
1628
2373
  for (const doc of documents) {
1629
- const chunks = this.chunker.chunk(doc.content, {
1630
- docId: doc.docId,
1631
- metadata: doc.metadata
1632
- });
1633
- const vectors = await this.embeddingProvider.batchEmbed(chunks.map((c) => c.content));
1634
- const upsertDocs = chunks.map((chunk, i) => ({
1635
- id: chunk.id,
1636
- vector: vectors[i],
1637
- content: chunk.content,
1638
- metadata: chunk.metadata
1639
- }));
1640
- await this.vectorDB.batchUpsert(upsertDocs, ns);
1641
- results.push({ docId: doc.docId, chunksIngested: chunks.length });
2374
+ try {
2375
+ const chunks = this.chunker.chunk(doc.content, {
2376
+ docId: doc.docId,
2377
+ metadata: doc.metadata
2378
+ });
2379
+ const batchOptions = {
2380
+ batchSize: 50,
2381
+ // Embedding batch size
2382
+ maxRetries: 3,
2383
+ initialDelayMs: 100
2384
+ };
2385
+ const vectors = await BatchProcessor.mapWithRetry(
2386
+ chunks.map((c) => c.content),
2387
+ (text) => this.embeddingProvider.embed(text),
2388
+ batchOptions
2389
+ );
2390
+ if (vectors.length !== chunks.length) {
2391
+ throw new Error(`Embedding failed: got ${vectors.length} vectors for ${chunks.length} chunks`);
2392
+ }
2393
+ const upsertDocs = chunks.map((chunk, i) => ({
2394
+ id: chunk.id,
2395
+ vector: vectors[i],
2396
+ content: chunk.content,
2397
+ metadata: chunk.metadata
2398
+ }));
2399
+ const upsertBatchOptions = {
2400
+ batchSize: 100,
2401
+ maxRetries: 3,
2402
+ initialDelayMs: 100
2403
+ };
2404
+ const upsertResult = await BatchProcessor.processBatch(
2405
+ upsertDocs,
2406
+ (batch) => this.vectorDB.batchUpsert(batch, ns),
2407
+ upsertBatchOptions
2408
+ );
2409
+ if (upsertResult.errors.length > 0) {
2410
+ console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed for doc ${doc.docId}`);
2411
+ }
2412
+ results.push({
2413
+ docId: doc.docId,
2414
+ chunksIngested: upsertResult.totalProcessed
2415
+ });
2416
+ } catch (error) {
2417
+ console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
2418
+ results.push({
2419
+ docId: doc.docId,
2420
+ chunksIngested: 0
2421
+ });
2422
+ }
1642
2423
  }
1643
2424
  return results;
1644
2425
  }
@@ -1648,22 +2429,483 @@ var Pipeline = class {
1648
2429
  const ns = namespace != null ? namespace : this.config.projectId;
1649
2430
  const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
1650
2431
  const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
1651
- const queryVector = await this.embeddingProvider.embed(question);
1652
- const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
1653
- const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
1654
- const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
2432
+ try {
2433
+ const queryVector = await this.embeddingProvider.embed(question);
2434
+ const rawMatches = await this.vectorDB.query(queryVector, topK, ns);
2435
+ const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
2436
+ const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
1655
2437
  ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
1656
- const messages = [...history, { role: "user", content: question }];
1657
- const reply = await this.llmProvider.chat(messages, context);
1658
- return { reply, sources };
2438
+ const messages = [...history, { role: "user", content: question }];
2439
+ const reply = await this.llmProvider.chat(messages, context);
2440
+ return { reply, sources };
2441
+ } catch (error) {
2442
+ throw new Error(`[Pipeline] Chat failed: ${error instanceof Error ? error.message : String(error)}`);
2443
+ }
1659
2444
  }
1660
- async health() {
1661
- await this.initialize();
1662
- const [vectorDB, llm] = await Promise.all([
1663
- this.vectorDB.ping().catch(() => false),
1664
- this.llmProvider.ping().catch(() => false)
2445
+ };
2446
+
2447
+ // src/core/ProviderHealthCheck.ts
2448
+ var ProviderHealthCheck = class {
2449
+ /**
2450
+ * Validates vector database configuration before initialization.
2451
+ * Performs connectivity checks and verifies required resources exist.
2452
+ */
2453
+ static async checkVectorProvider(config) {
2454
+ const timestamp = Date.now();
2455
+ const configErrors = ConfigValidator.validate({
2456
+ projectId: "health-check",
2457
+ vectorDb: config,
2458
+ llm: { provider: "openai", model: "gpt-4o" },
2459
+ // dummy
2460
+ embedding: { provider: "openai", model: "text-embedding-3-small" }
2461
+ });
2462
+ const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
2463
+ if (vectorDbErrors.length > 0) {
2464
+ return {
2465
+ healthy: false,
2466
+ provider: config.provider,
2467
+ error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
2468
+ timestamp
2469
+ };
2470
+ }
2471
+ try {
2472
+ switch (config.provider) {
2473
+ case "pinecone":
2474
+ return await this.checkPinecone(config, timestamp);
2475
+ case "pgvector":
2476
+ case "postgresql":
2477
+ return await this.checkPostgres(config, timestamp);
2478
+ case "mongodb":
2479
+ return await this.checkMongoDB(config, timestamp);
2480
+ case "milvus":
2481
+ return await this.checkMilvus(config, timestamp);
2482
+ case "qdrant":
2483
+ return await this.checkQdrant(config, timestamp);
2484
+ case "chromadb":
2485
+ return await this.checkChromaDB(config, timestamp);
2486
+ case "redis":
2487
+ return await this.checkRedis(config, timestamp);
2488
+ case "weaviate":
2489
+ return await this.checkWeaviate(config, timestamp);
2490
+ case "rest":
2491
+ case "universal_rest":
2492
+ return await this.checkRestAPI(config, timestamp);
2493
+ default:
2494
+ return {
2495
+ healthy: false,
2496
+ provider: config.provider,
2497
+ error: `Unsupported provider: ${config.provider}`,
2498
+ timestamp
2499
+ };
2500
+ }
2501
+ } catch (error) {
2502
+ return {
2503
+ healthy: false,
2504
+ provider: config.provider,
2505
+ error: error instanceof Error ? error.message : String(error),
2506
+ timestamp
2507
+ };
2508
+ }
2509
+ }
2510
+ static async checkPinecone(config, timestamp) {
2511
+ var _a, _b;
2512
+ const opts = config.options;
2513
+ try {
2514
+ const { Pinecone: Pinecone2 } = await import("@pinecone-database/pinecone");
2515
+ const client = new Pinecone2({ apiKey: opts.apiKey });
2516
+ const indexes = await client.listIndexes();
2517
+ const indexNames = (_b = (_a = indexes.indexes) == null ? void 0 : _a.map((i) => i.name)) != null ? _b : [];
2518
+ if (!indexNames.includes(config.indexName)) {
2519
+ return {
2520
+ healthy: false,
2521
+ provider: "pinecone",
2522
+ error: `Index "${config.indexName}" not found. Available: ${indexNames.join(", ")}`,
2523
+ timestamp
2524
+ };
2525
+ }
2526
+ return {
2527
+ healthy: true,
2528
+ provider: "pinecone",
2529
+ capabilities: {
2530
+ indexes: indexNames.length,
2531
+ targetIndex: config.indexName
2532
+ },
2533
+ timestamp
2534
+ };
2535
+ } catch (error) {
2536
+ return {
2537
+ healthy: false,
2538
+ provider: "pinecone",
2539
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2540
+ timestamp
2541
+ };
2542
+ }
2543
+ }
2544
+ static async checkPostgres(config, timestamp) {
2545
+ const opts = config.options;
2546
+ try {
2547
+ const { Client } = await import("pg");
2548
+ const client = new Client({ connectionString: opts.connectionString });
2549
+ await client.connect();
2550
+ const result = await client.query(`
2551
+ SELECT EXISTS(
2552
+ SELECT 1 FROM pg_extension WHERE extname = 'vector'
2553
+ );
2554
+ `);
2555
+ const hasVector = result.rows[0].exists;
2556
+ await client.end();
2557
+ return {
2558
+ healthy: true,
2559
+ provider: "postgresql",
2560
+ capabilities: { pgvectorInstalled: hasVector },
2561
+ timestamp
2562
+ };
2563
+ } catch (error) {
2564
+ return {
2565
+ healthy: false,
2566
+ provider: "postgresql",
2567
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2568
+ timestamp
2569
+ };
2570
+ }
2571
+ }
2572
+ static async checkMongoDB(config, timestamp) {
2573
+ const opts = config.options;
2574
+ try {
2575
+ const { MongoClient: MongoClient2 } = await import("mongodb");
2576
+ const client = new MongoClient2(opts.uri);
2577
+ await client.connect();
2578
+ const db = client.db(opts.database);
2579
+ const collections = await db.listCollections().toArray();
2580
+ const collectionNames = collections.map((c) => c.name);
2581
+ const hasCollection = collectionNames.includes(opts.collection);
2582
+ await client.close();
2583
+ return {
2584
+ healthy: true,
2585
+ provider: "mongodb",
2586
+ capabilities: {
2587
+ collections: collectionNames.length,
2588
+ targetCollection: hasCollection ? opts.collection : "NOT FOUND"
2589
+ },
2590
+ timestamp
2591
+ };
2592
+ } catch (error) {
2593
+ return {
2594
+ healthy: false,
2595
+ provider: "mongodb",
2596
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2597
+ timestamp
2598
+ };
2599
+ }
2600
+ }
2601
+ static async checkMilvus(config, timestamp) {
2602
+ const opts = config.options;
2603
+ const host = opts.host || "localhost";
2604
+ const port = opts.port || 19530;
2605
+ try {
2606
+ const controller = new AbortController();
2607
+ const timeoutId = setTimeout(() => controller.abort(), 5e3);
2608
+ const response = await fetch(`http://${host}:${port}/healthz`, {
2609
+ signal: controller.signal
2610
+ });
2611
+ clearTimeout(timeoutId);
2612
+ if (!response.ok) {
2613
+ throw new Error(`Health check returned ${response.status}`);
2614
+ }
2615
+ return {
2616
+ healthy: true,
2617
+ provider: "milvus",
2618
+ capabilities: { endpoint: `${host}:${port}` },
2619
+ timestamp
2620
+ };
2621
+ } catch (error) {
2622
+ return {
2623
+ healthy: false,
2624
+ provider: "milvus",
2625
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2626
+ timestamp
2627
+ };
2628
+ }
2629
+ }
2630
+ static async checkQdrant(config, timestamp) {
2631
+ const opts = config.options;
2632
+ const baseUrl = opts.baseUrl || opts.url || "http://localhost:6333";
2633
+ try {
2634
+ const response = await fetch(`${baseUrl}/health`);
2635
+ if (!response.ok) {
2636
+ throw new Error(`Health check returned ${response.status}`);
2637
+ }
2638
+ const health = await response.json();
2639
+ return {
2640
+ healthy: true,
2641
+ provider: "qdrant",
2642
+ capabilities: health,
2643
+ timestamp
2644
+ };
2645
+ } catch (error) {
2646
+ return {
2647
+ healthy: false,
2648
+ provider: "qdrant",
2649
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2650
+ timestamp
2651
+ };
2652
+ }
2653
+ }
2654
+ static async checkChromaDB(config, timestamp) {
2655
+ const opts = config.options;
2656
+ const host = opts.host || "localhost";
2657
+ const port = opts.port || 8e3;
2658
+ try {
2659
+ const response = await fetch(`http://${host}:${port}/api/v1/heartbeat`);
2660
+ if (!response.ok) {
2661
+ throw new Error(`Health check returned ${response.status}`);
2662
+ }
2663
+ return {
2664
+ healthy: true,
2665
+ provider: "chromadb",
2666
+ capabilities: { endpoint: `${host}:${port}` },
2667
+ timestamp
2668
+ };
2669
+ } catch (error) {
2670
+ return {
2671
+ healthy: false,
2672
+ provider: "chromadb",
2673
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2674
+ timestamp
2675
+ };
2676
+ }
2677
+ }
2678
+ static async checkRedis(config, timestamp) {
2679
+ const opts = config.options;
2680
+ const url = opts.url || `redis://${opts.host}:${opts.port}`;
2681
+ try {
2682
+ await fetch(url);
2683
+ return {
2684
+ healthy: true,
2685
+ provider: "redis",
2686
+ capabilities: { endpoint: url },
2687
+ timestamp
2688
+ };
2689
+ } catch (error) {
2690
+ return {
2691
+ healthy: false,
2692
+ provider: "redis",
2693
+ error: `Connection check failed: ${error instanceof Error ? error.message : String(error)}. Ensure Redis is running at ${url}`,
2694
+ timestamp
2695
+ };
2696
+ }
2697
+ }
2698
+ static async checkWeaviate(config, timestamp) {
2699
+ const opts = config.options;
2700
+ const url = (opts.url || "http://localhost:8080").replace(/\/$/, "");
2701
+ try {
2702
+ const response = await fetch(`${url}/.well-known/ready`);
2703
+ if (!response.ok) {
2704
+ throw new Error(`Health check returned ${response.status}`);
2705
+ }
2706
+ return {
2707
+ healthy: true,
2708
+ provider: "weaviate",
2709
+ capabilities: { endpoint: url },
2710
+ timestamp
2711
+ };
2712
+ } catch (error) {
2713
+ return {
2714
+ healthy: false,
2715
+ provider: "weaviate",
2716
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2717
+ timestamp
2718
+ };
2719
+ }
2720
+ }
2721
+ static async checkRestAPI(config, timestamp) {
2722
+ const opts = config.options;
2723
+ const baseUrl = (opts.baseUrl || "").replace(/\/$/, "");
2724
+ if (!baseUrl) {
2725
+ return {
2726
+ healthy: false,
2727
+ provider: "rest",
2728
+ error: "baseUrl is required",
2729
+ timestamp
2730
+ };
2731
+ }
2732
+ try {
2733
+ const response = await fetch(`${baseUrl}/health`, {
2734
+ headers: opts.headers ? JSON.parse(opts.headers) : {}
2735
+ });
2736
+ return {
2737
+ healthy: response.ok,
2738
+ provider: "rest",
2739
+ error: response.ok ? void 0 : `Health check returned ${response.status}`,
2740
+ timestamp
2741
+ };
2742
+ } catch (error) {
2743
+ return {
2744
+ healthy: false,
2745
+ provider: "rest",
2746
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2747
+ timestamp
2748
+ };
2749
+ }
2750
+ }
2751
+ /**
2752
+ * Validates LLM provider configuration.
2753
+ */
2754
+ static async checkLLMProvider(config) {
2755
+ const timestamp = Date.now();
2756
+ try {
2757
+ switch (config.provider) {
2758
+ case "openai":
2759
+ return await this.checkOpenAI(config, timestamp);
2760
+ case "anthropic":
2761
+ return await this.checkAnthropic(config, timestamp);
2762
+ case "ollama":
2763
+ return await this.checkOllama(config, timestamp);
2764
+ case "rest":
2765
+ case "universal_rest":
2766
+ return await this.checkLLMRestAPI(config, timestamp);
2767
+ default:
2768
+ return {
2769
+ healthy: false,
2770
+ provider: config.provider,
2771
+ error: `Unsupported provider: ${config.provider}`,
2772
+ timestamp
2773
+ };
2774
+ }
2775
+ } catch (error) {
2776
+ return {
2777
+ healthy: false,
2778
+ provider: config.provider,
2779
+ error: error instanceof Error ? error.message : String(error),
2780
+ timestamp
2781
+ };
2782
+ }
2783
+ }
2784
+ static async checkOpenAI(config, timestamp) {
2785
+ try {
2786
+ const OpenAI2 = await import("openai");
2787
+ const client = new OpenAI2.default({ apiKey: config.apiKey });
2788
+ const models = await client.models.list();
2789
+ const hasModel = models.data.some((m) => m.id === config.model);
2790
+ return {
2791
+ healthy: true,
2792
+ provider: "openai",
2793
+ capabilities: {
2794
+ model: config.model,
2795
+ available: hasModel,
2796
+ totalModels: models.data.length
2797
+ },
2798
+ timestamp
2799
+ };
2800
+ } catch (error) {
2801
+ return {
2802
+ healthy: false,
2803
+ provider: "openai",
2804
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2805
+ timestamp
2806
+ };
2807
+ }
2808
+ }
2809
+ static async checkAnthropic(config, timestamp) {
2810
+ try {
2811
+ const { default: Anthropic2 } = await import("@anthropic-ai/sdk");
2812
+ const client = new Anthropic2({ apiKey: config.apiKey });
2813
+ await client.messages.create({
2814
+ model: config.model,
2815
+ max_tokens: 10,
2816
+ messages: [{ role: "user", content: "ping" }]
2817
+ });
2818
+ return {
2819
+ healthy: true,
2820
+ provider: "anthropic",
2821
+ capabilities: { model: config.model },
2822
+ timestamp
2823
+ };
2824
+ } catch (error) {
2825
+ return {
2826
+ healthy: false,
2827
+ provider: "anthropic",
2828
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2829
+ timestamp
2830
+ };
2831
+ }
2832
+ }
2833
+ static async checkOllama(config, timestamp) {
2834
+ const baseUrl = (config.baseUrl || "http://localhost:11434").replace(/\/$/, "");
2835
+ try {
2836
+ const response = await fetch(`${baseUrl}/api/tags`);
2837
+ if (!response.ok) {
2838
+ throw new Error(`Health check returned ${response.status}`);
2839
+ }
2840
+ const data = await response.json();
2841
+ const models = data.models || [];
2842
+ const hasModel = models.some((m) => m.name === config.model);
2843
+ return {
2844
+ healthy: true,
2845
+ provider: "ollama",
2846
+ capabilities: {
2847
+ model: config.model,
2848
+ available: hasModel,
2849
+ totalModels: models.length
2850
+ },
2851
+ timestamp
2852
+ };
2853
+ } catch (error) {
2854
+ return {
2855
+ healthy: false,
2856
+ provider: "ollama",
2857
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2858
+ timestamp
2859
+ };
2860
+ }
2861
+ }
2862
+ static async checkLLMRestAPI(config, timestamp) {
2863
+ var _a, _b;
2864
+ const baseUrlRaw = config.baseUrl || ((_a = config.options) == null ? void 0 : _a.baseUrl);
2865
+ const baseUrl = (typeof baseUrlRaw === "string" ? baseUrlRaw : "").replace(/\/$/, "");
2866
+ if (!baseUrl) {
2867
+ return {
2868
+ healthy: false,
2869
+ provider: config.provider,
2870
+ error: "baseUrl is required",
2871
+ timestamp
2872
+ };
2873
+ }
2874
+ try {
2875
+ const headers = (_b = config.options) == null ? void 0 : _b.headers;
2876
+ const response = await fetch(`${baseUrl}/health`, {
2877
+ headers: headers || void 0
2878
+ });
2879
+ return {
2880
+ healthy: response.ok,
2881
+ provider: config.provider,
2882
+ error: response.ok ? void 0 : `Health check returned ${response.status}`,
2883
+ timestamp
2884
+ };
2885
+ } catch (error) {
2886
+ return {
2887
+ healthy: false,
2888
+ provider: config.provider,
2889
+ error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
2890
+ timestamp
2891
+ };
2892
+ }
2893
+ }
2894
+ /**
2895
+ * Runs comprehensive health checks on all configured providers.
2896
+ */
2897
+ static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
2898
+ const [vectorDb, llm, embedding] = await Promise.all([
2899
+ this.checkVectorProvider(vectorDbConfig),
2900
+ this.checkLLMProvider(llmConfig),
2901
+ embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
1665
2902
  ]);
1666
- return { vectorDB, llm };
2903
+ return {
2904
+ vectorDb,
2905
+ llm,
2906
+ embedding,
2907
+ allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
2908
+ };
1667
2909
  }
1668
2910
  };
1669
2911
 
@@ -1672,10 +2914,26 @@ var VectorPlugin = class {
1672
2914
  /**
1673
2915
  * Initializes the plugin with the host configuration.
1674
2916
  * @param hostConfig - Configuration object passed from the host application.
2917
+ * @throws Error if configuration is invalid or providers are unhealthy
2918
+ *
2919
+ * @example
2920
+ * const plugin = new VectorPlugin({
2921
+ * projectId: 'my-app',
2922
+ * vectorDb: {
2923
+ * provider: 'pinecone',
2924
+ * indexName: 'my-index',
2925
+ * options: { apiKey: process.env.PINECONE_API_KEY }
2926
+ * },
2927
+ * llm: {
2928
+ * provider: 'openai',
2929
+ * model: 'gpt-4o',
2930
+ * apiKey: process.env.OPENAI_API_KEY
2931
+ * }
2932
+ * });
1675
2933
  */
1676
2934
  constructor(hostConfig) {
1677
2935
  this.config = ConfigResolver.resolve(hostConfig);
1678
- ConfigResolver.validate(this.config);
2936
+ ConfigValidator.validateAndThrow(this.config);
1679
2937
  this.pipeline = new Pipeline(this.config);
1680
2938
  }
1681
2939
  /**
@@ -1684,6 +2942,19 @@ var VectorPlugin = class {
1684
2942
  getConfig() {
1685
2943
  return this.config;
1686
2944
  }
2945
+ /**
2946
+ * Perform pre-flight health checks on all configured providers.
2947
+ * Useful to verify connectivity before running operations.
2948
+ *
2949
+ * @returns Health status for vector DB, LLM, and embedding providers
2950
+ */
2951
+ async checkHealth() {
2952
+ return ProviderHealthCheck.checkAll(
2953
+ this.config.vectorDb,
2954
+ this.config.llm,
2955
+ this.config.embedding
2956
+ );
2957
+ }
1687
2958
  /**
1688
2959
  * Run a chat query.
1689
2960
  */
@@ -1696,12 +2967,6 @@ var VectorPlugin = class {
1696
2967
  async ingest(documents, namespace) {
1697
2968
  return this.pipeline.ingest(documents, namespace);
1698
2969
  }
1699
- /**
1700
- * Check the health of the connected providers.
1701
- */
1702
- async health() {
1703
- return this.pipeline.health();
1704
- }
1705
2970
  };
1706
2971
 
1707
2972
  // src/server.ts
@@ -1820,8 +3085,8 @@ function createHealthHandler(config) {
1820
3085
  const plugin = new VectorPlugin(config);
1821
3086
  return async function GET() {
1822
3087
  try {
1823
- const health = await plugin.health();
1824
- const status = health.vectorDB && health.llm ? "ok" : "degraded";
3088
+ const health = await plugin.checkHealth();
3089
+ const status = health.allHealthy ? "ok" : "degraded";
1825
3090
  return import_server.NextResponse.json(__spreadProps(__spreadValues({
1826
3091
  status
1827
3092
  }, health), {