@lov3kaizen/agentsea-embeddings 1.0.1 → 1.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1500,20 +1500,33 @@ function createVoyageProvider(config) {
1500
1500
  return new VoyageProvider(config);
1501
1501
  }
1502
1502
 
1503
+ // src/core/optional-import.ts
1504
+ function importOptional(name) {
1505
+ return import(
1506
+ /* @vite-ignore */
1507
+ /* webpackIgnore: true */
1508
+ name
1509
+ );
1510
+ }
1511
+
1503
1512
  // src/providers/LocalProvider.ts
1504
1513
  var LocalProvider = class extends BaseProvider {
1505
1514
  modelInfo;
1506
1515
  embedFn = null;
1507
1516
  normalize;
1508
1517
  batchSize;
1518
+ modelPath;
1519
+ /** Lazily-built ONNX extractor (only when loading from `modelPath`). */
1520
+ extractorPromise;
1509
1521
  constructor(config) {
1510
1522
  super({ ...config, type: "local" });
1511
1523
  if (!config.embedFn && !config.modelPath) {
1512
1524
  throw new Error(
1513
- "Either embedFn or modelPath is required for local provider"
1525
+ "`embedFn` or `modelPath` (ONNX) is required for the local provider"
1514
1526
  );
1515
1527
  }
1516
1528
  this.embedFn = config.embedFn ?? null;
1529
+ this.modelPath = config.modelPath;
1517
1530
  this.normalize = config.normalize ?? true;
1518
1531
  this.batchSize = config.batchSize ?? 32;
1519
1532
  this.modelInfo = {
@@ -1527,10 +1540,43 @@ var LocalProvider = class extends BaseProvider {
1527
1540
  description: "Local embedding model"
1528
1541
  };
1529
1542
  }
1543
+ /**
1544
+ * Lazily load the ONNX feature-extraction pipeline from `modelPath` via
1545
+ * Transformers.js and adapt it into a {@link LocalEmbeddingFn}.
1546
+ */
1547
+ getOnnxEmbedFn() {
1548
+ if (!this.extractorPromise) {
1549
+ this.extractorPromise = (async () => {
1550
+ let mod;
1551
+ try {
1552
+ mod = await importOptional("@xenova/transformers");
1553
+ } catch {
1554
+ throw new Error(
1555
+ 'Loading local ONNX models from `modelPath` requires the "@xenova/transformers" package. Install it, or pass an `embedFn`.'
1556
+ );
1557
+ }
1558
+ const transformers = mod;
1559
+ return transformers.pipeline("feature-extraction", this.modelPath);
1560
+ })();
1561
+ }
1562
+ return this.extractorPromise;
1563
+ }
1530
1564
  get info() {
1531
1565
  return this.modelInfo;
1532
1566
  }
1533
1567
  async doEmbed(texts, options) {
1568
+ if (!this.embedFn && this.modelPath) {
1569
+ const extractor = await this.getOnnxEmbedFn();
1570
+ this.embedFn = async (input) => {
1571
+ const output = await extractor(input, {
1572
+ pooling: "mean",
1573
+ normalize: false
1574
+ // we normalize below if configured
1575
+ });
1576
+ const list = output.tolist();
1577
+ return Array.isArray(list[0]) ? list : [list];
1578
+ };
1579
+ }
1534
1580
  if (!this.embedFn) {
1535
1581
  throw new Error("No embedding function configured");
1536
1582
  }
@@ -1768,8 +1814,37 @@ function createHuggingFaceProvider(config) {
1768
1814
  return new HuggingFaceProvider(config);
1769
1815
  }
1770
1816
 
1817
+ // ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/index.js
1818
+ var import_node_crypto = require("crypto");
1819
+
1820
+ // ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/url-alphabet/index.js
1821
+ var urlAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
1822
+
1823
+ // ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/index.js
1824
+ var POOL_SIZE_MULTIPLIER = 128;
1825
+ var pool;
1826
+ var poolOffset;
1827
+ function fillPool(bytes) {
1828
+ if (!pool || pool.length < bytes) {
1829
+ pool = Buffer.allocUnsafe(bytes * POOL_SIZE_MULTIPLIER);
1830
+ import_node_crypto.webcrypto.getRandomValues(pool);
1831
+ poolOffset = 0;
1832
+ } else if (poolOffset + bytes > pool.length) {
1833
+ import_node_crypto.webcrypto.getRandomValues(pool);
1834
+ poolOffset = 0;
1835
+ }
1836
+ poolOffset += bytes;
1837
+ }
1838
+ function nanoid(size = 21) {
1839
+ fillPool(size |= 0);
1840
+ let id = "";
1841
+ for (let i = poolOffset - size; i < poolOffset; i++) {
1842
+ id += urlAlphabet[pool[i] & 63];
1843
+ }
1844
+ return id;
1845
+ }
1846
+
1771
1847
  // src/chunking/BaseChunker.ts
1772
- var import_nanoid = require("nanoid");
1773
1848
  var defaultTokenCounter = (text) => {
1774
1849
  return Math.ceil(text.length / 4);
1775
1850
  };
@@ -1811,7 +1886,7 @@ var BaseChunker = class {
1811
1886
  if (options.source) metadata.source = options.source;
1812
1887
  if (options.type) metadata.type = options.type;
1813
1888
  return {
1814
- id: (0, import_nanoid.nanoid)(),
1889
+ id: nanoid(),
1815
1890
  text,
1816
1891
  index,
1817
1892
  startPosition,
@@ -1919,7 +1994,7 @@ function splitLargeChunks(chunks, maxTokens, tokenCounter) {
1919
1994
  if (testTokens > maxTokens && currentText) {
1920
1995
  result.push({
1921
1996
  ...chunk2,
1922
- id: (0, import_nanoid.nanoid)(),
1997
+ id: nanoid(),
1923
1998
  text: currentText,
1924
1999
  startPosition: currentStart,
1925
2000
  endPosition: currentStart + currentText.length,
@@ -1935,7 +2010,7 @@ function splitLargeChunks(chunks, maxTokens, tokenCounter) {
1935
2010
  if (currentText) {
1936
2011
  result.push({
1937
2012
  ...chunk2,
1938
- id: (0, import_nanoid.nanoid)(),
2013
+ id: nanoid(),
1939
2014
  text: currentText,
1940
2015
  startPosition: currentStart,
1941
2016
  endPosition: currentStart + currentText.length,
@@ -3433,7 +3508,13 @@ var SQLiteCache = class extends BaseCache {
3433
3508
  `);
3434
3509
  updateStmt.run(Date.now(), key);
3435
3510
  const vectorBuffer = row.vector;
3436
- const vector = Array.from(new Float32Array(vectorBuffer.buffer));
3511
+ const vector = Array.from(
3512
+ new Float32Array(
3513
+ vectorBuffer.buffer,
3514
+ vectorBuffer.byteOffset,
3515
+ vectorBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT
3516
+ )
3517
+ );
3437
3518
  return {
3438
3519
  key: row.key,
3439
3520
  vector,
@@ -3984,6 +4065,8 @@ var MemoryStore = class extends BaseStore {
3984
4065
  }
3985
4066
  return Promise.resolve({
3986
4067
  deletedCount,
4068
+ requestedCount: ids.length,
4069
+ countExact: true,
3987
4070
  durationMs: performance.now() - startTime
3988
4071
  });
3989
4072
  }
@@ -3996,6 +4079,8 @@ var MemoryStore = class extends BaseStore {
3996
4079
  this.namespaces.clear();
3997
4080
  return Promise.resolve({
3998
4081
  deletedCount: count2,
4082
+ requestedCount: count2,
4083
+ countExact: true,
3999
4084
  durationMs: performance.now() - startTime
4000
4085
  });
4001
4086
  }
@@ -4003,6 +4088,8 @@ var MemoryStore = class extends BaseStore {
4003
4088
  if (!nsIds) {
4004
4089
  return Promise.resolve({
4005
4090
  deletedCount: 0,
4091
+ requestedCount: 0,
4092
+ countExact: true,
4006
4093
  durationMs: performance.now() - startTime
4007
4094
  });
4008
4095
  }
@@ -4013,6 +4100,8 @@ var MemoryStore = class extends BaseStore {
4013
4100
  this.namespaces.delete(namespace);
4014
4101
  return Promise.resolve({
4015
4102
  deletedCount: count,
4103
+ requestedCount: count,
4104
+ countExact: true,
4016
4105
  durationMs: performance.now() - startTime
4017
4106
  });
4018
4107
  }
@@ -4207,6 +4296,8 @@ var PineconeStore = class extends BaseStore {
4207
4296
  );
4208
4297
  return {
4209
4298
  deletedCount: ids.length,
4299
+ requestedCount: ids.length,
4300
+ countExact: false,
4210
4301
  durationMs: performance.now() - startTime
4211
4302
  };
4212
4303
  }
@@ -4214,13 +4305,15 @@ var PineconeStore = class extends BaseStore {
4214
4305
  await this.ensureInitialized();
4215
4306
  const startTime = performance.now();
4216
4307
  const namespace = options?.namespace ?? this.namespace;
4308
+ const before = await this.getStats().catch(() => void 0);
4217
4309
  const ns = this.index.namespace(
4218
4310
  namespace
4219
4311
  );
4220
4312
  await ns.deleteAll();
4221
4313
  return {
4222
- deletedCount: -1,
4223
- // Unknown count
4314
+ deletedCount: before?.vectorCount ?? 0,
4315
+ requestedCount: before?.vectorCount,
4316
+ countExact: before !== void 0,
4224
4317
  durationMs: performance.now() - startTime
4225
4318
  };
4226
4319
  }
@@ -4381,12 +4474,15 @@ var ChromaStore = class extends BaseStore {
4381
4474
  await this.collection.delete({ ids });
4382
4475
  return {
4383
4476
  deletedCount: ids.length,
4477
+ requestedCount: ids.length,
4478
+ countExact: false,
4384
4479
  durationMs: performance.now() - startTime
4385
4480
  };
4386
4481
  }
4387
4482
  async deleteAll(_options) {
4388
4483
  await this.ensureInitialized();
4389
4484
  const startTime = performance.now();
4485
+ const before = await this.getStats().catch(() => void 0);
4390
4486
  await this.client.deleteCollection({ name: this.collectionName });
4391
4487
  this.collection = await this.client.createCollection({
4392
4488
  name: this.collectionName,
@@ -4395,7 +4491,9 @@ var ChromaStore = class extends BaseStore {
4395
4491
  }
4396
4492
  });
4397
4493
  return {
4398
- deletedCount: -1,
4494
+ deletedCount: before?.vectorCount ?? 0,
4495
+ requestedCount: before?.vectorCount,
4496
+ countExact: before !== void 0,
4399
4497
  durationMs: performance.now() - startTime
4400
4498
  };
4401
4499
  }
@@ -4585,12 +4683,15 @@ var QdrantStore = class extends BaseStore {
4585
4683
  });
4586
4684
  return {
4587
4685
  deletedCount: ids.length,
4686
+ requestedCount: ids.length,
4687
+ countExact: false,
4588
4688
  durationMs: performance.now() - startTime
4589
4689
  };
4590
4690
  }
4591
4691
  async deleteAll(_options) {
4592
4692
  await this.ensureInitialized();
4593
4693
  const startTime = performance.now();
4694
+ const before = await this.getStats().catch(() => void 0);
4594
4695
  await this.client.deleteCollection(this.collectionName);
4595
4696
  if (this.dimensions) {
4596
4697
  await this.client.createCollection(this.collectionName, {
@@ -4601,7 +4702,9 @@ var QdrantStore = class extends BaseStore {
4601
4702
  });
4602
4703
  }
4603
4704
  return {
4604
- deletedCount: -1,
4705
+ deletedCount: before?.vectorCount ?? 0,
4706
+ requestedCount: before?.vectorCount,
4707
+ countExact: before !== void 0,
4605
4708
  durationMs: performance.now() - startTime
4606
4709
  };
4607
4710
  }
@@ -4645,6 +4748,729 @@ function createQdrantStore(config) {
4645
4748
  return new QdrantStore(config);
4646
4749
  }
4647
4750
 
4751
+ // src/stores/PgVectorStore.ts
4752
+ var PgVectorStore = class extends BaseStore {
4753
+ storeType = "pgvector";
4754
+ pool;
4755
+ injectedPool;
4756
+ table;
4757
+ vectorColumn;
4758
+ contentColumn;
4759
+ metadataColumn;
4760
+ pgConfig;
4761
+ initialized = false;
4762
+ constructor(config) {
4763
+ super(config);
4764
+ if (!config.tableName) {
4765
+ throw new Error("pgvector store requires a `tableName`");
4766
+ }
4767
+ this.pgConfig = config;
4768
+ this.injectedPool = config.pool;
4769
+ this.table = config.tableName;
4770
+ this.vectorColumn = config.vectorColumn ?? "embedding";
4771
+ this.contentColumn = config.contentColumn ?? "content";
4772
+ this.metadataColumn = config.metadataColumn ?? "metadata";
4773
+ }
4774
+ /** The pgvector distance operator for the configured metric. */
4775
+ get distanceOperator() {
4776
+ switch (this.metric) {
4777
+ case "euclidean":
4778
+ return "<->";
4779
+ case "dot_product":
4780
+ return "<#>";
4781
+ case "cosine":
4782
+ default:
4783
+ return "<=>";
4784
+ }
4785
+ }
4786
+ async init() {
4787
+ if (this.initialized) return;
4788
+ if (this.injectedPool) {
4789
+ this.pool = this.injectedPool;
4790
+ } else {
4791
+ let mod;
4792
+ try {
4793
+ mod = await importOptional("pg");
4794
+ } catch {
4795
+ throw new Error(
4796
+ 'pgvector store requires the "pg" package. Install it, or pass a pre-built `pool` to the store.'
4797
+ );
4798
+ }
4799
+ const pg = mod.default ?? mod;
4800
+ const Pool = pg.Pool;
4801
+ this.pool = new Pool(
4802
+ this.pgConfig.connectionString ? { connectionString: this.pgConfig.connectionString } : {
4803
+ host: this.pgConfig.host,
4804
+ port: this.pgConfig.port,
4805
+ database: this.pgConfig.database,
4806
+ user: this.pgConfig.user,
4807
+ password: this.pgConfig.password
4808
+ }
4809
+ );
4810
+ }
4811
+ await this.pool.query("CREATE EXTENSION IF NOT EXISTS vector");
4812
+ const dims = this.dimensions;
4813
+ const vectorType = dims ? `vector(${dims})` : "vector";
4814
+ await this.pool.query(
4815
+ `CREATE TABLE IF NOT EXISTS ${this.ident(this.table)} (
4816
+ id text PRIMARY KEY,
4817
+ ${this.ident(this.contentColumn)} text,
4818
+ ${this.ident(this.metadataColumn)} jsonb,
4819
+ ${this.ident(this.vectorColumn)} ${vectorType}
4820
+ )`
4821
+ );
4822
+ this.initialized = true;
4823
+ }
4824
+ async ensureInitialized() {
4825
+ if (!this.initialized) await this.init();
4826
+ }
4827
+ /** Quote an SQL identifier to guard against injection via config names. */
4828
+ ident(name) {
4829
+ return `"${name.replace(/"/g, '""')}"`;
4830
+ }
4831
+ toVectorLiteral(vector) {
4832
+ return `[${Array.from(vector).join(",")}]`;
4833
+ }
4834
+ async upsert(records, options) {
4835
+ await this.ensureInitialized();
4836
+ const start = performance.now();
4837
+ const batchSize = options?.batchSize ?? 100;
4838
+ const upsertedIds = [];
4839
+ const errors = [];
4840
+ let completed = 0;
4841
+ for (const group of batch(records, batchSize)) {
4842
+ for (const record of group) {
4843
+ try {
4844
+ await this.pool.query(
4845
+ `INSERT INTO ${this.ident(this.table)} (id, ${this.ident(this.contentColumn)}, ${this.ident(this.metadataColumn)}, ${this.ident(this.vectorColumn)})
4846
+ VALUES ($1, $2, $3, $4)
4847
+ ON CONFLICT (id) DO UPDATE SET
4848
+ ${this.ident(this.contentColumn)} = EXCLUDED.${this.ident(this.contentColumn)},
4849
+ ${this.ident(this.metadataColumn)} = EXCLUDED.${this.ident(this.metadataColumn)},
4850
+ ${this.ident(this.vectorColumn)} = EXCLUDED.${this.ident(this.vectorColumn)}`,
4851
+ [
4852
+ record.id,
4853
+ record.text ?? null,
4854
+ JSON.stringify(record.metadata ?? {}),
4855
+ this.toVectorLiteral(record.vector)
4856
+ ]
4857
+ );
4858
+ upsertedIds.push(record.id);
4859
+ } catch (e) {
4860
+ errors.push({ id: record.id, error: e.message });
4861
+ }
4862
+ }
4863
+ completed += group.length;
4864
+ options?.onProgress?.({ completed, total: records.length });
4865
+ }
4866
+ return {
4867
+ upsertedIds,
4868
+ upsertedCount: upsertedIds.length,
4869
+ errors,
4870
+ durationMs: performance.now() - start
4871
+ };
4872
+ }
4873
+ async query(vector, options) {
4874
+ await this.ensureInitialized();
4875
+ const start = performance.now();
4876
+ const topK = options?.topK ?? 10;
4877
+ const params = [this.toVectorLiteral(vector)];
4878
+ let where = "";
4879
+ if (options?.filter && Object.keys(options.filter).length > 0) {
4880
+ params.push(JSON.stringify(options.filter));
4881
+ where = `WHERE ${this.ident(this.metadataColumn)} @> $${params.length}::jsonb`;
4882
+ }
4883
+ params.push(topK);
4884
+ const op = this.distanceOperator;
4885
+ const sql = `SELECT id, ${this.ident(this.contentColumn)} AS content, ${this.ident(this.metadataColumn)} AS metadata, ${this.ident(this.vectorColumn)} ${op} $1 AS distance FROM ${this.ident(this.table)} ${where} ORDER BY ${this.ident(this.vectorColumn)} ${op} $1 ASC LIMIT $${params.length}`;
4886
+ const result = await this.pool.query(sql, params);
4887
+ const matches = result.rows.map((row) => {
4888
+ const distance = Number(row.distance);
4889
+ const score = this.metric === "cosine" ? 1 - distance : 1 / (1 + distance);
4890
+ return {
4891
+ id: String(row.id),
4892
+ text: row.content ?? "",
4893
+ score,
4894
+ distance,
4895
+ metadata: row.metadata ?? {}
4896
+ };
4897
+ }).filter(
4898
+ (m) => options?.minScore === void 0 || m.score >= options.minScore
4899
+ );
4900
+ return {
4901
+ matches,
4902
+ namespace: this.namespace,
4903
+ durationMs: performance.now() - start
4904
+ };
4905
+ }
4906
+ async delete(ids, _options) {
4907
+ await this.ensureInitialized();
4908
+ const start = performance.now();
4909
+ const result = await this.pool.query(
4910
+ `DELETE FROM ${this.ident(this.table)} WHERE id = ANY($1)`,
4911
+ [ids]
4912
+ );
4913
+ const deleted = result.rowCount ?? ids.length;
4914
+ return {
4915
+ deletedCount: deleted,
4916
+ requestedCount: ids.length,
4917
+ countExact: result.rowCount !== null,
4918
+ durationMs: performance.now() - start
4919
+ };
4920
+ }
4921
+ async deleteAll(_options) {
4922
+ await this.ensureInitialized();
4923
+ const start = performance.now();
4924
+ const result = await this.pool.query(
4925
+ `DELETE FROM ${this.ident(this.table)}`
4926
+ );
4927
+ return {
4928
+ deletedCount: result.rowCount ?? 0,
4929
+ requestedCount: result.rowCount ?? void 0,
4930
+ countExact: result.rowCount !== null,
4931
+ durationMs: performance.now() - start
4932
+ };
4933
+ }
4934
+ async getStats() {
4935
+ await this.ensureInitialized();
4936
+ const result = await this.pool.query(
4937
+ `SELECT COUNT(*)::int AS count FROM ${this.ident(this.table)}`
4938
+ );
4939
+ const vectorCount = Number(result.rows[0]?.count ?? 0);
4940
+ return {
4941
+ type: this.storeType,
4942
+ vectorCount,
4943
+ namespaceCount: 1,
4944
+ dimensions: this.dimensions ?? 0,
4945
+ metric: this.metric,
4946
+ lastUpdated: Date.now()
4947
+ };
4948
+ }
4949
+ async checkHealth() {
4950
+ const start = performance.now();
4951
+ try {
4952
+ await this.ensureInitialized();
4953
+ await this.pool.query("SELECT 1");
4954
+ return {
4955
+ healthy: true,
4956
+ latencyMs: performance.now() - start,
4957
+ lastCheck: Date.now()
4958
+ };
4959
+ } catch (e) {
4960
+ return {
4961
+ healthy: false,
4962
+ latencyMs: performance.now() - start,
4963
+ lastCheck: Date.now(),
4964
+ error: e.message
4965
+ };
4966
+ }
4967
+ }
4968
+ async close() {
4969
+ if (this.pool && !this.injectedPool) {
4970
+ await this.pool.end();
4971
+ }
4972
+ this.initialized = false;
4973
+ }
4974
+ };
4975
+
4976
+ // src/stores/WeaviateStore.ts
4977
+ var WeaviateStore = class extends BaseStore {
4978
+ storeType = "weaviate";
4979
+ backend;
4980
+ injectedBackend;
4981
+ className;
4982
+ url;
4983
+ apiKey;
4984
+ initialized = false;
4985
+ constructor(config) {
4986
+ super(config);
4987
+ if (!config.url) throw new Error("Weaviate store requires a `url`");
4988
+ if (!config.className) {
4989
+ throw new Error("Weaviate store requires a `className`");
4990
+ }
4991
+ this.url = config.url;
4992
+ this.className = config.className;
4993
+ this.apiKey = config.apiKey;
4994
+ this.injectedBackend = config.backend;
4995
+ }
4996
+ async init() {
4997
+ if (this.initialized) return;
4998
+ this.backend = this.injectedBackend ?? await this.buildSdkBackend();
4999
+ await this.backend.ensureClass(this.className, this.dimensions);
5000
+ this.initialized = true;
5001
+ }
5002
+ async ensureInitialized() {
5003
+ if (!this.initialized) await this.init();
5004
+ }
5005
+ /** Build a {@link WeaviateBackend} backed by the real `weaviate-ts-client`. */
5006
+ async buildSdkBackend() {
5007
+ let mod;
5008
+ try {
5009
+ mod = await importOptional("weaviate-ts-client");
5010
+ } catch {
5011
+ throw new Error(
5012
+ 'Weaviate store requires the "weaviate-ts-client" package. Install it, or pass a custom `backend`.'
5013
+ );
5014
+ }
5015
+ const weaviate = mod.default ?? mod;
5016
+ const u = new URL(this.url);
5017
+ const client = weaviate.client({
5018
+ scheme: u.protocol.replace(":", ""),
5019
+ host: u.host,
5020
+ apiKey: this.apiKey && weaviate.ApiKey ? new weaviate.ApiKey(this.apiKey) : void 0
5021
+ });
5022
+ return new WeaviateSdkBackend(client);
5023
+ }
5024
+ async upsert(records, options) {
5025
+ await this.ensureInitialized();
5026
+ const start = performance.now();
5027
+ const batchSize = options?.batchSize ?? 100;
5028
+ const upsertedIds = [];
5029
+ const errors = [];
5030
+ let completed = 0;
5031
+ for (const group of batch(records, batchSize)) {
5032
+ try {
5033
+ await this.backend.upsert(
5034
+ this.className,
5035
+ group.map((r) => ({
5036
+ id: r.id,
5037
+ vector: Array.from(r.vector),
5038
+ properties: { ...r.metadata ?? {}, text: r.text ?? "" }
5039
+ }))
5040
+ );
5041
+ upsertedIds.push(...group.map((r) => r.id));
5042
+ } catch (e) {
5043
+ for (const r of group)
5044
+ errors.push({ id: r.id, error: e.message });
5045
+ }
5046
+ completed += group.length;
5047
+ options?.onProgress?.({ completed, total: records.length });
5048
+ }
5049
+ return {
5050
+ upsertedIds,
5051
+ upsertedCount: upsertedIds.length,
5052
+ errors,
5053
+ durationMs: performance.now() - start
5054
+ };
5055
+ }
5056
+ async query(vector, options) {
5057
+ await this.ensureInitialized();
5058
+ const start = performance.now();
5059
+ const topK = options?.topK ?? 10;
5060
+ const hits = await this.backend.nearVector(
5061
+ this.className,
5062
+ Array.from(vector),
5063
+ topK,
5064
+ options?.filter
5065
+ );
5066
+ const matches = hits.map((h) => ({
5067
+ id: h.id,
5068
+ text: h.properties.text ?? "",
5069
+ score: h.score,
5070
+ metadata: h.properties
5071
+ })).filter(
5072
+ (m) => options?.minScore === void 0 || m.score >= options.minScore
5073
+ );
5074
+ return {
5075
+ matches,
5076
+ namespace: this.className,
5077
+ durationMs: performance.now() - start
5078
+ };
5079
+ }
5080
+ async delete(ids, _options) {
5081
+ await this.ensureInitialized();
5082
+ const start = performance.now();
5083
+ const deleted = await this.backend.deleteByIds(this.className, ids);
5084
+ return {
5085
+ deletedCount: deleted,
5086
+ requestedCount: ids.length,
5087
+ countExact: true,
5088
+ durationMs: performance.now() - start
5089
+ };
5090
+ }
5091
+ async deleteAll(_options) {
5092
+ await this.ensureInitialized();
5093
+ const start = performance.now();
5094
+ const deleted = await this.backend.deleteAll(this.className);
5095
+ return {
5096
+ deletedCount: deleted,
5097
+ requestedCount: deleted,
5098
+ countExact: true,
5099
+ durationMs: performance.now() - start
5100
+ };
5101
+ }
5102
+ async getStats() {
5103
+ await this.ensureInitialized();
5104
+ return {
5105
+ type: this.storeType,
5106
+ vectorCount: await this.backend.count(this.className),
5107
+ namespaceCount: 1,
5108
+ dimensions: this.dimensions ?? 0,
5109
+ metric: this.metric,
5110
+ lastUpdated: Date.now()
5111
+ };
5112
+ }
5113
+ async checkHealth() {
5114
+ const start = performance.now();
5115
+ try {
5116
+ await this.ensureInitialized();
5117
+ await this.backend.ping();
5118
+ return {
5119
+ healthy: true,
5120
+ latencyMs: performance.now() - start,
5121
+ lastCheck: Date.now()
5122
+ };
5123
+ } catch (e) {
5124
+ return {
5125
+ healthy: false,
5126
+ latencyMs: performance.now() - start,
5127
+ lastCheck: Date.now(),
5128
+ error: e.message
5129
+ };
5130
+ }
5131
+ }
5132
+ close() {
5133
+ this.initialized = false;
5134
+ return Promise.resolve();
5135
+ }
5136
+ };
5137
+ var WeaviateSdkBackend = class {
5138
+ constructor(client) {
5139
+ this.client = client;
5140
+ }
5141
+ async ensureClass(className, _dimensions) {
5142
+ const schema = await this.client.schema.getter().do();
5143
+ if (schema.classes?.some((c) => c.class === className)) return;
5144
+ await this.client.schema.classCreator().withClass({ class: className, vectorizer: "none" }).do();
5145
+ }
5146
+ async upsert(className, objects) {
5147
+ let batcher = this.client.batch.objectsBatcher();
5148
+ for (const o of objects) {
5149
+ batcher = batcher.withObject({
5150
+ class: className,
5151
+ id: o.id,
5152
+ vector: o.vector,
5153
+ properties: o.properties
5154
+ });
5155
+ }
5156
+ await batcher.do();
5157
+ }
5158
+ async nearVector(className, vector, limit, filter) {
5159
+ let q = this.client.graphql.get().withClassName(className).withFields("_additional { id certainty } text").withNearVector({ vector }).withLimit(limit);
5160
+ if (filter) q = q.withWhere(this.toWhere(filter));
5161
+ const res = await q.do();
5162
+ const rows = res.data?.Get?.[className] ?? [];
5163
+ return rows.map((row) => {
5164
+ const additional = row._additional;
5165
+ const { _additional, ...properties } = row;
5166
+ void _additional;
5167
+ return {
5168
+ id: additional.id,
5169
+ score: additional.certainty ?? 0,
5170
+ properties
5171
+ };
5172
+ });
5173
+ }
5174
+ async deleteByIds(className, ids) {
5175
+ let n = 0;
5176
+ for (const id of ids) {
5177
+ await this.client.data.deleter().withClassName(className).withId(id).do();
5178
+ n++;
5179
+ }
5180
+ return n;
5181
+ }
5182
+ async deleteAll(className) {
5183
+ const count = await this.count(className);
5184
+ await this.client.schema.classCreator().withClass({ class: className, vectorizer: "none" }).do().catch(() => void 0);
5185
+ return count;
5186
+ }
5187
+ async count(className) {
5188
+ const res = await this.client.graphql.aggregate().withClassName(className).withFields("meta { count }").do();
5189
+ const agg = res.data?.Aggregate?.[className] ?? [];
5190
+ return agg[0]?.meta?.count ?? 0;
5191
+ }
5192
+ async ping() {
5193
+ await this.client.misc.liveChecker().do();
5194
+ }
5195
+ toWhere(filter) {
5196
+ const operands = Object.entries(filter).map(([path, value]) => ({
5197
+ path: [path],
5198
+ operator: "Equal",
5199
+ ...typeof value === "number" ? { valueNumber: value } : typeof value === "boolean" ? { valueBoolean: value } : { valueText: String(value) }
5200
+ }));
5201
+ return operands.length === 1 ? operands[0] : { operator: "And", operands };
5202
+ }
5203
+ };
5204
+
5205
+ // src/stores/MilvusStore.ts
5206
+ var MilvusStore = class extends BaseStore {
5207
+ storeType = "milvus";
5208
+ backend;
5209
+ injectedBackend;
5210
+ collection;
5211
+ milvusConfig;
5212
+ initialized = false;
5213
+ constructor(config) {
5214
+ super(config);
5215
+ if (!config.url) throw new Error("Milvus store requires a `url`");
5216
+ if (!config.collectionName) {
5217
+ throw new Error("Milvus store requires a `collectionName`");
5218
+ }
5219
+ if (!config.dimensions) {
5220
+ throw new Error("Milvus store requires `dimensions`");
5221
+ }
5222
+ this.milvusConfig = config;
5223
+ this.collection = config.collectionName;
5224
+ this.injectedBackend = config.backend;
5225
+ }
5226
+ /** Milvus metric type string for the configured distance metric. */
5227
+ get metricType() {
5228
+ switch (this.metric) {
5229
+ case "euclidean":
5230
+ return "L2";
5231
+ case "dot_product":
5232
+ return "IP";
5233
+ case "cosine":
5234
+ default:
5235
+ return "COSINE";
5236
+ }
5237
+ }
5238
+ async init() {
5239
+ if (this.initialized) return;
5240
+ this.backend = this.injectedBackend ?? await this.buildSdkBackend();
5241
+ await this.backend.ensureCollection(
5242
+ this.collection,
5243
+ this.dimensions,
5244
+ this.metricType
5245
+ );
5246
+ this.initialized = true;
5247
+ }
5248
+ async ensureInitialized() {
5249
+ if (!this.initialized) await this.init();
5250
+ }
5251
+ async buildSdkBackend() {
5252
+ let mod;
5253
+ try {
5254
+ mod = await importOptional("@zilliz/milvus2-sdk-node");
5255
+ } catch {
5256
+ throw new Error(
5257
+ 'Milvus store requires the "@zilliz/milvus2-sdk-node" package. Install it, or pass a custom `backend`.'
5258
+ );
5259
+ }
5260
+ const sdk = mod;
5261
+ const client = new sdk.MilvusClient({
5262
+ address: this.milvusConfig.url,
5263
+ username: this.milvusConfig.username,
5264
+ password: this.milvusConfig.password
5265
+ });
5266
+ return new MilvusSdkBackend(client);
5267
+ }
5268
+ async upsert(records, options) {
5269
+ await this.ensureInitialized();
5270
+ const start = performance.now();
5271
+ const batchSize = options?.batchSize ?? 100;
5272
+ const upsertedIds = [];
5273
+ const errors = [];
5274
+ let completed = 0;
5275
+ for (const group of batch(records, batchSize)) {
5276
+ try {
5277
+ await this.backend.upsert(
5278
+ this.collection,
5279
+ group.map((r) => ({
5280
+ id: r.id,
5281
+ vector: Array.from(r.vector),
5282
+ text: r.text ?? "",
5283
+ metadata: r.metadata ?? {}
5284
+ }))
5285
+ );
5286
+ upsertedIds.push(...group.map((r) => r.id));
5287
+ } catch (e) {
5288
+ for (const r of group)
5289
+ errors.push({ id: r.id, error: e.message });
5290
+ }
5291
+ completed += group.length;
5292
+ options?.onProgress?.({ completed, total: records.length });
5293
+ }
5294
+ return {
5295
+ upsertedIds,
5296
+ upsertedCount: upsertedIds.length,
5297
+ errors,
5298
+ durationMs: performance.now() - start
5299
+ };
5300
+ }
5301
+ async query(vector, options) {
5302
+ await this.ensureInitialized();
5303
+ const start = performance.now();
5304
+ const topK = options?.topK ?? 10;
5305
+ const hits = await this.backend.search(
5306
+ this.collection,
5307
+ Array.from(vector),
5308
+ topK,
5309
+ options?.filter ? this.toExpr(options.filter) : void 0
5310
+ );
5311
+ const matches = hits.map((h) => ({
5312
+ id: h.id,
5313
+ text: h.text,
5314
+ score: h.score,
5315
+ metadata: h.metadata
5316
+ })).filter(
5317
+ (m) => options?.minScore === void 0 || m.score >= options.minScore
5318
+ );
5319
+ return {
5320
+ matches,
5321
+ namespace: this.collection,
5322
+ durationMs: performance.now() - start
5323
+ };
5324
+ }
5325
+ async delete(ids, _options) {
5326
+ await this.ensureInitialized();
5327
+ const start = performance.now();
5328
+ const deleted = await this.backend.deleteByIds(this.collection, ids);
5329
+ return {
5330
+ deletedCount: deleted,
5331
+ requestedCount: ids.length,
5332
+ countExact: true,
5333
+ durationMs: performance.now() - start
5334
+ };
5335
+ }
5336
+ async deleteAll(_options) {
5337
+ await this.ensureInitialized();
5338
+ const start = performance.now();
5339
+ const deleted = await this.backend.deleteAll(this.collection);
5340
+ return {
5341
+ deletedCount: deleted,
5342
+ requestedCount: deleted,
5343
+ countExact: true,
5344
+ durationMs: performance.now() - start
5345
+ };
5346
+ }
5347
+ async getStats() {
5348
+ await this.ensureInitialized();
5349
+ return {
5350
+ type: this.storeType,
5351
+ vectorCount: await this.backend.count(this.collection),
5352
+ namespaceCount: 1,
5353
+ dimensions: this.dimensions ?? 0,
5354
+ metric: this.metric,
5355
+ lastUpdated: Date.now()
5356
+ };
5357
+ }
5358
+ async checkHealth() {
5359
+ const start = performance.now();
5360
+ try {
5361
+ await this.ensureInitialized();
5362
+ await this.backend.ping();
5363
+ return {
5364
+ healthy: true,
5365
+ latencyMs: performance.now() - start,
5366
+ lastCheck: Date.now()
5367
+ };
5368
+ } catch (e) {
5369
+ return {
5370
+ healthy: false,
5371
+ latencyMs: performance.now() - start,
5372
+ lastCheck: Date.now(),
5373
+ error: e.message
5374
+ };
5375
+ }
5376
+ }
5377
+ close() {
5378
+ this.initialized = false;
5379
+ return Promise.resolve();
5380
+ }
5381
+ /** Translate a flat equality filter to a Milvus boolean expression. */
5382
+ toExpr(filter) {
5383
+ return Object.entries(filter).map(
5384
+ ([k, v]) => typeof v === "number" ? `metadata["${k}"] == ${v}` : `metadata["${k}"] == "${String(v)}"`
5385
+ ).join(" && ");
5386
+ }
5387
+ };
5388
+ var MilvusSdkBackend = class {
5389
+ constructor(client) {
5390
+ this.client = client;
5391
+ }
5392
+ async ensureCollection(collection, dimensions, metric) {
5393
+ const has = await this.client.hasCollection({
5394
+ collection_name: collection
5395
+ });
5396
+ if (!has.value) {
5397
+ await this.client.createCollection({
5398
+ collection_name: collection,
5399
+ fields: [
5400
+ {
5401
+ name: "id",
5402
+ data_type: "VarChar",
5403
+ is_primary_key: true,
5404
+ max_length: 512
5405
+ },
5406
+ { name: "vector", data_type: "FloatVector", dim: dimensions },
5407
+ { name: "text", data_type: "VarChar", max_length: 65535 },
5408
+ { name: "metadata", data_type: "JSON" }
5409
+ ]
5410
+ });
5411
+ await this.client.createIndex({
5412
+ collection_name: collection,
5413
+ field_name: "vector",
5414
+ index_type: "HNSW",
5415
+ metric_type: metric,
5416
+ params: { M: 16, efConstruction: 200 }
5417
+ });
5418
+ }
5419
+ await this.client.loadCollectionSync({ collection_name: collection });
5420
+ }
5421
+ async upsert(collection, rows) {
5422
+ await this.client.insert({
5423
+ collection_name: collection,
5424
+ data: rows.map((r) => ({
5425
+ id: r.id,
5426
+ vector: r.vector,
5427
+ text: r.text,
5428
+ metadata: r.metadata
5429
+ }))
5430
+ });
5431
+ }
5432
+ async search(collection, vector, limit, filter) {
5433
+ const res = await this.client.search({
5434
+ collection_name: collection,
5435
+ data: [vector],
5436
+ limit,
5437
+ filter,
5438
+ output_fields: ["text", "metadata"]
5439
+ });
5440
+ return res.results.map((r) => ({
5441
+ id: String(r.id),
5442
+ score: Number(r.score),
5443
+ text: r.text ?? "",
5444
+ metadata: r.metadata ?? {}
5445
+ }));
5446
+ }
5447
+ async deleteByIds(collection, ids) {
5448
+ const list = ids.map((id) => `"${id}"`).join(", ");
5449
+ await this.client.deleteEntities({
5450
+ collection_name: collection,
5451
+ expr: `id in [${list}]`
5452
+ });
5453
+ return ids.length;
5454
+ }
5455
+ async deleteAll(collection) {
5456
+ const count = await this.count(collection);
5457
+ await this.client.deleteEntities({
5458
+ collection_name: collection,
5459
+ expr: 'id != ""'
5460
+ });
5461
+ return count;
5462
+ }
5463
+ async count(collection) {
5464
+ const stats = await this.client.getCollectionStatistics({
5465
+ collection_name: collection
5466
+ });
5467
+ return Number(stats.data?.row_count ?? 0);
5468
+ }
5469
+ async ping() {
5470
+ await this.client.hasCollection({ collection_name: "__ping__" });
5471
+ }
5472
+ };
5473
+
4648
5474
  // src/stores/index.ts
4649
5475
  function createStore(type, config) {
4650
5476
  switch (type) {
@@ -4656,13 +5482,20 @@ function createStore(type, config) {
4656
5482
  return new ChromaStore(config);
4657
5483
  case "qdrant":
4658
5484
  return new QdrantStore(config);
5485
+ case "weaviate":
5486
+ return new WeaviateStore(config);
5487
+ case "milvus":
5488
+ return new MilvusStore(config);
5489
+ case "pgvector":
5490
+ return new PgVectorStore(config);
4659
5491
  default:
4660
- return new MemoryStore(config);
5492
+ throw new Error(
5493
+ `Unknown vector store type "${String(type)}". Supported stores: memory, pinecone, chroma, qdrant, weaviate, milvus, pgvector.`
5494
+ );
4661
5495
  }
4662
5496
  }
4663
5497
 
4664
5498
  // src/versioning/VersionRegistry.ts
4665
- var import_nanoid2 = require("nanoid");
4666
5499
  var import_eventemitter32 = __toESM(require("eventemitter3"));
4667
5500
  var VersionRegistry = class extends import_eventemitter32.default {
4668
5501
  versions = /* @__PURE__ */ new Map();
@@ -4683,7 +5516,7 @@ var VersionRegistry = class extends import_eventemitter32.default {
4683
5516
  register(version) {
4684
5517
  const newVersion = {
4685
5518
  ...version,
4686
- id: (0, import_nanoid2.nanoid)(),
5519
+ id: nanoid(),
4687
5520
  createdAt: Date.now(),
4688
5521
  active: false,
4689
5522
  deprecated: false
@@ -4914,7 +5747,6 @@ function createVersionRegistry(options) {
4914
5747
 
4915
5748
  // src/quality/DriftDetector.ts
4916
5749
  var import_eventemitter33 = __toESM(require("eventemitter3"));
4917
- var import_nanoid3 = require("nanoid");
4918
5750
  var DriftDetector = class extends import_eventemitter33.default {
4919
5751
  reference = null;
4920
5752
  config;
@@ -4949,7 +5781,7 @@ var DriftDetector = class extends import_eventemitter33.default {
4949
5781
  varianceVector.push(variance(values));
4950
5782
  }
4951
5783
  this.reference = {
4952
- id: (0, import_nanoid3.nanoid)(),
5784
+ id: nanoid(),
4953
5785
  model,
4954
5786
  version,
4955
5787
  sampleCount: embeddings.length,
@@ -5124,7 +5956,7 @@ var DriftDetector = class extends import_eventemitter33.default {
5124
5956
  */
5125
5957
  emitAlert(result) {
5126
5958
  const alert = {
5127
- id: (0, import_nanoid3.nanoid)(),
5959
+ id: nanoid(),
5128
5960
  type: "drift_detected",
5129
5961
  severity: result.severity,
5130
5962
  message: `Embedding drift detected with score ${result.driftScore.toFixed(3)}`,