@lov3kaizen/agentsea-embeddings 1.0.1 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/caching/index.js +7 -1
- package/dist/caching/index.mjs +1 -1
- package/dist/chunk-2TCNSTX3.mjs +12 -0
- package/dist/{chunk-NBHIRTJT.mjs → chunk-5GTQFVEI.mjs} +41 -1
- package/dist/chunk-JHWMXQ56.mjs +1650 -0
- package/dist/{chunk-VPSMDBHH.mjs → chunk-MNJPAUDC.mjs} +7 -1
- package/dist/{chunk-DJAURHAS.mjs → chunk-U6EYWYUD.mjs} +31 -1
- package/dist/chunking/index.js +33 -4
- package/dist/chunking/index.mjs +1 -1
- package/dist/{index-DmEEUzJg.d.mts → index-DGzfvyHY.d.mts} +3 -0
- package/dist/{index-G-KgyvZT.d.ts → index-_uJcyK8e.d.ts} +3 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +848 -16
- package/dist/index.mjs +22 -22
- package/dist/providers/index.d.mts +1 -1
- package/dist/providers/index.d.ts +1 -1
- package/dist/providers/index.js +47 -1
- package/dist/providers/index.mjs +2 -1
- package/dist/stores/index.d.mts +255 -1
- package/dist/stores/index.d.ts +255 -1
- package/dist/stores/index.js +794 -7
- package/dist/stores/index.mjs +20 -3
- package/package.json +9 -6
- package/dist/chunk-TER262ST.mjs +0 -877
package/dist/index.js
CHANGED
|
@@ -1500,20 +1500,33 @@ function createVoyageProvider(config) {
|
|
|
1500
1500
|
return new VoyageProvider(config);
|
|
1501
1501
|
}
|
|
1502
1502
|
|
|
1503
|
+
// src/core/optional-import.ts
|
|
1504
|
+
function importOptional(name) {
|
|
1505
|
+
return import(
|
|
1506
|
+
/* @vite-ignore */
|
|
1507
|
+
/* webpackIgnore: true */
|
|
1508
|
+
name
|
|
1509
|
+
);
|
|
1510
|
+
}
|
|
1511
|
+
|
|
1503
1512
|
// src/providers/LocalProvider.ts
|
|
1504
1513
|
var LocalProvider = class extends BaseProvider {
|
|
1505
1514
|
modelInfo;
|
|
1506
1515
|
embedFn = null;
|
|
1507
1516
|
normalize;
|
|
1508
1517
|
batchSize;
|
|
1518
|
+
modelPath;
|
|
1519
|
+
/** Lazily-built ONNX extractor (only when loading from `modelPath`). */
|
|
1520
|
+
extractorPromise;
|
|
1509
1521
|
constructor(config) {
|
|
1510
1522
|
super({ ...config, type: "local" });
|
|
1511
1523
|
if (!config.embedFn && !config.modelPath) {
|
|
1512
1524
|
throw new Error(
|
|
1513
|
-
"
|
|
1525
|
+
"`embedFn` or `modelPath` (ONNX) is required for the local provider"
|
|
1514
1526
|
);
|
|
1515
1527
|
}
|
|
1516
1528
|
this.embedFn = config.embedFn ?? null;
|
|
1529
|
+
this.modelPath = config.modelPath;
|
|
1517
1530
|
this.normalize = config.normalize ?? true;
|
|
1518
1531
|
this.batchSize = config.batchSize ?? 32;
|
|
1519
1532
|
this.modelInfo = {
|
|
@@ -1527,10 +1540,43 @@ var LocalProvider = class extends BaseProvider {
|
|
|
1527
1540
|
description: "Local embedding model"
|
|
1528
1541
|
};
|
|
1529
1542
|
}
|
|
1543
|
+
/**
|
|
1544
|
+
* Lazily load the ONNX feature-extraction pipeline from `modelPath` via
|
|
1545
|
+
* Transformers.js and adapt it into a {@link LocalEmbeddingFn}.
|
|
1546
|
+
*/
|
|
1547
|
+
getOnnxEmbedFn() {
|
|
1548
|
+
if (!this.extractorPromise) {
|
|
1549
|
+
this.extractorPromise = (async () => {
|
|
1550
|
+
let mod;
|
|
1551
|
+
try {
|
|
1552
|
+
mod = await importOptional("@xenova/transformers");
|
|
1553
|
+
} catch {
|
|
1554
|
+
throw new Error(
|
|
1555
|
+
'Loading local ONNX models from `modelPath` requires the "@xenova/transformers" package. Install it, or pass an `embedFn`.'
|
|
1556
|
+
);
|
|
1557
|
+
}
|
|
1558
|
+
const transformers = mod;
|
|
1559
|
+
return transformers.pipeline("feature-extraction", this.modelPath);
|
|
1560
|
+
})();
|
|
1561
|
+
}
|
|
1562
|
+
return this.extractorPromise;
|
|
1563
|
+
}
|
|
1530
1564
|
get info() {
|
|
1531
1565
|
return this.modelInfo;
|
|
1532
1566
|
}
|
|
1533
1567
|
async doEmbed(texts, options) {
|
|
1568
|
+
if (!this.embedFn && this.modelPath) {
|
|
1569
|
+
const extractor = await this.getOnnxEmbedFn();
|
|
1570
|
+
this.embedFn = async (input) => {
|
|
1571
|
+
const output = await extractor(input, {
|
|
1572
|
+
pooling: "mean",
|
|
1573
|
+
normalize: false
|
|
1574
|
+
// we normalize below if configured
|
|
1575
|
+
});
|
|
1576
|
+
const list = output.tolist();
|
|
1577
|
+
return Array.isArray(list[0]) ? list : [list];
|
|
1578
|
+
};
|
|
1579
|
+
}
|
|
1534
1580
|
if (!this.embedFn) {
|
|
1535
1581
|
throw new Error("No embedding function configured");
|
|
1536
1582
|
}
|
|
@@ -1768,8 +1814,37 @@ function createHuggingFaceProvider(config) {
|
|
|
1768
1814
|
return new HuggingFaceProvider(config);
|
|
1769
1815
|
}
|
|
1770
1816
|
|
|
1817
|
+
// ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/index.js
|
|
1818
|
+
var import_node_crypto = require("crypto");
|
|
1819
|
+
|
|
1820
|
+
// ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/url-alphabet/index.js
|
|
1821
|
+
var urlAlphabet = "useandom-26T198340PX75pxJACKVERYMINDBUSHWOLF_GQZbfghjklqvwyzrict";
|
|
1822
|
+
|
|
1823
|
+
// ../../node_modules/.pnpm/nanoid@5.1.6/node_modules/nanoid/index.js
|
|
1824
|
+
var POOL_SIZE_MULTIPLIER = 128;
|
|
1825
|
+
var pool;
|
|
1826
|
+
var poolOffset;
|
|
1827
|
+
function fillPool(bytes) {
|
|
1828
|
+
if (!pool || pool.length < bytes) {
|
|
1829
|
+
pool = Buffer.allocUnsafe(bytes * POOL_SIZE_MULTIPLIER);
|
|
1830
|
+
import_node_crypto.webcrypto.getRandomValues(pool);
|
|
1831
|
+
poolOffset = 0;
|
|
1832
|
+
} else if (poolOffset + bytes > pool.length) {
|
|
1833
|
+
import_node_crypto.webcrypto.getRandomValues(pool);
|
|
1834
|
+
poolOffset = 0;
|
|
1835
|
+
}
|
|
1836
|
+
poolOffset += bytes;
|
|
1837
|
+
}
|
|
1838
|
+
function nanoid(size = 21) {
|
|
1839
|
+
fillPool(size |= 0);
|
|
1840
|
+
let id = "";
|
|
1841
|
+
for (let i = poolOffset - size; i < poolOffset; i++) {
|
|
1842
|
+
id += urlAlphabet[pool[i] & 63];
|
|
1843
|
+
}
|
|
1844
|
+
return id;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1771
1847
|
// src/chunking/BaseChunker.ts
|
|
1772
|
-
var import_nanoid = require("nanoid");
|
|
1773
1848
|
var defaultTokenCounter = (text) => {
|
|
1774
1849
|
return Math.ceil(text.length / 4);
|
|
1775
1850
|
};
|
|
@@ -1811,7 +1886,7 @@ var BaseChunker = class {
|
|
|
1811
1886
|
if (options.source) metadata.source = options.source;
|
|
1812
1887
|
if (options.type) metadata.type = options.type;
|
|
1813
1888
|
return {
|
|
1814
|
-
id:
|
|
1889
|
+
id: nanoid(),
|
|
1815
1890
|
text,
|
|
1816
1891
|
index,
|
|
1817
1892
|
startPosition,
|
|
@@ -1919,7 +1994,7 @@ function splitLargeChunks(chunks, maxTokens, tokenCounter) {
|
|
|
1919
1994
|
if (testTokens > maxTokens && currentText) {
|
|
1920
1995
|
result.push({
|
|
1921
1996
|
...chunk2,
|
|
1922
|
-
id:
|
|
1997
|
+
id: nanoid(),
|
|
1923
1998
|
text: currentText,
|
|
1924
1999
|
startPosition: currentStart,
|
|
1925
2000
|
endPosition: currentStart + currentText.length,
|
|
@@ -1935,7 +2010,7 @@ function splitLargeChunks(chunks, maxTokens, tokenCounter) {
|
|
|
1935
2010
|
if (currentText) {
|
|
1936
2011
|
result.push({
|
|
1937
2012
|
...chunk2,
|
|
1938
|
-
id:
|
|
2013
|
+
id: nanoid(),
|
|
1939
2014
|
text: currentText,
|
|
1940
2015
|
startPosition: currentStart,
|
|
1941
2016
|
endPosition: currentStart + currentText.length,
|
|
@@ -3433,7 +3508,13 @@ var SQLiteCache = class extends BaseCache {
|
|
|
3433
3508
|
`);
|
|
3434
3509
|
updateStmt.run(Date.now(), key);
|
|
3435
3510
|
const vectorBuffer = row.vector;
|
|
3436
|
-
const vector = Array.from(
|
|
3511
|
+
const vector = Array.from(
|
|
3512
|
+
new Float32Array(
|
|
3513
|
+
vectorBuffer.buffer,
|
|
3514
|
+
vectorBuffer.byteOffset,
|
|
3515
|
+
vectorBuffer.byteLength / Float32Array.BYTES_PER_ELEMENT
|
|
3516
|
+
)
|
|
3517
|
+
);
|
|
3437
3518
|
return {
|
|
3438
3519
|
key: row.key,
|
|
3439
3520
|
vector,
|
|
@@ -3984,6 +4065,8 @@ var MemoryStore = class extends BaseStore {
|
|
|
3984
4065
|
}
|
|
3985
4066
|
return Promise.resolve({
|
|
3986
4067
|
deletedCount,
|
|
4068
|
+
requestedCount: ids.length,
|
|
4069
|
+
countExact: true,
|
|
3987
4070
|
durationMs: performance.now() - startTime
|
|
3988
4071
|
});
|
|
3989
4072
|
}
|
|
@@ -3996,6 +4079,8 @@ var MemoryStore = class extends BaseStore {
|
|
|
3996
4079
|
this.namespaces.clear();
|
|
3997
4080
|
return Promise.resolve({
|
|
3998
4081
|
deletedCount: count2,
|
|
4082
|
+
requestedCount: count2,
|
|
4083
|
+
countExact: true,
|
|
3999
4084
|
durationMs: performance.now() - startTime
|
|
4000
4085
|
});
|
|
4001
4086
|
}
|
|
@@ -4003,6 +4088,8 @@ var MemoryStore = class extends BaseStore {
|
|
|
4003
4088
|
if (!nsIds) {
|
|
4004
4089
|
return Promise.resolve({
|
|
4005
4090
|
deletedCount: 0,
|
|
4091
|
+
requestedCount: 0,
|
|
4092
|
+
countExact: true,
|
|
4006
4093
|
durationMs: performance.now() - startTime
|
|
4007
4094
|
});
|
|
4008
4095
|
}
|
|
@@ -4013,6 +4100,8 @@ var MemoryStore = class extends BaseStore {
|
|
|
4013
4100
|
this.namespaces.delete(namespace);
|
|
4014
4101
|
return Promise.resolve({
|
|
4015
4102
|
deletedCount: count,
|
|
4103
|
+
requestedCount: count,
|
|
4104
|
+
countExact: true,
|
|
4016
4105
|
durationMs: performance.now() - startTime
|
|
4017
4106
|
});
|
|
4018
4107
|
}
|
|
@@ -4207,6 +4296,8 @@ var PineconeStore = class extends BaseStore {
|
|
|
4207
4296
|
);
|
|
4208
4297
|
return {
|
|
4209
4298
|
deletedCount: ids.length,
|
|
4299
|
+
requestedCount: ids.length,
|
|
4300
|
+
countExact: false,
|
|
4210
4301
|
durationMs: performance.now() - startTime
|
|
4211
4302
|
};
|
|
4212
4303
|
}
|
|
@@ -4214,13 +4305,15 @@ var PineconeStore = class extends BaseStore {
|
|
|
4214
4305
|
await this.ensureInitialized();
|
|
4215
4306
|
const startTime = performance.now();
|
|
4216
4307
|
const namespace = options?.namespace ?? this.namespace;
|
|
4308
|
+
const before = await this.getStats().catch(() => void 0);
|
|
4217
4309
|
const ns = this.index.namespace(
|
|
4218
4310
|
namespace
|
|
4219
4311
|
);
|
|
4220
4312
|
await ns.deleteAll();
|
|
4221
4313
|
return {
|
|
4222
|
-
deletedCount:
|
|
4223
|
-
|
|
4314
|
+
deletedCount: before?.vectorCount ?? 0,
|
|
4315
|
+
requestedCount: before?.vectorCount,
|
|
4316
|
+
countExact: before !== void 0,
|
|
4224
4317
|
durationMs: performance.now() - startTime
|
|
4225
4318
|
};
|
|
4226
4319
|
}
|
|
@@ -4381,12 +4474,15 @@ var ChromaStore = class extends BaseStore {
|
|
|
4381
4474
|
await this.collection.delete({ ids });
|
|
4382
4475
|
return {
|
|
4383
4476
|
deletedCount: ids.length,
|
|
4477
|
+
requestedCount: ids.length,
|
|
4478
|
+
countExact: false,
|
|
4384
4479
|
durationMs: performance.now() - startTime
|
|
4385
4480
|
};
|
|
4386
4481
|
}
|
|
4387
4482
|
async deleteAll(_options) {
|
|
4388
4483
|
await this.ensureInitialized();
|
|
4389
4484
|
const startTime = performance.now();
|
|
4485
|
+
const before = await this.getStats().catch(() => void 0);
|
|
4390
4486
|
await this.client.deleteCollection({ name: this.collectionName });
|
|
4391
4487
|
this.collection = await this.client.createCollection({
|
|
4392
4488
|
name: this.collectionName,
|
|
@@ -4395,7 +4491,9 @@ var ChromaStore = class extends BaseStore {
|
|
|
4395
4491
|
}
|
|
4396
4492
|
});
|
|
4397
4493
|
return {
|
|
4398
|
-
deletedCount:
|
|
4494
|
+
deletedCount: before?.vectorCount ?? 0,
|
|
4495
|
+
requestedCount: before?.vectorCount,
|
|
4496
|
+
countExact: before !== void 0,
|
|
4399
4497
|
durationMs: performance.now() - startTime
|
|
4400
4498
|
};
|
|
4401
4499
|
}
|
|
@@ -4585,12 +4683,15 @@ var QdrantStore = class extends BaseStore {
|
|
|
4585
4683
|
});
|
|
4586
4684
|
return {
|
|
4587
4685
|
deletedCount: ids.length,
|
|
4686
|
+
requestedCount: ids.length,
|
|
4687
|
+
countExact: false,
|
|
4588
4688
|
durationMs: performance.now() - startTime
|
|
4589
4689
|
};
|
|
4590
4690
|
}
|
|
4591
4691
|
async deleteAll(_options) {
|
|
4592
4692
|
await this.ensureInitialized();
|
|
4593
4693
|
const startTime = performance.now();
|
|
4694
|
+
const before = await this.getStats().catch(() => void 0);
|
|
4594
4695
|
await this.client.deleteCollection(this.collectionName);
|
|
4595
4696
|
if (this.dimensions) {
|
|
4596
4697
|
await this.client.createCollection(this.collectionName, {
|
|
@@ -4601,7 +4702,9 @@ var QdrantStore = class extends BaseStore {
|
|
|
4601
4702
|
});
|
|
4602
4703
|
}
|
|
4603
4704
|
return {
|
|
4604
|
-
deletedCount:
|
|
4705
|
+
deletedCount: before?.vectorCount ?? 0,
|
|
4706
|
+
requestedCount: before?.vectorCount,
|
|
4707
|
+
countExact: before !== void 0,
|
|
4605
4708
|
durationMs: performance.now() - startTime
|
|
4606
4709
|
};
|
|
4607
4710
|
}
|
|
@@ -4645,6 +4748,729 @@ function createQdrantStore(config) {
|
|
|
4645
4748
|
return new QdrantStore(config);
|
|
4646
4749
|
}
|
|
4647
4750
|
|
|
4751
|
+
// src/stores/PgVectorStore.ts
|
|
4752
|
+
var PgVectorStore = class extends BaseStore {
|
|
4753
|
+
storeType = "pgvector";
|
|
4754
|
+
pool;
|
|
4755
|
+
injectedPool;
|
|
4756
|
+
table;
|
|
4757
|
+
vectorColumn;
|
|
4758
|
+
contentColumn;
|
|
4759
|
+
metadataColumn;
|
|
4760
|
+
pgConfig;
|
|
4761
|
+
initialized = false;
|
|
4762
|
+
constructor(config) {
|
|
4763
|
+
super(config);
|
|
4764
|
+
if (!config.tableName) {
|
|
4765
|
+
throw new Error("pgvector store requires a `tableName`");
|
|
4766
|
+
}
|
|
4767
|
+
this.pgConfig = config;
|
|
4768
|
+
this.injectedPool = config.pool;
|
|
4769
|
+
this.table = config.tableName;
|
|
4770
|
+
this.vectorColumn = config.vectorColumn ?? "embedding";
|
|
4771
|
+
this.contentColumn = config.contentColumn ?? "content";
|
|
4772
|
+
this.metadataColumn = config.metadataColumn ?? "metadata";
|
|
4773
|
+
}
|
|
4774
|
+
/** The pgvector distance operator for the configured metric. */
|
|
4775
|
+
get distanceOperator() {
|
|
4776
|
+
switch (this.metric) {
|
|
4777
|
+
case "euclidean":
|
|
4778
|
+
return "<->";
|
|
4779
|
+
case "dot_product":
|
|
4780
|
+
return "<#>";
|
|
4781
|
+
case "cosine":
|
|
4782
|
+
default:
|
|
4783
|
+
return "<=>";
|
|
4784
|
+
}
|
|
4785
|
+
}
|
|
4786
|
+
async init() {
|
|
4787
|
+
if (this.initialized) return;
|
|
4788
|
+
if (this.injectedPool) {
|
|
4789
|
+
this.pool = this.injectedPool;
|
|
4790
|
+
} else {
|
|
4791
|
+
let mod;
|
|
4792
|
+
try {
|
|
4793
|
+
mod = await importOptional("pg");
|
|
4794
|
+
} catch {
|
|
4795
|
+
throw new Error(
|
|
4796
|
+
'pgvector store requires the "pg" package. Install it, or pass a pre-built `pool` to the store.'
|
|
4797
|
+
);
|
|
4798
|
+
}
|
|
4799
|
+
const pg = mod.default ?? mod;
|
|
4800
|
+
const Pool = pg.Pool;
|
|
4801
|
+
this.pool = new Pool(
|
|
4802
|
+
this.pgConfig.connectionString ? { connectionString: this.pgConfig.connectionString } : {
|
|
4803
|
+
host: this.pgConfig.host,
|
|
4804
|
+
port: this.pgConfig.port,
|
|
4805
|
+
database: this.pgConfig.database,
|
|
4806
|
+
user: this.pgConfig.user,
|
|
4807
|
+
password: this.pgConfig.password
|
|
4808
|
+
}
|
|
4809
|
+
);
|
|
4810
|
+
}
|
|
4811
|
+
await this.pool.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
4812
|
+
const dims = this.dimensions;
|
|
4813
|
+
const vectorType = dims ? `vector(${dims})` : "vector";
|
|
4814
|
+
await this.pool.query(
|
|
4815
|
+
`CREATE TABLE IF NOT EXISTS ${this.ident(this.table)} (
|
|
4816
|
+
id text PRIMARY KEY,
|
|
4817
|
+
${this.ident(this.contentColumn)} text,
|
|
4818
|
+
${this.ident(this.metadataColumn)} jsonb,
|
|
4819
|
+
${this.ident(this.vectorColumn)} ${vectorType}
|
|
4820
|
+
)`
|
|
4821
|
+
);
|
|
4822
|
+
this.initialized = true;
|
|
4823
|
+
}
|
|
4824
|
+
async ensureInitialized() {
|
|
4825
|
+
if (!this.initialized) await this.init();
|
|
4826
|
+
}
|
|
4827
|
+
/** Quote an SQL identifier to guard against injection via config names. */
|
|
4828
|
+
ident(name) {
|
|
4829
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
4830
|
+
}
|
|
4831
|
+
toVectorLiteral(vector) {
|
|
4832
|
+
return `[${Array.from(vector).join(",")}]`;
|
|
4833
|
+
}
|
|
4834
|
+
async upsert(records, options) {
|
|
4835
|
+
await this.ensureInitialized();
|
|
4836
|
+
const start = performance.now();
|
|
4837
|
+
const batchSize = options?.batchSize ?? 100;
|
|
4838
|
+
const upsertedIds = [];
|
|
4839
|
+
const errors = [];
|
|
4840
|
+
let completed = 0;
|
|
4841
|
+
for (const group of batch(records, batchSize)) {
|
|
4842
|
+
for (const record of group) {
|
|
4843
|
+
try {
|
|
4844
|
+
await this.pool.query(
|
|
4845
|
+
`INSERT INTO ${this.ident(this.table)} (id, ${this.ident(this.contentColumn)}, ${this.ident(this.metadataColumn)}, ${this.ident(this.vectorColumn)})
|
|
4846
|
+
VALUES ($1, $2, $3, $4)
|
|
4847
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
4848
|
+
${this.ident(this.contentColumn)} = EXCLUDED.${this.ident(this.contentColumn)},
|
|
4849
|
+
${this.ident(this.metadataColumn)} = EXCLUDED.${this.ident(this.metadataColumn)},
|
|
4850
|
+
${this.ident(this.vectorColumn)} = EXCLUDED.${this.ident(this.vectorColumn)}`,
|
|
4851
|
+
[
|
|
4852
|
+
record.id,
|
|
4853
|
+
record.text ?? null,
|
|
4854
|
+
JSON.stringify(record.metadata ?? {}),
|
|
4855
|
+
this.toVectorLiteral(record.vector)
|
|
4856
|
+
]
|
|
4857
|
+
);
|
|
4858
|
+
upsertedIds.push(record.id);
|
|
4859
|
+
} catch (e) {
|
|
4860
|
+
errors.push({ id: record.id, error: e.message });
|
|
4861
|
+
}
|
|
4862
|
+
}
|
|
4863
|
+
completed += group.length;
|
|
4864
|
+
options?.onProgress?.({ completed, total: records.length });
|
|
4865
|
+
}
|
|
4866
|
+
return {
|
|
4867
|
+
upsertedIds,
|
|
4868
|
+
upsertedCount: upsertedIds.length,
|
|
4869
|
+
errors,
|
|
4870
|
+
durationMs: performance.now() - start
|
|
4871
|
+
};
|
|
4872
|
+
}
|
|
4873
|
+
async query(vector, options) {
|
|
4874
|
+
await this.ensureInitialized();
|
|
4875
|
+
const start = performance.now();
|
|
4876
|
+
const topK = options?.topK ?? 10;
|
|
4877
|
+
const params = [this.toVectorLiteral(vector)];
|
|
4878
|
+
let where = "";
|
|
4879
|
+
if (options?.filter && Object.keys(options.filter).length > 0) {
|
|
4880
|
+
params.push(JSON.stringify(options.filter));
|
|
4881
|
+
where = `WHERE ${this.ident(this.metadataColumn)} @> $${params.length}::jsonb`;
|
|
4882
|
+
}
|
|
4883
|
+
params.push(topK);
|
|
4884
|
+
const op = this.distanceOperator;
|
|
4885
|
+
const sql = `SELECT id, ${this.ident(this.contentColumn)} AS content, ${this.ident(this.metadataColumn)} AS metadata, ${this.ident(this.vectorColumn)} ${op} $1 AS distance FROM ${this.ident(this.table)} ${where} ORDER BY ${this.ident(this.vectorColumn)} ${op} $1 ASC LIMIT $${params.length}`;
|
|
4886
|
+
const result = await this.pool.query(sql, params);
|
|
4887
|
+
const matches = result.rows.map((row) => {
|
|
4888
|
+
const distance = Number(row.distance);
|
|
4889
|
+
const score = this.metric === "cosine" ? 1 - distance : 1 / (1 + distance);
|
|
4890
|
+
return {
|
|
4891
|
+
id: String(row.id),
|
|
4892
|
+
text: row.content ?? "",
|
|
4893
|
+
score,
|
|
4894
|
+
distance,
|
|
4895
|
+
metadata: row.metadata ?? {}
|
|
4896
|
+
};
|
|
4897
|
+
}).filter(
|
|
4898
|
+
(m) => options?.minScore === void 0 || m.score >= options.minScore
|
|
4899
|
+
);
|
|
4900
|
+
return {
|
|
4901
|
+
matches,
|
|
4902
|
+
namespace: this.namespace,
|
|
4903
|
+
durationMs: performance.now() - start
|
|
4904
|
+
};
|
|
4905
|
+
}
|
|
4906
|
+
async delete(ids, _options) {
|
|
4907
|
+
await this.ensureInitialized();
|
|
4908
|
+
const start = performance.now();
|
|
4909
|
+
const result = await this.pool.query(
|
|
4910
|
+
`DELETE FROM ${this.ident(this.table)} WHERE id = ANY($1)`,
|
|
4911
|
+
[ids]
|
|
4912
|
+
);
|
|
4913
|
+
const deleted = result.rowCount ?? ids.length;
|
|
4914
|
+
return {
|
|
4915
|
+
deletedCount: deleted,
|
|
4916
|
+
requestedCount: ids.length,
|
|
4917
|
+
countExact: result.rowCount !== null,
|
|
4918
|
+
durationMs: performance.now() - start
|
|
4919
|
+
};
|
|
4920
|
+
}
|
|
4921
|
+
async deleteAll(_options) {
|
|
4922
|
+
await this.ensureInitialized();
|
|
4923
|
+
const start = performance.now();
|
|
4924
|
+
const result = await this.pool.query(
|
|
4925
|
+
`DELETE FROM ${this.ident(this.table)}`
|
|
4926
|
+
);
|
|
4927
|
+
return {
|
|
4928
|
+
deletedCount: result.rowCount ?? 0,
|
|
4929
|
+
requestedCount: result.rowCount ?? void 0,
|
|
4930
|
+
countExact: result.rowCount !== null,
|
|
4931
|
+
durationMs: performance.now() - start
|
|
4932
|
+
};
|
|
4933
|
+
}
|
|
4934
|
+
async getStats() {
|
|
4935
|
+
await this.ensureInitialized();
|
|
4936
|
+
const result = await this.pool.query(
|
|
4937
|
+
`SELECT COUNT(*)::int AS count FROM ${this.ident(this.table)}`
|
|
4938
|
+
);
|
|
4939
|
+
const vectorCount = Number(result.rows[0]?.count ?? 0);
|
|
4940
|
+
return {
|
|
4941
|
+
type: this.storeType,
|
|
4942
|
+
vectorCount,
|
|
4943
|
+
namespaceCount: 1,
|
|
4944
|
+
dimensions: this.dimensions ?? 0,
|
|
4945
|
+
metric: this.metric,
|
|
4946
|
+
lastUpdated: Date.now()
|
|
4947
|
+
};
|
|
4948
|
+
}
|
|
4949
|
+
async checkHealth() {
|
|
4950
|
+
const start = performance.now();
|
|
4951
|
+
try {
|
|
4952
|
+
await this.ensureInitialized();
|
|
4953
|
+
await this.pool.query("SELECT 1");
|
|
4954
|
+
return {
|
|
4955
|
+
healthy: true,
|
|
4956
|
+
latencyMs: performance.now() - start,
|
|
4957
|
+
lastCheck: Date.now()
|
|
4958
|
+
};
|
|
4959
|
+
} catch (e) {
|
|
4960
|
+
return {
|
|
4961
|
+
healthy: false,
|
|
4962
|
+
latencyMs: performance.now() - start,
|
|
4963
|
+
lastCheck: Date.now(),
|
|
4964
|
+
error: e.message
|
|
4965
|
+
};
|
|
4966
|
+
}
|
|
4967
|
+
}
|
|
4968
|
+
async close() {
|
|
4969
|
+
if (this.pool && !this.injectedPool) {
|
|
4970
|
+
await this.pool.end();
|
|
4971
|
+
}
|
|
4972
|
+
this.initialized = false;
|
|
4973
|
+
}
|
|
4974
|
+
};
|
|
4975
|
+
|
|
4976
|
+
// src/stores/WeaviateStore.ts
|
|
4977
|
+
var WeaviateStore = class extends BaseStore {
|
|
4978
|
+
storeType = "weaviate";
|
|
4979
|
+
backend;
|
|
4980
|
+
injectedBackend;
|
|
4981
|
+
className;
|
|
4982
|
+
url;
|
|
4983
|
+
apiKey;
|
|
4984
|
+
initialized = false;
|
|
4985
|
+
constructor(config) {
|
|
4986
|
+
super(config);
|
|
4987
|
+
if (!config.url) throw new Error("Weaviate store requires a `url`");
|
|
4988
|
+
if (!config.className) {
|
|
4989
|
+
throw new Error("Weaviate store requires a `className`");
|
|
4990
|
+
}
|
|
4991
|
+
this.url = config.url;
|
|
4992
|
+
this.className = config.className;
|
|
4993
|
+
this.apiKey = config.apiKey;
|
|
4994
|
+
this.injectedBackend = config.backend;
|
|
4995
|
+
}
|
|
4996
|
+
async init() {
|
|
4997
|
+
if (this.initialized) return;
|
|
4998
|
+
this.backend = this.injectedBackend ?? await this.buildSdkBackend();
|
|
4999
|
+
await this.backend.ensureClass(this.className, this.dimensions);
|
|
5000
|
+
this.initialized = true;
|
|
5001
|
+
}
|
|
5002
|
+
async ensureInitialized() {
|
|
5003
|
+
if (!this.initialized) await this.init();
|
|
5004
|
+
}
|
|
5005
|
+
/** Build a {@link WeaviateBackend} backed by the real `weaviate-ts-client`. */
|
|
5006
|
+
async buildSdkBackend() {
|
|
5007
|
+
let mod;
|
|
5008
|
+
try {
|
|
5009
|
+
mod = await importOptional("weaviate-ts-client");
|
|
5010
|
+
} catch {
|
|
5011
|
+
throw new Error(
|
|
5012
|
+
'Weaviate store requires the "weaviate-ts-client" package. Install it, or pass a custom `backend`.'
|
|
5013
|
+
);
|
|
5014
|
+
}
|
|
5015
|
+
const weaviate = mod.default ?? mod;
|
|
5016
|
+
const u = new URL(this.url);
|
|
5017
|
+
const client = weaviate.client({
|
|
5018
|
+
scheme: u.protocol.replace(":", ""),
|
|
5019
|
+
host: u.host,
|
|
5020
|
+
apiKey: this.apiKey && weaviate.ApiKey ? new weaviate.ApiKey(this.apiKey) : void 0
|
|
5021
|
+
});
|
|
5022
|
+
return new WeaviateSdkBackend(client);
|
|
5023
|
+
}
|
|
5024
|
+
async upsert(records, options) {
|
|
5025
|
+
await this.ensureInitialized();
|
|
5026
|
+
const start = performance.now();
|
|
5027
|
+
const batchSize = options?.batchSize ?? 100;
|
|
5028
|
+
const upsertedIds = [];
|
|
5029
|
+
const errors = [];
|
|
5030
|
+
let completed = 0;
|
|
5031
|
+
for (const group of batch(records, batchSize)) {
|
|
5032
|
+
try {
|
|
5033
|
+
await this.backend.upsert(
|
|
5034
|
+
this.className,
|
|
5035
|
+
group.map((r) => ({
|
|
5036
|
+
id: r.id,
|
|
5037
|
+
vector: Array.from(r.vector),
|
|
5038
|
+
properties: { ...r.metadata ?? {}, text: r.text ?? "" }
|
|
5039
|
+
}))
|
|
5040
|
+
);
|
|
5041
|
+
upsertedIds.push(...group.map((r) => r.id));
|
|
5042
|
+
} catch (e) {
|
|
5043
|
+
for (const r of group)
|
|
5044
|
+
errors.push({ id: r.id, error: e.message });
|
|
5045
|
+
}
|
|
5046
|
+
completed += group.length;
|
|
5047
|
+
options?.onProgress?.({ completed, total: records.length });
|
|
5048
|
+
}
|
|
5049
|
+
return {
|
|
5050
|
+
upsertedIds,
|
|
5051
|
+
upsertedCount: upsertedIds.length,
|
|
5052
|
+
errors,
|
|
5053
|
+
durationMs: performance.now() - start
|
|
5054
|
+
};
|
|
5055
|
+
}
|
|
5056
|
+
async query(vector, options) {
|
|
5057
|
+
await this.ensureInitialized();
|
|
5058
|
+
const start = performance.now();
|
|
5059
|
+
const topK = options?.topK ?? 10;
|
|
5060
|
+
const hits = await this.backend.nearVector(
|
|
5061
|
+
this.className,
|
|
5062
|
+
Array.from(vector),
|
|
5063
|
+
topK,
|
|
5064
|
+
options?.filter
|
|
5065
|
+
);
|
|
5066
|
+
const matches = hits.map((h) => ({
|
|
5067
|
+
id: h.id,
|
|
5068
|
+
text: h.properties.text ?? "",
|
|
5069
|
+
score: h.score,
|
|
5070
|
+
metadata: h.properties
|
|
5071
|
+
})).filter(
|
|
5072
|
+
(m) => options?.minScore === void 0 || m.score >= options.minScore
|
|
5073
|
+
);
|
|
5074
|
+
return {
|
|
5075
|
+
matches,
|
|
5076
|
+
namespace: this.className,
|
|
5077
|
+
durationMs: performance.now() - start
|
|
5078
|
+
};
|
|
5079
|
+
}
|
|
5080
|
+
async delete(ids, _options) {
|
|
5081
|
+
await this.ensureInitialized();
|
|
5082
|
+
const start = performance.now();
|
|
5083
|
+
const deleted = await this.backend.deleteByIds(this.className, ids);
|
|
5084
|
+
return {
|
|
5085
|
+
deletedCount: deleted,
|
|
5086
|
+
requestedCount: ids.length,
|
|
5087
|
+
countExact: true,
|
|
5088
|
+
durationMs: performance.now() - start
|
|
5089
|
+
};
|
|
5090
|
+
}
|
|
5091
|
+
async deleteAll(_options) {
|
|
5092
|
+
await this.ensureInitialized();
|
|
5093
|
+
const start = performance.now();
|
|
5094
|
+
const deleted = await this.backend.deleteAll(this.className);
|
|
5095
|
+
return {
|
|
5096
|
+
deletedCount: deleted,
|
|
5097
|
+
requestedCount: deleted,
|
|
5098
|
+
countExact: true,
|
|
5099
|
+
durationMs: performance.now() - start
|
|
5100
|
+
};
|
|
5101
|
+
}
|
|
5102
|
+
async getStats() {
|
|
5103
|
+
await this.ensureInitialized();
|
|
5104
|
+
return {
|
|
5105
|
+
type: this.storeType,
|
|
5106
|
+
vectorCount: await this.backend.count(this.className),
|
|
5107
|
+
namespaceCount: 1,
|
|
5108
|
+
dimensions: this.dimensions ?? 0,
|
|
5109
|
+
metric: this.metric,
|
|
5110
|
+
lastUpdated: Date.now()
|
|
5111
|
+
};
|
|
5112
|
+
}
|
|
5113
|
+
async checkHealth() {
|
|
5114
|
+
const start = performance.now();
|
|
5115
|
+
try {
|
|
5116
|
+
await this.ensureInitialized();
|
|
5117
|
+
await this.backend.ping();
|
|
5118
|
+
return {
|
|
5119
|
+
healthy: true,
|
|
5120
|
+
latencyMs: performance.now() - start,
|
|
5121
|
+
lastCheck: Date.now()
|
|
5122
|
+
};
|
|
5123
|
+
} catch (e) {
|
|
5124
|
+
return {
|
|
5125
|
+
healthy: false,
|
|
5126
|
+
latencyMs: performance.now() - start,
|
|
5127
|
+
lastCheck: Date.now(),
|
|
5128
|
+
error: e.message
|
|
5129
|
+
};
|
|
5130
|
+
}
|
|
5131
|
+
}
|
|
5132
|
+
close() {
|
|
5133
|
+
this.initialized = false;
|
|
5134
|
+
return Promise.resolve();
|
|
5135
|
+
}
|
|
5136
|
+
};
|
|
5137
|
+
var WeaviateSdkBackend = class {
|
|
5138
|
+
constructor(client) {
|
|
5139
|
+
this.client = client;
|
|
5140
|
+
}
|
|
5141
|
+
async ensureClass(className, _dimensions) {
|
|
5142
|
+
const schema = await this.client.schema.getter().do();
|
|
5143
|
+
if (schema.classes?.some((c) => c.class === className)) return;
|
|
5144
|
+
await this.client.schema.classCreator().withClass({ class: className, vectorizer: "none" }).do();
|
|
5145
|
+
}
|
|
5146
|
+
async upsert(className, objects) {
|
|
5147
|
+
let batcher = this.client.batch.objectsBatcher();
|
|
5148
|
+
for (const o of objects) {
|
|
5149
|
+
batcher = batcher.withObject({
|
|
5150
|
+
class: className,
|
|
5151
|
+
id: o.id,
|
|
5152
|
+
vector: o.vector,
|
|
5153
|
+
properties: o.properties
|
|
5154
|
+
});
|
|
5155
|
+
}
|
|
5156
|
+
await batcher.do();
|
|
5157
|
+
}
|
|
5158
|
+
async nearVector(className, vector, limit, filter) {
|
|
5159
|
+
let q = this.client.graphql.get().withClassName(className).withFields("_additional { id certainty } text").withNearVector({ vector }).withLimit(limit);
|
|
5160
|
+
if (filter) q = q.withWhere(this.toWhere(filter));
|
|
5161
|
+
const res = await q.do();
|
|
5162
|
+
const rows = res.data?.Get?.[className] ?? [];
|
|
5163
|
+
return rows.map((row) => {
|
|
5164
|
+
const additional = row._additional;
|
|
5165
|
+
const { _additional, ...properties } = row;
|
|
5166
|
+
void _additional;
|
|
5167
|
+
return {
|
|
5168
|
+
id: additional.id,
|
|
5169
|
+
score: additional.certainty ?? 0,
|
|
5170
|
+
properties
|
|
5171
|
+
};
|
|
5172
|
+
});
|
|
5173
|
+
}
|
|
5174
|
+
async deleteByIds(className, ids) {
|
|
5175
|
+
let n = 0;
|
|
5176
|
+
for (const id of ids) {
|
|
5177
|
+
await this.client.data.deleter().withClassName(className).withId(id).do();
|
|
5178
|
+
n++;
|
|
5179
|
+
}
|
|
5180
|
+
return n;
|
|
5181
|
+
}
|
|
5182
|
+
async deleteAll(className) {
|
|
5183
|
+
const count = await this.count(className);
|
|
5184
|
+
await this.client.schema.classCreator().withClass({ class: className, vectorizer: "none" }).do().catch(() => void 0);
|
|
5185
|
+
return count;
|
|
5186
|
+
}
|
|
5187
|
+
async count(className) {
|
|
5188
|
+
const res = await this.client.graphql.aggregate().withClassName(className).withFields("meta { count }").do();
|
|
5189
|
+
const agg = res.data?.Aggregate?.[className] ?? [];
|
|
5190
|
+
return agg[0]?.meta?.count ?? 0;
|
|
5191
|
+
}
|
|
5192
|
+
async ping() {
|
|
5193
|
+
await this.client.misc.liveChecker().do();
|
|
5194
|
+
}
|
|
5195
|
+
toWhere(filter) {
|
|
5196
|
+
const operands = Object.entries(filter).map(([path, value]) => ({
|
|
5197
|
+
path: [path],
|
|
5198
|
+
operator: "Equal",
|
|
5199
|
+
...typeof value === "number" ? { valueNumber: value } : typeof value === "boolean" ? { valueBoolean: value } : { valueText: String(value) }
|
|
5200
|
+
}));
|
|
5201
|
+
return operands.length === 1 ? operands[0] : { operator: "And", operands };
|
|
5202
|
+
}
|
|
5203
|
+
};
|
|
5204
|
+
|
|
5205
|
+
// src/stores/MilvusStore.ts
|
|
5206
|
+
var MilvusStore = class extends BaseStore {
|
|
5207
|
+
storeType = "milvus";
|
|
5208
|
+
backend;
|
|
5209
|
+
injectedBackend;
|
|
5210
|
+
collection;
|
|
5211
|
+
milvusConfig;
|
|
5212
|
+
initialized = false;
|
|
5213
|
+
constructor(config) {
|
|
5214
|
+
super(config);
|
|
5215
|
+
if (!config.url) throw new Error("Milvus store requires a `url`");
|
|
5216
|
+
if (!config.collectionName) {
|
|
5217
|
+
throw new Error("Milvus store requires a `collectionName`");
|
|
5218
|
+
}
|
|
5219
|
+
if (!config.dimensions) {
|
|
5220
|
+
throw new Error("Milvus store requires `dimensions`");
|
|
5221
|
+
}
|
|
5222
|
+
this.milvusConfig = config;
|
|
5223
|
+
this.collection = config.collectionName;
|
|
5224
|
+
this.injectedBackend = config.backend;
|
|
5225
|
+
}
|
|
5226
|
+
/** Milvus metric type string for the configured distance metric. */
|
|
5227
|
+
get metricType() {
|
|
5228
|
+
switch (this.metric) {
|
|
5229
|
+
case "euclidean":
|
|
5230
|
+
return "L2";
|
|
5231
|
+
case "dot_product":
|
|
5232
|
+
return "IP";
|
|
5233
|
+
case "cosine":
|
|
5234
|
+
default:
|
|
5235
|
+
return "COSINE";
|
|
5236
|
+
}
|
|
5237
|
+
}
|
|
5238
|
+
async init() {
|
|
5239
|
+
if (this.initialized) return;
|
|
5240
|
+
this.backend = this.injectedBackend ?? await this.buildSdkBackend();
|
|
5241
|
+
await this.backend.ensureCollection(
|
|
5242
|
+
this.collection,
|
|
5243
|
+
this.dimensions,
|
|
5244
|
+
this.metricType
|
|
5245
|
+
);
|
|
5246
|
+
this.initialized = true;
|
|
5247
|
+
}
|
|
5248
|
+
async ensureInitialized() {
|
|
5249
|
+
if (!this.initialized) await this.init();
|
|
5250
|
+
}
|
|
5251
|
+
async buildSdkBackend() {
|
|
5252
|
+
let mod;
|
|
5253
|
+
try {
|
|
5254
|
+
mod = await importOptional("@zilliz/milvus2-sdk-node");
|
|
5255
|
+
} catch {
|
|
5256
|
+
throw new Error(
|
|
5257
|
+
'Milvus store requires the "@zilliz/milvus2-sdk-node" package. Install it, or pass a custom `backend`.'
|
|
5258
|
+
);
|
|
5259
|
+
}
|
|
5260
|
+
const sdk = mod;
|
|
5261
|
+
const client = new sdk.MilvusClient({
|
|
5262
|
+
address: this.milvusConfig.url,
|
|
5263
|
+
username: this.milvusConfig.username,
|
|
5264
|
+
password: this.milvusConfig.password
|
|
5265
|
+
});
|
|
5266
|
+
return new MilvusSdkBackend(client);
|
|
5267
|
+
}
|
|
5268
|
+
async upsert(records, options) {
|
|
5269
|
+
await this.ensureInitialized();
|
|
5270
|
+
const start = performance.now();
|
|
5271
|
+
const batchSize = options?.batchSize ?? 100;
|
|
5272
|
+
const upsertedIds = [];
|
|
5273
|
+
const errors = [];
|
|
5274
|
+
let completed = 0;
|
|
5275
|
+
for (const group of batch(records, batchSize)) {
|
|
5276
|
+
try {
|
|
5277
|
+
await this.backend.upsert(
|
|
5278
|
+
this.collection,
|
|
5279
|
+
group.map((r) => ({
|
|
5280
|
+
id: r.id,
|
|
5281
|
+
vector: Array.from(r.vector),
|
|
5282
|
+
text: r.text ?? "",
|
|
5283
|
+
metadata: r.metadata ?? {}
|
|
5284
|
+
}))
|
|
5285
|
+
);
|
|
5286
|
+
upsertedIds.push(...group.map((r) => r.id));
|
|
5287
|
+
} catch (e) {
|
|
5288
|
+
for (const r of group)
|
|
5289
|
+
errors.push({ id: r.id, error: e.message });
|
|
5290
|
+
}
|
|
5291
|
+
completed += group.length;
|
|
5292
|
+
options?.onProgress?.({ completed, total: records.length });
|
|
5293
|
+
}
|
|
5294
|
+
return {
|
|
5295
|
+
upsertedIds,
|
|
5296
|
+
upsertedCount: upsertedIds.length,
|
|
5297
|
+
errors,
|
|
5298
|
+
durationMs: performance.now() - start
|
|
5299
|
+
};
|
|
5300
|
+
}
|
|
5301
|
+
async query(vector, options) {
|
|
5302
|
+
await this.ensureInitialized();
|
|
5303
|
+
const start = performance.now();
|
|
5304
|
+
const topK = options?.topK ?? 10;
|
|
5305
|
+
const hits = await this.backend.search(
|
|
5306
|
+
this.collection,
|
|
5307
|
+
Array.from(vector),
|
|
5308
|
+
topK,
|
|
5309
|
+
options?.filter ? this.toExpr(options.filter) : void 0
|
|
5310
|
+
);
|
|
5311
|
+
const matches = hits.map((h) => ({
|
|
5312
|
+
id: h.id,
|
|
5313
|
+
text: h.text,
|
|
5314
|
+
score: h.score,
|
|
5315
|
+
metadata: h.metadata
|
|
5316
|
+
})).filter(
|
|
5317
|
+
(m) => options?.minScore === void 0 || m.score >= options.minScore
|
|
5318
|
+
);
|
|
5319
|
+
return {
|
|
5320
|
+
matches,
|
|
5321
|
+
namespace: this.collection,
|
|
5322
|
+
durationMs: performance.now() - start
|
|
5323
|
+
};
|
|
5324
|
+
}
|
|
5325
|
+
async delete(ids, _options) {
|
|
5326
|
+
await this.ensureInitialized();
|
|
5327
|
+
const start = performance.now();
|
|
5328
|
+
const deleted = await this.backend.deleteByIds(this.collection, ids);
|
|
5329
|
+
return {
|
|
5330
|
+
deletedCount: deleted,
|
|
5331
|
+
requestedCount: ids.length,
|
|
5332
|
+
countExact: true,
|
|
5333
|
+
durationMs: performance.now() - start
|
|
5334
|
+
};
|
|
5335
|
+
}
|
|
5336
|
+
async deleteAll(_options) {
|
|
5337
|
+
await this.ensureInitialized();
|
|
5338
|
+
const start = performance.now();
|
|
5339
|
+
const deleted = await this.backend.deleteAll(this.collection);
|
|
5340
|
+
return {
|
|
5341
|
+
deletedCount: deleted,
|
|
5342
|
+
requestedCount: deleted,
|
|
5343
|
+
countExact: true,
|
|
5344
|
+
durationMs: performance.now() - start
|
|
5345
|
+
};
|
|
5346
|
+
}
|
|
5347
|
+
async getStats() {
|
|
5348
|
+
await this.ensureInitialized();
|
|
5349
|
+
return {
|
|
5350
|
+
type: this.storeType,
|
|
5351
|
+
vectorCount: await this.backend.count(this.collection),
|
|
5352
|
+
namespaceCount: 1,
|
|
5353
|
+
dimensions: this.dimensions ?? 0,
|
|
5354
|
+
metric: this.metric,
|
|
5355
|
+
lastUpdated: Date.now()
|
|
5356
|
+
};
|
|
5357
|
+
}
|
|
5358
|
+
async checkHealth() {
|
|
5359
|
+
const start = performance.now();
|
|
5360
|
+
try {
|
|
5361
|
+
await this.ensureInitialized();
|
|
5362
|
+
await this.backend.ping();
|
|
5363
|
+
return {
|
|
5364
|
+
healthy: true,
|
|
5365
|
+
latencyMs: performance.now() - start,
|
|
5366
|
+
lastCheck: Date.now()
|
|
5367
|
+
};
|
|
5368
|
+
} catch (e) {
|
|
5369
|
+
return {
|
|
5370
|
+
healthy: false,
|
|
5371
|
+
latencyMs: performance.now() - start,
|
|
5372
|
+
lastCheck: Date.now(),
|
|
5373
|
+
error: e.message
|
|
5374
|
+
};
|
|
5375
|
+
}
|
|
5376
|
+
}
|
|
5377
|
+
close() {
|
|
5378
|
+
this.initialized = false;
|
|
5379
|
+
return Promise.resolve();
|
|
5380
|
+
}
|
|
5381
|
+
/** Translate a flat equality filter to a Milvus boolean expression. */
|
|
5382
|
+
toExpr(filter) {
|
|
5383
|
+
return Object.entries(filter).map(
|
|
5384
|
+
([k, v]) => typeof v === "number" ? `metadata["${k}"] == ${v}` : `metadata["${k}"] == "${String(v)}"`
|
|
5385
|
+
).join(" && ");
|
|
5386
|
+
}
|
|
5387
|
+
};
|
|
5388
|
+
var MilvusSdkBackend = class {
|
|
5389
|
+
constructor(client) {
|
|
5390
|
+
this.client = client;
|
|
5391
|
+
}
|
|
5392
|
+
async ensureCollection(collection, dimensions, metric) {
|
|
5393
|
+
const has = await this.client.hasCollection({
|
|
5394
|
+
collection_name: collection
|
|
5395
|
+
});
|
|
5396
|
+
if (!has.value) {
|
|
5397
|
+
await this.client.createCollection({
|
|
5398
|
+
collection_name: collection,
|
|
5399
|
+
fields: [
|
|
5400
|
+
{
|
|
5401
|
+
name: "id",
|
|
5402
|
+
data_type: "VarChar",
|
|
5403
|
+
is_primary_key: true,
|
|
5404
|
+
max_length: 512
|
|
5405
|
+
},
|
|
5406
|
+
{ name: "vector", data_type: "FloatVector", dim: dimensions },
|
|
5407
|
+
{ name: "text", data_type: "VarChar", max_length: 65535 },
|
|
5408
|
+
{ name: "metadata", data_type: "JSON" }
|
|
5409
|
+
]
|
|
5410
|
+
});
|
|
5411
|
+
await this.client.createIndex({
|
|
5412
|
+
collection_name: collection,
|
|
5413
|
+
field_name: "vector",
|
|
5414
|
+
index_type: "HNSW",
|
|
5415
|
+
metric_type: metric,
|
|
5416
|
+
params: { M: 16, efConstruction: 200 }
|
|
5417
|
+
});
|
|
5418
|
+
}
|
|
5419
|
+
await this.client.loadCollectionSync({ collection_name: collection });
|
|
5420
|
+
}
|
|
5421
|
+
async upsert(collection, rows) {
|
|
5422
|
+
await this.client.insert({
|
|
5423
|
+
collection_name: collection,
|
|
5424
|
+
data: rows.map((r) => ({
|
|
5425
|
+
id: r.id,
|
|
5426
|
+
vector: r.vector,
|
|
5427
|
+
text: r.text,
|
|
5428
|
+
metadata: r.metadata
|
|
5429
|
+
}))
|
|
5430
|
+
});
|
|
5431
|
+
}
|
|
5432
|
+
async search(collection, vector, limit, filter) {
|
|
5433
|
+
const res = await this.client.search({
|
|
5434
|
+
collection_name: collection,
|
|
5435
|
+
data: [vector],
|
|
5436
|
+
limit,
|
|
5437
|
+
filter,
|
|
5438
|
+
output_fields: ["text", "metadata"]
|
|
5439
|
+
});
|
|
5440
|
+
return res.results.map((r) => ({
|
|
5441
|
+
id: String(r.id),
|
|
5442
|
+
score: Number(r.score),
|
|
5443
|
+
text: r.text ?? "",
|
|
5444
|
+
metadata: r.metadata ?? {}
|
|
5445
|
+
}));
|
|
5446
|
+
}
|
|
5447
|
+
async deleteByIds(collection, ids) {
|
|
5448
|
+
const list = ids.map((id) => `"${id}"`).join(", ");
|
|
5449
|
+
await this.client.deleteEntities({
|
|
5450
|
+
collection_name: collection,
|
|
5451
|
+
expr: `id in [${list}]`
|
|
5452
|
+
});
|
|
5453
|
+
return ids.length;
|
|
5454
|
+
}
|
|
5455
|
+
async deleteAll(collection) {
|
|
5456
|
+
const count = await this.count(collection);
|
|
5457
|
+
await this.client.deleteEntities({
|
|
5458
|
+
collection_name: collection,
|
|
5459
|
+
expr: 'id != ""'
|
|
5460
|
+
});
|
|
5461
|
+
return count;
|
|
5462
|
+
}
|
|
5463
|
+
async count(collection) {
|
|
5464
|
+
const stats = await this.client.getCollectionStatistics({
|
|
5465
|
+
collection_name: collection
|
|
5466
|
+
});
|
|
5467
|
+
return Number(stats.data?.row_count ?? 0);
|
|
5468
|
+
}
|
|
5469
|
+
async ping() {
|
|
5470
|
+
await this.client.hasCollection({ collection_name: "__ping__" });
|
|
5471
|
+
}
|
|
5472
|
+
};
|
|
5473
|
+
|
|
4648
5474
|
// src/stores/index.ts
|
|
4649
5475
|
function createStore(type, config) {
|
|
4650
5476
|
switch (type) {
|
|
@@ -4656,13 +5482,20 @@ function createStore(type, config) {
|
|
|
4656
5482
|
return new ChromaStore(config);
|
|
4657
5483
|
case "qdrant":
|
|
4658
5484
|
return new QdrantStore(config);
|
|
5485
|
+
case "weaviate":
|
|
5486
|
+
return new WeaviateStore(config);
|
|
5487
|
+
case "milvus":
|
|
5488
|
+
return new MilvusStore(config);
|
|
5489
|
+
case "pgvector":
|
|
5490
|
+
return new PgVectorStore(config);
|
|
4659
5491
|
default:
|
|
4660
|
-
|
|
5492
|
+
throw new Error(
|
|
5493
|
+
`Unknown vector store type "${String(type)}". Supported stores: memory, pinecone, chroma, qdrant, weaviate, milvus, pgvector.`
|
|
5494
|
+
);
|
|
4661
5495
|
}
|
|
4662
5496
|
}
|
|
4663
5497
|
|
|
4664
5498
|
// src/versioning/VersionRegistry.ts
|
|
4665
|
-
var import_nanoid2 = require("nanoid");
|
|
4666
5499
|
var import_eventemitter32 = __toESM(require("eventemitter3"));
|
|
4667
5500
|
var VersionRegistry = class extends import_eventemitter32.default {
|
|
4668
5501
|
versions = /* @__PURE__ */ new Map();
|
|
@@ -4683,7 +5516,7 @@ var VersionRegistry = class extends import_eventemitter32.default {
|
|
|
4683
5516
|
register(version) {
|
|
4684
5517
|
const newVersion = {
|
|
4685
5518
|
...version,
|
|
4686
|
-
id:
|
|
5519
|
+
id: nanoid(),
|
|
4687
5520
|
createdAt: Date.now(),
|
|
4688
5521
|
active: false,
|
|
4689
5522
|
deprecated: false
|
|
@@ -4914,7 +5747,6 @@ function createVersionRegistry(options) {
|
|
|
4914
5747
|
|
|
4915
5748
|
// src/quality/DriftDetector.ts
|
|
4916
5749
|
var import_eventemitter33 = __toESM(require("eventemitter3"));
|
|
4917
|
-
var import_nanoid3 = require("nanoid");
|
|
4918
5750
|
var DriftDetector = class extends import_eventemitter33.default {
|
|
4919
5751
|
reference = null;
|
|
4920
5752
|
config;
|
|
@@ -4949,7 +5781,7 @@ var DriftDetector = class extends import_eventemitter33.default {
|
|
|
4949
5781
|
varianceVector.push(variance(values));
|
|
4950
5782
|
}
|
|
4951
5783
|
this.reference = {
|
|
4952
|
-
id:
|
|
5784
|
+
id: nanoid(),
|
|
4953
5785
|
model,
|
|
4954
5786
|
version,
|
|
4955
5787
|
sampleCount: embeddings.length,
|
|
@@ -5124,7 +5956,7 @@ var DriftDetector = class extends import_eventemitter33.default {
|
|
|
5124
5956
|
*/
|
|
5125
5957
|
emitAlert(result) {
|
|
5126
5958
|
const alert = {
|
|
5127
|
-
id:
|
|
5959
|
+
id: nanoid(),
|
|
5128
5960
|
type: "drift_detected",
|
|
5129
5961
|
severity: result.severity,
|
|
5130
5962
|
message: `Embedding drift detected with score ${result.driftScore.toFixed(3)}`,
|