@adonis-agora/agent 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +151 -0
- package/README.md +210 -0
- package/dist/configure.d.ts +4 -3
- package/dist/configure.d.ts.map +1 -1
- package/dist/configure.js +5 -3
- package/dist/configure.js.map +1 -1
- package/dist/providers/agent_provider.d.ts +6 -4
- package/dist/providers/agent_provider.d.ts.map +1 -1
- package/dist/providers/agent_provider.js +23 -16
- package/dist/providers/agent_provider.js.map +1 -1
- package/dist/providers/mcp_provider.d.ts +22 -0
- package/dist/providers/mcp_provider.d.ts.map +1 -0
- package/dist/providers/mcp_provider.js +229 -0
- package/dist/providers/mcp_provider.js.map +1 -0
- package/dist/src/agent-deps-factory.d.ts +6 -0
- package/dist/src/agent-deps-factory.d.ts.map +1 -1
- package/dist/src/agent-deps-factory.js +5 -1
- package/dist/src/agent-deps-factory.js.map +1 -1
- package/dist/src/agent-deps.d.ts +6 -1
- package/dist/src/agent-deps.d.ts.map +1 -1
- package/dist/src/agent-deps.js.map +1 -1
- package/dist/src/agent-loop.d.ts +11 -2
- package/dist/src/agent-loop.d.ts.map +1 -1
- package/dist/src/agent-loop.js +69 -2
- package/dist/src/agent-loop.js.map +1 -1
- package/dist/src/data/tenant-scope.d.ts +16 -1
- package/dist/src/data/tenant-scope.d.ts.map +1 -1
- package/dist/src/data/tenant-scope.js +50 -7
- package/dist/src/data/tenant-scope.js.map +1 -1
- package/dist/src/define_config.d.ts +25 -9
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/governance-gate.d.ts +24 -8
- package/dist/src/governance-gate.d.ts.map +1 -1
- package/dist/src/governance-gate.js +14 -3
- package/dist/src/governance-gate.js.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/mcp/auth.d.ts +90 -0
- package/dist/src/mcp/auth.d.ts.map +1 -0
- package/dist/src/mcp/auth.js +108 -0
- package/dist/src/mcp/auth.js.map +1 -0
- package/dist/src/mcp/define_config.d.ts +64 -0
- package/dist/src/mcp/define_config.d.ts.map +1 -0
- package/dist/src/mcp/define_config.js +8 -0
- package/dist/src/mcp/define_config.js.map +1 -0
- package/dist/src/mcp/index.d.ts +7 -0
- package/dist/src/mcp/index.d.ts.map +1 -0
- package/dist/src/mcp/index.js +4 -0
- package/dist/src/mcp/index.js.map +1 -0
- package/dist/src/mcp/server.d.ts +47 -0
- package/dist/src/mcp/server.d.ts.map +1 -0
- package/dist/src/mcp/server.js +94 -0
- package/dist/src/mcp/server.js.map +1 -0
- package/dist/src/rag/chunk.d.ts +35 -0
- package/dist/src/rag/chunk.d.ts.map +1 -1
- package/dist/src/rag/chunk.js +60 -0
- package/dist/src/rag/chunk.js.map +1 -1
- package/dist/src/rag/index.d.ts +1 -1
- package/dist/src/rag/index.d.ts.map +1 -1
- package/dist/src/rag/index.js +1 -1
- package/dist/src/rag/index.js.map +1 -1
- package/dist/src/rag/memory-vector-store.d.ts +20 -1
- package/dist/src/rag/memory-vector-store.d.ts.map +1 -1
- package/dist/src/rag/memory-vector-store.js +56 -1
- package/dist/src/rag/memory-vector-store.js.map +1 -1
- package/dist/src/rag/pg-vector-store.d.ts +32 -1
- package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
- package/dist/src/rag/pg-vector-store.js +69 -0
- package/dist/src/rag/pg-vector-store.js.map +1 -1
- package/dist/src/rag/qdrant-store.d.ts +81 -2
- package/dist/src/rag/qdrant-store.d.ts.map +1 -1
- package/dist/src/rag/qdrant-store.js +148 -1
- package/dist/src/rag/qdrant-store.js.map +1 -1
- package/dist/src/rag/vector-store.d.ts +144 -0
- package/dist/src/rag/vector-store.d.ts.map +1 -1
- package/dist/src/rag/vector-store.js +73 -0
- package/dist/src/rag/vector-store.js.map +1 -1
- package/dist/stubs/config/mcp.stub +42 -0
- package/package.json +26 -10
package/dist/src/rag/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { chunkText } from './chunk.js';
|
|
2
|
-
export { documentIdOf, } from './vector-store.js';
|
|
2
|
+
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, } from './vector-store.js';
|
|
3
3
|
export { matchesFilter } from './filter.js';
|
|
4
4
|
export { MemoryVectorStore, cosineSimilarity } from './memory-vector-store.js';
|
|
5
5
|
export { EmbeddingRetriever } from './embedding-retriever.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,YAAY,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,kBAAkB,GAMnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,GAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,GAKhB,MAAM,aAAa,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Passage } from '../spi/retriever.js';
|
|
2
|
-
import { type IndexedDocument, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
|
|
2
|
+
import { type IndexedDocument, type MetadataPatch, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
|
|
3
3
|
/**
|
|
4
4
|
* An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
|
|
5
5
|
* tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
|
|
@@ -8,7 +8,26 @@ export declare class MemoryVectorStore implements VectorStore {
|
|
|
8
8
|
private readonly records;
|
|
9
9
|
upsert(records: VectorRecord[]): Promise<void>;
|
|
10
10
|
remove(documentId: string): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
13
|
+
* neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
|
|
14
|
+
* stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
|
|
15
|
+
* principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
|
|
16
|
+
* drift from the other two.
|
|
17
|
+
*/
|
|
18
|
+
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
11
19
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
20
|
+
/**
|
|
21
|
+
* {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
|
|
22
|
+
* per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
|
|
23
|
+
*/
|
|
24
|
+
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
25
|
+
/**
|
|
26
|
+
* {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
|
|
27
|
+
* Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
|
|
28
|
+
* identical argument, as the one in {@link MemoryVectorStore.search}.
|
|
29
|
+
*/
|
|
30
|
+
removeWhere(filter: Record<string, unknown>): Promise<number>;
|
|
12
31
|
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
|
|
13
32
|
}
|
|
14
33
|
/** Cosine similarity of two vectors; `0` when either has zero magnitude. Pure. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,
|
|
1
|
+
{"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAMjB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAErD,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C;;;;;;OAMG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAezE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiBjF;;;OAGG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAW1E;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAsBpF;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { matchesFilter } from './filter.js';
|
|
2
|
-
import { documentIdOf, } from './vector-store.js';
|
|
2
|
+
import { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, } from './vector-store.js';
|
|
3
3
|
/**
|
|
4
4
|
* An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
|
|
5
5
|
* tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
|
|
@@ -18,6 +18,27 @@ export class MemoryVectorStore {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
23
|
+
* neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
|
|
24
|
+
* stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
|
|
25
|
+
* principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
|
|
26
|
+
* drift from the other two.
|
|
27
|
+
*/
|
|
28
|
+
async updateMetadata(documentId, patch) {
|
|
29
|
+
if (effectivePatchKeys(patch).length === 0) {
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
let written = 0;
|
|
33
|
+
for (const [id, record] of this.records) {
|
|
34
|
+
if (documentIdOf(id) !== documentId) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
this.records.set(id, { ...record, metadata: applyMetadataPatch(record.metadata, patch) });
|
|
38
|
+
written += 1;
|
|
39
|
+
}
|
|
40
|
+
return written;
|
|
41
|
+
}
|
|
21
42
|
async listDocuments(filter) {
|
|
22
43
|
const documents = new Map();
|
|
23
44
|
for (const record of this.records.values()) {
|
|
@@ -34,6 +55,40 @@ export class MemoryVectorStore {
|
|
|
34
55
|
}
|
|
35
56
|
return [...documents.values()];
|
|
36
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
|
|
60
|
+
* per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
|
|
61
|
+
*/
|
|
62
|
+
async listDocumentIds(filter) {
|
|
63
|
+
const ids = new Set();
|
|
64
|
+
for (const record of this.records.values()) {
|
|
65
|
+
if (filter !== undefined && !matchesFilter(record.metadata, filter)) {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
ids.add(documentIdOf(record.id));
|
|
69
|
+
}
|
|
70
|
+
return [...ids];
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
|
|
74
|
+
* Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
|
|
75
|
+
* identical argument, as the one in {@link MemoryVectorStore.search}.
|
|
76
|
+
*/
|
|
77
|
+
async removeWhere(filter) {
|
|
78
|
+
assertRemovalFilter(filter);
|
|
79
|
+
if (filterDeniesAll(filter)) {
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
let removed = 0;
|
|
83
|
+
for (const [id, record] of [...this.records]) {
|
|
84
|
+
if (!matchesFilter(record.metadata, filter)) {
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
this.records.delete(id);
|
|
88
|
+
removed += 1;
|
|
89
|
+
}
|
|
90
|
+
return removed;
|
|
91
|
+
}
|
|
37
92
|
async search(embedding, options) {
|
|
38
93
|
const scored = [];
|
|
39
94
|
for (const record of this.records.values()) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,
|
|
1
|
+
{"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAML,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAkB,EAAE,KAAoB;QAC3D,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgC;QAClD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;oBAChB,EAAE;oBACF,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,MAAgC;QACpD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,MAA+B;QAC/C,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,OAA4B;QAC5D,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,2FAA2F;YAC3F,4DAA4D;YAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/D,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK;gBACL,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;QACvB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;QACzB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;AACnD,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { EmbeddingProvider } from '../spi/embedding-provider.js';
|
|
|
2
2
|
import type { Passage } from '../spi/retriever.js';
|
|
3
3
|
import type { LucidDatabaseLike } from '../stores/lucid.js';
|
|
4
4
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
5
|
-
import type { IndexedDocument, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
5
|
+
import type { IndexedDocument, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
6
6
|
/**
|
|
7
7
|
* The similarity metric — which pgvector distance operator ranks results (and which index opclass the
|
|
8
8
|
* migration should build). `cosine` (the default) is scale-invariant and the right choice for most
|
|
@@ -71,7 +71,38 @@ export declare class PgVectorStore implements VectorStore {
|
|
|
71
71
|
schemaStatements(): string[];
|
|
72
72
|
upsert(records: VectorRecord[]): Promise<void>;
|
|
73
73
|
remove(documentId: string): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
|
|
76
|
+
* {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
|
|
77
|
+
* into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
|
|
78
|
+
* at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
|
|
79
|
+
* removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
|
|
80
|
+
* lost-update window between a `SELECT` and an `UPDATE`.
|
|
81
|
+
*
|
|
82
|
+
* The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
|
|
83
|
+
* this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
|
|
84
|
+
* pgvector never re-parses a vector literal either.
|
|
85
|
+
*
|
|
86
|
+
* `RETURNING` the id column yields the chunk count without a second query.
|
|
87
|
+
*/
|
|
88
|
+
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
74
89
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
90
|
+
/**
|
|
91
|
+
* {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
|
|
92
|
+
* {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
|
|
93
|
+
* transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
|
|
94
|
+
* The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
|
|
95
|
+
*/
|
|
96
|
+
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
97
|
+
/**
|
|
98
|
+
* {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
|
|
99
|
+
* same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
|
|
100
|
+
*
|
|
101
|
+
* Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
|
|
102
|
+
* by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
|
|
103
|
+
* clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
|
|
104
|
+
*/
|
|
105
|
+
removeWhere(filter: Record<string, unknown>): Promise<number>;
|
|
75
106
|
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
|
|
76
107
|
}
|
|
77
108
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAyFD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAQ7C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAPrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;gBAG7B,EAAE,EAAE,iBAAiB,EACtC,OAAO,GAAE,oBAAyB;IAgBpC;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAMnC,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE;IAgBtB,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C;;;;;;;;;;;;;OAaG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBzE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoBjF;;;;;OAKG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa1E;;;;;;;OAOG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAa7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CA+CpF;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;gBAE3C,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa;CAG9D"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
2
|
+
import { assertRemovalFilter, effectivePatchKeys, filterDeniesAll } from './vector-store.js';
|
|
2
3
|
const METRICS = {
|
|
3
4
|
cosine: {
|
|
4
5
|
operator: '<=>',
|
|
@@ -203,6 +204,42 @@ export class PgVectorStore {
|
|
|
203
204
|
documentId,
|
|
204
205
|
]);
|
|
205
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
|
|
209
|
+
* {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
|
|
210
|
+
* into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
|
|
211
|
+
* at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
|
|
212
|
+
* removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
|
|
213
|
+
* lost-update window between a `SELECT` and an `UPDATE`.
|
|
214
|
+
*
|
|
215
|
+
* The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
|
|
216
|
+
* this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
|
|
217
|
+
* pgvector never re-parses a vector literal either.
|
|
218
|
+
*
|
|
219
|
+
* `RETURNING` the id column yields the chunk count without a second query.
|
|
220
|
+
*/
|
|
221
|
+
async updateMetadata(documentId, patch) {
|
|
222
|
+
const keys = effectivePatchKeys(patch);
|
|
223
|
+
if (keys.length === 0) {
|
|
224
|
+
return 0;
|
|
225
|
+
}
|
|
226
|
+
const set = {};
|
|
227
|
+
const removed = [];
|
|
228
|
+
for (const key of keys) {
|
|
229
|
+
if (patch[key] === null) {
|
|
230
|
+
removed.push(key);
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
set[key] = patch[key];
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const c = this.col;
|
|
237
|
+
const raw = await this.db.rawQuery(`UPDATE ${this.table}
|
|
238
|
+
SET ${c.metadata} = (COALESCE(${c.metadata}, '{}'::jsonb) || ?::jsonb) - ?::text[]
|
|
239
|
+
WHERE ${documentIdExpr(c.id)} = ?
|
|
240
|
+
RETURNING ${c.id} AS id`, [JSON.stringify(set), removed, documentId]);
|
|
241
|
+
return normalizeRows(raw).length;
|
|
242
|
+
}
|
|
206
243
|
async listDocuments(filter) {
|
|
207
244
|
const c = this.col;
|
|
208
245
|
const docExpr = documentIdExpr(c.id);
|
|
@@ -219,6 +256,38 @@ export class PgVectorStore {
|
|
|
219
256
|
};
|
|
220
257
|
});
|
|
221
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
|
|
261
|
+
* {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
|
|
262
|
+
* transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
|
|
263
|
+
* The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
|
|
264
|
+
*/
|
|
265
|
+
async listDocumentIds(filter) {
|
|
266
|
+
const docExpr = documentIdExpr(this.col.id);
|
|
267
|
+
const where = buildMetadataWhere(filter, this.col.metadata);
|
|
268
|
+
const raw = await this.db.rawQuery(`SELECT DISTINCT ${docExpr} AS doc_id
|
|
269
|
+
FROM ${this.table}
|
|
270
|
+
${where.sql}
|
|
271
|
+
ORDER BY doc_id`, where.bindings);
|
|
272
|
+
return normalizeRows(raw).map((row) => String(row.doc_id));
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
|
|
276
|
+
* same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
|
|
277
|
+
*
|
|
278
|
+
* Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
|
|
279
|
+
* by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
|
|
280
|
+
* clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
|
|
281
|
+
*/
|
|
282
|
+
async removeWhere(filter) {
|
|
283
|
+
assertRemovalFilter(filter);
|
|
284
|
+
if (filterDeniesAll(filter)) {
|
|
285
|
+
return 0;
|
|
286
|
+
}
|
|
287
|
+
const where = buildMetadataWhere(filter, this.col.metadata);
|
|
288
|
+
const raw = await this.db.rawQuery(`DELETE FROM ${this.table} ${where.sql} RETURNING ${this.col.id} AS id`, where.bindings);
|
|
289
|
+
return normalizeRows(raw).length;
|
|
290
|
+
}
|
|
222
291
|
async search(embedding, options) {
|
|
223
292
|
const c = this.col;
|
|
224
293
|
const vector = toVectorLiteral(embedding);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AA4D7F,MAAM,OAAO,GAAuC;IAClD,MAAM,EAAE;QACN,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,mBAAmB;QAC5B,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,GAAG,iBAAiB;KAC7C;IACD,EAAE,EAAE;QACF,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,eAAe;QACxB,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,iBAAiB;KAC1C;IACD,KAAK,EAAE;QACL,kGAAkG;QAClG,iGAAiG;QACjG,QAAQ,EAAE,KAAK;QACf,OAAO,EAAE,eAAe;QACxB,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,iBAAiB;KAC1C;CACF,CAAC;AAEF,+GAA+G;AAC/G,MAAM,UAAU,GAAG,0BAA0B,CAAC;AAE9C;;;;;GAKG;AACH,SAAS,gBAAgB,CAAC,IAAY,EAAE,IAAY;IAClD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,WAAW,IAAI,KAAK,IAAI,uCAAuC,UAAU,CAAC,MAAM,EAAE,CACnF,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,eAAe,CAAC,SAAmB;IACjD,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,SAAS,kBAAkB,CACzB,MAA2C,EAC3C,cAAsB;IAEtB,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACnC,CAAC;IACD,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAClD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,2FAA2F;YAC3F,0EAA0E;YAC1E,OAAO,CAAC,IAAI,CACV,2CAA2C,cAAc,iBAAiB;gBACxE,QAAQ,cAAc,8BAA8B,cAAc,sBAAsB,CAC3F,CAAC;YACF,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,GAAG,cAAc,cAAc,CAAC,CAAC;QAC9C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,EAAE,GAAG,EAAE,SAAS,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC;AAC7D,CAAC;AAED,wGAAwG;AACxG,SAAS,aAAa,CAAC,GAAY;IACjC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5C,OAAO,GAAG,CAAC,CAAC,CAA8B,CAAC;QAC7C,CAAC;QACD,OAAO,GAAgC,CAAC;IAC1C,CAAC;IACD,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAE,GAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/F,OAAQ,GAA2C,CAAC,IAAI,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wGAAwG;AACxG,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC5D,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAgC,CAAC;IACvE,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAA4B,CAAC;QACtD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,SAAS,cAAc,CAAC,GAAW;IACjC,OAAO,kBAAkB,GAAG,mBAAmB,CAAC;AAClD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,aAAa;IAQL;IAPF,KAAK,CAAS;IACd,SAAS,CAAS;IAClB,MAAM,CAAa;IACnB,UAAU,CAAiB;IAC3B,GAAG,CAA4B;IAEhD,YACmB,EAAqB,EACtC,UAAgC,EAAE;QADjB,OAAE,GAAF,EAAE,CAAmB;QAGtC,IAAI,CAAC,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,KAAK,IAAI,kBAAkB,EAAE,OAAO,CAAC,CAAC;QAC5E,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;QAC3C,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,IAAI,CAAC,GAAG,GAAG;YACT,EAAE,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,IAAI,IAAI,EAAE,QAAQ,CAAC;YAClD,IAAI,EAAE,gBAAgB,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,EAAE,QAAQ,CAAC;YACxD,MAAM,EAAE,gBAAgB,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,QAAQ,CAAC;YAC9D,QAAQ,EAAE,gBAAgB,CAAC,OAAO,CAAC,QAAQ,IAAI,UAAU,EAAE,QAAQ,CAAC;YACpE,SAAS,EAAE,gBAAgB,CAAC,OAAO,CAAC,SAAS,IAAI,WAAW,EAAE,QAAQ,CAAC;SACxE,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY;QAChB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,gBAAgB,EAAE,EAAE,CAAC;YAChD,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,uGAAuG;IACvG,gBAAgB;QACd,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,OAAO;YACL,uCAAuC;YACvC,8BAA8B,IAAI,CAAC,KAAK;UACpC,CAAC,CAAC,EAAE;UACJ,CAAC,CAAC,IAAI;UACN,CAAC,CAAC,MAAM;UACR,CAAC,CAAC,QAAQ;UACV,CAAC,CAAC,SAAS,WAAW,IAAI,CAAC,SAAS;QACtC;YACF,8BAA8B,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS;aAChD,IAAI,CAAC,KAAK,gBAAgB,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG;SACvE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CACpB,eAAe,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,SAAS;;wBAEzE,CAAC,CAAC,EAAE;aACf,CAAC,CAAC,IAAI,eAAe,CAAC,CAAC,IAAI;aAC3B,CAAC,CAAC,MAAM,eAAe,CAAC,CAAC,MAAM;aAC/B,CAAC,CAAC,QAAQ,eAAe,CAAC,CAAC,QAAQ;aACnC,CAAC,CAAC,SAAS,eAAe,CAAC,CAAC,SAAS,EAAE,EAC5C;gBACE,MAAM,CAAC,EAAE;gBACT,MAAM,CAAC,IAAI;gBACX,MAAM,CAAC,MAAM,IAAI,IAAI;gBACrB,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;gBACtE,eAAe,CAAC,MAAM,CAAC,SAAS,CAAC;aAClC,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC7B,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,KAAK,UAAU,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE;YAC3F,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,KAAK,CAAC,cAAc,CAAC,UAAkB,EAAE,KAAoB;QAC3D,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;QACvC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,GAAG,GAA4B,EAAE,CAAC;QACxC,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChC,UAAU,IAAI,CAAC,KAAK;gBACV,CAAC,CAAC,QAAQ,gBAAgB,CAAC,CAAC,QAAQ;gBACpC,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC;oBAChB,CAAC,CAAC,EAAE,QAAQ,EAC1B,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,OAAO,EAAE,UAAU,CAAC,CAC3C,CAAC;QACF,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgC;QAClD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChC,uBAAuB,OAAO,KAAK,OAAO,eAAe,CAAC,CAAC,QAAQ;cAC3D,IAAI,CAAC,KAAK;SACf,KAAK,CAAC,GAAG;kBACA,OAAO,EAAE,EACrB,KAAK,CAAC,QAAQ,CACf,CAAC;QACF,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACpC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,OAAO;gBACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBACtB,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,eAAe,CAAC,MAAgC;QACpD,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChC,mBAAmB,OAAO;gBAChB,IAAI,CAAC,KAAK;WACf,KAAK,CAAC,GAAG;wBACI,EAClB,KAAK,CAAC,QAAQ,CACf,CAAC;QACF,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7D,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,WAAW,CAAC,MAA+B;QAC/C,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5D,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChC,eAAe,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,cAAc,IAAI,CAAC,GAAG,CAAC,EAAE,QAAQ,EACvE,KAAK,CAAC,QAAQ,CACf,CAAC;QACF,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,OAA4B;QAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QACnB,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;QAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC5D,gGAAgG;QAChG,gGAAgG;QAChG,mGAAmG;QACnG,mGAAmG;QACnG,uDAAuD;QACvD,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,MAAM,aAAa,GAAc,EAAE,CAAC;QACpC,IAAI,IAAI,CAAC,GAAG,KAAK,EAAE,EAAE,CAAC;YACpB,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;YAC9C,aAAa,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACnC,0FAA0F;YAC1F,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,OAAO,CAAC,CAAC;YAClC,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;QAC/C,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,kGAAkG;QAClG,gGAAgG;QAChG,iGAAiG;QACjG,MAAM,QAAQ,GAAc,CAAC,MAAM,EAAE,GAAG,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7E,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAChC,UAAU,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC,MAAM,eAAe,CAAC,CAAC,QAAQ;gBACnE,SAAS;cACX,IAAI,CAAC,KAAK;SACf,QAAQ;kBACC,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ;eACtC,EACT,QAAQ,CACT,CAAC;QACF,OAAO,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACpC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YAC1B,OAAO;gBACL,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;gBACtB,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACxB,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9E,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAChD,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,iBAAkB,SAAQ,kBAAkB;IACvD,kQAAkQ;IAClQ,YAAY,QAA2B,EAAE,KAAoB;QAC3D,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;CACF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { EmbeddingProvider } from '../spi/embedding-provider.js';
|
|
2
2
|
import type { Passage } from '../spi/retriever.js';
|
|
3
3
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
4
|
-
import type { IndexedDocument, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
4
|
+
import type { IndexedDocument, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
5
5
|
/** Similaridade → nome de distância do Qdrant. Cosine (default), Dot (inner), Euclid (l2). */
|
|
6
6
|
export type QdrantMetric = 'cosine' | 'inner' | 'l2';
|
|
7
7
|
export interface QdrantStoreOptions {
|
|
@@ -73,10 +73,12 @@ export interface QdrantClientLike {
|
|
|
73
73
|
}>;
|
|
74
74
|
delete(collection: string, args: {
|
|
75
75
|
filter: QdrantFilter;
|
|
76
|
+
wait?: boolean;
|
|
76
77
|
}): Promise<unknown>;
|
|
77
78
|
scroll(collection: string, args: {
|
|
78
79
|
filter?: QdrantFilter;
|
|
79
|
-
|
|
80
|
+
/** `true` for the whole payload, or a key list to transfer only those keys. */
|
|
81
|
+
with_payload: boolean | string[];
|
|
80
82
|
with_vector: boolean;
|
|
81
83
|
limit: number;
|
|
82
84
|
offset?: unknown;
|
|
@@ -86,6 +88,32 @@ export interface QdrantClientLike {
|
|
|
86
88
|
}[];
|
|
87
89
|
next_page_offset?: unknown;
|
|
88
90
|
}>;
|
|
91
|
+
/**
|
|
92
|
+
* Set payload keys on specific points, leaving every other key — and **the vector** — alone. Backs
|
|
93
|
+
* {@link QdrantStore.updateMetadata}; the real `@qdrant/js-client-rest` provides it.
|
|
94
|
+
*
|
|
95
|
+
* OPTIONAL here, mirroring the `expire?` precedent on this package's Redis client shim, so adding it
|
|
96
|
+
* cannot break a host that hand-rolled this structural interface. `updateMetadata` throws a named
|
|
97
|
+
* error rather than degrading if it is absent: the alternative degradation would be scroll-with-vector
|
|
98
|
+
* plus `upsert`, which rewrites the embedding — the exact thing the method promises not to do.
|
|
99
|
+
*/
|
|
100
|
+
setPayload?(collection: string, args: {
|
|
101
|
+
payload: Record<string, unknown>;
|
|
102
|
+
points: string[];
|
|
103
|
+
wait?: boolean;
|
|
104
|
+
}): Promise<unknown>;
|
|
105
|
+
/**
|
|
106
|
+
* Count the points matching a filter WITHOUT transferring them. Backs the chunk count
|
|
107
|
+
* {@link QdrantStore.removeWhere} resolves to — Qdrant's delete response carries no count, and this is
|
|
108
|
+
* an aggregate, not an enumeration: no payloads, no vectors, one cheap round trip. Optional for the
|
|
109
|
+
* same reason as {@link QdrantClientLike.setPayload}; the real `@qdrant/js-client-rest` provides it.
|
|
110
|
+
*/
|
|
111
|
+
count?(collection: string, args: {
|
|
112
|
+
filter?: QdrantFilter;
|
|
113
|
+
exact?: boolean;
|
|
114
|
+
}): Promise<{
|
|
115
|
+
count: number;
|
|
116
|
+
}>;
|
|
89
117
|
}
|
|
90
118
|
/**
|
|
91
119
|
* Traduz o filtro de metadata (`Record<string, unknown>`) para um {@link QdrantFilter}, preservando a
|
|
@@ -115,6 +143,57 @@ export declare class QdrantStore implements VectorStore {
|
|
|
115
143
|
upsert(records: VectorRecord[]): Promise<void>;
|
|
116
144
|
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
|
|
117
145
|
remove(documentId: string): Promise<void>;
|
|
146
|
+
/**
|
|
147
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
148
|
+
* neither `text` nor the **vector**.
|
|
149
|
+
*
|
|
150
|
+
* Qdrant is the one adapter here that cannot do the merge server-side. `set_payload` merges at the TOP
|
|
151
|
+
* level of a point's payload, and this store nests the caller's metadata one level down (under
|
|
152
|
+
* `metadata`, so a filter key is `metadata.<k>` — see {@link buildQdrantFilter}), so a top-level set of
|
|
153
|
+
* `metadata` replaces the whole object rather than merging into it. The merge therefore happens in JS:
|
|
154
|
+
* scroll the document's points for their current metadata, apply the shared
|
|
155
|
+
* {@link import('./vector-store.js').applyMetadataPatch}, write back. Cost: one scroll (payload only,
|
|
156
|
+
* **`with_vector: false`**, and only the two keys needed) plus one `set_payload` per distinct resulting
|
|
157
|
+
* metadata — which is ONE for any corpus built by `chunkDocuments`, since that stamps the same document
|
|
158
|
+
* metadata on every chunk.
|
|
159
|
+
*
|
|
160
|
+
* The vector is untouched by construction, not by care: `set_payload` has no vector field, and the
|
|
161
|
+
* points are never re-`upsert`ed (upsert is the call that would carry a vector). The scroll does not
|
|
162
|
+
* even fetch the vectors, so this method never has one in hand to write back.
|
|
163
|
+
*
|
|
164
|
+
* Point ids are re-derived with {@link chunkIdToPointId} from the chunk id in the payload, so the
|
|
165
|
+
* scroll needs no point id from the response and this store keeps working against a client whose
|
|
166
|
+
* `scroll` returns only payloads.
|
|
167
|
+
*
|
|
168
|
+
* `wait: true` because the returned count is a claim that the writes landed; Qdrant's default is to
|
|
169
|
+
* acknowledge before applying.
|
|
170
|
+
*/
|
|
171
|
+
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
172
|
+
/**
|
|
173
|
+
* {@link VectorStore.listDocumentIds} — the same scroll {@link QdrantStore.listDocuments} does, but
|
|
174
|
+
* transferring ONE payload key (`documentId`) instead of the whole payload, so the per-chunk metadata
|
|
175
|
+
* blob never crosses the wire or gets parsed. Qdrant has no `DISTINCT`, so the collapse still happens
|
|
176
|
+
* here; what this saves is the payload, which is the bulk of it. Filter comes from the same
|
|
177
|
+
* {@link buildQdrantFilter} `search` uses.
|
|
178
|
+
*/
|
|
179
|
+
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
180
|
+
/**
|
|
181
|
+
* {@link VectorStore.removeWhere} — ONE filtered `delete`, no enumeration: Qdrant deletes by filter
|
|
182
|
+
* server-side, so this never scrolls the points it is about to remove.
|
|
183
|
+
*
|
|
184
|
+
* Filter parity with {@link QdrantStore.search} is by construction — the same
|
|
185
|
+
* {@link buildQdrantFilter} call produces the predicate for both, so `metadata.<k>` keying, scalar
|
|
186
|
+
* `match.value`, array `match.any` set-membership and the empty-array deny are the same object here as
|
|
187
|
+
* there.
|
|
188
|
+
*
|
|
189
|
+
* It DOES cost one extra round trip that memory and pgvector do not: Qdrant's delete response carries
|
|
190
|
+
* no count, so the chunk count comes from a preceding `count` (an aggregate — no payloads, no vectors,
|
|
191
|
+
* not an enumeration). Consequence worth knowing: the count is taken just before the delete, so under
|
|
192
|
+
* concurrent writes it is a very slightly stale count of what matched, where memory and pgvector return
|
|
193
|
+
* an exact count from the deleting statement itself. The delete is filter-scoped either way; only the
|
|
194
|
+
* reported number can drift.
|
|
195
|
+
*/
|
|
196
|
+
removeWhere(filter: Record<string, unknown>): Promise<number>;
|
|
118
197
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
119
198
|
}
|
|
120
199
|
/** Um {@link import('../spi/retriever.js').Retriever} sobre uma {@link QdrantStore}: embeda a query,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"qdrant-store.d.ts","sourceRoot":"","sources":["../../../src/rag/qdrant-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"qdrant-store.d.ts","sourceRoot":"","sources":["../../../src/rag/qdrant-store.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAQ9D,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B,8FAA8F;AAC9F,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,OAAO,GAAG,IAAI,CAAC;AAGrD,MAAM,WAAW,kBAAkB;IACjC,sDAAsD;IACtD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qGAAqG;IACrG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,iDAAiD;AACjD,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,4CAA4C;AAC5C,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE;QAAE,KAAK,EAAE,OAAO,CAAA;KAAE,GAAG;QAAE,GAAG,EAAE,OAAO,EAAE,CAAA;KAAE,GAAG;QAAE,MAAM,EAAE,OAAO,EAAE,CAAA;KAAE,CAAC;CACxE;AACD,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,EAAE,eAAe,EAAE,CAAC;CAC1B;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,cAAc,IAAI,OAAO,CAAC;QAAE,WAAW,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC,CAAC;IAC/D,gBAAgB,CACd,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE;QAAE,OAAO,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,GACtD,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC9E,KAAK,CACH,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;QACJ,KAAK,EAAE,MAAM,EAAE,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,OAAO,CAAC;QACtB,MAAM,CAAC,EAAE,YAAY,CAAC;QACtB,eAAe,CAAC,EAAE,MAAM,CAAC;KAC1B,GACA,OAAO,CAAC;QAAE,MAAM,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,EAAE,CAAA;KAAE,CAAC,CAAC;IAC/E,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE;QAAE,MAAM,EAAE,YAAY,CAAC;QAAC,IAAI,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7F,MAAM,CACJ,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;QACJ,MAAM,CAAC,EAAE,YAAY,CAAC;QACtB,+EAA+E;QAC/E,YAAY,EAAE,OAAO,GAAG,MAAM,EAAE,CAAC;QACjC,WAAW,EAAE,OAAO,CAAC;QACrB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,OAAO,CAAC;KAClB,GACA,OAAO,CAAC;QAAE,MAAM,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,EAAE,CAAC;QAAC,gBAAgB,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAC;IAC5F;;;;;;;;OAQG;IACH,UAAU,CAAC,CACT,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACjC,MAAM,EAAE,MAAM,EAAE,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;KAChB,GACA,OAAO,CAAC,OAAO,CAAC,CAAC;IACpB;;;;;OAKG;IACH,KAAK,CAAC,CACJ,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE;QAAE,MAAM,CAAC,EAAE,YAAY,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,GAC/C,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/B;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAC1C,YAAY,GAAG,SAAS,CAO1B;AASD,2EAA2E;AAC3E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CASxD;AAED;;;;;GAKG;AACH,qBAAa,WAAY,YAAW,WAAW;IAO3C,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;gBAGtB,MAAM,EAAE,gBAAgB,EACzC,OAAO,GAAE,kBAAuB;IAQlC,gFAAgF;IAC1E,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQjC,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAoB9C,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAwC7E,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAyD/E;;;;;;OAMG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA0B1E;;;;;;;;;;;;;;;OAeG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAsB7D,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;CAmClF;AAED;4DAC4D;AAC5D,qBAAa,eAAgB,SAAQ,kBAAkB;gBAEzC,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,WAAW;CAG5D"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
3
|
-
import { documentIdOf } from './vector-store.js';
|
|
3
|
+
import { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, } from './vector-store.js';
|
|
4
4
|
const DISTANCE = { cosine: 'Cosine', inner: 'Dot', l2: 'Euclid' };
|
|
5
5
|
/**
|
|
6
6
|
* Traduz o filtro de metadata (`Record<string, unknown>`) para um {@link QdrantFilter}, preservando a
|
|
@@ -127,6 +127,153 @@ export class QdrantStore {
|
|
|
127
127
|
filter: { must: [{ key: 'documentId', match: { value: documentId } }] },
|
|
128
128
|
});
|
|
129
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
132
|
+
* neither `text` nor the **vector**.
|
|
133
|
+
*
|
|
134
|
+
* Qdrant is the one adapter here that cannot do the merge server-side. `set_payload` merges at the TOP
|
|
135
|
+
* level of a point's payload, and this store nests the caller's metadata one level down (under
|
|
136
|
+
* `metadata`, so a filter key is `metadata.<k>` — see {@link buildQdrantFilter}), so a top-level set of
|
|
137
|
+
* `metadata` replaces the whole object rather than merging into it. The merge therefore happens in JS:
|
|
138
|
+
* scroll the document's points for their current metadata, apply the shared
|
|
139
|
+
* {@link import('./vector-store.js').applyMetadataPatch}, write back. Cost: one scroll (payload only,
|
|
140
|
+
* **`with_vector: false`**, and only the two keys needed) plus one `set_payload` per distinct resulting
|
|
141
|
+
* metadata — which is ONE for any corpus built by `chunkDocuments`, since that stamps the same document
|
|
142
|
+
* metadata on every chunk.
|
|
143
|
+
*
|
|
144
|
+
* The vector is untouched by construction, not by care: `set_payload` has no vector field, and the
|
|
145
|
+
* points are never re-`upsert`ed (upsert is the call that would carry a vector). The scroll does not
|
|
146
|
+
* even fetch the vectors, so this method never has one in hand to write back.
|
|
147
|
+
*
|
|
148
|
+
* Point ids are re-derived with {@link chunkIdToPointId} from the chunk id in the payload, so the
|
|
149
|
+
* scroll needs no point id from the response and this store keeps working against a client whose
|
|
150
|
+
* `scroll` returns only payloads.
|
|
151
|
+
*
|
|
152
|
+
* `wait: true` because the returned count is a claim that the writes landed; Qdrant's default is to
|
|
153
|
+
* acknowledge before applying.
|
|
154
|
+
*/
|
|
155
|
+
async updateMetadata(documentId, patch) {
|
|
156
|
+
if (effectivePatchKeys(patch).length === 0) {
|
|
157
|
+
return 0;
|
|
158
|
+
}
|
|
159
|
+
const setPayload = this.client.setPayload?.bind(this.client);
|
|
160
|
+
if (setPayload === undefined) {
|
|
161
|
+
throw new Error('QdrantStore.updateMetadata requires a client with `setPayload` (the real ' +
|
|
162
|
+
'@qdrant/js-client-rest has it). Refusing to fall back to upsert, which would rewrite the ' +
|
|
163
|
+
'embedding this method exists to preserve.');
|
|
164
|
+
}
|
|
165
|
+
// Group by the RESULTING metadata so identical outcomes share one write — the normal case.
|
|
166
|
+
const groups = new Map();
|
|
167
|
+
let written = 0;
|
|
168
|
+
let offset = undefined;
|
|
169
|
+
for (let pageCount = 0;; pageCount++) {
|
|
170
|
+
if (pageCount >= MAX_SCROLL_PAGES) {
|
|
171
|
+
throw new Error('updateMetadata: scroll excedeu MAX_SCROLL_PAGES (offset não terminou)');
|
|
172
|
+
}
|
|
173
|
+
const page = await this.client.scroll(this.collection, {
|
|
174
|
+
filter: { must: [{ key: 'documentId', match: { value: documentId } }] },
|
|
175
|
+
with_payload: ['id', 'metadata'],
|
|
176
|
+
with_vector: false,
|
|
177
|
+
limit: 256,
|
|
178
|
+
...(offset !== undefined ? { offset } : {}),
|
|
179
|
+
});
|
|
180
|
+
for (const point of page.points) {
|
|
181
|
+
const payload = point.payload ?? {};
|
|
182
|
+
const chunkId = payload.id;
|
|
183
|
+
if (chunkId === undefined || chunkId === null)
|
|
184
|
+
continue;
|
|
185
|
+
const current = payload.metadata;
|
|
186
|
+
const next = applyMetadataPatch(current !== undefined && current !== null
|
|
187
|
+
? current
|
|
188
|
+
: undefined, patch);
|
|
189
|
+
const key = JSON.stringify(next);
|
|
190
|
+
const group = groups.get(key) ?? { metadata: next, points: [] };
|
|
191
|
+
group.points.push(chunkIdToPointId(String(chunkId)));
|
|
192
|
+
groups.set(key, group);
|
|
193
|
+
written += 1;
|
|
194
|
+
}
|
|
195
|
+
if (page.next_page_offset === undefined || page.next_page_offset === null)
|
|
196
|
+
break;
|
|
197
|
+
offset = page.next_page_offset;
|
|
198
|
+
}
|
|
199
|
+
for (const group of groups.values()) {
|
|
200
|
+
await setPayload(this.collection, {
|
|
201
|
+
payload: { metadata: group.metadata },
|
|
202
|
+
points: group.points,
|
|
203
|
+
wait: true,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
return written;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* {@link VectorStore.listDocumentIds} — the same scroll {@link QdrantStore.listDocuments} does, but
|
|
210
|
+
* transferring ONE payload key (`documentId`) instead of the whole payload, so the per-chunk metadata
|
|
211
|
+
* blob never crosses the wire or gets parsed. Qdrant has no `DISTINCT`, so the collapse still happens
|
|
212
|
+
* here; what this saves is the payload, which is the bulk of it. Filter comes from the same
|
|
213
|
+
* {@link buildQdrantFilter} `search` uses.
|
|
214
|
+
*/
|
|
215
|
+
async listDocumentIds(filter) {
|
|
216
|
+
const qFilter = buildQdrantFilter(filter);
|
|
217
|
+
const ids = new Set();
|
|
218
|
+
let offset = undefined;
|
|
219
|
+
for (let pageCount = 0;; pageCount++) {
|
|
220
|
+
if (pageCount >= MAX_SCROLL_PAGES) {
|
|
221
|
+
throw new Error('listDocumentIds: scroll excedeu MAX_SCROLL_PAGES (offset não terminou)');
|
|
222
|
+
}
|
|
223
|
+
const page = await this.client.scroll(this.collection, {
|
|
224
|
+
with_payload: ['documentId'],
|
|
225
|
+
with_vector: false,
|
|
226
|
+
limit: 256,
|
|
227
|
+
...(qFilter !== undefined ? { filter: qFilter } : {}),
|
|
228
|
+
...(offset !== undefined ? { offset } : {}),
|
|
229
|
+
});
|
|
230
|
+
for (const point of page.points) {
|
|
231
|
+
const docId = point.payload?.documentId;
|
|
232
|
+
if (docId === undefined || docId === null)
|
|
233
|
+
continue;
|
|
234
|
+
ids.add(String(docId));
|
|
235
|
+
}
|
|
236
|
+
if (page.next_page_offset === undefined || page.next_page_offset === null)
|
|
237
|
+
break;
|
|
238
|
+
offset = page.next_page_offset;
|
|
239
|
+
}
|
|
240
|
+
return [...ids];
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* {@link VectorStore.removeWhere} — ONE filtered `delete`, no enumeration: Qdrant deletes by filter
|
|
244
|
+
* server-side, so this never scrolls the points it is about to remove.
|
|
245
|
+
*
|
|
246
|
+
* Filter parity with {@link QdrantStore.search} is by construction — the same
|
|
247
|
+
* {@link buildQdrantFilter} call produces the predicate for both, so `metadata.<k>` keying, scalar
|
|
248
|
+
* `match.value`, array `match.any` set-membership and the empty-array deny are the same object here as
|
|
249
|
+
* there.
|
|
250
|
+
*
|
|
251
|
+
* It DOES cost one extra round trip that memory and pgvector do not: Qdrant's delete response carries
|
|
252
|
+
* no count, so the chunk count comes from a preceding `count` (an aggregate — no payloads, no vectors,
|
|
253
|
+
* not an enumeration). Consequence worth knowing: the count is taken just before the delete, so under
|
|
254
|
+
* concurrent writes it is a very slightly stale count of what matched, where memory and pgvector return
|
|
255
|
+
* an exact count from the deleting statement itself. The delete is filter-scoped either way; only the
|
|
256
|
+
* reported number can drift.
|
|
257
|
+
*/
|
|
258
|
+
async removeWhere(filter) {
|
|
259
|
+
assertRemovalFilter(filter);
|
|
260
|
+
if (filterDeniesAll(filter)) {
|
|
261
|
+
return 0;
|
|
262
|
+
}
|
|
263
|
+
const count = this.client.count?.bind(this.client);
|
|
264
|
+
if (count === undefined) {
|
|
265
|
+
throw new Error('QdrantStore.removeWhere requires a client with `count` (the real @qdrant/js-client-rest has ' +
|
|
266
|
+
'it) to report how many chunks it removed, because Qdrant’s delete response carries no count.');
|
|
267
|
+
}
|
|
268
|
+
const qFilter = buildQdrantFilter(filter);
|
|
269
|
+
if (qFilter === undefined) {
|
|
270
|
+
// Unreachable: assertRemovalFilter already rejected the only input that yields undefined.
|
|
271
|
+
throw new Error('removeWhere: filtro vazio escapou do assertRemovalFilter');
|
|
272
|
+
}
|
|
273
|
+
const { count: matched } = await count(this.collection, { filter: qFilter, exact: true });
|
|
274
|
+
await this.client.delete(this.collection, { filter: qFilter, wait: true });
|
|
275
|
+
return matched;
|
|
276
|
+
}
|
|
130
277
|
async listDocuments(filter) {
|
|
131
278
|
const qFilter = buildQdrantFilter(filter);
|
|
132
279
|
const seen = new Map();
|