@semiont/vectors 0.5.24 → 0.5.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-VFNOQC4M.js → chunk-Q2LGLRYN.js} +40 -19
- package/dist/chunk-Q2LGLRYN.js.map +1 -0
- package/dist/index.d.ts +46 -37
- package/dist/index.js +23 -56
- package/dist/index.js.map +1 -1
- package/dist/qdrant-GNSEAGLI.js +7 -0
- package/package.json +3 -3
- package/dist/chunk-VFNOQC4M.js.map +0 -1
- package/dist/qdrant-A765QRQF.js +0 -7
- /package/dist/{qdrant-A765QRQF.js.map → qdrant-GNSEAGLI.js.map} +0 -0
|
@@ -4,6 +4,16 @@ function toQdrantId(input) {
|
|
|
4
4
|
const hex = createHash("md5").update(input).digest("hex");
|
|
5
5
|
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;
|
|
6
6
|
}
|
|
7
|
+
var STAMP_FIELDS = ["resourceId", "contentChecksum", "entityTypes", "machineRead"];
|
|
8
|
+
function toStamp(payload) {
|
|
9
|
+
const checksum = payload?.contentChecksum;
|
|
10
|
+
const entityTypes = payload?.entityTypes;
|
|
11
|
+
return {
|
|
12
|
+
contentChecksum: typeof checksum === "string" ? checksum : void 0,
|
|
13
|
+
entityTypes: Array.isArray(entityTypes) ? entityTypes.filter((t) => typeof t === "string") : [],
|
|
14
|
+
...payload?.machineRead ? { machineRead: true } : {}
|
|
15
|
+
};
|
|
16
|
+
}
|
|
7
17
|
var QdrantVectorStore = class {
|
|
8
18
|
client = null;
|
|
9
19
|
config;
|
|
@@ -64,7 +74,7 @@ var QdrantVectorStore = class {
|
|
|
64
74
|
} catch {
|
|
65
75
|
}
|
|
66
76
|
}
|
|
67
|
-
async upsertResourceVectors(resourceId, chunks, contentChecksum, entityTypes) {
|
|
77
|
+
async upsertResourceVectors(resourceId, chunks, contentChecksum, entityTypes, machineRead) {
|
|
68
78
|
await this.deleteResourceVectors(resourceId);
|
|
69
79
|
if (chunks.length === 0) return;
|
|
70
80
|
const points = chunks.map((chunk) => ({
|
|
@@ -75,7 +85,10 @@ var QdrantVectorStore = class {
|
|
|
75
85
|
chunkIndex: chunk.chunkIndex,
|
|
76
86
|
text: chunk.text,
|
|
77
87
|
contentChecksum,
|
|
78
|
-
entityTypes
|
|
88
|
+
entityTypes,
|
|
89
|
+
// Only stamped when true: absence is the common case and carries no
|
|
90
|
+
// claim, so a native extraction stores nothing extra.
|
|
91
|
+
...machineRead ? { machineRead: true } : {}
|
|
79
92
|
}
|
|
80
93
|
}));
|
|
81
94
|
await this.qdrant.upsert("resources", { points });
|
|
@@ -90,7 +103,8 @@ var QdrantVectorStore = class {
|
|
|
90
103
|
resourceId: String(payload.resourceId),
|
|
91
104
|
motivation: payload.motivation,
|
|
92
105
|
entityTypes: payload.entityTypes,
|
|
93
|
-
text: payload.exactText
|
|
106
|
+
text: payload.exactText,
|
|
107
|
+
...payload.machineRead ? { machineRead: true } : {}
|
|
94
108
|
}
|
|
95
109
|
}]
|
|
96
110
|
});
|
|
@@ -129,6 +143,16 @@ var QdrantVectorStore = class {
|
|
|
129
143
|
}
|
|
130
144
|
});
|
|
131
145
|
}
|
|
146
|
+
async getResourceStamp(resourceId) {
|
|
147
|
+
const page = await this.qdrant.scroll("resources", {
|
|
148
|
+
limit: 1,
|
|
149
|
+
filter: { must: [{ key: "resourceId", match: { value: String(resourceId) } }] },
|
|
150
|
+
with_payload: STAMP_FIELDS,
|
|
151
|
+
with_vector: false
|
|
152
|
+
});
|
|
153
|
+
const point = page.points[0];
|
|
154
|
+
return point ? toStamp(point.payload) : void 0;
|
|
155
|
+
}
|
|
132
156
|
async listResourceStamps() {
|
|
133
157
|
const stamps = /* @__PURE__ */ new Map();
|
|
134
158
|
let offset = void 0;
|
|
@@ -136,18 +160,13 @@ var QdrantVectorStore = class {
|
|
|
136
160
|
const page = await this.qdrant.scroll("resources", {
|
|
137
161
|
limit: 1e3,
|
|
138
162
|
offset,
|
|
139
|
-
with_payload:
|
|
163
|
+
with_payload: STAMP_FIELDS,
|
|
140
164
|
with_vector: false
|
|
141
165
|
});
|
|
142
166
|
for (const point of page.points) {
|
|
143
167
|
const rid = point.payload?.resourceId;
|
|
144
168
|
if (typeof rid !== "string" || stamps.has(rid)) continue;
|
|
145
|
-
|
|
146
|
-
const entityTypes = point.payload?.entityTypes;
|
|
147
|
-
stamps.set(rid, {
|
|
148
|
-
contentChecksum: typeof checksum === "string" ? checksum : void 0,
|
|
149
|
-
entityTypes: Array.isArray(entityTypes) ? entityTypes.filter((t) => typeof t === "string") : []
|
|
150
|
-
});
|
|
169
|
+
stamps.set(rid, toStamp(point.payload));
|
|
151
170
|
}
|
|
152
171
|
offset = page.next_page_offset ?? void 0;
|
|
153
172
|
} while (offset !== void 0 && offset !== null);
|
|
@@ -200,16 +219,16 @@ var QdrantVectorStore = class {
|
|
|
200
219
|
if (queryVectors.length === 0) return [];
|
|
201
220
|
const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });
|
|
202
221
|
const searches = queryVectors.map((vector) => ({
|
|
203
|
-
vector,
|
|
222
|
+
query: vector,
|
|
204
223
|
limit: opts.limit,
|
|
205
224
|
score_threshold: opts.scoreThreshold,
|
|
206
225
|
filter: filter ?? void 0,
|
|
207
226
|
with_payload: true
|
|
208
227
|
}));
|
|
209
|
-
const batches = await this.qdrant.
|
|
228
|
+
const batches = await this.qdrant.queryBatch("resources", { searches });
|
|
210
229
|
const bestByResource = /* @__PURE__ */ new Map();
|
|
211
230
|
for (const batch of batches) {
|
|
212
|
-
for (const r of batch) {
|
|
231
|
+
for (const r of batch.points) {
|
|
213
232
|
const payload = r.payload ?? {};
|
|
214
233
|
const tid = String(payload.resourceId);
|
|
215
234
|
const prev = bestByResource.get(tid);
|
|
@@ -224,19 +243,20 @@ var QdrantVectorStore = class {
|
|
|
224
243
|
resourceId: m.payload.resourceId,
|
|
225
244
|
annotationId: m.payload.annotationId,
|
|
226
245
|
text: m.payload.text,
|
|
227
|
-
entityTypes: m.payload.entityTypes
|
|
246
|
+
entityTypes: m.payload.entityTypes,
|
|
247
|
+
...m.payload.machineRead ? { machineRead: true } : {}
|
|
228
248
|
}));
|
|
229
249
|
}
|
|
230
250
|
async search(collection, embedding, opts) {
|
|
231
251
|
const filter = this.buildFilter(opts.filter);
|
|
232
|
-
const
|
|
233
|
-
|
|
252
|
+
const { points } = await this.qdrant.query(collection, {
|
|
253
|
+
query: embedding,
|
|
234
254
|
limit: opts.limit,
|
|
235
255
|
score_threshold: opts.scoreThreshold,
|
|
236
256
|
filter: filter ?? void 0,
|
|
237
257
|
with_payload: true
|
|
238
258
|
});
|
|
239
|
-
return
|
|
259
|
+
return points.map((r) => {
|
|
240
260
|
const payload = r.payload ?? {};
|
|
241
261
|
return {
|
|
242
262
|
id: String(r.id),
|
|
@@ -244,7 +264,8 @@ var QdrantVectorStore = class {
|
|
|
244
264
|
resourceId: payload.resourceId,
|
|
245
265
|
annotationId: payload.annotationId,
|
|
246
266
|
text: payload.text,
|
|
247
|
-
entityTypes: payload.entityTypes
|
|
267
|
+
entityTypes: payload.entityTypes,
|
|
268
|
+
...payload.machineRead ? { machineRead: true } : {}
|
|
248
269
|
};
|
|
249
270
|
});
|
|
250
271
|
}
|
|
@@ -278,4 +299,4 @@ var QdrantVectorStore = class {
|
|
|
278
299
|
export {
|
|
279
300
|
QdrantVectorStore
|
|
280
301
|
};
|
|
281
|
-
//# sourceMappingURL=chunk-
|
|
302
|
+
//# sourceMappingURL=chunk-Q2LGLRYN.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions, ResourceStamp } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\n/** The payload fields a stamp is read from — one list for the scroll and the\n * targeted read, so the two cannot drift. */\nconst STAMP_FIELDS = ['resourceId', 'contentChecksum', 'entityTypes', 'machineRead'];\n\nfunction toStamp(payload: Record<string, unknown> | null | undefined): ResourceStamp {\n const checksum = payload?.contentChecksum;\n const entityTypes = payload?.entityTypes;\n return {\n contentChecksum: typeof checksum === 'string' ? checksum : undefined,\n entityTypes: Array.isArray(entityTypes)\n ? entityTypes.filter((t): t is string => typeof t === 'string')\n : [],\n ...(payload?.machineRead ? { machineRead: true } : {}),\n };\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[], machineRead?: boolean): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n // Only stamped when true: absence is the common case and carries no\n // claim, so a native extraction stores nothing extra.\n ...(machineRead ? { machineRead: true } : {}),\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n ...(payload.machineRead ? { machineRead: true } : {}),\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n // Payload-only rewrite across the resource's points — no vectors touched,\n // no embedding involved (S13). Qdrant set_payload with a filter is a\n // no-op when the resource has no points.\n await this.qdrant.setPayload('resources', {\n payload: { entityTypes },\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async getResourceStamp(resourceId: ResourceId): Promise<ResourceStamp | undefined> {\n const page = await this.qdrant.scroll('resources', {\n limit: 1,\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_payload: STAMP_FIELDS,\n with_vector: false,\n });\n const point = page.points[0];\n return point ? toStamp(point.payload) : undefined;\n }\n\n async listResourceStamps(): Promise<Map<string, ResourceStamp>> {\n const stamps = new Map<string, ResourceStamp>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: STAMP_FIELDS,\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || stamps.has(rid)) continue;\n stamps.set(rid, toStamp(point.payload));\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n query: vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.queryBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch.points) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n ...(m.payload.machineRead ? { machineRead: true } : {}),\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n // `query`, not `search`: the REST client dropped `search`/`searchBatch` in\n // 1.19.0 in favour of the universal query endpoint. Same request but the\n // vector moves to `query`, and the response is wrapped in `{ points }`.\n const { points } = await this.qdrant.query(collection, {\n query: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return points.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n ...(payload.machineRead ? { machineRead: true } : {}),\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAUA,IAAM,eAAe,CAAC,cAAc,mBAAmB,eAAe,aAAa;AAEnF,SAAS,QAAQ,SAAoE;AACnF,QAAM,WAAW,SAAS;AAC1B,QAAM,cAAc,SAAS;AAC7B,SAAO;AAAA,IACL,iBAAiB,OAAO,aAAa,WAAW,WAAW;AAAA,IAC3D,aAAa,MAAM,QAAQ,WAAW,IAClC,YAAY,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,IAC5D,CAAC;AAAA,IACL,GAAI,SAAS,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,EACtD;AACF;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAuB,aAAsC;AAGlK,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA;AAAA;AAAA,QAGA,GAAI,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MAC7C;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,UACd,GAAI,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,QACrD;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAI5F,UAAM,KAAK,OAAO,WAAW,aAAa;AAAA,MACxC,SAAS,EAAE,YAAY;AAAA,MACvB,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,iBAAiB,YAA4D;AACjF,UAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACjD,OAAO;AAAA,MACP,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,MAC9E,cAAc;AAAA,MACd,aAAa;AAAA,IACf,CAAC;AACD,UAAM,QAAQ,KAAK,OAAO,CAAC;AAC3B,WAAO,QAAQ,QAAQ,MAAM,OAAO,IAAI;AAAA,EAC1C;AAAA,EAEA,MAAM,qBAA0D;AAC9D,UAAM,SAAS,oBAAI,IAA2B;AAC9C,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc;AAAA,QACd,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,OAAO,IAAI,GAAG,EAAG;AAChD,eAAO,IAAI,KAAK,QAAQ,MAAM,OAAO,CAAC;AAAA,MACxC;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C,OAAO;AAAA,MACP,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,WAAW,aAAa,EAAE,SAAS,CAAC;AAItE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,MAAM,QAAQ;AAC5B,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,MACvB,GAAI,EAAE,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,IACvD,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAK3C,UAAM,EAAE,OAAO,IAAI,MAAM,KAAK,OAAO,MAAM,YAAY;AAAA,MACrD,OAAO;AAAA,MACP,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,OAAO,IAAI,CAAC,MAAM;AACvB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,QACrB,GAAI,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MACrD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
|
package/dist/index.d.ts
CHANGED
|
@@ -19,6 +19,20 @@ interface AnnotationPayload {
|
|
|
19
19
|
motivation: string;
|
|
20
20
|
entityTypes: string[];
|
|
21
21
|
exactText: string;
|
|
22
|
+
/** True when the quoted text was recognized from pixels — see
|
|
23
|
+
* `VectorSearchResult.machineRead`. An annotation over a scanned page
|
|
24
|
+
* quotes OCR'd text, and annotation-focus gather searches these vectors. */
|
|
25
|
+
machineRead?: boolean;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* What a resource's vectors record about themselves: how fresh they are
|
|
29
|
+
* (`contentChecksum`), what they are tagged with (`entityTypes`), and how
|
|
30
|
+
* their text was obtained (`machineRead`).
|
|
31
|
+
*/
|
|
32
|
+
interface ResourceStamp {
|
|
33
|
+
contentChecksum: string | undefined;
|
|
34
|
+
entityTypes: string[];
|
|
35
|
+
machineRead?: boolean;
|
|
22
36
|
}
|
|
23
37
|
interface VectorSearchResult {
|
|
24
38
|
id: string;
|
|
@@ -27,6 +41,14 @@ interface VectorSearchResult {
|
|
|
27
41
|
annotationId?: AnnotationId;
|
|
28
42
|
text: string;
|
|
29
43
|
entityTypes?: string[];
|
|
44
|
+
/**
|
|
45
|
+
* Set when this passage's text was recognized from pixels rather than read
|
|
46
|
+
* from the document. Absent means read directly — the common case — so the
|
|
47
|
+
* flag is only ever present where it changes how the text should be
|
|
48
|
+
* trusted. Carried here because a chunk reaches its consumers with no
|
|
49
|
+
* document attached, and no consumer can recompute it.
|
|
50
|
+
*/
|
|
51
|
+
machineRead?: boolean;
|
|
30
52
|
}
|
|
31
53
|
interface SearchOptions {
|
|
32
54
|
limit: number;
|
|
@@ -60,7 +82,14 @@ interface VectorStore {
|
|
|
60
82
|
* entity-type set, stamped onto every point so `searchResources` can
|
|
61
83
|
* discriminate by kind (e.g. exclude `['Question']` from recall).
|
|
62
84
|
*/
|
|
63
|
-
|
|
85
|
+
/**
|
|
86
|
+
* Stamp a resource's chunks. `machineRead` records that the text was
|
|
87
|
+
* recognized from pixels (OCR) rather than read from the document — see
|
|
88
|
+
* `VectorSearchResult.machineRead`. It rides the embed because that is the
|
|
89
|
+
* moment extraction provenance is known; `reconcile()` restores it on a
|
|
90
|
+
* rebuild the same way it restores the checksum.
|
|
91
|
+
*/
|
|
92
|
+
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[], machineRead?: boolean): Promise<void>;
|
|
64
93
|
/**
|
|
65
94
|
* Rewrite the `entityTypes` stamp on a resource's existing points —
|
|
66
95
|
* payload-only, no embedding involved (SMELTER-AXIOMS.md, S13: a tag edit
|
|
@@ -102,10 +131,14 @@ interface VectorStore {
|
|
|
102
131
|
* and the entity-type set (missing stamp reads as `[]`). Drives both the
|
|
103
132
|
* S12 content-staleness diff and the S13 tag-staleness diff.
|
|
104
133
|
*/
|
|
105
|
-
listResourceStamps(): Promise<Map<string,
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
134
|
+
listResourceStamps(): Promise<Map<string, ResourceStamp>>;
|
|
135
|
+
/**
|
|
136
|
+
* One resource's stamp, or undefined when it has no vectors. Targeted so
|
|
137
|
+
* callers that need a single resource — the annotation index path, which
|
|
138
|
+
* must stamp provenance it cannot derive locally — do not scan the
|
|
139
|
+
* collection.
|
|
140
|
+
*/
|
|
141
|
+
getResourceStamp(resourceId: ResourceId): Promise<ResourceStamp | undefined>;
|
|
109
142
|
/** Distinct annotationIds present in the annotations collection. */
|
|
110
143
|
listAnnotationIds(): Promise<Set<string>>;
|
|
111
144
|
}
|
|
@@ -138,17 +171,15 @@ declare class QdrantVectorStore implements VectorStore {
|
|
|
138
171
|
* back-fills the index on collections created before the field was indexed.
|
|
139
172
|
*/
|
|
140
173
|
private ensurePayloadIndex;
|
|
141
|
-
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void>;
|
|
174
|
+
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[], machineRead?: boolean): Promise<void>;
|
|
142
175
|
upsertAnnotationVector(annotationId: AnnotationId, embedding: number[], payload: AnnotationPayload): Promise<void>;
|
|
143
176
|
deleteResourceVectors(resourceId: ResourceId): Promise<void>;
|
|
144
177
|
deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
|
|
145
178
|
deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
|
|
146
179
|
count(): Promise<number>;
|
|
147
180
|
updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
entityTypes: string[];
|
|
151
|
-
}>>;
|
|
181
|
+
getResourceStamp(resourceId: ResourceId): Promise<ResourceStamp | undefined>;
|
|
182
|
+
listResourceStamps(): Promise<Map<string, ResourceStamp>>;
|
|
152
183
|
listAnnotationIds(): Promise<Set<string>>;
|
|
153
184
|
/** Collect the distinct values of one payload field across a collection. */
|
|
154
185
|
private scrollPayloadField;
|
|
@@ -174,17 +205,15 @@ declare class MemoryVectorStore implements VectorStore {
|
|
|
174
205
|
disconnect(): Promise<void>;
|
|
175
206
|
clearAll(): Promise<void>;
|
|
176
207
|
isConnected(): boolean;
|
|
177
|
-
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void>;
|
|
208
|
+
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[], machineRead?: boolean): Promise<void>;
|
|
178
209
|
upsertAnnotationVector(annotationId: AnnotationId, embedding: number[], payload: AnnotationPayload): Promise<void>;
|
|
179
210
|
deleteResourceVectors(resourceId: ResourceId): Promise<void>;
|
|
180
211
|
deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
|
|
181
212
|
deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
|
|
182
213
|
count(): Promise<number>;
|
|
183
214
|
updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
|
|
184
|
-
listResourceStamps(): Promise<Map<string,
|
|
185
|
-
|
|
186
|
-
entityTypes: string[];
|
|
187
|
-
}>>;
|
|
215
|
+
listResourceStamps(): Promise<Map<string, ResourceStamp>>;
|
|
216
|
+
getResourceStamp(resourceId: ResourceId): Promise<ResourceStamp | undefined>;
|
|
188
217
|
listAnnotationIds(): Promise<Set<string>>;
|
|
189
218
|
searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
|
|
190
219
|
searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
|
|
@@ -279,25 +308,5 @@ interface EmbeddingConfig {
|
|
|
279
308
|
}
|
|
280
309
|
declare function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider>;
|
|
281
310
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
*
|
|
285
|
-
* Splits long text into overlapping chunks for embedding.
|
|
286
|
-
* Each chunk is a passage that fits within the embedding model's context window.
|
|
287
|
-
*/
|
|
288
|
-
interface ChunkingConfig {
|
|
289
|
-
chunkSize: number;
|
|
290
|
-
overlap: number;
|
|
291
|
-
}
|
|
292
|
-
declare const DEFAULT_CHUNKING_CONFIG: ChunkingConfig;
|
|
293
|
-
/**
|
|
294
|
-
* Split text into overlapping chunks.
|
|
295
|
-
*
|
|
296
|
-
* Splits on paragraph boundaries when possible, falling back to sentence
|
|
297
|
-
* boundaries, then word boundaries. Each chunk overlaps with the previous
|
|
298
|
-
* by `overlap` tokens worth of text.
|
|
299
|
-
*/
|
|
300
|
-
declare function chunkText(text: string, config?: ChunkingConfig): string[];
|
|
301
|
-
|
|
302
|
-
export { DEFAULT_CHUNKING_CONFIG, MemoryVectorStore, OllamaEmbeddingProvider, QdrantVectorStore, VoyageEmbeddingProvider, chunkText, createEmbeddingProvider, createVectorStore };
|
|
303
|
-
export type { AnnotationPayload, ChunkingConfig, EmbeddingChunk, EmbeddingConfig, EmbeddingProvider, OllamaEmbeddingConfig, QdrantConfig, SearchOptions, VectorSearchResult, VectorStore, VectorStoreConfig, VoyageConfig };
|
|
311
|
+
export { MemoryVectorStore, OllamaEmbeddingProvider, QdrantVectorStore, VoyageEmbeddingProvider, createEmbeddingProvider, createVectorStore };
|
|
312
|
+
export type { AnnotationPayload, EmbeddingChunk, EmbeddingConfig, EmbeddingProvider, OllamaEmbeddingConfig, QdrantConfig, SearchOptions, VectorSearchResult, VectorStore, VectorStoreConfig, VoyageConfig };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
QdrantVectorStore
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-Q2LGLRYN.js";
|
|
4
4
|
import {
|
|
5
5
|
VoyageEmbeddingProvider
|
|
6
6
|
} from "./chunk-UM3RNDW4.js";
|
|
@@ -9,6 +9,13 @@ import {
|
|
|
9
9
|
} from "./chunk-LBAPGZOW.js";
|
|
10
10
|
|
|
11
11
|
// src/store/memory.ts
|
|
12
|
+
function toStamp(point) {
|
|
13
|
+
return {
|
|
14
|
+
contentChecksum: point.payload.contentChecksum,
|
|
15
|
+
entityTypes: point.payload.entityTypes ?? [],
|
|
16
|
+
...point.payload.machineRead ? { machineRead: true } : {}
|
|
17
|
+
};
|
|
18
|
+
}
|
|
12
19
|
function cosineSimilarity(a, b) {
|
|
13
20
|
let dotProduct = 0;
|
|
14
21
|
let normA = 0;
|
|
@@ -38,7 +45,7 @@ var MemoryVectorStore = class {
|
|
|
38
45
|
isConnected() {
|
|
39
46
|
return this.connected;
|
|
40
47
|
}
|
|
41
|
-
async upsertResourceVectors(resourceId, chunks, contentChecksum, entityTypes) {
|
|
48
|
+
async upsertResourceVectors(resourceId, chunks, contentChecksum, entityTypes, machineRead) {
|
|
42
49
|
this.resources = this.resources.filter((p) => p.payload.resourceId !== String(resourceId));
|
|
43
50
|
for (const chunk of chunks) {
|
|
44
51
|
this.resources.push({
|
|
@@ -49,7 +56,10 @@ var MemoryVectorStore = class {
|
|
|
49
56
|
chunkIndex: chunk.chunkIndex,
|
|
50
57
|
text: chunk.text,
|
|
51
58
|
contentChecksum,
|
|
52
|
-
entityTypes
|
|
59
|
+
entityTypes,
|
|
60
|
+
// Only stamped when true: absence is the common case and carries no
|
|
61
|
+
// claim, so a native extraction stores nothing extra.
|
|
62
|
+
...machineRead ? { machineRead: true } : {}
|
|
53
63
|
}
|
|
54
64
|
});
|
|
55
65
|
}
|
|
@@ -64,7 +74,8 @@ var MemoryVectorStore = class {
|
|
|
64
74
|
resourceId: String(payload.resourceId),
|
|
65
75
|
motivation: payload.motivation,
|
|
66
76
|
entityTypes: payload.entityTypes,
|
|
67
|
-
text: payload.exactText
|
|
77
|
+
text: payload.exactText,
|
|
78
|
+
...payload.machineRead ? { machineRead: true } : {}
|
|
68
79
|
}
|
|
69
80
|
});
|
|
70
81
|
}
|
|
@@ -91,14 +102,15 @@ var MemoryVectorStore = class {
|
|
|
91
102
|
const stamps = /* @__PURE__ */ new Map();
|
|
92
103
|
for (const p of this.resources) {
|
|
93
104
|
if (!stamps.has(p.payload.resourceId)) {
|
|
94
|
-
stamps.set(p.payload.resourceId,
|
|
95
|
-
contentChecksum: p.payload.contentChecksum,
|
|
96
|
-
entityTypes: p.payload.entityTypes ?? []
|
|
97
|
-
});
|
|
105
|
+
stamps.set(p.payload.resourceId, toStamp(p));
|
|
98
106
|
}
|
|
99
107
|
}
|
|
100
108
|
return stamps;
|
|
101
109
|
}
|
|
110
|
+
async getResourceStamp(resourceId) {
|
|
111
|
+
const point = this.resources.find((p) => p.payload.resourceId === String(resourceId));
|
|
112
|
+
return point ? toStamp(point) : void 0;
|
|
113
|
+
}
|
|
102
114
|
async listAnnotationIds() {
|
|
103
115
|
const ids = /* @__PURE__ */ new Set();
|
|
104
116
|
for (const p of this.annotations) {
|
|
@@ -174,7 +186,8 @@ var MemoryVectorStore = class {
|
|
|
174
186
|
resourceId: s.payload.resourceId,
|
|
175
187
|
annotationId: s.payload.annotationId,
|
|
176
188
|
text: s.payload.text,
|
|
177
|
-
entityTypes: s.payload.entityTypes
|
|
189
|
+
entityTypes: s.payload.entityTypes,
|
|
190
|
+
...s.payload.machineRead ? { machineRead: true } : {}
|
|
178
191
|
};
|
|
179
192
|
}
|
|
180
193
|
};
|
|
@@ -183,7 +196,7 @@ var MemoryVectorStore = class {
|
|
|
183
196
|
async function createVectorStore(config) {
|
|
184
197
|
let store;
|
|
185
198
|
if (config.type === "qdrant") {
|
|
186
|
-
const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-
|
|
199
|
+
const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-GNSEAGLI.js");
|
|
187
200
|
store = new QdrantVectorStore2({
|
|
188
201
|
host: config.host ?? "localhost",
|
|
189
202
|
port: config.port ?? 6333,
|
|
@@ -216,57 +229,11 @@ async function createEmbeddingProvider(config) {
|
|
|
216
229
|
}
|
|
217
230
|
throw new Error(`Unknown embedding provider type: ${config.type}`);
|
|
218
231
|
}
|
|
219
|
-
|
|
220
|
-
// src/chunking.ts
|
|
221
|
-
var DEFAULT_CHUNKING_CONFIG = {
|
|
222
|
-
chunkSize: 512,
|
|
223
|
-
overlap: 64
|
|
224
|
-
};
|
|
225
|
-
function estimateTokens(text) {
|
|
226
|
-
return Math.ceil(text.length / 4);
|
|
227
|
-
}
|
|
228
|
-
function chunkText(text, config = DEFAULT_CHUNKING_CONFIG) {
|
|
229
|
-
if (text.length === 0) return [];
|
|
230
|
-
const totalTokens = estimateTokens(text);
|
|
231
|
-
if (totalTokens <= config.chunkSize) {
|
|
232
|
-
return [text];
|
|
233
|
-
}
|
|
234
|
-
const chunkChars = config.chunkSize * 4;
|
|
235
|
-
const overlapChars = config.overlap * 4;
|
|
236
|
-
const chunks = [];
|
|
237
|
-
let start = 0;
|
|
238
|
-
while (start < text.length) {
|
|
239
|
-
let end = Math.min(start + chunkChars, text.length);
|
|
240
|
-
if (end < text.length) {
|
|
241
|
-
const paraBreak = text.lastIndexOf("\n\n", end);
|
|
242
|
-
if (paraBreak > start + chunkChars / 2) {
|
|
243
|
-
end = paraBreak;
|
|
244
|
-
} else {
|
|
245
|
-
const sentenceBreak = text.lastIndexOf(". ", end);
|
|
246
|
-
if (sentenceBreak > start + chunkChars / 2) {
|
|
247
|
-
end = sentenceBreak + 1;
|
|
248
|
-
} else {
|
|
249
|
-
const wordBreak = text.lastIndexOf(" ", end);
|
|
250
|
-
if (wordBreak > start + chunkChars / 2) {
|
|
251
|
-
end = wordBreak;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
chunks.push(text.slice(start, end).trim());
|
|
257
|
-
const nextStart = end - overlapChars;
|
|
258
|
-
start = nextStart > start ? nextStart : end;
|
|
259
|
-
if (start >= text.length) break;
|
|
260
|
-
}
|
|
261
|
-
return chunks.filter((c) => c.length > 0);
|
|
262
|
-
}
|
|
263
232
|
export {
|
|
264
|
-
DEFAULT_CHUNKING_CONFIG,
|
|
265
233
|
MemoryVectorStore,
|
|
266
234
|
OllamaEmbeddingProvider,
|
|
267
235
|
QdrantVectorStore,
|
|
268
236
|
VoyageEmbeddingProvider,
|
|
269
|
-
chunkText,
|
|
270
237
|
createEmbeddingProvider,
|
|
271
238
|
createVectorStore
|
|
272
239
|
};
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts","../src/chunking.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n for (const p of this.resources) {\n if (p.payload.resourceId === String(resourceId)) {\n p.payload.entityTypes = entityTypes;\n }\n }\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n for (const p of this.resources) {\n if (!stamps.has(p.payload.resourceId)) {\n stamps.set(p.payload.resourceId, {\n contentChecksum: p.payload.contentChecksum,\n entityTypes: p.payload.entityTypes ?? [],\n });\n }\n }\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n","/**\n * Text Chunking Utilities\n *\n * Splits long text into overlapping chunks for embedding.\n * Each chunk is a passage that fits within the embedding model's context window.\n */\n\nexport interface ChunkingConfig {\n chunkSize: number; // approximate tokens per chunk\n overlap: number; // tokens of overlap between adjacent chunks\n}\n\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n chunkSize: 512,\n overlap: 64,\n};\n\n/**\n * Rough token count estimate: ~4 characters per token for English text.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into overlapping chunks.\n *\n * Splits on paragraph boundaries when possible, falling back to sentence\n * boundaries, then word boundaries. Each chunk overlaps with the previous\n * by `overlap` tokens worth of text.\n */\nexport function chunkText(text: string, config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG): string[] {\n if (text.length === 0) return [];\n const totalTokens = estimateTokens(text);\n if (totalTokens <= config.chunkSize) {\n return [text];\n }\n\n const chunkChars = config.chunkSize * 4;\n const overlapChars = config.overlap * 4;\n const chunks: string[] = [];\n let start = 0;\n\n while (start < text.length) {\n let end = Math.min(start + chunkChars, text.length);\n\n // Try to break at a paragraph boundary\n if (end < text.length) {\n const paraBreak = text.lastIndexOf('\\n\\n', end);\n if (paraBreak > start + chunkChars / 2) {\n end = paraBreak;\n } else {\n // Try sentence boundary\n const sentenceBreak = text.lastIndexOf('. ', end);\n if (sentenceBreak > start + chunkChars / 2) {\n end = sentenceBreak + 1;\n } else {\n // Try word boundary\n const wordBreak = text.lastIndexOf(' ', end);\n if (wordBreak > start + chunkChars / 2) {\n end = wordBreak;\n }\n }\n }\n }\n\n chunks.push(text.slice(start, end).trim());\n const nextStart = end - overlapChars;\n start = nextStart > start ? nextStart : end;\n if (start >= text.length) break;\n }\n\n return chunks.filter(c => c.length > 0);\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAE3I,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAC5F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,EAAE,QAAQ,eAAe,OAAO,UAAU,GAAG;AAC/C,UAAE,QAAQ,cAAc;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,OAAO,IAAI,EAAE,QAAQ,UAAU,GAAG;AACrC,eAAO,IAAI,EAAE,QAAQ,YAAY;AAAA,UAC/B,iBAAiB,EAAE,QAAQ;AAAA,UAC3B,aAAa,EAAE,QAAQ,eAAe,CAAC;AAAA,QACzC,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;;;ACxNA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;;;ACtBO,IAAM,0BAA0C;AAAA,EACrD,WAAW;AAAA,EACX,SAAS;AACX;AAKA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASO,SAAS,UAAU,MAAc,SAAyB,yBAAmC;AAClG,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAC/B,QAAM,cAAc,eAAe,IAAI;AACvC,MAAI,eAAe,OAAO,WAAW;AACnC,WAAO,CAAC,IAAI;AAAA,EACd;AAEA,QAAM,aAAa,OAAO,YAAY;AACtC,QAAM,eAAe,OAAO,UAAU;AACtC,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,QAAI,MAAM,KAAK,IAAI,QAAQ,YAAY,KAAK,MAAM;AAGlD,QAAI,MAAM,KAAK,QAAQ;AACrB,YAAM,YAAY,KAAK,YAAY,QAAQ,GAAG;AAC9C,UAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,cAAM;AAAA,MACR,OAAO;AAEL,cAAM,gBAAgB,KAAK,YAAY,MAAM,GAAG;AAChD,YAAI,gBAAgB,QAAQ,aAAa,GAAG;AAC1C,gBAAM,gBAAgB;AAAA,QACxB,OAAO;AAEL,gBAAM,YAAY,KAAK,YAAY,KAAK,GAAG;AAC3C,cAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC;AACzC,UAAM,YAAY,MAAM;AACxB,YAAQ,YAAY,QAAQ,YAAY;AACxC,QAAI,SAAS,KAAK,OAAQ;AAAA,EAC5B;AAEA,SAAO,OAAO,OAAO,OAAK,EAAE,SAAS,CAAC;AACxC;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
|
|
1
|
+
{"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions, ResourceStamp } from './interface';\n\nfunction toStamp(point: { payload: { contentChecksum?: string; entityTypes?: string[]; machineRead?: boolean } }): ResourceStamp {\n return {\n contentChecksum: point.payload.contentChecksum,\n entityTypes: point.payload.entityTypes ?? [],\n ...(point.payload.machineRead ? { machineRead: true } : {}),\n };\n}\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n machineRead?: boolean;\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[], machineRead?: boolean): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n // Only stamped when true: absence is the common case and carries no\n // claim, so a native extraction stores nothing extra.\n ...(machineRead ? { machineRead: true } : {}),\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n ...(payload.machineRead ? { machineRead: true } : {}),\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n for (const p of this.resources) {\n if (p.payload.resourceId === String(resourceId)) {\n p.payload.entityTypes = entityTypes;\n }\n }\n }\n\n async listResourceStamps(): Promise<Map<string, ResourceStamp>> {\n const stamps = new Map<string, ResourceStamp>();\n for (const p of this.resources) {\n if (!stamps.has(p.payload.resourceId)) {\n stamps.set(p.payload.resourceId, toStamp(p));\n }\n }\n return stamps;\n }\n\n async getResourceStamp(resourceId: ResourceId): Promise<ResourceStamp | undefined> {\n const point = this.resources.find(p => p.payload.resourceId === String(resourceId));\n return point ? toStamp(point) : undefined;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n ...(s.payload.machineRead ? { machineRead: true } : {}),\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n"],"mappings":";;;;;;;;;;;AAUA,SAAS,QAAQ,OAAgH;AAC/H,SAAO;AAAA,IACL,iBAAiB,MAAM,QAAQ;AAAA,IAC/B,aAAa,MAAM,QAAQ,eAAe,CAAC;AAAA,IAC3C,GAAI,MAAM,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,EAC3D;AACF;AAiBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAuB,aAAsC;AAElK,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA;AAAA;AAAA,UAGA,GAAI,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,QAC7C;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,QACd,GAAI,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MACrD;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAC5F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,EAAE,QAAQ,eAAe,OAAO,UAAU,GAAG;AAC/C,UAAE,QAAQ,cAAc;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,qBAA0D;AAC9D,UAAM,SAAS,oBAAI,IAA2B;AAC9C,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,OAAO,IAAI,EAAE,QAAQ,UAAU,GAAG;AACrC,eAAO,IAAI,EAAE,QAAQ,YAAY,QAAQ,CAAC,CAAC;AAAA,MAC7C;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,iBAAiB,YAA4D;AACjF,UAAM,QAAQ,KAAK,UAAU,KAAK,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAClF,WAAO,QAAQ,QAAQ,KAAK,IAAI;AAAA,EAClC;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,MACvB,GAAI,EAAE,QAAQ,cAAc,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,IACvD;AAAA,EACF;AACF;;;ACxOA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@semiont/vectors",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.25",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24.0.0"
|
|
6
6
|
},
|
|
@@ -25,11 +25,11 @@
|
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@qdrant/js-client-rest": "^1.18.0",
|
|
28
|
-
"@semiont/core": "0.5.
|
|
28
|
+
"@semiont/core": "0.5.25"
|
|
29
29
|
},
|
|
30
30
|
"devDependencies": {
|
|
31
31
|
"@vitest/coverage-v8": "^4.1.8",
|
|
32
|
-
"rollup": "^4.
|
|
32
|
+
"rollup": "^4.62.3",
|
|
33
33
|
"rollup-plugin-dts": "^6.4.1",
|
|
34
34
|
"tsup": "^8.5.1",
|
|
35
35
|
"typescript": "^6.0.2",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n // Payload-only rewrite across the resource's points — no vectors touched,\n // no embedding involved (S13). Qdrant set_payload with a filter is a\n // no-op when the resource has no points.\n await this.qdrant.setPayload('resources', {\n payload: { entityTypes },\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: ['resourceId', 'contentChecksum', 'entityTypes'],\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || stamps.has(rid)) continue;\n const checksum = point.payload?.contentChecksum;\n const entityTypes = point.payload?.entityTypes;\n stamps.set(rid, {\n contentChecksum: typeof checksum === 'string' ? checksum : undefined,\n entityTypes: Array.isArray(entityTypes)\n ? entityTypes.filter((t): t is string => typeof t === 'string')\n : [],\n });\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.searchBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n const results = await this.qdrant.search(collection, {\n vector: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return results.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAQO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAG3I,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,MACF;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAI5F,UAAM,KAAK,OAAO,WAAW,aAAa;AAAA,MACxC,SAAS,EAAE,YAAY;AAAA,MACvB,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,cAAc,mBAAmB,aAAa;AAAA,QAC7D,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,OAAO,IAAI,GAAG,EAAG;AAChD,cAAM,WAAW,MAAM,SAAS;AAChC,cAAM,cAAc,MAAM,SAAS;AACnC,eAAO,IAAI,KAAK;AAAA,UACd,iBAAiB,OAAO,aAAa,WAAW,WAAW;AAAA,UAC3D,aAAa,MAAM,QAAQ,WAAW,IAClC,YAAY,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,IAC5D,CAAC;AAAA,QACP,CAAC;AAAA,MACH;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,YAAY,aAAa,EAAE,SAAS,CAAC;AAIvE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,OAAO;AACrB,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAE3C,UAAM,UAAU,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,MACnD,QAAQ;AAAA,MACR,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
|
package/dist/qdrant-A765QRQF.js
DELETED
|
File without changes
|