@semiont/vectors 0.5.11 → 0.5.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-TYDOHCBS.js → chunk-VFNOQC4M.js} +19 -7
- package/dist/chunk-VFNOQC4M.js.map +1 -0
- package/dist/index.d.ts +25 -5
- package/dist/index.js +17 -7
- package/dist/index.js.map +1 -1
- package/dist/qdrant-A765QRQF.js +7 -0
- package/package.json +3 -3
- package/dist/chunk-TYDOHCBS.js.map +0 -1
- package/dist/qdrant-R7IW5NII.js +0 -7
- /package/dist/{qdrant-R7IW5NII.js.map → qdrant-A765QRQF.js.map} +0 -0
|
@@ -121,25 +121,37 @@ var QdrantVectorStore = class {
|
|
|
121
121
|
]);
|
|
122
122
|
return resources.count + annotations.count;
|
|
123
123
|
}
|
|
124
|
-
async
|
|
125
|
-
|
|
124
|
+
async updateResourceEntityTypes(resourceId, entityTypes) {
|
|
125
|
+
await this.qdrant.setPayload("resources", {
|
|
126
|
+
payload: { entityTypes },
|
|
127
|
+
filter: {
|
|
128
|
+
must: [{ key: "resourceId", match: { value: String(resourceId) } }]
|
|
129
|
+
}
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
async listResourceStamps() {
|
|
133
|
+
const stamps = /* @__PURE__ */ new Map();
|
|
126
134
|
let offset = void 0;
|
|
127
135
|
do {
|
|
128
136
|
const page = await this.qdrant.scroll("resources", {
|
|
129
137
|
limit: 1e3,
|
|
130
138
|
offset,
|
|
131
|
-
with_payload: ["resourceId", "contentChecksum"],
|
|
139
|
+
with_payload: ["resourceId", "contentChecksum", "entityTypes"],
|
|
132
140
|
with_vector: false
|
|
133
141
|
});
|
|
134
142
|
for (const point of page.points) {
|
|
135
143
|
const rid = point.payload?.resourceId;
|
|
136
|
-
if (typeof rid !== "string" ||
|
|
144
|
+
if (typeof rid !== "string" || stamps.has(rid)) continue;
|
|
137
145
|
const checksum = point.payload?.contentChecksum;
|
|
138
|
-
|
|
146
|
+
const entityTypes = point.payload?.entityTypes;
|
|
147
|
+
stamps.set(rid, {
|
|
148
|
+
contentChecksum: typeof checksum === "string" ? checksum : void 0,
|
|
149
|
+
entityTypes: Array.isArray(entityTypes) ? entityTypes.filter((t) => typeof t === "string") : []
|
|
150
|
+
});
|
|
139
151
|
}
|
|
140
152
|
offset = page.next_page_offset ?? void 0;
|
|
141
153
|
} while (offset !== void 0 && offset !== null);
|
|
142
|
-
return
|
|
154
|
+
return stamps;
|
|
143
155
|
}
|
|
144
156
|
async listAnnotationIds() {
|
|
145
157
|
return this.scrollPayloadField("annotations", "annotationId");
|
|
@@ -266,4 +278,4 @@ var QdrantVectorStore = class {
|
|
|
266
278
|
export {
|
|
267
279
|
QdrantVectorStore
|
|
268
280
|
};
|
|
269
|
-
//# sourceMappingURL=chunk-
|
|
281
|
+
//# sourceMappingURL=chunk-VFNOQC4M.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n // Payload-only rewrite across the resource's points — no vectors touched,\n // no embedding involved (S13). Qdrant set_payload with a filter is a\n // no-op when the resource has no points.\n await this.qdrant.setPayload('resources', {\n payload: { entityTypes },\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: ['resourceId', 'contentChecksum', 'entityTypes'],\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || stamps.has(rid)) continue;\n const checksum = point.payload?.contentChecksum;\n const entityTypes = point.payload?.entityTypes;\n stamps.set(rid, {\n contentChecksum: typeof checksum === 'string' ? checksum : undefined,\n entityTypes: Array.isArray(entityTypes)\n ? entityTypes.filter((t): t is string => typeof t === 'string')\n : [],\n });\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.searchBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n const results = await this.qdrant.search(collection, {\n vector: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return results.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAQO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAG3I,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,MACF;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAI5F,UAAM,KAAK,OAAO,WAAW,aAAa;AAAA,MACxC,SAAS,EAAE,YAAY;AAAA,MACvB,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,cAAc,mBAAmB,aAAa;AAAA,QAC7D,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,OAAO,IAAI,GAAG,EAAG;AAChD,cAAM,WAAW,MAAM,SAAS;AAChC,cAAM,cAAc,MAAM,SAAS;AACnC,eAAO,IAAI,KAAK;AAAA,UACd,iBAAiB,OAAO,aAAa,WAAW,WAAW;AAAA,UAC3D,aAAa,MAAM,QAAQ,WAAW,IAClC,YAAY,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,IAC5D,CAAC;AAAA,QACP,CAAC;AAAA,MACH;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,YAAY,aAAa,EAAE,SAAS,CAAC;AAIvE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,OAAO;AACrB,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAE3C,UAAM,UAAU,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,MACnD,QAAQ;AAAA,MACR,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
|
package/dist/index.d.ts
CHANGED
|
@@ -61,6 +61,13 @@ interface VectorStore {
|
|
|
61
61
|
* discriminate by kind (e.g. exclude `['Question']` from recall).
|
|
62
62
|
*/
|
|
63
63
|
upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void>;
|
|
64
|
+
/**
|
|
65
|
+
* Rewrite the `entityTypes` stamp on a resource's existing points —
|
|
66
|
+
* payload-only, no embedding involved (SMELTER-AXIOMS.md, S13: a tag edit
|
|
67
|
+
* must never trigger an embedding call). No-op when the resource has no
|
|
68
|
+
* points: the stamp rides the next embed.
|
|
69
|
+
*/
|
|
70
|
+
updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
|
|
64
71
|
upsertAnnotationVector(annotationId: AnnotationId, embedding: number[], payload: AnnotationPayload): Promise<void>;
|
|
65
72
|
deleteResourceVectors(resourceId: ResourceId): Promise<void>;
|
|
66
73
|
deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
|
|
@@ -90,10 +97,15 @@ interface VectorStore {
|
|
|
90
97
|
count(): Promise<number>;
|
|
91
98
|
/**
|
|
92
99
|
* Distinct resourceIds present in the resources collection, each with its
|
|
93
|
-
*
|
|
94
|
-
* existed — reconciliation treats those as stale and re-embeds
|
|
100
|
+
* stamps: the content checksum (undefined for points written before
|
|
101
|
+
* stamping existed — reconciliation treats those as stale and re-embeds)
|
|
102
|
+
* and the entity-type set (missing stamp reads as `[]`). Drives both the
|
|
103
|
+
* S12 content-staleness diff and the S13 tag-staleness diff.
|
|
95
104
|
*/
|
|
96
|
-
|
|
105
|
+
listResourceStamps(): Promise<Map<string, {
|
|
106
|
+
contentChecksum: string | undefined;
|
|
107
|
+
entityTypes: string[];
|
|
108
|
+
}>>;
|
|
97
109
|
/** Distinct annotationIds present in the annotations collection. */
|
|
98
110
|
listAnnotationIds(): Promise<Set<string>>;
|
|
99
111
|
}
|
|
@@ -132,7 +144,11 @@ declare class QdrantVectorStore implements VectorStore {
|
|
|
132
144
|
deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
|
|
133
145
|
deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
|
|
134
146
|
count(): Promise<number>;
|
|
135
|
-
|
|
147
|
+
updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
|
|
148
|
+
listResourceStamps(): Promise<Map<string, {
|
|
149
|
+
contentChecksum: string | undefined;
|
|
150
|
+
entityTypes: string[];
|
|
151
|
+
}>>;
|
|
136
152
|
listAnnotationIds(): Promise<Set<string>>;
|
|
137
153
|
/** Collect the distinct values of one payload field across a collection. */
|
|
138
154
|
private scrollPayloadField;
|
|
@@ -164,7 +180,11 @@ declare class MemoryVectorStore implements VectorStore {
|
|
|
164
180
|
deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
|
|
165
181
|
deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
|
|
166
182
|
count(): Promise<number>;
|
|
167
|
-
|
|
183
|
+
updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
|
|
184
|
+
listResourceStamps(): Promise<Map<string, {
|
|
185
|
+
contentChecksum: string | undefined;
|
|
186
|
+
entityTypes: string[];
|
|
187
|
+
}>>;
|
|
168
188
|
listAnnotationIds(): Promise<Set<string>>;
|
|
169
189
|
searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
|
|
170
190
|
searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
QdrantVectorStore
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-VFNOQC4M.js";
|
|
4
4
|
import {
|
|
5
5
|
VoyageEmbeddingProvider
|
|
6
6
|
} from "./chunk-UM3RNDW4.js";
|
|
@@ -80,14 +80,24 @@ var MemoryVectorStore = class {
|
|
|
80
80
|
async count() {
|
|
81
81
|
return this.resources.length + this.annotations.length;
|
|
82
82
|
}
|
|
83
|
-
async
|
|
84
|
-
const checksums = /* @__PURE__ */ new Map();
|
|
83
|
+
async updateResourceEntityTypes(resourceId, entityTypes) {
|
|
85
84
|
for (const p of this.resources) {
|
|
86
|
-
if (
|
|
87
|
-
|
|
85
|
+
if (p.payload.resourceId === String(resourceId)) {
|
|
86
|
+
p.payload.entityTypes = entityTypes;
|
|
88
87
|
}
|
|
89
88
|
}
|
|
90
|
-
|
|
89
|
+
}
|
|
90
|
+
async listResourceStamps() {
|
|
91
|
+
const stamps = /* @__PURE__ */ new Map();
|
|
92
|
+
for (const p of this.resources) {
|
|
93
|
+
if (!stamps.has(p.payload.resourceId)) {
|
|
94
|
+
stamps.set(p.payload.resourceId, {
|
|
95
|
+
contentChecksum: p.payload.contentChecksum,
|
|
96
|
+
entityTypes: p.payload.entityTypes ?? []
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return stamps;
|
|
91
101
|
}
|
|
92
102
|
async listAnnotationIds() {
|
|
93
103
|
const ids = /* @__PURE__ */ new Set();
|
|
@@ -173,7 +183,7 @@ var MemoryVectorStore = class {
|
|
|
173
183
|
async function createVectorStore(config) {
|
|
174
184
|
let store;
|
|
175
185
|
if (config.type === "qdrant") {
|
|
176
|
-
const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-
|
|
186
|
+
const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-A765QRQF.js");
|
|
177
187
|
store = new QdrantVectorStore2({
|
|
178
188
|
host: config.host ?? "localhost",
|
|
179
189
|
port: config.port ?? 6333,
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts","../src/chunking.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async listResourceChecksums(): Promise<Map<string, string | undefined>> {\n const checksums = new Map<string, string | undefined>();\n for (const p of this.resources) {\n if (!checksums.has(p.payload.resourceId)) {\n checksums.set(p.payload.resourceId, p.payload.contentChecksum);\n }\n }\n return checksums;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n","/**\n * Text Chunking Utilities\n *\n * Splits long text into overlapping chunks for embedding.\n * Each chunk is a passage that fits within the embedding model's context window.\n */\n\nexport interface ChunkingConfig {\n chunkSize: number; // approximate tokens per chunk\n overlap: number; // tokens of overlap between adjacent chunks\n}\n\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n chunkSize: 512,\n overlap: 64,\n};\n\n/**\n * Rough token count estimate: ~4 characters per token for English text.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into overlapping chunks.\n *\n * Splits on paragraph boundaries when possible, falling back to sentence\n * boundaries, then word boundaries. Each chunk overlaps with the previous\n * by `overlap` tokens worth of text.\n */\nexport function chunkText(text: string, config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG): string[] {\n if (text.length === 0) return [];\n const totalTokens = estimateTokens(text);\n if (totalTokens <= config.chunkSize) {\n return [text];\n }\n\n const chunkChars = config.chunkSize * 4;\n const overlapChars = config.overlap * 4;\n const chunks: string[] = [];\n let start = 0;\n\n while (start < text.length) {\n let end = Math.min(start + chunkChars, text.length);\n\n // Try to break at a paragraph boundary\n if (end < text.length) {\n const paraBreak = text.lastIndexOf('\\n\\n', end);\n if (paraBreak > start + chunkChars / 2) {\n end = paraBreak;\n } else {\n // Try sentence boundary\n const sentenceBreak = text.lastIndexOf('. ', end);\n if (sentenceBreak > start + chunkChars / 2) {\n end = sentenceBreak + 1;\n } else {\n // Try word boundary\n const wordBreak = text.lastIndexOf(' ', end);\n if (wordBreak > start + chunkChars / 2) {\n end = wordBreak;\n }\n }\n }\n }\n\n chunks.push(text.slice(start, end).trim());\n const nextStart = end - overlapChars;\n start = nextStart > start ? nextStart : end;\n if (start >= text.length) break;\n }\n\n return chunks.filter(c => c.length > 0);\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAE3I,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,wBAAkE;AACtE,UAAM,YAAY,oBAAI,IAAgC;AACtD,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,UAAU,IAAI,EAAE,QAAQ,UAAU,GAAG;AACxC,kBAAU,IAAI,EAAE,QAAQ,YAAY,EAAE,QAAQ,eAAe;AAAA,MAC/D;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;;;AC7MA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;;;ACtBO,IAAM,0BAA0C;AAAA,EACrD,WAAW;AAAA,EACX,SAAS;AACX;AAKA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASO,SAAS,UAAU,MAAc,SAAyB,yBAAmC;AAClG,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAC/B,QAAM,cAAc,eAAe,IAAI;AACvC,MAAI,eAAe,OAAO,WAAW;AACnC,WAAO,CAAC,IAAI;AAAA,EACd;AAEA,QAAM,aAAa,OAAO,YAAY;AACtC,QAAM,eAAe,OAAO,UAAU;AACtC,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,QAAI,MAAM,KAAK,IAAI,QAAQ,YAAY,KAAK,MAAM;AAGlD,QAAI,MAAM,KAAK,QAAQ;AACrB,YAAM,YAAY,KAAK,YAAY,QAAQ,GAAG;AAC9C,UAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,cAAM;AAAA,MACR,OAAO;AAEL,cAAM,gBAAgB,KAAK,YAAY,MAAM,GAAG;AAChD,YAAI,gBAAgB,QAAQ,aAAa,GAAG;AAC1C,gBAAM,gBAAgB;AAAA,QACxB,OAAO;AAEL,gBAAM,YAAY,KAAK,YAAY,KAAK,GAAG;AAC3C,cAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC;AACzC,UAAM,YAAY,MAAM;AACxB,YAAQ,YAAY,QAAQ,YAAY;AACxC,QAAI,SAAS,KAAK,OAAQ;AAAA,EAC5B;AAEA,SAAO,OAAO,OAAO,OAAK,EAAE,SAAS,CAAC;AACxC;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
|
|
1
|
+
{"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts","../src/chunking.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n for (const p of this.resources) {\n if (p.payload.resourceId === String(resourceId)) {\n p.payload.entityTypes = entityTypes;\n }\n }\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n for (const p of this.resources) {\n if (!stamps.has(p.payload.resourceId)) {\n stamps.set(p.payload.resourceId, {\n contentChecksum: p.payload.contentChecksum,\n entityTypes: p.payload.entityTypes ?? [],\n });\n }\n }\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n","/**\n * Text Chunking Utilities\n *\n * Splits long text into overlapping chunks for embedding.\n * Each chunk is a passage that fits within the embedding model's context window.\n */\n\nexport interface ChunkingConfig {\n chunkSize: number; // approximate tokens per chunk\n overlap: number; // tokens of overlap between adjacent chunks\n}\n\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n chunkSize: 512,\n overlap: 64,\n};\n\n/**\n * Rough token count estimate: ~4 characters per token for English text.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into overlapping chunks.\n *\n * Splits on paragraph boundaries when possible, falling back to sentence\n * boundaries, then word boundaries. Each chunk overlaps with the previous\n * by `overlap` tokens worth of text.\n */\nexport function chunkText(text: string, config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG): string[] {\n if (text.length === 0) return [];\n const totalTokens = estimateTokens(text);\n if (totalTokens <= config.chunkSize) {\n return [text];\n }\n\n const chunkChars = config.chunkSize * 4;\n const overlapChars = config.overlap * 4;\n const chunks: string[] = [];\n let start = 0;\n\n while (start < text.length) {\n let end = Math.min(start + chunkChars, text.length);\n\n // Try to break at a paragraph boundary\n if (end < text.length) {\n const paraBreak = text.lastIndexOf('\\n\\n', end);\n if (paraBreak > start + chunkChars / 2) {\n end = paraBreak;\n } else {\n // Try sentence boundary\n const sentenceBreak = text.lastIndexOf('. ', end);\n if (sentenceBreak > start + chunkChars / 2) {\n end = sentenceBreak + 1;\n } else {\n // Try word boundary\n const wordBreak = text.lastIndexOf(' ', end);\n if (wordBreak > start + chunkChars / 2) {\n end = wordBreak;\n }\n }\n }\n }\n\n chunks.push(text.slice(start, end).trim());\n const nextStart = end - overlapChars;\n start = nextStart > start ? nextStart : end;\n if (start >= text.length) break;\n }\n\n return chunks.filter(c => c.length > 0);\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAE3I,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAC5F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,EAAE,QAAQ,eAAe,OAAO,UAAU,GAAG;AAC/C,UAAE,QAAQ,cAAc;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,OAAO,IAAI,EAAE,QAAQ,UAAU,GAAG;AACrC,eAAO,IAAI,EAAE,QAAQ,YAAY;AAAA,UAC/B,iBAAiB,EAAE,QAAQ;AAAA,UAC3B,aAAa,EAAE,QAAQ,eAAe,CAAC;AAAA,QACzC,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;;;ACxNA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;;;ACtBO,IAAM,0BAA0C;AAAA,EACrD,WAAW;AAAA,EACX,SAAS;AACX;AAKA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASO,SAAS,UAAU,MAAc,SAAyB,yBAAmC;AAClG,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAC/B,QAAM,cAAc,eAAe,IAAI;AACvC,MAAI,eAAe,OAAO,WAAW;AACnC,WAAO,CAAC,IAAI;AAAA,EACd;AAEA,QAAM,aAAa,OAAO,YAAY;AACtC,QAAM,eAAe,OAAO,UAAU;AACtC,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,QAAI,MAAM,KAAK,IAAI,QAAQ,YAAY,KAAK,MAAM;AAGlD,QAAI,MAAM,KAAK,QAAQ;AACrB,YAAM,YAAY,KAAK,YAAY,QAAQ,GAAG;AAC9C,UAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,cAAM;AAAA,MACR,OAAO;AAEL,cAAM,gBAAgB,KAAK,YAAY,MAAM,GAAG;AAChD,YAAI,gBAAgB,QAAQ,aAAa,GAAG;AAC1C,gBAAM,gBAAgB;AAAA,QACxB,OAAO;AAEL,gBAAM,YAAY,KAAK,YAAY,KAAK,GAAG;AAC3C,cAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC;AACzC,UAAM,YAAY,MAAM;AACxB,YAAQ,YAAY,QAAQ,YAAY;AACxC,QAAI,SAAS,KAAK,OAAQ;AAAA,EAC5B;AAEA,SAAO,OAAO,OAAO,OAAK,EAAE,SAAS,CAAC;AACxC;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@semiont/vectors",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.13",
|
|
4
4
|
"engines": {
|
|
5
5
|
"node": ">=24.0.0"
|
|
6
6
|
},
|
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
27
|
"@qdrant/js-client-rest": "^1.18.0",
|
|
28
|
-
"@semiont/core": "0.5.
|
|
28
|
+
"@semiont/core": "0.5.13"
|
|
29
29
|
},
|
|
30
30
|
"devDependencies": {
|
|
31
31
|
"@vitest/coverage-v8": "^4.1.8",
|
|
@@ -33,7 +33,7 @@
|
|
|
33
33
|
"rollup-plugin-dts": "^6.4.1",
|
|
34
34
|
"tsup": "^8.5.1",
|
|
35
35
|
"typescript": "^6.0.2",
|
|
36
|
-
"vitest": "^4.1.
|
|
36
|
+
"vitest": "^4.1.10"
|
|
37
37
|
},
|
|
38
38
|
"files": [
|
|
39
39
|
"dist",
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async listResourceChecksums(): Promise<Map<string, string | undefined>> {\n const checksums = new Map<string, string | undefined>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: ['resourceId', 'contentChecksum'],\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || checksums.has(rid)) continue;\n const checksum = point.payload?.contentChecksum;\n checksums.set(rid, typeof checksum === 'string' ? checksum : undefined);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return checksums;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.searchBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n const results = await this.qdrant.search(collection, {\n vector: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return results.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAQO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAG3I,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,MACF;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,wBAAkE;AACtE,UAAM,YAAY,oBAAI,IAAgC;AACtD,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,cAAc,iBAAiB;AAAA,QAC9C,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,UAAU,IAAI,GAAG,EAAG;AACnD,cAAM,WAAW,MAAM,SAAS;AAChC,kBAAU,IAAI,KAAK,OAAO,aAAa,WAAW,WAAW,MAAS;AAAA,MACxE;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,YAAY,aAAa,EAAE,SAAS,CAAC;AAIvE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,OAAO;AACrB,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAE3C,UAAM,UAAU,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,MACnD,QAAQ;AAAA,MACR,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
|
package/dist/qdrant-R7IW5NII.js
DELETED
|
File without changes
|