@semiont/vectors 0.5.11 → 0.5.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -121,25 +121,37 @@ var QdrantVectorStore = class {
121
121
  ]);
122
122
  return resources.count + annotations.count;
123
123
  }
124
- async listResourceChecksums() {
125
- const checksums = /* @__PURE__ */ new Map();
124
+ async updateResourceEntityTypes(resourceId, entityTypes) {
125
+ await this.qdrant.setPayload("resources", {
126
+ payload: { entityTypes },
127
+ filter: {
128
+ must: [{ key: "resourceId", match: { value: String(resourceId) } }]
129
+ }
130
+ });
131
+ }
132
+ async listResourceStamps() {
133
+ const stamps = /* @__PURE__ */ new Map();
126
134
  let offset = void 0;
127
135
  do {
128
136
  const page = await this.qdrant.scroll("resources", {
129
137
  limit: 1e3,
130
138
  offset,
131
- with_payload: ["resourceId", "contentChecksum"],
139
+ with_payload: ["resourceId", "contentChecksum", "entityTypes"],
132
140
  with_vector: false
133
141
  });
134
142
  for (const point of page.points) {
135
143
  const rid = point.payload?.resourceId;
136
- if (typeof rid !== "string" || checksums.has(rid)) continue;
144
+ if (typeof rid !== "string" || stamps.has(rid)) continue;
137
145
  const checksum = point.payload?.contentChecksum;
138
- checksums.set(rid, typeof checksum === "string" ? checksum : void 0);
146
+ const entityTypes = point.payload?.entityTypes;
147
+ stamps.set(rid, {
148
+ contentChecksum: typeof checksum === "string" ? checksum : void 0,
149
+ entityTypes: Array.isArray(entityTypes) ? entityTypes.filter((t) => typeof t === "string") : []
150
+ });
139
151
  }
140
152
  offset = page.next_page_offset ?? void 0;
141
153
  } while (offset !== void 0 && offset !== null);
142
- return checksums;
154
+ return stamps;
143
155
  }
144
156
  async listAnnotationIds() {
145
157
  return this.scrollPayloadField("annotations", "annotationId");
@@ -266,4 +278,4 @@ var QdrantVectorStore = class {
266
278
  export {
267
279
  QdrantVectorStore
268
280
  };
269
- //# sourceMappingURL=chunk-TYDOHCBS.js.map
281
+ //# sourceMappingURL=chunk-VFNOQC4M.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n // Payload-only rewrite across the resource's points — no vectors touched,\n // no embedding involved (S13). Qdrant set_payload with a filter is a\n // no-op when the resource has no points.\n await this.qdrant.setPayload('resources', {\n payload: { entityTypes },\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: ['resourceId', 'contentChecksum', 'entityTypes'],\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || stamps.has(rid)) continue;\n const checksum = point.payload?.contentChecksum;\n const entityTypes = point.payload?.entityTypes;\n stamps.set(rid, {\n contentChecksum: typeof checksum === 'string' ? checksum : undefined,\n entityTypes: Array.isArray(entityTypes)\n ? entityTypes.filter((t): t is string => typeof t === 'string')\n : [],\n });\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.searchBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n const results = await this.qdrant.search(collection, {\n vector: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return results.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAQO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAG3I,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,MACF;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAI5F,UAAM,KAAK,OAAO,WAAW,aAAa;AAAA,MACxC,SAAS,EAAE,YAAY;AAAA,MACvB,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,cAAc,mBAAmB,aAAa;AAAA,QAC7D,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,OAAO,IAAI,GAAG,EAAG;AAChD,cAAM,WAAW,MAAM,SAAS;AAChC,cAAM,cAAc,MAAM,SAAS;AACnC,eAAO,IAAI,KAAK;AAAA,UACd,iBAAiB,OAAO,aAAa,WAAW,WAAW;AAAA,UAC3D,aAAa,MAAM,QAAQ,WAAW,IAClC,YAAY,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,IAC5D,CAAC;AAAA,QACP,CAAC;AAAA,MACH;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,YAAY,aAAa,EAAE,SAAS,CAAC;AAIvE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,OAAO;AACrB,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAE3C,UAAM,UAAU,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,MACnD,QAAQ;AAAA,MACR,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
package/dist/index.d.ts CHANGED
@@ -61,6 +61,13 @@ interface VectorStore {
61
61
  * discriminate by kind (e.g. exclude `['Question']` from recall).
62
62
  */
63
63
  upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void>;
64
+ /**
65
+ * Rewrite the `entityTypes` stamp on a resource's existing points —
66
+ * payload-only, no embedding involved (SMELTER-AXIOMS.md, S13: a tag edit
67
+ * must never trigger an embedding call). No-op when the resource has no
68
+ * points: the stamp rides the next embed.
69
+ */
70
+ updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
64
71
  upsertAnnotationVector(annotationId: AnnotationId, embedding: number[], payload: AnnotationPayload): Promise<void>;
65
72
  deleteResourceVectors(resourceId: ResourceId): Promise<void>;
66
73
  deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
@@ -90,10 +97,15 @@ interface VectorStore {
90
97
  count(): Promise<number>;
91
98
  /**
92
99
  * Distinct resourceIds present in the resources collection, each with its
93
- * stamped content checksum (undefined for points written before stamping
94
- * existed — reconciliation treats those as stale and re-embeds them).
100
+ * stamps: the content checksum (undefined for points written before
101
+ * stamping existed — reconciliation treats those as stale and re-embeds)
102
+ * and the entity-type set (missing stamp reads as `[]`). Drives both the
103
+ * S12 content-staleness diff and the S13 tag-staleness diff.
95
104
  */
96
- listResourceChecksums(): Promise<Map<string, string | undefined>>;
105
+ listResourceStamps(): Promise<Map<string, {
106
+ contentChecksum: string | undefined;
107
+ entityTypes: string[];
108
+ }>>;
97
109
  /** Distinct annotationIds present in the annotations collection. */
98
110
  listAnnotationIds(): Promise<Set<string>>;
99
111
  }
@@ -132,7 +144,11 @@ declare class QdrantVectorStore implements VectorStore {
132
144
  deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
133
145
  deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
134
146
  count(): Promise<number>;
135
- listResourceChecksums(): Promise<Map<string, string | undefined>>;
147
+ updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
148
+ listResourceStamps(): Promise<Map<string, {
149
+ contentChecksum: string | undefined;
150
+ entityTypes: string[];
151
+ }>>;
136
152
  listAnnotationIds(): Promise<Set<string>>;
137
153
  /** Collect the distinct values of one payload field across a collection. */
138
154
  private scrollPayloadField;
@@ -164,7 +180,11 @@ declare class MemoryVectorStore implements VectorStore {
164
180
  deleteAnnotationVector(annotationId: AnnotationId): Promise<void>;
165
181
  deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void>;
166
182
  count(): Promise<number>;
167
- listResourceChecksums(): Promise<Map<string, string | undefined>>;
183
+ updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void>;
184
+ listResourceStamps(): Promise<Map<string, {
185
+ contentChecksum: string | undefined;
186
+ entityTypes: string[];
187
+ }>>;
168
188
  listAnnotationIds(): Promise<Set<string>>;
169
189
  searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
170
190
  searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]>;
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  QdrantVectorStore
3
- } from "./chunk-TYDOHCBS.js";
3
+ } from "./chunk-VFNOQC4M.js";
4
4
  import {
5
5
  VoyageEmbeddingProvider
6
6
  } from "./chunk-UM3RNDW4.js";
@@ -80,14 +80,24 @@ var MemoryVectorStore = class {
80
80
  async count() {
81
81
  return this.resources.length + this.annotations.length;
82
82
  }
83
- async listResourceChecksums() {
84
- const checksums = /* @__PURE__ */ new Map();
83
+ async updateResourceEntityTypes(resourceId, entityTypes) {
85
84
  for (const p of this.resources) {
86
- if (!checksums.has(p.payload.resourceId)) {
87
- checksums.set(p.payload.resourceId, p.payload.contentChecksum);
85
+ if (p.payload.resourceId === String(resourceId)) {
86
+ p.payload.entityTypes = entityTypes;
88
87
  }
89
88
  }
90
- return checksums;
89
+ }
90
+ async listResourceStamps() {
91
+ const stamps = /* @__PURE__ */ new Map();
92
+ for (const p of this.resources) {
93
+ if (!stamps.has(p.payload.resourceId)) {
94
+ stamps.set(p.payload.resourceId, {
95
+ contentChecksum: p.payload.contentChecksum,
96
+ entityTypes: p.payload.entityTypes ?? []
97
+ });
98
+ }
99
+ }
100
+ return stamps;
91
101
  }
92
102
  async listAnnotationIds() {
93
103
  const ids = /* @__PURE__ */ new Set();
@@ -173,7 +183,7 @@ var MemoryVectorStore = class {
173
183
  async function createVectorStore(config) {
174
184
  let store;
175
185
  if (config.type === "qdrant") {
176
- const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-R7IW5NII.js");
186
+ const { QdrantVectorStore: QdrantVectorStore2 } = await import("./qdrant-A765QRQF.js");
177
187
  store = new QdrantVectorStore2({
178
188
  host: config.host ?? "localhost",
179
189
  port: config.port ?? 6333,
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts","../src/chunking.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async listResourceChecksums(): Promise<Map<string, string | undefined>> {\n const checksums = new Map<string, string | undefined>();\n for (const p of this.resources) {\n if (!checksums.has(p.payload.resourceId)) {\n checksums.set(p.payload.resourceId, p.payload.contentChecksum);\n }\n }\n return checksums;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n","/**\n * Text Chunking Utilities\n *\n * Splits long text into overlapping chunks for embedding.\n * Each chunk is a passage that fits within the embedding model's context window.\n */\n\nexport interface ChunkingConfig {\n chunkSize: number; // approximate tokens per chunk\n overlap: number; // tokens of overlap between adjacent chunks\n}\n\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n chunkSize: 512,\n overlap: 64,\n};\n\n/**\n * Rough token count estimate: ~4 characters per token for English text.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into overlapping chunks.\n *\n * Splits on paragraph boundaries when possible, falling back to sentence\n * boundaries, then word boundaries. Each chunk overlaps with the previous\n * by `overlap` tokens worth of text.\n */\nexport function chunkText(text: string, config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG): string[] {\n if (text.length === 0) return [];\n const totalTokens = estimateTokens(text);\n if (totalTokens <= config.chunkSize) {\n return [text];\n }\n\n const chunkChars = config.chunkSize * 4;\n const overlapChars = config.overlap * 4;\n const chunks: string[] = [];\n let start = 0;\n\n while (start < text.length) {\n let end = Math.min(start + chunkChars, text.length);\n\n // Try to break at a paragraph boundary\n if (end < text.length) {\n const paraBreak = text.lastIndexOf('\\n\\n', end);\n if (paraBreak > start + chunkChars / 2) {\n end = paraBreak;\n } else {\n // Try sentence boundary\n const sentenceBreak = text.lastIndexOf('. ', end);\n if (sentenceBreak > start + chunkChars / 2) {\n end = sentenceBreak + 1;\n } else {\n // Try word boundary\n const wordBreak = text.lastIndexOf(' ', end);\n if (wordBreak > start + chunkChars / 2) {\n end = wordBreak;\n }\n }\n }\n }\n\n chunks.push(text.slice(start, end).trim());\n const nextStart = end - overlapChars;\n start = nextStart > start ? nextStart : end;\n if (start >= text.length) break;\n }\n\n return chunks.filter(c => c.length > 0);\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAE3I,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,wBAAkE;AACtE,UAAM,YAAY,oBAAI,IAAgC;AACtD,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,UAAU,IAAI,EAAE,QAAQ,UAAU,GAAG;AACxC,kBAAU,IAAI,EAAE,QAAQ,YAAY,EAAE,QAAQ,eAAe;AAAA,MAC/D;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;;;AC7MA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;;;ACtBO,IAAM,0BAA0C;AAAA,EACrD,WAAW;AAAA,EACX,SAAS;AACX;AAKA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASO,SAAS,UAAU,MAAc,SAAyB,yBAAmC;AAClG,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAC/B,QAAM,cAAc,eAAe,IAAI;AACvC,MAAI,eAAe,OAAO,WAAW;AACnC,WAAO,CAAC,IAAI;AAAA,EACd;AAEA,QAAM,aAAa,OAAO,YAAY;AACtC,QAAM,eAAe,OAAO,UAAU;AACtC,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,QAAI,MAAM,KAAK,IAAI,QAAQ,YAAY,KAAK,MAAM;AAGlD,QAAI,MAAM,KAAK,QAAQ;AACrB,YAAM,YAAY,KAAK,YAAY,QAAQ,GAAG;AAC9C,UAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,cAAM;AAAA,MACR,OAAO;AAEL,cAAM,gBAAgB,KAAK,YAAY,MAAM,GAAG;AAChD,YAAI,gBAAgB,QAAQ,aAAa,GAAG;AAC1C,gBAAM,gBAAgB;AAAA,QACxB,OAAO;AAEL,gBAAM,YAAY,KAAK,YAAY,KAAK,GAAG;AAC3C,cAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC;AACzC,UAAM,YAAY,MAAM;AACxB,YAAQ,YAAY,QAAQ,YAAY;AACxC,QAAI,SAAS,KAAK,OAAQ;AAAA,EAC5B;AAEA,SAAO,OAAO,OAAO,OAAK,EAAE,SAAS,CAAC;AACxC;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
1
+ {"version":3,"sources":["../src/store/memory.ts","../src/store/factory.ts","../src/embedding/factory.ts","../src/chunking.ts"],"sourcesContent":["/**\n * In-Memory VectorStore Implementation\n *\n * For testing and development without a running Qdrant instance.\n * Uses brute-force cosine similarity search.\n */\n\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\ninterface StoredPoint {\n id: string;\n vector: number[];\n payload: {\n resourceId: string;\n annotationId?: string;\n chunkIndex?: number;\n text: string;\n contentChecksum?: string;\n motivation?: string;\n entityTypes?: string[];\n };\n}\n\nfunction cosineSimilarity(a: number[], b: number[]): number {\n let dotProduct = 0;\n let normA = 0;\n let normB = 0;\n for (let i = 0; i < a.length; i++) {\n dotProduct += a[i] * b[i];\n normA += a[i] * a[i];\n normB += b[i] * b[i];\n }\n const denom = Math.sqrt(normA) * Math.sqrt(normB);\n return denom === 0 ? 0 : dotProduct / denom;\n}\n\nexport class MemoryVectorStore implements VectorStore {\n private resources: StoredPoint[] = [];\n private annotations: StoredPoint[] = [];\n private connected = false;\n\n async connect(): Promise<void> {\n this.connected = true;\n }\n\n async disconnect(): Promise<void> {\n this.connected = false;\n }\n\n async clearAll(): Promise<void> {\n this.resources = [];\n this.annotations = [];\n }\n\n isConnected(): boolean {\n return this.connected;\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Remove existing vectors for this resource\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n\n for (const chunk of chunks) {\n this.resources.push({\n id: `${resourceId}-${chunk.chunkIndex}`,\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n });\n }\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n this.annotations.push({\n id: String(annotationId),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n this.resources = this.resources.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.id !== String(annotationId));\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n this.annotations = this.annotations.filter(p => p.payload.resourceId !== String(resourceId));\n }\n\n async count(): Promise<number> {\n return this.resources.length + this.annotations.length;\n }\n\n async updateResourceEntityTypes(resourceId: ResourceId, entityTypes: string[]): Promise<void> {\n for (const p of this.resources) {\n if (p.payload.resourceId === String(resourceId)) {\n p.payload.entityTypes = entityTypes;\n }\n }\n }\n\n async listResourceStamps(): Promise<Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>> {\n const stamps = new Map<string, { contentChecksum: string | undefined; entityTypes: string[] }>();\n for (const p of this.resources) {\n if (!stamps.has(p.payload.resourceId)) {\n stamps.set(p.payload.resourceId, {\n contentChecksum: p.payload.contentChecksum,\n entityTypes: p.payload.entityTypes ?? [],\n });\n }\n }\n return stamps;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n const ids = new Set<string>();\n for (const p of this.annotations) {\n if (p.payload.annotationId) ids.add(p.payload.annotationId);\n }\n return ids;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.resources, embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search(this.annotations, embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n const rid = String(resourceId);\n const queryPoints = this.resources.filter(p => p.payload.resourceId === rid);\n if (queryPoints.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter: SearchOptions['filter'] = { ...opts.filter, excludeResourceId: resourceId };\n\n // Per-chunk max-sim, merged by resource: each candidate point scores as the\n // best similarity to any of the source's query chunks; keep, per target\n // resource, the single best-matching point (its score and its text).\n const bestByResource = new Map<string, StoredPoint & { score: number }>();\n for (const cand of this.resources) {\n if (!this.passesFilter(cand, filter)) continue;\n let best = -Infinity;\n for (const q of queryPoints) {\n const score = cosineSimilarity(q.vector, cand.vector);\n if (score > best) best = score;\n }\n const prev = bestByResource.get(cand.payload.resourceId);\n if (!prev || best > prev.score) {\n bestByResource.set(cand.payload.resourceId, { ...cand, score: best });\n }\n }\n\n let merged = [...bestByResource.values()];\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n merged = merged.filter(s => s.score >= threshold);\n }\n merged.sort((a, b) => b.score - a.score);\n return merged.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private passesFilter(p: StoredPoint, filter: SearchOptions['filter']): boolean {\n if (!filter) return true;\n const f = filter;\n if (f.resourceId && p.payload.resourceId !== String(f.resourceId)) return false;\n if (f.excludeResourceId && p.payload.resourceId === String(f.excludeResourceId)) return false;\n if (f.motivation && p.payload.motivation !== f.motivation) return false;\n if (f.entityTypes && f.entityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (!f.entityTypes.some(t => pTypes.includes(t))) return false;\n }\n if (f.excludeEntityTypes && f.excludeEntityTypes.length > 0) {\n const pTypes = p.payload.entityTypes ?? [];\n if (f.excludeEntityTypes.some(t => pTypes.includes(t))) return false;\n }\n return true;\n }\n\n private search(points: StoredPoint[], embedding: number[], opts: SearchOptions): VectorSearchResult[] {\n const filtered = points.filter(p => this.passesFilter(p, opts.filter));\n\n const scored = filtered.map(p => ({\n ...p,\n score: cosineSimilarity(embedding, p.vector),\n }));\n\n scored.sort((a, b) => b.score - a.score);\n\n if (opts.scoreThreshold !== undefined) {\n const threshold = opts.scoreThreshold;\n return scored\n .filter(s => s.score >= threshold)\n .slice(0, opts.limit)\n .map(s => this.toResult(s));\n }\n\n return scored.slice(0, opts.limit).map(s => this.toResult(s));\n }\n\n private toResult(s: StoredPoint & { score: number }): VectorSearchResult {\n return {\n id: s.id,\n score: s.score,\n resourceId: s.payload.resourceId as ResourceId,\n annotationId: s.payload.annotationId as AnnotationId | undefined,\n text: s.payload.text,\n entityTypes: s.payload.entityTypes,\n };\n }\n}\n","/**\n * VectorStore Factory\n *\n * Creates a connected VectorStore instance based on configuration.\n */\n\nimport type { VectorStore } from './interface';\nimport { MemoryVectorStore } from './memory';\n\nexport interface VectorStoreConfig {\n type: 'qdrant' | 'memory';\n host?: string;\n port?: number;\n dimensions: number;\n}\n\nexport async function createVectorStore(config: VectorStoreConfig): Promise<VectorStore> {\n let store: VectorStore;\n\n if (config.type === 'qdrant') {\n const { QdrantVectorStore } = await import('./qdrant');\n store = new QdrantVectorStore({\n host: config.host ?? 'localhost',\n port: config.port ?? 6333,\n dimensions: config.dimensions,\n });\n } else {\n store = new MemoryVectorStore();\n }\n\n await store.connect();\n return store;\n}\n","/**\n * EmbeddingProvider Factory\n */\n\nimport type { EmbeddingProvider } from './interface';\n\nexport interface EmbeddingConfig {\n type: 'voyage' | 'ollama';\n model: string;\n apiKey?: string;\n baseURL?: string;\n endpoint?: string;\n}\n\nexport async function createEmbeddingProvider(config: EmbeddingConfig): Promise<EmbeddingProvider> {\n if (config.type === 'voyage') {\n const { VoyageEmbeddingProvider } = await import('./voyage');\n if (!config.apiKey) throw new Error('apiKey is required for Voyage embedding provider');\n return new VoyageEmbeddingProvider({\n apiKey: config.apiKey,\n model: config.model,\n endpoint: config.endpoint,\n });\n }\n\n if (config.type === 'ollama') {\n const { OllamaEmbeddingProvider } = await import('./ollama');\n return new OllamaEmbeddingProvider({\n model: config.model,\n baseURL: config.baseURL,\n });\n }\n\n throw new Error(`Unknown embedding provider type: ${config.type}`);\n}\n","/**\n * Text Chunking Utilities\n *\n * Splits long text into overlapping chunks for embedding.\n * Each chunk is a passage that fits within the embedding model's context window.\n */\n\nexport interface ChunkingConfig {\n chunkSize: number; // approximate tokens per chunk\n overlap: number; // tokens of overlap between adjacent chunks\n}\n\nexport const DEFAULT_CHUNKING_CONFIG: ChunkingConfig = {\n chunkSize: 512,\n overlap: 64,\n};\n\n/**\n * Rough token count estimate: ~4 characters per token for English text.\n */\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n/**\n * Split text into overlapping chunks.\n *\n * Splits on paragraph boundaries when possible, falling back to sentence\n * boundaries, then word boundaries. Each chunk overlaps with the previous\n * by `overlap` tokens worth of text.\n */\nexport function chunkText(text: string, config: ChunkingConfig = DEFAULT_CHUNKING_CONFIG): string[] {\n if (text.length === 0) return [];\n const totalTokens = estimateTokens(text);\n if (totalTokens <= config.chunkSize) {\n return [text];\n }\n\n const chunkChars = config.chunkSize * 4;\n const overlapChars = config.overlap * 4;\n const chunks: string[] = [];\n let start = 0;\n\n while (start < text.length) {\n let end = Math.min(start + chunkChars, text.length);\n\n // Try to break at a paragraph boundary\n if (end < text.length) {\n const paraBreak = text.lastIndexOf('\\n\\n', end);\n if (paraBreak > start + chunkChars / 2) {\n end = paraBreak;\n } else {\n // Try sentence boundary\n const sentenceBreak = text.lastIndexOf('. ', end);\n if (sentenceBreak > start + chunkChars / 2) {\n end = sentenceBreak + 1;\n } else {\n // Try word boundary\n const wordBreak = text.lastIndexOf(' ', end);\n if (wordBreak > start + chunkChars / 2) {\n end = wordBreak;\n }\n }\n }\n }\n\n chunks.push(text.slice(start, end).trim());\n const nextStart = end - overlapChars;\n start = nextStart > start ? nextStart : end;\n if (start >= text.length) break;\n }\n\n return chunks.filter(c => c.length > 0);\n}\n"],"mappings":";;;;;;;;;;;AAwBA,SAAS,iBAAiB,GAAa,GAAqB;AAC1D,MAAI,aAAa;AACjB,MAAI,QAAQ;AACZ,MAAI,QAAQ;AACZ,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,kBAAc,EAAE,CAAC,IAAI,EAAE,CAAC;AACxB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AACnB,aAAS,EAAE,CAAC,IAAI,EAAE,CAAC;AAAA,EACrB;AACA,QAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;AAChD,SAAO,UAAU,IAAI,IAAI,aAAa;AACxC;AAEO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,YAA2B,CAAC;AAAA,EAC5B,cAA6B,CAAC;AAAA,EAC9B,YAAY;AAAA,EAEpB,MAAM,UAAyB;AAC7B,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,YAAY;AAAA,EACnB;AAAA,EAEA,MAAM,WAA0B;AAC9B,SAAK,YAAY,CAAC;AAClB,SAAK,cAAc,CAAC;AAAA,EACtB;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAE3I,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAEvF,eAAW,SAAS,QAAQ;AAC1B,WAAK,UAAU,KAAK;AAAA,QAClB,IAAI,GAAG,UAAU,IAAI,MAAM,UAAU;AAAA,QACrC,QAAQ,MAAM;AAAA,QACd,SAAS;AAAA,UACP,YAAY,OAAO,UAAU;AAAA,UAC7B,YAAY,MAAM;AAAA,UAClB,MAAM,MAAM;AAAA,UACZ;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAC7E,SAAK,YAAY,KAAK;AAAA,MACpB,IAAI,OAAO,YAAY;AAAA,MACvB,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,cAAc,OAAO,QAAQ,YAAY;AAAA,QACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,QACrC,YAAY,QAAQ;AAAA,QACpB,aAAa,QAAQ;AAAA,QACrB,MAAM,QAAQ;AAAA,MAChB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,SAAK,YAAY,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,OAAO,OAAO,YAAY,CAAC;AAAA,EAC/E;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,SAAK,cAAc,KAAK,YAAY,OAAO,OAAK,EAAE,QAAQ,eAAe,OAAO,UAAU,CAAC;AAAA,EAC7F;AAAA,EAEA,MAAM,QAAyB;AAC7B,WAAO,KAAK,UAAU,SAAS,KAAK,YAAY;AAAA,EAClD;AAAA,EAEA,MAAM,0BAA0B,YAAwB,aAAsC;AAC5F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,EAAE,QAAQ,eAAe,OAAO,UAAU,GAAG;AAC/C,UAAE,QAAQ,cAAc;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,qBAA2G;AAC/G,UAAM,SAAS,oBAAI,IAA4E;AAC/F,eAAW,KAAK,KAAK,WAAW;AAC9B,UAAI,CAAC,OAAO,IAAI,EAAE,QAAQ,UAAU,GAAG;AACrC,eAAO,IAAI,EAAE,QAAQ,YAAY;AAAA,UAC/B,iBAAiB,EAAE,QAAQ;AAAA,UAC3B,aAAa,EAAE,QAAQ,eAAe,CAAC;AAAA,QACzC,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,UAAM,MAAM,oBAAI,IAAY;AAC5B,eAAW,KAAK,KAAK,aAAa;AAChC,UAAI,EAAE,QAAQ,aAAc,KAAI,IAAI,EAAE,QAAQ,YAAY;AAAA,IAC5D;AACA,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,KAAK,WAAW,WAAW,IAAI;AAAA,EACpD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,KAAK,aAAa,WAAW,IAAI;AAAA,EACtD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AACjG,UAAM,MAAM,OAAO,UAAU;AAC7B,UAAM,cAAc,KAAK,UAAU,OAAO,OAAK,EAAE,QAAQ,eAAe,GAAG;AAC3E,QAAI,YAAY,WAAW,EAAG,QAAO,CAAC;AAGtC,UAAM,SAAkC,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW;AAKxF,UAAM,iBAAiB,oBAAI,IAA6C;AACxE,eAAW,QAAQ,KAAK,WAAW;AACjC,UAAI,CAAC,KAAK,aAAa,MAAM,MAAM,EAAG;AACtC,UAAI,OAAO;AACX,iBAAW,KAAK,aAAa;AAC3B,cAAM,QAAQ,iBAAiB,EAAE,QAAQ,KAAK,MAAM;AACpD,YAAI,QAAQ,KAAM,QAAO;AAAA,MAC3B;AACA,YAAM,OAAO,eAAe,IAAI,KAAK,QAAQ,UAAU;AACvD,UAAI,CAAC,QAAQ,OAAO,KAAK,OAAO;AAC9B,uBAAe,IAAI,KAAK,QAAQ,YAAY,EAAE,GAAG,MAAM,OAAO,KAAK,CAAC;AAAA,MACtE;AAAA,IACF;AAEA,QAAI,SAAS,CAAC,GAAG,eAAe,OAAO,CAAC;AACxC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,eAAS,OAAO,OAAO,OAAK,EAAE,SAAS,SAAS;AAAA,IAClD;AACA,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AACvC,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,aAAa,GAAgB,QAA0C;AAC7E,QAAI,CAAC,OAAQ,QAAO;AACpB,UAAM,IAAI;AACV,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,OAAO,EAAE,UAAU,EAAG,QAAO;AAC1E,QAAI,EAAE,qBAAqB,EAAE,QAAQ,eAAe,OAAO,EAAE,iBAAiB,EAAG,QAAO;AACxF,QAAI,EAAE,cAAc,EAAE,QAAQ,eAAe,EAAE,WAAY,QAAO;AAClE,QAAI,EAAE,eAAe,EAAE,YAAY,SAAS,GAAG;AAC7C,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,CAAC,EAAE,YAAY,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IAC3D;AACA,QAAI,EAAE,sBAAsB,EAAE,mBAAmB,SAAS,GAAG;AAC3D,YAAM,SAAS,EAAE,QAAQ,eAAe,CAAC;AACzC,UAAI,EAAE,mBAAmB,KAAK,OAAK,OAAO,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,IACjE;AACA,WAAO;AAAA,EACT;AAAA,EAEQ,OAAO,QAAuB,WAAqB,MAA2C;AACpG,UAAM,WAAW,OAAO,OAAO,OAAK,KAAK,aAAa,GAAG,KAAK,MAAM,CAAC;AAErE,UAAM,SAAS,SAAS,IAAI,QAAM;AAAA,MAChC,GAAG;AAAA,MACH,OAAO,iBAAiB,WAAW,EAAE,MAAM;AAAA,IAC7C,EAAE;AAEF,WAAO,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAEvC,QAAI,KAAK,mBAAmB,QAAW;AACrC,YAAM,YAAY,KAAK;AACvB,aAAO,OACJ,OAAO,OAAK,EAAE,SAAS,SAAS,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,IAC9B;AAEA,WAAO,OAAO,MAAM,GAAG,KAAK,KAAK,EAAE,IAAI,OAAK,KAAK,SAAS,CAAC,CAAC;AAAA,EAC9D;AAAA,EAEQ,SAAS,GAAwD;AACvE,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB;AAAA,EACF;AACF;;;ACxNA,eAAsB,kBAAkB,QAAiD;AACvF,MAAI;AAEJ,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,mBAAAA,mBAAkB,IAAI,MAAM,OAAO,sBAAU;AACrD,YAAQ,IAAIA,mBAAkB;AAAA,MAC5B,MAAM,OAAO,QAAQ;AAAA,MACrB,MAAM,OAAO,QAAQ;AAAA,MACrB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH,OAAO;AACL,YAAQ,IAAI,kBAAkB;AAAA,EAChC;AAEA,QAAM,MAAM,QAAQ;AACpB,SAAO;AACT;;;AClBA,eAAsB,wBAAwB,QAAqD;AACjG,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,QAAI,CAAC,OAAO,OAAQ,OAAM,IAAI,MAAM,kDAAkD;AACtF,WAAO,IAAIA,yBAAwB;AAAA,MACjC,QAAQ,OAAO;AAAA,MACf,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,SAAS,UAAU;AAC5B,UAAM,EAAE,yBAAAC,yBAAwB,IAAI,MAAM,OAAO,sBAAU;AAC3D,WAAO,IAAIA,yBAAwB;AAAA,MACjC,OAAO,OAAO;AAAA,MACd,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AAEA,QAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AACnE;;;ACtBO,IAAM,0BAA0C;AAAA,EACrD,WAAW;AAAA,EACX,SAAS;AACX;AAKA,SAAS,eAAe,MAAsB;AAC5C,SAAO,KAAK,KAAK,KAAK,SAAS,CAAC;AAClC;AASO,SAAS,UAAU,MAAc,SAAyB,yBAAmC;AAClG,MAAI,KAAK,WAAW,EAAG,QAAO,CAAC;AAC/B,QAAM,cAAc,eAAe,IAAI;AACvC,MAAI,eAAe,OAAO,WAAW;AACnC,WAAO,CAAC,IAAI;AAAA,EACd;AAEA,QAAM,aAAa,OAAO,YAAY;AACtC,QAAM,eAAe,OAAO,UAAU;AACtC,QAAM,SAAmB,CAAC;AAC1B,MAAI,QAAQ;AAEZ,SAAO,QAAQ,KAAK,QAAQ;AAC1B,QAAI,MAAM,KAAK,IAAI,QAAQ,YAAY,KAAK,MAAM;AAGlD,QAAI,MAAM,KAAK,QAAQ;AACrB,YAAM,YAAY,KAAK,YAAY,QAAQ,GAAG;AAC9C,UAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,cAAM;AAAA,MACR,OAAO;AAEL,cAAM,gBAAgB,KAAK,YAAY,MAAM,GAAG;AAChD,YAAI,gBAAgB,QAAQ,aAAa,GAAG;AAC1C,gBAAM,gBAAgB;AAAA,QACxB,OAAO;AAEL,gBAAM,YAAY,KAAK,YAAY,KAAK,GAAG;AAC3C,cAAI,YAAY,QAAQ,aAAa,GAAG;AACtC,kBAAM;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,KAAK,MAAM,OAAO,GAAG,EAAE,KAAK,CAAC;AACzC,UAAM,YAAY,MAAM;AACxB,YAAQ,YAAY,QAAQ,YAAY;AACxC,QAAI,SAAS,KAAK,OAAQ;AAAA,EAC5B;AAEA,SAAO,OAAO,OAAO,OAAK,EAAE,SAAS,CAAC;AACxC;","names":["QdrantVectorStore","VoyageEmbeddingProvider","OllamaEmbeddingProvider"]}
@@ -0,0 +1,7 @@
1
+ import {
2
+ QdrantVectorStore
3
+ } from "./chunk-VFNOQC4M.js";
4
+ export {
5
+ QdrantVectorStore
6
+ };
7
+ //# sourceMappingURL=qdrant-A765QRQF.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@semiont/vectors",
3
- "version": "0.5.11",
3
+ "version": "0.5.13",
4
4
  "engines": {
5
5
  "node": ">=24.0.0"
6
6
  },
@@ -25,7 +25,7 @@
25
25
  },
26
26
  "dependencies": {
27
27
  "@qdrant/js-client-rest": "^1.18.0",
28
- "@semiont/core": "0.5.11"
28
+ "@semiont/core": "0.5.13"
29
29
  },
30
30
  "devDependencies": {
31
31
  "@vitest/coverage-v8": "^4.1.8",
@@ -33,7 +33,7 @@
33
33
  "rollup-plugin-dts": "^6.4.1",
34
34
  "tsup": "^8.5.1",
35
35
  "typescript": "^6.0.2",
36
- "vitest": "^4.1.8"
36
+ "vitest": "^4.1.10"
37
37
  },
38
38
  "files": [
39
39
  "dist",
@@ -1 +0,0 @@
1
- {"version":3,"sources":["../src/store/qdrant.ts"],"sourcesContent":["/**\n * Qdrant VectorStore Implementation\n *\n * Uses the Qdrant REST API via @qdrant/js-client-rest.\n * Manages two collections: 'resources' and 'annotations'.\n */\n\nimport { createHash } from 'crypto';\nimport type { QdrantClient, Schemas } from '@qdrant/js-client-rest';\nimport type { ResourceId, AnnotationId } from '@semiont/core';\nimport type { VectorStore, EmbeddingChunk, AnnotationPayload, VectorSearchResult, SearchOptions } from './interface';\n\n/**\n * Generate a deterministic UUID v5-style ID from an arbitrary string.\n * Qdrant requires point IDs to be UUIDs or unsigned integers.\n */\nfunction toQdrantId(input: string): string {\n const hex = createHash('md5').update(input).digest('hex');\n return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20, 32)}`;\n}\n\nexport interface QdrantConfig {\n host: string;\n port: number;\n dimensions: number;\n}\n\nexport class QdrantVectorStore implements VectorStore {\n private client: QdrantClient | null = null;\n private config: QdrantConfig;\n\n constructor(config: QdrantConfig) {\n this.config = config;\n }\n\n private get qdrant(): QdrantClient {\n if (!this.client) throw new Error('QdrantVectorStore is not connected');\n return this.client;\n }\n\n async connect(): Promise<void> {\n const { QdrantClient } = await import('@qdrant/js-client-rest');\n this.client = new QdrantClient({\n host: this.config.host,\n port: this.config.port,\n });\n\n // Ensure collections exist\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n // Payload indexes so filtered operations scale:\n // - entityTypes: the excludeEntityTypes recall filter.\n // - resourceId: searchByResource's by-resource scroll + self-exclusion, and\n // the per-resource delete paths (deleteResourceVectors /\n // deleteAnnotationVectorsForResource).\n await this.ensurePayloadIndex('resources', 'entityTypes');\n await this.ensurePayloadIndex('resources', 'resourceId');\n await this.ensurePayloadIndex('annotations', 'resourceId');\n }\n\n async disconnect(): Promise<void> {\n this.client = null;\n }\n\n async clearAll(): Promise<void> {\n try { await this.qdrant.deleteCollection('resources'); } catch { /* may not exist */ }\n try { await this.qdrant.deleteCollection('annotations'); } catch { /* may not exist */ }\n await this.ensureCollection('resources', this.config.dimensions);\n await this.ensureCollection('annotations', this.config.dimensions);\n }\n\n isConnected(): boolean {\n return this.client !== null;\n }\n\n private async ensureCollection(name: string, dimensions: number): Promise<void> {\n try {\n await this.qdrant.getCollection(name);\n } catch {\n await this.qdrant.createCollection(name, {\n vectors: { size: dimensions, distance: 'Cosine' },\n });\n }\n }\n\n /**\n * Idempotently create a keyword payload index. Qdrant accepts a repeat call\n * for an already-indexed field, so this runs safely on every connect and\n * back-fills the index on collections created before the field was indexed.\n */\n private async ensurePayloadIndex(collection: string, field: string): Promise<void> {\n try {\n await this.qdrant.createPayloadIndex(collection, { field_name: field, field_schema: 'keyword' });\n } catch { /* already indexed, or created concurrently */ }\n }\n\n async upsertResourceVectors(resourceId: ResourceId, chunks: EmbeddingChunk[], contentChecksum: string, entityTypes: string[]): Promise<void> {\n // Replace semantics: purge existing chunks first, or a resource that\n // shrinks leaves orphan points at the higher chunk indices.\n await this.deleteResourceVectors(resourceId);\n if (chunks.length === 0) return;\n\n const points = chunks.map((chunk) => ({\n id: toQdrantId(`${resourceId}-${chunk.chunkIndex}`),\n vector: chunk.embedding,\n payload: {\n resourceId: String(resourceId),\n chunkIndex: chunk.chunkIndex,\n text: chunk.text,\n contentChecksum,\n entityTypes,\n },\n }));\n\n await this.qdrant.upsert('resources', { points });\n }\n\n async upsertAnnotationVector(\n annotationId: AnnotationId,\n embedding: number[],\n payload: AnnotationPayload\n ): Promise<void> {\n await this.qdrant.upsert('annotations', {\n points: [{\n id: toQdrantId(String(annotationId)),\n vector: embedding,\n payload: {\n annotationId: String(payload.annotationId),\n resourceId: String(payload.resourceId),\n motivation: payload.motivation,\n entityTypes: payload.entityTypes,\n text: payload.exactText,\n },\n }],\n });\n }\n\n async deleteResourceVectors(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('resources', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async deleteAnnotationVector(annotationId: AnnotationId): Promise<void> {\n await this.qdrant.delete('annotations', {\n points: [toQdrantId(String(annotationId))],\n });\n }\n\n async deleteAnnotationVectorsForResource(resourceId: ResourceId): Promise<void> {\n await this.qdrant.delete('annotations', {\n filter: {\n must: [{ key: 'resourceId', match: { value: String(resourceId) } }],\n },\n });\n }\n\n async count(): Promise<number> {\n const [resources, annotations] = await Promise.all([\n this.qdrant.count('resources', { exact: true }),\n this.qdrant.count('annotations', { exact: true }),\n ]);\n return resources.count + annotations.count;\n }\n\n async listResourceChecksums(): Promise<Map<string, string | undefined>> {\n const checksums = new Map<string, string | undefined>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n limit: 1000,\n offset,\n with_payload: ['resourceId', 'contentChecksum'],\n with_vector: false,\n });\n for (const point of page.points) {\n const rid = point.payload?.resourceId;\n if (typeof rid !== 'string' || checksums.has(rid)) continue;\n const checksum = point.payload?.contentChecksum;\n checksums.set(rid, typeof checksum === 'string' ? checksum : undefined);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return checksums;\n }\n\n async listAnnotationIds(): Promise<Set<string>> {\n return this.scrollPayloadField('annotations', 'annotationId');\n }\n\n /** Collect the distinct values of one payload field across a collection. */\n private async scrollPayloadField(collection: string, field: string): Promise<Set<string>> {\n const values = new Set<string>();\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll(collection, {\n limit: 1000,\n offset,\n with_payload: [field],\n with_vector: false,\n });\n for (const point of page.points) {\n const value = point.payload?.[field];\n if (typeof value === 'string') values.add(value);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n return values;\n }\n\n async searchResources(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('resources', embedding, opts);\n }\n\n async searchAnnotations(embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n return this.search('annotations', embedding, opts);\n }\n\n async searchByResource(resourceId: ResourceId, opts: SearchOptions): Promise<VectorSearchResult[]> {\n // Fetch the resource's stored chunk vectors (page through all of them so a\n // long resource isn't silently truncated).\n const queryVectors: number[][] = [];\n let offset: Schemas['ScrollRequest']['offset'] = undefined;\n do {\n const page = await this.qdrant.scroll('resources', {\n filter: { must: [{ key: 'resourceId', match: { value: String(resourceId) } }] },\n with_vector: true,\n with_payload: false,\n limit: 256,\n offset,\n });\n for (const point of page.points) {\n if (Array.isArray(point.vector)) queryVectors.push(point.vector as number[]);\n }\n offset = page.next_page_offset ?? undefined;\n } while (offset !== undefined && offset !== null);\n\n if (queryVectors.length === 0) return [];\n\n // Self-exclude the source; carry the caller's filter (e.g. excludeEntityTypes).\n const filter = this.buildFilter({ ...opts.filter, excludeResourceId: resourceId });\n\n // One batched search per query chunk (single round-trip), top-`limit` each;\n // over-fetch beyond `limit` is unnecessary because the max-sim merge only\n // needs a target in some chunk's top-K to surface.\n const searches = queryVectors.map((vector) => ({\n vector,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n }));\n const batches = await this.qdrant.searchBatch('resources', { searches });\n\n // Max-sim merge: dedup by resourceId, keep the best (query-chunk × target-chunk)\n // score and the best-matching target chunk's payload.\n const bestByResource = new Map<string, { id: string; score: number; payload: Record<string, unknown> }>();\n for (const batch of batches) {\n for (const r of batch) {\n const payload = r.payload ?? {};\n const tid = String(payload.resourceId);\n const prev = bestByResource.get(tid);\n if (!prev || r.score > prev.score) {\n bestByResource.set(tid, { id: String(r.id), score: r.score, payload });\n }\n }\n }\n\n return [...bestByResource.values()]\n .sort((a, b) => b.score - a.score)\n .slice(0, opts.limit)\n .map((m) => ({\n id: m.id,\n score: m.score,\n resourceId: m.payload.resourceId as ResourceId,\n annotationId: m.payload.annotationId as AnnotationId | undefined,\n text: m.payload.text as string,\n entityTypes: m.payload.entityTypes as string[] | undefined,\n }));\n }\n\n private async search(collection: string, embedding: number[], opts: SearchOptions): Promise<VectorSearchResult[]> {\n const filter = this.buildFilter(opts.filter);\n\n const results = await this.qdrant.search(collection, {\n vector: embedding,\n limit: opts.limit,\n score_threshold: opts.scoreThreshold,\n filter: filter ?? undefined,\n with_payload: true,\n });\n\n return results.map((r) => {\n const payload = r.payload ?? {};\n return {\n id: String(r.id),\n score: r.score,\n resourceId: payload.resourceId as ResourceId,\n annotationId: payload.annotationId as AnnotationId | undefined,\n text: payload.text as string,\n entityTypes: payload.entityTypes as string[] | undefined,\n };\n });\n }\n\n private buildFilter(filter?: SearchOptions['filter']): Schemas['Filter'] | null {\n if (!filter) return null;\n\n const must: Schemas['FieldCondition'][] = [];\n\n if (filter.entityTypes && filter.entityTypes.length > 0) {\n // any-of: match payloads whose `entityTypes` array contains at least one\n // of the requested types. Matches the memory store's `some(t => ...)`\n // semantics; pushing one `must` clause per type would mean all-of.\n must.push({ key: 'entityTypes', match: { any: filter.entityTypes } });\n }\n\n if (filter.resourceId) {\n must.push({ key: 'resourceId', match: { value: String(filter.resourceId) } });\n }\n\n if (filter.motivation) {\n must.push({ key: 'motivation', match: { value: filter.motivation } });\n }\n\n const must_not: Schemas['FieldCondition'][] = [];\n\n if (filter.excludeResourceId) {\n must_not.push({ key: 'resourceId', match: { value: String(filter.excludeResourceId) } });\n }\n\n if (filter.excludeEntityTypes && filter.excludeEntityTypes.length > 0) {\n // any-of exclusion: drop points whose entityTypes contain any of these.\n must_not.push({ key: 'entityTypes', match: { any: filter.excludeEntityTypes } });\n }\n\n if (must.length === 0 && must_not.length === 0) return null;\n\n return {\n ...(must.length > 0 ? { must } : {}),\n ...(must_not.length > 0 ? { must_not } : {}),\n };\n }\n}\n"],"mappings":";AAOA,SAAS,kBAAkB;AAS3B,SAAS,WAAW,OAAuB;AACzC,QAAM,MAAM,WAAW,KAAK,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK;AACxD,SAAO,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,MAAM,GAAG,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC,IAAI,IAAI,MAAM,IAAI,EAAE,CAAC;AAC9G;AAQO,IAAM,oBAAN,MAA+C;AAAA,EAC5C,SAA8B;AAAA,EAC9B;AAAA,EAER,YAAY,QAAsB;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,IAAY,SAAuB;AACjC,QAAI,CAAC,KAAK,OAAQ,OAAM,IAAI,MAAM,oCAAoC;AACtE,WAAO,KAAK;AAAA,EACd;AAAA,EAEA,MAAM,UAAyB;AAC7B,UAAM,EAAE,aAAa,IAAI,MAAM,OAAO,wBAAwB;AAC9D,SAAK,SAAS,IAAI,aAAa;AAAA,MAC7B,MAAM,KAAK,OAAO;AAAA,MAClB,MAAM,KAAK,OAAO;AAAA,IACpB,CAAC;AAGD,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAMjE,UAAM,KAAK,mBAAmB,aAAa,aAAa;AACxD,UAAM,KAAK,mBAAmB,aAAa,YAAY;AACvD,UAAM,KAAK,mBAAmB,eAAe,YAAY;AAAA,EAC3D;AAAA,EAEA,MAAM,aAA4B;AAChC,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,WAA0B;AAC9B,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,WAAW;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACrF,QAAI;AAAE,YAAM,KAAK,OAAO,iBAAiB,aAAa;AAAA,IAAG,QAAQ;AAAA,IAAsB;AACvF,UAAM,KAAK,iBAAiB,aAAa,KAAK,OAAO,UAAU;AAC/D,UAAM,KAAK,iBAAiB,eAAe,KAAK,OAAO,UAAU;AAAA,EACnE;AAAA,EAEA,cAAuB;AACrB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,MAAc,iBAAiB,MAAc,YAAmC;AAC9E,QAAI;AACF,YAAM,KAAK,OAAO,cAAc,IAAI;AAAA,IACtC,QAAQ;AACN,YAAM,KAAK,OAAO,iBAAiB,MAAM;AAAA,QACvC,SAAS,EAAE,MAAM,YAAY,UAAU,SAAS;AAAA,MAClD,CAAC;AAAA,IACH;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,MAAc,mBAAmB,YAAoB,OAA8B;AACjF,QAAI;AACF,YAAM,KAAK,OAAO,mBAAmB,YAAY,EAAE,YAAY,OAAO,cAAc,UAAU,CAAC;AAAA,IACjG,QAAQ;AAAA,IAAiD;AAAA,EAC3D;AAAA,EAEA,MAAM,sBAAsB,YAAwB,QAA0B,iBAAyB,aAAsC;AAG3I,UAAM,KAAK,sBAAsB,UAAU;AAC3C,QAAI,OAAO,WAAW,EAAG;AAEzB,UAAM,SAAS,OAAO,IAAI,CAAC,WAAW;AAAA,MACpC,IAAI,WAAW,GAAG,UAAU,IAAI,MAAM,UAAU,EAAE;AAAA,MAClD,QAAQ,MAAM;AAAA,MACd,SAAS;AAAA,QACP,YAAY,OAAO,UAAU;AAAA,QAC7B,YAAY,MAAM;AAAA,QAClB,MAAM,MAAM;AAAA,QACZ;AAAA,QACA;AAAA,MACF;AAAA,IACF,EAAE;AAEF,UAAM,KAAK,OAAO,OAAO,aAAa,EAAE,OAAO,CAAC;AAAA,EAClD;AAAA,EAEA,MAAM,uBACJ,cACA,WACA,SACe;AACf,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC;AAAA,QACP,IAAI,WAAW,OAAO,YAAY,CAAC;AAAA,QACnC,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,cAAc,OAAO,QAAQ,YAAY;AAAA,UACzC,YAAY,OAAO,QAAQ,UAAU;AAAA,UACrC,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,MAAM,QAAQ;AAAA,QAChB;AAAA,MACF,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,sBAAsB,YAAuC;AACjE,UAAM,KAAK,OAAO,OAAO,aAAa;AAAA,MACpC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,uBAAuB,cAA2C;AACtE,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ,CAAC,WAAW,OAAO,YAAY,CAAC,CAAC;AAAA,IAC3C,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,mCAAmC,YAAuC;AAC9E,UAAM,KAAK,OAAO,OAAO,eAAe;AAAA,MACtC,QAAQ;AAAA,QACN,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,MACpE;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,QAAyB;AAC7B,UAAM,CAAC,WAAW,WAAW,IAAI,MAAM,QAAQ,IAAI;AAAA,MACjD,KAAK,OAAO,MAAM,aAAa,EAAE,OAAO,KAAK,CAAC;AAAA,MAC9C,KAAK,OAAO,MAAM,eAAe,EAAE,OAAO,KAAK,CAAC;AAAA,IAClD,CAAC;AACD,WAAO,UAAU,QAAQ,YAAY;AAAA,EACvC;AAAA,EAEA,MAAM,wBAAkE;AACtE,UAAM,YAAY,oBAAI,IAAgC;AACtD,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,cAAc,iBAAiB;AAAA,QAC9C,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,MAAM,MAAM,SAAS;AAC3B,YAAI,OAAO,QAAQ,YAAY,UAAU,IAAI,GAAG,EAAG;AACnD,cAAM,WAAW,MAAM,SAAS;AAChC,kBAAU,IAAI,KAAK,OAAO,aAAa,WAAW,WAAW,MAAS;AAAA,MACxE;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,oBAA0C;AAC9C,WAAO,KAAK,mBAAmB,eAAe,cAAc;AAAA,EAC9D;AAAA;AAAA,EAGA,MAAc,mBAAmB,YAAoB,OAAqC;AACxF,UAAM,SAAS,oBAAI,IAAY;AAC/B,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,QAChD,OAAO;AAAA,QACP;AAAA,QACA,cAAc,CAAC,KAAK;AAAA,QACpB,aAAa;AAAA,MACf,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,cAAM,QAAQ,MAAM,UAAU,KAAK;AACnC,YAAI,OAAO,UAAU,SAAU,QAAO,IAAI,KAAK;AAAA,MACjD;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAC5C,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,gBAAgB,WAAqB,MAAoD;AAC7F,WAAO,KAAK,OAAO,aAAa,WAAW,IAAI;AAAA,EACjD;AAAA,EAEA,MAAM,kBAAkB,WAAqB,MAAoD;AAC/F,WAAO,KAAK,OAAO,eAAe,WAAW,IAAI;AAAA,EACnD;AAAA,EAEA,MAAM,iBAAiB,YAAwB,MAAoD;AAGjG,UAAM,eAA2B,CAAC;AAClC,QAAI,SAA6C;AACjD,OAAG;AACD,YAAM,OAAO,MAAM,KAAK,OAAO,OAAO,aAAa;AAAA,QACjD,QAAQ,EAAE,MAAM,CAAC,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC,EAAE;AAAA,QAC9E,aAAa;AAAA,QACb,cAAc;AAAA,QACd,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AACD,iBAAW,SAAS,KAAK,QAAQ;AAC/B,YAAI,MAAM,QAAQ,MAAM,MAAM,EAAG,cAAa,KAAK,MAAM,MAAkB;AAAA,MAC7E;AACA,eAAS,KAAK,oBAAoB;AAAA,IACpC,SAAS,WAAW,UAAa,WAAW;AAE5C,QAAI,aAAa,WAAW,EAAG,QAAO,CAAC;AAGvC,UAAM,SAAS,KAAK,YAAY,EAAE,GAAG,KAAK,QAAQ,mBAAmB,WAAW,CAAC;AAKjF,UAAM,WAAW,aAAa,IAAI,CAAC,YAAY;AAAA,MAC7C;AAAA,MACA,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,EAAE;AACF,UAAM,UAAU,MAAM,KAAK,OAAO,YAAY,aAAa,EAAE,SAAS,CAAC;AAIvE,UAAM,iBAAiB,oBAAI,IAA6E;AACxG,eAAW,SAAS,SAAS;AAC3B,iBAAW,KAAK,OAAO;AACrB,cAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,cAAM,MAAM,OAAO,QAAQ,UAAU;AACrC,cAAM,OAAO,eAAe,IAAI,GAAG;AACnC,YAAI,CAAC,QAAQ,EAAE,QAAQ,KAAK,OAAO;AACjC,yBAAe,IAAI,KAAK,EAAE,IAAI,OAAO,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,QAAQ,CAAC;AAAA,QACvE;AAAA,MACF;AAAA,IACF;AAEA,WAAO,CAAC,GAAG,eAAe,OAAO,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAChC,MAAM,GAAG,KAAK,KAAK,EACnB,IAAI,CAAC,OAAO;AAAA,MACX,IAAI,EAAE;AAAA,MACN,OAAO,EAAE;AAAA,MACT,YAAY,EAAE,QAAQ;AAAA,MACtB,cAAc,EAAE,QAAQ;AAAA,MACxB,MAAM,EAAE,QAAQ;AAAA,MAChB,aAAa,EAAE,QAAQ;AAAA,IACzB,EAAE;AAAA,EACN;AAAA,EAEA,MAAc,OAAO,YAAoB,WAAqB,MAAoD;AAChH,UAAM,SAAS,KAAK,YAAY,KAAK,MAAM;AAE3C,UAAM,UAAU,MAAM,KAAK,OAAO,OAAO,YAAY;AAAA,MACnD,QAAQ;AAAA,MACR,OAAO,KAAK;AAAA,MACZ,iBAAiB,KAAK;AAAA,MACtB,QAAQ,UAAU;AAAA,MAClB,cAAc;AAAA,IAChB,CAAC;AAED,WAAO,QAAQ,IAAI,CAAC,MAAM;AACxB,YAAM,UAAU,EAAE,WAAW,CAAC;AAC9B,aAAO;AAAA,QACL,IAAI,OAAO,EAAE,EAAE;AAAA,QACf,OAAO,EAAE;AAAA,QACT,YAAY,QAAQ;AAAA,QACpB,cAAc,QAAQ;AAAA,QACtB,MAAM,QAAQ;AAAA,QACd,aAAa,QAAQ;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,EACH;AAAA,EAEQ,YAAY,QAA4D;AAC9E,QAAI,CAAC,OAAQ,QAAO;AAEpB,UAAM,OAAoC,CAAC;AAE3C,QAAI,OAAO,eAAe,OAAO,YAAY,SAAS,GAAG;AAIvD,WAAK,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,YAAY,EAAE,CAAC;AAAA,IACtE;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,UAAU,EAAE,EAAE,CAAC;AAAA,IAC9E;AAEA,QAAI,OAAO,YAAY;AACrB,WAAK,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,WAAW,EAAE,CAAC;AAAA,IACtE;AAEA,UAAM,WAAwC,CAAC;AAE/C,QAAI,OAAO,mBAAmB;AAC5B,eAAS,KAAK,EAAE,KAAK,cAAc,OAAO,EAAE,OAAO,OAAO,OAAO,iBAAiB,EAAE,EAAE,CAAC;AAAA,IACzF;AAEA,QAAI,OAAO,sBAAsB,OAAO,mBAAmB,SAAS,GAAG;AAErE,eAAS,KAAK,EAAE,KAAK,eAAe,OAAO,EAAE,KAAK,OAAO,mBAAmB,EAAE,CAAC;AAAA,IACjF;AAEA,QAAI,KAAK,WAAW,KAAK,SAAS,WAAW,EAAG,QAAO;AAEvD,WAAO;AAAA,MACL,GAAI,KAAK,SAAS,IAAI,EAAE,KAAK,IAAI,CAAC;AAAA,MAClC,GAAI,SAAS,SAAS,IAAI,EAAE,SAAS,IAAI,CAAC;AAAA,IAC5C;AAAA,EACF;AACF;","names":[]}
@@ -1,7 +0,0 @@
1
- import {
2
- QdrantVectorStore
3
- } from "./chunk-TYDOHCBS.js";
4
- export {
5
- QdrantVectorStore
6
- };
7
- //# sourceMappingURL=qdrant-R7IW5NII.js.map