@statewalker/indexer-mem 0.1.4 → 0.2.0

package/dist/mem-vector-index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mem-vector-index.d.ts","names":["EmbeddingBlock","EmbeddingSearchParams","EmbeddingSearchResult","EmbeddingIndexInfo","EmbeddingIndex"],"sources":["../src/mem-vector-index.ts"],"mappings":";;;qBAwCa,0BAETI,aACA,uBAAuBJ,aAAgBC,aAAuBC;mBAE/C;mBACA;UACT;;;;;;;;UAQA;UACA;EAER,YAAY,MAAMC;;;;;;EASlB;UAIQ;UAMC;EAYH,gBAAgB,QAAQA;EAKvB,OAAO,QAAQF,cAAwB,eAAeC;EA+BvD,YAAY,QAAQF,gBAAmB;EAgBvC,aACJ,QAAQ,SAASA,iBAAoB,cAAcA,iBAClD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;EAoBG,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAeA;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;;;;;;EAiBrB,oBAAoB;;;;;EAsCb,aAAa,cAAc;;;;;;EAS5B,SAAS,SAAS,SAAS,oBAAoB;;;;;;;SAiC9C,qBAAqB,MAAMG,iBAAoB,MAAM,aAAa"}
+{"version":3,"file":"mem-vector-index.d.ts","names":["EmbeddingBlock","EmbeddingSearchParams","EmbeddingSearchResult","EmbeddingIndexInfo","EmbeddingIndex"],"sources":["../src/mem-vector-index.ts"],"mappings":";;;qBAwCa,0BAETI,aACA,uBAAuBJ,aAAgBC,aAAuBC;mBAE/C;mBACA;UACT;;;;;;;;UAQA;UACA;EAER,YAAY,MAAMC;;;;;;EASlB;UAIQ;UAMC;EAYH,gBAAgB,QAAQA;EAKvB,OAAO,QAAQF,cAAwB,eAAeC;EA+BvD,YAAY,QAAQF,gBAAmB;EAkBvC,aACJ,QAAQ,SAASA,iBAAoB,cAAcA,iBAClD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;EAoBG,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAeA;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;;;;;;EAiBrB,oBAAoB;;;;;EAsCb,aAAa,cAAc;;;;;;EAS5B,SAAS,SAAS,SAAS,oBAAoB;;;;;;;SAiC9C,qBAAqB,MAAMG,iBAAoB,MAAM,aAAa"}

package/dist/mem-vector-index.js

@@ -62,8 +62,9 @@ var MemVectorIndex = class MemVectorIndex {
 	async addDocument(blocks) {
 		this.ensureOpen();
 		if (blocks.length === 0) return;
+		for (const block of blocks) validateDimensionality(this.info, block.embedding);
+		this.dirty = true;
 		for (const block of blocks) {
-			validateDimensionality(this.info, block.embedding);
 			const key = compositeKey(block.path, block.blockId);
 			this.entries.set(key, {
 				path: block.path,
@@ -72,7 +73,6 @@ var MemVectorIndex = class MemVectorIndex {
 				metadata: block.metadata
 			});
 		}
-		this.dirty = true;
 	}
 	async addDocuments(blocks) {
 		this.ensureOpen();

package/dist/mem-vector-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"mem-vector-index.js","names":[],"sources":["../src/mem-vector-index.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  Metadata,\n  PathSelector,\n  PersistableSearchIndex,\n  PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n  compositeKey,\n  matchesPrefix,\n  readEntryBytes,\n  singleChunk,\n  toAsyncIterable,\n  validateDimensionality,\n} from \"@statewalker/indexer-core\";\nimport type {\n  VectorBlock as EmbeddingBlock,\n  VectorIndex as EmbeddingIndex,\n  VectorIndexInfo as EmbeddingIndexInfo,\n  VectorQuery as EmbeddingSearchParams,\n  VectorResult as EmbeddingSearchResult,\n} from \"@statewalker/indexer-vector\";\nimport {\n  fixedSizeList,\n  float32,\n  tableFromArrays,\n  tableFromIPC,\n  tableToIPC,\n  utf8,\n} from \"@uwdata/flechette\";\nimport { bruteForceSearch } from \"./vector-search.js\";\n\ninterface StoredEntry {\n  path: DocumentPath;\n  blockId: string;\n  embedding: Float32Array;\n  metadata?: Metadata;\n}\n\nexport class MemVectorIndex\n  implements\n    EmbeddingIndex,\n    PersistableSearchIndex<EmbeddingBlock, EmbeddingSearchParams, EmbeddingSearchResult>\n{\n  private readonly info: EmbeddingIndexInfo;\n  private readonly entries = new Map<string, StoredEntry>();\n  private closed = false;\n  /**\n   * Dirty bit. `true` whenever `entries` has been mutated since the last\n   * `serializeToArrow()`. The serializer caches the last produced\n   * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync\n   * skips the expensive Arrow IPC encoding for an unchanged index.\n   * Starts `true` so the first serialize after construction always runs.\n   */\n  private dirty = true;\n  private cachedSerialized?: Uint8Array;\n\n  constructor(info: EmbeddingIndexInfo) {\n    this.info = info;\n  }\n\n  /**\n   * `true` when the in-memory state has changed since the last serialize.\n   * Callers (e.g. the persistence layer) may use it to skip serialization\n   * and writes when the index has not been touched.\n   */\n  isDirty(): boolean {\n    return this.dirty;\n  }\n\n  private ensureOpen(): void {\n    if (this.closed) {\n      throw new Error(\"EmbeddingIndex is closed\");\n    }\n  }\n\n  private *filteredEntries(pathPrefixes?: DocumentPath[]): Iterable<StoredEntry> {\n    if (!pathPrefixes || pathPrefixes.length === 0) {\n      yield* this.entries.values();\n      return;\n    }\n    for (const entry of this.entries.values()) {\n      if (pathPrefixes.some((p) => matchesPrefix(entry.path, p))) {\n        yield entry;\n      }\n    }\n  }\n\n  async getIndexInfo(): Promise<EmbeddingIndexInfo> {\n    this.ensureOpen();\n    return { ...this.info };\n  }\n\n  async *search(params: EmbeddingSearchParams): AsyncGenerator<EmbeddingSearchResult> {\n    this.ensureOpen();\n    const { embeddings, paths } = params;\n    // Sub-queries may omit `topK` to defer to the composite's top-level cutoff;\n    // when unset we retrieve generously so RRF has material to fuse.\n    const topK = params.topK ?? 100;\n\n    if (!embeddings || embeddings.length === 0) return;\n\n    // Multi-embedding: search for each, merge by best score\n    const bestScores = new Map<string, EmbeddingSearchResult>();\n\n    for (const queryEmb of embeddings) {\n      validateDimensionality(this.info, queryEmb);\n      const filtered = [...this.filteredEntries(paths)];\n      const results = bruteForceSearch(queryEmb, filtered, topK);\n      for (const r of results) {\n        const key = compositeKey(r.path, r.blockId);\n        const existing = bestScores.get(key);\n        if (!existing || r.score > existing.score) {\n          bestScores.set(key, r);\n        }\n      }\n    }\n\n    const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score);\n    for (const r of sorted.slice(0, topK)) {\n      yield r;\n    }\n  }\n\n  async addDocument(blocks: EmbeddingBlock[]): Promise<void> {\n    this.ensureOpen();\n    if (blocks.length === 0) return;\n    for (const block of blocks) {\n      validateDimensionality(this.info, block.embedding);\n      const key = compositeKey(block.path, block.blockId);\n      this.entries.set(key, {\n        path: block.path,\n        blockId: block.blockId,\n        embedding: new Float32Array(block.embedding),\n        metadata: block.metadata,\n      });\n    }\n    this.dirty = true;\n  }\n\n  async addDocuments(\n    blocks: Iterable<EmbeddingBlock[]> | AsyncIterable<EmbeddingBlock[]>,\n  ): Promise<void> {\n    this.ensureOpen();\n    for await (const batch of blocks) {\n      await this.addDocument(batch);\n    }\n  }\n\n  async deleteDocuments(\n    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n  ): Promise<void> {\n    this.ensureOpen();\n    let removed = 0;\n    for await (const sel of toAsyncIterable(pathSelectors)) {\n      if (sel.blockId !== undefined) {\n        if (this.entries.delete(compositeKey(sel.path, sel.blockId))) {\n          removed += 1;\n        }\n      } else {\n        for (const [key, entry] of this.entries) {\n          if (matchesPrefix(entry.path, sel.path)) {\n            this.entries.delete(key);\n            removed += 1;\n          }\n        }\n      }\n    }\n    if (removed > 0) this.dirty = true;\n  }\n\n  async getSize(pathPrefix?: DocumentPath): Promise<number> {\n    this.ensureOpen();\n    if (pathPrefix === undefined) return this.entries.size;\n    let count = 0;\n    for (const entry of this.entries.values()) {\n      if (matchesPrefix(entry.path, pathPrefix)) count++;\n    }\n    return count;\n  }\n\n  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n    this.ensureOpen();\n    const paths = new Set<DocumentPath>();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        paths.add(entry.path);\n      }\n    }\n    for (const p of paths) yield p;\n  }\n\n  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield { path: entry.path, blockId: entry.blockId };\n      }\n    }\n  }\n\n  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<EmbeddingBlock> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield {\n          path: entry.path,\n          blockId: entry.blockId,\n          embedding: entry.embedding,\n          metadata: entry.metadata,\n        };\n      }\n    }\n  }\n\n  async close(_options?: { force?: boolean }): Promise<void> {\n    this.closed = true;\n  }\n\n  async flush(): Promise<void> {\n    this.ensureOpen();\n    // no-op for in-memory\n  }\n\n  async deleteIndex(): Promise<void> {\n    this.ensureOpen();\n    this.entries.clear();\n    this.cachedSerialized = undefined;\n    this.dirty = true;\n    this.closed = true;\n  }\n\n  /**\n   * Serialize embeddings to Arrow IPC format.\n   *\n   * Caches the result; while `dirty` is false (no mutations since the\n   * last call) returns the same `Uint8Array` instance, skipping the\n   * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the\n   * persistence layer still calls into us each flush, but the heavy\n   * encoding only runs when the index actually changed.\n   */\n  serializeToArrow(): Uint8Array {\n    if (!this.dirty && this.cachedSerialized !== undefined) {\n      return this.cachedSerialized;\n    }\n    const dim = this.info.dimensionality;\n    const paths: string[] = [];\n    const blockIds: string[] = [];\n    const embeddingArrays: number[][] = [];\n    const metadata: Array<string | null> = [];\n    for (const entry of this.entries.values()) {\n      paths.push(entry.path);\n      blockIds.push(entry.blockId);\n      embeddingArrays.push(Array.from(entry.embedding));\n      metadata.push(entry.metadata === undefined ? null : JSON.stringify(entry.metadata));\n    }\n    const table = tableFromArrays(\n      { path: paths, blockId: blockIds, embedding: embeddingArrays, metadata },\n      {\n        types: {\n          path: utf8(),\n          blockId: utf8(),\n          embedding: fixedSizeList(float32(), dim),\n          metadata: utf8(),\n        },\n      },\n    );\n    const bytes = tableToIPC(table, { format: \"stream\" }) as Uint8Array;\n    this.cachedSerialized = bytes;\n    this.dirty = false;\n    return bytes;\n  }\n\n  // --- PersistableSearchIndex --------------------------------------------------\n\n  /**\n   * Stream the vector sub-index's state as a single named entry. The composite\n   * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n   */\n  async *serialise(): AsyncIterable<PersistenceEntry> {\n    yield { name: \"arrow\", content: singleChunk(this.serializeToArrow()) };\n  }\n\n  /**\n   * Restore the vector sub-index from a previously-produced entry set. Expects\n   * one entry named `\"arrow\"`; ignores entries with other names so future\n   * additions remain forward-compatible.\n   */\n  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n    this.ensureOpen();\n    for (const e of entries) {\n      if (e.name !== \"arrow\") continue;\n      const bytes = await readEntryBytes(e);\n      const table = tableFromIPC(bytes);\n      const pathCol = table.getChild(\"path\");\n      const blockIdCol = table.getChild(\"blockId\");\n      const embCol = table.getChild(\"embedding\");\n      const metaCol = table.getChild(\"metadata\");\n      this.entries.clear();\n      for (let i = 0; i < table.numRows; i++) {\n        const path = pathCol.at(i) as string as DocumentPath;\n        const blockId = blockIdCol.at(i) as string;\n        const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n        const metaRaw = metaCol?.at(i) as string | null | undefined;\n        const metadata =\n          metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n        const key = compositeKey(path, blockId);\n        this.entries.set(key, { path, blockId, embedding, metadata });\n      }\n      this.cachedSerialized = bytes;\n      this.dirty = false;\n      return;\n    }\n  }\n\n  /**\n   * Deserialize embeddings from Arrow IPC format. The reconstructed index\n   * starts clean (`dirty = false`) and primes its serialize cache with the\n   * input bytes — a subsequent `serializeToArrow()` on the just-loaded\n   * index returns the same bytes without re-encoding.\n   */\n  static deserializeFromArrow(info: EmbeddingIndexInfo, data: Uint8Array): MemVectorIndex {\n    const table = tableFromIPC(data);\n    const vec = new MemVectorIndex(info);\n    const pathCol = table.getChild(\"path\");\n    const blockIdCol = table.getChild(\"blockId\");\n    const embCol = table.getChild(\"embedding\");\n    const metaCol = table.getChild(\"metadata\");\n    for (let i = 0; i < table.numRows; i++) {\n      const path = pathCol.at(i) as string as DocumentPath;\n      const blockId = blockIdCol.at(i) as string;\n      const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n      const metaRaw = metaCol?.at(i) as string | null | undefined;\n      const metadata =\n        metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n      const key = compositeKey(path, blockId);\n      vec.entries.set(key, { path, blockId, embedding, metadata });\n    }\n    vec.cachedSerialized = data;\n    vec.dirty = false;\n    return vec;\n  }\n}\n"],"mappings":";;;;AAwCA,IAAa,iBAAb,MAAa,eAIb;CACE;CACA,0BAA2B,IAAI,IAAyB;CACxD,SAAiB;;;;;;;;CAQjB,QAAgB;CAChB;CAEA,YAAY,MAA0B;EACpC,KAAK,OAAO;CACd;;;;;;CAOA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,0BAA0B;CAE9C;CAEA,CAAS,gBAAgB,cAAsD;EAC7E,IAAI,CAAC,gBAAgB,aAAa,WAAW,GAAG;GAC9C,OAAO,KAAK,QAAQ,OAAO;GAC3B;EACF;EACA,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC,GACvD,MAAM;CAGZ;CAEA,MAAM,eAA4C;EAChD,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAsE;EAClF,KAAK,WAAW;EAChB,MAAM,EAAE,YAAY,UAAU;EAG9B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,cAAc,WAAW,WAAW,GAAG;EAG5C,MAAM,6BAAa,IAAI,IAAmC;EAE1D,KAAK,MAAM,YAAY,YAAY;GACjC,uBAAuB,KAAK,MAAM,QAAQ;GAC1C,MAAM,WAAW,CAAC,GAAG,KAAK,gBAAgB,KAAK,CAAC;GAChD,MAAM,UAAU,iBAAiB,UAAU,UAAU,IAAI;GACzD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,WAAW,IAAI,GAAG;IACnC,IAAI,CAAC,YAAY,EAAE,QAAQ,SAAS,OAClC,WAAW,IAAI,KAAK,CAAC;GAEzB;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,WAAW,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACxE,KAAK,MAAM,KAAK,OAAO,MAAM,GAAG,IAAI,GAClC,MAAM;CAEV;CAEA,MAAM,YAAY,QAAyC;EACzD,KAAK,WAAW;EAChB,IAAI,OAAO,WAAW,GAAG;EACzB,KAAK,MAAM,SAAS,QAAQ;GAC1B,uBAAuB,KAAK,MAAM,MAAM,SAAS;GACjD,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,KAAK,QAAQ,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,WAAW,IAAI,aAAa,MAAM,SAAS;IAC3C,UAAU,MAAM;GAClB,CAAC;EACH;EACA,KAAK,QAAQ;CACf;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GACd;OAAA,KAAK,QAAQ,OAAO,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC,GACzD,WAAW;EAAA,OAGb,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,SAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GAAG;GACvC,KAAK,QAAQ,OAAO,GAAG;GACvB,WAAW;EACb;EAIN,IAAI,UAAU,GAAG,KAAK,QAAQ;CAChC;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,QAAQ;EAClD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA2D;EACnF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,WAAW,MAAM;GACjB,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,QAAQ,MAAM;EACnB,KAAK,mBAAmB,KAAA;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS;CAChB;;;;;;;;;;CAWA,mBAA+B;EAC7B,IAAI,CAAC,KAAK,SAAS,KAAK,qBAAqB,KAAA,GAC3C,OAAO,KAAK;EAEd,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,QAAkB,CAAC;EACzB,MAAM,WAAqB,CAAC;EAC5B,MAAM,kBAA8B,CAAC;EACrC,MAAM,WAAiC,CAAC;EACxC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GAAG;GACzC,MAAM,KAAK,MAAM,IAAI;GACrB,SAAS,KAAK,MAAM,OAAO;GAC3B,gBAAgB,KAAK,MAAM,KAAK,MAAM,SAAS,CAAC;GAChD,SAAS,KAAK,MAAM,aAAa,KAAA,IAAY,OAAO,KAAK,UAAU,MAAM,QAAQ,CAAC;EACpF;EACA,MAAM,QAAQ,gBACZ;GAAE,MAAM;GAAO,SAAS;GAAU,WAAW;GAAiB;EAAS,GACvE,EACE,OAAO;GACL,MAAM,KAAK;GACX,SAAS,KAAK;GACd,WAAW,cAAc,QAAQ,GAAG,GAAG;GACvC,UAAU,KAAK;EACjB,EACF,CACF;EACA,MAAM,QAAQ,WAAW,OAAO,EAAE,QAAQ,SAAS,CAAC;EACpD,KAAK,mBAAmB;EACxB,KAAK,QAAQ;EACb,OAAO;CACT;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAS,SAAS,YAAY,KAAK,iBAAiB,CAAC;EAAE;CACvE;;;;;;CAOA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,SAAS;GACxB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,QAAQ,aAAa,KAAK;GAChC,MAAM,UAAU,MAAM,SAAS,MAAM;GACrC,MAAM,aAAa,MAAM,SAAS,SAAS;GAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;GACzC,MAAM,UAAU,MAAM,SAAS,UAAU;GACzC,KAAK,QAAQ,MAAM;GACnB,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;IACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;IACzB,MAAM,UAAU,WAAW,GAAG,CAAC;IAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;IACpE,MAAM,UAAU,SAAS,GAAG,CAAC;IAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;IACnD,MAAM,MAAM,aAAa,MAAM,OAAO;IACtC,KAAK,QAAQ,IAAI,KAAK;KAAE;KAAM;KAAS;KAAW;IAAS,CAAC;GAC9D;GACA,KAAK,mBAAmB;GACxB,KAAK,QAAQ;GACb;EACF;CACF;;;;;;;CAQA,OAAO,qBAAqB,MAA0B,MAAkC;EACtF,MAAM,QAAQ,aAAa,IAAI;EAC/B,MAAM,MAAM,IAAI,eAAe,IAAI;EACnC,MAAM,UAAU,MAAM,SAAS,MAAM;EACrC,MAAM,aAAa,MAAM,SAAS,SAAS;EAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;EACzC,MAAM,UAAU,MAAM,SAAS,UAAU;EACzC,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;GACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;GACzB,MAAM,UAAU,WAAW,GAAG,CAAC;GAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;GACpE,MAAM,UAAU,SAAS,GAAG,CAAC;GAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;GACnD,MAAM,MAAM,aAAa,MAAM,OAAO;GACtC,IAAI,QAAQ,IAAI,KAAK;IAAE;IAAM;IAAS;IAAW;GAAS,CAAC;EAC7D;EACA,IAAI,mBAAmB;EACvB,IAAI,QAAQ;EACZ,OAAO;CACT;AACF"}
+{"version":3,"file":"mem-vector-index.js","names":[],"sources":["../src/mem-vector-index.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  Metadata,\n  PathSelector,\n  PersistableSearchIndex,\n  PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n  compositeKey,\n  matchesPrefix,\n  readEntryBytes,\n  singleChunk,\n  toAsyncIterable,\n  validateDimensionality,\n} from \"@statewalker/indexer-core\";\nimport type {\n  VectorBlock as EmbeddingBlock,\n  VectorIndex as EmbeddingIndex,\n  VectorIndexInfo as EmbeddingIndexInfo,\n  VectorQuery as EmbeddingSearchParams,\n  VectorResult as EmbeddingSearchResult,\n} from \"@statewalker/indexer-vector\";\nimport {\n  fixedSizeList,\n  float32,\n  tableFromArrays,\n  tableFromIPC,\n  tableToIPC,\n  utf8,\n} from \"@uwdata/flechette\";\nimport { bruteForceSearch } from \"./vector-search.js\";\n\ninterface StoredEntry {\n  path: DocumentPath;\n  blockId: string;\n  embedding: Float32Array;\n  metadata?: Metadata;\n}\n\nexport class MemVectorIndex\n  implements\n    EmbeddingIndex,\n    PersistableSearchIndex<EmbeddingBlock, EmbeddingSearchParams, EmbeddingSearchResult>\n{\n  private readonly info: EmbeddingIndexInfo;\n  private readonly entries = new Map<string, StoredEntry>();\n  private closed = false;\n  /**\n   * Dirty bit. `true` whenever `entries` has been mutated since the last\n   * `serializeToArrow()`. The serializer caches the last produced\n   * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync\n   * skips the expensive Arrow IPC encoding for an unchanged index.\n   * Starts `true` so the first serialize after construction always runs.\n   */\n  private dirty = true;\n  private cachedSerialized?: Uint8Array;\n\n  constructor(info: EmbeddingIndexInfo) {\n    this.info = info;\n  }\n\n  /**\n   * `true` when the in-memory state has changed since the last serialize.\n   * Callers (e.g. the persistence layer) may use it to skip serialization\n   * and writes when the index has not been touched.\n   */\n  isDirty(): boolean {\n    return this.dirty;\n  }\n\n  private ensureOpen(): void {\n    if (this.closed) {\n      throw new Error(\"EmbeddingIndex is closed\");\n    }\n  }\n\n  private *filteredEntries(pathPrefixes?: DocumentPath[]): Iterable<StoredEntry> {\n    if (!pathPrefixes || pathPrefixes.length === 0) {\n      yield* this.entries.values();\n      return;\n    }\n    for (const entry of this.entries.values()) {\n      if (pathPrefixes.some((p) => matchesPrefix(entry.path, p))) {\n        yield entry;\n      }\n    }\n  }\n\n  async getIndexInfo(): Promise<EmbeddingIndexInfo> {\n    this.ensureOpen();\n    return { ...this.info };\n  }\n\n  async *search(params: EmbeddingSearchParams): AsyncGenerator<EmbeddingSearchResult> {\n    this.ensureOpen();\n    const { embeddings, paths } = params;\n    // Sub-queries may omit `topK` to defer to the composite's top-level cutoff;\n    // when unset we retrieve generously so RRF has material to fuse.\n    const topK = params.topK ?? 100;\n\n    if (!embeddings || embeddings.length === 0) return;\n\n    // Multi-embedding: search for each, merge by best score\n    const bestScores = new Map<string, EmbeddingSearchResult>();\n\n    for (const queryEmb of embeddings) {\n      validateDimensionality(this.info, queryEmb);\n      const filtered = [...this.filteredEntries(paths)];\n      const results = bruteForceSearch(queryEmb, filtered, topK);\n      for (const r of results) {\n        const key = compositeKey(r.path, r.blockId);\n        const existing = bestScores.get(key);\n        if (!existing || r.score > existing.score) {\n          bestScores.set(key, r);\n        }\n      }\n    }\n\n    const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score);\n    for (const r of sorted.slice(0, topK)) {\n      yield r;\n    }\n  }\n\n  async addDocument(blocks: EmbeddingBlock[]): Promise<void> {\n    this.ensureOpen();\n    if (blocks.length === 0) return;\n    // Validate the whole batch before writing, and mark dirty first, so a failure cannot leave\n    // blocks written that the cached serialisation does not include.\n    for (const block of blocks) validateDimensionality(this.info, block.embedding);\n    this.dirty = true;\n    for (const block of blocks) {\n      const key = compositeKey(block.path, block.blockId);\n      this.entries.set(key, {\n        path: block.path,\n        blockId: block.blockId,\n        embedding: new Float32Array(block.embedding),\n        metadata: block.metadata,\n      });\n    }\n  }\n\n  async addDocuments(\n    blocks: Iterable<EmbeddingBlock[]> | AsyncIterable<EmbeddingBlock[]>,\n  ): Promise<void> {\n    this.ensureOpen();\n    for await (const batch of blocks) {\n      await this.addDocument(batch);\n    }\n  }\n\n  async deleteDocuments(\n    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n  ): Promise<void> {\n    this.ensureOpen();\n    let removed = 0;\n    for await (const sel of toAsyncIterable(pathSelectors)) {\n      if (sel.blockId !== undefined) {\n        if (this.entries.delete(compositeKey(sel.path, sel.blockId))) {\n          removed += 1;\n        }\n      } else {\n        for (const [key, entry] of this.entries) {\n          if (matchesPrefix(entry.path, sel.path)) {\n            this.entries.delete(key);\n            removed += 1;\n          }\n        }\n      }\n    }\n    if (removed > 0) this.dirty = true;\n  }\n\n  async getSize(pathPrefix?: DocumentPath): Promise<number> {\n    this.ensureOpen();\n    if (pathPrefix === undefined) return this.entries.size;\n    let count = 0;\n    for (const entry of this.entries.values()) {\n      if (matchesPrefix(entry.path, pathPrefix)) count++;\n    }\n    return count;\n  }\n\n  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n    this.ensureOpen();\n    const paths = new Set<DocumentPath>();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        paths.add(entry.path);\n      }\n    }\n    for (const p of paths) yield p;\n  }\n\n  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield { path: entry.path, blockId: entry.blockId };\n      }\n    }\n  }\n\n  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<EmbeddingBlock> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield {\n          path: entry.path,\n          blockId: entry.blockId,\n          embedding: entry.embedding,\n          metadata: entry.metadata,\n        };\n      }\n    }\n  }\n\n  async close(_options?: { force?: boolean }): Promise<void> {\n    this.closed = true;\n  }\n\n  async flush(): Promise<void> {\n    this.ensureOpen();\n    // no-op for in-memory\n  }\n\n  async deleteIndex(): Promise<void> {\n    this.ensureOpen();\n    this.entries.clear();\n    this.cachedSerialized = undefined;\n    this.dirty = true;\n    this.closed = true;\n  }\n\n  /**\n   * Serialize embeddings to Arrow IPC format.\n   *\n   * Caches the result; while `dirty` is false (no mutations since the\n   * last call) returns the same `Uint8Array` instance, skipping the\n   * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the\n   * persistence layer still calls into us each flush, but the heavy\n   * encoding only runs when the index actually changed.\n   */\n  serializeToArrow(): Uint8Array {\n    if (!this.dirty && this.cachedSerialized !== undefined) {\n      return this.cachedSerialized;\n    }\n    const dim = this.info.dimensionality;\n    const paths: string[] = [];\n    const blockIds: string[] = [];\n    const embeddingArrays: number[][] = [];\n    const metadata: Array<string | null> = [];\n    for (const entry of this.entries.values()) {\n      paths.push(entry.path);\n      blockIds.push(entry.blockId);\n      embeddingArrays.push(Array.from(entry.embedding));\n      metadata.push(entry.metadata === undefined ? null : JSON.stringify(entry.metadata));\n    }\n    const table = tableFromArrays(\n      { path: paths, blockId: blockIds, embedding: embeddingArrays, metadata },\n      {\n        types: {\n          path: utf8(),\n          blockId: utf8(),\n          embedding: fixedSizeList(float32(), dim),\n          metadata: utf8(),\n        },\n      },\n    );\n    const bytes = tableToIPC(table, { format: \"stream\" }) as Uint8Array;\n    this.cachedSerialized = bytes;\n    this.dirty = false;\n    return bytes;\n  }\n\n  // --- PersistableSearchIndex --------------------------------------------------\n\n  /**\n   * Stream the vector sub-index's state as a single named entry. The composite\n   * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n   */\n  async *serialise(): AsyncIterable<PersistenceEntry> {\n    yield { name: \"arrow\", content: singleChunk(this.serializeToArrow()) };\n  }\n\n  /**\n   * Restore the vector sub-index from a previously-produced entry set. Expects\n   * one entry named `\"arrow\"`; ignores entries with other names so future\n   * additions remain forward-compatible.\n   */\n  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n    this.ensureOpen();\n    for (const e of entries) {\n      if (e.name !== \"arrow\") continue;\n      const bytes = await readEntryBytes(e);\n      const table = tableFromIPC(bytes);\n      const pathCol = table.getChild(\"path\");\n      const blockIdCol = table.getChild(\"blockId\");\n      const embCol = table.getChild(\"embedding\");\n      const metaCol = table.getChild(\"metadata\");\n      this.entries.clear();\n      for (let i = 0; i < table.numRows; i++) {\n        const path = pathCol.at(i) as string as DocumentPath;\n        const blockId = blockIdCol.at(i) as string;\n        const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n        const metaRaw = metaCol?.at(i) as string | null | undefined;\n        const metadata =\n          metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n        const key = compositeKey(path, blockId);\n        this.entries.set(key, { path, blockId, embedding, metadata });\n      }\n      this.cachedSerialized = bytes;\n      this.dirty = false;\n      return;\n    }\n  }\n\n  /**\n   * Deserialize embeddings from Arrow IPC format. The reconstructed index\n   * starts clean (`dirty = false`) and primes its serialize cache with the\n   * input bytes — a subsequent `serializeToArrow()` on the just-loaded\n   * index returns the same bytes without re-encoding.\n   */\n  static deserializeFromArrow(info: EmbeddingIndexInfo, data: Uint8Array): MemVectorIndex {\n    const table = tableFromIPC(data);\n    const vec = new MemVectorIndex(info);\n    const pathCol = table.getChild(\"path\");\n    const blockIdCol = table.getChild(\"blockId\");\n    const embCol = table.getChild(\"embedding\");\n    const metaCol = table.getChild(\"metadata\");\n    for (let i = 0; i < table.numRows; i++) {\n      const path = pathCol.at(i) as string as DocumentPath;\n      const blockId = blockIdCol.at(i) as string;\n      const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n      const metaRaw = metaCol?.at(i) as string | null | undefined;\n      const metadata =\n        metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n      const key = compositeKey(path, blockId);\n      vec.entries.set(key, { path, blockId, embedding, metadata });\n    }\n    vec.cachedSerialized = data;\n    vec.dirty = false;\n    return vec;\n  }\n}\n"],"mappings":";;;;AAwCA,IAAa,iBAAb,MAAa,eAIb;CACE;CACA,0BAA2B,IAAI,IAAyB;CACxD,SAAiB;;;;;;;;CAQjB,QAAgB;CAChB;CAEA,YAAY,MAA0B;EACpC,KAAK,OAAO;CACd;;;;;;CAOA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,0BAA0B;CAE9C;CAEA,CAAS,gBAAgB,cAAsD;EAC7E,IAAI,CAAC,gBAAgB,aAAa,WAAW,GAAG;GAC9C,OAAO,KAAK,QAAQ,OAAO;GAC3B;EACF;EACA,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC,GACvD,MAAM;CAGZ;CAEA,MAAM,eAA4C;EAChD,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAsE;EAClF,KAAK,WAAW;EAChB,MAAM,EAAE,YAAY,UAAU;EAG9B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,cAAc,WAAW,WAAW,GAAG;EAG5C,MAAM,6BAAa,IAAI,IAAmC;EAE1D,KAAK,MAAM,YAAY,YAAY;GACjC,uBAAuB,KAAK,MAAM,QAAQ;GAC1C,MAAM,WAAW,CAAC,GAAG,KAAK,gBAAgB,KAAK,CAAC;GAChD,MAAM,UAAU,iBAAiB,UAAU,UAAU,IAAI;GACzD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,WAAW,IAAI,GAAG;IACnC,IAAI,CAAC,YAAY,EAAE,QAAQ,SAAS,OAClC,WAAW,IAAI,KAAK,CAAC;GAEzB;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,WAAW,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACxE,KAAK,MAAM,KAAK,OAAO,MAAM,GAAG,IAAI,GAClC,MAAM;CAEV;CAEA,MAAM,YAAY,QAAyC;EACzD,KAAK,WAAW;EAChB,IAAI,OAAO,WAAW,GAAG;EAGzB,KAAK,MAAM,SAAS,QAAQ,uBAAuB,KAAK,MAAM,MAAM,SAAS;EAC7E,KAAK,QAAQ;EACb,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,KAAK,QAAQ,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,WAAW,IAAI,aAAa,MAAM,SAAS;IAC3C,UAAU,MAAM;GAClB,CAAC;EACH;CACF;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GACd;OAAA,KAAK,QAAQ,OAAO,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC,GACzD,WAAW;EAAA,OAGb,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,SAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GAAG;GACvC,KAAK,QAAQ,OAAO,GAAG;GACvB,WAAW;EACb;EAIN,IAAI,UAAU,GAAG,KAAK,QAAQ;CAChC;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,QAAQ;EAClD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA2D;EACnF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,WAAW,MAAM;GACjB,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,QAAQ,MAAM;EACnB,KAAK,mBAAmB,KAAA;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS;CAChB;;;;;;;;;;CAWA,mBAA+B;EAC7B,IAAI,CAAC,KAAK,SAAS,KAAK,qBAAqB,KAAA,GAC3C,OAAO,KAAK;EAEd,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,QAAkB,CAAC;EACzB,MAAM,WAAqB,CAAC;EAC5B,MAAM,kBAA8B,CAAC;EACrC,MAAM,WAAiC,CAAC;EACxC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GAAG;GACzC,MAAM,KAAK,MAAM,IAAI;GACrB,SAAS,KAAK,MAAM,OAAO;GAC3B,gBAAgB,KAAK,MAAM,KAAK,MAAM,SAAS,CAAC;GAChD,SAAS,KAAK,MAAM,aAAa,KAAA,IAAY,OAAO,KAAK,UAAU,MAAM,QAAQ,CAAC;EACpF;EACA,MAAM,QAAQ,gBACZ;GAAE,MAAM;GAAO,SAAS;GAAU,WAAW;GAAiB;EAAS,GACvE,EACE,OAAO;GACL,MAAM,KAAK;GACX,SAAS,KAAK;GACd,WAAW,cAAc,QAAQ,GAAG,GAAG;GACvC,UAAU,KAAK;EACjB,EACF,CACF;EACA,MAAM,QAAQ,WAAW,OAAO,EAAE,QAAQ,SAAS,CAAC;EACpD,KAAK,mBAAmB;EACxB,KAAK,QAAQ;EACb,OAAO;CACT;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAS,SAAS,YAAY,KAAK,iBAAiB,CAAC;EAAE;CACvE;;;;;;CAOA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,SAAS;GACxB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,QAAQ,aAAa,KAAK;GAChC,MAAM,UAAU,MAAM,SAAS,MAAM;GACrC,MAAM,aAAa,MAAM,SAAS,SAAS;GAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;GACzC,MAAM,UAAU,MAAM,SAAS,UAAU;GACzC,KAAK,QAAQ,MAAM;GACnB,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;IACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;IACzB,MAAM,UAAU,WAAW,GAAG,CAAC;IAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;IACpE,MAAM,UAAU,SAAS,GAAG,CAAC;IAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;IACnD,MAAM,MAAM,aAAa,MAAM,OAAO;IACtC,KAAK,QAAQ,IAAI,KAAK;KAAE;KAAM;KAAS;KAAW;IAAS,CAAC;GAC9D;GACA,KAAK,mBAAmB;GACxB,KAAK,QAAQ;GACb;EACF;CACF;;;;;;;CAQA,OAAO,qBAAqB,MAA0B,MAAkC;EACtF,MAAM,QAAQ,aAAa,IAAI;EAC/B,MAAM,MAAM,IAAI,eAAe,IAAI;EACnC,MAAM,UAAU,MAAM,SAAS,MAAM;EACrC,MAAM,aAAa,MAAM,SAAS,SAAS;EAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;EACzC,MAAM,UAAU,MAAM,SAAS,UAAU;EACzC,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;GACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;GACzB,MAAM,UAAU,WAAW,GAAG,CAAC;GAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;GACpE,MAAM,UAAU,SAAS,GAAG,CAAC;GAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;GACnD,MAAM,MAAM,aAAa,MAAM,OAAO;GACtC,IAAI,QAAQ,IAAI,KAAK;IAAE;IAAM;IAAS;IAAW;GAAS,CAAC;EAC7D;EACA,IAAI,mBAAmB;EACvB,IAAI,QAAQ;EACZ,OAAO;CACT;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@statewalker/indexer-mem",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "private": false,
   "type": "module",
   "description": "In-memory vector sub-index (MemVectorIndex, brute-force cosine, Arrow persistence) and its VectorProvider.",
@@ -32,9 +32,9 @@
   ],
   "dependencies": {
     "@uwdata/flechette": "^2.5.0",
-    "@statewalker/indexer-api": "^0.1.3",
-    "@statewalker/indexer-core": "^0.1.3",
-    "@statewalker/indexer-vector": "^0.1.4"
+    "@statewalker/indexer-api": "^0.1.5",
+    "@statewalker/indexer-vector": "^0.1.6",
+    "@statewalker/indexer-core": "^0.2.0"
   },
   "devDependencies": {
     "rimraf": "^6.1.3",

package/src/mem-vector-index.ts

@@ -126,8 +126,11 @@ export class MemVectorIndex
   async addDocument(blocks: EmbeddingBlock[]): Promise<void> {
     this.ensureOpen();
     if (blocks.length === 0) return;
+    // Validate the whole batch before writing, and mark dirty first, so a failure cannot leave
+    // blocks written that the cached serialisation does not include.
+    for (const block of blocks) validateDimensionality(this.info, block.embedding);
+    this.dirty = true;
     for (const block of blocks) {
-      validateDimensionality(this.info, block.embedding);
       const key = compositeKey(block.path, block.blockId);
       this.entries.set(key, {
         path: block.path,
@@ -136,7 +139,6 @@ export class MemVectorIndex
         metadata: block.metadata,
       });
     }
-    this.dirty = true;
   }
 
   async addDocuments(