@statewalker/indexer-mem 0.1.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Kotelnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,37 @@
+# @statewalker/indexer-mem
+
+In-memory vector sub-index (`MemVectorIndex`) implementing the `EmbeddingIndex` contract from `@statewalker/indexer-api`. Consumed by the FTS-backed mem indexers (`indexer-mem-flexsearch`, `indexer-mem-minisearch`), which combine it with a full-text sub-index via `@statewalker/indexer-core`'s composite-index factory.
+
+## Usage
+
+This package is a piece of scaffolding — consumers should reach for the concrete indexer packages:
+
+```ts
+import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch";
+// or
+import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch";
+
+const indexer = createFlexSearchIndexer();
+const index = await indexer.createIndex({
+  name: "docs",
+  fulltext: { language: "en" },
+  vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" },
+});
+
+await index.addDocument([
+  { path: "/docs/a", blockId: "1", content: "hello world", embedding: queryEmbedding },
+]);
+
+for await (const hit of index.search({ queries: ["hello"], embeddings: [queryEmbedding], topK: 10 })) {
+  console.log(hit);
+}
+```
+
+## Exports
+
+- `MemVectorIndex` — in-memory vector sub-index with brute-force cosine search and Arrow IPC serialization (used as the vector side of both mem-flexsearch and mem-minisearch).
+
+## Related
+
+- `@statewalker/indexer-api` — public types and contract
+- `@statewalker/indexer-mem-flexsearch` / `@statewalker/indexer-mem-minisearch` — end-user factories combining `MemVectorIndex` with a full-text backend

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+import { MemVectorIndex } from "./mem-vector-index.js";
+import { memVectorProvider } from "./mem-vector-provider.js";
+export { MemVectorIndex, memVectorProvider };

package/dist/index.js

@@ -0,0 +1,3 @@
+import { MemVectorIndex } from "./mem-vector-index.js";
+import { memVectorProvider } from "./mem-vector-provider.js";
+export { MemVectorIndex, memVectorProvider };

package/dist/mem-vector-index.d.ts

@@ -0,0 +1,70 @@
+import { VectorBlock, VectorIndex, VectorIndexInfo, VectorQuery, VectorResult } from "@statewalker/indexer-vector";
+import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
+//#region src/mem-vector-index.d.ts
+export declare class MemVectorIndex implements VectorIndex, PersistableSearchIndex<VectorBlock, VectorQuery, VectorResult> {
+  private readonly info;
+  private readonly entries;
+  private closed;
+  /**
+   * Dirty bit. `true` whenever `entries` has been mutated since the last
+   * `serializeToArrow()`. The serializer caches the last produced
+   * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync
+   * skips the expensive Arrow IPC encoding for an unchanged index.
+   * Starts `true` so the first serialize after construction always runs.
+   */
+  private dirty;
+  private cachedSerialized?;
+  constructor(info: VectorIndexInfo);
+  /**
+   * `true` when the in-memory state has changed since the last serialize.
+   * Callers (e.g. the persistence layer) may use it to skip serialization
+   * and writes when the index has not been touched.
+   */
+  isDirty(): boolean;
+  private ensureOpen;
+  private filteredEntries;
+  getIndexInfo(): Promise<VectorIndexInfo>;
+  search(params: VectorQuery): AsyncGenerator<VectorResult>;
+  addDocument(blocks: VectorBlock[]): Promise<void>;
+  addDocuments(blocks: Iterable<VectorBlock[]> | AsyncIterable<VectorBlock[]>): Promise<void>;
+  deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
+  getSize(pathPrefix?: DocumentPath): Promise<number>;
+  getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
+  getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
+  getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<VectorBlock>;
+  close(_options?: {
+    force?: boolean;
+  }): Promise<void>;
+  flush(): Promise<void>;
+  deleteIndex(): Promise<void>;
+  /**
+   * Serialize embeddings to Arrow IPC format.
+   *
+   * Caches the result; while `dirty` is false (no mutations since the
+   * last call) returns the same `Uint8Array` instance, skipping the
+   * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the
+   * persistence layer still calls into us each flush, but the heavy
+   * encoding only runs when the index actually changed.
+   */
+  serializeToArrow(): Uint8Array;
+  /**
+   * Stream the vector sub-index's state as a single named entry. The composite
+   * prefixes this with `${indexName}/${subName}/` before it hits persistence.
+   */
+  serialise(): AsyncIterable<PersistenceEntry>;
+  /**
+   * Restore the vector sub-index from a previously-produced entry set. Expects
+   * one entry named `"arrow"`; ignores entries with other names so future
+   * additions remain forward-compatible.
+   */
+  loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
+  /**
+   * Deserialize embeddings from Arrow IPC format. The reconstructed index
+   * starts clean (`dirty = false`) and primes its serialize cache with the
+   * input bytes — a subsequent `serializeToArrow()` on the just-loaded
+   * index returns the same bytes without re-encoding.
+   */
+  static deserializeFromArrow(info: VectorIndexInfo, data: Uint8Array): MemVectorIndex;
+}
+//#endregion
+//# sourceMappingURL=mem-vector-index.d.ts.map

package/dist/mem-vector-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-vector-index.d.ts","names":["EmbeddingBlock","EmbeddingSearchParams","EmbeddingSearchResult","EmbeddingIndexInfo","EmbeddingIndex"],"sources":["../src/mem-vector-index.ts"],"mappings":";;;qBAwCa,0BAETI,aACA,uBAAuBJ,aAAgBC,aAAuBC;mBAE/C;mBACA;UACT;;;;;;;;UAQA;UACA;EAER,YAAY,MAAMC;;;;;;EASlB;UAIQ;UAMC;EAYH,gBAAgB,QAAQA;EAKvB,OAAO,QAAQF,cAAwB,eAAeC;EA+BvD,YAAY,QAAQF,gBAAmB;EAgBvC,aACJ,QAAQ,SAASA,iBAAoB,cAAcA,iBAClD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;EAoBG,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAeA;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;;;;;;EAiBrB,oBAAoB;;;;;EAsCb,aAAa,cAAc;;;;;;EAS5B,SAAS,SAAS,SAAS,oBAAoB;;;;;;;SAiC9C,qBAAqB,MAAMG,iBAAoB,MAAM,aAAa"}

package/dist/mem-vector-index.js

@@ -0,0 +1,252 @@
+import { bruteForceSearch } from "./vector-search.js";
+import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core";
+import { fixedSizeList, float32, tableFromArrays, tableFromIPC, tableToIPC, utf8 } from "@uwdata/flechette";
+//#region src/mem-vector-index.ts
+var MemVectorIndex = class MemVectorIndex {
+	info;
+	entries = /* @__PURE__ */ new Map();
+	closed = false;
+	/**
+	* Dirty bit. `true` whenever `entries` has been mutated since the last
+	* `serializeToArrow()`. The serializer caches the last produced
+	* `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync
+	* skips the expensive Arrow IPC encoding for an unchanged index.
+	* Starts `true` so the first serialize after construction always runs.
+	*/
+	dirty = true;
+	cachedSerialized;
+	constructor(info) {
+		this.info = info;
+	}
+	/**
+	* `true` when the in-memory state has changed since the last serialize.
+	* Callers (e.g. the persistence layer) may use it to skip serialization
+	* and writes when the index has not been touched.
+	*/
+	isDirty() {
+		return this.dirty;
+	}
+	ensureOpen() {
+		if (this.closed) throw new Error("EmbeddingIndex is closed");
+	}
+	*filteredEntries(pathPrefixes) {
+		if (!pathPrefixes || pathPrefixes.length === 0) {
+			yield* this.entries.values();
+			return;
+		}
+		for (const entry of this.entries.values()) if (pathPrefixes.some((p) => matchesPrefix(entry.path, p))) yield entry;
+	}
+	async getIndexInfo() {
+		this.ensureOpen();
+		return { ...this.info };
+	}
+	async *search(params) {
+		this.ensureOpen();
+		const { embeddings, paths } = params;
+		const topK = params.topK ?? 100;
+		if (!embeddings || embeddings.length === 0) return;
+		const bestScores = /* @__PURE__ */ new Map();
+		for (const queryEmb of embeddings) {
+			validateDimensionality(this.info, queryEmb);
+			const filtered = [...this.filteredEntries(paths)];
+			const results = bruteForceSearch(queryEmb, filtered, topK);
+			for (const r of results) {
+				const key = compositeKey(r.path, r.blockId);
+				const existing = bestScores.get(key);
+				if (!existing || r.score > existing.score) bestScores.set(key, r);
+			}
+		}
+		const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score);
+		for (const r of sorted.slice(0, topK)) yield r;
+	}
+	async addDocument(blocks) {
+		this.ensureOpen();
+		if (blocks.length === 0) return;
+		for (const block of blocks) {
+			validateDimensionality(this.info, block.embedding);
+			const key = compositeKey(block.path, block.blockId);
+			this.entries.set(key, {
+				path: block.path,
+				blockId: block.blockId,
+				embedding: new Float32Array(block.embedding),
+				metadata: block.metadata
+			});
+		}
+		this.dirty = true;
+	}
+	async addDocuments(blocks) {
+		this.ensureOpen();
+		for await (const batch of blocks) await this.addDocument(batch);
+	}
+	async deleteDocuments(pathSelectors) {
+		this.ensureOpen();
+		let removed = 0;
+		for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
+			if (this.entries.delete(compositeKey(sel.path, sel.blockId))) removed += 1;
+		} else for (const [key, entry] of this.entries) if (matchesPrefix(entry.path, sel.path)) {
+			this.entries.delete(key);
+			removed += 1;
+		}
+		if (removed > 0) this.dirty = true;
+	}
+	async getSize(pathPrefix) {
+		this.ensureOpen();
+		if (pathPrefix === void 0) return this.entries.size;
+		let count = 0;
+		for (const entry of this.entries.values()) if (matchesPrefix(entry.path, pathPrefix)) count++;
+		return count;
+	}
+	async *getDocumentPaths(pathPrefix) {
+		this.ensureOpen();
+		const paths = /* @__PURE__ */ new Set();
+		for (const entry of this.entries.values()) if (pathPrefix === void 0 || matchesPrefix(entry.path, pathPrefix)) paths.add(entry.path);
+		for (const p of paths) yield p;
+	}
+	async *getDocumentBlocksRefs(pathPrefix) {
+		this.ensureOpen();
+		for (const entry of this.entries.values()) if (pathPrefix === void 0 || matchesPrefix(entry.path, pathPrefix)) yield {
+			path: entry.path,
+			blockId: entry.blockId
+		};
+	}
+	async *getDocumentsBlocks(pathPrefix) {
+		this.ensureOpen();
+		for (const entry of this.entries.values()) if (pathPrefix === void 0 || matchesPrefix(entry.path, pathPrefix)) yield {
+			path: entry.path,
+			blockId: entry.blockId,
+			embedding: entry.embedding,
+			metadata: entry.metadata
+		};
+	}
+	async close(_options) {
+		this.closed = true;
+	}
+	async flush() {
+		this.ensureOpen();
+	}
+	async deleteIndex() {
+		this.ensureOpen();
+		this.entries.clear();
+		this.cachedSerialized = void 0;
+		this.dirty = true;
+		this.closed = true;
+	}
+	/**
+	* Serialize embeddings to Arrow IPC format.
+	*
+	* Caches the result; while `dirty` is false (no mutations since the
+	* last call) returns the same `Uint8Array` instance, skipping the
+	* Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the
+	* persistence layer still calls into us each flush, but the heavy
+	* encoding only runs when the index actually changed.
+	*/
+	serializeToArrow() {
+		if (!this.dirty && this.cachedSerialized !== void 0) return this.cachedSerialized;
+		const dim = this.info.dimensionality;
+		const paths = [];
+		const blockIds = [];
+		const embeddingArrays = [];
+		const metadata = [];
+		for (const entry of this.entries.values()) {
+			paths.push(entry.path);
+			blockIds.push(entry.blockId);
+			embeddingArrays.push(Array.from(entry.embedding));
+			metadata.push(entry.metadata === void 0 ? null : JSON.stringify(entry.metadata));
+		}
+		const table = tableFromArrays({
+			path: paths,
+			blockId: blockIds,
+			embedding: embeddingArrays,
+			metadata
+		}, { types: {
+			path: utf8(),
+			blockId: utf8(),
+			embedding: fixedSizeList(float32(), dim),
+			metadata: utf8()
+		} });
+		const bytes = tableToIPC(table, { format: "stream" });
+		this.cachedSerialized = bytes;
+		this.dirty = false;
+		return bytes;
+	}
+	/**
+	* Stream the vector sub-index's state as a single named entry. The composite
+	* prefixes this with `${indexName}/${subName}/` before it hits persistence.
+	*/
+	async *serialise() {
+		yield {
+			name: "arrow",
+			content: singleChunk(this.serializeToArrow())
+		};
+	}
+	/**
+	* Restore the vector sub-index from a previously-produced entry set. Expects
+	* one entry named `"arrow"`; ignores entries with other names so future
+	* additions remain forward-compatible.
+	*/
+	async loadFrom(entries) {
+		this.ensureOpen();
+		for (const e of entries) {
+			if (e.name !== "arrow") continue;
+			const bytes = await readEntryBytes(e);
+			const table = tableFromIPC(bytes);
+			const pathCol = table.getChild("path");
+			const blockIdCol = table.getChild("blockId");
+			const embCol = table.getChild("embedding");
+			const metaCol = table.getChild("metadata");
+			this.entries.clear();
+			for (let i = 0; i < table.numRows; i++) {
+				const path = pathCol.at(i);
+				const blockId = blockIdCol.at(i);
+				const embedding = new Float32Array(embCol.at(i));
+				const metaRaw = metaCol?.at(i);
+				const metadata = metaRaw == null ? void 0 : JSON.parse(metaRaw);
+				const key = compositeKey(path, blockId);
+				this.entries.set(key, {
+					path,
+					blockId,
+					embedding,
+					metadata
+				});
+			}
+			this.cachedSerialized = bytes;
+			this.dirty = false;
+			return;
+		}
+	}
+	/**
+	* Deserialize embeddings from Arrow IPC format. The reconstructed index
+	* starts clean (`dirty = false`) and primes its serialize cache with the
+	* input bytes — a subsequent `serializeToArrow()` on the just-loaded
+	* index returns the same bytes without re-encoding.
+	*/
+	static deserializeFromArrow(info, data) {
+		const table = tableFromIPC(data);
+		const vec = new MemVectorIndex(info);
+		const pathCol = table.getChild("path");
+		const blockIdCol = table.getChild("blockId");
+		const embCol = table.getChild("embedding");
+		const metaCol = table.getChild("metadata");
+		for (let i = 0; i < table.numRows; i++) {
+			const path = pathCol.at(i);
+			const blockId = blockIdCol.at(i);
+			const embedding = new Float32Array(embCol.at(i));
+			const metaRaw = metaCol?.at(i);
+			const metadata = metaRaw == null ? void 0 : JSON.parse(metaRaw);
+			const key = compositeKey(path, blockId);
+			vec.entries.set(key, {
+				path,
+				blockId,
+				embedding,
+				metadata
+			});
+		}
+		vec.cachedSerialized = data;
+		vec.dirty = false;
+		return vec;
+	}
+};
+//#endregion
+export { MemVectorIndex };
+
+//# sourceMappingURL=mem-vector-index.js.map

package/dist/mem-vector-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-vector-index.js","names":[],"sources":["../src/mem-vector-index.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  Metadata,\n  PathSelector,\n  PersistableSearchIndex,\n  PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n  compositeKey,\n  matchesPrefix,\n  readEntryBytes,\n  singleChunk,\n  toAsyncIterable,\n  validateDimensionality,\n} from \"@statewalker/indexer-core\";\nimport type {\n  VectorBlock as EmbeddingBlock,\n  VectorIndex as EmbeddingIndex,\n  VectorIndexInfo as EmbeddingIndexInfo,\n  VectorQuery as EmbeddingSearchParams,\n  VectorResult as EmbeddingSearchResult,\n} from \"@statewalker/indexer-vector\";\nimport {\n  fixedSizeList,\n  float32,\n  tableFromArrays,\n  tableFromIPC,\n  tableToIPC,\n  utf8,\n} from \"@uwdata/flechette\";\nimport { bruteForceSearch } from \"./vector-search.js\";\n\ninterface StoredEntry {\n  path: DocumentPath;\n  blockId: string;\n  embedding: Float32Array;\n  metadata?: Metadata;\n}\n\nexport class MemVectorIndex\n  implements\n    EmbeddingIndex,\n    PersistableSearchIndex<EmbeddingBlock, EmbeddingSearchParams, EmbeddingSearchResult>\n{\n  private readonly info: EmbeddingIndexInfo;\n  private readonly entries = new Map<string, StoredEntry>();\n  private closed = false;\n  /**\n   * Dirty bit. `true` whenever `entries` has been mutated since the last\n   * `serializeToArrow()`. The serializer caches the last produced\n   * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync\n   * skips the expensive Arrow IPC encoding for an unchanged index.\n   * Starts `true` so the first serialize after construction always runs.\n   */\n  private dirty = true;\n  private cachedSerialized?: Uint8Array;\n\n  constructor(info: EmbeddingIndexInfo) {\n    this.info = info;\n  }\n\n  /**\n   * `true` when the in-memory state has changed since the last serialize.\n   * Callers (e.g. the persistence layer) may use it to skip serialization\n   * and writes when the index has not been touched.\n   */\n  isDirty(): boolean {\n    return this.dirty;\n  }\n\n  private ensureOpen(): void {\n    if (this.closed) {\n      throw new Error(\"EmbeddingIndex is closed\");\n    }\n  }\n\n  private *filteredEntries(pathPrefixes?: DocumentPath[]): Iterable<StoredEntry> {\n    if (!pathPrefixes || pathPrefixes.length === 0) {\n      yield* this.entries.values();\n      return;\n    }\n    for (const entry of this.entries.values()) {\n      if (pathPrefixes.some((p) => matchesPrefix(entry.path, p))) {\n        yield entry;\n      }\n    }\n  }\n\n  async getIndexInfo(): Promise<EmbeddingIndexInfo> {\n    this.ensureOpen();\n    return { ...this.info };\n  }\n\n  async *search(params: EmbeddingSearchParams): AsyncGenerator<EmbeddingSearchResult> {\n    this.ensureOpen();\n    const { embeddings, paths } = params;\n    // Sub-queries may omit `topK` to defer to the composite's top-level cutoff;\n    // when unset we retrieve generously so RRF has material to fuse.\n    const topK = params.topK ?? 100;\n\n    if (!embeddings || embeddings.length === 0) return;\n\n    // Multi-embedding: search for each, merge by best score\n    const bestScores = new Map<string, EmbeddingSearchResult>();\n\n    for (const queryEmb of embeddings) {\n      validateDimensionality(this.info, queryEmb);\n      const filtered = [...this.filteredEntries(paths)];\n      const results = bruteForceSearch(queryEmb, filtered, topK);\n      for (const r of results) {\n        const key = compositeKey(r.path, r.blockId);\n        const existing = bestScores.get(key);\n        if (!existing || r.score > existing.score) {\n          bestScores.set(key, r);\n        }\n      }\n    }\n\n    const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score);\n    for (const r of sorted.slice(0, topK)) {\n      yield r;\n    }\n  }\n\n  async addDocument(blocks: EmbeddingBlock[]): Promise<void> {\n    this.ensureOpen();\n    if (blocks.length === 0) return;\n    for (const block of blocks) {\n      validateDimensionality(this.info, block.embedding);\n      const key = compositeKey(block.path, block.blockId);\n      this.entries.set(key, {\n        path: block.path,\n        blockId: block.blockId,\n        embedding: new Float32Array(block.embedding),\n        metadata: block.metadata,\n      });\n    }\n    this.dirty = true;\n  }\n\n  async addDocuments(\n    blocks: Iterable<EmbeddingBlock[]> | AsyncIterable<EmbeddingBlock[]>,\n  ): Promise<void> {\n    this.ensureOpen();\n    for await (const batch of blocks) {\n      await this.addDocument(batch);\n    }\n  }\n\n  async deleteDocuments(\n    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n  ): Promise<void> {\n    this.ensureOpen();\n    let removed = 0;\n    for await (const sel of toAsyncIterable(pathSelectors)) {\n      if (sel.blockId !== undefined) {\n        if (this.entries.delete(compositeKey(sel.path, sel.blockId))) {\n          removed += 1;\n        }\n      } else {\n        for (const [key, entry] of this.entries) {\n          if (matchesPrefix(entry.path, sel.path)) {\n            this.entries.delete(key);\n            removed += 1;\n          }\n        }\n      }\n    }\n    if (removed > 0) this.dirty = true;\n  }\n\n  async getSize(pathPrefix?: DocumentPath): Promise<number> {\n    this.ensureOpen();\n    if (pathPrefix === undefined) return this.entries.size;\n    let count = 0;\n    for (const entry of this.entries.values()) {\n      if (matchesPrefix(entry.path, pathPrefix)) count++;\n    }\n    return count;\n  }\n\n  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n    this.ensureOpen();\n    const paths = new Set<DocumentPath>();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        paths.add(entry.path);\n      }\n    }\n    for (const p of paths) yield p;\n  }\n\n  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield { path: entry.path, blockId: entry.blockId };\n      }\n    }\n  }\n\n  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<EmbeddingBlock> {\n    this.ensureOpen();\n    for (const entry of this.entries.values()) {\n      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {\n        yield {\n          path: entry.path,\n          blockId: entry.blockId,\n          embedding: entry.embedding,\n          metadata: entry.metadata,\n        };\n      }\n    }\n  }\n\n  async close(_options?: { force?: boolean }): Promise<void> {\n    this.closed = true;\n  }\n\n  async flush(): Promise<void> {\n    this.ensureOpen();\n    // no-op for in-memory\n  }\n\n  async deleteIndex(): Promise<void> {\n    this.ensureOpen();\n    this.entries.clear();\n    this.cachedSerialized = undefined;\n    this.dirty = true;\n    this.closed = true;\n  }\n\n  /**\n   * Serialize embeddings to Arrow IPC format.\n   *\n   * Caches the result; while `dirty` is false (no mutations since the\n   * last call) returns the same `Uint8Array` instance, skipping the\n   * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the\n   * persistence layer still calls into us each flush, but the heavy\n   * encoding only runs when the index actually changed.\n   */\n  serializeToArrow(): Uint8Array {\n    if (!this.dirty && this.cachedSerialized !== undefined) {\n      return this.cachedSerialized;\n    }\n    const dim = this.info.dimensionality;\n    const paths: string[] = [];\n    const blockIds: string[] = [];\n    const embeddingArrays: number[][] = [];\n    const metadata: Array<string | null> = [];\n    for (const entry of this.entries.values()) {\n      paths.push(entry.path);\n      blockIds.push(entry.blockId);\n      embeddingArrays.push(Array.from(entry.embedding));\n      metadata.push(entry.metadata === undefined ? null : JSON.stringify(entry.metadata));\n    }\n    const table = tableFromArrays(\n      { path: paths, blockId: blockIds, embedding: embeddingArrays, metadata },\n      {\n        types: {\n          path: utf8(),\n          blockId: utf8(),\n          embedding: fixedSizeList(float32(), dim),\n          metadata: utf8(),\n        },\n      },\n    );\n    const bytes = tableToIPC(table, { format: \"stream\" }) as Uint8Array;\n    this.cachedSerialized = bytes;\n    this.dirty = false;\n    return bytes;\n  }\n\n  // --- PersistableSearchIndex --------------------------------------------------\n\n  /**\n   * Stream the vector sub-index's state as a single named entry. The composite\n   * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n   */\n  async *serialise(): AsyncIterable<PersistenceEntry> {\n    yield { name: \"arrow\", content: singleChunk(this.serializeToArrow()) };\n  }\n\n  /**\n   * Restore the vector sub-index from a previously-produced entry set. Expects\n   * one entry named `\"arrow\"`; ignores entries with other names so future\n   * additions remain forward-compatible.\n   */\n  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n    this.ensureOpen();\n    for (const e of entries) {\n      if (e.name !== \"arrow\") continue;\n      const bytes = await readEntryBytes(e);\n      const table = tableFromIPC(bytes);\n      const pathCol = table.getChild(\"path\");\n      const blockIdCol = table.getChild(\"blockId\");\n      const embCol = table.getChild(\"embedding\");\n      const metaCol = table.getChild(\"metadata\");\n      this.entries.clear();\n      for (let i = 0; i < table.numRows; i++) {\n        const path = pathCol.at(i) as string as DocumentPath;\n        const blockId = blockIdCol.at(i) as string;\n        const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n        const metaRaw = metaCol?.at(i) as string | null | undefined;\n        const metadata =\n          metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n        const key = compositeKey(path, blockId);\n        this.entries.set(key, { path, blockId, embedding, metadata });\n      }\n      this.cachedSerialized = bytes;\n      this.dirty = false;\n      return;\n    }\n  }\n\n  /**\n   * Deserialize embeddings from Arrow IPC format. The reconstructed index\n   * starts clean (`dirty = false`) and primes its serialize cache with the\n   * input bytes — a subsequent `serializeToArrow()` on the just-loaded\n   * index returns the same bytes without re-encoding.\n   */\n  static deserializeFromArrow(info: EmbeddingIndexInfo, data: Uint8Array): MemVectorIndex {\n    const table = tableFromIPC(data);\n    const vec = new MemVectorIndex(info);\n    const pathCol = table.getChild(\"path\");\n    const blockIdCol = table.getChild(\"blockId\");\n    const embCol = table.getChild(\"embedding\");\n    const metaCol = table.getChild(\"metadata\");\n    for (let i = 0; i < table.numRows; i++) {\n      const path = pathCol.at(i) as string as DocumentPath;\n      const blockId = blockIdCol.at(i) as string;\n      const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);\n      const metaRaw = metaCol?.at(i) as string | null | undefined;\n      const metadata =\n        metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry[\"metadata\"]);\n      const key = compositeKey(path, blockId);\n      vec.entries.set(key, { path, blockId, embedding, metadata });\n    }\n    vec.cachedSerialized = data;\n    vec.dirty = false;\n    return vec;\n  }\n}\n"],"mappings":";;;;AAwCA,IAAa,iBAAb,MAAa,eAIb;CACE;CACA,0BAA2B,IAAI,IAAyB;CACxD,SAAiB;;;;;;;;CAQjB,QAAgB;CAChB;CAEA,YAAY,MAA0B;EACpC,KAAK,OAAO;CACd;;;;;;CAOA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,0BAA0B;CAE9C;CAEA,CAAS,gBAAgB,cAAsD;EAC7E,IAAI,CAAC,gBAAgB,aAAa,WAAW,GAAG;GAC9C,OAAO,KAAK,QAAQ,OAAO;GAC3B;EACF;EACA,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC,GACvD,MAAM;CAGZ;CAEA,MAAM,eAA4C;EAChD,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAsE;EAClF,KAAK,WAAW;EAChB,MAAM,EAAE,YAAY,UAAU;EAG9B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,cAAc,WAAW,WAAW,GAAG;EAG5C,MAAM,6BAAa,IAAI,IAAmC;EAE1D,KAAK,MAAM,YAAY,YAAY;GACjC,uBAAuB,KAAK,MAAM,QAAQ;GAC1C,MAAM,WAAW,CAAC,GAAG,KAAK,gBAAgB,KAAK,CAAC;GAChD,MAAM,UAAU,iBAAiB,UAAU,UAAU,IAAI;GACzD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,WAAW,IAAI,GAAG;IACnC,IAAI,CAAC,YAAY,EAAE,QAAQ,SAAS,OAClC,WAAW,IAAI,KAAK,CAAC;GAEzB;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,WAAW,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACxE,KAAK,MAAM,KAAK,OAAO,MAAM,GAAG,IAAI,GAClC,MAAM;CAEV;CAEA,MAAM,YAAY,QAAyC;EACzD,KAAK,WAAW;EAChB,IAAI,OAAO,WAAW,GAAG;EACzB,KAAK,MAAM,SAAS,QAAQ;GAC1B,uBAAuB,KAAK,MAAM,MAAM,SAAS;GACjD,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,KAAK,QAAQ,IAAI,KAAK;IACpB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,WAAW,IAAI,aAAa,MAAM,SAAS;IAC3C,UAAU,MAAM;GAClB,CAAC;EACH;EACA,KAAK,QAAQ;CACf;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GACd;OAAA,KAAK,QAAQ,OAAO,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC,GACzD,WAAW;EAAA,OAGb,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,SAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GAAG;GACvC,KAAK,QAAQ,OAAO,GAAG;GACvB,WAAW;EACb;EAIN,IAAI,UAAU,GAAG,KAAK,QAAQ;CAChC;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,QAAQ;EAClD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA2D;EACnF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GACtC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,WAAW,MAAM;GACjB,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,QAAQ,MAAM;EACnB,KAAK,mBAAmB,KAAA;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS;CAChB;;;;;;;;;;CAWA,mBAA+B;EAC7B,IAAI,CAAC,KAAK,SAAS,KAAK,qBAAqB,KAAA,GAC3C,OAAO,KAAK;EAEd,MAAM,MAAM,KAAK,KAAK;EACtB,MAAM,QAAkB,CAAC;EACzB,MAAM,WAAqB,CAAC;EAC5B,MAAM,kBAA8B,CAAC;EACrC,MAAM,WAAiC,CAAC;EACxC,KAAK,MAAM,SAAS,KAAK,QAAQ,OAAO,GAAG;GACzC,MAAM,KAAK,MAAM,IAAI;GACrB,SAAS,KAAK,MAAM,OAAO;GAC3B,gBAAgB,KAAK,MAAM,KAAK,MAAM,SAAS,CAAC;GAChD,SAAS,KAAK,MAAM,aAAa,KAAA,IAAY,OAAO,KAAK,UAAU,MAAM,QAAQ,CAAC;EACpF;EACA,MAAM,QAAQ,gBACZ;GAAE,MAAM;GAAO,SAAS;GAAU,WAAW;GAAiB;EAAS,GACvE,EACE,OAAO;GACL,MAAM,KAAK;GACX,SAAS,KAAK;GACd,WAAW,cAAc,QAAQ,GAAG,GAAG;GACvC,UAAU,KAAK;EACjB,EACF,CACF;EACA,MAAM,QAAQ,WAAW,OAAO,EAAE,QAAQ,SAAS,CAAC;EACpD,KAAK,mBAAmB;EACxB,KAAK,QAAQ;EACb,OAAO;CACT;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAS,SAAS,YAAY,KAAK,iBAAiB,CAAC;EAAE;CACvE;;;;;;CAOA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,SAAS;GACxB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,QAAQ,aAAa,KAAK;GAChC,MAAM,UAAU,MAAM,SAAS,MAAM;GACrC,MAAM,aAAa,MAAM,SAAS,SAAS;GAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;GACzC,MAAM,UAAU,MAAM,SAAS,UAAU;GACzC,KAAK,QAAQ,MAAM;GACnB,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;IACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;IACzB,MAAM,UAAU,WAAW,GAAG,CAAC;IAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;IACpE,MAAM,UAAU,SAAS,GAAG,CAAC;IAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;IACnD,MAAM,MAAM,aAAa,MAAM,OAAO;IACtC,KAAK,QAAQ,IAAI,KAAK;KAAE;KAAM;KAAS;KAAW;IAAS,CAAC;GAC9D;GACA,KAAK,mBAAmB;GACxB,KAAK,QAAQ;GACb;EACF;CACF;;;;;;;CAQA,OAAO,qBAAqB,MAA0B,MAAkC;EACtF,MAAM,QAAQ,aAAa,IAAI;EAC/B,MAAM,MAAM,IAAI,eAAe,IAAI;EACnC,MAAM,UAAU,MAAM,SAAS,MAAM;EACrC,MAAM,aAAa,MAAM,SAAS,SAAS;EAC3C,MAAM,SAAS,MAAM,SAAS,WAAW;EACzC,MAAM,UAAU,MAAM,SAAS,UAAU;EACzC,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,SAAS,KAAK;GACtC,MAAM,OAAO,QAAQ,GAAG,CAAC;GACzB,MAAM,UAAU,WAAW,GAAG,CAAC;GAC/B,MAAM,YAAY,IAAI,aAAa,OAAO,GAAG,CAAC,CAAsB;GACpE,MAAM,UAAU,SAAS,GAAG,CAAC;GAC7B,MAAM,WACJ,WAAW,OAAO,KAAA,IAAa,KAAK,MAAM,OAAO;GACnD,MAAM,MAAM,aAAa,MAAM,OAAO;GACtC,IAAI,QAAQ,IAAI,KAAK;IAAE;IAAM;IAAS;IAAW;GAAS,CAAC;EAC7D;EACA,IAAI,mBAAmB;EACvB,IAAI,QAAQ;EACZ,OAAO;CACT;AACF"}

package/dist/mem-vector-provider.d.ts

@@ -0,0 +1,10 @@
+import { VectorProvider } from "@statewalker/indexer-vector";
+//#region src/mem-vector-provider.d.ts
+/**
+ * `VectorProvider` backed by {@link MemVectorIndex} — the in-memory brute-force
+ * vector store. Wired into an `Indexer` at construction time via
+ * `createPersistenceBackedIndexer({ providers: [memVectorProvider] })`.
+ */
+export declare const memVectorProvider: VectorProvider;
+//#endregion
+//# sourceMappingURL=mem-vector-provider.d.ts.map

package/dist/mem-vector-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-vector-provider.d.ts","names":[],"sources":["../src/mem-vector-provider.ts"],"mappings":";;;;;;;qBAQa,mBAAmB"}

package/dist/mem-vector-provider.js

@@ -0,0 +1,22 @@
+import { MemVectorIndex } from "./mem-vector-index.js";
+import { VECTOR_TYPE } from "@statewalker/indexer-vector";
+//#region src/mem-vector-provider.ts
+/**
+* `VectorProvider` backed by {@link MemVectorIndex} — the in-memory brute-force
+* vector store. Wired into an `Indexer` at construction time via
+* `createPersistenceBackedIndexer({ providers: [memVectorProvider] })`.
+*/
+const memVectorProvider = {
+	type: VECTOR_TYPE,
+	create(config) {
+		return new MemVectorIndex({
+			dimensionality: config.dimensionality,
+			model: config.model,
+			metadata: config.metadata
+		});
+	}
+};
+//#endregion
+export { memVectorProvider };
+
+//# sourceMappingURL=mem-vector-provider.js.map

package/dist/mem-vector-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-vector-provider.js","names":[],"sources":["../src/mem-vector-provider.ts"],"sourcesContent":["import { VECTOR_TYPE, type VectorConfig, type VectorProvider } from \"@statewalker/indexer-vector\";\nimport { MemVectorIndex } from \"./mem-vector-index.js\";\n\n/**\n * `VectorProvider` backed by {@link MemVectorIndex} — the in-memory brute-force\n * vector store. Wired into an `Indexer` at construction time via\n * `createPersistenceBackedIndexer({ providers: [memVectorProvider] })`.\n */\nexport const memVectorProvider: VectorProvider = {\n  type: VECTOR_TYPE,\n  create(config: VectorConfig): MemVectorIndex {\n    return new MemVectorIndex({\n      dimensionality: config.dimensionality,\n      model: config.model,\n      metadata: config.metadata,\n    });\n  },\n};\n"],"mappings":";;;;;;;;AAQA,MAAa,oBAAoC;CAC/C,MAAM;CACN,OAAO,QAAsC;EAC3C,OAAO,IAAI,eAAe;GACxB,gBAAgB,OAAO;GACvB,OAAO,OAAO;GACd,UAAU,OAAO;EACnB,CAAC;CACH;AACF"}

package/dist/vector-search.js

@@ -0,0 +1,34 @@
+//#region src/vector-search.ts
+function cosineSimilarity(a, b) {
+	if (a.length !== b.length) throw new Error(`cosineSimilarity: dimensionality mismatch (${a.length} vs ${b.length})`);
+	let dot = 0;
+	let normA = 0;
+	let normB = 0;
+	for (let i = 0; i < a.length; i++) {
+		const ai = a[i] ?? 0;
+		const bi = b[i] ?? 0;
+		dot += ai * bi;
+		normA += ai * ai;
+		normB += bi * bi;
+	}
+	const denom = Math.sqrt(normA) * Math.sqrt(normB);
+	if (denom === 0) return 0;
+	return dot / denom;
+}
+function bruteForceSearch(query, entries, topK) {
+	const scored = [];
+	for (const entry of entries) {
+		const score = cosineSimilarity(query, entry.embedding);
+		scored.push({
+			path: entry.path,
+			blockId: entry.blockId,
+			score
+		});
+	}
+	scored.sort((a, b) => b.score - a.score);
+	return scored.slice(0, topK);
+}
+//#endregion
+export { bruteForceSearch, cosineSimilarity };
+
+//# sourceMappingURL=vector-search.js.map

package/dist/vector-search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-search.js","names":[],"sources":["../src/vector-search.ts"],"sourcesContent":["import type { DocumentPath } from \"@statewalker/indexer-api\";\nimport type { VectorResult as EmbeddingSearchResult } from \"@statewalker/indexer-vector\";\n\nexport function cosineSimilarity(a: Float32Array, b: Float32Array): number {\n  if (a.length !== b.length) {\n    throw new Error(`cosineSimilarity: dimensionality mismatch (${a.length} vs ${b.length})`);\n  }\n  let dot = 0;\n  let normA = 0;\n  let normB = 0;\n  for (let i = 0; i < a.length; i++) {\n    const ai = a[i] ?? 0;\n    const bi = b[i] ?? 0;\n    dot += ai * bi;\n    normA += ai * ai;\n    normB += bi * bi;\n  }\n  const denom = Math.sqrt(normA) * Math.sqrt(normB);\n  if (denom === 0) return 0;\n  return dot / denom;\n}\n\nexport function bruteForceSearch(\n  query: Float32Array,\n  entries: Iterable<{\n    path: DocumentPath;\n    blockId: string;\n    embedding: Float32Array;\n  }>,\n  topK: number,\n): EmbeddingSearchResult[] {\n  const scored: EmbeddingSearchResult[] = [];\n  for (const entry of entries) {\n    const score = cosineSimilarity(query, entry.embedding);\n    scored.push({ path: entry.path, blockId: entry.blockId, score });\n  }\n  scored.sort((a, b) => b.score - a.score);\n  return scored.slice(0, topK);\n}\n"],"mappings":";AAGA,SAAgB,iBAAiB,GAAiB,GAAyB;CACzE,IAAI,EAAE,WAAW,EAAE,QACjB,MAAM,IAAI,MAAM,8CAA8C,EAAE,OAAO,MAAM,EAAE,OAAO,EAAE;CAE1F,IAAI,MAAM;CACV,IAAI,QAAQ;CACZ,IAAI,QAAQ;CACZ,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;EACjC,MAAM,KAAK,EAAE,MAAM;EACnB,MAAM,KAAK,EAAE,MAAM;EACnB,OAAO,KAAK;EACZ,SAAS,KAAK;EACd,SAAS,KAAK;CAChB;CACA,MAAM,QAAQ,KAAK,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK;CAChD,IAAI,UAAU,GAAG,OAAO;CACxB,OAAO,MAAM;AACf;AAEA,SAAgB,iBACd,OACA,SAKA,MACyB;CACzB,MAAM,SAAkC,CAAC;CACzC,KAAK,MAAM,SAAS,SAAS;EAC3B,MAAM,QAAQ,iBAAiB,OAAO,MAAM,SAAS;EACrD,OAAO,KAAK;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;GAAS;EAAM,CAAC;CACjE;CACA,OAAO,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;CACvC,OAAO,OAAO,MAAM,GAAG,IAAI;AAC7B"}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@statewalker/indexer-mem",
+  "version": "0.1.2",
+  "private": false,
+  "type": "module",
+  "description": "In-memory base implementation of @statewalker/indexer-api. Reusable scaffold for full-text and vector in-memory indexers.",
+  "homepage": "https://github.com/statewalker/statewalker-indexer",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git"
+  },
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@uwdata/flechette": "^2.5.0",
+    "@statewalker/indexer-api": "^0.1.1",
+    "@statewalker/indexer-vector": "^0.1.1",
+    "@statewalker/indexer-core": "^0.1.1"
+  },
+  "devDependencies": {
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.23.0",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "test": "vitest run --passWithNoTests",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export { MemVectorIndex } from "./mem-vector-index.js";
+export { memVectorProvider } from "./mem-vector-provider.js";

package/src/mem-vector-index.ts

@@ -0,0 +1,344 @@
+import type {
+  BlockReference,
+  DocumentPath,
+  Metadata,
+  PathSelector,
+  PersistableSearchIndex,
+  PersistenceEntry,
+} from "@statewalker/indexer-api";
+import {
+  compositeKey,
+  matchesPrefix,
+  readEntryBytes,
+  singleChunk,
+  toAsyncIterable,
+  validateDimensionality,
+} from "@statewalker/indexer-core";
+import type {
+  VectorBlock as EmbeddingBlock,
+  VectorIndex as EmbeddingIndex,
+  VectorIndexInfo as EmbeddingIndexInfo,
+  VectorQuery as EmbeddingSearchParams,
+  VectorResult as EmbeddingSearchResult,
+} from "@statewalker/indexer-vector";
+import {
+  fixedSizeList,
+  float32,
+  tableFromArrays,
+  tableFromIPC,
+  tableToIPC,
+  utf8,
+} from "@uwdata/flechette";
+import { bruteForceSearch } from "./vector-search.js";
+
+interface StoredEntry {
+  path: DocumentPath;
+  blockId: string;
+  embedding: Float32Array;
+  metadata?: Metadata;
+}
+
+export class MemVectorIndex
+  implements
+    EmbeddingIndex,
+    PersistableSearchIndex<EmbeddingBlock, EmbeddingSearchParams, EmbeddingSearchResult>
+{
+  private readonly info: EmbeddingIndexInfo;
+  private readonly entries = new Map<string, StoredEntry>();
+  private closed = false;
+  /**
+   * Dirty bit. `true` whenever `entries` has been mutated since the last
+   * `serializeToArrow()`. The serializer caches the last produced
+   * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync
+   * skips the expensive Arrow IPC encoding for an unchanged index.
+   * Starts `true` so the first serialize after construction always runs.
+   */
+  private dirty = true;
+  private cachedSerialized?: Uint8Array;
+
+  constructor(info: EmbeddingIndexInfo) {
+    this.info = info;
+  }
+
+  /**
+   * `true` when the in-memory state has changed since the last serialize.
+   * Callers (e.g. the persistence layer) may use it to skip serialization
+   * and writes when the index has not been touched.
+   */
+  isDirty(): boolean {
+    return this.dirty;
+  }
+
+  private ensureOpen(): void {
+    if (this.closed) {
+      throw new Error("EmbeddingIndex is closed");
+    }
+  }
+
+  private *filteredEntries(pathPrefixes?: DocumentPath[]): Iterable<StoredEntry> {
+    if (!pathPrefixes || pathPrefixes.length === 0) {
+      yield* this.entries.values();
+      return;
+    }
+    for (const entry of this.entries.values()) {
+      if (pathPrefixes.some((p) => matchesPrefix(entry.path, p))) {
+        yield entry;
+      }
+    }
+  }
+
+  async getIndexInfo(): Promise<EmbeddingIndexInfo> {
+    this.ensureOpen();
+    return { ...this.info };
+  }
+
+  async *search(params: EmbeddingSearchParams): AsyncGenerator<EmbeddingSearchResult> {
+    this.ensureOpen();
+    const { embeddings, paths } = params;
+    // Sub-queries may omit `topK` to defer to the composite's top-level cutoff;
+    // when unset we retrieve generously so RRF has material to fuse.
+    const topK = params.topK ?? 100;
+
+    if (!embeddings || embeddings.length === 0) return;
+
+    // Multi-embedding: search for each, merge by best score
+    const bestScores = new Map<string, EmbeddingSearchResult>();
+
+    for (const queryEmb of embeddings) {
+      validateDimensionality(this.info, queryEmb);
+      const filtered = [...this.filteredEntries(paths)];
+      const results = bruteForceSearch(queryEmb, filtered, topK);
+      for (const r of results) {
+        const key = compositeKey(r.path, r.blockId);
+        const existing = bestScores.get(key);
+        if (!existing || r.score > existing.score) {
+          bestScores.set(key, r);
+        }
+      }
+    }
+
+    const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score);
+    for (const r of sorted.slice(0, topK)) {
+      yield r;
+    }
+  }
+
+  async addDocument(blocks: EmbeddingBlock[]): Promise<void> {
+    this.ensureOpen();
+    if (blocks.length === 0) return;
+    for (const block of blocks) {
+      validateDimensionality(this.info, block.embedding);
+      const key = compositeKey(block.path, block.blockId);
+      this.entries.set(key, {
+        path: block.path,
+        blockId: block.blockId,
+        embedding: new Float32Array(block.embedding),
+        metadata: block.metadata,
+      });
+    }
+    this.dirty = true;
+  }
+
+  async addDocuments(
+    blocks: Iterable<EmbeddingBlock[]> | AsyncIterable<EmbeddingBlock[]>,
+  ): Promise<void> {
+    this.ensureOpen();
+    for await (const batch of blocks) {
+      await this.addDocument(batch);
+    }
+  }
+
+  async deleteDocuments(
+    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
+  ): Promise<void> {
+    this.ensureOpen();
+    let removed = 0;
+    for await (const sel of toAsyncIterable(pathSelectors)) {
+      if (sel.blockId !== undefined) {
+        if (this.entries.delete(compositeKey(sel.path, sel.blockId))) {
+          removed += 1;
+        }
+      } else {
+        for (const [key, entry] of this.entries) {
+          if (matchesPrefix(entry.path, sel.path)) {
+            this.entries.delete(key);
+            removed += 1;
+          }
+        }
+      }
+    }
+    if (removed > 0) this.dirty = true;
+  }
+
+  async getSize(pathPrefix?: DocumentPath): Promise<number> {
+    this.ensureOpen();
+    if (pathPrefix === undefined) return this.entries.size;
+    let count = 0;
+    for (const entry of this.entries.values()) {
+      if (matchesPrefix(entry.path, pathPrefix)) count++;
+    }
+    return count;
+  }
+
+  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
+    this.ensureOpen();
+    const paths = new Set<DocumentPath>();
+    for (const entry of this.entries.values()) {
+      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {
+        paths.add(entry.path);
+      }
+    }
+    for (const p of paths) yield p;
+  }
+
+  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
+    this.ensureOpen();
+    for (const entry of this.entries.values()) {
+      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {
+        yield { path: entry.path, blockId: entry.blockId };
+      }
+    }
+  }
+
+  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<EmbeddingBlock> {
+    this.ensureOpen();
+    for (const entry of this.entries.values()) {
+      if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) {
+        yield {
+          path: entry.path,
+          blockId: entry.blockId,
+          embedding: entry.embedding,
+          metadata: entry.metadata,
+        };
+      }
+    }
+  }
+
+  async close(_options?: { force?: boolean }): Promise<void> {
+    this.closed = true;
+  }
+
+  async flush(): Promise<void> {
+    this.ensureOpen();
+    // no-op for in-memory
+  }
+
+  async deleteIndex(): Promise<void> {
+    this.ensureOpen();
+    this.entries.clear();
+    this.cachedSerialized = undefined;
+    this.dirty = true;
+    this.closed = true;
+  }
+
+  /**
+   * Serialize embeddings to Arrow IPC format.
+   *
+   * Caches the result; while `dirty` is false (no mutations since the
+   * last call) returns the same `Uint8Array` instance, skipping the
+   * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the
+   * persistence layer still calls into us each flush, but the heavy
+   * encoding only runs when the index actually changed.
+   */
+  serializeToArrow(): Uint8Array {
+    if (!this.dirty && this.cachedSerialized !== undefined) {
+      return this.cachedSerialized;
+    }
+    const dim = this.info.dimensionality;
+    const paths: string[] = [];
+    const blockIds: string[] = [];
+    const embeddingArrays: number[][] = [];
+    const metadata: Array<string | null> = [];
+    for (const entry of this.entries.values()) {
+      paths.push(entry.path);
+      blockIds.push(entry.blockId);
+      embeddingArrays.push(Array.from(entry.embedding));
+      metadata.push(entry.metadata === undefined ? null : JSON.stringify(entry.metadata));
+    }
+    const table = tableFromArrays(
+      { path: paths, blockId: blockIds, embedding: embeddingArrays, metadata },
+      {
+        types: {
+          path: utf8(),
+          blockId: utf8(),
+          embedding: fixedSizeList(float32(), dim),
+          metadata: utf8(),
+        },
+      },
+    );
+    const bytes = tableToIPC(table, { format: "stream" }) as Uint8Array;
+    this.cachedSerialized = bytes;
+    this.dirty = false;
+    return bytes;
+  }
+
+  // --- PersistableSearchIndex --------------------------------------------------
+
+  /**
+   * Stream the vector sub-index's state as a single named entry. The composite
+   * prefixes this with `${indexName}/${subName}/` before it hits persistence.
+   */
+  async *serialise(): AsyncIterable<PersistenceEntry> {
+    yield { name: "arrow", content: singleChunk(this.serializeToArrow()) };
+  }
+
+  /**
+   * Restore the vector sub-index from a previously-produced entry set. Expects
+   * one entry named `"arrow"`; ignores entries with other names so future
+   * additions remain forward-compatible.
+   */
+  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
+    this.ensureOpen();
+    for (const e of entries) {
+      if (e.name !== "arrow") continue;
+      const bytes = await readEntryBytes(e);
+      const table = tableFromIPC(bytes);
+      const pathCol = table.getChild("path");
+      const blockIdCol = table.getChild("blockId");
+      const embCol = table.getChild("embedding");
+      const metaCol = table.getChild("metadata");
+      this.entries.clear();
+      for (let i = 0; i < table.numRows; i++) {
+        const path = pathCol.at(i) as string as DocumentPath;
+        const blockId = blockIdCol.at(i) as string;
+        const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);
+        const metaRaw = metaCol?.at(i) as string | null | undefined;
+        const metadata =
+          metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry["metadata"]);
+        const key = compositeKey(path, blockId);
+        this.entries.set(key, { path, blockId, embedding, metadata });
+      }
+      this.cachedSerialized = bytes;
+      this.dirty = false;
+      return;
+    }
+  }
+
+  /**
+   * Deserialize embeddings from Arrow IPC format. The reconstructed index
+   * starts clean (`dirty = false`) and primes its serialize cache with the
+   * input bytes — a subsequent `serializeToArrow()` on the just-loaded
+   * index returns the same bytes without re-encoding.
+   */
+  static deserializeFromArrow(info: EmbeddingIndexInfo, data: Uint8Array): MemVectorIndex {
+    const table = tableFromIPC(data);
+    const vec = new MemVectorIndex(info);
+    const pathCol = table.getChild("path");
+    const blockIdCol = table.getChild("blockId");
+    const embCol = table.getChild("embedding");
+    const metaCol = table.getChild("metadata");
+    for (let i = 0; i < table.numRows; i++) {
+      const path = pathCol.at(i) as string as DocumentPath;
+      const blockId = blockIdCol.at(i) as string;
+      const embedding = new Float32Array(embCol.at(i) as ArrayLike<number>);
+      const metaRaw = metaCol?.at(i) as string | null | undefined;
+      const metadata =
+        metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry["metadata"]);
+      const key = compositeKey(path, blockId);
+      vec.entries.set(key, { path, blockId, embedding, metadata });
+    }
+    vec.cachedSerialized = data;
+    vec.dirty = false;
+    return vec;
+  }
+}

package/src/mem-vector-provider.ts

@@ -0,0 +1,18 @@
+import { VECTOR_TYPE, type VectorConfig, type VectorProvider } from "@statewalker/indexer-vector";
+import { MemVectorIndex } from "./mem-vector-index.js";
+
+/**
+ * `VectorProvider` backed by {@link MemVectorIndex} — the in-memory brute-force
+ * vector store. Wired into an `Indexer` at construction time via
+ * `createPersistenceBackedIndexer({ providers: [memVectorProvider] })`.
+ */
+export const memVectorProvider: VectorProvider = {
+  type: VECTOR_TYPE,
+  create(config: VectorConfig): MemVectorIndex {
+    return new MemVectorIndex({
+      dimensionality: config.dimensionality,
+      model: config.model,
+      metadata: config.metadata,
+    });
+  },
+};

package/src/text-codec.d.ts

@@ -0,0 +1,14 @@
+// See ../../indexer-core/src/text-codec.d.ts for rationale.
+declare const TextEncoder: {
+  new (): {
+    encode(input: string): Uint8Array;
+  };
+};
+
+declare const TextDecoder: {
+  new (
+    label?: string,
+  ): {
+    decode(input?: Uint8Array): string;
+  };
+};

package/src/vector-search.ts

@@ -0,0 +1,39 @@
+import type { DocumentPath } from "@statewalker/indexer-api";
+import type { VectorResult as EmbeddingSearchResult } from "@statewalker/indexer-vector";
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  if (a.length !== b.length) {
+    throw new Error(`cosineSimilarity: dimensionality mismatch (${a.length} vs ${b.length})`);
+  }
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    const ai = a[i] ?? 0;
+    const bi = b[i] ?? 0;
+    dot += ai * bi;
+    normA += ai * ai;
+    normB += bi * bi;
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  if (denom === 0) return 0;
+  return dot / denom;
+}
+
+export function bruteForceSearch(
+  query: Float32Array,
+  entries: Iterable<{
+    path: DocumentPath;
+    blockId: string;
+    embedding: Float32Array;
+  }>,
+  topK: number,
+): EmbeddingSearchResult[] {
+  const scored: EmbeddingSearchResult[] = [];
+  for (const entry of entries) {
+    const score = cosineSimilarity(query, entry.embedding);
+    scored.push({ path: entry.path, blockId: entry.blockId, score });
+  }
+  scored.sort((a, b) => b.score - a.score);
+  return scored.slice(0, topK);
+}