@statewalker/indexer-mem 0.1.5 → 0.3.0

package/README.md

@@ -4,9 +4,11 @@
 
 An in-memory vector sub-index, `MemVectorIndex`, and `memVectorProvider`, the `VectorProvider` that creates it. Search is a brute-force cosine scan over all stored embeddings. The index can save itself as one Apache Arrow IPC stream.
 
+It also holds the in-memory block store behind the full-text indexes. `MemBlockStore` owns the blocks keyed by path + blockId, path-prefix selection, enumeration, multi-query score fusion (scores summed across queries), dirty tracking and the persistence framing (`{ version: 3, ...engineState, blocks }`). `MemFullTextIndex` is the `FullTextIndex` over that store. A full-text engine plugs in as a `MemBlockEngine` adapter that supplies only `add`, `remove`, `search` and `exportState`.
+
 ## Why it exists
 
-The in-memory indexers (`@statewalker/indexer-mem-flexsearch`, `@statewalker/indexer-mem-minisearch`) differ only in their full-text engine. Both use this package for the vector side, so the vector code and its file format exist once.
+The in-memory indexers (`@statewalker/indexer-mem-flexsearch`, `@statewalker/indexer-mem-minisearch`) differ only in their full-text engine. Both use this package for the vector side and for the full-text block store, so the vector code, the block and path rules and the file formats exist once, and another engine costs only an adapter.
 
 ## How to use
 
@@ -14,7 +16,7 @@ The in-memory indexers (`@statewalker/indexer-mem-flexsearch`, `@statewalker/ind
 pnpm add @statewalker/indexer-mem
 ```
 
-One entry point, `@statewalker/indexer-mem`: `MemVectorIndex`, `memVectorProvider`. Runs anywhere (browser, Node, workers).
+One entry point, `@statewalker/indexer-mem`: `MemVectorIndex`, `memVectorProvider`, `MemBlockStore`, `MemBlockEngine`, `MemFullTextIndex`. Runs anywhere (browser, Node, workers).
 
 Most applications use `createFlexSearchIndexer` or `createMiniSearchIndexer`, which already include `memVectorProvider`. Use this package directly when you build your own indexer from providers.
 

package/dist/index.d.ts

@@ -1,3 +1,5 @@
+import { MEM_BLOCK_STORE_VERSION, MemBlockEngine, MemBlockHit, MemBlockStore, MemBlockStoreOptions, RankedKey } from "./mem-block-store.js";
+import { MemFullTextEngine, MemFullTextIndex } from "./mem-full-text-index.js";
 import { MemVectorIndex } from "./mem-vector-index.js";
 import { memVectorProvider } from "./mem-vector-provider.js";
-export { MemVectorIndex, memVectorProvider };
+export { MEM_BLOCK_STORE_VERSION, type MemBlockEngine, type MemBlockHit, MemBlockStore, type MemBlockStoreOptions, type MemFullTextEngine, MemFullTextIndex, MemVectorIndex, type RankedKey, memVectorProvider };

package/dist/index.js

@@ -1,3 +1,5 @@
+import { MEM_BLOCK_STORE_VERSION, MemBlockStore } from "./mem-block-store.js";
+import { MemFullTextIndex } from "./mem-full-text-index.js";
 import { MemVectorIndex } from "./mem-vector-index.js";
 import { memVectorProvider } from "./mem-vector-provider.js";
-export { MemVectorIndex, memVectorProvider };
+export { MEM_BLOCK_STORE_VERSION, MemBlockStore, MemFullTextIndex, MemVectorIndex, memVectorProvider };

package/dist/mem-block-store.d.ts

@@ -0,0 +1,114 @@
+import { BlockReference, DocumentPath, PathSelector } from "@statewalker/indexer-api";
+//#region src/mem-block-store.d.ts
+/** One engine hit: the store key of a block and the engine's native score. */
+export interface RankedKey {
+  key: string;
+  score: number;
+}
+/**
+ * The engine side of a {@link MemBlockStore}: what one search engine
+ * (FlexSearch, MiniSearch, …) supplies. The store owns the blocks and calls
+ * the engine only to index, unindex and rank them, addressed by the store's
+ * key for a block.
+ */
+export interface MemBlockEngine<B> {
+  /** Index `block` under `key`. A previous version is always removed first. */
+  add(key: string, block: B): void;
+  /** Unindex the block under `key`. Called only for keys the store holds. */
+  remove(key: string): void;
+  /**
+   * Rank the indexed blocks for one query, best first. `limit` is the depth
+   * the caller wants after path filtering; an engine may return more.
+   * Keys the store no longer holds are ignored.
+   */
+  search(query: string, limit: number): RankedKey[];
+  /**
+   * The engine's own state as JSON-safe fields, framed with the blocks by
+   * {@link MemBlockStore.serialize}. Must not use the names `version` or `blocks`.
+   */
+  exportState(): Record<string, unknown>;
+}
+/** A fused hit from {@link MemBlockStore.search}. */
+export interface MemBlockHit<B> {
+  block: B;
+  /** The sum of the block's scores over every query it matched. */
+  score: number;
+}
+export interface MemBlockStoreOptions<B> {
+  /** Named in errors, e.g. the class of the index that owns the store. */
+  label: string;
+  engine: MemBlockEngine<B>;
+}
+/** The persistence format version written by {@link MemBlockStore.serialize}. */
+export declare const MEM_BLOCK_STORE_VERSION = 3;
+/**
+ * In-memory block store shared by the in-memory search engines.
+ *
+ * Owns the blocks keyed by `path` + `blockId`, path-prefix selection,
+ * enumeration, multi-query score fusion, dirty tracking and the persistence
+ * framing; the {@link MemBlockEngine} supplies only add, remove and search.
+ *
+ * Dirty tracking: the store starts dirty, becomes clean after
+ * {@link serialize} or {@link restore}, and becomes dirty again only on a
+ * change. Empty batches and deletes that remove nothing are not changes.
+ * While clean, `serialize` returns the cached output without asking the
+ * engine to export again.
+ */
+export declare class MemBlockStore<B extends BlockReference> {
+  private readonly label;
+  private engineRef;
+  private blocks;
+  private dirty;
+  private cached?;
+  constructor(options: MemBlockStoreOptions<B>);
+  /** The engine currently indexing the blocks (replaced by {@link restore}). */
+  get engine(): MemBlockEngine<B>;
+  /** `true` when the blocks changed since the last serialize or restore. */
+  isDirty(): boolean;
+  private markDirty;
+  /** The block under `key`, if any. */
+  get(key: string): B | undefined;
+  /**
+   * Add or replace a batch of blocks. An empty batch changes nothing and
+   * leaves the store clean.
+   */
+  put(batch: readonly B[]): void;
+  /**
+   * Delete the blocks the selectors match: one block when `blockId` is set,
+   * otherwise every block under the path prefix. Returns how many blocks were
+   * removed; the store becomes dirty only when that is more than zero.
+   */
+  delete(selectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<number>;
+  private removeKey;
+  /** Drop every block, from the store and the engine. */
+  clear(): void;
+  private keys;
+  /** The blocks under `pathPrefix` (all blocks without one), in insertion order. */
+  select(pathPrefix?: DocumentPath): Iterable<B>;
+  /** How many blocks lie under `pathPrefix` (all blocks without one). */
+  count(pathPrefix?: DocumentPath): number;
+  /** The distinct document paths under `pathPrefix`, in first-seen order. */
+  paths(pathPrefix?: DocumentPath): Iterable<DocumentPath>;
+  /**
+   * Multi-query search with score fusion. Each query is ranked by the engine,
+   * filtered to `pathPrefixes` (when given) and cut to `topK`; the per-block
+   * scores are then **summed** across queries, so a block matching more
+   * queries ranks higher, and the fused list is cut to `topK` again.
+   */
+  search(queries: readonly string[], topK: number, pathPrefixes?: DocumentPath[]): MemBlockHit<B>[];
+  /**
+   * Serialize the blocks and the engine state into one JSON document,
+   * `{ version: 3, ...engineState, blocks }`. While the store is clean the
+   * previous output is returned as is.
+   */
+  serialize(): string;
+  /**
+   * Replace the store's content with a document written by {@link serialize}.
+   * `restoreEngine` rebuilds the engine from the document's fields. Throws on
+   * a document of another version, leaving the store unchanged. The store is
+   * clean afterwards and re-serializes to `json` until it changes.
+   */
+  restore(json: string, restoreEngine: (state: Record<string, unknown>) => MemBlockEngine<B>): void;
+}
+//#endregion
+//# sourceMappingURL=mem-block-store.d.ts.map

package/dist/mem-block-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-block-store.d.ts","names":[],"sources":["../src/mem-block-store.ts"],"mappings":";;;iBAIiB;EACf;EACA;;;;;;;;iBASe,eAAe;;EAE9B,IAAI,aAAa,OAAO;;EAExB,OAAO;;;;;;EAMP,OAAO,eAAe,gBAAgB;;;;;EAKtC,eAAe;;;iBAIA,YAAY;EAC3B,OAAO;;EAEP;;iBAGe,qBAAqB;;EAEpC;EACA,QAAQ,eAAe;;;qBAIZ;;;;;;;;;;;;;;qBAeA,cAAc,UAAU;mBAClB;UACT;UACA;UACA;UACA;EAER,YAAY,SAAS,qBAAqB;;MAMtC,UAAU,eAAe;;EAK7B;UAIQ;;EAMR,IAAI,cAAc;;;;;EAQlB,IAAI,gBAAgB;;;;;;EAgBd,OAAO,WAAW,iBAAiB,cAAc,gBAAgB;UAe/D;;EAQR;UAMS;;EAOR,OAAO,aAAa,eAAe,SAAS;;EAQ7C,MAAM,aAAa;;EAQlB,MAAM,aAAa,eAAe,SAAS;;;;;;;EAe5C,OACE,4BACA,cACA,eAAe,iBACd,YAAY;;;;;;EAwBf;;;;;;;EAkBA,QACE,cACA,gBAAgB,OAAO,4BAA4B,eAAe"}

package/dist/mem-block-store.js

@@ -0,0 +1,173 @@
+import { compositeKey, matchesPrefix, toAsyncIterable } from "@statewalker/indexer-core";
+//#region src/mem-block-store.ts
+/** The persistence format version written by {@link MemBlockStore.serialize}. */
+const MEM_BLOCK_STORE_VERSION = 3;
+/**
+* In-memory block store shared by the in-memory search engines.
+*
+* Owns the blocks keyed by `path` + `blockId`, path-prefix selection,
+* enumeration, multi-query score fusion, dirty tracking and the persistence
+* framing; the {@link MemBlockEngine} supplies only add, remove and search.
+*
+* Dirty tracking: the store starts dirty, becomes clean after
+* {@link serialize} or {@link restore}, and becomes dirty again only on a
+* change. Empty batches and deletes that remove nothing are not changes.
+* While clean, `serialize` returns the cached output without asking the
+* engine to export again.
+*/
+var MemBlockStore = class {
+	label;
+	engineRef;
+	blocks = /* @__PURE__ */ new Map();
+	dirty = true;
+	cached;
+	constructor(options) {
+		this.label = options.label;
+		this.engineRef = options.engine;
+	}
+	/** The engine currently indexing the blocks (replaced by {@link restore}). */
+	get engine() {
+		return this.engineRef;
+	}
+	/** `true` when the blocks changed since the last serialize or restore. */
+	isDirty() {
+		return this.dirty;
+	}
+	markDirty() {
+		this.dirty = true;
+		this.cached = void 0;
+	}
+	/** The block under `key`, if any. */
+	get(key) {
+		return this.blocks.get(key);
+	}
+	/**
+	* Add or replace a batch of blocks. An empty batch changes nothing and
+	* leaves the store clean.
+	*/
+	put(batch) {
+		if (batch.length === 0) return;
+		for (const block of batch) {
+			const key = compositeKey(block.path, block.blockId);
+			if (this.blocks.has(key)) this.engineRef.remove(key);
+			this.engineRef.add(key, block);
+			this.blocks.set(key, block);
+		}
+		this.markDirty();
+	}
+	/**
+	* Delete the blocks the selectors match: one block when `blockId` is set,
+	* otherwise every block under the path prefix. Returns how many blocks were
+	* removed; the store becomes dirty only when that is more than zero.
+	*/
+	async delete(selectors) {
+		let removed = 0;
+		for await (const sel of toAsyncIterable(selectors)) if (sel.blockId !== void 0) {
+			if (this.removeKey(compositeKey(sel.path, sel.blockId))) removed += 1;
+		} else for (const key of [...this.keys(sel.path)]) if (this.removeKey(key)) removed += 1;
+		if (removed > 0) this.markDirty();
+		return removed;
+	}
+	removeKey(key) {
+		if (!this.blocks.has(key)) return false;
+		this.engineRef.remove(key);
+		this.blocks.delete(key);
+		return true;
+	}
+	/** Drop every block, from the store and the engine. */
+	clear() {
+		for (const key of this.blocks.keys()) this.engineRef.remove(key);
+		this.blocks.clear();
+		this.markDirty();
+	}
+	*keys(pathPrefix) {
+		for (const [key, block] of this.blocks) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield key;
+	}
+	/** The blocks under `pathPrefix` (all blocks without one), in insertion order. */
+	*select(pathPrefix) {
+		for (const key of this.keys(pathPrefix)) {
+			const block = this.blocks.get(key);
+			if (block) yield block;
+		}
+	}
+	/** How many blocks lie under `pathPrefix` (all blocks without one). */
+	count(pathPrefix) {
+		if (pathPrefix === void 0) return this.blocks.size;
+		let n = 0;
+		for (const _ of this.keys(pathPrefix)) n += 1;
+		return n;
+	}
+	/** The distinct document paths under `pathPrefix`, in first-seen order. */
+	*paths(pathPrefix) {
+		const seen = /* @__PURE__ */ new Set();
+		for (const block of this.select(pathPrefix)) {
+			if (seen.has(block.path)) continue;
+			seen.add(block.path);
+			yield block.path;
+		}
+	}
+	/**
+	* Multi-query search with score fusion. Each query is ranked by the engine,
+	* filtered to `pathPrefixes` (when given) and cut to `topK`; the per-block
+	* scores are then **summed** across queries, so a block matching more
+	* queries ranks higher, and the fused list is cut to `topK` again.
+	*/
+	search(queries, topK, pathPrefixes) {
+		const totals = /* @__PURE__ */ new Map();
+		const scoped = pathPrefixes !== void 0 && pathPrefixes.length > 0;
+		for (const query of queries) {
+			let taken = 0;
+			for (const { key, score } of this.engineRef.search(query, topK)) {
+				if (taken >= topK) break;
+				const block = this.blocks.get(key);
+				if (!block) continue;
+				if (scoped && !pathPrefixes.some((p) => matchesPrefix(block.path, p))) continue;
+				taken += 1;
+				const hit = totals.get(key);
+				if (hit) hit.score += score;
+				else totals.set(key, {
+					block,
+					score
+				});
+			}
+		}
+		return [...totals.values()].sort((a, b) => b.score - a.score).slice(0, topK);
+	}
+	/**
+	* Serialize the blocks and the engine state into one JSON document,
+	* `{ version: 3, ...engineState, blocks }`. While the store is clean the
+	* previous output is returned as is.
+	*/
+	serialize() {
+		if (!this.dirty && this.cached !== void 0) return this.cached;
+		const out = JSON.stringify({
+			version: 3,
+			...this.engineRef.exportState(),
+			blocks: [...this.blocks.values()]
+		});
+		this.cached = out;
+		this.dirty = false;
+		return out;
+	}
+	/**
+	* Replace the store's content with a document written by {@link serialize}.
+	* `restoreEngine` rebuilds the engine from the document's fields. Throws on
+	* a document of another version, leaving the store unchanged. The store is
+	* clean afterwards and re-serializes to `json` until it changes.
+	*/
+	restore(json, restoreEngine) {
+		const parsed = JSON.parse(json);
+		if (parsed.version !== 3) throw new Error(`${this.label}: unsupported serialised version ${String(parsed.version)} (expected 3)`);
+		const engine = restoreEngine(parsed);
+		const blocks = /* @__PURE__ */ new Map();
+		for (const block of parsed.blocks ?? []) blocks.set(compositeKey(block.path, block.blockId), block);
+		this.engineRef = engine;
+		this.blocks = blocks;
+		this.cached = json;
+		this.dirty = false;
+	}
+};
+//#endregion
+export { MEM_BLOCK_STORE_VERSION, MemBlockStore };
+
+//# sourceMappingURL=mem-block-store.js.map

package/dist/mem-block-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-block-store.js","names":[],"sources":["../src/mem-block-store.ts"],"sourcesContent":["import type { BlockReference, DocumentPath, PathSelector } from \"@statewalker/indexer-api\";\nimport { compositeKey, matchesPrefix, toAsyncIterable } from \"@statewalker/indexer-core\";\n\n/** One engine hit: the store key of a block and the engine's native score. */\nexport interface RankedKey {\n  key: string;\n  score: number;\n}\n\n/**\n * The engine side of a {@link MemBlockStore}: what one search engine\n * (FlexSearch, MiniSearch, …) supplies. The store owns the blocks and calls\n * the engine only to index, unindex and rank them, addressed by the store's\n * key for a block.\n */\nexport interface MemBlockEngine<B> {\n  /** Index `block` under `key`. A previous version is always removed first. */\n  add(key: string, block: B): void;\n  /** Unindex the block under `key`. Called only for keys the store holds. */\n  remove(key: string): void;\n  /**\n   * Rank the indexed blocks for one query, best first. `limit` is the depth\n   * the caller wants after path filtering; an engine may return more.\n   * Keys the store no longer holds are ignored.\n   */\n  search(query: string, limit: number): RankedKey[];\n  /**\n   * The engine's own state as JSON-safe fields, framed with the blocks by\n   * {@link MemBlockStore.serialize}. Must not use the names `version` or `blocks`.\n   */\n  exportState(): Record<string, unknown>;\n}\n\n/** A fused hit from {@link MemBlockStore.search}. */\nexport interface MemBlockHit<B> {\n  block: B;\n  /** The sum of the block's scores over every query it matched. */\n  score: number;\n}\n\nexport interface MemBlockStoreOptions<B> {\n  /** Named in errors, e.g. the class of the index that owns the store. */\n  label: string;\n  engine: MemBlockEngine<B>;\n}\n\n/** The persistence format version written by {@link MemBlockStore.serialize}. */\nexport const MEM_BLOCK_STORE_VERSION = 3;\n\n/**\n * In-memory block store shared by the in-memory search engines.\n *\n * Owns the blocks keyed by `path` + `blockId`, path-prefix selection,\n * enumeration, multi-query score fusion, dirty tracking and the persistence\n * framing; the {@link MemBlockEngine} supplies only add, remove and search.\n *\n * Dirty tracking: the store starts dirty, becomes clean after\n * {@link serialize} or {@link restore}, and becomes dirty again only on a\n * change. Empty batches and deletes that remove nothing are not changes.\n * While clean, `serialize` returns the cached output without asking the\n * engine to export again.\n */\nexport class MemBlockStore<B extends BlockReference> {\n  private readonly label: string;\n  private engineRef: MemBlockEngine<B>;\n  private blocks = new Map<string, B>();\n  private dirty = true;\n  private cached?: string;\n\n  constructor(options: MemBlockStoreOptions<B>) {\n    this.label = options.label;\n    this.engineRef = options.engine;\n  }\n\n  /** The engine currently indexing the blocks (replaced by {@link restore}). */\n  get engine(): MemBlockEngine<B> {\n    return this.engineRef;\n  }\n\n  /** `true` when the blocks changed since the last serialize or restore. */\n  isDirty(): boolean {\n    return this.dirty;\n  }\n\n  private markDirty(): void {\n    this.dirty = true;\n    this.cached = undefined;\n  }\n\n  /** The block under `key`, if any. */\n  get(key: string): B | undefined {\n    return this.blocks.get(key);\n  }\n\n  /**\n   * Add or replace a batch of blocks. An empty batch changes nothing and\n   * leaves the store clean.\n   */\n  put(batch: readonly B[]): void {\n    if (batch.length === 0) return;\n    for (const block of batch) {\n      const key = compositeKey(block.path, block.blockId);\n      if (this.blocks.has(key)) this.engineRef.remove(key);\n      this.engineRef.add(key, block);\n      this.blocks.set(key, block);\n    }\n    this.markDirty();\n  }\n\n  /**\n   * Delete the blocks the selectors match: one block when `blockId` is set,\n   * otherwise every block under the path prefix. Returns how many blocks were\n   * removed; the store becomes dirty only when that is more than zero.\n   */\n  async delete(selectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<number> {\n    let removed = 0;\n    for await (const sel of toAsyncIterable(selectors)) {\n      if (sel.blockId !== undefined) {\n        if (this.removeKey(compositeKey(sel.path, sel.blockId))) removed += 1;\n      } else {\n        for (const key of [...this.keys(sel.path)]) {\n          if (this.removeKey(key)) removed += 1;\n        }\n      }\n    }\n    if (removed > 0) this.markDirty();\n    return removed;\n  }\n\n  private removeKey(key: string): boolean {\n    if (!this.blocks.has(key)) return false;\n    this.engineRef.remove(key);\n    this.blocks.delete(key);\n    return true;\n  }\n\n  /** Drop every block, from the store and the engine. */\n  clear(): void {\n    for (const key of this.blocks.keys()) this.engineRef.remove(key);\n    this.blocks.clear();\n    this.markDirty();\n  }\n\n  private *keys(pathPrefix?: DocumentPath): Iterable<string> {\n    for (const [key, block] of this.blocks) {\n      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) yield key;\n    }\n  }\n\n  /** The blocks under `pathPrefix` (all blocks without one), in insertion order. */\n  *select(pathPrefix?: DocumentPath): Iterable<B> {\n    for (const key of this.keys(pathPrefix)) {\n      const block = this.blocks.get(key);\n      if (block) yield block;\n    }\n  }\n\n  /** How many blocks lie under `pathPrefix` (all blocks without one). */\n  count(pathPrefix?: DocumentPath): number {\n    if (pathPrefix === undefined) return this.blocks.size;\n    let n = 0;\n    for (const _ of this.keys(pathPrefix)) n += 1;\n    return n;\n  }\n\n  /** The distinct document paths under `pathPrefix`, in first-seen order. */\n  *paths(pathPrefix?: DocumentPath): Iterable<DocumentPath> {\n    const seen = new Set<DocumentPath>();\n    for (const block of this.select(pathPrefix)) {\n      if (seen.has(block.path)) continue;\n      seen.add(block.path);\n      yield block.path;\n    }\n  }\n\n  /**\n   * Multi-query search with score fusion. Each query is ranked by the engine,\n   * filtered to `pathPrefixes` (when given) and cut to `topK`; the per-block\n   * scores are then **summed** across queries, so a block matching more\n   * queries ranks higher, and the fused list is cut to `topK` again.\n   */\n  search(\n    queries: readonly string[],\n    topK: number,\n    pathPrefixes?: DocumentPath[],\n  ): MemBlockHit<B>[] {\n    const totals = new Map<string, MemBlockHit<B>>();\n    const scoped = pathPrefixes !== undefined && pathPrefixes.length > 0;\n    for (const query of queries) {\n      let taken = 0;\n      for (const { key, score } of this.engineRef.search(query, topK)) {\n        if (taken >= topK) break;\n        const block = this.blocks.get(key);\n        if (!block) continue;\n        if (scoped && !pathPrefixes.some((p) => matchesPrefix(block.path, p))) continue;\n        taken += 1;\n        const hit = totals.get(key);\n        if (hit) hit.score += score;\n        else totals.set(key, { block, score });\n      }\n    }\n    return [...totals.values()].sort((a, b) => b.score - a.score).slice(0, topK);\n  }\n\n  /**\n   * Serialize the blocks and the engine state into one JSON document,\n   * `{ version: 3, ...engineState, blocks }`. While the store is clean the\n   * previous output is returned as is.\n   */\n  serialize(): string {\n    if (!this.dirty && this.cached !== undefined) return this.cached;\n    const out = JSON.stringify({\n      version: MEM_BLOCK_STORE_VERSION,\n      ...this.engineRef.exportState(),\n      blocks: [...this.blocks.values()],\n    });\n    this.cached = out;\n    this.dirty = false;\n    return out;\n  }\n\n  /**\n   * Replace the store's content with a document written by {@link serialize}.\n   * `restoreEngine` rebuilds the engine from the document's fields. Throws on\n   * a document of another version, leaving the store unchanged. The store is\n   * clean afterwards and re-serializes to `json` until it changes.\n   */\n  restore(\n    json: string,\n    restoreEngine: (state: Record<string, unknown>) => MemBlockEngine<B>,\n  ): void {\n    const parsed = JSON.parse(json) as { version?: unknown; blocks?: B[] } & Record<\n      string,\n      unknown\n    >;\n    if (parsed.version !== MEM_BLOCK_STORE_VERSION) {\n      throw new Error(\n        `${this.label}: unsupported serialised version ${String(parsed.version)} (expected ${MEM_BLOCK_STORE_VERSION})`,\n      );\n    }\n    const engine = restoreEngine(parsed);\n    const blocks = new Map<string, B>();\n    for (const block of parsed.blocks ?? []) {\n      blocks.set(compositeKey(block.path, block.blockId), block);\n    }\n    this.engineRef = engine;\n    this.blocks = blocks;\n    this.cached = json;\n    this.dirty = false;\n  }\n}\n"],"mappings":";;;AA+CA,MAAa,0BAA0B;;;;;;;;;;;;;;AAevC,IAAa,gBAAb,MAAqD;CACnD;CACA;CACA,yBAAiB,IAAI,IAAe;CACpC,QAAgB;CAChB;CAEA,YAAY,SAAkC;EAC5C,KAAK,QAAQ,QAAQ;EACrB,KAAK,YAAY,QAAQ;CAC3B;;CAGA,IAAI,SAA4B;EAC9B,OAAO,KAAK;CACd;;CAGA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,YAA0B;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS,KAAA;CAChB;;CAGA,IAAI,KAA4B;EAC9B,OAAO,KAAK,OAAO,IAAI,GAAG;CAC5B;;;;;CAMA,IAAI,OAA2B;EAC7B,IAAI,MAAM,WAAW,GAAG;EACxB,KAAK,MAAM,SAAS,OAAO;GACzB,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,IAAI,KAAK,OAAO,IAAI,GAAG,GAAG,KAAK,UAAU,OAAO,GAAG;GACnD,KAAK,UAAU,IAAI,KAAK,KAAK;GAC7B,KAAK,OAAO,IAAI,KAAK,KAAK;EAC5B;EACA,KAAK,UAAU;CACjB;;;;;;CAOA,MAAM,OAAO,WAA0E;EACrF,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,SAAS,GAC/C,IAAI,IAAI,YAAY,KAAA,GACd;OAAA,KAAK,UAAU,aAAa,IAAI,MAAM,IAAI,OAAO,CAAC,GAAG,WAAW;EAAA,OAEpE,KAAK,MAAM,OAAO,CAAC,GAAG,KAAK,KAAK,IAAI,IAAI,CAAC,GACvC,IAAI,KAAK,UAAU,GAAG,GAAG,WAAW;EAI1C,IAAI,UAAU,GAAG,KAAK,UAAU;EAChC,OAAO;CACT;CAEA,UAAkB,KAAsB;EACtC,IAAI,CAAC,KAAK,OAAO,IAAI,GAAG,GAAG,OAAO;EAClC,KAAK,UAAU,OAAO,GAAG;EACzB,KAAK,OAAO,OAAO,GAAG;EACtB,OAAO;CACT;;CAGA,QAAc;EACZ,KAAK,MAAM,OAAO,KAAK,OAAO,KAAK,GAAG,KAAK,UAAU,OAAO,GAAG;EAC/D,KAAK,OAAO,MAAM;EAClB,KAAK,UAAU;CACjB;CAEA,CAAS,KAAK,YAA6C;EACzD,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAAG,MAAM;CAEjF;;CAGA,CAAC,OAAO,YAAwC;EAC9C,KAAK,MAAM,OAAO,KAAK,KAAK,UAAU,GAAG;GACvC,MAAM,QAAQ,KAAK,OAAO,IAAI,GAAG;GACjC,IAAI,OAAO,MAAM;EACnB;CACF;;CAGA,MAAM,YAAmC;EACvC,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,IAAI;EACR,KAAK,MAAM,KAAK,KAAK,KAAK,UAAU,GAAG,KAAK;EAC5C,OAAO;CACT;;CAGA,CAAC,MAAM,YAAmD;EACxD,MAAM,uBAAO,IAAI,IAAkB;EACnC,KAAK,MAAM,SAAS,KAAK,OAAO,UAAU,GAAG;GAC3C,IAAI,KAAK,IAAI,MAAM,IAAI,GAAG;GAC1B,KAAK,IAAI,MAAM,IAAI;GACnB,MAAM,MAAM;EACd;CACF;;;;;;;CAQA,OACE,SACA,MACA,cACkB;EAClB,MAAM,yBAAS,IAAI,IAA4B;EAC/C,MAAM,SAAS,iBAAiB,KAAA,KAAa,aAAa,SAAS;EACnE,KAAK,MAAM,SAAS,SAAS;GAC3B,IAAI,QAAQ;GACZ,KAAK,MAAM,EAAE,KAAK,WAAW,KAAK,UAAU,OAAO,OAAO,IAAI,GAAG;IAC/D,IAAI,SAAS,MAAM;IACnB,MAAM,QAAQ,KAAK,OAAO,IAAI,GAAG;IACjC,IAAI,CAAC,OAAO;IACZ,IAAI,UAAU,CAAC,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC,GAAG;IACvE,SAAS;IACT,MAAM,MAAM,OAAO,IAAI,GAAG;IAC1B,IAAI,KAAK,IAAI,SAAS;SACjB,OAAO,IAAI,KAAK;KAAE;KAAO;IAAM,CAAC;GACvC;EACF;EACA,OAAO,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,MAAM,GAAG,IAAI;CAC7E;;;;;;CAOA,YAAoB;EAClB,IAAI,CAAC,KAAK,SAAS,KAAK,WAAW,KAAA,GAAW,OAAO,KAAK;EAC1D,MAAM,MAAM,KAAK,UAAU;GACzB,SAAA;GACA,GAAG,KAAK,UAAU,YAAY;GAC9B,QAAQ,CAAC,GAAG,KAAK,OAAO,OAAO,CAAC;EAClC,CAAC;EACD,KAAK,SAAS;EACd,KAAK,QAAQ;EACb,OAAO;CACT;;;;;;;CAQA,QACE,MACA,eACM;EACN,MAAM,SAAS,KAAK,MAAM,IAAI;EAI9B,IAAI,OAAO,YAAA,GACT,MAAM,IAAI,MACR,GAAG,KAAK,MAAM,mCAAmC,OAAO,OAAO,OAAO,EAAE,cAC1E;EAEF,MAAM,SAAS,cAAc,MAAM;EACnC,MAAM,yBAAS,IAAI,IAAe;EAClC,KAAK,MAAM,SAAS,OAAO,UAAU,CAAC,GACpC,OAAO,IAAI,aAAa,MAAM,MAAM,MAAM,OAAO,GAAG,KAAK;EAE3D,KAAK,YAAY;EACjB,KAAK,SAAS;EACd,KAAK,SAAS;EACd,KAAK,QAAQ;CACf;AACF"}

package/dist/mem-full-text-index.d.ts

@@ -0,0 +1,58 @@
+import { MemBlockEngine, MemBlockStore } from "./mem-block-store.js";
+import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
+import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
+//#region src/mem-full-text-index.d.ts
+/** The engine of an in-memory full-text index: a {@link MemBlockEngine} over text blocks. */
+export type MemFullTextEngine = MemBlockEngine<FullTextBlock>;
+/**
+ * An in-memory {@link FullTextIndex} over a {@link MemBlockStore}. The store
+ * does everything but text indexing; a subclass supplies only the engine
+ * adapter (FlexSearch, MiniSearch, …) and how to rebuild it from saved state.
+ *
+ * Persistence: one entry named `json`, holding the store's framed document.
+ */
+export declare abstract class MemFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
+  protected readonly info: FullTextIndexInfo;
+  protected readonly store: MemBlockStore<FullTextBlock>;
+  private closed;
+  /**
+   * @param label  named in errors (usually the subclass name)
+   * @param engine the engine adapter for a fresh, empty index
+   */
+  protected constructor(info: FullTextIndexInfo, label: string, engine: MemFullTextEngine);
+  /** Rebuild the engine adapter from the fields it exported. */
+  protected abstract restoreEngine(state: Record<string, unknown>): MemFullTextEngine;
+  /**
+   * `true` when the in-memory state has changed since the last serialize or
+   * load. Callers (e.g. the persistence layer) may use it to skip writes.
+   */
+  isDirty(): boolean;
+  protected ensureOpen(): void;
+  /** The framed JSON document; cached while the index is unchanged. */
+  protected serializeState(): string;
+  /** Replace the content with a document from {@link serializeState}. */
+  protected restoreState(json: string): void;
+  getIndexInfo(): Promise<FullTextIndexInfo>;
+  search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
+  addDocument(blocks: FullTextBlock[]): Promise<void>;
+  addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
+  deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
+  getSize(pathPrefix?: DocumentPath): Promise<number>;
+  getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
+  getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
+  getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
+  close(_options?: {
+    force?: boolean;
+  }): Promise<void>;
+  flush(): Promise<void>;
+  deleteIndex(): Promise<void>;
+  /**
+   * Stream the sub-index's state as a single entry named `json`. The composite
+   * prefixes it with `${indexName}/${subName}/` before it reaches persistence.
+   */
+  serialise(): AsyncIterable<PersistenceEntry>;
+  /** Restore the sub-index from a previously produced entry set. */
+  loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
+}
+//#endregion
+//# sourceMappingURL=mem-full-text-index.d.ts.map

package/dist/mem-full-text-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-full-text-index.d.ts","names":[],"sources":["../src/mem-full-text-index.ts"],"mappings":";;;;;YAkBY,oBAAoB,eAAe;;;;;;;;8BASzB,4BACT,eAAe,uBAAuB,eAAe,eAAe;qBAE5D,MAAM;qBACN,OAAO,cAAc;UAChC;;;;;YAMC,YAAa,MAAM,mBAAmB,eAAe,QAAQ;;qBAMnD,cAAc,OAAO,0BAA0B;;;;;EAMlE;YAIU;;YAKA;;YAKA,aAAa;EAIjB,gBAAgB,QAAQ;EAKvB,OAAO,QAAQ,gBAAgB,eAAe;EAS/C,YAAY,QAAQ,kBAAkB;EAYtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;EAKG,QAAQ,aAAa,eAAe;EAKnC,iBAAiB,aAAa,eAAe,eAAe;EAK5D,sBAAsB,aAAa,eAAe,eAAe;EAOjE,mBAAmB,aAAa,eAAe,eAAe;EAO/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;EAYd,aAAa,cAAc;;EAK5B,SAAS,SAAS,SAAS,oBAAoB"}

package/dist/mem-full-text-index.js

@@ -0,0 +1,130 @@
+import { MemBlockStore } from "./mem-block-store.js";
+import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core";
+//#region src/mem-full-text-index.ts
+/**
+* An in-memory {@link FullTextIndex} over a {@link MemBlockStore}. The store
+* does everything but text indexing; a subclass supplies only the engine
+* adapter (FlexSearch, MiniSearch, …) and how to rebuild it from saved state.
+*
+* Persistence: one entry named `json`, holding the store's framed document.
+*/
+var MemFullTextIndex = class {
+	info;
+	store;
+	closed = false;
+	/**
+	* @param label  named in errors (usually the subclass name)
+	* @param engine the engine adapter for a fresh, empty index
+	*/
+	constructor(info, label, engine) {
+		this.info = info;
+		this.store = new MemBlockStore({
+			label,
+			engine
+		});
+	}
+	/**
+	* `true` when the in-memory state has changed since the last serialize or
+	* load. Callers (e.g. the persistence layer) may use it to skip writes.
+	*/
+	isDirty() {
+		return this.store.isDirty();
+	}
+	ensureOpen() {
+		if (this.closed) throw new Error("FullTextIndex is closed");
+	}
+	/** The framed JSON document; cached while the index is unchanged. */
+	serializeState() {
+		return this.store.serialize();
+	}
+	/** Replace the content with a document from {@link serializeState}. */
+	restoreState(json) {
+		this.store.restore(json, (state) => this.restoreEngine(state));
+	}
+	async getIndexInfo() {
+		this.ensureOpen();
+		return { ...this.info };
+	}
+	async *search(params) {
+		this.ensureOpen();
+		const { queries, paths } = params;
+		if (!queries || queries.length === 0) return;
+		for (const { block, score } of this.store.search(queries, params.topK ?? 100, paths)) yield {
+			path: block.path,
+			blockId: block.blockId,
+			snippet: block.content,
+			score
+		};
+	}
+	async addDocument(blocks) {
+		this.ensureOpen();
+		this.store.put(blocks.map((b) => ({
+			path: b.path,
+			blockId: b.blockId,
+			content: b.content,
+			metadata: b.metadata
+		})));
+	}
+	async addDocuments(blocks) {
+		this.ensureOpen();
+		for await (const batch of blocks) await this.addDocument(batch);
+	}
+	async deleteDocuments(pathSelectors) {
+		this.ensureOpen();
+		await this.store.delete(pathSelectors);
+	}
+	async getSize(pathPrefix) {
+		this.ensureOpen();
+		return this.store.count(pathPrefix);
+	}
+	async *getDocumentPaths(pathPrefix) {
+		this.ensureOpen();
+		yield* this.store.paths(pathPrefix);
+	}
+	async *getDocumentBlocksRefs(pathPrefix) {
+		this.ensureOpen();
+		for (const block of this.store.select(pathPrefix)) yield {
+			path: block.path,
+			blockId: block.blockId
+		};
+	}
+	async *getDocumentsBlocks(pathPrefix) {
+		this.ensureOpen();
+		for (const block of this.store.select(pathPrefix)) yield { ...block };
+	}
+	async close(_options) {
+		this.closed = true;
+	}
+	async flush() {
+		this.ensureOpen();
+	}
+	async deleteIndex() {
+		this.ensureOpen();
+		this.store.clear();
+		this.closed = true;
+	}
+	/**
+	* Stream the sub-index's state as a single entry named `json`. The composite
+	* prefixes it with `${indexName}/${subName}/` before it reaches persistence.
+	*/
+	async *serialise() {
+		yield {
+			name: "json",
+			content: singleChunk(toBytes(this.serializeState()))
+		};
+	}
+	/** Restore the sub-index from a previously produced entry set. */
+	async loadFrom(entries) {
+		this.ensureOpen();
+		for (const e of entries) {
+			if (e.name !== "json") continue;
+			const bytes = await readEntryBytes(e);
+			this.restoreState(new TextDecoder().decode(bytes));
+			return;
+		}
+	}
+};
+//#endregion
+export { MemFullTextIndex };
+
+//# sourceMappingURL=mem-full-text-index.js.map

package/dist/mem-full-text-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mem-full-text-index.js","names":[],"sources":["../src/mem-full-text-index.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  PathSelector,\n  PersistableSearchIndex,\n  PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport { readEntryBytes, singleChunk, toBytes } from \"@statewalker/indexer-core\";\nimport type {\n  FullTextBlock,\n  FullTextIndex,\n  FullTextIndexInfo,\n  FulltextQuery,\n  FulltextResult,\n} from \"@statewalker/indexer-fulltext\";\nimport { type MemBlockEngine, MemBlockStore } from \"./mem-block-store.js\";\n\n/** The engine of an in-memory full-text index: a {@link MemBlockEngine} over text blocks. */\nexport type MemFullTextEngine = MemBlockEngine<FullTextBlock>;\n\n/**\n * An in-memory {@link FullTextIndex} over a {@link MemBlockStore}. The store\n * does everything but text indexing; a subclass supplies only the engine\n * adapter (FlexSearch, MiniSearch, …) and how to rebuild it from saved state.\n *\n * Persistence: one entry named `json`, holding the store's framed document.\n */\nexport abstract class MemFullTextIndex\n  implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult>\n{\n  protected readonly info: FullTextIndexInfo;\n  protected readonly store: MemBlockStore<FullTextBlock>;\n  private closed = false;\n\n  /**\n   * @param label  named in errors (usually the subclass name)\n   * @param engine the engine adapter for a fresh, empty index\n   */\n  protected constructor(info: FullTextIndexInfo, label: string, engine: MemFullTextEngine) {\n    this.info = info;\n    this.store = new MemBlockStore<FullTextBlock>({ label, engine });\n  }\n\n  /** Rebuild the engine adapter from the fields it exported. */\n  protected abstract restoreEngine(state: Record<string, unknown>): MemFullTextEngine;\n\n  /**\n   * `true` when the in-memory state has changed since the last serialize or\n   * load. Callers (e.g. the persistence layer) may use it to skip writes.\n   */\n  isDirty(): boolean {\n    return this.store.isDirty();\n  }\n\n  protected ensureOpen(): void {\n    if (this.closed) throw new Error(\"FullTextIndex is closed\");\n  }\n\n  /** The framed JSON document; cached while the index is unchanged. */\n  protected serializeState(): string {\n    return this.store.serialize();\n  }\n\n  /** Replace the content with a document from {@link serializeState}. */\n  protected restoreState(json: string): void {\n    this.store.restore(json, (state) => this.restoreEngine(state));\n  }\n\n  async getIndexInfo(): Promise<FullTextIndexInfo> {\n    this.ensureOpen();\n    return { ...this.info };\n  }\n\n  async *search(params: FulltextQuery): AsyncGenerator<FulltextResult> {\n    this.ensureOpen();\n    const { queries, paths } = params;\n    if (!queries || queries.length === 0) return;\n    for (const { block, score } of this.store.search(queries, params.topK ?? 100, paths)) {\n      yield { path: block.path, blockId: block.blockId, snippet: block.content, score };\n    }\n  }\n\n  async addDocument(blocks: FullTextBlock[]): Promise<void> {\n    this.ensureOpen();\n    this.store.put(\n      blocks.map((b) => ({\n        path: b.path,\n        blockId: b.blockId,\n        content: b.content,\n        metadata: b.metadata,\n      })),\n    );\n  }\n\n  async addDocuments(\n    blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n  ): Promise<void> {\n    this.ensureOpen();\n    for await (const batch of blocks) {\n      await this.addDocument(batch);\n    }\n  }\n\n  async deleteDocuments(\n    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n  ): Promise<void> {\n    this.ensureOpen();\n    await this.store.delete(pathSelectors);\n  }\n\n  async getSize(pathPrefix?: DocumentPath): Promise<number> {\n    this.ensureOpen();\n    return this.store.count(pathPrefix);\n  }\n\n  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n    this.ensureOpen();\n    yield* this.store.paths(pathPrefix);\n  }\n\n  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n    this.ensureOpen();\n    for (const block of this.store.select(pathPrefix)) {\n      yield { path: block.path, blockId: block.blockId };\n    }\n  }\n\n  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n    this.ensureOpen();\n    for (const block of this.store.select(pathPrefix)) {\n      yield { ...block };\n    }\n  }\n\n  async close(_options?: { force?: boolean }): Promise<void> {\n    this.closed = true;\n  }\n\n  async flush(): Promise<void> {\n    this.ensureOpen();\n    // no-op for in-memory\n  }\n\n  async deleteIndex(): Promise<void> {\n    this.ensureOpen();\n    this.store.clear();\n    this.closed = true;\n  }\n\n  // --- PersistableSearchIndex ------------------------------------------------\n\n  /**\n   * Stream the sub-index's state as a single entry named `json`. The composite\n   * prefixes it with `${indexName}/${subName}/` before it reaches persistence.\n   */\n  async *serialise(): AsyncIterable<PersistenceEntry> {\n    yield { name: \"json\", content: singleChunk(toBytes(this.serializeState())) };\n  }\n\n  /** Restore the sub-index from a previously produced entry set. */\n  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n    this.ensureOpen();\n    for (const e of entries) {\n      if (e.name !== \"json\") continue;\n      const bytes = await readEntryBytes(e);\n      this.restoreState(new TextDecoder().decode(bytes));\n      return;\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;AA2BA,IAAsB,mBAAtB,MAEA;CACE;CACA;CACA,SAAiB;;;;;CAMjB,YAAsB,MAAyB,OAAe,QAA2B;EACvF,KAAK,OAAO;EACZ,KAAK,QAAQ,IAAI,cAA6B;GAAE;GAAO;EAAO,CAAC;CACjE;;;;;CASA,UAAmB;EACjB,OAAO,KAAK,MAAM,QAAQ;CAC5B;CAEA,aAA6B;EAC3B,IAAI,KAAK,QAAQ,MAAM,IAAI,MAAM,yBAAyB;CAC5D;;CAGA,iBAAmC;EACjC,OAAO,KAAK,MAAM,UAAU;CAC9B;;CAGA,aAAuB,MAAoB;EACzC,KAAK,MAAM,QAAQ,OAAO,UAAU,KAAK,cAAc,KAAK,CAAC;CAC/D;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAuD;EACnE,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EACtC,KAAK,MAAM,EAAE,OAAO,WAAW,KAAK,MAAM,OAAO,SAAS,OAAO,QAAQ,KAAK,KAAK,GACjF,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;GAAS,SAAS,MAAM;GAAS;EAAM;CAEpF;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,KAAK,MAAM,IACT,OAAO,KAAK,OAAO;GACjB,MAAM,EAAE;GACR,SAAS,EAAE;GACX,SAAS,EAAE;GACX,UAAU,EAAE;EACd,EAAE,CACJ;CACF;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,MAAM,KAAK,MAAM,OAAO,aAAa;CACvC;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,OAAO,KAAK,MAAM,MAAM,UAAU;CACpC;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,OAAO,KAAK,MAAM,MAAM,UAAU;CACpC;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,MAAM,OAAO,UAAU,GAC9C,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAErD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,MAAM,OAAO,UAAU,GAC9C,MAAM,EAAE,GAAG,MAAM;CAErB;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,MAAM,MAAM;EACjB,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,KAAK,eAAe,CAAC,CAAC;EAAE;CAC7E;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,KAAK,aAAa,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK,CAAC;GACjD;EACF;CACF;AACF"}

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@statewalker/indexer-mem",
-  "version": "0.1.5",
+  "version": "0.3.0",
   "private": false,
   "type": "module",
-  "description": "In-memory vector sub-index (MemVectorIndex, brute-force cosine, Arrow persistence) and its VectorProvider.",
+  "description": "In-memory sub-indexes: the block store and full-text base behind the FlexSearch and MiniSearch indexes, and the vector sub-index (MemVectorIndex, brute-force cosine, Arrow persistence) with its VectorProvider.",
   "bugs": {
     "url": "https://github.com/statewalker/statewalker-search/issues"
   },
@@ -32,9 +32,10 @@
   ],
   "dependencies": {
     "@uwdata/flechette": "^2.5.0",
-    "@statewalker/indexer-api": "^0.1.4",
-    "@statewalker/indexer-core": "^0.1.4",
-    "@statewalker/indexer-vector": "^0.1.5"
+    "@statewalker/indexer-api": "^0.1.5",
+    "@statewalker/indexer-core": "^0.2.0",
+    "@statewalker/indexer-fulltext": "^0.1.6",
+    "@statewalker/indexer-vector": "^0.1.6"
   },
   "devDependencies": {
     "rimraf": "^6.1.3",

package/src/index.ts

@@ -1,2 +1,11 @@
+export {
+  MEM_BLOCK_STORE_VERSION,
+  type MemBlockEngine,
+  type MemBlockHit,
+  MemBlockStore,
+  type MemBlockStoreOptions,
+  type RankedKey,
+} from "./mem-block-store.js";
+export { type MemFullTextEngine, MemFullTextIndex } from "./mem-full-text-index.js";
 export { MemVectorIndex } from "./mem-vector-index.js";
 export { memVectorProvider } from "./mem-vector-provider.js";

package/src/mem-block-store.ts

@@ -0,0 +1,251 @@
+import type { BlockReference, DocumentPath, PathSelector } from "@statewalker/indexer-api";
+import { compositeKey, matchesPrefix, toAsyncIterable } from "@statewalker/indexer-core";
+
+/** One engine hit: the store key of a block and the engine's native score. */
+export interface RankedKey {
+  key: string;
+  score: number;
+}
+
+/**
+ * The engine side of a {@link MemBlockStore}: what one search engine
+ * (FlexSearch, MiniSearch, …) supplies. The store owns the blocks and calls
+ * the engine only to index, unindex and rank them, addressed by the store's
+ * key for a block.
+ */
+export interface MemBlockEngine<B> {
+  /** Index `block` under `key`. A previous version is always removed first. */
+  add(key: string, block: B): void;
+  /** Unindex the block under `key`. Called only for keys the store holds. */
+  remove(key: string): void;
+  /**
+   * Rank the indexed blocks for one query, best first. `limit` is the depth
+   * the caller wants after path filtering; an engine may return more.
+   * Keys the store no longer holds are ignored.
+   */
+  search(query: string, limit: number): RankedKey[];
+  /**
+   * The engine's own state as JSON-safe fields, framed with the blocks by
+   * {@link MemBlockStore.serialize}. Must not use the names `version` or `blocks`.
+   */
+  exportState(): Record<string, unknown>;
+}
+
+/** A fused hit from {@link MemBlockStore.search}. */
+export interface MemBlockHit<B> {
+  block: B;
+  /** The sum of the block's scores over every query it matched. */
+  score: number;
+}
+
+export interface MemBlockStoreOptions<B> {
+  /** Named in errors, e.g. the class of the index that owns the store. */
+  label: string;
+  engine: MemBlockEngine<B>;
+}
+
+/** The persistence format version written by {@link MemBlockStore.serialize}. */
+export const MEM_BLOCK_STORE_VERSION = 3;
+
+/**
+ * In-memory block store shared by the in-memory search engines.
+ *
+ * Owns the blocks keyed by `path` + `blockId`, path-prefix selection,
+ * enumeration, multi-query score fusion, dirty tracking and the persistence
+ * framing; the {@link MemBlockEngine} supplies only add, remove and search.
+ *
+ * Dirty tracking: the store starts dirty, becomes clean after
+ * {@link serialize} or {@link restore}, and becomes dirty again only on a
+ * change. Empty batches and deletes that remove nothing are not changes.
+ * While clean, `serialize` returns the cached output without asking the
+ * engine to export again.
+ */
+export class MemBlockStore<B extends BlockReference> {
+  private readonly label: string;
+  private engineRef: MemBlockEngine<B>;
+  private blocks = new Map<string, B>();
+  private dirty = true;
+  private cached?: string;
+
+  constructor(options: MemBlockStoreOptions<B>) {
+    this.label = options.label;
+    this.engineRef = options.engine;
+  }
+
+  /** The engine currently indexing the blocks (replaced by {@link restore}). */
+  get engine(): MemBlockEngine<B> {
+    return this.engineRef;
+  }
+
+  /** `true` when the blocks changed since the last serialize or restore. */
+  isDirty(): boolean {
+    return this.dirty;
+  }
+
+  private markDirty(): void {
+    this.dirty = true;
+    this.cached = undefined;
+  }
+
+  /** The block under `key`, if any. */
+  get(key: string): B | undefined {
+    return this.blocks.get(key);
+  }
+
+  /**
+   * Add or replace a batch of blocks. An empty batch changes nothing and
+   * leaves the store clean.
+   */
+  put(batch: readonly B[]): void {
+    if (batch.length === 0) return;
+    for (const block of batch) {
+      const key = compositeKey(block.path, block.blockId);
+      if (this.blocks.has(key)) this.engineRef.remove(key);
+      this.engineRef.add(key, block);
+      this.blocks.set(key, block);
+    }
+    this.markDirty();
+  }
+
+  /**
+   * Delete the blocks the selectors match: one block when `blockId` is set,
+   * otherwise every block under the path prefix. Returns how many blocks were
+   * removed; the store becomes dirty only when that is more than zero.
+   */
+  async delete(selectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<number> {
+    let removed = 0;
+    for await (const sel of toAsyncIterable(selectors)) {
+      if (sel.blockId !== undefined) {
+        if (this.removeKey(compositeKey(sel.path, sel.blockId))) removed += 1;
+      } else {
+        for (const key of [...this.keys(sel.path)]) {
+          if (this.removeKey(key)) removed += 1;
+        }
+      }
+    }
+    if (removed > 0) this.markDirty();
+    return removed;
+  }
+
+  private removeKey(key: string): boolean {
+    if (!this.blocks.has(key)) return false;
+    this.engineRef.remove(key);
+    this.blocks.delete(key);
+    return true;
+  }
+
+  /** Drop every block, from the store and the engine. */
+  clear(): void {
+    for (const key of this.blocks.keys()) this.engineRef.remove(key);
+    this.blocks.clear();
+    this.markDirty();
+  }
+
+  private *keys(pathPrefix?: DocumentPath): Iterable<string> {
+    for (const [key, block] of this.blocks) {
+      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) yield key;
+    }
+  }
+
+  /** The blocks under `pathPrefix` (all blocks without one), in insertion order. */
+  *select(pathPrefix?: DocumentPath): Iterable<B> {
+    for (const key of this.keys(pathPrefix)) {
+      const block = this.blocks.get(key);
+      if (block) yield block;
+    }
+  }
+
+  /** How many blocks lie under `pathPrefix` (all blocks without one). */
+  count(pathPrefix?: DocumentPath): number {
+    if (pathPrefix === undefined) return this.blocks.size;
+    let n = 0;
+    for (const _ of this.keys(pathPrefix)) n += 1;
+    return n;
+  }
+
+  /** The distinct document paths under `pathPrefix`, in first-seen order. */
+  *paths(pathPrefix?: DocumentPath): Iterable<DocumentPath> {
+    const seen = new Set<DocumentPath>();
+    for (const block of this.select(pathPrefix)) {
+      if (seen.has(block.path)) continue;
+      seen.add(block.path);
+      yield block.path;
+    }
+  }
+
+  /**
+   * Multi-query search with score fusion. Each query is ranked by the engine,
+   * filtered to `pathPrefixes` (when given) and cut to `topK`; the per-block
+   * scores are then **summed** across queries, so a block matching more
+   * queries ranks higher, and the fused list is cut to `topK` again.
+   */
+  search(
+    queries: readonly string[],
+    topK: number,
+    pathPrefixes?: DocumentPath[],
+  ): MemBlockHit<B>[] {
+    const totals = new Map<string, MemBlockHit<B>>();
+    const scoped = pathPrefixes !== undefined && pathPrefixes.length > 0;
+    for (const query of queries) {
+      let taken = 0;
+      for (const { key, score } of this.engineRef.search(query, topK)) {
+        if (taken >= topK) break;
+        const block = this.blocks.get(key);
+        if (!block) continue;
+        if (scoped && !pathPrefixes.some((p) => matchesPrefix(block.path, p))) continue;
+        taken += 1;
+        const hit = totals.get(key);
+        if (hit) hit.score += score;
+        else totals.set(key, { block, score });
+      }
+    }
+    return [...totals.values()].sort((a, b) => b.score - a.score).slice(0, topK);
+  }
+
+  /**
+   * Serialize the blocks and the engine state into one JSON document,
+   * `{ version: 3, ...engineState, blocks }`. While the store is clean the
+   * previous output is returned as is.
+   */
+  serialize(): string {
+    if (!this.dirty && this.cached !== undefined) return this.cached;
+    const out = JSON.stringify({
+      version: MEM_BLOCK_STORE_VERSION,
+      ...this.engineRef.exportState(),
+      blocks: [...this.blocks.values()],
+    });
+    this.cached = out;
+    this.dirty = false;
+    return out;
+  }
+
+  /**
+   * Replace the store's content with a document written by {@link serialize}.
+   * `restoreEngine` rebuilds the engine from the document's fields. Throws on
+   * a document of another version, leaving the store unchanged. The store is
+   * clean afterwards and re-serializes to `json` until it changes.
+   */
+  restore(
+    json: string,
+    restoreEngine: (state: Record<string, unknown>) => MemBlockEngine<B>,
+  ): void {
+    const parsed = JSON.parse(json) as { version?: unknown; blocks?: B[] } & Record<
+      string,
+      unknown
+    >;
+    if (parsed.version !== MEM_BLOCK_STORE_VERSION) {
+      throw new Error(
+        `${this.label}: unsupported serialised version ${String(parsed.version)} (expected ${MEM_BLOCK_STORE_VERSION})`,
+      );
+    }
+    const engine = restoreEngine(parsed);
+    const blocks = new Map<string, B>();
+    for (const block of parsed.blocks ?? []) {
+      blocks.set(compositeKey(block.path, block.blockId), block);
+    }
+    this.engineRef = engine;
+    this.blocks = blocks;
+    this.cached = json;
+    this.dirty = false;
+  }
+}

package/src/mem-full-text-index.ts

@@ -0,0 +1,170 @@
+import type {
+  BlockReference,
+  DocumentPath,
+  PathSelector,
+  PersistableSearchIndex,
+  PersistenceEntry,
+} from "@statewalker/indexer-api";
+import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core";
+import type {
+  FullTextBlock,
+  FullTextIndex,
+  FullTextIndexInfo,
+  FulltextQuery,
+  FulltextResult,
+} from "@statewalker/indexer-fulltext";
+import { type MemBlockEngine, MemBlockStore } from "./mem-block-store.js";
+
+/** The engine of an in-memory full-text index: a {@link MemBlockEngine} over text blocks. */
+export type MemFullTextEngine = MemBlockEngine<FullTextBlock>;
+
+/**
+ * An in-memory {@link FullTextIndex} over a {@link MemBlockStore}. The store
+ * does everything but text indexing; a subclass supplies only the engine
+ * adapter (FlexSearch, MiniSearch, …) and how to rebuild it from saved state.
+ *
+ * Persistence: one entry named `json`, holding the store's framed document.
+ */
+export abstract class MemFullTextIndex
+  implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult>
+{
+  protected readonly info: FullTextIndexInfo;
+  protected readonly store: MemBlockStore<FullTextBlock>;
+  private closed = false;
+
+  /**
+   * @param label  named in errors (usually the subclass name)
+   * @param engine the engine adapter for a fresh, empty index
+   */
+  protected constructor(info: FullTextIndexInfo, label: string, engine: MemFullTextEngine) {
+    this.info = info;
+    this.store = new MemBlockStore<FullTextBlock>({ label, engine });
+  }
+
+  /** Rebuild the engine adapter from the fields it exported. */
+  protected abstract restoreEngine(state: Record<string, unknown>): MemFullTextEngine;
+
+  /**
+   * `true` when the in-memory state has changed since the last serialize or
+   * load. Callers (e.g. the persistence layer) may use it to skip writes.
+   */
+  isDirty(): boolean {
+    return this.store.isDirty();
+  }
+
+  protected ensureOpen(): void {
+    if (this.closed) throw new Error("FullTextIndex is closed");
+  }
+
+  /** The framed JSON document; cached while the index is unchanged. */
+  protected serializeState(): string {
+    return this.store.serialize();
+  }
+
+  /** Replace the content with a document from {@link serializeState}. */
+  protected restoreState(json: string): void {
+    this.store.restore(json, (state) => this.restoreEngine(state));
+  }
+
+  async getIndexInfo(): Promise<FullTextIndexInfo> {
+    this.ensureOpen();
+    return { ...this.info };
+  }
+
+  async *search(params: FulltextQuery): AsyncGenerator<FulltextResult> {
+    this.ensureOpen();
+    const { queries, paths } = params;
+    if (!queries || queries.length === 0) return;
+    for (const { block, score } of this.store.search(queries, params.topK ?? 100, paths)) {
+      yield { path: block.path, blockId: block.blockId, snippet: block.content, score };
+    }
+  }
+
+  async addDocument(blocks: FullTextBlock[]): Promise<void> {
+    this.ensureOpen();
+    this.store.put(
+      blocks.map((b) => ({
+        path: b.path,
+        blockId: b.blockId,
+        content: b.content,
+        metadata: b.metadata,
+      })),
+    );
+  }
+
+  async addDocuments(
+    blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
+  ): Promise<void> {
+    this.ensureOpen();
+    for await (const batch of blocks) {
+      await this.addDocument(batch);
+    }
+  }
+
+  async deleteDocuments(
+    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
+  ): Promise<void> {
+    this.ensureOpen();
+    await this.store.delete(pathSelectors);
+  }
+
+  async getSize(pathPrefix?: DocumentPath): Promise<number> {
+    this.ensureOpen();
+    return this.store.count(pathPrefix);
+  }
+
+  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
+    this.ensureOpen();
+    yield* this.store.paths(pathPrefix);
+  }
+
+  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
+    this.ensureOpen();
+    for (const block of this.store.select(pathPrefix)) {
+      yield { path: block.path, blockId: block.blockId };
+    }
+  }
+
+  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
+    this.ensureOpen();
+    for (const block of this.store.select(pathPrefix)) {
+      yield { ...block };
+    }
+  }
+
+  async close(_options?: { force?: boolean }): Promise<void> {
+    this.closed = true;
+  }
+
+  async flush(): Promise<void> {
+    this.ensureOpen();
+    // no-op for in-memory
+  }
+
+  async deleteIndex(): Promise<void> {
+    this.ensureOpen();
+    this.store.clear();
+    this.closed = true;
+  }
+
+  // --- PersistableSearchIndex ------------------------------------------------
+
+  /**
+   * Stream the sub-index's state as a single entry named `json`. The composite
+   * prefixes it with `${indexName}/${subName}/` before it reaches persistence.
+   */
+  async *serialise(): AsyncIterable<PersistenceEntry> {
+    yield { name: "json", content: singleChunk(toBytes(this.serializeState())) };
+  }
+
+  /** Restore the sub-index from a previously produced entry set. */
+  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
+    this.ensureOpen();
+    for (const e of entries) {
+      if (e.name !== "json") continue;
+      const bytes = await readEntryBytes(e);
+      this.restoreState(new TextDecoder().decode(bytes));
+      return;
+    }
+  }
+}