@statewalker/indexer-mem-flexsearch 0.1.2

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Kotelnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,55 @@
+# @statewalker/indexer-mem-flexsearch
+
+In-memory implementation of `@statewalker/indexer-api` combining [FlexSearch](https://github.com/nextapps-de/flexsearch) for full-text search with `@statewalker/indexer-mem`'s `MemVectorIndex` for embeddings. Optional streaming persistence via `IndexerPersistence`.
+
+## Installation
+
+```sh
+pnpm add @statewalker/indexer-mem-flexsearch
+```
+
+## Usage
+
+```ts
+import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch";
+
+const indexer = createFlexSearchIndexer();
+const index = await indexer.createIndex({
+  name: "docs",
+  fulltext: { language: "en" },
+  vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" },
+});
+
+await index.addDocument([
+  { path: "/docs/a", blockId: "1", content: "hello world" },
+]);
+
+for await (const hit of index.search({ queries: ["hello"], topK: 10 })) {
+  console.log(hit.path, hit.blockId, hit.score);
+}
+```
+
+### With persistence
+
+```ts
+import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch";
+import type { IndexerPersistence } from "@statewalker/indexer-api";
+
+const persistence: IndexerPersistence = /* … your save/load adapter … */;
+const indexer = createFlexSearchIndexer({ persistence });
+
+// First use loads existing state; `indexer.flush()` writes current state back.
+```
+
+Wire format (stable, byte-compatible across releases): `__manifest__` (JSON array of index names), `${name}/__config__`, `${name}/fts` (FlexSearch serialized state), `${name}/vec` (Arrow IPC from `MemVectorIndex`).
+
+## API
+
+- `createFlexSearchIndexer(options?)` — returns an `Indexer`. Options: `persistence?: IndexerPersistence`.
+- `FlexSearchIndexerOptions` — option type.
+
+## Related
+
+- `@statewalker/indexer-api` — the pluggable contract.
+- `@statewalker/indexer-mem-minisearch` — drop-in alternative using MiniSearch.
+- `@statewalker/indexer-mem` — provides the vector sub-index (`MemVectorIndex`).

package/dist/flexsearch-full-text-index.d.ts

@@ -0,0 +1,67 @@
+import { FullTextBlock, FullTextIndex, FullTextIndexInfo, FulltextQuery, FulltextResult } from "@statewalker/indexer-fulltext";
+import { BlockReference, DocumentPath, PathSelector, PersistableSearchIndex, PersistenceEntry } from "@statewalker/indexer-api";
+//#region src/flexsearch-full-text-index.d.ts
+export declare class FlexSearchFullTextIndex implements FullTextIndex, PersistableSearchIndex<FullTextBlock, FulltextQuery, FulltextResult> {
+  private readonly info;
+  private flexIndex;
+  private keyToNum;
+  private numToKey;
+  private blocks;
+  private nextNum;
+  private closed;
+  /**
+   * Dirty bit. `true` whenever the FTS index has been mutated since the
+   * last `serialize()`. The serializer caches the last produced JSON and
+   * returns it directly when `!dirty`, so a no-op re-sync skips the
+   * expensive FlexSearch export entirely. Starts `true` so the first
+   * serialize after construction always runs.
+   */
+  private dirty;
+  private cachedSerialized?;
+  constructor(info: FullTextIndexInfo);
+  /**
+   * `true` when the in-memory state has changed since the last serialize.
+   * Callers (e.g. the persistence layer) may use it to skip serialization
+   * and writes when the index has not been touched.
+   */
+  isDirty(): boolean;
+  private ensureOpen;
+  private getOrAssignNum;
+  private searchInternal;
+  getIndexInfo(): Promise<FullTextIndexInfo>;
+  search(params: FulltextQuery): AsyncGenerator<FulltextResult>;
+  addDocument(blocks: FullTextBlock[]): Promise<void>;
+  addDocuments(blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>): Promise<void>;
+  deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable<PathSelector>): Promise<void>;
+  /** Returns true when the key was present and got removed. */
+  private removeByKey;
+  getSize(pathPrefix?: DocumentPath): Promise<number>;
+  getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath>;
+  getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference>;
+  getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock>;
+  close(_options?: {
+    force?: boolean;
+  }): Promise<void>;
+  flush(): Promise<void>;
+  deleteIndex(): Promise<void>;
+  /**
+   * Stream the FTS sub-index's state as a single named entry. The composite
+   * prefixes this with `${indexName}/${subName}/` before it hits persistence.
+   */
+  serialise(): AsyncIterable<PersistenceEntry>;
+  /** Restore the FTS sub-index from a previously-produced entry set. */
+  loadFrom(entries: Iterable<PersistenceEntry>): Promise<void>;
+  /**
+   * Serialize to JSON string (v3 format with path data).
+   *
+   * Caches the result; while `dirty` is false (no mutations since the
+   * last call) returns the same string, skipping the FlexSearch export
+   * entirely. The persistence layer still calls into us each flush, but
+   * the heavy work only runs when the index actually changed.
+   */
+  serialize(): Promise<string>;
+  /** Deserialize from JSON — handles v3 format */
+  static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex;
+}
+//#endregion
+//# sourceMappingURL=flexsearch-full-text-index.d.ts.map

package/dist/flexsearch-full-text-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-full-text-index.d.ts","names":["FullTextSearchParams","FullTextSearchResult"],"sources":["../src/flexsearch-full-text-index.ts"],"mappings":";;;qBAgCa,mCAET,eACA,uBAAuB,eAAeA,eAAsBC;mBAE7C;UACT;UACA;UACA;UACA;UACA;UACA;;;;;;;;UAQA;UACA;EAER,YAAY,MAAM;;;;;;EAclB;UAIQ;UAMA;UAUA;EAwCF,gBAAgB,QAAQ;EAKvB,OAAO,QAAQD,gBAAuB,eAAeC;EAqCtD,YAAY,QAAQ,kBAAkB;EAqBtC,aACJ,QAAQ,SAAS,mBAAmB,cAAc,mBACjD;EAOG,gBACJ,eAAe,iBAAiB,cAAc,gBAC7C;;UAuBK;EAYF,QAAQ,aAAa,eAAe;EAUnC,iBAAiB,aAAa,eAAe,eAAe;EAW5D,sBAAsB,aAAa,eAAe,eAAe;EASjE,mBAAmB,aAAa,eAAe,eAAe;EAc/D,MAAM;IAAa;MAAoB;EAIvC,SAAS;EAKT,eAAe;;;;;EAgBd,aAAa,cAAc;;EAM5B,SAAS,SAAS,SAAS,oBAAoB;;;;;;;;;EA2B/C,aAAa;;SAqCZ,YAAY,MAAM,mBAAmB,eAAe"}

package/dist/flexsearch-full-text-index.js

@@ -0,0 +1,284 @@
+import { compositeKey, matchesPrefix, readEntryBytes, singleChunk, toAsyncIterable, toBytes } from "@statewalker/indexer-core";
+import FlexSearch from "flexsearch";
+//#region src/flexsearch-full-text-index.ts
+var FlexSearchFullTextIndex = class FlexSearchFullTextIndex {
+	info;
+	flexIndex;
+	keyToNum = /* @__PURE__ */ new Map();
+	numToKey = /* @__PURE__ */ new Map();
+	blocks = /* @__PURE__ */ new Map();
+	nextNum = 1;
+	closed = false;
+	/**
+	* Dirty bit. `true` whenever the FTS index has been mutated since the
+	* last `serialize()`. The serializer caches the last produced JSON and
+	* returns it directly when `!dirty`, so a no-op re-sync skips the
+	* expensive FlexSearch export entirely. Starts `true` so the first
+	* serialize after construction always runs.
+	*/
+	dirty = true;
+	cachedSerialized;
+	constructor(info) {
+		this.info = info;
+		this.flexIndex = new FlexSearch.Index({
+			tokenize: "forward",
+			resolution: 9,
+			cache: true
+		});
+	}
+	/**
+	* `true` when the in-memory state has changed since the last serialize.
+	* Callers (e.g. the persistence layer) may use it to skip serialization
+	* and writes when the index has not been touched.
+	*/
+	isDirty() {
+		return this.dirty;
+	}
+	ensureOpen() {
+		if (this.closed) throw new Error("FullTextIndex is closed");
+	}
+	getOrAssignNum(key) {
+		let num = this.keyToNum.get(key);
+		if (num === void 0) {
+			num = this.nextNum++;
+			this.keyToNum.set(key, num);
+			this.numToKey.set(num, key);
+		}
+		return num;
+	}
+	searchInternal(query, topK, pathPrefixes) {
+		const ids = this.flexIndex.search(query, topK * 3, { suggest: true });
+		let results = ids.map((numId, rank) => ({
+			key: this.numToKey.get(numId) ?? "",
+			score: 1 - rank / ids.length
+		})).filter((r) => r.key !== "");
+		if (pathPrefixes && pathPrefixes.length > 0) results = results.filter((r) => {
+			const block = this.blocks.get(r.key);
+			return block !== void 0 && pathPrefixes.some((p) => matchesPrefix(block.path, p));
+		});
+		results.sort((a, b) => b.score - a.score);
+		return results.slice(0, topK).map((r) => {
+			const block = this.blocks.get(r.key);
+			return {
+				path: block?.path ?? "/",
+				blockId: block?.blockId ?? "",
+				snippet: block?.content ?? "",
+				score: r.score
+			};
+		});
+	}
+	async getIndexInfo() {
+		this.ensureOpen();
+		return { ...this.info };
+	}
+	async *search(params) {
+		this.ensureOpen();
+		const { queries, paths } = params;
+		const topK = params.topK ?? 100;
+		if (!queries || queries.length === 0) return;
+		const merged = /* @__PURE__ */ new Map();
+		for (const query of queries) {
+			const results = this.searchInternal(query, topK, paths);
+			for (const r of results) {
+				const key = compositeKey(r.path, r.blockId);
+				const existing = merged.get(key);
+				if (!existing) merged.set(key, {
+					result: r,
+					best: r.score,
+					total: r.score
+				});
+				else {
+					existing.total += r.score;
+					if (r.score > existing.best) {
+						existing.best = r.score;
+						existing.result = r;
+					}
+				}
+			}
+		}
+		const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
+		for (const { result, total } of sorted.slice(0, topK)) yield {
+			...result,
+			score: total
+		};
+	}
+	async addDocument(blocks) {
+		this.ensureOpen();
+		if (blocks.length === 0) return;
+		for (const block of blocks) {
+			const key = compositeKey(block.path, block.blockId);
+			const num = this.getOrAssignNum(key);
+			if (this.blocks.has(key)) this.flexIndex.remove(num);
+			this.flexIndex.add(num, block.content);
+			this.blocks.set(key, {
+				path: block.path,
+				blockId: block.blockId,
+				content: block.content,
+				metadata: block.metadata
+			});
+		}
+		this.dirty = true;
+	}
+	async addDocuments(blocks) {
+		this.ensureOpen();
+		for await (const batch of blocks) await this.addDocument(batch);
+	}
+	async deleteDocuments(pathSelectors) {
+		this.ensureOpen();
+		let removed = 0;
+		for await (const sel of toAsyncIterable(pathSelectors)) if (sel.blockId !== void 0) {
+			const key = compositeKey(sel.path, sel.blockId);
+			if (this.removeByKey(key)) removed += 1;
+		} else {
+			const keysToDelete = [];
+			for (const [key, block] of this.blocks) if (matchesPrefix(block.path, sel.path)) keysToDelete.push(key);
+			for (const key of keysToDelete) if (this.removeByKey(key)) removed += 1;
+		}
+		if (removed > 0) this.dirty = true;
+	}
+	/** Returns true when the key was present and got removed. */
+	removeByKey(key) {
+		const num = this.keyToNum.get(key);
+		const had = this.blocks.has(key);
+		if (num !== void 0) {
+			this.flexIndex.remove(num);
+			this.keyToNum.delete(key);
+			this.numToKey.delete(num);
+		}
+		this.blocks.delete(key);
+		return had;
+	}
+	async getSize(pathPrefix) {
+		this.ensureOpen();
+		if (pathPrefix === void 0) return this.blocks.size;
+		let count = 0;
+		for (const block of this.blocks.values()) if (matchesPrefix(block.path, pathPrefix)) count++;
+		return count;
+	}
+	async *getDocumentPaths(pathPrefix) {
+		this.ensureOpen();
+		const paths = /* @__PURE__ */ new Set();
+		for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) paths.add(block.path);
+		for (const p of paths) yield p;
+	}
+	async *getDocumentBlocksRefs(pathPrefix) {
+		this.ensureOpen();
+		for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
+			path: block.path,
+			blockId: block.blockId
+		};
+	}
+	async *getDocumentsBlocks(pathPrefix) {
+		this.ensureOpen();
+		for (const block of this.blocks.values()) if (pathPrefix === void 0 || matchesPrefix(block.path, pathPrefix)) yield {
+			path: block.path,
+			blockId: block.blockId,
+			content: block.content,
+			metadata: block.metadata
+		};
+	}
+	async close(_options) {
+		this.closed = true;
+	}
+	async flush() {
+		this.ensureOpen();
+	}
+	async deleteIndex() {
+		this.ensureOpen();
+		this.blocks.clear();
+		this.keyToNum.clear();
+		this.numToKey.clear();
+		this.cachedSerialized = void 0;
+		this.dirty = true;
+		this.closed = true;
+	}
+	/**
+	* Stream the FTS sub-index's state as a single named entry. The composite
+	* prefixes this with `${indexName}/${subName}/` before it hits persistence.
+	*/
+	async *serialise() {
+		const json = await this.serialize();
+		yield {
+			name: "json",
+			content: singleChunk(toBytes(json))
+		};
+	}
+	/** Restore the FTS sub-index from a previously-produced entry set. */
+	async loadFrom(entries) {
+		this.ensureOpen();
+		for (const e of entries) {
+			if (e.name !== "json") continue;
+			const bytes = await readEntryBytes(e);
+			const json = new TextDecoder().decode(bytes);
+			const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);
+			this.flexIndex = rebuilt.flexIndex;
+			this.keyToNum = rebuilt.keyToNum;
+			this.numToKey = rebuilt.numToKey;
+			this.blocks = rebuilt.blocks;
+			this.nextNum = rebuilt.nextNum;
+			this.cachedSerialized = json;
+			this.dirty = false;
+			return;
+		}
+	}
+	/**
+	* Serialize to JSON string (v3 format with path data).
+	*
+	* Caches the result; while `dirty` is false (no mutations since the
+	* last call) returns the same string, skipping the FlexSearch export
+	* entirely. The persistence layer still calls into us each flush, but
+	* the heavy work only runs when the index actually changed.
+	*/
+	async serialize() {
+		if (!this.dirty && this.cachedSerialized !== void 0) return this.cachedSerialized;
+		const chunks = /* @__PURE__ */ new Map();
+		await this.flexIndex.export((key, data) => {
+			chunks.set(String(key), data);
+		});
+		const blocksData = [];
+		for (const block of this.blocks.values()) blocksData.push({
+			path: block.path,
+			blockId: block.blockId,
+			content: block.content,
+			metadata: block.metadata
+		});
+		const out = JSON.stringify({
+			version: 3,
+			chunks: Object.fromEntries(chunks),
+			blocks: blocksData,
+			keyToNum: [...this.keyToNum.entries()],
+			nextNum: this.nextNum
+		});
+		this.cachedSerialized = out;
+		this.dirty = false;
+		return out;
+	}
+	/** Deserialize from JSON — handles v3 format */
+	static deserialize(info, json) {
+		const parsed = JSON.parse(json);
+		if (parsed.version !== 3) throw new Error(`FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`);
+		const fts = new FlexSearchFullTextIndex(info);
+		for (const [key, data] of Object.entries(parsed.chunks)) fts.flexIndex.import(key, data);
+		for (const [key, num] of parsed.keyToNum) {
+			fts.keyToNum.set(key, num);
+			fts.numToKey.set(num, key);
+		}
+		fts.nextNum = parsed.nextNum;
+		for (const block of parsed.blocks) {
+			const key = compositeKey(block.path, block.blockId);
+			fts.blocks.set(key, {
+				path: block.path,
+				blockId: block.blockId,
+				content: block.content,
+				metadata: block.metadata
+			});
+		}
+		fts.cachedSerialized = json;
+		fts.dirty = false;
+		return fts;
+	}
+};
+//#endregion
+export { FlexSearchFullTextIndex };
+
+//# sourceMappingURL=flexsearch-full-text-index.js.map

package/dist/flexsearch-full-text-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-full-text-index.js","names":[],"sources":["../src/flexsearch-full-text-index.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  Metadata,\n  PathSelector,\n  PersistableSearchIndex,\n  PersistenceEntry,\n} from \"@statewalker/indexer-api\";\nimport {\n  compositeKey,\n  matchesPrefix,\n  readEntryBytes,\n  singleChunk,\n  toAsyncIterable,\n  toBytes,\n} from \"@statewalker/indexer-core\";\nimport type {\n  FullTextBlock,\n  FullTextIndex,\n  FullTextIndexInfo,\n  FulltextQuery as FullTextSearchParams,\n  FulltextResult as FullTextSearchResult,\n} from \"@statewalker/indexer-fulltext\";\nimport FlexSearch, { type Index as FlexIndex } from \"flexsearch\";\n\ninterface StoredBlock {\n  path: DocumentPath;\n  blockId: string;\n  content: string;\n  metadata?: Metadata;\n}\n\nexport class FlexSearchFullTextIndex\n  implements\n    FullTextIndex,\n    PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>\n{\n  private readonly info: FullTextIndexInfo;\n  private flexIndex: FlexIndex;\n  private keyToNum = new Map<string, number>();\n  private numToKey = new Map<number, string>();\n  private blocks = new Map<string, StoredBlock>();\n  private nextNum = 1;\n  private closed = false;\n  /**\n   * Dirty bit. `true` whenever the FTS index has been mutated since the\n   * last `serialize()`. The serializer caches the last produced JSON and\n   * returns it directly when `!dirty`, so a no-op re-sync skips the\n   * expensive FlexSearch export entirely. Starts `true` so the first\n   * serialize after construction always runs.\n   */\n  private dirty = true;\n  private cachedSerialized?: string;\n\n  constructor(info: FullTextIndexInfo) {\n    this.info = info;\n    this.flexIndex = new FlexSearch.Index({\n      tokenize: \"forward\",\n      resolution: 9,\n      cache: true,\n    });\n  }\n\n  /**\n   * `true` when the in-memory state has changed since the last serialize.\n   * Callers (e.g. the persistence layer) may use it to skip serialization\n   * and writes when the index has not been touched.\n   */\n  isDirty(): boolean {\n    return this.dirty;\n  }\n\n  private ensureOpen(): void {\n    if (this.closed) {\n      throw new Error(\"FullTextIndex is closed\");\n    }\n  }\n\n  private getOrAssignNum(key: string): number {\n    let num = this.keyToNum.get(key);\n    if (num === undefined) {\n      num = this.nextNum++;\n      this.keyToNum.set(key, num);\n      this.numToKey.set(num, key);\n    }\n    return num;\n  }\n\n  private searchInternal(\n    query: string,\n    topK: number,\n    pathPrefixes?: DocumentPath[],\n  ): FullTextSearchResult[] {\n    // `suggest: true` relaxes the default all-terms-AND for a multi-word query into a\n    // ranked partial match — sections are ordered by how much of the term set they match\n    // (FlexSearch's own resolution scoring), instead of returning nothing unless one\n    // section contains every term. For single-word or fully-intersecting queries it is\n    // identical to the default, so it only ever broadens recall, never narrows it.\n    const ids = this.flexIndex.search(query, topK * 3, { suggest: true }) as number[];\n\n    let results = ids\n      .map((numId, rank) => ({\n        key: this.numToKey.get(numId) ?? \"\",\n        score: 1 - rank / ids.length,\n      }))\n      .filter((r) => r.key !== \"\");\n\n    // Filter by path prefixes\n    if (pathPrefixes && pathPrefixes.length > 0) {\n      results = results.filter((r) => {\n        const block = this.blocks.get(r.key);\n        return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));\n      });\n    }\n\n    results.sort((a, b) => b.score - a.score);\n\n    return results.slice(0, topK).map((r) => {\n      const block = this.blocks.get(r.key);\n      return {\n        path: block?.path ?? (\"/\" as DocumentPath),\n        blockId: block?.blockId ?? \"\",\n        snippet: block?.content ?? \"\",\n        score: r.score,\n      };\n    });\n  }\n\n  async getIndexInfo(): Promise<FullTextIndexInfo> {\n    this.ensureOpen();\n    return { ...this.info };\n  }\n\n  async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {\n    this.ensureOpen();\n    const { queries, paths } = params;\n    const topK = params.topK ?? 100;\n\n    if (!queries || queries.length === 0) return;\n\n    // Multi-query: search for each and ACCUMULATE per-block scores, so a block\n    // matching more queries ranks higher (the FulltextQuery contract). Taking the\n    // max instead would let a block that is the top hit for one query outrank a\n    // block matching several queries. The snippet is kept from the strongest\n    // single-query match.\n    const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();\n\n    for (const query of queries) {\n      const results = this.searchInternal(query, topK, paths);\n      for (const r of results) {\n        const key = compositeKey(r.path, r.blockId);\n        const existing = merged.get(key);\n        if (!existing) {\n          merged.set(key, { result: r, best: r.score, total: r.score });\n        } else {\n          existing.total += r.score;\n          if (r.score > existing.best) {\n            existing.best = r.score;\n            existing.result = r;\n          }\n        }\n      }\n    }\n\n    const sorted = [...merged.values()].sort((a, b) => b.total - a.total);\n    for (const { result, total } of sorted.slice(0, topK)) {\n      yield { ...result, score: total };\n    }\n  }\n\n  async addDocument(blocks: FullTextBlock[]): Promise<void> {\n    this.ensureOpen();\n    if (blocks.length === 0) return;\n    for (const block of blocks) {\n      const key = compositeKey(block.path, block.blockId);\n      const num = this.getOrAssignNum(key);\n      // Remove old content if exists\n      if (this.blocks.has(key)) {\n        this.flexIndex.remove(num);\n      }\n      this.flexIndex.add(num, block.content);\n      this.blocks.set(key, {\n        path: block.path,\n        blockId: block.blockId,\n        content: block.content,\n        metadata: block.metadata,\n      });\n    }\n    this.dirty = true;\n  }\n\n  async addDocuments(\n    blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,\n  ): Promise<void> {\n    this.ensureOpen();\n    for await (const batch of blocks) {\n      await this.addDocument(batch);\n    }\n  }\n\n  async deleteDocuments(\n    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,\n  ): Promise<void> {\n    this.ensureOpen();\n    let removed = 0;\n    for await (const sel of toAsyncIterable(pathSelectors)) {\n      if (sel.blockId !== undefined) {\n        const key = compositeKey(sel.path, sel.blockId);\n        if (this.removeByKey(key)) removed += 1;\n      } else {\n        const keysToDelete: string[] = [];\n        for (const [key, block] of this.blocks) {\n          if (matchesPrefix(block.path, sel.path)) {\n            keysToDelete.push(key);\n          }\n        }\n        for (const key of keysToDelete) {\n          if (this.removeByKey(key)) removed += 1;\n        }\n      }\n    }\n    if (removed > 0) this.dirty = true;\n  }\n\n  /** Returns true when the key was present and got removed. */\n  private removeByKey(key: string): boolean {\n    const num = this.keyToNum.get(key);\n    const had = this.blocks.has(key);\n    if (num !== undefined) {\n      this.flexIndex.remove(num);\n      this.keyToNum.delete(key);\n      this.numToKey.delete(num);\n    }\n    this.blocks.delete(key);\n    return had;\n  }\n\n  async getSize(pathPrefix?: DocumentPath): Promise<number> {\n    this.ensureOpen();\n    if (pathPrefix === undefined) return this.blocks.size;\n    let count = 0;\n    for (const block of this.blocks.values()) {\n      if (matchesPrefix(block.path, pathPrefix)) count++;\n    }\n    return count;\n  }\n\n  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {\n    this.ensureOpen();\n    const paths = new Set<DocumentPath>();\n    for (const block of this.blocks.values()) {\n      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n        paths.add(block.path);\n      }\n    }\n    for (const p of paths) yield p;\n  }\n\n  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {\n    this.ensureOpen();\n    for (const block of this.blocks.values()) {\n      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n        yield { path: block.path, blockId: block.blockId };\n      }\n    }\n  }\n\n  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {\n    this.ensureOpen();\n    for (const block of this.blocks.values()) {\n      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {\n        yield {\n          path: block.path,\n          blockId: block.blockId,\n          content: block.content,\n          metadata: block.metadata,\n        };\n      }\n    }\n  }\n\n  async close(_options?: { force?: boolean }): Promise<void> {\n    this.closed = true;\n  }\n\n  async flush(): Promise<void> {\n    this.ensureOpen();\n    // no-op for in-memory\n  }\n\n  async deleteIndex(): Promise<void> {\n    this.ensureOpen();\n    this.blocks.clear();\n    this.keyToNum.clear();\n    this.numToKey.clear();\n    this.cachedSerialized = undefined;\n    this.dirty = true;\n    this.closed = true;\n  }\n\n  // --- PersistableSearchIndex --------------------------------------------------\n\n  /**\n   * Stream the FTS sub-index's state as a single named entry. The composite\n   * prefixes this with `${indexName}/${subName}/` before it hits persistence.\n   */\n  async *serialise(): AsyncIterable<PersistenceEntry> {\n    const json = await this.serialize();\n    yield { name: \"json\", content: singleChunk(toBytes(json)) };\n  }\n\n  /** Restore the FTS sub-index from a previously-produced entry set. */\n  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {\n    this.ensureOpen();\n    for (const e of entries) {\n      if (e.name !== \"json\") continue;\n      const bytes = await readEntryBytes(e);\n      const json = new TextDecoder().decode(bytes);\n      const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);\n      // Adopt the rebuilt instance's internal state.\n      this.flexIndex = rebuilt.flexIndex;\n      this.keyToNum = rebuilt.keyToNum;\n      this.numToKey = rebuilt.numToKey;\n      this.blocks = rebuilt.blocks;\n      this.nextNum = rebuilt.nextNum;\n      this.cachedSerialized = json;\n      this.dirty = false;\n      return;\n    }\n  }\n\n  /**\n   * Serialize to JSON string (v3 format with path data).\n   *\n   * Caches the result; while `dirty` is false (no mutations since the\n   * last call) returns the same string, skipping the FlexSearch export\n   * entirely. The persistence layer still calls into us each flush, but\n   * the heavy work only runs when the index actually changed.\n   */\n  async serialize(): Promise<string> {\n    if (!this.dirty && this.cachedSerialized !== undefined) {\n      return this.cachedSerialized;\n    }\n    const chunks = new Map<string, string>();\n    await this.flexIndex.export((key: string | number, data: string) => {\n      chunks.set(String(key), data);\n    });\n\n    const blocksData: Array<{\n      path: string;\n      blockId: string;\n      content: string;\n      metadata?: Metadata;\n    }> = [];\n    for (const block of this.blocks.values()) {\n      blocksData.push({\n        path: block.path,\n        blockId: block.blockId,\n        content: block.content,\n        metadata: block.metadata,\n      });\n    }\n\n    const out = JSON.stringify({\n      version: 3,\n      chunks: Object.fromEntries(chunks),\n      blocks: blocksData,\n      keyToNum: [...this.keyToNum.entries()],\n      nextNum: this.nextNum,\n    });\n    this.cachedSerialized = out;\n    this.dirty = false;\n    return out;\n  }\n\n  /** Deserialize from JSON — handles v3 format */\n  static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex {\n    const parsed = JSON.parse(json) as {\n      version?: number;\n      chunks: Record<string, string>;\n      blocks: Array<{\n        path: string;\n        blockId: string;\n        content: string;\n        metadata?: Metadata;\n      }>;\n      keyToNum: Array<[string, number]>;\n      nextNum: number;\n    };\n\n    if (parsed.version !== 3) {\n      throw new Error(\n        `FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,\n      );\n    }\n\n    const fts = new FlexSearchFullTextIndex(info);\n    for (const [key, data] of Object.entries(parsed.chunks)) {\n      fts.flexIndex.import(key, data);\n    }\n    for (const [key, num] of parsed.keyToNum) {\n      fts.keyToNum.set(key, num);\n      fts.numToKey.set(num, key);\n    }\n    fts.nextNum = parsed.nextNum;\n    for (const block of parsed.blocks) {\n      const key = compositeKey(block.path as DocumentPath, block.blockId);\n      fts.blocks.set(key, {\n        path: block.path as DocumentPath,\n        blockId: block.blockId,\n        content: block.content,\n        metadata: block.metadata,\n      });\n    }\n    // Prime the cache: a deserialized index that's never mutated should\n    // return the same JSON it was loaded from when re-serialized.\n    fts.cachedSerialized = json;\n    fts.dirty = false;\n    return fts;\n  }\n}\n"],"mappings":";;;AAgCA,IAAa,0BAAb,MAAa,wBAIb;CACE;CACA;CACA,2BAAmB,IAAI,IAAoB;CAC3C,2BAAmB,IAAI,IAAoB;CAC3C,yBAAiB,IAAI,IAAyB;CAC9C,UAAkB;CAClB,SAAiB;;;;;;;;CAQjB,QAAgB;CAChB;CAEA,YAAY,MAAyB;EACnC,KAAK,OAAO;EACZ,KAAK,YAAY,IAAI,WAAW,MAAM;GACpC,UAAU;GACV,YAAY;GACZ,OAAO;EACT,CAAC;CACH;;;;;;CAOA,UAAmB;EACjB,OAAO,KAAK;CACd;CAEA,aAA2B;EACzB,IAAI,KAAK,QACP,MAAM,IAAI,MAAM,yBAAyB;CAE7C;CAEA,eAAuB,KAAqB;EAC1C,IAAI,MAAM,KAAK,SAAS,IAAI,GAAG;EAC/B,IAAI,QAAQ,KAAA,GAAW;GACrB,MAAM,KAAK;GACX,KAAK,SAAS,IAAI,KAAK,GAAG;GAC1B,KAAK,SAAS,IAAI,KAAK,GAAG;EAC5B;EACA,OAAO;CACT;CAEA,eACE,OACA,MACA,cACwB;EAMxB,MAAM,MAAM,KAAK,UAAU,OAAO,OAAO,OAAO,GAAG,EAAE,SAAS,KAAK,CAAC;EAEpE,IAAI,UAAU,IACX,KAAK,OAAO,UAAU;GACrB,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK;GACjC,OAAO,IAAI,OAAO,IAAI;EACxB,EAAE,CAAC,CACF,QAAQ,MAAM,EAAE,QAAQ,EAAE;EAG7B,IAAI,gBAAgB,aAAa,SAAS,GACxC,UAAU,QAAQ,QAAQ,MAAM;GAC9B,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO,UAAU,KAAA,KAAa,aAAa,MAAM,MAAM,cAAc,MAAM,MAAM,CAAC,CAAC;EACrF,CAAC;EAGH,QAAQ,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EAExC,OAAO,QAAQ,MAAM,GAAG,IAAI,CAAC,CAAC,KAAK,MAAM;GACvC,MAAM,QAAQ,KAAK,OAAO,IAAI,EAAE,GAAG;GACnC,OAAO;IACL,MAAM,OAAO,QAAS;IACtB,SAAS,OAAO,WAAW;IAC3B,SAAS,OAAO,WAAW;IAC3B,OAAO,EAAE;GACX;EACF,CAAC;CACH;CAEA,MAAM,eAA2C;EAC/C,KAAK,WAAW;EAChB,OAAO,EAAE,GAAG,KAAK,KAAK;CACxB;CAEA,OAAO,OAAO,QAAoE;EAChF,KAAK,WAAW;EAChB,MAAM,EAAE,SAAS,UAAU;EAC3B,MAAM,OAAO,OAAO,QAAQ;EAE5B,IAAI,CAAC,WAAW,QAAQ,WAAW,GAAG;EAOtC,MAAM,yBAAS,IAAI,IAA2E;EAE9F,KAAK,MAAM,SAAS,SAAS;GAC3B,MAAM,UAAU,KAAK,eAAe,OAAO,MAAM,KAAK;GACtD,KAAK,MAAM,KAAK,SAAS;IACvB,MAAM,MAAM,aAAa,EAAE,MAAM,EAAE,OAAO;IAC1C,MAAM,WAAW,OAAO,IAAI,GAAG;IAC/B,IAAI,CAAC,UACH,OAAO,IAAI,KAAK;KAAE,QAAQ;KAAG,MAAM,EAAE;KAAO,OAAO,EAAE;IAAM,CAAC;SACvD;KACL,SAAS,SAAS,EAAE;KACpB,IAAI,EAAE,QAAQ,SAAS,MAAM;MAC3B,SAAS,OAAO,EAAE;MAClB,SAAS,SAAS;KACpB;IACF;GACF;EACF;EAEA,MAAM,SAAS,CAAC,GAAG,OAAO,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;EACpE,KAAK,MAAM,EAAE,QAAQ,WAAW,OAAO,MAAM,GAAG,IAAI,GAClD,MAAM;GAAE,GAAG;GAAQ,OAAO;EAAM;CAEpC;CAEA,MAAM,YAAY,QAAwC;EACxD,KAAK,WAAW;EAChB,IAAI,OAAO,WAAW,GAAG;EACzB,KAAK,MAAM,SAAS,QAAQ;GAC1B,MAAM,MAAM,aAAa,MAAM,MAAM,MAAM,OAAO;GAClD,MAAM,MAAM,KAAK,eAAe,GAAG;GAEnC,IAAI,KAAK,OAAO,IAAI,GAAG,GACrB,KAAK,UAAU,OAAO,GAAG;GAE3B,KAAK,UAAU,IAAI,KAAK,MAAM,OAAO;GACrC,KAAK,OAAO,IAAI,KAAK;IACnB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EACA,KAAK,QAAQ;CACf;CAEA,MAAM,aACJ,QACe;EACf,KAAK,WAAW;EAChB,WAAW,MAAM,SAAS,QACxB,MAAM,KAAK,YAAY,KAAK;CAEhC;CAEA,MAAM,gBACJ,eACe;EACf,KAAK,WAAW;EAChB,IAAI,UAAU;EACd,WAAW,MAAM,OAAO,gBAAgB,aAAa,GACnD,IAAI,IAAI,YAAY,KAAA,GAAW;GAC7B,MAAM,MAAM,aAAa,IAAI,MAAM,IAAI,OAAO;GAC9C,IAAI,KAAK,YAAY,GAAG,GAAG,WAAW;EACxC,OAAO;GACL,MAAM,eAAyB,CAAC;GAChC,KAAK,MAAM,CAAC,KAAK,UAAU,KAAK,QAC9B,IAAI,cAAc,MAAM,MAAM,IAAI,IAAI,GACpC,aAAa,KAAK,GAAG;GAGzB,KAAK,MAAM,OAAO,cAChB,IAAI,KAAK,YAAY,GAAG,GAAG,WAAW;EAE1C;EAEF,IAAI,UAAU,GAAG,KAAK,QAAQ;CAChC;;CAGA,YAAoB,KAAsB;EACxC,MAAM,MAAM,KAAK,SAAS,IAAI,GAAG;EACjC,MAAM,MAAM,KAAK,OAAO,IAAI,GAAG;EAC/B,IAAI,QAAQ,KAAA,GAAW;GACrB,KAAK,UAAU,OAAO,GAAG;GACzB,KAAK,SAAS,OAAO,GAAG;GACxB,KAAK,SAAS,OAAO,GAAG;EAC1B;EACA,KAAK,OAAO,OAAO,GAAG;EACtB,OAAO;CACT;CAEA,MAAM,QAAQ,YAA4C;EACxD,KAAK,WAAW;EAChB,IAAI,eAAe,KAAA,GAAW,OAAO,KAAK,OAAO;EACjD,IAAI,QAAQ;EACZ,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,cAAc,MAAM,MAAM,UAAU,GAAG;EAE7C,OAAO;CACT;CAEA,OAAO,iBAAiB,YAAyD;EAC/E,KAAK,WAAW;EAChB,MAAM,wBAAQ,IAAI,IAAkB;EACpC,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM,IAAI,MAAM,IAAI;EAGxB,KAAK,MAAM,KAAK,OAAO,MAAM;CAC/B;CAEA,OAAO,sBAAsB,YAA2D;EACtF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ;CAGvD;CAEA,OAAO,mBAAmB,YAA0D;EAClF,KAAK,WAAW;EAChB,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,IAAI,eAAe,KAAA,KAAa,cAAc,MAAM,MAAM,UAAU,GAClE,MAAM;GACJ,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB;CAGN;CAEA,MAAM,MAAM,UAA+C;EACzD,KAAK,SAAS;CAChB;CAEA,MAAM,QAAuB;EAC3B,KAAK,WAAW;CAElB;CAEA,MAAM,cAA6B;EACjC,KAAK,WAAW;EAChB,KAAK,OAAO,MAAM;EAClB,KAAK,SAAS,MAAM;EACpB,KAAK,SAAS,MAAM;EACpB,KAAK,mBAAmB,KAAA;EACxB,KAAK,QAAQ;EACb,KAAK,SAAS;CAChB;;;;;CAQA,OAAO,YAA6C;EAClD,MAAM,OAAO,MAAM,KAAK,UAAU;EAClC,MAAM;GAAE,MAAM;GAAQ,SAAS,YAAY,QAAQ,IAAI,CAAC;EAAE;CAC5D;;CAGA,MAAM,SAAS,SAAoD;EACjE,KAAK,WAAW;EAChB,KAAK,MAAM,KAAK,SAAS;GACvB,IAAI,EAAE,SAAS,QAAQ;GACvB,MAAM,QAAQ,MAAM,eAAe,CAAC;GACpC,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,KAAK;GAC3C,MAAM,UAAU,wBAAwB,YAAY,KAAK,MAAM,IAAI;GAEnE,KAAK,YAAY,QAAQ;GACzB,KAAK,WAAW,QAAQ;GACxB,KAAK,WAAW,QAAQ;GACxB,KAAK,SAAS,QAAQ;GACtB,KAAK,UAAU,QAAQ;GACvB,KAAK,mBAAmB;GACxB,KAAK,QAAQ;GACb;EACF;CACF;;;;;;;;;CAUA,MAAM,YAA6B;EACjC,IAAI,CAAC,KAAK,SAAS,KAAK,qBAAqB,KAAA,GAC3C,OAAO,KAAK;EAEd,MAAM,yBAAS,IAAI,IAAoB;EACvC,MAAM,KAAK,UAAU,QAAQ,KAAsB,SAAiB;GAClE,OAAO,IAAI,OAAO,GAAG,GAAG,IAAI;EAC9B,CAAC;EAED,MAAM,aAKD,CAAC;EACN,KAAK,MAAM,SAAS,KAAK,OAAO,OAAO,GACrC,WAAW,KAAK;GACd,MAAM,MAAM;GACZ,SAAS,MAAM;GACf,SAAS,MAAM;GACf,UAAU,MAAM;EAClB,CAAC;EAGH,MAAM,MAAM,KAAK,UAAU;GACzB,SAAS;GACT,QAAQ,OAAO,YAAY,MAAM;GACjC,QAAQ;GACR,UAAU,CAAC,GAAG,KAAK,SAAS,QAAQ,CAAC;GACrC,SAAS,KAAK;EAChB,CAAC;EACD,KAAK,mBAAmB;EACxB,KAAK,QAAQ;EACb,OAAO;CACT;;CAGA,OAAO,YAAY,MAAyB,MAAuC;EACjF,MAAM,SAAS,KAAK,MAAM,IAAI;EAa9B,IAAI,OAAO,YAAY,GACrB,MAAM,IAAI,MACR,2DAA2D,OAAO,OAAO,OAAO,EAAE,cACpF;EAGF,MAAM,MAAM,IAAI,wBAAwB,IAAI;EAC5C,KAAK,MAAM,CAAC,KAAK,SAAS,OAAO,QAAQ,OAAO,MAAM,GACpD,IAAI,UAAU,OAAO,KAAK,IAAI;EAEhC,KAAK,MAAM,CAAC,KAAK,QAAQ,OAAO,UAAU;GACxC,IAAI,SAAS,IAAI,KAAK,GAAG;GACzB,IAAI,SAAS,IAAI,KAAK,GAAG;EAC3B;EACA,IAAI,UAAU,OAAO;EACrB,KAAK,MAAM,SAAS,OAAO,QAAQ;GACjC,MAAM,MAAM,aAAa,MAAM,MAAsB,MAAM,OAAO;GAClE,IAAI,OAAO,IAAI,KAAK;IAClB,MAAM,MAAM;IACZ,SAAS,MAAM;IACf,SAAS,MAAM;IACf,UAAU,MAAM;GAClB,CAAC;EACH;EAGA,IAAI,mBAAmB;EACvB,IAAI,QAAQ;EACZ,OAAO;CACT;AACF"}

package/dist/flexsearch-indexer.d.ts

@@ -0,0 +1,13 @@
+import { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
+//#region src/flexsearch-indexer.d.ts
+export interface FlexSearchIndexerOptions {
+  persistence?: IndexerPersistence;
+}
+/**
+ * In-memory Indexer combining FlexSearch full-text and brute-force vector
+ * sub-indexes via the Provider model. Multiple sub-indexes of either modality
+ * can be created on one Index (e.g. English and French FTS sub-indexes).
+ */
+export declare function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Indexer;
+//#endregion
+//# sourceMappingURL=flexsearch-indexer.d.ts.map

package/dist/flexsearch-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-indexer.d.ts","names":[],"sources":["../src/flexsearch-indexer.ts"],"mappings":";;iBAKiB;EACf,cAAc;;;;;;;wBAQA,wBAAwB,UAAU,2BAA2B"}

package/dist/flexsearch-indexer.js

@@ -0,0 +1,19 @@
+import { flexSearchFullTextProvider } from "./flexsearch-provider.js";
+import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
+import { memVectorProvider } from "@statewalker/indexer-mem";
+//#region src/flexsearch-indexer.ts
+/**
+* In-memory Indexer combining FlexSearch full-text and brute-force vector
+* sub-indexes via the Provider model. Multiple sub-indexes of either modality
+* can be created on one Index (e.g. English and French FTS sub-indexes).
+*/
+function createFlexSearchIndexer(options) {
+	return createPersistenceBackedIndexer({
+		persistence: options?.persistence,
+		providers: [flexSearchFullTextProvider, memVectorProvider]
+	});
+}
+//#endregion
+export { createFlexSearchIndexer };
+
+//# sourceMappingURL=flexsearch-indexer.js.map

package/dist/flexsearch-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-indexer.js","names":[],"sources":["../src/flexsearch-indexer.ts"],"sourcesContent":["import type { Indexer, IndexerPersistence } from \"@statewalker/indexer-api\";\nimport { createPersistenceBackedIndexer } from \"@statewalker/indexer-core\";\nimport { memVectorProvider } from \"@statewalker/indexer-mem\";\nimport { flexSearchFullTextProvider } from \"./flexsearch-provider.js\";\n\nexport interface FlexSearchIndexerOptions {\n  persistence?: IndexerPersistence;\n}\n\n/**\n * In-memory Indexer combining FlexSearch full-text and brute-force vector\n * sub-indexes via the Provider model. Multiple sub-indexes of either modality\n * can be created on one Index (e.g. English and French FTS sub-indexes).\n */\nexport function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Indexer {\n  return createPersistenceBackedIndexer({\n    persistence: options?.persistence,\n    providers: [flexSearchFullTextProvider, memVectorProvider],\n  });\n}\n"],"mappings":";;;;;;;;;AAcA,SAAgB,wBAAwB,SAA6C;CACnF,OAAO,+BAA+B;EACpC,aAAa,SAAS;EACtB,WAAW,CAAC,4BAA4B,iBAAiB;CAC3D,CAAC;AACH"}

package/dist/flexsearch-provider.d.ts

@@ -0,0 +1,10 @@
+import { FullTextProvider } from "@statewalker/indexer-fulltext";
+//#region src/flexsearch-provider.d.ts
+/**
+ * `FullTextProvider` backed by {@link FlexSearchFullTextIndex}. Wired into an
+ * `Indexer` alongside a `VectorProvider` to compose a full-text + vector
+ * Indexer.
+ */
+export declare const flexSearchFullTextProvider: FullTextProvider;
+//#endregion
+//# sourceMappingURL=flexsearch-provider.d.ts.map

package/dist/flexsearch-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-provider.d.ts","names":[],"sources":["../src/flexsearch-provider.ts"],"mappings":";;;;;;;qBAYa,4BAA4B"}

package/dist/flexsearch-provider.js

@@ -0,0 +1,21 @@
+import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js";
+import { FULL_TEXT_TYPE } from "@statewalker/indexer-fulltext";
+//#region src/flexsearch-provider.ts
+/**
+* `FullTextProvider` backed by {@link FlexSearchFullTextIndex}. Wired into an
+* `Indexer` alongside a `VectorProvider` to compose a full-text + vector
+* Indexer.
+*/
+const flexSearchFullTextProvider = {
+	type: FULL_TEXT_TYPE,
+	create(config) {
+		return new FlexSearchFullTextIndex({
+			language: config.language,
+			metadata: config.metadata
+		});
+	}
+};
+//#endregion
+export { flexSearchFullTextProvider };
+
+//# sourceMappingURL=flexsearch-provider.js.map

package/dist/flexsearch-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flexsearch-provider.js","names":[],"sources":["../src/flexsearch-provider.ts"],"sourcesContent":["import {\n  FULL_TEXT_TYPE,\n  type FullTextConfig,\n  type FullTextProvider,\n} from \"@statewalker/indexer-fulltext\";\nimport { FlexSearchFullTextIndex } from \"./flexsearch-full-text-index.js\";\n\n/**\n * `FullTextProvider` backed by {@link FlexSearchFullTextIndex}. Wired into an\n * `Indexer` alongside a `VectorProvider` to compose a full-text + vector\n * Indexer.\n */\nexport const flexSearchFullTextProvider: FullTextProvider = {\n  type: FULL_TEXT_TYPE,\n  create(config: FullTextConfig): FlexSearchFullTextIndex {\n    return new FlexSearchFullTextIndex({\n      language: config.language,\n      metadata: config.metadata,\n    });\n  },\n};\n"],"mappings":";;;;;;;;AAYA,MAAa,6BAA+C;CAC1D,MAAM;CACN,OAAO,QAAiD;EACtD,OAAO,IAAI,wBAAwB;GACjC,UAAU,OAAO;GACjB,UAAU,OAAO;EACnB,CAAC;CACH;AACF"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js";
+import { FlexSearchIndexerOptions, createFlexSearchIndexer } from "./flexsearch-indexer.js";
+import { flexSearchFullTextProvider } from "./flexsearch-provider.js";
+export { FlexSearchFullTextIndex, type FlexSearchIndexerOptions, createFlexSearchIndexer, flexSearchFullTextProvider };

package/dist/index.js

@@ -0,0 +1,4 @@
+import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js";
+import { flexSearchFullTextProvider } from "./flexsearch-provider.js";
+import { createFlexSearchIndexer } from "./flexsearch-indexer.js";
+export { FlexSearchFullTextIndex, createFlexSearchIndexer, flexSearchFullTextProvider };

package/package.json

@@ -0,0 +1,59 @@
+{
+  "name": "@statewalker/indexer-mem-flexsearch",
+  "version": "0.1.2",
+  "private": false,
+  "type": "module",
+  "description": "In-memory indexer implementation backed by FlexSearch for full-text search.",
+  "homepage": "https://github.com/statewalker/statewalker-indexer",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git"
+  },
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "flexsearch": "^0.8.212",
+    "@statewalker/indexer-api": "^0.1.1",
+    "@statewalker/indexer-core": "^0.1.1",
+    "@statewalker/indexer-mem": "^0.1.2",
+    "@statewalker/indexer-fulltext": "^0.1.1"
+  },
+  "devDependencies": {
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.23.0",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3",
+    "@statewalker/indexer-search": "^0.1.0",
+    "@statewalker/indexer-tests": "^0.0.0",
+    "@statewalker/indexer-vector": "^0.1.1"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "dev": "tsdown --watch",
+    "test": "vitest run --passWithNoTests",
+    "test:watch": "vitest",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
+}

package/src/flexsearch-full-text-index.ts

@@ -0,0 +1,421 @@
+import type {
+  BlockReference,
+  DocumentPath,
+  Metadata,
+  PathSelector,
+  PersistableSearchIndex,
+  PersistenceEntry,
+} from "@statewalker/indexer-api";
+import {
+  compositeKey,
+  matchesPrefix,
+  readEntryBytes,
+  singleChunk,
+  toAsyncIterable,
+  toBytes,
+} from "@statewalker/indexer-core";
+import type {
+  FullTextBlock,
+  FullTextIndex,
+  FullTextIndexInfo,
+  FulltextQuery as FullTextSearchParams,
+  FulltextResult as FullTextSearchResult,
+} from "@statewalker/indexer-fulltext";
+import FlexSearch, { type Index as FlexIndex } from "flexsearch";
+
+interface StoredBlock {
+  path: DocumentPath;
+  blockId: string;
+  content: string;
+  metadata?: Metadata;
+}
+
+export class FlexSearchFullTextIndex
+  implements
+    FullTextIndex,
+    PersistableSearchIndex<FullTextBlock, FullTextSearchParams, FullTextSearchResult>
+{
+  private readonly info: FullTextIndexInfo;
+  private flexIndex: FlexIndex;
+  private keyToNum = new Map<string, number>();
+  private numToKey = new Map<number, string>();
+  private blocks = new Map<string, StoredBlock>();
+  private nextNum = 1;
+  private closed = false;
+  /**
+   * Dirty bit. `true` whenever the FTS index has been mutated since the
+   * last `serialize()`. The serializer caches the last produced JSON and
+   * returns it directly when `!dirty`, so a no-op re-sync skips the
+   * expensive FlexSearch export entirely. Starts `true` so the first
+   * serialize after construction always runs.
+   */
+  private dirty = true;
+  private cachedSerialized?: string;
+
+  constructor(info: FullTextIndexInfo) {
+    this.info = info;
+    this.flexIndex = new FlexSearch.Index({
+      tokenize: "forward",
+      resolution: 9,
+      cache: true,
+    });
+  }
+
+  /**
+   * `true` when the in-memory state has changed since the last serialize.
+   * Callers (e.g. the persistence layer) may use it to skip serialization
+   * and writes when the index has not been touched.
+   */
+  isDirty(): boolean {
+    return this.dirty;
+  }
+
+  private ensureOpen(): void {
+    if (this.closed) {
+      throw new Error("FullTextIndex is closed");
+    }
+  }
+
+  private getOrAssignNum(key: string): number {
+    let num = this.keyToNum.get(key);
+    if (num === undefined) {
+      num = this.nextNum++;
+      this.keyToNum.set(key, num);
+      this.numToKey.set(num, key);
+    }
+    return num;
+  }
+
+  private searchInternal(
+    query: string,
+    topK: number,
+    pathPrefixes?: DocumentPath[],
+  ): FullTextSearchResult[] {
+    // `suggest: true` relaxes the default all-terms-AND for a multi-word query into a
+    // ranked partial match — sections are ordered by how much of the term set they match
+    // (FlexSearch's own resolution scoring), instead of returning nothing unless one
+    // section contains every term. For single-word or fully-intersecting queries it is
+    // identical to the default, so it only ever broadens recall, never narrows it.
+    const ids = this.flexIndex.search(query, topK * 3, { suggest: true }) as number[];
+
+    let results = ids
+      .map((numId, rank) => ({
+        key: this.numToKey.get(numId) ?? "",
+        score: 1 - rank / ids.length,
+      }))
+      .filter((r) => r.key !== "");
+
+    // Filter by path prefixes
+    if (pathPrefixes && pathPrefixes.length > 0) {
+      results = results.filter((r) => {
+        const block = this.blocks.get(r.key);
+        return block !== undefined && pathPrefixes.some((p) => matchesPrefix(block.path, p));
+      });
+    }
+
+    results.sort((a, b) => b.score - a.score);
+
+    return results.slice(0, topK).map((r) => {
+      const block = this.blocks.get(r.key);
+      return {
+        path: block?.path ?? ("/" as DocumentPath),
+        blockId: block?.blockId ?? "",
+        snippet: block?.content ?? "",
+        score: r.score,
+      };
+    });
+  }
+
+  async getIndexInfo(): Promise<FullTextIndexInfo> {
+    this.ensureOpen();
+    return { ...this.info };
+  }
+
+  async *search(params: FullTextSearchParams): AsyncGenerator<FullTextSearchResult> {
+    this.ensureOpen();
+    const { queries, paths } = params;
+    const topK = params.topK ?? 100;
+
+    if (!queries || queries.length === 0) return;
+
+    // Multi-query: search for each and ACCUMULATE per-block scores, so a block
+    // matching more queries ranks higher (the FulltextQuery contract). Taking the
+    // max instead would let a block that is the top hit for one query outrank a
+    // block matching several queries. The snippet is kept from the strongest
+    // single-query match.
+    const merged = new Map<string, { result: FullTextSearchResult; best: number; total: number }>();
+
+    for (const query of queries) {
+      const results = this.searchInternal(query, topK, paths);
+      for (const r of results) {
+        const key = compositeKey(r.path, r.blockId);
+        const existing = merged.get(key);
+        if (!existing) {
+          merged.set(key, { result: r, best: r.score, total: r.score });
+        } else {
+          existing.total += r.score;
+          if (r.score > existing.best) {
+            existing.best = r.score;
+            existing.result = r;
+          }
+        }
+      }
+    }
+
+    const sorted = [...merged.values()].sort((a, b) => b.total - a.total);
+    for (const { result, total } of sorted.slice(0, topK)) {
+      yield { ...result, score: total };
+    }
+  }
+
+  async addDocument(blocks: FullTextBlock[]): Promise<void> {
+    this.ensureOpen();
+    if (blocks.length === 0) return;
+    for (const block of blocks) {
+      const key = compositeKey(block.path, block.blockId);
+      const num = this.getOrAssignNum(key);
+      // Remove old content if exists
+      if (this.blocks.has(key)) {
+        this.flexIndex.remove(num);
+      }
+      this.flexIndex.add(num, block.content);
+      this.blocks.set(key, {
+        path: block.path,
+        blockId: block.blockId,
+        content: block.content,
+        metadata: block.metadata,
+      });
+    }
+    this.dirty = true;
+  }
+
+  async addDocuments(
+    blocks: Iterable<FullTextBlock[]> | AsyncIterable<FullTextBlock[]>,
+  ): Promise<void> {
+    this.ensureOpen();
+    for await (const batch of blocks) {
+      await this.addDocument(batch);
+    }
+  }
+
+  async deleteDocuments(
+    pathSelectors: PathSelector[] | AsyncIterable<PathSelector>,
+  ): Promise<void> {
+    this.ensureOpen();
+    let removed = 0;
+    for await (const sel of toAsyncIterable(pathSelectors)) {
+      if (sel.blockId !== undefined) {
+        const key = compositeKey(sel.path, sel.blockId);
+        if (this.removeByKey(key)) removed += 1;
+      } else {
+        const keysToDelete: string[] = [];
+        for (const [key, block] of this.blocks) {
+          if (matchesPrefix(block.path, sel.path)) {
+            keysToDelete.push(key);
+          }
+        }
+        for (const key of keysToDelete) {
+          if (this.removeByKey(key)) removed += 1;
+        }
+      }
+    }
+    if (removed > 0) this.dirty = true;
+  }
+
+  /** Returns true when the key was present and got removed. */
+  private removeByKey(key: string): boolean {
+    const num = this.keyToNum.get(key);
+    const had = this.blocks.has(key);
+    if (num !== undefined) {
+      this.flexIndex.remove(num);
+      this.keyToNum.delete(key);
+      this.numToKey.delete(num);
+    }
+    this.blocks.delete(key);
+    return had;
+  }
+
+  async getSize(pathPrefix?: DocumentPath): Promise<number> {
+    this.ensureOpen();
+    if (pathPrefix === undefined) return this.blocks.size;
+    let count = 0;
+    for (const block of this.blocks.values()) {
+      if (matchesPrefix(block.path, pathPrefix)) count++;
+    }
+    return count;
+  }
+
+  async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator<DocumentPath> {
+    this.ensureOpen();
+    const paths = new Set<DocumentPath>();
+    for (const block of this.blocks.values()) {
+      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
+        paths.add(block.path);
+      }
+    }
+    for (const p of paths) yield p;
+  }
+
+  async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator<BlockReference> {
+    this.ensureOpen();
+    for (const block of this.blocks.values()) {
+      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
+        yield { path: block.path, blockId: block.blockId };
+      }
+    }
+  }
+
+  async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator<FullTextBlock> {
+    this.ensureOpen();
+    for (const block of this.blocks.values()) {
+      if (pathPrefix === undefined || matchesPrefix(block.path, pathPrefix)) {
+        yield {
+          path: block.path,
+          blockId: block.blockId,
+          content: block.content,
+          metadata: block.metadata,
+        };
+      }
+    }
+  }
+
+  async close(_options?: { force?: boolean }): Promise<void> {
+    this.closed = true;
+  }
+
+  async flush(): Promise<void> {
+    this.ensureOpen();
+    // no-op for in-memory
+  }
+
+  async deleteIndex(): Promise<void> {
+    this.ensureOpen();
+    this.blocks.clear();
+    this.keyToNum.clear();
+    this.numToKey.clear();
+    this.cachedSerialized = undefined;
+    this.dirty = true;
+    this.closed = true;
+  }
+
+  // --- PersistableSearchIndex --------------------------------------------------
+
+  /**
+   * Stream the FTS sub-index's state as a single named entry. The composite
+   * prefixes this with `${indexName}/${subName}/` before it hits persistence.
+   */
+  async *serialise(): AsyncIterable<PersistenceEntry> {
+    const json = await this.serialize();
+    yield { name: "json", content: singleChunk(toBytes(json)) };
+  }
+
+  /** Restore the FTS sub-index from a previously-produced entry set. */
+  async loadFrom(entries: Iterable<PersistenceEntry>): Promise<void> {
+    this.ensureOpen();
+    for (const e of entries) {
+      if (e.name !== "json") continue;
+      const bytes = await readEntryBytes(e);
+      const json = new TextDecoder().decode(bytes);
+      const rebuilt = FlexSearchFullTextIndex.deserialize(this.info, json);
+      // Adopt the rebuilt instance's internal state.
+      this.flexIndex = rebuilt.flexIndex;
+      this.keyToNum = rebuilt.keyToNum;
+      this.numToKey = rebuilt.numToKey;
+      this.blocks = rebuilt.blocks;
+      this.nextNum = rebuilt.nextNum;
+      this.cachedSerialized = json;
+      this.dirty = false;
+      return;
+    }
+  }
+
+  /**
+   * Serialize to JSON string (v3 format with path data).
+   *
+   * Caches the result; while `dirty` is false (no mutations since the
+   * last call) returns the same string, skipping the FlexSearch export
+   * entirely. The persistence layer still calls into us each flush, but
+   * the heavy work only runs when the index actually changed.
+   */
+  async serialize(): Promise<string> {
+    if (!this.dirty && this.cachedSerialized !== undefined) {
+      return this.cachedSerialized;
+    }
+    const chunks = new Map<string, string>();
+    await this.flexIndex.export((key: string | number, data: string) => {
+      chunks.set(String(key), data);
+    });
+
+    const blocksData: Array<{
+      path: string;
+      blockId: string;
+      content: string;
+      metadata?: Metadata;
+    }> = [];
+    for (const block of this.blocks.values()) {
+      blocksData.push({
+        path: block.path,
+        blockId: block.blockId,
+        content: block.content,
+        metadata: block.metadata,
+      });
+    }
+
+    const out = JSON.stringify({
+      version: 3,
+      chunks: Object.fromEntries(chunks),
+      blocks: blocksData,
+      keyToNum: [...this.keyToNum.entries()],
+      nextNum: this.nextNum,
+    });
+    this.cachedSerialized = out;
+    this.dirty = false;
+    return out;
+  }
+
+  /** Deserialize from JSON — handles v3 format */
+  static deserialize(info: FullTextIndexInfo, json: string): FlexSearchFullTextIndex {
+    const parsed = JSON.parse(json) as {
+      version?: number;
+      chunks: Record<string, string>;
+      blocks: Array<{
+        path: string;
+        blockId: string;
+        content: string;
+        metadata?: Metadata;
+      }>;
+      keyToNum: Array<[string, number]>;
+      nextNum: number;
+    };
+
+    if (parsed.version !== 3) {
+      throw new Error(
+        `FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`,
+      );
+    }
+
+    const fts = new FlexSearchFullTextIndex(info);
+    for (const [key, data] of Object.entries(parsed.chunks)) {
+      fts.flexIndex.import(key, data);
+    }
+    for (const [key, num] of parsed.keyToNum) {
+      fts.keyToNum.set(key, num);
+      fts.numToKey.set(num, key);
+    }
+    fts.nextNum = parsed.nextNum;
+    for (const block of parsed.blocks) {
+      const key = compositeKey(block.path as DocumentPath, block.blockId);
+      fts.blocks.set(key, {
+        path: block.path as DocumentPath,
+        blockId: block.blockId,
+        content: block.content,
+        metadata: block.metadata,
+      });
+    }
+    // Prime the cache: a deserialized index that's never mutated should
+    // return the same JSON it was loaded from when re-serialized.
+    fts.cachedSerialized = json;
+    fts.dirty = false;
+    return fts;
+  }
+}

package/src/flexsearch-indexer.ts

@@ -0,0 +1,20 @@
+import type { Indexer, IndexerPersistence } from "@statewalker/indexer-api";
+import { createPersistenceBackedIndexer } from "@statewalker/indexer-core";
+import { memVectorProvider } from "@statewalker/indexer-mem";
+import { flexSearchFullTextProvider } from "./flexsearch-provider.js";
+
+export interface FlexSearchIndexerOptions {
+  persistence?: IndexerPersistence;
+}
+
+/**
+ * In-memory Indexer combining FlexSearch full-text and brute-force vector
+ * sub-indexes via the Provider model. Multiple sub-indexes of either modality
+ * can be created on one Index (e.g. English and French FTS sub-indexes).
+ */
+export function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Indexer {
+  return createPersistenceBackedIndexer({
+    persistence: options?.persistence,
+    providers: [flexSearchFullTextProvider, memVectorProvider],
+  });
+}

package/src/flexsearch-provider.ts

@@ -0,0 +1,21 @@
+import {
+  FULL_TEXT_TYPE,
+  type FullTextConfig,
+  type FullTextProvider,
+} from "@statewalker/indexer-fulltext";
+import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js";
+
+/**
+ * `FullTextProvider` backed by {@link FlexSearchFullTextIndex}. Wired into an
+ * `Indexer` alongside a `VectorProvider` to compose a full-text + vector
+ * Indexer.
+ */
+export const flexSearchFullTextProvider: FullTextProvider = {
+  type: FULL_TEXT_TYPE,
+  create(config: FullTextConfig): FlexSearchFullTextIndex {
+    return new FlexSearchFullTextIndex({
+      language: config.language,
+      metadata: config.metadata,
+    });
+  },
+};

package/src/index.ts

@@ -0,0 +1,3 @@
+export { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js";
+export { createFlexSearchIndexer, type FlexSearchIndexerOptions } from "./flexsearch-indexer.js";
+export { flexSearchFullTextProvider } from "./flexsearch-provider.js";

package/src/text-codec.d.ts

@@ -0,0 +1,3 @@
+// See ../../indexer-core/src/text-codec.d.ts for rationale.
+declare const TextEncoder: { new (): { encode(input: string): Uint8Array } };
+declare const TextDecoder: { new (label?: string): { decode(input?: Uint8Array): string } };