@statewalker/indexer-ngram 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mikhail Kotelnikov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,86 @@
-# Temporary Holding Version
+# @statewalker/indexer-ngram
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+## What it is
+
+The n-gram / regex modality contract for `@statewalker/indexer-api`: the block, query and result
+types, the `NGRAM_TYPE` discriminator, config helpers, a typed access handle (`newNgramAccess`) and
+a `grep` helper. It holds no engine; a backend registers a sub-index of type `"ngram"` that
+implements `NgramIndex`. `@statewalker/indexer-sqlite` (`newSqliteNgramProvider()`) is such a backend.
+
+## Why exact-match search needs its own modality
+
+Full-text search drops punctuation and stems words, so it cannot find `fooBar(`, `TODO:` or a
+regular expression. Grep needs a modality that matches the original text exactly and reports where
+each match sits (offsets and line numbers). Backends implement that behind the types in this
+package, so callers write against the contract and fuse grep hits with full-text and vector hits in
+one `SearchRequest`.
+
+## How to use
+
+```ts
+import { grep, newNgramAccess, setNgramConfig } from "@statewalker/indexer-ngram";
+```
+
+- `setNgramConfig(params, name, { store? })` declares the sub-index in `CreateIndexParams`.
+- `newNgramAccess(name).get(index)` returns the typed `NgramIndex` for ingestion
+  (`addDocument`, ...) and `search`.
+- `grep(index, name, pattern, options?)` runs a search and returns `{ hits, truncated }`.
+
+## Examples
+
+Config, ingest, grep:
+
+```ts
+import { grep, newNgramAccess, setNgramConfig } from "@statewalker/indexer-ngram";
+
+const params = { name: "code" /* ... */ };
+setNgramConfig(params, "grep", {}); // backend picks the store
+
+// `indexer` is a backend that handles type "ngram"; `index` came from it.
+const ngram = newNgramAccess("grep");
+await ngram.get(index).addDocument([
+  { path: "/src/a.ts", blockId: "0", content: "const x = fooBar(1); // TODO: fix" },
+]);
+
+const { hits, truncated } = await grep(index, "grep", "TODO:");
+// hits[0] -> { path: "/src/a.ts", blockId: "0", score: 1, count: 1,
+//              matches: [{ start: 24, end: 29, line: 1 }] }
+
+const re = await grep(index, "grep", "foo[A-Z]\\w+\\(", { regex: true, topK: 20 });
+```
+
+Fused with other modalities in one request:
+
+```ts
+import type { SearchRequest } from "@statewalker/indexer-api";
+
+const request: SearchRequest = { topK: 10 };
+ngram.setQuery(request, { pattern: "fooBar(", caseSensitive: false });
+// ... set the full-text / vector sub-queries on the same request ...
+for await (const entry of index.search(request)) {
+  const hit = ngram.getResult(entry); // NgramResult | undefined
+}
+```
+
+## Internals
+
+- **Score is the match count.** `NgramResult.score === count`; `matches` lists every match found in
+  the block.
+- **Matches never cross blocks.** A pattern spanning two blocks does not match.
+- **Offsets.** `start`/`end` are UTF-16 code-unit offsets into the original block text, `end`
+  exclusive; `line` is 1-based.
+- **Case.** `caseSensitive` defaults to `true` (grep semantics).
+- **Limits.** `topK` defaults to 100 and caps returned blocks; `topK: 0` returns no hits. A backend
+  may stop verifying candidates early (the SQLite backend's candidate budget is 10,000). Either cut
+  sets `truncated: true`.
+- **Regex dialect.** `regex: true` compiles a JavaScript regular expression without the `u` flag:
+  `.` matches one UTF-16 code unit and `\u{...}` is not a code-point escape. Patterns containing
+  `\p{` or `\P{` are rejected with a `SyntaxError`; Unicode-property escapes are not supported.
+- **`truncated` travels as the generator's return value.** `NgramIndex.search` yields hits and
+  returns `{ truncated }`. A `for await` over `search` silently ignores truncation and shows
+  possibly incomplete results as complete; use `grep`, which reads the return value.
+- **Dependencies.** Only `@statewalker/indexer-api`; no engine, no database package.
+
+## License
+
+MIT

package/dist/access.d.ts

@@ -0,0 +1,18 @@
+import { NgramConfig, NgramIndex, NgramQuery, NgramResult } from "./types.js";
+import { CreateIndexParams, Index, SearchRequest, SearchResult } from "@statewalker/indexer-api";
+//#region src/access.d.ts
+/** User-facing typed bundle bound to one n-gram sub-index **name**. */
+export interface NgramAccess {
+  readonly name: string;
+  /** Throws if no sub-index named `name` is registered on `index`. */
+  get(index: Index): NgramIndex;
+  tryGet(index: Index): NgramIndex | undefined;
+  setConfig(params: CreateIndexParams, config: NgramConfig): void;
+  getConfig(params: CreateIndexParams): NgramConfig | undefined;
+  setQuery(request: SearchRequest, query: NgramQuery): void;
+  getQuery(request: SearchRequest): NgramQuery | undefined;
+  getResult(entry: SearchResult): NgramResult | undefined;
+}
+export declare function newNgramAccess(name: string): NgramAccess;
+//#endregion
+//# sourceMappingURL=access.d.ts.map

package/dist/access.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"access.d.ts","names":[],"sources":["../src/access.ts"],"mappings":";;;;iBAWiB;WACN;;EAET,IAAI,OAAO,QAAQ;EACnB,OAAO,OAAO,QAAQ;EACtB,UAAU,QAAQ,mBAAmB,QAAQ;EAC7C,UAAU,QAAQ,oBAAoB;EACtC,SAAS,SAAS,eAAe,OAAO;EACxC,SAAS,SAAS,gBAAgB;EAClC,UAAU,OAAO,eAAe;;wBAGlB,eAAe,eAAe"}

package/dist/access.js

@@ -0,0 +1,23 @@
+import { getNgramConfig, setNgramConfig } from "./config.js";
+import { getSubQuery, getSubResult, setSubQuery } from "@statewalker/indexer-api";
+//#region src/access.ts
+function newNgramAccess(name) {
+	return {
+		name,
+		get(index) {
+			const binding = index.getBinding(name);
+			if (!binding) throw new Error(`No sub-index named "${name}" is registered on index "${index.name}"`);
+			return binding.index;
+		},
+		tryGet: (index) => index.getBinding(name)?.index,
+		setConfig: (params, config) => setNgramConfig(params, name, config),
+		getConfig: (params) => getNgramConfig(params, name),
+		setQuery: (request, query) => setSubQuery(request, name, query),
+		getQuery: (request) => getSubQuery(request, name),
+		getResult: (entry) => getSubResult(entry, name)
+	};
+}
+//#endregion
+export { newNgramAccess };
+
+//# sourceMappingURL=access.js.map

package/dist/access.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"access.js","names":[],"sources":["../src/access.ts"],"sourcesContent":["import type {\n  CreateIndexParams,\n  Index,\n  SearchRequest,\n  SearchResult,\n} from \"@statewalker/indexer-api\";\nimport { getSubQuery, getSubResult, setSubQuery } from \"@statewalker/indexer-api\";\nimport { getNgramConfig, setNgramConfig } from \"./config.js\";\nimport type { NgramConfig, NgramIndex, NgramQuery, NgramResult } from \"./types.js\";\n\n/** User-facing typed bundle bound to one n-gram sub-index **name**. */\nexport interface NgramAccess {\n  readonly name: string;\n  /** Throws if no sub-index named `name` is registered on `index`. */\n  get(index: Index): NgramIndex;\n  tryGet(index: Index): NgramIndex | undefined;\n  setConfig(params: CreateIndexParams, config: NgramConfig): void;\n  getConfig(params: CreateIndexParams): NgramConfig | undefined;\n  setQuery(request: SearchRequest, query: NgramQuery): void;\n  getQuery(request: SearchRequest): NgramQuery | undefined;\n  getResult(entry: SearchResult): NgramResult | undefined;\n}\n\nexport function newNgramAccess(name: string): NgramAccess {\n  return {\n    name,\n    get(index: Index): NgramIndex {\n      const binding = index.getBinding(name);\n      if (!binding) {\n        throw new Error(`No sub-index named \"${name}\" is registered on index \"${index.name}\"`);\n      }\n      return binding.index as NgramIndex;\n    },\n    tryGet: (index) => index.getBinding(name)?.index as NgramIndex | undefined,\n    setConfig: (params, config) => setNgramConfig(params, name, config),\n    getConfig: (params) => getNgramConfig(params, name),\n    setQuery: (request, query) => setSubQuery<NgramQuery>(request, name, query),\n    getQuery: (request) => getSubQuery<NgramQuery>(request, name),\n    getResult: (entry) => getSubResult<NgramResult>(entry, name),\n  };\n}\n"],"mappings":";;;AAuBA,SAAgB,eAAe,MAA2B;CACxD,OAAO;EACL;EACA,IAAI,OAA0B;GAC5B,MAAM,UAAU,MAAM,WAAW,IAAI;GACrC,IAAI,CAAC,SACH,MAAM,IAAI,MAAM,uBAAuB,KAAK,4BAA4B,MAAM,KAAK,EAAE;GAEvF,OAAO,QAAQ;EACjB;EACA,SAAS,UAAU,MAAM,WAAW,IAAI,CAAC,EAAE;EAC3C,YAAY,QAAQ,WAAW,eAAe,QAAQ,MAAM,MAAM;EAClE,YAAY,WAAW,eAAe,QAAQ,IAAI;EAClD,WAAW,SAAS,UAAU,YAAwB,SAAS,MAAM,KAAK;EAC1E,WAAW,YAAY,YAAwB,SAAS,IAAI;EAC5D,YAAY,UAAU,aAA0B,OAAO,IAAI;CAC7D;AACF"}

package/dist/config.d.ts

@@ -0,0 +1,9 @@
+import { NgramConfig } from "./types.js";
+import { CreateIndexParams } from "@statewalker/indexer-api";
+//#region src/config.d.ts
+/** Write an n-gram config for the sub-index named `name` into `params.subIndexes`. */
+export declare function setNgramConfig(params: CreateIndexParams, name: string, config: NgramConfig): void;
+/** Read the n-gram config for the sub-index named `name`, if any. */
+export declare function getNgramConfig(params: CreateIndexParams, name: string): NgramConfig | undefined;
+//#endregion
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","names":[],"sources":["../src/config.ts"],"mappings":";;;;wBAIgB,eAAe,QAAQ,mBAAmB,cAAc,QAAQ;;wBAMhE,eAAe,QAAQ,mBAAmB,eAAe"}

package/dist/config.js

@@ -0,0 +1,21 @@
+import { NGRAM_TYPE } from "./types.js";
+//#region src/config.ts
+/** Write an n-gram config for the sub-index named `name` into `params.subIndexes`. */
+function setNgramConfig(params, name, config) {
+	if (!params.subIndexes) params.subIndexes = {};
+	params.subIndexes[name] = {
+		type: NGRAM_TYPE,
+		...config
+	};
+}
+/** Read the n-gram config for the sub-index named `name`, if any. */
+function getNgramConfig(params, name) {
+	const entry = params.subIndexes?.[name];
+	if (!entry || entry.type !== "ngram") return void 0;
+	const { type: _type, ...rest } = entry;
+	return rest;
+}
+//#endregion
+export { getNgramConfig, setNgramConfig };
+
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","names":[],"sources":["../src/config.ts"],"sourcesContent":["import type { CreateIndexParams } from \"@statewalker/indexer-api\";\nimport { NGRAM_TYPE, type NgramConfig } from \"./types.js\";\n\n/** Write an n-gram config for the sub-index named `name` into `params.subIndexes`. */\nexport function setNgramConfig(params: CreateIndexParams, name: string, config: NgramConfig): void {\n  if (!params.subIndexes) params.subIndexes = {};\n  params.subIndexes[name] = { type: NGRAM_TYPE, ...config };\n}\n\n/** Read the n-gram config for the sub-index named `name`, if any. */\nexport function getNgramConfig(params: CreateIndexParams, name: string): NgramConfig | undefined {\n  const entry = params.subIndexes?.[name];\n  if (!entry || entry.type !== NGRAM_TYPE) return undefined;\n  const { type: _type, ...rest } = entry;\n  return rest as NgramConfig;\n}\n"],"mappings":";;;AAIA,SAAgB,eAAe,QAA2B,MAAc,QAA2B;CACjG,IAAI,CAAC,OAAO,YAAY,OAAO,aAAa,CAAC;CAC7C,OAAO,WAAW,QAAQ;EAAE,MAAM;EAAY,GAAG;CAAO;AAC1D;;AAGA,SAAgB,eAAe,QAA2B,MAAuC;CAC/F,MAAM,QAAQ,OAAO,aAAa;CAClC,IAAI,CAAC,SAAS,MAAM,SAAA,SAAqB,OAAO,KAAA;CAChD,MAAM,EAAE,MAAM,OAAO,GAAG,SAAS;CACjC,OAAO;AACT"}

package/dist/grep.d.ts

@@ -0,0 +1,13 @@
+import { NgramQuery, NgramResult } from "./types.js";
+import { Index } from "@statewalker/indexer-api";
+//#region src/grep.d.ts
+export type GrepOptions = Omit<NgramQuery, "pattern">;
+export interface GrepResult {
+  hits: NgramResult[];
+  /** `true` when the backend stopped early (candidate budget or `topK`); `hits` may be incomplete. */
+  truncated: boolean;
+}
+/** Run `pattern` on the n-gram sub-index `name` of `index` and collect every hit plus the truncation flag. */
+export declare function grep(index: Index, name: string, pattern: string, options?: GrepOptions): Promise<GrepResult>;
+//#endregion
+//# sourceMappingURL=grep.d.ts.map

package/dist/grep.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grep.d.ts","names":[],"sources":["../src/grep.ts"],"mappings":";;;YAIY,cAAc,KAAK;iBAEd;EACf,MAAM;;EAEN;;;wBAIoB,KACpB,OAAO,OACP,cACA,iBACA,UAAS,cACR,QAAQ"}

package/dist/grep.js

@@ -0,0 +1,22 @@
+import { newNgramAccess } from "./access.js";
+//#region src/grep.ts
+/** Run `pattern` on the n-gram sub-index `name` of `index` and collect every hit plus the truncation flag. */
+async function grep(index, name, pattern, options = {}) {
+	const it = newNgramAccess(name).get(index).search({
+		...options,
+		pattern
+	});
+	const hits = [];
+	for (;;) {
+		const step = await it.next();
+		if (step.done) return {
+			hits,
+			truncated: step.value?.truncated ?? false
+		};
+		hits.push(step.value);
+	}
+}
+//#endregion
+export { grep };
+
+//# sourceMappingURL=grep.js.map

package/dist/grep.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grep.js","names":[],"sources":["../src/grep.ts"],"sourcesContent":["import type { Index } from \"@statewalker/indexer-api\";\nimport { newNgramAccess } from \"./access.js\";\nimport type { NgramQuery, NgramResult } from \"./types.js\";\n\nexport type GrepOptions = Omit<NgramQuery, \"pattern\">;\n\nexport interface GrepResult {\n  hits: NgramResult[];\n  /** `true` when the backend stopped early (candidate budget or `topK`); `hits` may be incomplete. */\n  truncated: boolean;\n}\n\n/** Run `pattern` on the n-gram sub-index `name` of `index` and collect every hit plus the truncation flag. */\nexport async function grep(\n  index: Index,\n  name: string,\n  pattern: string,\n  options: GrepOptions = {},\n): Promise<GrepResult> {\n  const it = newNgramAccess(name)\n    .get(index)\n    .search({ ...options, pattern });\n  const hits: NgramResult[] = [];\n  for (;;) {\n    const step = await it.next();\n    if (step.done) return { hits, truncated: step.value?.truncated ?? false };\n    hits.push(step.value);\n  }\n}\n"],"mappings":";;;AAaA,eAAsB,KACpB,OACA,MACA,SACA,UAAuB,CAAC,GACH;CACrB,MAAM,KAAK,eAAe,IAAI,CAAC,CAC5B,IAAI,KAAK,CAAC,CACV,OAAO;EAAE,GAAG;EAAS;CAAQ,CAAC;CACjC,MAAM,OAAsB,CAAC;CAC7B,SAAS;EACP,MAAM,OAAO,MAAM,GAAG,KAAK;EAC3B,IAAI,KAAK,MAAM,OAAO;GAAE;GAAM,WAAW,KAAK,OAAO,aAAa;EAAM;EACxE,KAAK,KAAK,KAAK,KAAK;CACtB;AACF"}

package/dist/index.d.ts

@@ -0,0 +1,5 @@
+import { NGRAM_TYPE, NgramBlock, NgramConfig, NgramIndex, NgramIndexInfo, NgramMatch, NgramQuery, NgramResult, NgramSearchSummary } from "./types.js";
+import { NgramAccess, newNgramAccess } from "./access.js";
+import { getNgramConfig, setNgramConfig } from "./config.js";
+import { GrepOptions, GrepResult, grep } from "./grep.js";
+export { type GrepOptions, type GrepResult, NGRAM_TYPE, type NgramAccess, type NgramBlock, type NgramConfig, type NgramIndex, type NgramIndexInfo, type NgramMatch, type NgramQuery, type NgramResult, type NgramSearchSummary, getNgramConfig, grep, newNgramAccess, setNgramConfig };

package/dist/index.js

@@ -0,0 +1,5 @@
+import { NGRAM_TYPE } from "./types.js";
+import { getNgramConfig, setNgramConfig } from "./config.js";
+import { newNgramAccess } from "./access.js";
+import { grep } from "./grep.js";
+export { NGRAM_TYPE, getNgramConfig, grep, newNgramAccess, setNgramConfig };

package/dist/types.d.ts

@@ -0,0 +1,63 @@
+import { BlockReference, DocumentPath, Metadata, ScoredHit, SearchIndex } from "@statewalker/indexer-api";
+//#region src/types.d.ts
+/** Stable modality-type discriminator for the n-gram (grep) modality. */
+export declare const NGRAM_TYPE: "ngram";
+/** A block of text for the n-gram sub-index. Matches never cross block boundaries. */
+export interface NgramBlock extends BlockReference {
+  content: string;
+  metadata?: Metadata;
+}
+/**
+ * N-gram sub-query: an exact pattern, literal by default.
+ *
+ * `regex: true` reads `pattern` as a JavaScript regular expression compiled
+ * without the `u` flag (`.` is one UTF-16 code unit; `\u{…}` is not a
+ * code-point escape); patterns containing `\p{` / `\P{` are rejected with a
+ * `SyntaxError`. `caseSensitive` defaults to `true` (grep semantics). `topK`
+ * (default 100) caps the returned blocks; `0` returns no hits. `topK` / `paths`
+ * override the top-level request values for this sub-index. A backend may stop
+ * early (a candidate budget — 10,000 in the SQLite backend); the search then
+ * reports `truncated: true`, as it does when `topK` cut the result short.
+ */
+export interface NgramQuery {
+  pattern: string;
+  regex?: boolean;
+  caseSensitive?: boolean;
+  topK?: number;
+  paths?: DocumentPath[];
+}
+/** One match: UTF-16 offsets into the block's content, `end` exclusive; `line` is 1-based. */
+export interface NgramMatch {
+  start: number;
+  end: number;
+  line: number;
+}
+/** A block with at least one match. `score` equals `count`. */
+export interface NgramResult extends ScoredHit {
+  count: number;
+  matches: NgramMatch[];
+}
+/** Returned by `NgramIndex.search` when the generator finishes. */
+export interface NgramSearchSummary {
+  /** `true` when the candidate budget or `topK` cut the search short — results may be incomplete. */
+  truncated: boolean;
+}
+export interface NgramIndexInfo {
+  store: "fts5" | "postings";
+  metadata?: Metadata;
+}
+/**
+ * N-gram / regex sub-index. `search` yields hits and *returns* a summary;
+ * `for await` drops the summary — use {@link grep} to read it.
+ */
+export interface NgramIndex extends SearchIndex<NgramBlock, NgramQuery, NgramResult> {
+  search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary | undefined>;
+  getIndexInfo(): Promise<NgramIndexInfo>;
+}
+/** Creation config. `store` picks the backend's candidate store; backends choose the default. */
+export interface NgramConfig {
+  store?: "fts5" | "postings";
+  metadata?: Metadata;
+}
+//#endregion
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;qBASa;;iBAGI,mBAAmB;EAClC;EACA,WAAW;;;;;;;;;;;;;;iBAeI;EACf;EACA;EACA;EACA;EACA,QAAQ;;;iBAIO;EACf;EACA;EACA;;;iBAIe,oBAAoB;EACnC;EACA,SAAS;;;iBAIM;;EAEf;;iBAGe;EACf;EACA,WAAW;;;;;;iBAOI,mBAAmB,YAAY,YAAY,YAAY;EACtE,OAAO,QAAQ,aAAa,eAAe,aAAa;EACxD,gBAAgB,QAAQ;;;iBAIT;EACf;EACA,WAAW"}

package/dist/types.js

@@ -0,0 +1,7 @@
+//#region src/types.ts
+/** Stable modality-type discriminator for the n-gram (grep) modality. */
+const NGRAM_TYPE = "ngram";
+//#endregion
+export { NGRAM_TYPE };
+
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","names":[],"sources":["../src/types.ts"],"sourcesContent":["import type {\n  BlockReference,\n  DocumentPath,\n  Metadata,\n  ScoredHit,\n  SearchIndex,\n} from \"@statewalker/indexer-api\";\n\n/** Stable modality-type discriminator for the n-gram (grep) modality. */\nexport const NGRAM_TYPE = \"ngram\" as const;\n\n/** A block of text for the n-gram sub-index. Matches never cross block boundaries. */\nexport interface NgramBlock extends BlockReference {\n  content: string;\n  metadata?: Metadata;\n}\n\n/**\n * N-gram sub-query: an exact pattern, literal by default.\n *\n * `regex: true` reads `pattern` as a JavaScript regular expression compiled\n * without the `u` flag (`.` is one UTF-16 code unit; `\\u{…}` is not a\n * code-point escape); patterns containing `\\p{` / `\\P{` are rejected with a\n * `SyntaxError`. `caseSensitive` defaults to `true` (grep semantics). `topK`\n * (default 100) caps the returned blocks; `0` returns no hits. `topK` / `paths`\n * override the top-level request values for this sub-index. A backend may stop\n * early (a candidate budget — 10,000 in the SQLite backend); the search then\n * reports `truncated: true`, as it does when `topK` cut the result short.\n */\nexport interface NgramQuery {\n  pattern: string;\n  regex?: boolean;\n  caseSensitive?: boolean;\n  topK?: number;\n  paths?: DocumentPath[];\n}\n\n/** One match: UTF-16 offsets into the block's content, `end` exclusive; `line` is 1-based. */\nexport interface NgramMatch {\n  start: number;\n  end: number;\n  line: number;\n}\n\n/** A block with at least one match. `score` equals `count`. */\nexport interface NgramResult extends ScoredHit {\n  count: number;\n  matches: NgramMatch[];\n}\n\n/** Returned by `NgramIndex.search` when the generator finishes. */\nexport interface NgramSearchSummary {\n  /** `true` when the candidate budget or `topK` cut the search short — results may be incomplete. */\n  truncated: boolean;\n}\n\nexport interface NgramIndexInfo {\n  store: \"fts5\" | \"postings\";\n  metadata?: Metadata;\n}\n\n/**\n * N-gram / regex sub-index. `search` yields hits and *returns* a summary;\n * `for await` drops the summary — use {@link grep} to read it.\n */\nexport interface NgramIndex extends SearchIndex<NgramBlock, NgramQuery, NgramResult> {\n  search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary | undefined>;\n  getIndexInfo(): Promise<NgramIndexInfo>;\n}\n\n/** Creation config. `store` picks the backend's candidate store; backends choose the default. */\nexport interface NgramConfig {\n  store?: \"fts5\" | \"postings\";\n  metadata?: Metadata;\n}\n"],"mappings":";;AASA,MAAa,aAAa"}

package/package.json

@@ -1,6 +1,54 @@
 {
   "name": "@statewalker/indexer-ngram",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "version": "0.1.0",
+  "private": false,
+  "type": "module",
+  "description": "N-gram / regex (grep) modality contract for @statewalker/indexer-api: types, typed access handle and a grep helper.",
+  "bugs": {
+    "url": "https://github.com/statewalker/statewalker-search/issues"
+  },
+  "homepage": "https://github.com/statewalker/statewalker-search/tree/main/packages/indexer-ngram#readme",
+  "author": {
+    "name": "Mikhail Kotelnikov",
+    "email": "mikhail.kotelnikov@gmail.com"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/statewalker/statewalker-search.git",
+    "directory": "packages/indexer-ngram"
+  },
+  "exports": {
+    ".": {
+      "source": "./src/index.ts",
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@statewalker/indexer-api": "^0.1.3"
+  },
+  "devDependencies": {
+    "rimraf": "^6.1.3",
+    "tsdown": "^0.23.0",
+    "typescript": "^7.0.2",
+    "vitest": "^5.0.3"
+  },
+  "sideEffects": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsdown",
+    "test": "vitest run --passWithNoTests",
+    "typecheck": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "lint": "biome check --write .",
+    "format": "biome format --write ."
+  }
 }

package/src/access.ts

@@ -0,0 +1,41 @@
+import type {
+  CreateIndexParams,
+  Index,
+  SearchRequest,
+  SearchResult,
+} from "@statewalker/indexer-api";
+import { getSubQuery, getSubResult, setSubQuery } from "@statewalker/indexer-api";
+import { getNgramConfig, setNgramConfig } from "./config.js";
+import type { NgramConfig, NgramIndex, NgramQuery, NgramResult } from "./types.js";
+
+/** User-facing typed bundle bound to one n-gram sub-index **name**. */
+export interface NgramAccess {
+  readonly name: string;
+  /** Throws if no sub-index named `name` is registered on `index`. */
+  get(index: Index): NgramIndex;
+  tryGet(index: Index): NgramIndex | undefined;
+  setConfig(params: CreateIndexParams, config: NgramConfig): void;
+  getConfig(params: CreateIndexParams): NgramConfig | undefined;
+  setQuery(request: SearchRequest, query: NgramQuery): void;
+  getQuery(request: SearchRequest): NgramQuery | undefined;
+  getResult(entry: SearchResult): NgramResult | undefined;
+}
+
+export function newNgramAccess(name: string): NgramAccess {
+  return {
+    name,
+    get(index: Index): NgramIndex {
+      const binding = index.getBinding(name);
+      if (!binding) {
+        throw new Error(`No sub-index named "${name}" is registered on index "${index.name}"`);
+      }
+      return binding.index as NgramIndex;
+    },
+    tryGet: (index) => index.getBinding(name)?.index as NgramIndex | undefined,
+    setConfig: (params, config) => setNgramConfig(params, name, config),
+    getConfig: (params) => getNgramConfig(params, name),
+    setQuery: (request, query) => setSubQuery<NgramQuery>(request, name, query),
+    getQuery: (request) => getSubQuery<NgramQuery>(request, name),
+    getResult: (entry) => getSubResult<NgramResult>(entry, name),
+  };
+}

package/src/config.ts

@@ -0,0 +1,16 @@
+import type { CreateIndexParams } from "@statewalker/indexer-api";
+import { NGRAM_TYPE, type NgramConfig } from "./types.js";
+
+/** Write an n-gram config for the sub-index named `name` into `params.subIndexes`. */
+export function setNgramConfig(params: CreateIndexParams, name: string, config: NgramConfig): void {
+  if (!params.subIndexes) params.subIndexes = {};
+  params.subIndexes[name] = { type: NGRAM_TYPE, ...config };
+}
+
+/** Read the n-gram config for the sub-index named `name`, if any. */
+export function getNgramConfig(params: CreateIndexParams, name: string): NgramConfig | undefined {
+  const entry = params.subIndexes?.[name];
+  if (!entry || entry.type !== NGRAM_TYPE) return undefined;
+  const { type: _type, ...rest } = entry;
+  return rest as NgramConfig;
+}

package/src/grep.ts

@@ -0,0 +1,29 @@
+import type { Index } from "@statewalker/indexer-api";
+import { newNgramAccess } from "./access.js";
+import type { NgramQuery, NgramResult } from "./types.js";
+
+export type GrepOptions = Omit<NgramQuery, "pattern">;
+
+export interface GrepResult {
+  hits: NgramResult[];
+  /** `true` when the backend stopped early (candidate budget or `topK`); `hits` may be incomplete. */
+  truncated: boolean;
+}
+
+/** Run `pattern` on the n-gram sub-index `name` of `index` and collect every hit plus the truncation flag. */
+export async function grep(
+  index: Index,
+  name: string,
+  pattern: string,
+  options: GrepOptions = {},
+): Promise<GrepResult> {
+  const it = newNgramAccess(name)
+    .get(index)
+    .search({ ...options, pattern });
+  const hits: NgramResult[] = [];
+  for (;;) {
+    const step = await it.next();
+    if (step.done) return { hits, truncated: step.value?.truncated ?? false };
+    hits.push(step.value);
+  }
+}

package/src/index.ts

@@ -0,0 +1,14 @@
+export { type NgramAccess, newNgramAccess } from "./access.js";
+export { getNgramConfig, setNgramConfig } from "./config.js";
+export { type GrepOptions, type GrepResult, grep } from "./grep.js";
+export {
+  NGRAM_TYPE,
+  type NgramBlock,
+  type NgramConfig,
+  type NgramIndex,
+  type NgramIndexInfo,
+  type NgramMatch,
+  type NgramQuery,
+  type NgramResult,
+  type NgramSearchSummary,
+} from "./types.js";

package/src/types.ts

@@ -0,0 +1,75 @@
+import type {
+  BlockReference,
+  DocumentPath,
+  Metadata,
+  ScoredHit,
+  SearchIndex,
+} from "@statewalker/indexer-api";
+
+/** Stable modality-type discriminator for the n-gram (grep) modality. */
+export const NGRAM_TYPE = "ngram" as const;
+
+/** A block of text for the n-gram sub-index. Matches never cross block boundaries. */
+export interface NgramBlock extends BlockReference {
+  content: string;
+  metadata?: Metadata;
+}
+
+/**
+ * N-gram sub-query: an exact pattern, literal by default.
+ *
+ * `regex: true` reads `pattern` as a JavaScript regular expression compiled
+ * without the `u` flag (`.` is one UTF-16 code unit; `\u{…}` is not a
+ * code-point escape); patterns containing `\p{` / `\P{` are rejected with a
+ * `SyntaxError`. `caseSensitive` defaults to `true` (grep semantics). `topK`
+ * (default 100) caps the returned blocks; `0` returns no hits. `topK` / `paths`
+ * override the top-level request values for this sub-index. A backend may stop
+ * early (a candidate budget — 10,000 in the SQLite backend); the search then
+ * reports `truncated: true`, as it does when `topK` cut the result short.
+ */
+export interface NgramQuery {
+  pattern: string;
+  regex?: boolean;
+  caseSensitive?: boolean;
+  topK?: number;
+  paths?: DocumentPath[];
+}
+
+/** One match: UTF-16 offsets into the block's content, `end` exclusive; `line` is 1-based. */
+export interface NgramMatch {
+  start: number;
+  end: number;
+  line: number;
+}
+
+/** A block with at least one match. `score` equals `count`. */
+export interface NgramResult extends ScoredHit {
+  count: number;
+  matches: NgramMatch[];
+}
+
+/** Returned by `NgramIndex.search` when the generator finishes. */
+export interface NgramSearchSummary {
+  /** `true` when the candidate budget or `topK` cut the search short — results may be incomplete. */
+  truncated: boolean;
+}
+
+export interface NgramIndexInfo {
+  store: "fts5" | "postings";
+  metadata?: Metadata;
+}
+
+/**
+ * N-gram / regex sub-index. `search` yields hits and *returns* a summary;
+ * `for await` drops the summary — use {@link grep} to read it.
+ */
+export interface NgramIndex extends SearchIndex<NgramBlock, NgramQuery, NgramResult> {
+  search(params: NgramQuery): AsyncGenerator<NgramResult, NgramSearchSummary | undefined>;
+  getIndexInfo(): Promise<NgramIndexInfo>;
+}
+
+/** Creation config. `store` picks the backend's candidate store; backends choose the default. */
+export interface NgramConfig {
+  store?: "fts5" | "postings";
+  metadata?: Metadata;
+}