okengine 0.18.4 → 0.19.0

package/src/elements/store/resource.ts

@@ -13,6 +13,7 @@
  */
 
 import { flow, type AnyFlowDef, type FlowDef, type FlowErrorMap } from "../../kernel/flow.ts";
+import { applyBoundaryContract, stampBoundaryContract } from "../../kernel/boundary-contract.ts";
 import { fail } from "../../kernel/errors.ts";
 import type { Fx } from "../../kernel/fx.ts";
 import {
@@ -260,11 +261,9 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
   }
 
   const listFlow = flow("list", {
-    ...(breaking ? { breaking: true as const } : {}),
     // Loose record so the HTTP AoT infers `query` and lets every list URL
     // key through; real validation happens in parseListQuery (PostgREST
     // grammar).
-    in: z.record(z.string(), z.unknown()) as never,
     effects: { reads: [db.ref] },
     do: async (input, fx) => {
       const result = await runList(input, fx);
@@ -272,10 +271,15 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       return fx.json.with(result);
     },
   });
+  applyBoundaryContract(
+    listFlow,
+    stampBoundaryContract({
+      in: z.record(z.string(), z.unknown()) as never,
+      ...(breaking ? { breaking: true } : {}),
+    }),
+  );
 
   const createFlow = flow("create", {
-    ...(breaking ? { breaking: true as const } : {}),
-    in: options.in as never,
     effects: { writes: [db.ref] },
     do: async (input, fx) => {
       const store = fx.store(db) as {
@@ -288,10 +292,16 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       return fx.json.create(row);
     },
   });
+  applyBoundaryContract(
+    createFlow,
+    stampBoundaryContract({
+      in: options.in as never,
+      out: options.out as never,
+      ...(breaking ? { breaking: true } : {}),
+    }),
+  );
 
   const getFlow = flow("get", {
-    ...(breaking ? { breaking: true as const } : {}),
-    errors,
     effects: { reads: [db.ref] },
     do: async (input, fx) => {
       const id = (input as Record<string, unknown>)[idKey];
@@ -303,6 +313,14 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       return row;
     },
   });
+  applyBoundaryContract(
+    getFlow,
+    stampBoundaryContract({
+      out: options.out as never,
+      errors,
+      ...(breaking ? { breaking: true } : {}),
+    }),
+  );
 
   // Wire update body is `{ id, ...patch }`. The patch schema (`update`,
   // default `in`) describes the mutable fields; the path id rides along and
@@ -315,9 +333,6 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
         ? patchSchema.extend({ [idKey]: z.string() })
         : patchSchema;
   const updateFlow = flow("update", {
-    ...(breaking ? { breaking: true as const } : {}),
-    in: updateIn as never,
-    errors,
     effects: { reads: [db.ref], writes: [db.ref] },
     do: async (input, fx) => {
       const { [idKey]: id, ...patch } = input as Record<string, unknown>;
@@ -338,10 +353,17 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       return row;
     },
   });
+  applyBoundaryContract(
+    updateFlow,
+    stampBoundaryContract({
+      in: updateIn as never,
+      out: options.out as never,
+      errors,
+      ...(breaking ? { breaking: true } : {}),
+    }),
+  );
 
   const removeFlow = flow("remove", {
-    ...(breaking ? { breaking: true as const } : {}),
-    errors,
     effects: { writes: [db.ref] },
     do: async (input, fx) => {
       const id = (input as Record<string, unknown>)[idKey];
@@ -353,6 +375,13 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       return fx.json.empty();
     },
   });
+  applyBoundaryContract(
+    removeFlow,
+    stampBoundaryContract({
+      errors,
+      ...(breaking ? { breaking: true } : {}),
+    }),
+  );
 
   const defs: ResourceFlowDefs = {
     list: listFlow as FlowDef<any, any, any>,
@@ -389,7 +418,6 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
       if (kind !== undefined) columnKinds[c.key] = kind;
     }
     const liveFlow = flow(`_live_${tableName}`, {
-      ...(options.out !== undefined ? { out: options.out } : {}),
       effects: { reads: [db.ref, `signal:${signalName}`] },
       do: async (input: unknown, fx: Fx) => {
         const bridge = realtimeBridgeRuntime();
@@ -428,6 +456,9 @@ export function resource(db: SqlStoreDecl, table: unknown, options: ResourceOpti
         return fx.json.stream(stream.chunks);
       },
     }) as unknown as AnyFlowDef;
+    if (options.out !== undefined) {
+      applyBoundaryContract(liveFlow, stampBoundaryContract({ out: options.out as never }));
+    }
     return { signal: signalName, flow: liveFlow };
   };
 

package/src/elements/store/schema-decl.ts

@@ -167,6 +167,8 @@ export interface SchemaColumnDecl<
   readonly typeOptions?: FieldTypeOptions;
   /** Optional human description for Console / docs (falls back to the JS key). */
   readonly description?: string;
+  /** Hybrid search (BM25 ± LSH) when `.searchable()` / `.embed()` was chained. */
+  readonly search?: ColumnSearchDecl;
   /**
    * Drizzle `SQLWrapper` structural match — never invoked at runtime.
    *
@@ -215,6 +217,45 @@ export interface SchemaLiveDecl {
   readonly live: false;
 }
 
+/**
+ * Options for {@link FieldBuilder.searchable}.
+ *
+ * `weight` is a BM25F field weight applied to term frequency **before**
+ * saturation / IDF — pure SQL math, zero AI cost. Default `1`.
+ */
+export interface SearchableOptions {
+  readonly weight?: number;
+}
+
+/**
+ * Options for {@link FieldBuilder.embed} — separate from {@link SearchableOptions}
+ * because embedding is an async, costed AI pipeline.
+ *
+ * `model` / `dims` may be omitted when the project supplies
+ * `oke({ store: { search: { embed: { model, dims } } } })`. Per-field values
+ * override the project default.
+ */
+export interface EmbedFieldOptions {
+  /** Embedding model handle or name (inherits project default when omitted). */
+  readonly model?: { readonly name: string } | string;
+  /**
+   * Vector dimensionality (inherits project default when omitted).
+   * Mismatch with a computed embedding fails loud at write time.
+   */
+  readonly dims?: number;
+}
+
+/** Column-level hybrid search declaration stamped on {@link SchemaColumnDecl}. */
+export interface ColumnSearchDecl {
+  readonly searchable: true;
+  readonly weight: number;
+  readonly embed?: {
+    readonly model?: string;
+    /** Present after extract resolution; may be deferred to project default at declare time. */
+    readonly dims?: number;
+  };
+}
+
 /** Third-arg extra for {@link schemaTable}. */
 export type SchemaTableExtra =
   | SchemaPolicyDecl
@@ -299,6 +340,23 @@ export interface FieldBuilder<TData = unknown, TNotNull extends boolean = false>
   pii(): FieldBuilder<TData, TNotNull>;
   sensitive(): FieldBuilder<TData, TNotNull>;
   retain(duration: string): FieldBuilder<TData, TNotNull>;
+  /**
+   * Participate in built-in BM25 full-text ranking on the primary SQL table.
+   * Zero AI dependency. Optional {@link SearchableOptions.weight} is BM25F
+   * field weight (default `1`).
+   *
+   * Chain `.embed()` separately when semantic LSH is also required.
+   */
+  searchable(options?: SearchableOptions): FieldBuilder<TData, TNotNull>;
+  /**
+   * Opt into async embedding + LSH for this searchable field.
+   * Requires a prior `.searchable()` and a configured `ai` element.
+   * Never a boolean flag inside searchable options — cost/mechanism differ.
+   *
+   * Omit `model` / `dims` to inherit `oke({ store: { search: { embed } } })`.
+   * Pass either to override the project default for this field only.
+   */
+  embed(options?: EmbedFieldOptions): FieldBuilder<TData, TNotNull>;
   /** Override snake_case SQL name. */
   as(sqlName: string): FieldBuilder<TData, TNotNull>;
   /** Optional human description for Console / docs (falls back to the JS key). */
@@ -342,8 +400,11 @@ interface FieldState {
   readonly description?: string;
   readonly references?: ColumnReference;
   readonly typeOptions?: FieldTypeOptions;
+  readonly search?: ColumnSearchDecl;
 }
 
+const TEXT_SEARCHABLE_TYPES = new Set<FieldSqlType>(["text", "varchar", "char"]);
+
 function camelToSnake(key: string): string {
   return key
     .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
@@ -428,6 +489,61 @@ function createBuilder<TData, TNotNull extends boolean>(
       next<TNotNull>({
         classification: mergeClassification(state.classification, { retain: duration }),
       }),
+    searchable: (options) => {
+      if (!TEXT_SEARCHABLE_TYPES.has(state.sqlType)) {
+        throw new Error(
+          `field.${state.sqlType}().searchable() is only valid on text / varchar / char columns`,
+        );
+      }
+      const weight = options?.weight ?? 1;
+      if (!(typeof weight === "number" && Number.isFinite(weight) && weight > 0)) {
+        throw new Error(
+          `searchable({ weight }) must be a finite number > 0 (got ${String(weight)})`,
+        );
+      }
+      return next<TNotNull>({
+        search: {
+          searchable: true,
+          weight,
+          ...(state.search?.embed ? { embed: state.search.embed } : {}),
+        },
+      });
+    },
+    embed: (options) => {
+      if (!state.search?.searchable) {
+        throw new Error(
+          `.embed() requires a prior .searchable() on the same field — weight is free SQL math; embed is an async AI pipeline`,
+        );
+      }
+      if (!TEXT_SEARCHABLE_TYPES.has(state.sqlType)) {
+        throw new Error(
+          `field.${state.sqlType}().embed() is only valid on text / varchar / char columns`,
+        );
+      }
+      const dims = options?.dims;
+      if (dims !== undefined && !(typeof dims === "number" && Number.isInteger(dims) && dims > 0)) {
+        throw new Error(
+          `.embed({ dims }) requires a positive integer when set (got ${String(dims)})`,
+        );
+      }
+      const modelRaw = options?.model;
+      const model =
+        typeof modelRaw === "string"
+          ? modelRaw
+          : modelRaw && typeof modelRaw === "object" && "name" in modelRaw
+            ? String(modelRaw.name)
+            : undefined;
+      return next<TNotNull>({
+        search: {
+          searchable: true,
+          weight: state.search.weight,
+          embed: {
+            ...(dims !== undefined ? { dims } : {}),
+            ...(model !== undefined ? { model } : {}),
+          },
+        },
+      });
+    },
     as: (sqlName) => next<TNotNull>({ sqlName }),
     describe: (description) => next<TNotNull>({ description }),
     type: () => createBuilder<unknown, TNotNull>(state) as FieldBuilder<never, TNotNull> as never,
@@ -450,6 +566,7 @@ function createBuilder<TData, TNotNull extends boolean>(
         ...(state.defaultFnKind ? { defaultFnKind: state.defaultFnKind } : {}),
         ...(state.classification ? { classification: state.classification } : {}),
         ...(state.description !== undefined ? { description: state.description } : {}),
+        ...(state.search ? { search: state.search } : {}),
         ...(state.references ? { references: state.references } : {}),
         ...(state.typeOptions !== undefined && Object.keys(state.typeOptions).length > 0
           ? { typeOptions: state.typeOptions }

package/src/elements/store/search-backfill.ts

@@ -0,0 +1,205 @@
+/**
+ * `oke db search-backfill` — explicit, resumable corpus + embedding backfill.
+ * Never runs as a side effect of `oke db push`.
+ */
+
+import type { SqlConnection } from "../../drivers/types.ts";
+import type { DeclaredColumn, Manifest } from "../../manifest/types.ts";
+import { SEARCH_LOW_CORPUS_WARN_N } from "./search-errors.ts";
+import {
+  ensureHyperplaneInserts,
+  OKE_SEARCH_DF,
+  OKE_SEARCH_STATS,
+  OKE_TSV_COL,
+  searchDdlForTable,
+} from "./search-ddl.ts";
+import { tokenize } from "./search-bm25.ts";
+import { embColumn, lshColumn, OKE_SEARCH_PLANES } from "./search-ddl.ts";
+import { deserializePlanes, lshBucket, lshBucketToSql } from "./search-lsh.ts";
+import { LSH_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
+
+export interface SearchBackfillOptions {
+  readonly table: string;
+  readonly batchSize?: number;
+  /** Sleep between embed batches (ms) — rate-limit pacing. */
+  readonly embedPauseMs?: number;
+  readonly embed?: (model: string, text: string, dims: number) => Promise<readonly number[]>;
+  /**
+   * Optional abort — mid-backfill kill. Re-running the same command is safe
+   * (idempotent UPDATEs + DF/stats rebuild) and completes the remaining work.
+   */
+  readonly signal?: AbortSignal;
+}
+
+export interface SearchBackfillResult {
+  readonly table: string;
+  readonly rows: number;
+  readonly embedded: number;
+  readonly warnedLowCorpus: boolean;
+}
+
+/**
+ * Apply search DDL + rebuild stats/DF + optional embeddings for one table.
+ * Cursor-paginated; caller may wrap steps in Journal for resumability.
+ *
+ * @param conn - SQL connection
+ * @param manifest - Project manifest
+ * @param options - Table + batching
+ */
+export async function runSearchBackfill(
+  conn: SqlConnection,
+  manifest: Manifest,
+  options: SearchBackfillOptions,
+): Promise<SearchBackfillResult> {
+  const tableMeta = findTable(manifest, options.table);
+  if (!tableMeta) {
+    throw new Error(`search-backfill: table "${options.table}" not found in Manifest`);
+  }
+  const { columns, pk } = tableMeta;
+  const batchSize = options.batchSize ?? 32;
+  const pauseMs = options.embedPauseMs ?? 100;
+
+  for (const stmt of searchDdlForTable(options.table, columns)) {
+    await conn.exec(stmt);
+  }
+  for (const ins of ensureHyperplaneInserts(options.table, columns)) {
+    await conn.exec(ins.sql, ins.params);
+  }
+
+  // Rebuild DF + stats from full scan (pg_textsearch lesson: after data load).
+  await conn.exec(`DELETE FROM ${OKE_SEARCH_DF} WHERE table_name = ?`, [options.table]);
+  const df = new Map<string, number>();
+  let n = 0;
+  let totalLen = 0;
+  let cursor: string | null = null;
+  let embedded = 0;
+
+  const embedCols = Object.entries(columns)
+    .filter(([, c]) => c.embed)
+    .map(([key, c]) => ({
+      sqlName: c.sqlName ?? key,
+      dims: c.embed!.dims,
+      model: c.embed!.model ?? "default",
+    }));
+
+  for (;;) {
+    if (options.signal?.aborted) {
+      throw new DOMException("search-backfill aborted", "AbortError");
+    }
+    const params: unknown[] = [];
+    let sql = `SELECT * FROM ${options.table}`;
+    if (cursor !== null) {
+      sql += ` WHERE ${pk} > ?`;
+      params.push(cursor);
+    }
+    sql += ` ORDER BY ${pk} ASC LIMIT ${batchSize}`;
+    const rows = await conn.query(sql, params);
+    if (rows.length === 0) break;
+
+    for (const row of rows) {
+      if (options.signal?.aborted) {
+        throw new DOMException("search-backfill aborted", "AbortError");
+      }
+      n += 1;
+      const seenTerms = new Set<string>();
+      let docLen = 0;
+      for (const [key, col] of Object.entries(columns)) {
+        if (!col.searchable) continue;
+        const sqlName = col.sqlName ?? key;
+        const tokens = tokenize(String(row[sqlName] ?? ""));
+        docLen += tokens.length;
+        for (const t of new Set(tokens)) {
+          if (!seenTerms.has(t)) {
+            seenTerms.add(t);
+            df.set(t, (df.get(t) ?? 0) + 1);
+          }
+        }
+      }
+      totalLen += docLen;
+
+      if (embedCols.length > 0 && options.embed) {
+        const id = String(row[pk] ?? "");
+        for (const col of embedCols) {
+          const text = String(row[col.sqlName] ?? "");
+          const vector = await options.embed(col.model, text, col.dims);
+          if (vector.length !== col.dims) {
+            throw new SearchConfigError(
+              options.table,
+              col.sqlName,
+              `embedding length ${vector.length} !== declared dims ${col.dims}`,
+            );
+          }
+          const planesRows = await conn.query(
+            `SELECT k, planes FROM ${OKE_SEARCH_PLANES} WHERE table_name = ? AND column_name = ?`,
+            [options.table, col.sqlName],
+          );
+          const prow = planesRows[0];
+          if (!prow) {
+            throw new SearchConfigError(options.table, col.sqlName, "missing hyperplanes");
+          }
+          const k = Number(prow["k"] ?? LSH_DEFAULT_K);
+          const planes = deserializePlanes(Buffer.from(prow["planes"] as Buffer), k);
+          const bucket = lshBucket(vector, planes);
+          await conn.exec(
+            `UPDATE ${options.table} SET ${embColumn(col.sqlName)} = ?::real[], ${lshColumn(col.sqlName)} = ? WHERE ${pk} = ?`,
+            [`{${Array.from(vector).join(",")}}`, lshBucketToSql(bucket), id],
+          );
+          embedded += 1;
+        }
+      }
+      cursor = String(row[pk] ?? cursor);
+    }
+
+    if (embedCols.length > 0 && options.embed && pauseMs > 0) {
+      await new Promise((r) => setTimeout(r, pauseMs));
+    }
+    if (rows.length < batchSize) break;
+  }
+
+  const avgdl = n > 0 ? totalLen / n : 0;
+  await conn.exec(
+    `INSERT INTO ${OKE_SEARCH_STATS} (table_name, n, avgdl, updated_at)
+     VALUES (?, ?, ?, now())
+     ON CONFLICT (table_name) DO UPDATE SET n = EXCLUDED.n, avgdl = EXCLUDED.avgdl, updated_at = now()`,
+    [options.table, n, avgdl],
+  );
+  for (const [term, count] of df) {
+    await conn.exec(
+      `INSERT INTO ${OKE_SEARCH_DF} (table_name, term, df) VALUES (?, ?, ?)
+       ON CONFLICT (table_name, term) DO UPDATE SET df = EXCLUDED.df`,
+      [options.table, term, count],
+    );
+  }
+
+  const warnedLowCorpus = n < SEARCH_LOW_CORPUS_WARN_N;
+  if (warnedLowCorpus) {
+    console.warn(
+      `[oke db search-backfill] warn: table "${options.table}" has only ${n} rows — IDF/BM25 corpus statistics are not meaningful yet (threshold ${SEARCH_LOW_CORPUS_WARN_N})`,
+    );
+  }
+
+  // Touch generated tsv column by no-op update when needed — GENERATED ALWAYS is maintained by PG.
+  void OKE_TSV_COL;
+
+  return { table: options.table, rows: n, embedded, warnedLowCorpus };
+}
+
+function findTable(
+  manifest: Manifest,
+  tableName: string,
+): { columns: Record<string, DeclaredColumn>; pk: string } | undefined {
+  for (const store of Object.values(manifest.stores ?? {})) {
+    if (store.facet !== "sql" || !store.tables?.[tableName]) continue;
+    const table = store.tables[tableName]!;
+    const columns: Record<string, DeclaredColumn> = {};
+    let pk = "id";
+    for (const [key, col] of Object.entries(table.columns ?? {})) {
+      if (!col || typeof col !== "object") continue;
+      const c = col as DeclaredColumn;
+      columns[key] = c;
+      if (c.primaryKey) pk = c.sqlName ?? key;
+    }
+    return { columns, pk };
+  }
+  return undefined;
+}

package/src/elements/store/search-bind.ts

@@ -0,0 +1,66 @@
+/**
+ * Boot helper — register system-owned durable embed CDC consumers for every
+ * `.embed()` column in the Manifest. Writer flows stay free of `effects.embeds`.
+ */
+
+import type { Manifest } from "../../manifest/types.ts";
+import type { Binding } from "../../kernel/on.ts";
+import { flow } from "../../kernel/flow.ts";
+import {
+  applySearchEmbedCdc,
+  searchEmbedFlowName,
+  tablesNeedingSearchEmbed,
+} from "./search-embed-flow.ts";
+
+/**
+ * Adopt `_oke_search_embed_<table>` durable CDC bindings for each embed-declared
+ * table. Idempotent on flow name — safe to call once at boot after Manifest is
+ * known. Prefer {@link adopt} with real CDC {@link Binding}s so
+ * `dispatchCdc` runs them (bare `app.adopt([flow])` only registers the flow).
+ *
+ * @param adopt - App binding adopter (`adoptBinding` inside `oke()`)
+ * @param manifest - Project manifest
+ */
+export function bindSearchEmbedFlows(
+  adopt: (binding: Binding) => void,
+  manifest: Manifest,
+): readonly string[] {
+  const tables = tablesNeedingSearchEmbed(manifest);
+  const names: string[] = [];
+  for (const t of tables) {
+    const name = searchEmbedFlowName(t.table);
+    names.push(name);
+    const models = [...new Set(t.columns.map((c) => c.model ?? "default"))];
+    const sqlRef = `sql:${t.store}` as const;
+    const embedFlow = flow(name, {
+      plane: "operator",
+      durable: true,
+      effects: {
+        reads: [sqlRef],
+        writes: [sqlRef],
+        embeds: models,
+      },
+      do: async (payload: { before: unknown; after: unknown }, fx) => {
+        await fx.step(`embed:${t.table}`, async () => {
+          await applySearchEmbedCdc(
+            fx,
+            t.table,
+            t.pk,
+            t.columns,
+            {
+              before: (payload.before ?? null) as Record<string, unknown> | null,
+              after: (payload.after ?? null) as Record<string, unknown> | null,
+            },
+            sqlRef,
+          );
+        });
+      },
+    });
+    const binding: Binding = {
+      trigger: { kind: "cdc", table: t.table, store: t.store },
+      flow: embedFlow,
+    };
+    adopt(binding);
+  }
+  return names;
+}

package/src/elements/store/search-bm25.ts

@@ -0,0 +1,78 @@
+/**
+ * BM25F scoring — Robertson–Zaragoza with field weights applied to TF
+ * **before** saturation (not post-hoc score summing).
+ */
+
+import { BM25_B, BM25_K1 } from "./search-errors.ts";
+
+/** One field's contribution to a document. */
+export interface Bm25FieldTerms {
+  readonly weight: number;
+  /** term → raw term frequency in this field */
+  readonly tf: ReadonlyMap<string, number>;
+}
+
+/**
+ * BM25F score for one document given query terms and corpus stats.
+ *
+ * @param queryTerms - Tokenized query (unique terms scored once)
+ * @param fields - Per-field TF maps with BM25F weights
+ * @param docLen - Total document length (sum of field token counts)
+ * @param avgdl - Corpus average document length
+ * @param N - Corpus size
+ * @param df - Document frequency per term
+ */
+export function bm25fScore(
+  queryTerms: readonly string[],
+  fields: readonly Bm25FieldTerms[],
+  docLen: number,
+  avgdl: number,
+  N: number,
+  df: ReadonlyMap<string, number>,
+): number {
+  if (N <= 0 || avgdl <= 0) return 0;
+  let score = 0;
+  const seen = new Set<string>();
+  for (const term of queryTerms) {
+    if (seen.has(term)) continue;
+    seen.add(term);
+    let weightedTf = 0;
+    for (const field of fields) {
+      const tf = field.tf.get(term) ?? 0;
+      if (tf > 0) weightedTf += field.weight * tf;
+    }
+    if (weightedTf <= 0) continue;
+    const n = df.get(term) ?? 0;
+    const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));
+    const denom = weightedTf + BM25_K1 * (1 - BM25_B + BM25_B * (docLen / avgdl));
+    score += idf * ((weightedTf * (BM25_K1 + 1)) / denom);
+  }
+  return score;
+}
+
+/**
+ * Simple whitespace + lower-case tokenizer for BM25 (English-oriented).
+ * Used for in-process scoring and tests; SQL path uses to_tsvector terms.
+ *
+ * @param text - Raw text
+ */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]+/gu, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 0);
+}
+
+/**
+ * Build a TF map from tokens.
+ *
+ * @param tokens - Token list
+ */
+export function termFrequencies(tokens: readonly string[]): Map<string, number> {
+  const tf = new Map<string, number>();
+  for (const t of tokens) {
+    tf.set(t, (tf.get(t) ?? 0) + 1);
+  }
+  return tf;
+}