okengine 0.18.5 → 0.19.1

package/src/elements/store/search-bm25.ts

@@ -0,0 +1,78 @@
+/**
+ * BM25F scoring — Robertson–Zaragoza with field weights applied to TF
+ * **before** saturation (not post-hoc score summing).
+ */
+
+import { BM25_B, BM25_K1 } from "./search-errors.ts";
+
+/** One field's contribution to a document. */
+export interface Bm25FieldTerms {
+  readonly weight: number;
+  /** term → raw term frequency in this field */
+  readonly tf: ReadonlyMap<string, number>;
+}
+
+/**
+ * BM25F score for one document given query terms and corpus stats.
+ *
+ * @param queryTerms - Tokenized query (unique terms scored once)
+ * @param fields - Per-field TF maps with BM25F weights
+ * @param docLen - Total document length (sum of field token counts)
+ * @param avgdl - Corpus average document length
+ * @param N - Corpus size
+ * @param df - Document frequency per term
+ */
+export function bm25fScore(
+  queryTerms: readonly string[],
+  fields: readonly Bm25FieldTerms[],
+  docLen: number,
+  avgdl: number,
+  N: number,
+  df: ReadonlyMap<string, number>,
+): number {
+  if (N <= 0 || avgdl <= 0) return 0;
+  let score = 0;
+  const seen = new Set<string>();
+  for (const term of queryTerms) {
+    if (seen.has(term)) continue;
+    seen.add(term);
+    let weightedTf = 0;
+    for (const field of fields) {
+      const tf = field.tf.get(term) ?? 0;
+      if (tf > 0) weightedTf += field.weight * tf;
+    }
+    if (weightedTf <= 0) continue;
+    const n = df.get(term) ?? 0;
+    const idf = Math.log(1 + (N - n + 0.5) / (n + 0.5));
+    const denom = weightedTf + BM25_K1 * (1 - BM25_B + BM25_B * (docLen / avgdl));
+    score += idf * ((weightedTf * (BM25_K1 + 1)) / denom);
+  }
+  return score;
+}
+
+/**
+ * Simple whitespace + lower-case tokenizer for BM25 (English-oriented).
+ * Used for in-process scoring and tests; SQL path uses to_tsvector terms.
+ *
+ * @param text - Raw text
+ */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\p{L}\p{N}\s]+/gu, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 0);
+}
+
+/**
+ * Build a TF map from tokens.
+ *
+ * @param tokens - Token list
+ */
+export function termFrequencies(tokens: readonly string[]): Map<string, number> {
+  const tf = new Map<string, number>();
+  for (const t of tokens) {
+    tf.set(t, (tf.get(t) ?? 0) + 1);
+  }
+  return tf;
+}

package/src/elements/store/search-ddl.ts

@@ -0,0 +1,156 @@
+/**
+ * Plain-SQL DDL for built-in hybrid search (PostgreSQL 15+, zero extensions).
+ */
+
+import type { DeclaredColumn } from "../../manifest/types.ts";
+import { LSH_DEFAULT_K } from "./search-errors.ts";
+import { generateHyperplanes, hyperplaneSeed, serializePlanes } from "./search-lsh.ts";
+
+/** Shadow / system column and index names. */
+export const OKE_TSV_COL = "__oke_tsv";
+export const OKE_SEARCH_STATS = "oke_search_stats";
+export const OKE_SEARCH_DF = "oke_search_df";
+export const OKE_SEARCH_PLANES = "oke_search_planes";
+
+/**
+ * @param sqlName - Column SQL name
+ */
+export function embColumn(sqlName: string): string {
+  return `__oke_emb_${sqlName}`;
+}
+
+/**
+ * @param sqlName - Column SQL name
+ */
+export function lshColumn(sqlName: string): string {
+  return `__oke_lsh_${sqlName}`;
+}
+
+/**
+ * Emit CREATE TABLE / ALTER / INDEX statements for searchable columns.
+ *
+ * @param table - SQL table name
+ * @param columns - Manifest columns map
+ */
+export function searchDdlForTable(
+  table: string,
+  columns: Record<string, DeclaredColumn>,
+): string[] {
+  const searchable: Array<{ key: string; sqlName: string; weight: number }> = [];
+  const embedCols: Array<{ key: string; sqlName: string; dims: number; model?: string }> = [];
+  for (const [key, col] of Object.entries(columns)) {
+    if (!col.searchable) continue;
+    const sqlName = col.sqlName ?? key;
+    searchable.push({ key, sqlName, weight: col.searchable.weight });
+    if (col.embed) {
+      embedCols.push({
+        key,
+        sqlName,
+        dims: col.embed.dims,
+        ...(col.embed.model ? { model: col.embed.model } : {}),
+      });
+    }
+  }
+  if (searchable.length === 0) return [];
+
+  const stmts: string[] = [
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_STATS} (
+      table_name text PRIMARY KEY,
+      n bigint NOT NULL DEFAULT 0,
+      avgdl double precision NOT NULL DEFAULT 0,
+      updated_at timestamptz NOT NULL DEFAULT now()
+    )`,
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_DF} (
+      table_name text NOT NULL,
+      term text NOT NULL,
+      df bigint NOT NULL DEFAULT 0,
+      PRIMARY KEY (table_name, term)
+    )`,
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_PLANES} (
+      table_name text NOT NULL,
+      column_name text NOT NULL,
+      dims integer NOT NULL,
+      k integer NOT NULL,
+      seed text NOT NULL,
+      planes bytea NOT NULL,
+      PRIMARY KEY (table_name, column_name)
+    )`,
+  ];
+
+  // Weighted tsvector expression for GIN candidate retrieval.
+  const tsvParts = searchable.map(
+    (c) => `coalesce(to_tsvector('english', ${quoteIdent(c.sqlName)}), '')`,
+  );
+  const tsvExpr = tsvParts.join(" || ");
+  stmts.push(
+    `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${OKE_TSV_COL} tsvector GENERATED ALWAYS AS (${tsvExpr}) STORED`,
+  );
+  stmts.push(
+    `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_${OKE_TSV_COL}_gin`)} ON ${quoteIdent(table)} USING gin (${OKE_TSV_COL})`,
+  );
+
+  for (const c of embedCols) {
+    stmts.push(
+      `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${quoteIdent(embColumn(c.sqlName))} real[]`,
+    );
+    stmts.push(
+      `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${quoteIdent(lshColumn(c.sqlName))} bigint`,
+    );
+    stmts.push(
+      `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_${lshColumn(c.sqlName)}_btree`)} ON ${quoteIdent(table)} (${quoteIdent(lshColumn(c.sqlName))})`,
+    );
+  }
+
+  return stmts;
+}
+
+/**
+ * Insert-once hyperplane rows for embed columns (never regenerate).
+ *
+ * @param table - SQL table
+ * @param columns - Manifest columns
+ */
+export function ensureHyperplaneInserts(
+  table: string,
+  columns: Record<string, DeclaredColumn>,
+): Array<{ sql: string; params: unknown[] }> {
+  const out: Array<{ sql: string; params: unknown[] }> = [];
+  for (const [key, col] of Object.entries(columns)) {
+    if (!col.embed) continue;
+    const sqlName = col.sqlName ?? key;
+    const dims = col.embed.dims;
+    const k = LSH_DEFAULT_K;
+    const seed = hyperplaneSeed(table, sqlName, dims, k);
+    const planes = generateHyperplanes(seed, dims, k);
+    const blob = serializePlanes(planes);
+    const hex = blob.toString("hex");
+    out.push({
+      sql: `INSERT INTO ${OKE_SEARCH_PLANES} (table_name, column_name, dims, k, seed, planes)
+            VALUES (?, ?, ?, ?, ?, decode(?, 'hex'))
+            ON CONFLICT (table_name, column_name) DO NOTHING`,
+      params: [table, sqlName, dims, k, seed, hex],
+    });
+  }
+  return out;
+}
+
+/**
+ * Minimum PostgreSQL major version for hybrid search DDL.
+ */
+export const SEARCH_PG_MIN_MAJOR = 15;
+
+/**
+ * @param versionString - `SELECT version()` / `server_version` text
+ */
+export function parsePgMajor(versionString: string): number | undefined {
+  const m = /(\d+)\.\d+/.exec(versionString);
+  if (!m) return undefined;
+  return Number(m[1]);
+}
+
+function quoteIdent(name: string): string {
+  if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
+    return `"${name.replaceAll('"', '""')}"`;
+  }
+  return name;
+}

package/src/elements/store/search-embed-flow.ts

@@ -0,0 +1,141 @@
+/**
+ * System-owned durable CDC flow that embeds `.embed()` columns asynchronously.
+ * Writer flows never carry `effects.embeds` — only this flow does.
+ */
+
+import type { DeclaredColumn, Manifest } from "../../manifest/types.ts";
+import type { Fx } from "../../kernel/fx.ts";
+import { LSH_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
+import { embColumn, lshColumn, OKE_SEARCH_PLANES } from "./search-ddl.ts";
+import { deserializePlanes, lshBucket, lshBucketToSql } from "./search-lsh.ts";
+
+/** CDC payload shape from bindRealtimeBridge / dispatchCdc. */
+export interface SearchEmbedCdcPayload {
+  readonly before: Record<string, unknown> | null;
+  readonly after: Record<string, unknown> | null;
+}
+
+/**
+ * System flow name for a searchable-embed table.
+ *
+ * @param table - SQL table name
+ */
+export function searchEmbedFlowName(table: string): string {
+  return `_oke_search_embed_${table}`;
+}
+
+/**
+ * Collect tables that need the system embed CDC flow.
+ *
+ * @param manifest - Project manifest
+ */
+export function tablesNeedingSearchEmbed(manifest: Manifest): Array<{
+  readonly table: string;
+  readonly store: string;
+  readonly columns: Array<{ sqlName: string; dims: number; model?: string }>;
+  readonly pk: string;
+}> {
+  const out: Array<{
+    readonly table: string;
+    readonly store: string;
+    readonly columns: Array<{ sqlName: string; dims: number; model?: string }>;
+    readonly pk: string;
+  }> = [];
+  for (const [storeName, store] of Object.entries(manifest.stores ?? {})) {
+    if (store.facet !== "sql" || !store.tables) continue;
+    for (const [tableName, table] of Object.entries(store.tables)) {
+      const embedCols: Array<{ sqlName: string; dims: number; model?: string }> = [];
+      let pk = "id";
+      for (const [key, col] of Object.entries(table.columns ?? {})) {
+        if (!col || typeof col !== "object") continue;
+        const c = col as DeclaredColumn;
+        if (c.primaryKey) pk = c.sqlName ?? key;
+        if (c.embed) {
+          embedCols.push({
+            sqlName: c.sqlName ?? key,
+            dims: c.embed.dims,
+            ...(c.embed.model ? { model: c.embed.model } : {}),
+          });
+        }
+      }
+      if (embedCols.length > 0) {
+        out.push({ table: tableName, store: storeName, columns: embedCols, pk });
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Run one embed step for a CDC event (call inside `fx.step`).
+ *
+ * @param fx - Flow fx
+ * @param table - Table name
+ * @param pk - Primary key SQL name
+ * @param columns - Embed columns
+ * @param payload - CDC before/after
+ * @param sqlRef - Resource ref for raw SQL (`sql:app`)
+ */
+export async function applySearchEmbedCdc(
+  fx: Fx,
+  table: string,
+  pk: string,
+  columns: Array<{ sqlName: string; dims: number; model?: string }>,
+  payload: SearchEmbedCdcPayload,
+  sqlRef: `sql:${string}`,
+): Promise<void> {
+  const store = fx.store({
+    kind: "sql",
+    name: sqlRef.slice(4),
+    ref: sqlRef,
+    facet: "sql",
+  } as never);
+
+  if (!payload.after) {
+    // Delete — embeddings cleared with the row; nothing to do.
+    return;
+  }
+
+  const id = String(payload.after[pk] ?? "");
+  if (!id) return;
+
+  for (const col of columns) {
+    const text = String(payload.after[col.sqlName] ?? "");
+    const beforeText =
+      payload.before && typeof payload.before[col.sqlName] === "string"
+        ? String(payload.before[col.sqlName])
+        : undefined;
+    if (beforeText !== undefined && beforeText === text) continue;
+
+    const model = col.model ?? "default";
+    const vector = await fx.embed(model, text);
+    if (vector.length !== col.dims) {
+      throw new SearchConfigError(
+        table,
+        col.sqlName,
+        `embedding length ${vector.length} !== declared dims ${col.dims}`,
+      );
+    }
+
+    const planesRows = await store.raw(
+      `SELECT k, planes FROM ${OKE_SEARCH_PLANES} WHERE table_name = ? AND column_name = ?`,
+      [table, col.sqlName],
+    );
+    const prow = planesRows[0];
+    if (!prow) {
+      throw new SearchConfigError(
+        table,
+        col.sqlName,
+        "missing hyperplanes — ensure search DDL / search-backfill ran",
+      );
+    }
+    const k = Number(prow["k"] ?? LSH_DEFAULT_K);
+    const planes = deserializePlanes(Buffer.from(prow["planes"] as Buffer), k);
+    const bucket = lshBucket(vector, planes);
+
+    await store.raw(
+      `UPDATE ${table} SET ${embColumn(col.sqlName)} = ?::real[], ${lshColumn(col.sqlName)} = ? WHERE ${pk} = ?`,
+      [`{${Array.from(vector).join(",")}}`, lshBucketToSql(bucket), id],
+    );
+  }
+}

package/src/elements/store/search-errors.ts

@@ -0,0 +1,34 @@
+/**
+ * Fail-loud errors for built-in hybrid SQL search.
+ */
+
+/** Thrown when `.embed()` / search config is invalid or AI is missing. */
+export class SearchConfigError extends Error {
+  override readonly name = "SearchConfigError";
+  readonly table: string;
+  readonly column: string;
+
+  /**
+   * @param table - SQL table name
+   * @param column - JS or SQL column key
+   * @param message - Human-readable fix
+   */
+  constructor(table: string, column: string, message: string) {
+    super(`SearchConfigError: ${table}.${column}: ${message}`);
+    this.table = table;
+    this.column = column;
+  }
+}
+
+/** Corpus size below which IDF / BM25 stats are not meaningful. */
+export const SEARCH_LOW_CORPUS_WARN_N = 100;
+
+/** Default RRF damping constant (Cormack, Clarke, Büttcher — SIGIR 2009). */
+export const RRF_DEFAULT_K = 60;
+
+/** Fixed LSH hyperplane count (fits bigint bit packing). */
+export const LSH_DEFAULT_K = 64;
+
+/** BM25 saturation / length-norm defaults (Robertson–Zaragoza). */
+export const BM25_K1 = 1.2;
+export const BM25_B = 0.75;

package/src/elements/store/search-fusion.ts

@@ -0,0 +1,112 @@
+/**
+ * Rank fusion — RRF (default) and weighted combination (opt-in).
+ */
+
+import { RRF_DEFAULT_K } from "./search-errors.ts";
+
+/** One ranked list entry (1-based rank preferred for RRF). */
+export interface RankedHit {
+  readonly id: string;
+  readonly score: number;
+  readonly rank: number;
+}
+
+export type FuseStrategy = "rrf" | "weighted";
+
+export interface FuseOptions {
+  readonly strategy?: FuseStrategy;
+  /** RRF damping constant — default {@link RRF_DEFAULT_K} (Cormack et al. 2009). */
+  readonly k?: number;
+  /** Weighted fusion only. */
+  readonly weights?: { readonly bm25?: number; readonly vector?: number };
+}
+
+/**
+ * Reciprocal Rank Fusion: Σ 1/(k + rank).
+ *
+ * @param lists - Named ranked lists (e.g. bm25, vector)
+ * @param k - Damping constant
+ */
+export function fuseRrf(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  k: number = RRF_DEFAULT_K,
+): RankedHit[] {
+  const scores = new Map<string, number>();
+  for (const list of lists.values()) {
+    for (const hit of list) {
+      const contrib = 1 / (k + hit.rank);
+      scores.set(hit.id, (scores.get(hit.id) ?? 0) + contrib);
+    }
+  }
+  return toRanked(scores);
+}
+
+/**
+ * Weighted linear combination of raw scores (min-max normalized per list).
+ *
+ * @param lists - Named scored lists
+ * @param weights - Per-engine weights
+ */
+export function fuseWeighted(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  weights: { readonly bm25?: number; readonly vector?: number } = {},
+): RankedHit[] {
+  const wBm25 = weights.bm25 ?? 0.5;
+  const wVec = weights.vector ?? 0.5;
+  const norm = normalizeLists(lists);
+  const scores = new Map<string, number>();
+  for (const [name, list] of norm) {
+    const w = name === "vector" ? wVec : name === "bm25" ? wBm25 : 1;
+    for (const hit of list) {
+      scores.set(hit.id, (scores.get(hit.id) ?? 0) + w * hit.score);
+    }
+  }
+  return toRanked(scores);
+}
+
+/**
+ * Apply {@link FuseOptions} — omitted / rrf → RRF with k=60.
+ *
+ * @param lists - Ranked lists
+ * @param fuse - Options
+ */
+export function fuseLists(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  fuse?: FuseOptions,
+): { hits: RankedHit[]; strategy: FuseStrategy; k?: number } {
+  const strategy = fuse?.strategy ?? "rrf";
+  if (strategy === "weighted") {
+    return { hits: fuseWeighted(lists, fuse?.weights), strategy };
+  }
+  const k = fuse?.k ?? RRF_DEFAULT_K;
+  return { hits: fuseRrf(lists, k), strategy, k };
+}
+
+function normalizeLists(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+): Map<string, RankedHit[]> {
+  const out = new Map<string, RankedHit[]>();
+  for (const [name, list] of lists) {
+    let min = Infinity;
+    let max = -Infinity;
+    for (const h of list) {
+      if (h.score < min) min = h.score;
+      if (h.score > max) max = h.score;
+    }
+    const span = max - min || 1;
+    out.set(
+      name,
+      list.map((h) => ({
+        id: h.id,
+        rank: h.rank,
+        score: (h.score - min) / span,
+      })),
+    );
+  }
+  return out;
+}
+
+function toRanked(scores: Map<string, number>): RankedHit[] {
+  const sorted = [...scores.entries()].sort((a, b) => b[1]! - a[1]!);
+  return sorted.map(([id, score], i) => ({ id, score, rank: i + 1 }));
+}