okengine 0.18.4 → 0.19.0

package/src/elements/store/search-ddl.ts

@@ -0,0 +1,156 @@
+/**
+ * Plain-SQL DDL for built-in hybrid search (PostgreSQL 15+, zero extensions).
+ */
+
+import type { DeclaredColumn } from "../../manifest/types.ts";
+import { LSH_DEFAULT_K } from "./search-errors.ts";
+import { generateHyperplanes, hyperplaneSeed, serializePlanes } from "./search-lsh.ts";
+
+/** Shadow / system column and index names. */
+export const OKE_TSV_COL = "__oke_tsv";
+export const OKE_SEARCH_STATS = "oke_search_stats";
+export const OKE_SEARCH_DF = "oke_search_df";
+export const OKE_SEARCH_PLANES = "oke_search_planes";
+
+/**
+ * @param sqlName - Column SQL name
+ */
+export function embColumn(sqlName: string): string {
+  return `__oke_emb_${sqlName}`;
+}
+
+/**
+ * @param sqlName - Column SQL name
+ */
+export function lshColumn(sqlName: string): string {
+  return `__oke_lsh_${sqlName}`;
+}
+
+/**
+ * Emit CREATE TABLE / ALTER / INDEX statements for searchable columns.
+ *
+ * @param table - SQL table name
+ * @param columns - Manifest columns map
+ */
+export function searchDdlForTable(
+  table: string,
+  columns: Record<string, DeclaredColumn>,
+): string[] {
+  const searchable: Array<{ key: string; sqlName: string; weight: number }> = [];
+  const embedCols: Array<{ key: string; sqlName: string; dims: number; model?: string }> = [];
+  for (const [key, col] of Object.entries(columns)) {
+    if (!col.searchable) continue;
+    const sqlName = col.sqlName ?? key;
+    searchable.push({ key, sqlName, weight: col.searchable.weight });
+    if (col.embed) {
+      embedCols.push({
+        key,
+        sqlName,
+        dims: col.embed.dims,
+        ...(col.embed.model ? { model: col.embed.model } : {}),
+      });
+    }
+  }
+  if (searchable.length === 0) return [];
+
+  const stmts: string[] = [
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_STATS} (
+      table_name text PRIMARY KEY,
+      n bigint NOT NULL DEFAULT 0,
+      avgdl double precision NOT NULL DEFAULT 0,
+      updated_at timestamptz NOT NULL DEFAULT now()
+    )`,
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_DF} (
+      table_name text NOT NULL,
+      term text NOT NULL,
+      df bigint NOT NULL DEFAULT 0,
+      PRIMARY KEY (table_name, term)
+    )`,
+    `CREATE TABLE IF NOT EXISTS ${OKE_SEARCH_PLANES} (
+      table_name text NOT NULL,
+      column_name text NOT NULL,
+      dims integer NOT NULL,
+      k integer NOT NULL,
+      seed text NOT NULL,
+      planes bytea NOT NULL,
+      PRIMARY KEY (table_name, column_name)
+    )`,
+  ];
+
+  // Weighted tsvector expression for GIN candidate retrieval.
+  const tsvParts = searchable.map(
+    (c) => `coalesce(to_tsvector('english', ${quoteIdent(c.sqlName)}), '')`,
+  );
+  const tsvExpr = tsvParts.join(" || ");
+  stmts.push(
+    `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${OKE_TSV_COL} tsvector GENERATED ALWAYS AS (${tsvExpr}) STORED`,
+  );
+  stmts.push(
+    `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_${OKE_TSV_COL}_gin`)} ON ${quoteIdent(table)} USING gin (${OKE_TSV_COL})`,
+  );
+
+  for (const c of embedCols) {
+    stmts.push(
+      `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${quoteIdent(embColumn(c.sqlName))} real[]`,
+    );
+    stmts.push(
+      `ALTER TABLE ${quoteIdent(table)} ADD COLUMN IF NOT EXISTS ${quoteIdent(lshColumn(c.sqlName))} bigint`,
+    );
+    stmts.push(
+      `CREATE INDEX IF NOT EXISTS ${quoteIdent(`${table}_${lshColumn(c.sqlName)}_btree`)} ON ${quoteIdent(table)} (${quoteIdent(lshColumn(c.sqlName))})`,
+    );
+  }
+
+  return stmts;
+}
+
+/**
+ * Insert-once hyperplane rows for embed columns (never regenerate).
+ *
+ * @param table - SQL table
+ * @param columns - Manifest columns
+ */
+export function ensureHyperplaneInserts(
+  table: string,
+  columns: Record<string, DeclaredColumn>,
+): Array<{ sql: string; params: unknown[] }> {
+  const out: Array<{ sql: string; params: unknown[] }> = [];
+  for (const [key, col] of Object.entries(columns)) {
+    if (!col.embed) continue;
+    const sqlName = col.sqlName ?? key;
+    const dims = col.embed.dims;
+    const k = LSH_DEFAULT_K;
+    const seed = hyperplaneSeed(table, sqlName, dims, k);
+    const planes = generateHyperplanes(seed, dims, k);
+    const blob = serializePlanes(planes);
+    const hex = blob.toString("hex");
+    out.push({
+      sql: `INSERT INTO ${OKE_SEARCH_PLANES} (table_name, column_name, dims, k, seed, planes)
+            VALUES (?, ?, ?, ?, ?, decode(?, 'hex'))
+            ON CONFLICT (table_name, column_name) DO NOTHING`,
+      params: [table, sqlName, dims, k, seed, hex],
+    });
+  }
+  return out;
+}
+
+/**
+ * Minimum PostgreSQL major version for hybrid search DDL.
+ */
+export const SEARCH_PG_MIN_MAJOR = 15;
+
+/**
+ * @param versionString - `SELECT version()` / `server_version` text
+ */
+export function parsePgMajor(versionString: string): number | undefined {
+  const m = /(\d+)\.\d+/.exec(versionString);
+  if (!m) return undefined;
+  return Number(m[1]);
+}
+
+function quoteIdent(name: string): string {
+  if (!/^[a-z_][a-z0-9_]*$/i.test(name)) {
+    return `"${name.replaceAll('"', '""')}"`;
+  }
+  return name;
+}

package/src/elements/store/search-embed-flow.ts

@@ -0,0 +1,141 @@
+/**
+ * System-owned durable CDC flow that embeds `.embed()` columns asynchronously.
+ * Writer flows never carry `effects.embeds` — only this flow does.
+ */
+
+import type { DeclaredColumn, Manifest } from "../../manifest/types.ts";
+import type { Fx } from "../../kernel/fx.ts";
+import { LSH_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
+import { embColumn, lshColumn, OKE_SEARCH_PLANES } from "./search-ddl.ts";
+import { deserializePlanes, lshBucket, lshBucketToSql } from "./search-lsh.ts";
+
+/** CDC payload shape from bindRealtimeBridge / dispatchCdc. */
+export interface SearchEmbedCdcPayload {
+  readonly before: Record<string, unknown> | null;
+  readonly after: Record<string, unknown> | null;
+}
+
+/**
+ * System flow name for a searchable-embed table.
+ *
+ * @param table - SQL table name
+ */
+export function searchEmbedFlowName(table: string): string {
+  return `_oke_search_embed_${table}`;
+}
+
+/**
+ * Collect tables that need the system embed CDC flow.
+ *
+ * @param manifest - Project manifest
+ */
+export function tablesNeedingSearchEmbed(manifest: Manifest): Array<{
+  readonly table: string;
+  readonly store: string;
+  readonly columns: Array<{ sqlName: string; dims: number; model?: string }>;
+  readonly pk: string;
+}> {
+  const out: Array<{
+    readonly table: string;
+    readonly store: string;
+    readonly columns: Array<{ sqlName: string; dims: number; model?: string }>;
+    readonly pk: string;
+  }> = [];
+  for (const [storeName, store] of Object.entries(manifest.stores ?? {})) {
+    if (store.facet !== "sql" || !store.tables) continue;
+    for (const [tableName, table] of Object.entries(store.tables)) {
+      const embedCols: Array<{ sqlName: string; dims: number; model?: string }> = [];
+      let pk = "id";
+      for (const [key, col] of Object.entries(table.columns ?? {})) {
+        if (!col || typeof col !== "object") continue;
+        const c = col as DeclaredColumn;
+        if (c.primaryKey) pk = c.sqlName ?? key;
+        if (c.embed) {
+          embedCols.push({
+            sqlName: c.sqlName ?? key,
+            dims: c.embed.dims,
+            ...(c.embed.model ? { model: c.embed.model } : {}),
+          });
+        }
+      }
+      if (embedCols.length > 0) {
+        out.push({ table: tableName, store: storeName, columns: embedCols, pk });
+      }
+    }
+  }
+  return out;
+}
+
+/**
+ * Run one embed step for a CDC event (call inside `fx.step`).
+ *
+ * @param fx - Flow fx
+ * @param table - Table name
+ * @param pk - Primary key SQL name
+ * @param columns - Embed columns
+ * @param payload - CDC before/after
+ * @param sqlRef - Resource ref for raw SQL (`sql:app`)
+ */
+export async function applySearchEmbedCdc(
+  fx: Fx,
+  table: string,
+  pk: string,
+  columns: Array<{ sqlName: string; dims: number; model?: string }>,
+  payload: SearchEmbedCdcPayload,
+  sqlRef: `sql:${string}`,
+): Promise<void> {
+  const store = fx.store({
+    kind: "sql",
+    name: sqlRef.slice(4),
+    ref: sqlRef,
+    facet: "sql",
+  } as never);
+
+  if (!payload.after) {
+    // Delete — embeddings cleared with the row; nothing to do.
+    return;
+  }
+
+  const id = String(payload.after[pk] ?? "");
+  if (!id) return;
+
+  for (const col of columns) {
+    const text = String(payload.after[col.sqlName] ?? "");
+    const beforeText =
+      payload.before && typeof payload.before[col.sqlName] === "string"
+        ? String(payload.before[col.sqlName])
+        : undefined;
+    if (beforeText !== undefined && beforeText === text) continue;
+
+    const model = col.model ?? "default";
+    const vector = await fx.embed(model, text);
+    if (vector.length !== col.dims) {
+      throw new SearchConfigError(
+        table,
+        col.sqlName,
+        `embedding length ${vector.length} !== declared dims ${col.dims}`,
+      );
+    }
+
+    const planesRows = await store.raw(
+      `SELECT k, planes FROM ${OKE_SEARCH_PLANES} WHERE table_name = ? AND column_name = ?`,
+      [table, col.sqlName],
+    );
+    const prow = planesRows[0];
+    if (!prow) {
+      throw new SearchConfigError(
+        table,
+        col.sqlName,
+        "missing hyperplanes — ensure search DDL / search-backfill ran",
+      );
+    }
+    const k = Number(prow["k"] ?? LSH_DEFAULT_K);
+    const planes = deserializePlanes(Buffer.from(prow["planes"] as Buffer), k);
+    const bucket = lshBucket(vector, planes);
+
+    await store.raw(
+      `UPDATE ${table} SET ${embColumn(col.sqlName)} = ?::real[], ${lshColumn(col.sqlName)} = ? WHERE ${pk} = ?`,
+      [`{${Array.from(vector).join(",")}}`, lshBucketToSql(bucket), id],
+    );
+  }
+}

package/src/elements/store/search-errors.ts

@@ -0,0 +1,34 @@
+/**
+ * Fail-loud errors for built-in hybrid SQL search.
+ */
+
+/** Thrown when `.embed()` / search config is invalid or AI is missing. */
+export class SearchConfigError extends Error {
+  override readonly name = "SearchConfigError";
+  readonly table: string;
+  readonly column: string;
+
+  /**
+   * @param table - SQL table name
+   * @param column - JS or SQL column key
+   * @param message - Human-readable fix
+   */
+  constructor(table: string, column: string, message: string) {
+    super(`SearchConfigError: ${table}.${column}: ${message}`);
+    this.table = table;
+    this.column = column;
+  }
+}
+
+/** Corpus size below which IDF / BM25 stats are not meaningful. */
+export const SEARCH_LOW_CORPUS_WARN_N = 100;
+
+/** Default RRF damping constant (Cormack, Clarke, Büttcher — SIGIR 2009). */
+export const RRF_DEFAULT_K = 60;
+
+/** Fixed LSH hyperplane count (fits bigint bit packing). */
+export const LSH_DEFAULT_K = 64;
+
+/** BM25 saturation / length-norm defaults (Robertson–Zaragoza). */
+export const BM25_K1 = 1.2;
+export const BM25_B = 0.75;

package/src/elements/store/search-fusion.ts

@@ -0,0 +1,112 @@
+/**
+ * Rank fusion — RRF (default) and weighted combination (opt-in).
+ */
+
+import { RRF_DEFAULT_K } from "./search-errors.ts";
+
+/** One ranked list entry (1-based rank preferred for RRF). */
+export interface RankedHit {
+  readonly id: string;
+  readonly score: number;
+  readonly rank: number;
+}
+
+export type FuseStrategy = "rrf" | "weighted";
+
+export interface FuseOptions {
+  readonly strategy?: FuseStrategy;
+  /** RRF damping constant — default {@link RRF_DEFAULT_K} (Cormack et al. 2009). */
+  readonly k?: number;
+  /** Weighted fusion only. */
+  readonly weights?: { readonly bm25?: number; readonly vector?: number };
+}
+
+/**
+ * Reciprocal Rank Fusion: Σ 1/(k + rank).
+ *
+ * @param lists - Named ranked lists (e.g. bm25, vector)
+ * @param k - Damping constant
+ */
+export function fuseRrf(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  k: number = RRF_DEFAULT_K,
+): RankedHit[] {
+  const scores = new Map<string, number>();
+  for (const list of lists.values()) {
+    for (const hit of list) {
+      const contrib = 1 / (k + hit.rank);
+      scores.set(hit.id, (scores.get(hit.id) ?? 0) + contrib);
+    }
+  }
+  return toRanked(scores);
+}
+
+/**
+ * Weighted linear combination of raw scores (min-max normalized per list).
+ *
+ * @param lists - Named scored lists
+ * @param weights - Per-engine weights
+ */
+export function fuseWeighted(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  weights: { readonly bm25?: number; readonly vector?: number } = {},
+): RankedHit[] {
+  const wBm25 = weights.bm25 ?? 0.5;
+  const wVec = weights.vector ?? 0.5;
+  const norm = normalizeLists(lists);
+  const scores = new Map<string, number>();
+  for (const [name, list] of norm) {
+    const w = name === "vector" ? wVec : name === "bm25" ? wBm25 : 1;
+    for (const hit of list) {
+      scores.set(hit.id, (scores.get(hit.id) ?? 0) + w * hit.score);
+    }
+  }
+  return toRanked(scores);
+}
+
+/**
+ * Apply {@link FuseOptions} — omitted / rrf → RRF with k=60.
+ *
+ * @param lists - Ranked lists
+ * @param fuse - Options
+ */
+export function fuseLists(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+  fuse?: FuseOptions,
+): { hits: RankedHit[]; strategy: FuseStrategy; k?: number } {
+  const strategy = fuse?.strategy ?? "rrf";
+  if (strategy === "weighted") {
+    return { hits: fuseWeighted(lists, fuse?.weights), strategy };
+  }
+  const k = fuse?.k ?? RRF_DEFAULT_K;
+  return { hits: fuseRrf(lists, k), strategy, k };
+}
+
+function normalizeLists(
+  lists: ReadonlyMap<string, readonly RankedHit[]>,
+): Map<string, RankedHit[]> {
+  const out = new Map<string, RankedHit[]>();
+  for (const [name, list] of lists) {
+    let min = Infinity;
+    let max = -Infinity;
+    for (const h of list) {
+      if (h.score < min) min = h.score;
+      if (h.score > max) max = h.score;
+    }
+    const span = max - min || 1;
+    out.set(
+      name,
+      list.map((h) => ({
+        id: h.id,
+        rank: h.rank,
+        score: (h.score - min) / span,
+      })),
+    );
+  }
+  return out;
+}
+
+function toRanked(scores: Map<string, number>): RankedHit[] {
+  const sorted = [...scores.entries()].sort((a, b) => b[1]! - a[1]!);
+  return sorted.map(([id, score], i) => ({ id, score, rank: i + 1 }));
+}

package/src/elements/store/search-lsh.ts

@@ -0,0 +1,179 @@
+/**
+ * Random-hyperplane LSH — K fixed planes generated once per field lifetime.
+ */
+
+import { createHash } from "node:crypto";
+import { LSH_DEFAULT_K } from "./search-errors.ts";
+
+/**
+ * Stable seed string for a field's hyperplanes.
+ *
+ * @param table - SQL table
+ * @param column - SQL column
+ * @param dims - Embedding dimensionality
+ * @param k - Number of planes
+ */
+export function hyperplaneSeed(table: string, column: string, dims: number, k: number): string {
+  return `${table}|${column}|${dims}|${k}`;
+}
+
+/**
+ * Generate K unit-ish random hyperplanes in `dims` dimensions from a seed.
+ * Deterministic across hosts for the same seed.
+ *
+ * @param seed - Stable seed string
+ * @param dims - Vector dimensionality
+ * @param k - Plane count (default {@link LSH_DEFAULT_K})
+ */
+export function generateHyperplanes(
+  seed: string,
+  dims: number,
+  k: number = LSH_DEFAULT_K,
+): Float32Array[] {
+  const planes: Float32Array[] = [];
+  let counter = 0;
+  for (let i = 0; i < k; i++) {
+    const plane = new Float32Array(dims);
+    let filled = 0;
+    while (filled < dims) {
+      const h = createHash("sha256").update(`${seed}\0${counter++}`).digest();
+      for (let b = 0; b + 3 < h.length && filled < dims; b += 4) {
+        const u = h.readUInt32BE(b) / 0xffff_ffff;
+        plane[filled++] = u * 2 - 1;
+      }
+    }
+    // L2 normalize
+    let norm = 0;
+    for (let j = 0; j < dims; j++) norm += plane[j]! * plane[j]!;
+    norm = Math.sqrt(norm) || 1;
+    for (let j = 0; j < dims; j++) plane[j]! /= norm;
+    planes.push(plane);
+  }
+  return planes;
+}
+
+/**
+ * Pack LSH bucket bits into a bigint (K ≤ 64).
+ *
+ * @param vector - Embedding
+ * @param planes - Fixed hyperplanes
+ */
+export function lshBucket(vector: readonly number[], planes: readonly Float32Array[]): bigint {
+  if (planes.length > 64) {
+    throw new Error(`lshBucket: K=${planes.length} exceeds bigint packing (max 64)`);
+  }
+  let bits = 0n;
+  for (let i = 0; i < planes.length; i++) {
+    const plane = planes[i]!;
+    if (vector.length !== plane.length) {
+      throw new Error(`lshBucket: vector length ${vector.length} !== plane dims ${plane.length}`);
+    }
+    let dot = 0;
+    for (let j = 0; j < plane.length; j++) {
+      dot += vector[j]! * plane[j]!;
+    }
+    if (dot >= 0) bits |= 1n << BigInt(i);
+  }
+  return bits;
+}
+
+/**
+ * Encode an LSH bucket for PostgreSQL `bigint` (signed int64).
+ * K=64 bit patterns with the high bit set exceed unsigned range when passed
+ * as a decimal string — reinterpret as two's-complement signed.
+ *
+ * @param bucket - Unsigned bit pack from {@link lshBucket}
+ */
+export function lshBucketToSql(bucket: bigint): string {
+  const masked = bucket & 0xffff_ffff_ffff_ffffn;
+  if (masked >= 0x8000_0000_0000_0000n) {
+    return (masked - 0x1_0000_0000_0000_0000n).toString();
+  }
+  return masked.toString();
+}
+
+/**
+ * Decode a PostgreSQL `bigint` LSH value back to an unsigned bit pack.
+ *
+ * @param value - Driver value (string | number | bigint)
+ */
+export function lshBucketFromSql(value: unknown): bigint {
+  const n = typeof value === "bigint" ? value : BigInt(String(value));
+  return n < 0n ? n + 0x1_0000_0000_0000_0000n : n;
+}
+
+/**
+ * Cosine similarity in [−1, 1].
+ *
+ * @param a - Vector a
+ * @param b - Vector b
+ */
+export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
+  if (a.length !== b.length) {
+    throw new Error(`cosineSimilarity: length mismatch ${a.length} vs ${b.length}`);
+  }
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i]! * b[i]!;
+    na += a[i]! * a[i]!;
+    nb += b[i]! * b[i]!;
+  }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+/**
+ * Candidate buckets: exact match plus Hamming distance 1 flips.
+ *
+ * @param bucket - Query bucket
+ * @param k - Bit width
+ */
+export function neighborBuckets(bucket: bigint, k: number): bigint[] {
+  const out: bigint[] = [bucket];
+  for (let i = 0; i < k; i++) {
+    out.push(bucket ^ (1n << BigInt(i)));
+  }
+  return out;
+}
+
+/**
+ * Serialize planes for `oke_search_planes.planes` (Float32 little-endian).
+ *
+ * @param planes - Hyperplanes
+ */
+export function serializePlanes(planes: readonly Float32Array[]): Buffer {
+  const dims = planes[0]?.length ?? 0;
+  const buf = Buffer.allocUnsafe(4 + planes.length * dims * 4);
+  buf.writeUInt32LE(dims, 0);
+  let offset = 4;
+  for (const plane of planes) {
+    for (let i = 0; i < dims; i++) {
+      buf.writeFloatLE(plane[i]!, offset);
+      offset += 4;
+    }
+  }
+  return buf;
+}
+
+/**
+ * Deserialize planes from storage.
+ *
+ * @param buf - Stored bytes
+ * @param k - Expected plane count
+ */
+export function deserializePlanes(buf: Buffer, k: number): Float32Array[] {
+  const dims = buf.readUInt32LE(0);
+  const planes: Float32Array[] = [];
+  let offset = 4;
+  for (let i = 0; i < k; i++) {
+    const plane = new Float32Array(dims);
+    for (let j = 0; j < dims; j++) {
+      plane[j] = buf.readFloatLE(offset);
+      offset += 4;
+    }
+    planes.push(plane);
+  }
+  return planes;
+}