okengine 0.19.0 → 0.19.2

package/src/elements/store/search-runtime.ts

@@ -14,7 +14,7 @@ import {
   deserializePlanes,
   lshBucket,
   lshBucketToSql,
-  neighborBuckets,
+  lshHammingSql,
 } from "./search-lsh.ts";
 import {
   embColumn,
@@ -74,8 +74,9 @@ export interface RunSqlSearchDeps {
 }
 
 /**
- * Execute hybrid search against PostgreSQL using GIN + LSH B-tree candidates,
- * then BM25F / cosine / fusion in-process (correct, testable math).
+ * Execute hybrid search against PostgreSQL using GIN lexical candidates UNION
+ * SimHash Hamming-rank (K=64 `bit_count` ORDER BY LIMIT), then BM25F / cosine
+ * / fusion in-process (correct, testable math).
  *
  * @param deps - Connection, table meta, options
  */
@@ -103,11 +104,9 @@ export async function runSqlSearch(deps: RunSqlSearchDeps): Promise<SqlSearchRes
   const embedFields = searchable.filter((c) => c.embed);
   if (embedFields.length > 0) engines.push("lsh");
 
-  // Candidate retrieval: GIN tsvector match OR LSH buckets.
-  const params: unknown[] = [query];
-  let whereSql = `${OKE_TSV_COL} @@ plainto_tsquery('english', ?)`;
   let queryVec: readonly number[] | undefined;
-  const bucketParams: bigint[] = [];
+  const hamExprs: string[] = [];
+  const hamParams: unknown[] = [];
 
   if (embedFields.length > 0 && deps.embedQuery) {
     const model = embedFields[0]?.embed?.model;
@@ -137,20 +136,18 @@ export async function runSqlSearch(deps: RunSqlSearchDeps): Promise<SqlSearchRes
       const planesBuf = row["planes"] as Buffer;
       const planes = deserializePlanes(Buffer.from(planesBuf), k);
       const bucket = lshBucket(queryVec, planes);
-      for (const b of neighborBuckets(bucket, k)) {
-        bucketParams.push(b);
-      }
-      whereSql += ` OR ${lshColumn(field.sqlName)} = ANY(?::bigint[])`;
-      params.push(`{${bucketParams.map((b) => lshBucketToSql(b)).join(",")}}`);
+      hamExprs.push(lshHammingSql(lshColumn(field.sqlName), "?"));
+      hamParams.push(lshBucketToSql(bucket));
     }
   }
 
-  // Compose list-grammar filters.
+  let filterClause = "";
+  const filterParams: unknown[] = [];
   if (parsed.page.where) {
     const compiled = compileWhere(parsed.page.where);
     if (compiled.clause) {
-      whereSql = `(${whereSql}) AND (${compiled.clause})`;
-      params.push(...compiled.params);
+      filterClause = ` AND (${compiled.clause})`;
+      filterParams.push(...compiled.params);
     }
   }
 
@@ -159,10 +156,27 @@ export async function runSqlSearch(deps: RunSqlSearchDeps): Promise<SqlSearchRes
     ...searchable.map((c) => quoteIdent(c.sqlName)),
     ...embedFields.map((c) => quoteIdent(embColumn(c.sqlName))),
   ].join(", ");
+  const tableIdent = quoteIdent(tableName);
 
   // Oversample candidates for fusion then truncate.
   const fetchLimit = Math.min(Math.max(limit * 5, 50), 500);
-  const candidateSql = `SELECT ${selectCols} FROM ${quoteIdent(tableName)} WHERE ${whereSql} LIMIT ${fetchLimit}`;
+  const lexicalSql = `SELECT ${selectCols} FROM ${tableIdent} WHERE ${OKE_TSV_COL} @@ plainto_tsquery('english', ?)${filterClause} LIMIT ${fetchLimit}`;
+
+  // SimHash k-NN: rank by Hamming over the K=64 bit pack, then LIMIT.
+  // Hamming-1 equality (`= ANY(65 buckets)`) is not a candidate set — true
+  // near-neighbors sit at distance ~5–16, so that probe recovered ~0 rows.
+  let candidateSql: string;
+  const params: unknown[] = [];
+  if (hamExprs.length > 0) {
+    const lshPred = embedFields.map((f) => `${lshColumn(f.sqlName)} IS NOT NULL`).join(" OR ");
+    const orderExpr = hamExprs.length === 1 ? hamExprs[0]! : `LEAST(${hamExprs.join(", ")})`;
+    const lshSql = `SELECT ${selectCols} FROM ${tableIdent} WHERE (${lshPred})${filterClause} ORDER BY ${orderExpr} ASC NULLS LAST LIMIT ${fetchLimit}`;
+    candidateSql = `(${lexicalSql}) UNION (${lshSql})`;
+    params.push(query, ...filterParams, ...hamParams, ...filterParams);
+  } else {
+    candidateSql = lexicalSql;
+    params.push(query, ...filterParams);
+  }
   const candidates = await conn.query(candidateSql, params);
 
   const statsRows = await conn.query(

package/src/elements/store/search.test.ts

@@ -3,18 +3,23 @@
  */
 
 import { describe, expect, test } from "bun:test";
+import { createHash } from "node:crypto";
 import { field, store } from "../store.ts";
 import { bm25fScore, termFrequencies, tokenize } from "./search-bm25.ts";
 import { fuseLists, fuseRrf, fuseWeighted } from "./search-fusion.ts";
-import { RRF_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
+import { LSH_DEFAULT_K, RRF_DEFAULT_K, SearchConfigError } from "./search-errors.ts";
 import {
   cosineSimilarity,
+  deserializePlanes,
   generateHyperplanes,
+  hammingDistance,
   hyperplaneSeed,
   lshBucket,
   lshBucketFromSql,
   lshBucketToSql,
+  lshHammingSql,
   neighborBuckets,
+  serializePlanes,
 } from "./search-lsh.ts";
 
 describe("field.searchable / .embed", () => {
@@ -101,8 +106,85 @@ describe("LSH", () => {
     expect(lshBucketFromSql(sql)).toBe(high);
     expect(lshBucketToSql(42n)).toBe("42");
   });
+
+  test("write-time and query-time buckets are identical for the same vector", () => {
+    const seed = hyperplaneSeed("g17_docs", "body", 32, LSH_DEFAULT_K);
+    const planes = generateHyperplanes(seed, 32, LSH_DEFAULT_K);
+    const reloaded = deserializePlanes(serializePlanes(planes), LSH_DEFAULT_K);
+    const vector = synthHashBag(
+      "refund policy shipping delay. Document 0 discusses refund policy shipping delay.",
+    );
+    const write = lshBucket(vector, planes);
+    const query = lshBucket(vector, reloaded);
+    expect(write).toBe(query);
+    expect(lshBucketToSql(write)).toBe(lshBucketToSql(query));
+    expect(neighborBuckets(write, LSH_DEFAULT_K)).toHaveLength(LSH_DEFAULT_K + 1);
+  });
+
+  test("K=64 Hamming-1 misses G17-like neighbors; Hamming-rank recovers them", () => {
+    // Same deterministic hash-bag generator G17 uses — 40 docs, 8 topics.
+    const topics = [
+      "refund policy shipping delay",
+      "password reset two factor auth",
+      "invoice payment stripe webhook",
+      "search ranking bm25 hybrid",
+      "postgres gin btree index plan",
+      "tenant isolation row level security",
+      "durable flow journal resume",
+      "live query fanout subscriber",
+    ];
+    const planes = generateHyperplanes(hyperplaneSeed("g17_docs", "body", 32, 64), 32, 64);
+    const docs = topics.flatMap((topic, t) =>
+      [0, 1, 2, 3, 4].map((n) => {
+        const i = t + n * 8;
+        const body = `${topic}. Document ${i} discusses ${topic} with enough tokens for BM25.`;
+        const vec = synthHashBag(body);
+        return { i, vec, bucket: lshBucket(vec, planes) };
+      }),
+    );
+    const qVec = synthHashBag(topics[0]!);
+    const qBucket = lshBucket(qVec, planes);
+    const scored = docs.map((d) => ({
+      i: d.i,
+      cos: cosineSimilarity(qVec, d.vec),
+      ham: hammingDistance(qBucket, d.bucket),
+    }));
+    const exact = [...scored].sort((a, b) => b.cos - a.cos).slice(0, 10);
+    const exactSet = new Set(exact.map((x) => x.i));
+    const ham1 = scored.filter((s) => s.ham <= 1);
+    const ham1Hits = ham1.filter((s) => exactSet.has(s.i)).length;
+    const ranked = [...scored].sort((a, b) => a.ham - b.ham || b.cos - a.cos).slice(0, 50);
+    const rankedTop = [...ranked].sort((a, b) => b.cos - a.cos).slice(0, 10);
+    const rankedHits = rankedTop.filter((s) => exactSet.has(s.i)).length;
+    // Hamming-1 is the broken probe (G17 P@10 ≈ 0). Hamming-rank of the K=64
+    // pack recovers the exact cosine top-10 on this 40-row hand case.
+    expect(ham1Hits).toBe(0);
+    expect(rankedHits).toBe(10);
+    expect(exact.every((e) => e.ham > 1)).toBe(true);
+    expect(lshHammingSql("__oke_lsh_body")).toContain("bit_count");
+  });
 });
 
+/**
+ * G17 synthetic embedding — deterministic SHA-256 hash bag, L2-normalized.
+ *
+ * @param text - Source text
+ */
+function synthHashBag(text: string, dims = 32): number[] {
+  const v = Array.from({ length: dims }, () => 0);
+  const tokens = text.toLowerCase().split(/\W+/).filter(Boolean);
+  for (const t of tokens) {
+    const h = createHash("sha256").update(t).digest();
+    for (let i = 0; i < dims; i++) {
+      v[i]! += (h[i % h.length]! / 255) * 2 - 1;
+    }
+  }
+  let norm = 0;
+  for (const x of v) norm += x * x;
+  norm = Math.sqrt(norm) || 1;
+  return v.map((x) => x / norm);
+}
+
 describe("fusion", () => {
   test("RRF default k=60 matches Cormack hand ranks", () => {
     expect(RRF_DEFAULT_K).toBe(60);

package/src/elements/store.ts

@@ -197,7 +197,9 @@ export {
   lshBucketToSql,
   lshBucketFromSql,
   cosineSimilarity,
+  hammingDistance,
   neighborBuckets,
+  lshHammingSql,
   serializePlanes,
   deserializePlanes,
 } from "./store/search-lsh.ts";

package/src/full.ts

@@ -37,6 +37,9 @@ export {
   type OkeApp,
   type OkeOptions,
   type ReadyState,
+  type CdcAction,
+  type CdcPayload,
+  type CdcPayloadInput,
   type FlowDef,
   type FlowFailure,
   type FlowErrorValue,

package/src/http.ts

@@ -28,6 +28,9 @@ export { isFlowFailure } from "./kernel/hooks.ts";
 export { plugin, isPlugin, type PluginDef } from "./kernel/plugin.ts";
 export { createBunRuntime, APP_PORT, type Runtime, type ServeOptions } from "./runtime/index.ts";
 export type {
+  CdcAction,
+  CdcPayload,
+  CdcPayloadInput,
   OkeApp,
   OkeOptions,
   ReadyState,

package/src/index.ts

@@ -16,6 +16,9 @@
 
 export {
   oke,
+  type CdcAction,
+  type CdcPayload,
+  type CdcPayloadInput,
   type OkeApp,
   type OkeOptions,
   type ReadyState,

package/src/kernel/app.ts

@@ -67,7 +67,8 @@ import {
 import { GATE_PUBLIC_NAME } from "../elements/gate/flatten.ts";
 import type { JsonCodeAuth } from "../runtime/json-code-block.ts";
 import { resolveDurationMs } from "./elapsed.ts";
-import { fail, throwOke } from "./errors.ts";
+import { fail, throwOke, OkeError } from "./errors.ts";
+import { FLOW_NAME_DUPLICATE } from "./errors-flow-name.ts";
 import { consumeRegisteredFlowUnits, type FlowUnitBag } from "./flow-units.ts";
 import {
   isFlowFailure,
@@ -143,6 +144,14 @@ import { recordObservedEffect } from "./effects.ts";
 import { cacheDimensionOf, createRunTelemetry } from "./run-telemetry.ts";
 import type { RunsRuntime } from "../runs/runtime.ts";
 import type { Effects, ResourceRef } from "../manifest/types.ts";
+import {
+  enrichCdcPayload,
+  pkColumnByTableFromManifest,
+  type CdcPayloadInput,
+  type DeclaredPk,
+} from "./cdc-payload.ts";
+
+export type { CdcAction, CdcPayload, CdcPayloadInput } from "./cdc-payload.ts";
 
 /**
  * Auto-cache helpers — loaded only when a Store runtime is bound.
@@ -334,12 +343,6 @@ export interface OkeOptions {
 /** Readiness probe state — see `GET /_/ready`. */
 export type ReadyState = "booting" | "orphan_scan" | "ready";
 
-/** Payload for a CDC invocation. */
-export interface CdcPayload {
-  readonly before: Record<string, unknown> | null;
-  readonly after: Record<string, unknown> | null;
-}
-
 /** Result of executing a flow. */
 export interface ExecuteResult {
   readonly output: unknown;
@@ -582,10 +585,14 @@ export interface OkeApp<D extends Record<string, unknown> = {}, R extends AppRou
    * Invoke all flows bound to a CDC table change.
    *
    * @param tableName - Table name
-   * @param payload - before/after
+   * @param payload - `{ before, after }` (and optional `table` / `action` / `id`)
    * @param column - Optional column filter
    */
-  dispatchCdc(tableName: string, payload: CdcPayload, column?: string): Promise<ExecuteResult[]>;
+  dispatchCdc(
+    tableName: string,
+    payload: CdcPayloadInput,
+    column?: string,
+  ): Promise<ExecuteResult[]>;
   /**
    * Call a flow by name/handle (same path as `fx.call`).
    *
@@ -937,6 +944,7 @@ export function oke(options: OkeOptions): OkeApp {
   const flushedPlugins = new WeakMap<AnyFlowDef, Set<PluginDef>>();
   /** Runtime route table — types accumulate on the returned {@link OkeApp}. */
   const routes: RuntimeRouteMap = {};
+  let cdcPkByTable: ReadonlyMap<string, DeclaredPk> = pkColumnByTableFromManifest(options.manifest);
 
   let authMaterialization: AuthHttpMaterialization | undefined;
   let wiredAuth: WiredGateAuth | undefined;
@@ -1081,6 +1089,10 @@ export function oke(options: OkeOptions): OkeApp {
   }
 
   function registerFlow(flowDef: AnyFlowDef): void {
+    const existing = flowsByName.get(flowDef.name);
+    if (existing && existing !== flowDef && flowDef.name) {
+      throw new OkeError(FLOW_NAME_DUPLICATE, { flow: flowDef.name });
+    }
     flowsByName.set(flowDef.name, flowDef);
     flushFlowPlugins(flowDef);
   }
@@ -1348,6 +1360,8 @@ export function oke(options: OkeOptions): OkeApp {
     const baseChannel = overrides?.channel ?? effectiveChannel;
     const mergedCatalog = mergeTemplateCatalogs(baseChannel?.catalog, ...pluginChannelCatalogs);
 
+    cdcPkByTable = pkColumnByTableFromManifest(overrides?.manifest ?? options.manifest);
+
     const merged: BootOptions = {
       env: bootEnv,
       docker: overrides?.docker ?? options.docker,
@@ -2405,6 +2419,12 @@ export function oke(options: OkeOptions): OkeApp {
       return results;
     },
     async dispatchCdc(tableName, payload, column) {
+      const enriched = enrichCdcPayload(
+        tableName,
+        payload.before ?? null,
+        payload.after ?? null,
+        cdcPkByTable.get(tableName),
+      );
       const results: ExecuteResult[] = [];
       for (const b of adopted) {
         if (b.trigger.kind !== "cdc") continue;
@@ -2413,7 +2433,7 @@ export function oke(options: OkeOptions): OkeApp {
         if (column !== undefined && t.column !== undefined && t.column !== column) {
           continue;
         }
-        results.push(await execute(b.flow, payload, t));
+        results.push(await execute(b.flow, enriched, t));
       }
       return results;
     },

package/src/kernel/boot.ts

@@ -194,7 +194,7 @@ export interface BootOptions {
   };
   /**
    * Process instance id for Clock / Journal / fleet registry.
-   * When unset, boot mints one `inst-<uuid>` and passes it to every binder.
+   * When unset, boot mints one `inst-<okid>` and passes it to every binder.
    */
   readonly instanceId?: string;
   /** Injected fleet store (chaos / tests — activates the registry in `test`). */

package/src/kernel/cdc-payload.test.ts

@@ -0,0 +1,224 @@
+/**
+ * CdcPayload enrichment — action from images, PK from Manifest, additive fields.
+ */
+
+import { beforeEach, describe, expect, test } from "bun:test";
+import { field, store } from "../elements/store.ts";
+import type { Manifest } from "../manifest/types.ts";
+import { oke } from "./app.ts";
+import {
+  cdcActionFromImages,
+  cdcIdFromImages,
+  declaredPk,
+  declaredPkColumn,
+  enrichCdcPayload,
+  pkColumnByTableFromManifest,
+  type CdcPayload,
+} from "./cdc-payload.ts";
+import { flow, resetFlowSeq } from "./flow.ts";
+import { on, resetBindings } from "./on.ts";
+
+beforeEach(() => {
+  resetBindings();
+  resetFlowSeq();
+});
+
+const manifest: Manifest = {
+  oke: "1.0",
+  app: "cdc-enrich",
+  stores: {
+    db: {
+      facet: "sql",
+      tables: {
+        orders: {
+          columns: {
+            id: { type: "text", primaryKey: true, sqlName: "id" },
+            status: { type: "text", sqlName: "status" },
+          },
+        },
+        accounts: {
+          columns: {
+            code: { type: "text", primaryKey: true, sqlName: "code" },
+            name: { type: "text", sqlName: "name" },
+          },
+        },
+        members: {
+          columns: {
+            userId: { type: "text", primaryKey: true, sqlName: "user_id" },
+          },
+        },
+      },
+    },
+  },
+};
+
+describe("cdc-payload helpers", () => {
+  test("action is created / updated / deleted from before/after presence", () => {
+    expect(cdcActionFromImages(null, { id: "1" })).toBe("created");
+    expect(cdcActionFromImages({ id: "1" }, { id: "1" })).toBe("updated");
+    expect(cdcActionFromImages({ id: "1" }, null)).toBe("deleted");
+  });
+
+  test("id uses after for created/updated and before for deleted", () => {
+    expect(cdcIdFromImages(null, { code: "a1" }, "code")).toBe("a1");
+    expect(cdcIdFromImages({ code: "a1" }, { code: "a1", name: "x" }, "code")).toBe("a1");
+    expect(cdcIdFromImages({ code: "a1" }, null, "code")).toBe("a1");
+  });
+
+  test("id preserves number PKs and stringifies anything else", () => {
+    expect(cdcIdFromImages(null, { id: 42 }, "id")).toBe(42);
+    expect(cdcIdFromImages(null, { id: true }, "id")).toBe("true");
+    expect(cdcIdFromImages(null, {}, "id")).toBe("");
+  });
+
+  test("id tries JS key then SQL name when they differ", () => {
+    const pk = { key: "userId", sqlName: "user_id" };
+    expect(cdcIdFromImages(null, { userId: "u1" }, pk)).toBe("u1");
+    expect(cdcIdFromImages(null, { user_id: "u2" }, pk)).toBe("u2");
+  });
+
+  test("declaredPk reads Manifest primaryKey, else id", () => {
+    expect(declaredPk(manifest.stores!.db!.tables!.accounts!.columns)).toEqual({
+      key: "code",
+      sqlName: "code",
+    });
+    expect(declaredPkColumn(manifest.stores!.db!.tables!.members!.columns)).toBe("user_id");
+    expect(declaredPk(undefined)).toEqual({ key: "id", sqlName: "id" });
+    expect(declaredPk({ name: { type: "text" } })).toEqual({ key: "id", sqlName: "id" });
+  });
+
+  test("pkColumnByTableFromManifest maps every SQL table", () => {
+    const map = pkColumnByTableFromManifest(manifest);
+    expect(map.get("orders")).toEqual({ key: "id", sqlName: "id" });
+    expect(map.get("accounts")).toEqual({ key: "code", sqlName: "code" });
+    expect(map.get("members")).toEqual({ key: "userId", sqlName: "user_id" });
+    expect(pkColumnByTableFromManifest(undefined).size).toBe(0);
+  });
+
+  test("enrichCdcPayload fills table, action, and id", () => {
+    expect(enrichCdcPayload("orders", null, { id: "o1", status: "open" }, "id")).toEqual({
+      table: "orders",
+      action: "created",
+      id: "o1",
+      before: null,
+      after: { id: "o1", status: "open" },
+    });
+  });
+});
+
+describe("dispatchCdc — enriched payload", () => {
+  test("insert/update/delete carry table, action, and the declared PK for two tables", async () => {
+    const seen: CdcPayload[] = [];
+    const db = store.sql("main");
+    const orders = store.schema.table("orders", {
+      id: field.text().primaryKey(),
+      status: field.text(),
+    });
+    const accounts = store.schema.table("accounts", {
+      code: field.text().primaryKey(),
+      name: field.text(),
+    });
+
+    on(
+      db.table(orders).changed(),
+      flow("audit.orders", {
+        do: (p: CdcPayload) => {
+          seen.push(p);
+        },
+      }),
+    );
+    on(
+      db.table(accounts).changed(),
+      flow("audit.accounts", {
+        do: (p: CdcPayload) => {
+          seen.push(p);
+        },
+      }),
+    );
+
+    const app = oke({ autoBoot: false, name: "cdc-enrich", manifest });
+
+    await app.dispatchCdc("orders", { before: null, after: { id: "o1", status: "open" } });
+    await app.dispatchCdc("orders", {
+      before: { id: "o1", status: "open" },
+      after: { id: "o1", status: "paid" },
+    });
+    await app.dispatchCdc("orders", { before: { id: "o1", status: "paid" }, after: null });
+
+    await app.dispatchCdc("accounts", { before: null, after: { code: "acme", name: "Acme" } });
+    await app.dispatchCdc("accounts", {
+      before: { code: "acme", name: "Acme" },
+      after: { code: "acme", name: "Acme Inc" },
+    });
+    await app.dispatchCdc("accounts", { before: { code: "acme", name: "Acme Inc" }, after: null });
+
+    expect(seen).toEqual([
+      {
+        table: "orders",
+        action: "created",
+        id: "o1",
+        before: null,
+        after: { id: "o1", status: "open" },
+      },
+      {
+        table: "orders",
+        action: "updated",
+        id: "o1",
+        before: { id: "o1", status: "open" },
+        after: { id: "o1", status: "paid" },
+      },
+      {
+        table: "orders",
+        action: "deleted",
+        id: "o1",
+        before: { id: "o1", status: "paid" },
+        after: null,
+      },
+      {
+        table: "accounts",
+        action: "created",
+        id: "acme",
+        before: null,
+        after: { code: "acme", name: "Acme" },
+      },
+      {
+        table: "accounts",
+        action: "updated",
+        id: "acme",
+        before: { code: "acme", name: "Acme" },
+        after: { code: "acme", name: "Acme Inc" },
+      },
+      {
+        table: "accounts",
+        action: "deleted",
+        id: "acme",
+        before: { code: "acme", name: "Acme Inc" },
+        after: null,
+      },
+    ]);
+  });
+
+  test("a handler that only destructures { before, after } still runs", async () => {
+    const seen: string[] = [];
+    const db = store.sql("main");
+    const orders = store.schema.table("orders", {
+      id: field.text().primaryKey(),
+      status: field.text(),
+    });
+    on(
+      db.table(orders).changed("status"),
+      flow("t.cdc.bare", {
+        do: (p: { before: { status: string }; after: { status: string } }) => {
+          seen.push(`cdc:${p.before.status}->${p.after.status}`);
+        },
+      }),
+    );
+    const app = oke({ autoBoot: false, name: "cdc-bare" });
+    await app.dispatchCdc(
+      "orders",
+      { before: { status: "open" }, after: { status: "paid" } },
+      "status",
+    );
+    expect(seen).toEqual(["cdc:open->paid"]);
+  });
+});