okengine 0.11.2 → 0.12.0

package/src/kernel/instances.ts

@@ -0,0 +1,372 @@
+/**
+ * Fleet instance registry — one row per process, TTL liveness, no sweeper.
+ *
+ * Alive ⇔ `leaseExpiresAt > now`. Each process UPSERTs only its own row.
+ * SIGKILL leaves a stale row until TTL (same physics as Clock / Journal).
+ */
+
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import type { ConfigEnv } from "../config/index.ts";
+import type { CronStore } from "../elements/clock/reconcile.ts";
+import type { JournalStore } from "./journal.ts";
+
+/** Heartbeat write interval — ride the 1s scheduler, write every 5s. */
+export const INSTANCE_HEARTBEAT_MS = 5_000;
+
+/** Presence TTL — matches Clock / Journal / Vault rotate (30s). */
+export const INSTANCE_LEASE_MS = 30_000;
+
+/** One process row in `oke_instances`. */
+export interface InstanceRow {
+  readonly id: string;
+  readonly startedAt: number;
+  readonly heartbeatAt: number;
+  readonly leaseExpiresAt: number;
+  readonly env: ConfigEnv;
+  readonly pid?: number;
+}
+
+/** Durable store for {@link InstanceRow} (own-row UPSERT, not a claim). */
+export interface InstanceStore {
+  readonly kind: "memory" | "file" | "postgres";
+  /**
+   * Insert or refresh this process's row. Never overwrites `startedAt`.
+   *
+   * @param row - Full row
+   */
+  upsert(row: InstanceRow): Promise<void>;
+  /**
+   * @param id - Instance id
+   */
+  get(id: string): Promise<InstanceRow | undefined>;
+  /** Every row, including expired (readers filter). */
+  list(): Promise<readonly InstanceRow[]>;
+  /**
+   * Rows whose lease has not expired.
+   *
+   * @param now - Epoch-ms
+   */
+  listAlive(now: number): Promise<readonly InstanceRow[]>;
+  /**
+   * Delete this process's row (graceful release).
+   *
+   * @param id - Instance id
+   */
+  remove(id: string): Promise<void>;
+  /** Close the backing connection when the driver owns one. */
+  close?(): Promise<void>;
+}
+
+/** Clock lease currently held by an instance. */
+export interface InstanceClockLease {
+  readonly name: string;
+  readonly leaseUntil: number;
+}
+
+/** Journal run lease currently held by an instance. */
+export interface InstanceJournalLease {
+  readonly runId: string;
+  readonly flow: string;
+  readonly leaseUntil: number;
+}
+
+/** One live instance + read-time lease snapshot. */
+export interface InstanceDetail {
+  readonly id: string;
+  readonly startedAt: number;
+  readonly heartbeatAt: number;
+  readonly leaseExpiresAt: number;
+  readonly env: ConfigEnv;
+  readonly pid?: number;
+  readonly clock: readonly InstanceClockLease[];
+  readonly journal: readonly InstanceJournalLease[];
+}
+
+/** Registry unbound — not a zero count. */
+export type InstancesListEmpty = {
+  readonly kind: "empty";
+};
+
+/** Live fleet snapshot. */
+export type InstancesListFleet = {
+  readonly kind: "fleet";
+  readonly now: number;
+  readonly alive: number;
+  readonly instances: readonly InstanceDetail[];
+};
+
+/** Console / operator projection of the fleet. */
+export type InstancesList = InstancesListEmpty | InstancesListFleet;
+
+/** Options for {@link projectInstancesList}. */
+export interface ProjectInstancesListOptions {
+  readonly store: InstanceStore | null | undefined;
+  readonly clock?: CronStore | null;
+  readonly journal?: JournalStore | null;
+  readonly now?: () => number;
+}
+
+/**
+ * Project live instances + Clock / Journal leases joined by instance id.
+ *
+ * Unbound store → `{ kind: "empty" }` (never `{ alive: 0 }`).
+ *
+ * @param options - Registry + optional join stores
+ */
+export async function projectInstancesList(
+  options: ProjectInstancesListOptions,
+): Promise<InstancesList> {
+  if (!options.store) return { kind: "empty" };
+  const now = (options.now ?? (() => Date.now()))();
+  const alive = await options.store.listAlive(now);
+  const crons = options.clock ? await options.clock.list() : [];
+  const runs = options.journal ? await options.journal.list() : [];
+
+  const instances: InstanceDetail[] = alive.map((row) => {
+    const clock: InstanceClockLease[] = [];
+    for (const cron of crons) {
+      if (cron.leaderInstanceId === row.id && (cron.leaderLeaseUntil ?? 0) > now) {
+        clock.push({ name: cron.name, leaseUntil: cron.leaderLeaseUntil! });
+      }
+    }
+    const journal: InstanceJournalLease[] = [];
+    for (const run of runs) {
+      if (run.lockedBy === row.id && (run.leaseExpiresAt ?? 0) > now) {
+        journal.push({ runId: run.id, flow: run.flow, leaseUntil: run.leaseExpiresAt! });
+      }
+    }
+    return {
+      id: row.id,
+      startedAt: row.startedAt,
+      heartbeatAt: row.heartbeatAt,
+      leaseExpiresAt: row.leaseExpiresAt,
+      env: row.env,
+      ...(row.pid !== undefined ? { pid: row.pid } : {}),
+      clock,
+      journal,
+    };
+  });
+
+  return { kind: "fleet", now, alive: instances.length, instances };
+}
+
+/** Options for {@link createInstanceRuntime}. */
+export interface CreateInstanceRuntimeOptions {
+  readonly instanceId: string;
+  readonly store: InstanceStore;
+  readonly env: ConfigEnv;
+  readonly now?: () => number;
+  readonly heartbeatMs?: number;
+  readonly leaseMs?: number;
+  readonly pid?: number;
+}
+
+/** Process-local heartbeat driver over an {@link InstanceStore}. */
+export interface InstanceRuntime {
+  readonly instanceId: string;
+  readonly store: InstanceStore;
+  readonly env: ConfigEnv;
+  readonly heartbeatMs: number;
+  readonly leaseMs: number;
+  /**
+   * Write a heartbeat when the 5s interval has elapsed (or never written).
+   *
+   * @param at - Optional epoch-ms
+   */
+  maybeHeartbeat(at?: number): Promise<void>;
+  /**
+   * Force a heartbeat write.
+   *
+   * @param at - Optional epoch-ms
+   */
+  heartbeat(at?: number): Promise<void>;
+  /** Delete this process's row (graceful stop). */
+  release(): Promise<void>;
+  /** Close the backing store. */
+  close(): Promise<void>;
+}
+
+/**
+ * Create a heartbeat runtime for one process.
+ *
+ * @param options - Identity, store, env
+ */
+export function createInstanceRuntime(options: CreateInstanceRuntimeOptions): InstanceRuntime {
+  const now = options.now ?? (() => Date.now());
+  const heartbeatMs = options.heartbeatMs ?? INSTANCE_HEARTBEAT_MS;
+  const leaseMs = options.leaseMs ?? INSTANCE_LEASE_MS;
+  const pid = options.pid ?? process.pid;
+  const startedAt = now();
+  let lastHeartbeatAt = 0;
+
+  const runtime: InstanceRuntime = {
+    instanceId: options.instanceId,
+    store: options.store,
+    env: options.env,
+    heartbeatMs,
+    leaseMs,
+    async maybeHeartbeat(at) {
+      const t = at ?? now();
+      if (lastHeartbeatAt > 0 && t - lastHeartbeatAt < heartbeatMs) return;
+      await runtime.heartbeat(t);
+    },
+    async heartbeat(at) {
+      const t = at ?? now();
+      await options.store.upsert({
+        id: options.instanceId,
+        startedAt,
+        heartbeatAt: t,
+        leaseExpiresAt: t + leaseMs,
+        env: options.env,
+        pid,
+      });
+      lastHeartbeatAt = t;
+    },
+    async release() {
+      await options.store.remove(options.instanceId);
+    },
+    async close() {
+      await options.store.close?.();
+    },
+  };
+  return runtime;
+}
+
+function cloneRow(row: InstanceRow): InstanceRow {
+  return { ...row };
+}
+
+/**
+ * In-memory instance store (tests / single process).
+ *
+ * @param seed - Optional rows
+ */
+export function createMemoryInstanceStore(seed?: readonly InstanceRow[]): InstanceStore {
+  const rows = new Map<string, InstanceRow>();
+  for (const row of seed ?? []) {
+    rows.set(row.id, cloneRow(row));
+  }
+  return {
+    kind: "memory",
+    async upsert(row) {
+      const existing = rows.get(row.id);
+      rows.set(row.id, cloneRow(existing ? { ...row, startedAt: existing.startedAt } : row));
+    },
+    async get(id) {
+      const row = rows.get(id);
+      return row ? cloneRow(row) : undefined;
+    },
+    async list() {
+      return [...rows.values()].map(cloneRow);
+    },
+    async listAlive(now) {
+      return [...rows.values()].filter((r) => r.leaseExpiresAt > now).map(cloneRow);
+    },
+    async remove(id) {
+      rows.delete(id);
+    },
+  };
+}
+
+/** File snapshot shape. */
+interface FileInstanceSnapshot {
+  readonly rows: InstanceRow[];
+}
+
+/**
+ * File-backed instance store — multi-process chaos on one host.
+ *
+ * Every mutation runs under an exclusive lock directory (same as the file
+ * CronStore). Does not cache rows across calls.
+ *
+ * @param path - JSON snapshot path
+ */
+export function createFileInstanceStore(path: string): InstanceStore {
+  const lockDir = `${path}.lock`;
+
+  async function withFileLock<T>(fn: () => Promise<T>): Promise<T> {
+    for (let attempt = 0; attempt < 2_000; attempt++) {
+      try {
+        await mkdir(lockDir);
+        break;
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code;
+        if (code !== "EEXIST") throw err;
+        await Bun.sleep(2);
+        if (attempt === 1_999) {
+          throw new Error(`createFileInstanceStore: lock timeout on ${lockDir}`);
+        }
+      }
+    }
+    try {
+      return await fn();
+    } finally {
+      await rm(lockDir, { recursive: true, force: true });
+    }
+  }
+
+  async function loadUnlocked(): Promise<Map<string, InstanceRow>> {
+    const map = new Map<string, InstanceRow>();
+    const file = Bun.file(path);
+    if (!(await file.exists())) return map;
+    const text = await file.text();
+    if (!text.trim()) return map;
+    const snap = JSON.parse(text) as FileInstanceSnapshot;
+    for (const row of snap.rows ?? []) {
+      map.set(row.id, cloneRow(row));
+    }
+    return map;
+  }
+
+  async function flushUnlocked(map: Map<string, InstanceRow>): Promise<void> {
+    await mkdir(dirname(path), { recursive: true });
+    const tmp = join(dirname(path), `.${Bun.hash(path).toString(16)}.tmp`);
+    const body = JSON.stringify(
+      { rows: [...map.values()] } satisfies FileInstanceSnapshot,
+      null,
+      2,
+    );
+    await writeFile(tmp, body, "utf8");
+    await rename(tmp, path);
+  }
+
+  return {
+    kind: "file",
+    async upsert(row) {
+      await withFileLock(async () => {
+        const map = await loadUnlocked();
+        const existing = map.get(row.id);
+        map.set(row.id, cloneRow(existing ? { ...row, startedAt: existing.startedAt } : row));
+        await flushUnlocked(map);
+      });
+    },
+    async get(id) {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        const row = map.get(id);
+        return row ? cloneRow(row) : undefined;
+      });
+    },
+    async list() {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        return [...map.values()].map(cloneRow);
+      });
+    },
+    async listAlive(now) {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        return [...map.values()].filter((r) => r.leaseExpiresAt > now).map(cloneRow);
+      });
+    },
+    async remove(id) {
+      await withFileLock(async () => {
+        const map = await loadUnlocked();
+        map.delete(id);
+        await flushUnlocked(map);
+      });
+    },
+  };
+}

package/src/kernel/lazy-require.ts

@@ -0,0 +1,20 @@
+/**
+ * Computed `import.meta.require` so Bun.build cannot inline the target.
+ * Callers must pass a computed `stem` (e.g. `["list","page"].join("-")`).
+ */
+
+/**
+ * Sync-load `dir/stem.ts` (Bun src) or `dir/stem.js` (published chunk).
+ *
+ * @typeParam T - Module namespace
+ * @param dir - Directory (`import.meta.dir` or a relative walk)
+ * @param stem - Filename without extension
+ */
+export function lazyRequire<T>(dir: string, stem: string): T {
+  const req = (import.meta as ImportMeta & { require?: (id: string) => T }).require!;
+  try {
+    return req(`${dir}/${stem}.ts`);
+  } catch {
+    return req(`${dir}/${stem}.js`);
+  }
+}

package/src/kernel/list-page.test.ts

@@ -0,0 +1,194 @@
+import { describe, expect, test } from "bun:test";
+import { listPage } from "./list-page.ts";
+
+const rows = [
+  { id: "iss_1", title: "Ship", teamId: "eng" },
+  { id: "iss_2", title: "Shipping docs", teamId: "eng" },
+  { id: "iss_3", title: "Design", teamId: "des" },
+  { id: "iss_4", title: "Ship later", teamId: "eng" },
+];
+
+describe("listPage offset", () => {
+  test("pages with total / limit / offset", () => {
+    const page = listPage(rows, { limit: 2, offset: 0 });
+    expect(page.data).toEqual(rows.slice(0, 2));
+    expect(page.meta).toMatchObject({
+      mode: "offset",
+      total: 4,
+      limit: 2,
+      offset: 0,
+      prev: null,
+    });
+    expect(page.meta.next?.cursor).toBeString();
+    const next = listPage(rows, { limit: 2, offset: 2 });
+    expect(next.meta).toMatchObject({ offset: 2, next: null });
+    expect(next.meta.prev?.cursor).toBeString();
+  });
+
+  test("zero-config q searches every string field", () => {
+    const page = listPage(rows, { q: "ship", limit: 10 });
+    expect(page.data.map((r) => r.id)).toEqual(["iss_1", "iss_2", "iss_4"]);
+  });
+
+  test("extra keys auto-eq; id never auto-eqs", () => {
+    const comments = [
+      { id: "c_1", issueId: "iss_1", body: "Ship it" },
+      { id: "c_2", issueId: "iss_1", body: "Later" },
+      { id: "c_3", issueId: "iss_2", body: "Ship docs" },
+    ];
+    const byIssue = listPage(comments, { issueId: "iss_1", limit: 10 });
+    expect(byIssue.data.map((r) => r.id)).toEqual(["c_1", "c_2"]);
+
+    const nested = listPage(comments, { id: "c_1", q: "ship", limit: 10 });
+    expect(nested.data.map((r) => r.id)).toEqual(["c_1", "c_3"]);
+  });
+
+  test("search + extra filter + limit cap", () => {
+    const page = listPage(
+      rows,
+      { q: "ship", teamId: "eng", limit: 500 },
+      { search: ["title"], maxLimit: 2 },
+    );
+    expect(page.data.map((r) => r.id)).toEqual(["iss_1", "iss_2"]);
+    expect(page.meta).toMatchObject({ mode: "offset", total: 3, limit: 2 });
+  });
+
+  test("empty q returns every row", () => {
+    const page = listPage(rows, { q: "" });
+    expect(page.meta).toMatchObject({ total: 4 });
+  });
+
+  test("unknown orderBy is ignored; whitelist sorts", () => {
+    const raw = listPage(rows, { orderBy: "nope" }, { order: ["title"] });
+    expect(raw.data.map((r) => r.id)).toEqual(rows.map((r) => r.id));
+
+    const sorted = listPage(rows, { orderBy: "title", order: "asc" }, { order: ["title"] });
+    expect(sorted.data.map((r) => r.title)).toEqual([
+      "Design",
+      "Ship",
+      "Ship later",
+      "Shipping docs",
+    ]);
+  });
+
+  test("offset next / prev walk as request bags", () => {
+    const first = listPage(rows, { limit: 2 });
+    expect(first.meta.next).not.toBeNull();
+    expect(first.meta.prev).toBeNull();
+    const second = listPage(rows, { limit: 2, ...first.meta.next });
+    expect(second.data).toEqual(rows.slice(2, 4));
+    expect(second.meta.next).toBeNull();
+    const back = listPage(rows, { limit: 2, ...second.meta.prev });
+    expect(back.data).toEqual(first.data);
+    expect(() => listPage(rows, { cursor: "x" })).toThrow(/invalid cursor/);
+  });
+});
+
+describe("listPage cursor", () => {
+  test("requires cursor fields", () => {
+    expect(() => listPage(rows, {}, { mode: "cursor" })).toThrow(/spec.cursor/);
+  });
+
+  test("rejects offset in cursor mode", () => {
+    expect(() => listPage(rows, { offset: 1 }, { mode: "cursor", cursor: ["id"] })).toThrow(
+      /offset/,
+    );
+  });
+
+  test("keyset next / previous cursors", () => {
+    const first = listPage(rows, { limit: 2 }, { mode: "cursor", cursor: ["id"] });
+    expect(first.data.map((r) => r.id)).toEqual(["iss_1", "iss_2"]);
+    expect(first.meta.mode).toBe("cursor");
+    if (first.meta.mode !== "cursor") throw new Error("expected cursor meta");
+    expect(first.meta.prev).toBeNull();
+    expect(first.meta.next?.cursor).toBeString();
+
+    const second = listPage(
+      rows,
+      { limit: 2, ...first.meta.next },
+      { mode: "cursor", cursor: ["id"] },
+    );
+    expect(second.data.map((r) => r.id)).toEqual(["iss_3", "iss_4"]);
+    if (second.meta.mode !== "cursor") throw new Error("expected cursor meta");
+    expect(second.meta.next).toBeNull();
+    expect(second.meta.prev?.cursor).toBeString();
+
+    const back = listPage(
+      rows,
+      { limit: 2, ...second.meta.prev },
+      { mode: "cursor", cursor: ["id"] },
+    );
+    expect(back.data.map((r) => r.id)).toEqual(["iss_1", "iss_2"]);
+    if (back.meta.mode !== "cursor") throw new Error("expected cursor meta");
+    expect(back.meta.prev).toBeNull();
+    expect(back.meta.next).not.toBeNull();
+  });
+
+  test("invalid cursor throws", () => {
+    expect(() =>
+      listPage(rows, { cursor: "not-a-cursor" }, { mode: "cursor", cursor: ["id"] }),
+    ).toThrow(/invalid cursor/);
+  });
+});
+
+describe("PostgREST query", () => {
+  const people = [
+    { id: "1", age: 12, student: true, lastName: "Owen", title: "Kid" },
+    { id: "2", age: 18, student: true, lastName: "Patel", title: "Student" },
+    { id: "3", age: 22, student: false, lastName: "Owen", title: "Grad" },
+    { id: "4", age: 14, student: true, lastName: "Ng", title: null as string | null },
+  ];
+
+  test("eq / gte / is.true", () => {
+    const page = listPage(people, { age: "gte.18", student: "is.true" });
+    expect(page.data.map((r) => r.id)).toEqual(["2"]);
+  });
+
+  test("or= and not.and", () => {
+    const page = listPage(people, { or: "(age.lt.18,age.gt.21)" });
+    expect(page.data.map((r) => r.id)).toEqual(["1", "3", "4"]);
+
+    const nested = listPage(people, { or: "(age.eq.14,not.and(age.gte.11,age.lte.17))" });
+    expect(nested.data.map((r) => r.id)).toEqual(["2", "3", "4"]);
+  });
+
+  test("like * wildcard, in-list, not.eq", () => {
+    const like = listPage(people, { lastName: "like.O*" });
+    expect(like.data.map((r) => r.lastName)).toEqual(["Owen", "Owen"]);
+
+    const inn = listPage(people, { id: "in.(1,3)" });
+    expect(inn.data.map((r) => r.id)).toEqual(["1", "3"]);
+
+    const ne = listPage(people, { lastName: "not.eq.Owen" });
+    expect(ne.data.map((r) => r.lastName)).toEqual(["Patel", "Ng"]);
+  });
+
+  test("like(any) modifier", () => {
+    const page = listPage(people, { lastName: "like(any).{O*,P*}" });
+    expect(page.data.map((r) => r.lastName)).toEqual(["Owen", "Patel", "Owen"]);
+  });
+
+  test("order= col.desc and nullslast", () => {
+    const desc = listPage(people, { order: "age.desc" });
+    expect(desc.data.map((r) => r.age)).toEqual([22, 18, 14, 12]);
+
+    const nulls = listPage(people, { order: "title.asc.nullslast" });
+    expect(nulls.data.map((r) => r.title)).toEqual(["Grad", "Kid", "Student", null]);
+  });
+
+  test("select= projection and alias", () => {
+    const page = listPage(people, { select: "id,name:lastName" }, { filter: "none" });
+    expect(page.data as unknown).toEqual([
+      { id: "1", name: "Owen" },
+      { id: "2", name: "Patel" },
+      { id: "3", name: "Owen" },
+      { id: "4", name: "Ng" },
+    ]);
+  });
+
+  test("filter none rejects column params", () => {
+    expect(() => listPage(people, { age: "eq.18" }, { filter: "none" })).toThrow(
+      /unknown list param/,
+    );
+  });
+});