@inflexa-ai/harness 0.1.1

package/dist/runtime/dbos.js

@@ -0,0 +1,160 @@
+/**
+ * DBOS Transact bootstrap.
+ *
+ * This is the only module that imports `@dbos-inc/dbos-sdk` directly. The
+ * agent loop and routes depend on the `RunStep` shape (`harness/loop/types.ts`),
+ * so DBOS stays swappable and the chat path never reaches into the workflow
+ * runtime. `harness/loop/run-step.ts` wraps `DBOS.runStep` for the durable
+ * variant; nothing else here.
+ *
+ * Launch is dormant — this change registers no workflows. The durability seam
+ * exists; the engine is running; nothing is durable yet.
+ *
+ * See `openspec/changes/harness-dbos-runtime`.
+ */
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { DBOS_SYSTEM_POOL_SIZE } from "./pools.js";
+const state = {
+    launched: false,
+    recoveryStarted: false,
+};
+/**
+ * Build the Postgres URL DBOS uses for its system database. Reuses the
+ * existing `DB_PG_*` env vars; sslmode is propagated as a query string so
+ * DBOS's underlying `pg.Pool` honours it.
+ *
+ * The sslmode must mirror the app pool's mapping in `lib/storage.ts`:
+ * `require` means "encrypt, don't verify the chain" there, but
+ * `pg-connection-string` parses bare `require` as an alias for `verify-full`
+ * (full chain verification). Against RDS — whose CA isn't in Node's default
+ * trust store — that fails with SELF_SIGNED_CERT_IN_CHAIN. Emit `no-verify`
+ * so DBOS's pool gets `rejectUnauthorized: false`, matching the app pool.
+ */
+function dbosSslMode(mode) {
+    if (mode === "disable")
+        return "disable";
+    if (mode === "verify-ca" || mode === "verify-full")
+        return "verify-full";
+    return "no-verify";
+}
+function systemDatabaseUrl(config) {
+    const user = encodeURIComponent(config.dbUser);
+    const password = encodeURIComponent(config.dbPassword);
+    const host = config.dbHost;
+    const port = config.dbPort;
+    const database = encodeURIComponent(config.dbName);
+    return `postgresql://${user}:${password}@${host}:${port}/${database}?sslmode=${dbosSslMode(config.dbSslMode)}`;
+}
+/**
+ * Launch DBOS. Idempotent — a second call is a no-op so tests that drive
+ * the harness twice (or accidentally double-import) don't re-launch.
+ *
+ * Order is load-bearing: `setConfig` must precede `launch`, and `launch`
+ * must resolve before the HTTP listener accepts traffic (otherwise the
+ * readiness probe could 200 against a runtime that can't own workflows).
+ */
+export async function launchDbos({ config, logger }) {
+    if (state.launched)
+        return;
+    const executorID = config.executorId;
+    const adminPort = parseInt(config.adminPort, 10);
+    DBOS.setConfig({
+        name: config.appName,
+        systemDatabaseUrl: systemDatabaseUrl(config),
+        systemDatabasePoolSize: DBOS_SYSTEM_POOL_SIZE,
+        executorID,
+        applicationVersion: config.applicationVersion,
+        adminPort,
+        logLevel: config.logLevel,
+    });
+    const start = performance.now();
+    await DBOS.launch();
+    state.launched = true;
+    state.recoveryStarted = true;
+    logger.info({
+        executorID,
+        applicationVersion: config.applicationVersion,
+        adminPort,
+        durationMs: Math.round(performance.now() - start),
+    }, "[dbos] launched");
+}
+/**
+ * Shut DBOS down. In-flight workflows are marked recoverable so another
+ * replica (or this pod on restart) can adopt them. Never throws — the
+ * outer shutdown sequence must close the pg.Pool and flush exporters even
+ * if DBOS shutdown fails.
+ *
+ * Must run after HTTP has drained (so no inbound request is orphaned) and
+ * before the application pool closes (DBOS needs the system DB).
+ */
+export async function shutdownDbos({ logger }) {
+    if (!state.launched)
+        return;
+    const start = performance.now();
+    try {
+        await DBOS.shutdown();
+        logger.info({ durationMs: Math.round(performance.now() - start) }, "[dbos] shutdown");
+    }
+    catch (err) {
+        logger.error({
+            err: err instanceof Error ? err.message : String(err),
+            durationMs: Math.round(performance.now() - start),
+        }, "[dbos] shutdown failed");
+    }
+    finally {
+        state.launched = false;
+        state.recoveryStarted = false;
+    }
+}
+/**
+ * Cancel any `ephemeral:`-prefixed PENDING workflow this executor owns —
+ * called BEFORE `launchDbos`, whose recovery would otherwise re-dispatch them.
+ *
+ * Ephemeral runs are turn-scoped: a run whose pod died mid-flight has no live
+ * awaiter and must never re-execute. DBOS has no "zero recovery" knob, and
+ * `launch()` starts recovery itself, so there is no post-launch window to
+ * cancel from — the only race-free point is a direct system-DB UPDATE before
+ * launch. A CANCELLED row is excluded from the recovery query (which selects
+ * `status='PENDING'`). The `dbos.workflow_status` coupling is the price of
+ * pre-launch timing, and this is the sole module that owns DBOS. The system
+ * DB is the same database as the app pool, so the pool reaches it directly.
+ */
+export async function sweepEphemeralWorkflows({ pool, logger, executorId, }) {
+    const executorID = executorId;
+    try {
+        const { rowCount } = await pool.query({
+            text: `UPDATE dbos.workflow_status
+                SET status = 'CANCELLED', updated_at = $1
+              WHERE status = 'PENDING'
+                AND executor_id = $2
+                AND workflow_uuid LIKE 'ephemeral:%'`,
+            values: [Date.now(), executorID],
+        });
+        if (rowCount && rowCount > 0) {
+            logger.info({ executorID, swept: rowCount }, "[dbos] swept orphaned ephemeral workflows");
+        }
+    }
+    catch (err) {
+        // First-ever boot: DBOS has not created its schema yet — nothing to sweep.
+        if (err && typeof err === "object" && "code" in err && err.code === "42P01") {
+            return;
+        }
+        logger.error({ executorID, err: err instanceof Error ? err.message : String(err) }, "[dbos] ephemeral sweep failed");
+    }
+}
+/** Snapshot of DBOS lifecycle state — read by the readiness probe. */
+export function dbosState() {
+    return { ...state };
+}
+/** Test hook: force-reset state without calling DBOS. Test-only. */
+export function __resetDbosStateForTest() {
+    state.launched = false;
+    state.recoveryStarted = false;
+}
+/** Test hook: mark launched without calling DBOS. Test-only. */
+export function __setDbosStateForTest(next) {
+    if (next.launched !== undefined)
+        state.launched = next.launched;
+    if (next.recoveryStarted !== undefined)
+        state.recoveryStarted = next.recoveryStarted;
+}

package/dist/runtime/lifecycle.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Process-wide lifecycle flags shared between the readiness probe and the
+ * shutdown sequence.
+ *
+ * `draining` flips true the moment SIGTERM/SIGINT is received, so the
+ * readiness probe drops the pod out of LB rotation before the HTTP
+ * server starts refusing requests.
+ */
+export declare function isDraining(): boolean;
+export declare function markDraining(): void;
+/** Test hook — reset the flag between tests. */
+export declare function __resetLifecycleForTest(): void;

package/dist/runtime/lifecycle.js

@@ -0,0 +1,19 @@
+/**
+ * Process-wide lifecycle flags shared between the readiness probe and the
+ * shutdown sequence.
+ *
+ * `draining` flips true the moment SIGTERM/SIGINT is received, so the
+ * readiness probe drops the pod out of LB rotation before the HTTP
+ * server starts refusing requests.
+ */
+let draining = false;
+export function isDraining() {
+    return draining;
+}
+export function markDraining() {
+    draining = true;
+}
+/** Test hook — reset the flag between tests. */
+export function __resetLifecycleForTest() {
+    draining = false;
+}

package/dist/runtime/pools.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Per-pod Postgres pool sizing — the single source of truth.
+ *
+ * Cortex runs two pools per pod against the same cluster (see the postgres-storage-backend spec): the
+ * application pool (`lib/storage.ts::createPool()`) and the DBOS system-DB
+ * pool (`runtime/dbos.ts`). Both numbers live here so the value DBOS is
+ * configured with and the value the boot-time guard reasons about can never
+ * drift apart.
+ *
+ * Fleet-level capacity (how many replicas the cluster can hold) is NOT a
+ * per-pod concern and is deliberately absent here — it belongs to the
+ * database (`max_connections` sizing) or a connection pooler. See
+ * `runtime/connection-budget.ts`.
+ */
+/** DBOS system-DB pool size; passed verbatim to `DBOS.setConfig`. */
+export declare const DBOS_SYSTEM_POOL_SIZE = 10;
+/** Application pool size when `DB_POOL_MAX` is unset. */
+export declare const DEFAULT_APP_POOL_SIZE = 12;
+/**
+ * Connections reserved outside the two pools: psql admin sessions, a brief
+ * overlap during DBOS recovery, replication / monitoring users.
+ */
+export declare const SAFETY_MARGIN = 5;
+/** Resolve the app pool size from the optional `DB_POOL_MAX` override. */
+export declare function resolveAppPoolSize(dbPoolMax: string | undefined): number;

package/dist/runtime/pools.js

@@ -0,0 +1,30 @@
+/**
+ * Per-pod Postgres pool sizing — the single source of truth.
+ *
+ * Cortex runs two pools per pod against the same cluster (see the postgres-storage-backend spec): the
+ * application pool (`lib/storage.ts::createPool()`) and the DBOS system-DB
+ * pool (`runtime/dbos.ts`). Both numbers live here so the value DBOS is
+ * configured with and the value the boot-time guard reasons about can never
+ * drift apart.
+ *
+ * Fleet-level capacity (how many replicas the cluster can hold) is NOT a
+ * per-pod concern and is deliberately absent here — it belongs to the
+ * database (`max_connections` sizing) or a connection pooler. See
+ * `runtime/connection-budget.ts`.
+ */
+/** DBOS system-DB pool size; passed verbatim to `DBOS.setConfig`. */
+export const DBOS_SYSTEM_POOL_SIZE = 10;
+/** Application pool size when `DB_POOL_MAX` is unset. */
+export const DEFAULT_APP_POOL_SIZE = 12;
+/**
+ * Connections reserved outside the two pools: psql admin sessions, a brief
+ * overlap during DBOS recovery, replication / monitoring users.
+ */
+export const SAFETY_MARGIN = 5;
+/** Resolve the app pool size from the optional `DB_POOL_MAX` override. */
+export function resolveAppPoolSize(dbPoolMax) {
+    if (dbPoolMax === undefined || dbPoolMax === "")
+        return DEFAULT_APP_POOL_SIZE;
+    const n = Number.parseInt(dbPoolMax, 10);
+    return Number.isFinite(n) && n >= 1 ? n : DEFAULT_APP_POOL_SIZE;
+}

package/dist/runtime/shutdown.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Graceful-shutdown sequence.
+ *
+ * The order is load-bearing for durability:
+ *
+ *   1. Flip readiness to NOT_READY (so the LB stops sending new traffic).
+ *   2. Drain in-flight HTTP (close the listener, await pending requests).
+ *   3. `DBOS.shutdown()` — marks in-flight workflows recoverable.
+ *      Must run AFTER HTTP drains (so no inbound request is orphaned)
+ *      and BEFORE the application pool closes (DBOS needs the system DB,
+ *      though it owns its own pool, the app close happens after).
+ *   4. Close the application `pg.Pool`.
+ *   5. Flush log + OTel exporters.
+ *   6. `process.exit(0)`.
+ *
+ * Each side effect is injected so the sequence can be tested in isolation
+ * (no real server, no real DBOS). Errors at any step are logged and
+ * swallowed — once shutdown has started, exiting takes precedence over
+ * any single step's failure.
+ */
+import type pino from "pino";
+export interface ShutdownDeps {
+    signal: string;
+    logger: pino.Logger;
+    markDraining: () => void;
+    closeHttpServer: () => Promise<void>;
+    shutdownDbos: () => Promise<void>;
+    closePool: () => Promise<void>;
+    flushLogger: () => Promise<void>;
+    shutdownOtel: () => Promise<void>;
+    exit: (code: number) => void;
+}
+export declare function runShutdownSequence(deps: ShutdownDeps): Promise<void>;

package/dist/runtime/shutdown.js

@@ -0,0 +1,39 @@
+/**
+ * Graceful-shutdown sequence.
+ *
+ * The order is load-bearing for durability:
+ *
+ *   1. Flip readiness to NOT_READY (so the LB stops sending new traffic).
+ *   2. Drain in-flight HTTP (close the listener, await pending requests).
+ *   3. `DBOS.shutdown()` — marks in-flight workflows recoverable.
+ *      Must run AFTER HTTP drains (so no inbound request is orphaned)
+ *      and BEFORE the application pool closes (DBOS needs the system DB,
+ *      though it owns its own pool, the app close happens after).
+ *   4. Close the application `pg.Pool`.
+ *   5. Flush log + OTel exporters.
+ *   6. `process.exit(0)`.
+ *
+ * Each side effect is injected so the sequence can be tested in isolation
+ * (no real server, no real DBOS). Errors at any step are logged and
+ * swallowed — once shutdown has started, exiting takes precedence over
+ * any single step's failure.
+ */
+async function safely(logger, step, fn) {
+    try {
+        await fn();
+    }
+    catch (err) {
+        logger.error({ err: err instanceof Error ? err.message : String(err), step }, `[shutdown] ${step} failed`);
+    }
+}
+export async function runShutdownSequence(deps) {
+    const { logger, signal } = deps;
+    logger.info({ signal }, "[shutdown] requested");
+    deps.markDraining();
+    await safely(logger, "http-drain", deps.closeHttpServer);
+    await safely(logger, "dbos-shutdown", deps.shutdownDbos);
+    await safely(logger, "pool-close", deps.closePool);
+    await safely(logger, "otel-flush", deps.shutdownOtel);
+    await safely(logger, "logger-flush", deps.flushLogger);
+    deps.exit(0);
+}

package/dist/sandbox/await-exec.d.ts

@@ -0,0 +1,143 @@
+/**
+ * `awaitExec` — awaits a submitted exec's terminal result, forwarding progress
+ * events, under one of two transports selected by `options.transport`:
+ *
+ *   - `poll` (default): {@link awaitExecPoll} loops durable pull steps against
+ *     `GET /exec/{execId}?since={cursor}`. The sandbox initiates nothing; the
+ *     host asks. Inherently restart-proof — a recovered workflow just resumes
+ *     polling from its current identity — so the recovery wedge (#41) does not
+ *     apply. A machine that dies mid-exec fast-fails in-loop: sustained
+ *     `unavailable` polls escalate to a backend-inspect probe (`liveness.ts`)
+ *     and an observably dead machine returns the synthetic-failure result
+ *     instead of waiting out the deadline.
+ *   - `callback`: {@link awaitExecCallback} is the recv loop below, with the
+ *     pull as its recovery backstop.
+ *
+ * Both run in the workflow body (NOT a step) — `DBOS.recv`, `DBOS.writeStream`,
+ * and `DBOS.sleepms` are body-only — and both verify every result against the
+ * per-sandbox HMAC, treating a bad/stale signature as a hard cancel.
+ *
+ * ## Callback mode: the recv loop
+ *
+ * On each received envelope:
+ *
+ *   1. If it carries `signature: null` and the payload is a
+ *      `synthetic-failure` done-marker, accept it — the watchdog is an
+ *      in-process trusted sender. Any other null-signature message
+ *      hard-cancels.
+ *   2. Otherwise HMAC-verify against the per-sandbox `callbackSecret`.
+ *      A bad or stale signature throws `HardCancelError`, which DBOS
+ *      treats as a fatal workflow error (no retry).
+ *   3. If the verified payload is a done-marker, return its `result`.
+ *   4. Otherwise forward the payload via `emit` and continue.
+ *
+ * `deadline` is an absolute unix-ms timestamp — the durable backstop
+ * (typically `step.timeout`). `T` is liveness-agnostic recv pacing: we
+ * use `min(5s, remaining)` so the loop returns often enough to honour
+ * the deadline.
+ *
+ * ## Recv is not the only way home
+ *
+ * The completion callback is a *push* to an address baked into the container at
+ * creation. A host that dies mid-exec comes back with a different ingress port,
+ * so the push can never land — the sandbox retries into a void while the
+ * recovered recv waits for a message that will never arrive. That is the
+ * recovery wedge (#41), and it is silent: a `running` row that never advances.
+ *
+ * So whenever the topic falls quiet, this loop stops waiting and *asks*:
+ * `GET /exec/{execId}` on the sandbox itself returns the terminal result,
+ * signed fresh at request time. The bytes are the ones the callback would have
+ * carried, so the provenance frame survives the recovery path intact and one
+ * verification path serves both. Push remains the fast path; pull is what makes
+ * the protocol durable rather than merely optimistic.
+ *
+ * The pull is a DBOS step, not a bare `fetch`: a body-level call whose result
+ * varied between replays would desynchronise the recorded function-ID sequence.
+ */
+import { type ExecEmit, type ExecResult, type SandboxLiveness, type SandboxRef, type SandboxTransport } from "./types.js";
+export declare class HardCancelError extends Error {
+    readonly execId: string;
+    readonly reason: "bad-signature" | "stale-timestamp" | "missing" | "synthetic-without-marker";
+    constructor(execId: string, reason: "bad-signature" | "stale-timestamp" | "missing" | "synthetic-without-marker", detail?: string);
+}
+export declare class ExecTimeoutError extends Error {
+    readonly execId: string;
+    constructor(execId: string);
+}
+/**
+ * Dependency seams of the await loops — every member is an injectable effect
+ * (clock, recv, fetch, durable step, durable sleep, warn sink) plus the
+ * transport selector. Tuning values (freshness window, pull threshold, poll
+ * cadence) are module constants, not options.
+ */
+export interface AwaitExecOptions {
+    /**
+     * Clock source for the deadline check and HMAC freshness. Defaults to the
+     * checkpointed `DBOS.now()` so the loop's recv count is replay-stable (a raw
+     * `Date.now()` here drifts the recorded `DBOS.recv` function-ID sequence on
+     * recovery — see the harness-durable-runtime spec).
+     */
+    now?: () => number | Promise<number>;
+    /** Replaces `DBOS.recv` (callback mode). */
+    recv?: <T>(topic: string, timeoutSec: number) => Promise<T | null>;
+    /** Defaults to `globalThis.fetch`. */
+    fetch?: typeof fetch;
+    /** Defaults to `DBOS.runStep`. */
+    runStep?: <T>(fn: () => Promise<T>, config: {
+        name: string;
+    }) => Promise<T>;
+    /**
+     * Result transport. `poll` (default) runs the durable poll loop; `callback`
+     * runs the `DBOS.recv` loop with the pull as its recovery backstop. The
+     * composition root passes the same value it hands the container.
+     */
+    transport?: SandboxTransport;
+    /** Replaces the durable inter-poll sleep. Defaults to `DBOS.sleepms`. */
+    sleep?: (ms: number) => Promise<void>;
+    /**
+     * Sink for advisory warnings (poll-mode progress-event loss). Defaults to
+     * `DBOS.logger.warn`. Advisory only: the terminal result — not the event
+     * stream — is the authoritative outcome.
+     */
+    warn?: (message: string) => void;
+    /**
+     * Backend inspect for the poll loop's liveness escalation (`liveness.ts`).
+     * Wired by `createSandboxClient` from its backend ops; when absent the
+     * loop skips escalation and stays purely deadline-bounded. Throws on
+     * transient backend API errors — the escalation treats a throw as an
+     * inconclusive probe, never a failed exec.
+     */
+    isAlive?: (ref: SandboxRef) => Promise<SandboxLiveness>;
+}
+/**
+ * Await a submitted exec's terminal result. Dispatches on transport — `poll`
+ * (default) or `callback`. Both verify every result against the per-sandbox
+ * HMAC and are bounded by `deadlineMs`.
+ */
+export declare function awaitExec(ref: SandboxRef, execId: string, emit: ExecEmit, deadlineMs: number, options?: AwaitExecOptions): Promise<ExecResult>;
+/**
+ * Body of the recv loop (callback transport). Hoisted so it's testable without
+ * spinning up DBOS — `recv` is injectable and `now` is overridable.
+ */
+export declare function awaitExecCallback(ref: SandboxRef, execId: string, emit: ExecEmit, deadlineMs: number, options?: AwaitExecOptions): Promise<ExecResult>;
+/**
+ * Body of the poll loop (poll transport). No `DBOS.recv`, no per-exec topic, no
+ * callback handler: a sequence of durable pull steps named
+ * `sandbox.poll-exec-result.${execId}.${n}`, each fetching the signed
+ * `{ status, events, cursor, result? }`, verifying it exactly as a pushed
+ * completion is verified, forwarding new events via `emit`, advancing the
+ * cursor, and returning `result` once terminal. `sleep`, `now`, `fetch`,
+ * `runStep`, and `isAlive` are injectable so the loop is testable without DBOS.
+ *
+ * Liveness: when the `isAlive` seam is wired (the client always wires it),
+ * consecutive `unavailable` polls past the module threshold escalate to a
+ * durable probe step `sandbox.probe-liveness.${execId}.${k}`; a dead verdict
+ * returns the synthetic-failure result in-loop — the poll-mode analogue of the
+ * watchdog's synthetic-complete, minus the topic and the race gate, since the
+ * verdict is produced and consumed by the same workflow in sequence.
+ *
+ * The `cursor`, `pollAttempt`, and `probeAttempt` counters are pure functions
+ * of the cached step sequence, so a replay issues the same polls and probes in
+ * the same order and the DBOS function-ID sequence stays stable.
+ */
+export declare function awaitExecPoll(ref: SandboxRef, execId: string, emit: ExecEmit, deadlineMs: number, options?: AwaitExecOptions): Promise<ExecResult>;