@inflexa-ai/harness 0.42.1 → 0.43.0

package/dist/sandbox/hmac.js

@@ -1,72 +0,0 @@
-/**
- * HMAC verification for sandbox callbacks (see the harness-sandbox-exec spec).
- *
- * sandbox-server signs every outbound event/completion POST with
- * `X-Sandbox-Signature = hex(HMAC-SHA256(callbackSecret,
- *   "${execId}:${timestamp}:${sha256Hex(body)}"))` plus a
- * `X-Sandbox-Timestamp` header. The Cortex-side callback endpoint is
- * **dumb** — it forwards the headers verbatim onto the per-exec DBOS
- * topic. Verification happens here, in the workflow-body recv loop,
- * because that body holds the `callbackSecret` from the cached
- * `createSandbox` step output.
- *
- * A bad or stale signature triggers a hard cancel of the run — under
- * NetworkPolicy isolation a forged event implies a bug or a breach,
- * neither of which should keep a 3-hour sandbox burning.
- */
-import { createHmac, timingSafeEqual } from "node:crypto";
-import { digestBody } from "./digest.js";
-/**
- * Decode the per-sandbox secret. sandbox-server accepts both raw UTF-8
- * and `base64:` prefixed values (change 4 spec); we mirror the same
- * convention here so the Cortex and Go sides interpret the secret
- * identically.
- */
-function decodeSecret(secret) {
-    if (secret.startsWith("base64:")) {
-        return Buffer.from(secret.slice("base64:".length), "base64");
-    }
-    return Buffer.from(secret, "utf8");
-}
-function sha256Hex(input) {
-    return digestBody(input);
-}
-/**
- * Recompute the HMAC and compare in constant time, then check the
- * timestamp falls within a symmetric `freshnessSec` window of `nowSec`.
- *
- * Order: `missing` (one of signature/timestamp is null) is reported
- * separately from `bad-signature` so the caller can log it precisely.
- * Freshness is checked AFTER signature, so a stale-but-valid signature
- * is distinguishable from a forgery.
- */
-export function verifyCallback(input) {
-    if (input.signature === null || input.timestamp === null) {
-        return { valid: false, reason: "missing" };
-    }
-    const digest = input.bodyDigest ?? (input.body !== undefined ? sha256Hex(input.body) : undefined);
-    if (digest === undefined) {
-        return { valid: false, reason: "missing" };
-    }
-    const secretBytes = decodeSecret(input.secret);
-    const message = `${input.execId}:${input.timestamp}:${digest}`;
-    const expected = createHmac("sha256", secretBytes).update(message).digest("hex");
-    const provided = input.signature;
-    if (provided.length !== expected.length) {
-        return { valid: false, reason: "bad-signature" };
-    }
-    const matches = timingSafeEqual(Buffer.from(provided, "utf8"), Buffer.from(expected, "utf8"));
-    if (!matches) {
-        return { valid: false, reason: "bad-signature" };
-    }
-    if (Math.abs(input.nowSec - input.timestamp) > input.freshnessSec) {
-        return { valid: false, reason: "stale-timestamp" };
-    }
-    return { valid: true };
-}
-/** Convenience for the Cortex side to compute a signature for tests. */
-export function signCallback({ execId, body, timestamp, secret }) {
-    const secretBytes = decodeSecret(secret);
-    const message = `${execId}:${timestamp}:${sha256Hex(body)}`;
-    return createHmac("sha256", secretBytes).update(message).digest("hex");
-}

package/dist/sandbox/submit-exec.d.ts

@@ -1,38 +0,0 @@
-/**
- * `submitExec` — backend-agnostic DBOS step that POSTs a command to
- * sandbox-server's `/exec` and returns after the HTTP 202 ack (ADR
- * 0002). `submitExec` wraps its own POST in a `DBOS.runStep` keyed
- * `sandbox.submit-exec.${execId}`.
- *
- * Step name: `sandbox.submit-exec.${execId}` — `execId` is already unique
- * across the workflow (`${workflowId}:${stepId}:${functionId}`), so it
- * doubles as the DBOS step replay cache key. On replay the cached step
- * output is returned without re-POSTing; any duplicate POST that
- * actually reaches sandbox-server during a narrow in-flight window is
- * deduped server-side (PR #3 change 4) on the same `execId`.
- *
- * A non-202 status throws — the step boundary is throw-based by design
- * (the `resultStep` principle): durability records a step as failed only
- * on a thrown exception, and a throw lets DBOS cancellation propagate
- * untouched.
- */
-import type { SandboxRef, SubmitExecBody } from "./types.js";
-export interface SubmitExecDeps {
-    /** Injected for tests. Defaults to `globalThis.fetch`. */
-    fetch?: typeof fetch;
-    /** Injected for tests. Defaults to `DBOS.runStep`. */
-    runStep?: <T>(fn: () => Promise<T>, config: {
-        name: string;
-    }) => Promise<T>;
-    /**
-     * Per-stream retention budget to send when the body does not carry one.
-     * Defaults to `EXEC_STREAM_BYTE_CAP`, the maximum of a kept tool output.
-     */
-    execStreamByteCap?: number;
-}
-/**
- * Submit an exec as a DBOS step. The step name is unique per execId, so
- * a workflow that has already submitted the exec replays without issuing
- * another POST.
- */
-export declare function submitExec(ref: SandboxRef, body: SubmitExecBody, deps?: SubmitExecDeps): Promise<void>;

package/dist/sandbox/submit-exec.js

@@ -1,80 +0,0 @@
-/**
- * `submitExec` — backend-agnostic DBOS step that POSTs a command to
- * sandbox-server's `/exec` and returns after the HTTP 202 ack (ADR
- * 0002). `submitExec` wraps its own POST in a `DBOS.runStep` keyed
- * `sandbox.submit-exec.${execId}`.
- *
- * Step name: `sandbox.submit-exec.${execId}` — `execId` is already unique
- * across the workflow (`${workflowId}:${stepId}:${functionId}`), so it
- * doubles as the DBOS step replay cache key. On replay the cached step
- * output is returned without re-POSTing; any duplicate POST that
- * actually reaches sandbox-server during a narrow in-flight window is
- * deduped server-side (PR #3 change 4) on the same `execId`.
- *
- * A non-202 status throws — the step boundary is throw-based by design
- * (the `resultStep` principle): durability records a step as failed only
- * on a thrown exception, and a throw lets DBOS cancellation propagate
- * untouched.
- */
-import { DBOS } from "@dbos-inc/dbos-sdk";
-import { ATTR_INFLEXA_EXEC_ID, stableSpan, untracedFetch } from "../lib/otel-spans.js";
-import { EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
-import { signCallback } from "./hmac.js";
-const SUBMIT_HTTP_TIMEOUT_MS = 30_000;
-async function postExec(fetchImpl, ref, body) {
-    const controller = new AbortController();
-    const timer = setTimeout(() => controller.abort(), SUBMIT_HTTP_TIMEOUT_MS);
-    try {
-        // Sign the exact bytes we send, over the same HMAC construction the
-        // sandbox uses for its callbacks — sandbox-server authenticates every
-        // inbound request. `Date.now()` is safe here: this whole POST runs inside
-        // a `DBOS.runStep` whose cached result is replayed without re-executing
-        // the body, so the timestamp never re-computes on recovery.
-        const raw = JSON.stringify(body);
-        const timestamp = Math.floor(Date.now() / 1000);
-        const signature = signCallback({ execId: body.execId, body: raw, timestamp, secret: ref.callbackSecret });
-        const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec`, {
-            method: "POST",
-            headers: {
-                "content-type": "application/json",
-                "x-sandbox-signature": signature,
-                "x-sandbox-timestamp": String(timestamp),
-            },
-            body: raw,
-            signal: controller.signal,
-        });
-        if (res.status !== 202) {
-            const text = await res.text().catch(() => "");
-            throw Object.assign(new Error(`submitExec: sandbox-server returned ${res.status} for execId=${body.execId}: ${text}`), { statusCode: res.status });
-        }
-        // The 202 body carries either `{ status: "started" }` (fresh) or a
-        // dedup-hit summary. Both are success — read and discard so the
-        // socket is released.
-        await res.text().catch(() => "");
-    }
-    finally {
-        clearTimeout(timer);
-    }
-}
-/**
- * Submit an exec as a DBOS step. The step name is unique per execId, so
- * a workflow that has already submitted the exec replays without issuing
- * another POST.
- */
-export async function submitExec(ref, body, deps = {}) {
-    const fetchImpl = deps.fetch ?? fetch;
-    const runStep = deps.runStep ?? ((fn, c) => DBOS.runStep(fn, c));
-    // Every exec in the process funnels through here, so this is where the
-    // retention budget is attached rather than at each of the callers that build
-    // a body. Applied before signing, since the signature covers these bytes.
-    const capped = {
-        ...body,
-        stdoutByteCap: body.stdoutByteCap ?? deps.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP,
-        stderrByteCap: body.stderrByteCap ?? deps.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP,
-    };
-    const stepName = `sandbox.submit-exec.${body.execId}`;
-    await runStep(() => {
-        stableSpan(stepName, "sandbox.submit-exec", { [ATTR_INFLEXA_EXEC_ID]: body.execId });
-        return postExec(fetchImpl, ref, capped);
-    }, { name: stepName });
-}

package/dist/sandbox/watchdog.d.ts

@@ -1,71 +0,0 @@
-/**
- * Per-sandbox-machine liveness watchdog (CONTEXT.md "Sandbox exec").
- *
- * A `@DBOS.scheduled` parent fires every minute, lists the active-sandbox
- * registry, shards by `hash(sandboxId) % SHARD_COUNT`, and `startWorkflow`s
- * one child check workflow per non-empty shard. The fan-out invariant —
- * "no single invocation polls every sandbox" — is the user's explicit
- * constraint.
- *
- * Each child workflow walks its shard, calls `isAlive`, and on `false`
- * gates on `DBOS.getWorkflowStatus`: if the owning workflow is still
- * `PENDING` or `ENQUEUED`, the child sends a `synthetic-failure`
- * done-marker on `exec-event:${execId}` to unblock `awaitExec`. The
- * marker carries `signature: null` and `kind: "synthetic-failure"` so the
- * recv loop accepts it from the trusted in-process sender (see the harness-sandbox-exec spec).
- *
- * The pure sharding and check logic is hoisted so it can be unit-tested
- * without spinning a DBOS runtime; `registerWatchdog` is the thin
- * production wiring.
- */
-import type { ResultAsync } from "neverthrow";
-import type { Logger } from "../lib/logger.js";
-import type { DbError } from "../lib/db-result.js";
-import type { ActiveSandboxRow } from "../state/index.js";
-import type { SandboxClient } from "./client.js";
-import type { ExecEventMessage, ExecResult, SandboxLiveness, SandboxRef } from "./types.js";
-export declare const SHARD_COUNT = 8;
-/** Stable hash → shard mapping, exported for tests. */
-export declare function shardIndex(sandboxId: string, shardCount?: number): number;
-/** Partition the active-sandbox set into `shardCount` lists. */
-export declare function shardActiveSandboxes(rows: ActiveSandboxRow[], shardCount?: number): ActiveSandboxRow[][];
-export interface CheckShardDeps {
-    isAlive: (ref: SandboxRef) => Promise<SandboxLiveness>;
-    getStatus: (workflowId: string) => Promise<{
-        status: string;
-    } | null>;
-    /**
-     * Send a synthetic-failure marker on the per-exec topic. In production
-     * this is `DBOS.send`; injected here for testability.
-     */
-    sendSynthetic: (workflowId: string, execId: string, failure: ExecResult, reason: string) => Promise<void>;
-    logger?: Logger;
-}
-export interface CheckShardSummary {
-    activeCount: number;
-    deadCount: number;
-    syntheticSends: number;
-    liveWorkflowsSkipped: number;
-}
-/**
- * Pure check logic over a shard: enumerate, call `isAlive`, and on
- * dead-and-still-in-flight send a synthetic-failure. The in-flight guard
- * is critical — without it a real `complete` arriving microseconds before
- * the watchdog would race with the synthetic and produce a duplicate
- * notification.
- */
-export declare function checkShard(rows: ActiveSandboxRow[], deps: CheckShardDeps): Promise<CheckShardSummary>;
-/** Encodes the synthetic-failure marker on the per-exec topic envelope. */
-export declare function syntheticFailureMessage(execId: string, failure: ExecResult, reason: string, nowSec: number): ExecEventMessage;
-export interface WatchdogDeps {
-    queryActiveSandboxes: () => ResultAsync<ActiveSandboxRow[], DbError>;
-    sandboxClient: SandboxClient;
-    shardCount?: number;
-    logger?: Logger;
-}
-/**
- * Production registration. Builds the parent + child registered workflows
- * and the scheduled trigger. The parent fans out via `startWorkflow` so
- * each child is durable in its own right (and recovers independently).
- */
-export declare function registerWatchdog(deps: WatchdogDeps): void;

package/dist/sandbox/watchdog.js

@@ -1,161 +0,0 @@
-/**
- * Per-sandbox-machine liveness watchdog (CONTEXT.md "Sandbox exec").
- *
- * A `@DBOS.scheduled` parent fires every minute, lists the active-sandbox
- * registry, shards by `hash(sandboxId) % SHARD_COUNT`, and `startWorkflow`s
- * one child check workflow per non-empty shard. The fan-out invariant —
- * "no single invocation polls every sandbox" — is the user's explicit
- * constraint.
- *
- * Each child workflow walks its shard, calls `isAlive`, and on `false`
- * gates on `DBOS.getWorkflowStatus`: if the owning workflow is still
- * `PENDING` or `ENQUEUED`, the child sends a `synthetic-failure`
- * done-marker on `exec-event:${execId}` to unblock `awaitExec`. The
- * marker carries `signature: null` and `kind: "synthetic-failure"` so the
- * recv loop accepts it from the trusted in-process sender (see the harness-sandbox-exec spec).
- *
- * The pure sharding and check logic is hoisted so it can be unit-tested
- * without spinning a DBOS runtime; `registerWatchdog` is the thin
- * production wiring.
- */
-import { createHash } from "node:crypto";
-import { DBOS } from "@dbos-inc/dbos-sdk";
-import { createNoopLogger } from "../lib/console-logger.js";
-import { untracedWorkflow } from "../lib/otel-spans.js";
-import { unwrapOrThrow } from "../lib/result.js";
-import { workflowIdFromExec } from "./exec-id.js";
-import { syntheticFailureReason, syntheticFailureResult } from "./liveness.js";
-export const SHARD_COUNT = 8;
-const WATCHDOG_CRON = "*/60 * * * * *";
-const WATCHDOG_PARENT_WORKFLOW = "sandbox-watchdog-parent";
-const WATCHDOG_SHARD_WORKFLOW = "sandbox-watchdog-shard";
-/** Stable hash → shard mapping, exported for tests. */
-export function shardIndex(sandboxId, shardCount = SHARD_COUNT) {
-    const digest = createHash("sha1").update(sandboxId).digest();
-    const slice = digest.readUInt32BE(0);
-    return slice % shardCount;
-}
-/** Partition the active-sandbox set into `shardCount` lists. */
-export function shardActiveSandboxes(rows, shardCount = SHARD_COUNT) {
-    const shards = Array.from({ length: shardCount }, () => []);
-    for (const row of rows) {
-        shards[shardIndex(row.sandboxRef.sandboxId, shardCount)].push(row);
-    }
-    return shards;
-}
-const ACTIVE_WORKFLOW_STATUSES = new Set(["PENDING", "ENQUEUED"]);
-/**
- * Pure check logic over a shard: enumerate, call `isAlive`, and on
- * dead-and-still-in-flight send a synthetic-failure. The in-flight guard
- * is critical — without it a real `complete` arriving microseconds before
- * the watchdog would race with the synthetic and produce a duplicate
- * notification.
- */
-export async function checkShard(rows, deps) {
-    const logger = (deps.logger ?? createNoopLogger()).named("sandbox-watchdog");
-    let deadCount = 0;
-    let syntheticSends = 0;
-    let liveWorkflowsSkipped = 0;
-    for (const row of rows) {
-        // We don't know the secret from here; reconstruct a minimal in-memory
-        // SandboxRef from the persisted shape. `isAlive` does not consult
-        // the secret, so this is safe.
-        const ref = {
-            ...row.sandboxRef,
-            callbackSecret: "",
-        };
-        let liveness;
-        try {
-            liveness = await deps.isAlive(ref);
-        }
-        catch (err) {
-            logger.warn("isAlive threw — skipping this round", {
-                sandboxId: row.sandboxRef.sandboxId,
-                ...logger.errorFields(err),
-            });
-            continue;
-        }
-        if (liveness.alive)
-            continue;
-        deadCount++;
-        const workflowId = row.execId ? workflowIdFromExec(row.execId) : null;
-        if (!workflowId || !row.execId) {
-            // No exec was in flight when this row was tagged; nothing to
-            // unblock. Future change: clear the registry row separately.
-            continue;
-        }
-        const status = await deps.getStatus(workflowId);
-        if (!status || !ACTIVE_WORKFLOW_STATUSES.has(status.status)) {
-            liveWorkflowsSkipped++;
-            continue;
-        }
-        const reason = syntheticFailureReason(liveness);
-        const failure = syntheticFailureResult(row.execId, reason);
-        await deps.sendSynthetic(workflowId, row.execId, failure, reason);
-        syntheticSends++;
-    }
-    const summary = {
-        activeCount: rows.length,
-        deadCount,
-        syntheticSends,
-        liveWorkflowsSkipped,
-    };
-    // Spread: an interface has no implicit index signature, so it needs widening to `LogFields`.
-    logger.info("shard check completed", { ...summary });
-    return summary;
-}
-/** Encodes the synthetic-failure marker on the per-exec topic envelope. */
-export function syntheticFailureMessage(execId, failure, reason, nowSec) {
-    return {
-        payload: {
-            done: true,
-            result: failure,
-            kind: "synthetic-failure",
-            reason,
-        },
-        signature: null,
-        timestamp: nowSec,
-    };
-}
-/**
- * Production registration. Builds the parent + child registered workflows
- * and the scheduled trigger. The parent fans out via `startWorkflow` so
- * each child is durable in its own right (and recovers independently).
- */
-export function registerWatchdog(deps) {
-    const shardCount = deps.shardCount ?? SHARD_COUNT;
-    const childCheckShard = DBOS.registerWorkflow(async (rows) => {
-        return checkShard(rows, {
-            isAlive: (ref) => deps.sandboxClient.isAlive(ref),
-            getStatus: async (workflowId) => (await DBOS.getWorkflowStatus(workflowId)),
-            sendSynthetic: async (workflowId, execId, failure, reason) => {
-                const msg = syntheticFailureMessage(execId, failure, reason, Math.floor((await DBOS.now()) / 1000));
-                await DBOS.send(workflowId, msg, `exec-event:${execId}`);
-            },
-            logger: deps.logger,
-        });
-    }, { name: WATCHDOG_SHARD_WORKFLOW });
-    untracedWorkflow(WATCHDOG_SHARD_WORKFLOW);
-    // The scheduled target must itself be a registered workflow — DBOS's
-    // scheduler loop skips (and errors on) any `@scheduled` function that lacks
-    // a workflow registration. Register, then schedule the same callable.
-    const watchdogParent = DBOS.registerWorkflow(async () => {
-        // Checkpoint the live registry read: the row set gates how many
-        // `startWorkflow` calls fire below, so a differing set on replay would
-        // drift the recorded function-ID sequence (see the harness-durable-runtime spec).
-        const rows = await DBOS.runStep(async () => unwrapOrThrow(await deps.queryActiveSandboxes()), { name: "query-active-sandboxes" });
-        if (rows.length === 0)
-            return;
-        const shards = shardActiveSandboxes(rows, shardCount);
-        for (const shard of shards) {
-            if (shard.length === 0)
-                continue;
-            await DBOS.startWorkflow(childCheckShard)(shard);
-        }
-    }, { name: WATCHDOG_PARENT_WORKFLOW });
-    untracedWorkflow(WATCHDOG_PARENT_WORKFLOW);
-    DBOS.registerScheduled(watchdogParent, {
-        name: WATCHDOG_PARENT_WORKFLOW,
-        crontab: WATCHDOG_CRON,
-    });
-}