@inflexa-ai/harness 0.27.0 → 0.28.0

package/dist/sandbox/await-exec.js

@@ -55,6 +55,7 @@
  * varied between replays would desynchronise the recorded function-ID sequence.
  */
 import { DBOS } from "@dbos-inc/dbos-sdk";
+import { digestBody } from "./digest.js";
 import { signCallback, verifyCallback } from "./hmac.js";
 import { createEscalationPolicy, probeLiveness, syntheticFailureReason, syntheticFailureResult } from "./liveness.js";
 import { ExecResultSchema, isDoneMarker, isSyntheticFailure, PollResponseSchema, } from "./types.js";
@@ -233,13 +234,16 @@ export async function awaitExecCallback(ref, execId, emit, deadlineMs, options =
             throw new HardCancelError(execId, "synthetic-without-marker");
         }
         // Verify against the exact bytes sandbox-server signed. The route
-        // preserves them as `payloadRaw`; the JSON.stringify fallback exists
-        // only for messages persisted before that field was added — re-
-        // serializing diverges from Go's HTML-escaping encoder.
-        const bodyBytes = Buffer.from(msg.payloadRaw ?? JSON.stringify(msg.payload), "utf8");
+        // reduces them to `payloadDigest` at ingress — the signature covers that
+        // digest, so it is all verification needs and the message never carries
+        // a second copy of the body. `payloadRaw` is the superseded form, and
+        // the JSON.stringify fallback covers messages persisted before either
+        // field existed — re-serializing diverges from Go's HTML-escaping
+        // encoder, so it is a last resort, not a normal path.
+        const bodyDigest = msg.payloadDigest ?? digestBody(Buffer.from(msg.payloadRaw ?? JSON.stringify(msg.payload), "utf8"));
         const result = verifyCallback({
             execId,
-            body: bodyBytes,
+            bodyDigest,
             signature: msg.signature,
             timestamp: msg.timestamp,
             secret: callbackSecret,

package/dist/sandbox/create-sandbox.d.ts

@@ -45,6 +45,13 @@ export interface CreateSandboxClientConfig {
     transport?: SandboxTransport;
     /** Default sandbox-base image when the workflow doesn't override per-step. */
     image: string;
+    /**
+     * Per-stream retention budget sent with every exec, so the sandbox drops
+     * output past it instead of shipping it. Defaults to `EXEC_STREAM_BYTE_CAP`,
+     * the same value the host truncates to on receipt — sending more than the
+     * host will keep buys nothing.
+     */
+    execStreamByteCap?: number;
     /** Cluster resource ceilings; every sandbox request is clamped to these. */
     resourceLimits: ResourceLimits;
     /** K8s namespace; only used by the k8s backend. */

package/dist/sandbox/create-sandbox.js

@@ -18,6 +18,7 @@ import { tailWritePrefix } from "../workspace/paths.js";
 import { tryMutation } from "../lib/db-result.js";
 import { unwrapOrThrow } from "../lib/result.js";
 import { clearSandboxRef, setSandboxRef } from "../state/index.js";
+import { capExecStreams, EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
 import { awaitExec } from "./await-exec.js";
 import { createDockerSandboxOps } from "./docker-client.js";
 import { createK8sSandboxOps } from "./k8s-client.js";
@@ -145,8 +146,16 @@ export function createSandboxClient(config) {
             const resources = clampResources(meta.resources, config.resourceLimits);
             return unwrapOrThrow(await ops.createSandbox({ ...meta, resources }, identity));
         },
-        submitExec: async (ref, body) => submitExec(ref, body, config.submitDeps),
-        awaitExec: (ref, execId, emit, deadline) => awaitExec(ref, execId, emit, deadline, composeAwaitOptions(config.awaitOptions, transport, isAlive)),
+        submitExec: async (ref, body) => submitExec(ref, body, {
+            ...config.submitDeps,
+            execStreamByteCap: config.submitDeps?.execStreamByteCap ?? config.execStreamByteCap,
+        }),
+        // The one place an ExecResult crosses from the wire into the process, so
+        // the cap lands here rather than at each consumer. Every downstream use —
+        // tool results, workflow return values, durable step outputs — is bounded
+        // by construction, including against a sandbox image that predates the
+        // retention budget and still returns whole streams.
+        awaitExec: async (ref, execId, emit, deadline) => capExecStreams(await awaitExec(ref, execId, emit, deadline, composeAwaitOptions(config.awaitOptions, transport, isAlive)), config.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP),
         isAlive,
         isAliveById: async (sandboxId) => unwrapOrThrow(await ops.isAliveById(sandboxId)),
         teardown,

package/dist/sandbox/digest.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Body digest for sandbox callbacks.
+ *
+ * The callback signature is taken over the hex SHA-256 of the POSTed body
+ * rather than over the body itself, so the digest is a value the ingress can
+ * compute and forward while the bytes themselves are dropped — verification
+ * downstream needs nothing more.
+ *
+ * This lives apart from `hmac.ts` on purpose. Hashing carries no secret and
+ * decides nothing, so the route that only forwards can depend on it without
+ * taking on the verification machinery it is required to stay clear of.
+ */
+/** Hex SHA-256 of the POSTed bytes — the value the callback signature covers. */
+export declare function digestBody(input: Buffer | string): string;

package/dist/sandbox/digest.js

@@ -0,0 +1,17 @@
+/**
+ * Body digest for sandbox callbacks.
+ *
+ * The callback signature is taken over the hex SHA-256 of the POSTed body
+ * rather than over the body itself, so the digest is a value the ingress can
+ * compute and forward while the bytes themselves are dropped — verification
+ * downstream needs nothing more.
+ *
+ * This lives apart from `hmac.ts` on purpose. Hashing carries no secret and
+ * decides nothing, so the route that only forwards can depend on it without
+ * taking on the verification machinery it is required to stay clear of.
+ */
+import { createHash } from "node:crypto";
+/** Hex SHA-256 of the POSTed bytes — the value the callback signature covers. */
+export function digestBody(input) {
+    return createHash("sha256").update(input).digest("hex");
+}

package/dist/sandbox/hmac.d.ts

@@ -16,8 +16,16 @@
  */
 export interface VerifyCallbackInput {
     execId: string;
-    /** The bytes the sandbox-server POSTed. The dumb route preserves them. */
-    body: Buffer | string;
+    /**
+     * The bytes the sandbox-server POSTed. Supply this or `bodyDigest`.
+     *
+     * The signature covers the hex SHA-256 of the body, never the body itself,
+     * so a caller that has already hashed the bytes can verify from the digest
+     * alone — which is what keeps a large payload from being carried twice.
+     */
+    body?: Buffer | string;
+    /** Hex SHA-256 of the POSTed bytes. Takes precedence over `body`. */
+    bodyDigest?: string;
     /** `X-Sandbox-Signature` value (lowercase hex). */
     signature: string | null;
     /** `X-Sandbox-Timestamp` value (unix seconds). */

package/dist/sandbox/hmac.js

@@ -14,7 +14,8 @@
  * NetworkPolicy isolation a forged event implies a bug or a breach,
  * neither of which should keep a 3-hour sandbox burning.
  */
-import { createHash, createHmac, timingSafeEqual } from "node:crypto";
+import { createHmac, timingSafeEqual } from "node:crypto";
+import { digestBody } from "./digest.js";
 /**
  * Decode the per-sandbox secret. sandbox-server accepts both raw UTF-8
  * and `base64:` prefixed values (change 4 spec); we mirror the same
@@ -28,7 +29,7 @@ function decodeSecret(secret) {
     return Buffer.from(secret, "utf8");
 }
 function sha256Hex(input) {
-    return createHash("sha256").update(input).digest("hex");
+    return digestBody(input);
 }
 /**
  * Recompute the HMAC and compare in constant time, then check the
@@ -43,8 +44,12 @@ export function verifyCallback(input) {
     if (input.signature === null || input.timestamp === null) {
         return { valid: false, reason: "missing" };
     }
+    const digest = input.bodyDigest ?? (input.body !== undefined ? sha256Hex(input.body) : undefined);
+    if (digest === undefined) {
+        return { valid: false, reason: "missing" };
+    }
     const secretBytes = decodeSecret(input.secret);
-    const message = `${input.execId}:${input.timestamp}:${sha256Hex(input.body)}`;
+    const message = `${input.execId}:${input.timestamp}:${digest}`;
     const expected = createHmac("sha256", secretBytes).update(message).digest("hex");
     const provided = input.signature;
     if (provided.length !== expected.length) {

package/dist/sandbox/submit-exec.d.ts

@@ -24,6 +24,11 @@ export interface SubmitExecDeps {
     runStep?: <T>(fn: () => Promise<T>, config: {
         name: string;
     }) => Promise<T>;
+    /**
+     * Per-stream retention budget to send when the body does not carry one.
+     * Defaults to `EXEC_STREAM_BYTE_CAP`.
+     */
+    execStreamByteCap?: number;
 }
 /**
  * Submit an exec as a DBOS step. The step name is unique per execId, so

package/dist/sandbox/submit-exec.js

@@ -17,6 +17,7 @@
  * untouched.
  */
 import { DBOS } from "@dbos-inc/dbos-sdk";
+import { EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
 import { signCallback } from "./hmac.js";
 const SUBMIT_HTTP_TIMEOUT_MS = 30_000;
 async function postExec(fetchImpl, ref, body) {
@@ -62,7 +63,15 @@ async function postExec(fetchImpl, ref, body) {
 export async function submitExec(ref, body, deps = {}) {
     const fetchImpl = deps.fetch ?? fetch;
     const runStep = deps.runStep ?? ((fn, c) => DBOS.runStep(fn, c));
-    await runStep(() => postExec(fetchImpl, ref, body), {
+    // Every exec in the process funnels through here, so this is where the
+    // retention budget is attached rather than at each of the callers that build
+    // a body. Applied before signing, since the signature covers these bytes.
+    const capped = {
+        ...body,
+        stdoutByteCap: body.stdoutByteCap ?? deps.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP,
+        stderrByteCap: body.stderrByteCap ?? deps.execStreamByteCap ?? EXEC_STREAM_BYTE_CAP,
+    };
+    await runStep(() => postExec(fetchImpl, ref, capped), {
         name: `sandbox.submit-exec.${body.execId}`,
     });
 }

package/dist/sandbox/types.d.ts

@@ -103,6 +103,10 @@ export declare const ExecResultSchema: z.ZodObject<{
     stderr: z.ZodDefault<z.ZodString>;
     durationMs: z.ZodNullable<z.ZodNumber>;
     timedOut: z.ZodDefault<z.ZodBoolean>;
+    stdoutTruncated: z.ZodOptional<z.ZodBoolean>;
+    stderrTruncated: z.ZodOptional<z.ZodBoolean>;
+    stdoutTotalBytes: z.ZodOptional<z.ZodNumber>;
+    stderrTotalBytes: z.ZodOptional<z.ZodNumber>;
     syntheticFailure: z.ZodOptional<z.ZodObject<{
         reason: z.ZodString;
     }, z.core.$strip>>;
@@ -128,15 +132,21 @@ export type ExecResult = z.infer<typeof ExecResultSchema>;
  * a non-null `signature` and `timestamp`; the in-process watchdog uses a
  * `null` signature + a `synthetic-failure` payload (see `await-exec.ts`).
  *
- * `payloadRaw` is the exact bytes sandbox-server POSTed — sandbox-server's
- * HMAC signs the raw body, and Go's `encoding/json` HTML-escapes `<`, `>`,
- * `&` by default, so verifying against a JS re-serialization of the parsed
- * payload would diverge for any output containing those characters
- * (common in bioinformatics: FASTA headers, shell stderr, command pipelines).
- * Optional only for back-compat with messages that pre-date this field.
+ * `payloadDigest` is the hex SHA-256 of the exact bytes sandbox-server POSTed.
+ * The HMAC signs that digest rather than the body itself, so the digest is all
+ * verification needs — and carrying it instead of the bytes keeps the message
+ * from holding a second copy of a payload that can be arbitrarily large.
+ * Re-serializing the parsed payload would not do: Go's `encoding/json`
+ * HTML-escapes `<`, `>`, `&` by default, so a JS re-serialization diverges for
+ * any output containing those characters (common in bioinformatics: FASTA
+ * headers, shell stderr, command pipelines).
+ *
+ * `payloadRaw` is the superseded form, kept optional so messages already in
+ * flight or persisted still verify.
  */
 export declare const ExecEventMessageSchema: z.ZodObject<{
     payload: z.ZodUnknown;
+    payloadDigest: z.ZodOptional<z.ZodString>;
     payloadRaw: z.ZodOptional<z.ZodString>;
     signature: z.ZodNullable<z.ZodString>;
     timestamp: z.ZodNullable<z.ZodNumber>;
@@ -175,6 +185,10 @@ export declare const PollResponseSchema: z.ZodObject<{
         stderr: z.ZodDefault<z.ZodString>;
         durationMs: z.ZodNullable<z.ZodNumber>;
         timedOut: z.ZodDefault<z.ZodBoolean>;
+        stdoutTruncated: z.ZodOptional<z.ZodBoolean>;
+        stderrTruncated: z.ZodOptional<z.ZodBoolean>;
+        stdoutTotalBytes: z.ZodOptional<z.ZodNumber>;
+        stderrTotalBytes: z.ZodOptional<z.ZodNumber>;
         syntheticFailure: z.ZodOptional<z.ZodObject<{
             reason: z.ZodString;
         }, z.core.$strip>>;
@@ -220,6 +234,14 @@ export interface SubmitExecBody {
     cwd?: string;
     env?: Record<string, string>;
     timeoutSeconds?: number;
+    /**
+     * Per-stream retention budget for the sandbox. Omitting them leaves the
+     * sandbox unbounded, which is how a server that pre-dates the fields
+     * behaves; the host caps on receipt regardless, so the difference is how
+     * many bytes cross the wire, not what the caller ends up with.
+     */
+    stdoutByteCap?: number;
+    stderrByteCap?: number;
 }
 /** Step-meta passed to `createSandbox` — what the registry row needs. */
 export interface CreateSandboxMeta {

package/dist/sandbox/types.js

@@ -79,6 +79,19 @@ export const ExecResultSchema = z.object({
     stderr: z.string().default(""),
     durationMs: z.number().nullable(),
     timedOut: z.boolean().default(false),
+    /**
+     * Set when sandbox-server dropped output past the per-stream cap it was
+     * given. The total is what the command actually produced, which is what
+     * separates "printed nothing" from "printed more than we kept" — without it
+     * a capped stream and an empty one are indistinguishable downstream.
+     *
+     * Optional because a sandbox image that pre-dates the cap omits both, and
+     * because the watchdog's synthetic failures carry neither.
+     */
+    stdoutTruncated: z.boolean().optional(),
+    stderrTruncated: z.boolean().optional(),
+    stdoutTotalBytes: z.number().int().nonnegative().optional(),
+    stderrTotalBytes: z.number().int().nonnegative().optional(),
     /** Set when the watchdog synthesises a completion for a dead sandbox. */
     syntheticFailure: z
         .object({
@@ -97,15 +110,21 @@ export const ExecResultSchema = z.object({
  * a non-null `signature` and `timestamp`; the in-process watchdog uses a
  * `null` signature + a `synthetic-failure` payload (see `await-exec.ts`).
  *
- * `payloadRaw` is the exact bytes sandbox-server POSTed — sandbox-server's
- * HMAC signs the raw body, and Go's `encoding/json` HTML-escapes `<`, `>`,
- * `&` by default, so verifying against a JS re-serialization of the parsed
- * payload would diverge for any output containing those characters
- * (common in bioinformatics: FASTA headers, shell stderr, command pipelines).
- * Optional only for back-compat with messages that pre-date this field.
+ * `payloadDigest` is the hex SHA-256 of the exact bytes sandbox-server POSTed.
+ * The HMAC signs that digest rather than the body itself, so the digest is all
+ * verification needs — and carrying it instead of the bytes keeps the message
+ * from holding a second copy of a payload that can be arbitrarily large.
+ * Re-serializing the parsed payload would not do: Go's `encoding/json`
+ * HTML-escapes `<`, `>`, `&` by default, so a JS re-serialization diverges for
+ * any output containing those characters (common in bioinformatics: FASTA
+ * headers, shell stderr, command pipelines).
+ *
+ * `payloadRaw` is the superseded form, kept optional so messages already in
+ * flight or persisted still verify.
  */
 export const ExecEventMessageSchema = z.object({
     payload: z.unknown(),
+    payloadDigest: z.string().optional(),
     payloadRaw: z.string().optional(),
     signature: z.string().nullable(),
     timestamp: z.number().int().nullable(),

package/dist/tools/workspace/result-bounds.d.ts

@@ -8,9 +8,14 @@
  * The cap is well below the loop's overall result budget so a chatty
  * command cannot blow the context window even when stdout AND stderr both
  * cap out at the limit.
+ *
+ * The same value is sent to the sandbox as a retention budget, so the bytes are
+ * dropped at the producer and never cross the wire. Capping here as well is not
+ * redundant: a sandbox image that pre-dates the budget returns the whole stream,
+ * and this is the boundary that keeps it out of the process either way.
  */
 import type { ExecResult } from "../../sandbox/types.js";
-/** Per-stream cap. 8 KiB — multi-KB but well under the loop result budget. */
+/** Per-stream cap. 32 KiB — roomy for real output, well under the loop result budget. */
 export declare const EXEC_STREAM_BYTE_CAP: number;
 export interface BoundedStream {
     readonly content: string;
@@ -33,10 +38,24 @@ export interface BoundedExecResult {
         readonly reason: string;
     };
 }
+/**
+ * Bound an `ExecResult`'s streams while keeping its shape.
+ *
+ * `boundExecResult` produces the tool-facing form; this is for the boundaries
+ * that have to forward an `ExecResult` onward — notably a workflow return value,
+ * which DBOS serializes into its durable step output. Returns the input
+ * unchanged when nothing needed cutting, so the common path allocates nothing.
+ */
+export declare function capExecStreams(result: ExecResult, cap?: number): ExecResult;
 /**
  * Apply per-stream truncation to an `ExecResult`. `exitCode`, `durationMs`,
  * `timedOut`, and `syntheticFailure` pass through unchanged regardless of
  * stream truncation — the discriminants the loop and the model rely on are
  * preserved.
+ *
+ * When the sandbox already truncated at the producer, what arrives here is the
+ * retained slice, not the stream — so its length is the wrong number to report.
+ * The sandbox's own totals win where present, and truncation is the union of the
+ * two cuts: either side having dropped bytes makes the result truncated.
  */
 export declare function boundExecResult(result: ExecResult, cap?: number): BoundedExecResult;

package/dist/tools/workspace/result-bounds.js

@@ -8,16 +8,46 @@
  * The cap is well below the loop's overall result budget so a chatty
  * command cannot blow the context window even when stdout AND stderr both
  * cap out at the limit.
+ *
+ * The same value is sent to the sandbox as a retention budget, so the bytes are
+ * dropped at the producer and never cross the wire. Capping here as well is not
+ * redundant: a sandbox image that pre-dates the budget returns the whole stream,
+ * and this is the boundary that keeps it out of the process either way.
+ */
+/** Per-stream cap. 32 KiB — roomy for real output, well under the loop result budget. */
+export const EXEC_STREAM_BYTE_CAP = 32 * 1024;
+/**
+ * Decode `buf` as UTF-8, dropping a trailing partial sequence rather than
+ * letting it decode to U+FFFD. Only the last few bytes can be partial, so the
+ * scan is bounded by the maximum UTF-8 sequence length.
  */
-/** Per-stream cap. 8 KiB — multi-KB but well under the loop result budget. */
-export const EXEC_STREAM_BYTE_CAP = 8 * 1024;
+function trimPartialSequence(buf) {
+    const MAX_UTF8_SEQUENCE = 4;
+    for (let drop = 0; drop < MAX_UTF8_SEQUENCE && drop < buf.length; drop++) {
+        const candidate = buf.subarray(0, buf.length - drop);
+        const decoded = candidate.toString("utf8");
+        if (!decoded.endsWith("�")) {
+            return decoded;
+        }
+    }
+    // A stream that genuinely ends in U+FFFD is returned as-is; dropping more
+    // would start eating real characters.
+    return buf.toString("utf8");
+}
 function boundStream(stream, cap = EXEC_STREAM_BYTE_CAP) {
     const totalLength = Buffer.byteLength(stream, "utf8");
     if (totalLength <= cap) {
         return { content: stream, truncated: false, totalLength, returnedBytes: totalLength };
     }
-    const buf = Buffer.from(stream, "utf8").subarray(0, cap);
-    const content = buf.toString("utf8");
+    // Slice the string before encoding. Encoding first allocates a Buffer the
+    // size of the whole stream in order to keep `cap` bytes of it, which makes
+    // the function that exists to bound a stream the largest allocation on the
+    // path. One UTF-16 code unit encodes to at most 3 UTF-8 bytes, so `cap`
+    // units always cover `cap` bytes and the encode stays bounded.
+    const buf = Buffer.from(stream.slice(0, cap), "utf8").subarray(0, cap);
+    // The byte cut can land mid-sequence, and decoding that directly ends the
+    // content in U+FFFD. Trim back to the last whole character instead.
+    const content = trimPartialSequence(buf);
     return {
         content,
         truncated: true,
@@ -25,11 +55,40 @@ function boundStream(stream, cap = EXEC_STREAM_BYTE_CAP) {
         returnedBytes: Buffer.byteLength(content, "utf8"),
     };
 }
+/**
+ * Bound an `ExecResult`'s streams while keeping its shape.
+ *
+ * `boundExecResult` produces the tool-facing form; this is for the boundaries
+ * that have to forward an `ExecResult` onward — notably a workflow return value,
+ * which DBOS serializes into its durable step output. Returns the input
+ * unchanged when nothing needed cutting, so the common path allocates nothing.
+ */
+export function capExecStreams(result, cap = EXEC_STREAM_BYTE_CAP) {
+    const out = boundStream(result.stdout, cap);
+    const err = boundStream(result.stderr, cap);
+    if (!out.truncated && !err.truncated) {
+        return result;
+    }
+    return {
+        ...result,
+        stdout: out.content,
+        stderr: err.content,
+        stdoutTruncated: out.truncated || (result.stdoutTruncated ?? false),
+        stderrTruncated: err.truncated || (result.stderrTruncated ?? false),
+        stdoutTotalBytes: result.stdoutTotalBytes ?? out.totalLength,
+        stderrTotalBytes: result.stderrTotalBytes ?? err.totalLength,
+    };
+}
 /**
  * Apply per-stream truncation to an `ExecResult`. `exitCode`, `durationMs`,
  * `timedOut`, and `syntheticFailure` pass through unchanged regardless of
  * stream truncation — the discriminants the loop and the model rely on are
  * preserved.
+ *
+ * When the sandbox already truncated at the producer, what arrives here is the
+ * retained slice, not the stream — so its length is the wrong number to report.
+ * The sandbox's own totals win where present, and truncation is the union of the
+ * two cuts: either side having dropped bytes makes the result truncated.
  */
 export function boundExecResult(result, cap = EXEC_STREAM_BYTE_CAP) {
     const outStream = boundStream(result.stdout, cap);
@@ -39,10 +98,10 @@ export function boundExecResult(result, cap = EXEC_STREAM_BYTE_CAP) {
         exitCode: result.exitCode,
         stdout: outStream.content,
         stderr: errStream.content,
-        stdoutTruncated: outStream.truncated,
-        stderrTruncated: errStream.truncated,
-        stdoutTotalLength: outStream.totalLength,
-        stderrTotalLength: errStream.totalLength,
+        stdoutTruncated: outStream.truncated || (result.stdoutTruncated ?? false),
+        stderrTruncated: errStream.truncated || (result.stderrTruncated ?? false),
+        stdoutTotalLength: result.stdoutTotalBytes ?? outStream.totalLength,
+        stderrTotalLength: result.stderrTotalBytes ?? errStream.totalLength,
         durationMs: result.durationMs,
         timedOut: result.timedOut,
         ...(result.syntheticFailure ? { syntheticFailure: result.syntheticFailure } : {}),

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@inflexa-ai/harness",
-    "version": "0.27.0",
+    "version": "0.28.0",
     "description": "Host-agnostic bioinformatics agent harness — hand-rolled agent loop, DBOS-durable workflows, sandboxed compute.",
     "license": "Apache-2.0",
     "type": "module",