humanish 0.120.1 → 0.121.0

package/dist/routes/computer-use/types.d.ts

@@ -1,4 +1,5 @@
 import type { Verdict } from "../../run/judge.js";
+import type { HostSuspensionReading } from "../../run/host-suspension.js";
 import type { LocalCapacity } from "../../substrates/local/capacity.js";
 import type { RunDesktopRecording } from "../../evidence/desktop-recording-types.js";
 import type { CuaExecutor, CuaLiveMetadata, CuaLoopResult, CuaProvider } from "../../actors/computer-use/loop.js";
@@ -233,6 +234,11 @@ export interface CuaActorStudyResult extends AutomaticAnalysisResult, StudyResul
     rerun?: RunRerunLineage;
     observer?: ObserverResult;
     diagnostics?: CuaDiagnostics;
+    /**
+     * Present when the host was suspended during the run (a laptop that slept): when, and which
+     * participant failures it likely caused. Each participant's own result is unchanged.
+     */
+    hostSuspension?: HostSuspensionReading;
     warnings: string[];
     error?: {
         code: CuaActorStudyErrorCode;
@@ -391,8 +397,8 @@ export type CuaParticipantDeps = E2BDesktopDeps & ParticipantModelDeps & {
     /** The caller's inProcess executor with the run's config bound. Read by the in-process desktop. */
     inProcessExecutor?: (appUrl: string) => Promise<CuaExecutor>;
     runSession: (options: CuaActorSessionOptions) => Promise<CuaLoopResult>;
-    /** Lane-0 only: signal the pipeline gate after provisioning succeeds (true) or fails (false). */
-    signalProvisioned?: (ok: boolean) => void;
+    /** Called once, when the participant's desktop and model are ready and its session starts. */
+    signalReady?: () => void;
 };
 /** Why a sandbox was not confirmed released. */
 export interface SandboxReleaseFact {
@@ -439,6 +445,12 @@ export interface ParticipantRunOutcome {
     skippedReason?: string;
     /** When a fan-out participant was due and when its desktop was requested. */
     arrival?: ParticipantArrival;
+    /**
+     * When the participant's runner returned this outcome, on the deps clock (epoch ms): after its
+     * session, or after a setup failure that left it none. Absent on a skipped participant, which
+     * never ran, and on outcomes built outside the runner.
+     */
+    endedAt?: number;
     noEngagement: boolean;
     selfReportedBlocker: boolean;
     /** The inclusive friction read: blocker-shaped narration incl. self-resolved arcs.
@@ -470,6 +482,8 @@ export interface CuaFanoutBundleArgs {
     participantPlan: CuaParticipantPlan;
     rerun?: RunRerunLineage;
     failFastReason?: string;
+    /** The run's host suspensions and the participant failures they likely caused. */
+    hostSuspension?: HostSuspensionReading;
     publicRepo?: string;
     inProgress?: boolean;
 }

package/dist/run/host-suspension.d.ts

@@ -0,0 +1,56 @@
+import type { RunEvent } from "./bundle.js";
+/** The run's clock and heartbeat timer. Tests pass one whose wall clock they move. */
+export interface HostClock {
+    /** Wall-clock time in epoch milliseconds. */
+    now(): number;
+    /** Call `tick` every `intervalMs` until the returned function runs. Never keeps the process alive. */
+    every(intervalMs: number, tick: () => void): () => void;
+}
+export declare const systemHostClock: HostClock;
+/** A time the run's heartbeat did not run: from its last tick before the gap to its first after. */
+export interface HostSuspension {
+    startedAt: string;
+    endedAt: string;
+    durationMs: number;
+}
+export interface Heartbeat {
+    /** The suspensions so far. A read also checks the time since the last tick. */
+    suspensions(): readonly HostSuspension[];
+    /** Stop the timer. Later reads return the suspensions recorded until then. */
+    stop(): void;
+}
+/** Run `onTick` every `intervalMs` on `clock`, and record each gap of 30 s or more. */
+export declare function startHeartbeat(clock: HostClock, intervalMs: number, onTick: () => void): Heartbeat;
+/** One warn event per suspension, at its start, naming its start, end, length and offset. */
+export declare function hostSuspensionEvents(suspensions: readonly HostSuspension[], runStartMs: number): RunEvent[];
+/** One participant as the attribution reads it. */
+export interface SuspendedParticipant {
+    id: string;
+    /**
+     * How it did not pass, when a suspension can explain that: an error in the harness, its
+     * session's time limit, or a skip after another participant's harness error. Absent when it
+     * passed or its session ended on its own report.
+     */
+    failure?: "harness" | "time-limit" | "skipped";
+    /** When it started; absent when no record says. */
+    startedAtMs?: number;
+    /** When it stopped, after its session or after a setup failure; absent when no record says. */
+    endedAtMs?: number;
+}
+/** What a run says about its host suspensions. */
+export interface HostSuspensionReading {
+    /** One sentence for the run outcome, the CLI and the review. */
+    summary: string;
+    /** The participants whose failure is likely a suspension's effect, in the order given. */
+    participantIds: string[];
+    suspensions: HostSuspension[];
+}
+/**
+ * Which failures the run's suspensions likely caused, and the sentence that says so; undefined
+ * without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
+ * timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
+ * rule reads who was in flight across the suspension. A skip follows another participant's
+ * harness error (the pipeline gate, fail-fast), so skips count only when the run had harness
+ * errors and every one was in flight across a suspension.
+ */
+export declare function readHostSuspensions(suspensions: readonly HostSuspension[], participants: readonly SuspendedParticipant[], runStartMs: number): HostSuspensionReading | undefined;

package/dist/run/host-suspension.js

@@ -0,0 +1,106 @@
+// Host suspensions: when the machine running a study sleeps, every participant loop on it stops,
+// and on wake its timers and connections fail together. This module notices the gap and says
+// which participant failures it likely caused, so the run names the suspension as their cause.
+//
+// Detection reads the wall clock across a heartbeat. Date.now counts suspended time on macOS and
+// Linux. performance.now and Node's timers run on libuv's clock, which counts sleep on macOS
+// (mach_continuous_time), so the tick after a sleep fires at wake, and stops during a Linux
+// suspend (CLOCK_MONOTONIC), so the tick fires within one interval of resume. Either way the wall
+// clock between two ticks shows the gap.
+import { formatDuration } from "./projection.js";
+import { plural } from "./text.js";
+export const systemHostClock = {
+    now: () => Date.now(),
+    every(intervalMs, tick) {
+        const timer = setInterval(tick, intervalMs);
+        timer.unref?.();
+        return () => clearInterval(timer);
+    },
+};
+/**
+ * A wall-clock gap between two heartbeats this long or longer is a suspension. A shorter one is
+ * most likely a busy event loop.
+ */
+const HOST_SUSPENSION_MIN_MS = 30_000;
+/** Run `onTick` every `intervalMs` on `clock`, and record each gap of 30 s or more. */
+export function startHeartbeat(clock, intervalMs, onTick) {
+    const found = [];
+    let lastMs = clock.now();
+    let stopped = false;
+    const observe = () => {
+        if (stopped)
+            return;
+        const nowMs = clock.now();
+        if (nowMs - lastMs >= HOST_SUSPENSION_MIN_MS)
+            found.push({
+                startedAt: new Date(lastMs).toISOString(),
+                endedAt: new Date(nowMs).toISOString(),
+                durationMs: nowMs - lastMs,
+            });
+        lastMs = nowMs;
+    };
+    const stopTimer = clock.every(intervalMs, () => {
+        observe();
+        onTick();
+    });
+    return {
+        suspensions() {
+            observe();
+            return [...found];
+        },
+        stop() {
+            stopped = true;
+            stopTimer();
+        },
+    };
+}
+/** `6m 5s at +1m`: how long, and how far into the run it started. */
+function span(suspension, runStartMs) {
+    const offset = Date.parse(suspension.startedAt) - runStartMs;
+    return `${formatDuration(suspension.durationMs)} at +${formatDuration(offset)}`;
+}
+/** One warn event per suspension, at its start, naming its start, end, length and offset. */
+export function hostSuspensionEvents(suspensions, runStartMs) {
+    return suspensions.map((suspension, index) => ({
+        id: `event-host-suspended-${String(index + 1).padStart(3, "0")}`,
+        at: suspension.startedAt,
+        level: "warn",
+        type: "host.suspended",
+        message: `The host was suspended for ${formatDuration(suspension.durationMs)}, from ${suspension.startedAt} to ${suspension.endedAt} (+${formatDuration(Date.parse(suspension.startedAt) - runStartMs)} into the run): the run's heartbeat did not run in that time. A process that was stopped, or a clock set forward, reads the same way.`,
+    }));
+}
+/** The participant was running at some moment of the suspension. */
+function inFlight(participant, suspension) {
+    return ((participant.startedAtMs ?? -Infinity) <= Date.parse(suspension.endedAt) &&
+        (participant.endedAtMs ?? Infinity) >= Date.parse(suspension.startedAt));
+}
+/**
+ * Which failures the run's suspensions likely caused, and the sentence that says so; undefined
+ * without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
+ * timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
+ * rule reads who was in flight across the suspension. A skip follows another participant's
+ * harness error (the pipeline gate, fail-fast), so skips count only when the run had harness
+ * errors and every one was in flight across a suspension.
+ */
+export function readHostSuspensions(suspensions, participants, runStartMs) {
+    if (suspensions.length === 0)
+        return undefined;
+    const explained = new Set(participants
+        .filter((participant) => (participant.failure === "harness" || participant.failure === "time-limit") &&
+        suspensions.some((suspension) => inFlight(participant, suspension)))
+        .map((participant) => participant.id));
+    const harness = participants.filter((participant) => participant.failure === "harness");
+    const skipsFollow = harness.length > 0 && harness.every((participant) => explained.has(participant.id));
+    const participantIds = participants
+        .filter((participant) => explained.has(participant.id) || (participant.failure === "skipped" && skipsFollow))
+        .map((participant) => participant.id);
+    const one = suspensions.length === 1;
+    const what = one
+        ? `The host was suspended for ${span(suspensions[0], runStartMs)}`
+        : `The host was suspended ${suspensions.length} times (${suspensions.map((suspension) => span(suspension, runStartMs)).join(", ")})`;
+    const count = participantIds.length;
+    const effect = count === 0
+        ? "no participant failure followed"
+        : `the ${plural(count, "participant failure")} after ${one ? "it" : "them"} ${count === 1 ? "is" : "are"} likely ${one ? "its" : "their"} effect`;
+    return { summary: `${what}; ${effect}.`, participantIds, suspensions: [...suspensions] };
+}

package/dist/run/outcomes.js

@@ -193,7 +193,9 @@ export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
             const description = entry.label === undefined ? fallback : `interrupted (${entry.label})`;
             counts.set(description, (counts.get(description) ?? 0) + 1);
         }
-        for (const [description, n] of counts)
+        // Participants finish in any order, so the causes are sorted: most first, then by name.
+        const sorted = [...counts].sort(([a, m], [b, n]) => n - m || a.localeCompare(b));
+        for (const [description, n] of sorted)
             parts.push(`${n} ${description}`);
     };
     append(outcomes.abandoned, ["abandoned"], "gave up");

package/dist/run/run.d.ts

@@ -3,6 +3,7 @@ import { type RunBundle } from "./bundle.js";
 import { type ExecutionFailure, type ExecutionOutcome, type OutcomePolicy } from "./judge.js";
 import { type PreparedRunArtifactPaths, type RunIdInUse } from "./paths.js";
 import { RunSecrets } from "./secrets.js";
+import { type HostClock, type HostSuspension } from "./host-suspension.js";
 import type { RunStudyProvenance } from "./study-provenance.js";
 interface StartRunOptions {
     /** Project directory, in the form the route resolved it. */
@@ -21,8 +22,10 @@ interface StartRunOptions {
         open: boolean;
         render?: typeof renderObserver | undefined;
     };
-    /** Clock for `createdAt` and the latest pointer. */
+    /** Clock for `createdAt` and the latest pointer. Defaults to the clock's `now`. */
     now?: (() => number) | undefined;
+    /** The run's wall clock and heartbeat timer. Defaults to systemHostClock. */
+    clock?: HostClock | undefined;
     /** Warnings about the study's own fields. Every bundle write records each as a warn event. */
     warnings?: readonly string[] | undefined;
     /** The route's known values, created before the run so a refusal before it is scrubbed too. */
@@ -42,9 +45,13 @@ interface StartRunInput {
     readonly runId?: string | undefined;
     /** Open the Observer page once it renders. */
     readonly open?: boolean | undefined;
-    /** `renderObserver` is the seam FinishedRun.renderObserver calls. */
+    /**
+     * `renderObserver` is the seam FinishedRun.renderObserver calls; `hostClock` is the run's
+     * clock and heartbeat timer.
+     */
     readonly deps?: {
         readonly renderObserver?: typeof renderObserver | undefined;
+        readonly hostClock?: HostClock | undefined;
     } | undefined;
 }
 /** What a recorded route passes to startRun besides its plan and input. */
@@ -100,6 +107,12 @@ interface Run {
      * and a run that failed before any session (a refused E2B key) does not.
      */
     participantStarted(): void;
+    /**
+     * The times this process did not run while the run was alive, most likely because the host
+     * slept, as the run's heartbeat saw them up to this call. Every bundle write records each as a
+     * `host.suspended` event; a route reads them to say which participant failures they caused.
+     */
+    hostSuspensions(): readonly HostSuspension[];
     /**
      * The one final publication: run.json with `outcome` set from `outcome`, then the status
      * outcome copied from it, then review.json, review.md, events.ndjson,

package/dist/run/run.js

@@ -16,6 +16,7 @@ import { scrubRunSandboxIds, withPublicSandboxIds } from "./sandbox-ids.js";
 import { RunSecrets } from "./secrets.js";
 import { RUN_STATUS_FILE, beginRunStatus, runStatusOutcome, } from "./status.js";
 import { readRunJsonIfExists, runJsonValue } from "./locate.js";
+import { hostSuspensionEvents, systemHostClock, } from "./host-suspension.js";
 /** An entry point called in a state that does not admit it. */
 class RunLifecycleError extends Error {
     name = "RunLifecycleError";
@@ -127,10 +128,14 @@ function runPublisher(args) {
         type: "study.warning",
         message: redactText(message),
     }));
-    const withStudyWarnings = (events) => [
-        ...events.filter((event) => !studyWarnings.some((warning) => warning.id === event.id)),
-        ...studyWarnings,
-    ];
+    // So do the host suspensions the heartbeat has seen by the time of the write.
+    const withRunEvents = (events) => {
+        const owned = [
+            ...studyWarnings,
+            ...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
+        ];
+        return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
+    };
     let pointerWritten = false;
     // The bundle of the last write asked for, and the outcome every later write carries once a
     // signal stopped the run, so no write after the interrupt can drop it.
@@ -162,7 +167,7 @@ function runPublisher(args) {
         const recorded = interrupted ?? outcome;
         const publicBundle = await withPublicSandboxIds(paths, {
             ...evidence,
-            events: withStudyWarnings(evidence.events),
+            events: withRunEvents(evidence.events),
             cwd: PUBLIC_TARGET_CWD,
             ...(recorded === undefined ? {} : { outcome: recorded }),
         });
@@ -259,8 +264,8 @@ export async function runScope(fn) {
             const created = await createRunArtifactPaths(options.cwd, runId);
             if (!created.ok)
                 return created;
-            // beginRunStatus reads only the mode, the study provenance and the sandboxes field from
-            // the run's options.
+            // beginRunStatus reads only the mode, the study provenance, the sandboxes field and the
+            // clock from the run's options.
             const runStatus = beginRunStatus(created.paths, { ...options, runId });
             status = runStatus;
             let interruptBundle;
@@ -290,7 +295,7 @@ export async function runScope(fn) {
         return args.length === 1 ? startRunWith(args[0]) : startRecordedRun(startRunWith, ...args);
     }
     const openRun = (options, runId, paths, runStatus) => {
-        const now = options.now ?? Date.now;
+        const now = options.now ?? (options.clock ?? systemHostClock).now;
         const createdAt = new Date(now()).toISOString();
         let participantsRan = false;
         const publisher = runPublisher({
@@ -317,6 +322,7 @@ export async function runScope(fn) {
             participantStarted() {
                 participantsRan = true;
             },
+            hostSuspensions: () => runStatus.suspensions(),
             writeSnapshot(bundle) {
                 if (closed)
                     return refuse("The run scope has closed.");
@@ -374,6 +380,7 @@ async function startRecordedRun(start, plan, input, route) {
         renderReview: route.renderReview,
         observer: { open: input.open === true, render: input.deps?.renderObserver },
         now: route.now,
+        clock: input.deps?.hostClock,
         secrets: route.secrets,
     });
     if (!started.ok)

package/dist/run/status.d.ts

@@ -2,6 +2,7 @@ import type { RunBundle } from "./bundle.js";
 import { type RunStudyProvenance } from "./study-provenance.js";
 import { type PreparedOutputRoot } from "./contained-output.js";
 import type { ExecutionOutcome } from "./judge.js";
+import { type HostClock, type HostSuspension } from "./host-suspension.js";
 export declare const RUN_STATUS_SCHEMA = "humanish.run-status.v1";
 /** The file, relative to the run directory. */
 export declare const RUN_STATUS_FILE = "status.json";
@@ -87,17 +88,22 @@ export interface RunStatusHandle {
     interrupt(signal: RunInterruptSignal): Promise<boolean>;
     /** True from the moment `interrupt` accepts a signal, before its write lands. */
     readonly interrupted: boolean;
+    /** The host suspensions the cadence saw until now, or until it stopped. */
+    suspensions(): readonly HostSuspension[];
 }
 export interface BeginRunStatusOptions {
     runId: string;
     mode: "dry-run" | "live";
     study?: RunStudyProvenance | undefined;
     sandboxes?: "none" | undefined;
+    /** The clock its timestamps and cadence read. Defaults to systemHostClock. */
+    clock?: HostClock | undefined;
 }
 /**
  * Start a run's status record and keep it fresh. Fire-and-forget by design: a status write that
- * fails must never fail the run it describes, so every write swallows its error. The interval is
- * `unref`'d, so this file can never be the reason a process stays alive.
+ * fails must never fail the run it describes, so every write swallows its error. The cadence is
+ * the run's heartbeat, which also records each host suspension, and its timer never keeps the
+ * process alive.
  */
 export declare function beginRunStatus(runPaths: PreparedOutputRoot, options: BeginRunStatusOptions): RunStatusHandle;
 /** The three ways a run reads from disk. `interrupted` is a record its stopped process wrote, or a

package/dist/run/status.js

@@ -12,6 +12,7 @@
 // operator may share the run directory, so a share-safety gate must have nothing to strip here.
 import { isProvenanceField, studyFields } from "./study-provenance.js";
 import { writeContainedOutputFile } from "./contained-output.js";
+import { startHeartbeat, systemHostClock, } from "./host-suspension.js";
 export const RUN_STATUS_SCHEMA = "humanish.run-status.v1";
 /** The file, relative to the run directory. */
 export const RUN_STATUS_FILE = "status.json";
@@ -52,11 +53,13 @@ export function runStatusOutcome(bundle) {
 }
 /**
  * Start a run's status record and keep it fresh. Fire-and-forget by design: a status write that
- * fails must never fail the run it describes, so every write swallows its error. The interval is
- * `unref`'d, so this file can never be the reason a process stays alive.
+ * fails must never fail the run it describes, so every write swallows its error. The cadence is
+ * the run's heartbeat, which also records each host suspension, and its timer never keeps the
+ * process alive.
  */
 export function beginRunStatus(runPaths, options) {
-    const iso = () => new Date().toISOString();
+    const clock = options.clock ?? systemHostClock;
+    const iso = () => new Date(clock.now()).toISOString();
     const startedAt = iso();
     const base = {
         schema: RUN_STATUS_SCHEMA,
@@ -84,19 +87,18 @@ export function beginRunStatus(runPaths, options) {
         return writing;
     };
     const started = write(base);
-    const timer = setInterval(() => {
+    const heartbeat = startHeartbeat(clock, RUN_STATUS_TOUCH_MS, () => {
         if (finished)
             return;
         void write({ ...base, updatedAt: iso() });
-    }, RUN_STATUS_TOUCH_MS);
-    timer.unref?.();
+    });
     const handle = {
         started,
         async finish(outcome) {
             if (finished)
                 return;
             finished = true;
-            clearInterval(timer);
+            heartbeat.stop();
             const completedAt = iso();
             finishedRecord = {
                 ...base,
@@ -122,7 +124,7 @@ export function beginRunStatus(runPaths, options) {
             }
             finished = true;
             interrupted = true;
-            clearInterval(timer);
+            heartbeat.stop();
             const completedAt = iso();
             await write({ ...base, state: "interrupted", updatedAt: completedAt, completedAt, signal });
             return true;
@@ -130,6 +132,7 @@ export function beginRunStatus(runPaths, options) {
         get interrupted() {
             return interrupted;
         },
+        suspensions: () => heartbeat.suspensions(),
     };
     return handle;
 }

package/dist/study/study-deps.d.ts

@@ -4,6 +4,7 @@ import type { CuaActorSessionOptions } from "../actors/computer-use/actor.js";
 import type { AutomaticAnalysisDeps, runAutomaticAnalysis } from "../analysis/automatic.js";
 import type { CuaLoopResult } from "../actors/computer-use/loop.js";
 import type { renderObserver } from "../observer/render.js";
+import type { HostClock } from "../run/host-suspension.js";
 import type { TerminalCostProbe } from "../routes/terminal/types.js";
 import type { DetachedTimers } from "../substrates/detached.js";
 import type { E2BDesktopModule } from "../substrates/e2b/sdk.js";
@@ -17,6 +18,11 @@ export interface StudyDeps {
     readonly renderObserver?: typeof renderObserver;
     /** The clock, in epoch milliseconds. Defaults to Date.now. */
     readonly now?: () => number;
+    /**
+     * Every recorded route: the run's wall clock and heartbeat timer, which time the run and notice
+     * a host suspension. Defaults to systemHostClock.
+     */
+    readonly hostClock?: HostClock;
     /** Clock and sleep for detached provisioning steps. */
     readonly detachedTimers?: DetachedTimers;
     /**