humanish 0.130.0 → 0.131.1

package/CHANGELOG.md

@@ -8,6 +8,33 @@ The Unreleased section holds the full notes for the next version until it is tag
 
 ## Unreleased
 
+## 0.131.1: Hosted desktops keep their screens on while participants wait (2026-10-13)
+
+humanish 0.131.1 keeps a hosted desktop's screen on while a participant waits without input.
+About 1 in 8 stock E2B desktops started with a screen saver that turned the screen black after 5
+minutes, and a participant then spent its turns keeping the screen awake: one live hour cost
+$17.70 where its pair cost $2.03. humanish now turns the screen saver off once the participant's
+browser or terminal has opened. The refusal of an out-of-range `actor.maxWaitMs` also now says the
+field caps a turn of only waits.
+
+[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.131.1)
+
+## 0.131.0: Participants that wait watch the screen and wake when it changes (2026-10-13)
+
+humanish 0.131.0 watches the screen while a computer-use or shared-world participant waits. A
+turn of only waits sends no wait to the desktop: humanish looks every `idleWaitMs` and asks the
+model for the next turn when a change holds still, speech is heard, a condition matches, the
+participant's end arrives or the wait's time passes, and idle turns in a row grow toward
+`actor.maxWaitMs`. A clock, a spinner or a "2 minutes ago" label ends no wait. In a live hour on a
+support queue, new rows reached the participant a median of 7 seconds after they appeared, against
+30 seconds on 0.129.0, and an OpenAI participant made 50 requests where 0.129.0 made 43. Codex
+and Claude Code waits last the length they ask for, up to 10 minutes. This is a minor release with
+breaking changes: `idleWaitMs` now sets the time between looks, `actor.maxWaitMs` defaults to 10
+minutes for idle turns and accepts up to an hour, and a turn of only waits sends no `wait` action
+to a library caller's executor. The migration table says how to keep the old pacing.
+
+[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.131.0)
+
 ## 0.130.0: Analysis reads traces up to 32 MiB one at a time and names what it refused; the bundle estimate counts the mission (2026-10-13)
 
 humanish 0.130.0 lets analysis read long runs that 0.129.0 refused. Analysis reads each

package/dist/actors/codex/restricted-participant.d.ts

@@ -18,8 +18,8 @@ export interface RestrictedParticipantOptions {
     /** Admit speech actions and heard-speaker evidence for a speech-capable desktop only. */
     speechEnabled?: boolean;
     /**
-     * The study's actor.maxWaitMs, which the loop applies and the tool description states. Default
-     * defaultMaxWaitMs(speechEnabled), the same default the loop applies.
+     * The study's actor.maxWaitMs, which the loop applies and the tool description states with the
+     * defaults the loop applies without it (resolveWaitSettings).
      */
     maxWaitMs?: number;
 }

package/dist/actors/codex/restricted-participant.js

@@ -3,7 +3,7 @@ import { defaultCodexCliVersion, untestedOperatorReleaseWarning } from "./codex-
 import { IMPRESSIONS_ASK, } from "../computer-use/loop.js";
 import { ComputerUseProviderError, isComputerUseProviderError, } from "../computer-use/provider-error.js";
 import { validateBrowserControlPng, validateHeardSpeech } from "../../browser-control/protocol.js";
-import { defaultMaxWaitMs } from "../computer-use/wait.js";
+import { resolveWaitSettings } from "../computer-use/wait.js";
 import { createRestrictedCodexSession, } from "./restricted-session.js";
 import { PARTICIPANT_PROFILE, PARTICIPANT_LIMITS as L, PARTICIPANT_FINAL_SCHEMA, participantToolSchema, parseParticipantTool, parseParticipantFinal, } from "./restricted-participant-policy.js";
 import { truncatedFrameWarning, unknownNotificationsWarning } from "./restricted-notifications.js";
@@ -41,7 +41,7 @@ function deferred() {
     return { promise, resolve, reject };
 }
 // prose-check: model prompt (Codex reads this tool description, not a person)
-const toolDescription = (speechEnabled, maxWaitMs) => `Act on the participant's browser through humanish. Submit one to four UI actions${speechEnabled ? ", including speak when you need to reply aloud," : ""} and a short public comment; never private reasoning. humanish returns a JSON STRING with execution acknowledgments${speechEnabled ? ", speech heard from the actual participant speaker sink," : ""} and a fresh screenshot. In Code Mode use: const r = JSON.parse(await tools.humanish_ui({narration: "...", actions: [...]})); text({acknowledgments: r.acknowledgments${speechEnabled ? ", heardSpeech: r.heardSpeech" : ""}, contextHint: r.contextHint, closing: r.closing}); image(r.imageUrl). Call serially and inspect each returned screenshot${speechEnabled ? " and heardSpeech array" : ""} before deciding what to do next. A wait action lasts at most ${maxWaitMs} ms; a longer wait is shortened to ${maxWaitMs} ms, so wait longer with another call.${speechEnabled ? " Use speak only after the visible UI shows that you joined the call and your microphone is unmuted; it sends audio into that call." : ""} Acknowledged input does not prove an application outcome. If closing is true, stop calling tools and give your final account.`;
+const toolDescription = (speechEnabled, { maxWaitMs, watchMaxMs }) => `Act on the participant's browser through humanish. Submit one to four UI actions${speechEnabled ? ", including speak when you need to reply aloud," : ""} and a short public comment; never private reasoning. humanish returns a JSON STRING with execution acknowledgments${speechEnabled ? ", speech heard from the actual participant speaker sink," : ""} and a fresh screenshot. In Code Mode use: const r = JSON.parse(await tools.humanish_ui({narration: "...", actions: [...]})); text({acknowledgments: r.acknowledgments${speechEnabled ? ", heardSpeech: r.heardSpeech" : ""}, contextHint: r.contextHint, closing: r.closing}); image(r.imageUrl). Call serially and inspect each returned screenshot${speechEnabled ? " and heardSpeech array" : ""} before deciding what to do next. A call whose actions are only waits returns when the screen changes and holds still${speechEnabled ? ", when speech is heard" : ""}, or after its waits' ms, together at most ${watchMaxMs} ms. While no change on the screen holds still, each further such call in a row waits twice as long as the one before, up to ${watchMaxMs} ms, and contextHint says how long it waited and why it returned. A wait among other actions lasts its ms, at most ${maxWaitMs} ms.${speechEnabled ? " Use speak only after the visible UI shows that you joined the call and your microphone is unmuted; it sends audio into that call." : ""} Acknowledged input does not prove an application outcome. If closing is true, stop calling tools and give your final account.`;
 const participantInstructions = (prompt, speechEnabled) => `${prompt}\n\nYou are the study participant throughout this conversation, including its closing account. Use only the supplied screenshots and humanish_ui tool to interact. The tool returns a JSON string: parse it, inspect acknowledgments${speechEnabled ? " and heardSpeech captured from the actual participant speaker sink" : ""}, and display imageUrl with Code Mode image(). Do not print the image data URL as text. Keep your persona and earlier observations throughout the session.${speechEnabled ? " A speak action plays into the participant microphone; use it only after the visible UI shows that you joined the call and the microphone is unmuted." : ""} Speak publicly about your experience, never reveal private reasoning. Completed inputs do not prove application outcomes; verify on the next screenshot. When the task ends, return only the required final JSON with outcome, summary, frictionReports and impressions. Report observed confusion and recovered mistakes as well as blockers. ${IMPRESSIONS_ASK} Do not invent observations.`;
 /** Turns and failures from the native run, handed in order to the next provider request. */
 class ParticipantEvents {
@@ -192,7 +192,7 @@ function participantSession(options, speechEnabled, effort, call) {
             reasoningEffort: effort,
             tool: {
                 name: "humanish_ui",
-                description: toolDescription(speechEnabled, options.maxWaitMs ?? defaultMaxWaitMs(speechEnabled)),
+                description: toolDescription(speechEnabled, resolveWaitSettings({ maxWaitMs: options.maxWaitMs, speechEnabled })),
                 inputSchema: participantToolSchema(speechEnabled),
                 call,
             },

package/dist/actors/computer-use/loop/actions.d.ts

@@ -1,3 +1,4 @@
+import type { ActorWatch } from "../../contract.js";
 import type { LoopSession } from "./session.js";
 import type { CuaAction, CuaTurnRequest } from "./types.js";
 /** A public-safe one-line action label. Never includes raw typed text. */
@@ -21,9 +22,23 @@ export interface ActionBatch {
      */
     readonly hint: string | undefined;
 }
+/**
+ * The waits of a turn the loop watched in place of sending them to the desktop (watch.ts): the
+ * part of the watch each wait covered, in order, and the `watch` field of the wait during which
+ * the watch ended.
+ */
+export interface WatchedWaits {
+    readonly ms: readonly number[];
+    readonly endedOn: number;
+    readonly watch: ActorWatch;
+    /** The session ended during the watch, so the turn's screenshot actions are not sent. */
+    readonly sessionEnded: boolean;
+}
 /**
  * Dispatch one turn's actions in order. A declared pre-dispatch rejection stops the batch and
  * leaves the desktop usable; a failed desktop command skips its action; any other failure ends
- * the session.
+ * the session. The waits of a watched turn (`watched`) already ran: each is recorded with the part
+ * of the watch it covered and nothing is sent for it. When that watch ended the session, the
+ * turn's screenshots are acknowledged as not dispatched and not recorded.
  */
-export declare function runActionBatch(session: LoopSession, actions: readonly CuaAction[]): Promise<ActionBatch>;
+export declare function runActionBatch(session: LoopSession, actions: readonly CuaAction[], watched?: WatchedWaits): Promise<ActionBatch>;

package/dist/actors/computer-use/loop/actions.js

@@ -119,19 +119,34 @@ function recordShortenedWait(session, title, requestedMs) {
 /**
  * Dispatch one turn's actions in order. A declared pre-dispatch rejection stops the batch and
  * leaves the desktop usable; a failed desktop command skips its action; any other failure ends
- * the session.
+ * the session. The waits of a watched turn (`watched`) already ran: each is recorded with the part
+ * of the watch it covered and nothing is sent for it. When that watch ended the session, the
+ * turn's screenshots are acknowledged as not dispatched and not recorded.
  */
-export async function runActionBatch(session, actions) {
+export async function runActionBatch(session, actions, watched) {
     const execution = { actions: [] };
     const { actionHistory, trace } = session;
     const notes = [];
     const hint = () => (notes.length === 0 ? undefined : notes.join(" "));
     const idleTurn = isIdleTurn(actions);
+    let waits = 0;
     for (const [index, requested] of actions.entries()) {
         if (session.signal?.aborted)
             throw new CuaAbortError();
-        const wait = requested.kind === "wait" ? planWait(session.waits, requested.ms, idleTurn) : undefined;
-        const action = wait === undefined ? requested : { kind: "wait", ms: wait.ms };
+        if (watched?.sessionEnded === true && requested.kind !== "wait") {
+            execution.actions.push({ index, status: "not_dispatched" });
+            continue;
+        }
+        const waitIndex = requested.kind === "wait" ? waits++ : undefined;
+        const watchedMs = waitIndex === undefined ? undefined : watched?.ms[waitIndex];
+        const wait = requested.kind === "wait" && watchedMs === undefined
+            ? planWait(session.waits, requested.ms, idleTurn)
+            : undefined;
+        const action = watchedMs !== undefined
+            ? { kind: "wait", ms: watchedMs }
+            : wait === undefined
+                ? requested
+                : { kind: "wait", ms: wait.ms };
         const title = describeCuaAction(action);
         if (wait?.shortenedFromMs !== undefined)
             notes.push(recordShortenedWait(session, title, wait.shortenedFromMs));
@@ -147,7 +162,10 @@ export async function runActionBatch(session, actions) {
         session.phase = `executing ${title}`;
         let status;
         try {
-            status = await dispatchAction(session, action, title, wait?.steps);
+            status =
+                watchedMs === undefined
+                    ? await dispatchAction(session, action, title, wait?.steps)
+                    : "completed";
         }
         catch (error) {
             const declared = isComputerUseExecutorError(error)
@@ -190,6 +208,7 @@ export async function runActionBatch(session, actions) {
             ...(action.kind === "click" || action.kind === "double_click"
                 ? { coord: { x: action.x, y: action.y } }
                 : {}),
+            ...(watched !== undefined && waitIndex === watched.endedOn ? { watch: watched.watch } : {}),
         }));
     }
     return { execution, hint: hint() };

package/dist/actors/computer-use/loop/backstop.d.ts

@@ -12,6 +12,12 @@ import type { CuaAction, CuaObservation } from "./types.js";
  * mutates the input. (See docs/architecture/state-driven-executor.md.)
  */
 export declare function stableProgressKey(appState: Record<string, unknown>): string;
+/**
+ * The friction progress key: a stable projection of appState when present, else stateSignature.
+ * A state executor with a constant signature still registers progress when its appState changes,
+ * and a vision executor behaves exactly as before.
+ */
+export declare function progressKeyOf(observation: CuaObservation): string;
 export interface Backstop {
     readonly progressKey: string;
     /** The last few turns' action fingerprints, in memory only, for the corroboration rule below. */
@@ -37,11 +43,13 @@ export interface BackstopStep {
 /**
  * Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
  * participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
- * toward neither streak and earns no nudge. A turn that acts is folded as usual.
+ * toward neither streak and earns no nudge. A turn that acts is folded as usual. `screenChanged`
+ * says the turn's watch woke on a change it saw, which the progress key can miss.
  */
 export declare function advanceBackstop(previous: Backstop, turn: {
     actions: readonly CuaAction[];
     observation: CuaObservation;
     heardNewSpeech: boolean;
     waitingIsTheTask?: boolean;
+    screenChanged?: boolean;
 }, limits: BackstopLimits): BackstopStep;

package/dist/actors/computer-use/loop/backstop.js

@@ -103,7 +103,7 @@ export function stableProgressKey(appState) {
  * A state executor with a constant signature still registers progress when its appState changes,
  * and a vision executor behaves exactly as before.
  */
-function progressKeyOf(observation) {
+export function progressKeyOf(observation) {
     const base = observation.appState !== undefined
         ? stableProgressKey(observation.appState)
         : observation.stateSignature;
@@ -125,7 +125,8 @@ export function startBackstop(observation) {
 /**
  * Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
  * participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
- * toward neither streak and earns no nudge. A turn that acts is folded as usual.
+ * toward neither streak and earns no nudge. A turn that acts is folded as usual. `screenChanged`
+ * says the turn's watch woke on a change it saw, which the progress key can miss.
  */
 export function advanceBackstop(previous, turn, limits) {
     const idle = isIdleTurn(turn.actions);
@@ -157,7 +158,7 @@ export function advanceBackstop(previous, turn, limits) {
     // behavioral signal already (the agent took nothing but screenshots and waits), so it keeps
     // reading the frame on its own: a repeated screenshot is exactly what an idle streak is, and
     // feeding repetition into it would grant an extra forgiveness step for being idle.
-    const observedProgress = frameChanged || turn.heardNewSpeech;
+    const observedProgress = frameChanged || turn.heardNewSpeech || turn.screenChanged === true;
     const progressed = observedProgress || !repeatingRecentAction;
     // A screenshot/wait turn while the UI visibly changes may be patience through loading or a
     // transition, so grant a bounded recovery window. Do not grant infinite immunity: animated

package/dist/actors/computer-use/loop/clock.d.ts

@@ -14,8 +14,8 @@ export declare function clockHint(clock: ParticipantClock | undefined, now: () =
  */
 export declare function heldToScheduledEnd(session: LoopSession): boolean;
 /**
- * The stop at the participant's end, or undefined before it: its scheduled end or its session cap,
- * which is the stop cause. The session passes as a planned end and the participant is asked for
- * its closing report on the last observation.
+ * The stop at the participant's end, or undefined before `nowMs`, by default the session's clock,
+ * reaches it: its scheduled end or its session cap, which is the stop cause. The session passes as
+ * a planned end and the participant is asked for its closing report on the last observation.
  */
-export declare function clockEndStop(session: LoopSession, observation: CuaObservation): Stop | undefined;
+export declare function clockEndStop(session: LoopSession, observation: CuaObservation, nowMs?: number): Stop | undefined;

package/dist/actors/computer-use/loop/clock.js

@@ -37,13 +37,13 @@ export function heldToScheduledEnd(session) {
     return end !== undefined && end.kind !== "session_cap" && session.now() < end.atMs;
 }
 /**
- * The stop at the participant's end, or undefined before it: its scheduled end or its session cap,
- * which is the stop cause. The session passes as a planned end and the participant is asked for
- * its closing report on the last observation.
+ * The stop at the participant's end, or undefined before `nowMs`, by default the session's clock,
+ * reaches it: its scheduled end or its session cap, which is the stop cause. The session passes as
+ * a planned end and the participant is asked for its closing report on the last observation.
  */
-export function clockEndStop(session, observation) {
+export function clockEndStop(session, observation, nowMs) {
     const end = session.settings.clock?.end;
-    if (end === undefined || session.now() < end.atMs)
+    if (end === undefined || (nowMs ?? session.now()) < end.atMs)
         return undefined;
     const at = endTime(session.settings.clock, end.atMs);
     const kind = end.kind ?? "scheduled_end";

package/dist/actors/computer-use/loop/observation.d.ts

@@ -1,6 +1,14 @@
 import { type Stop } from "./ending.js";
 import type { LoopSession } from "./session.js";
 import type { CuaObservation } from "./types.js";
+/** What one look during a watch found (watch.ts). */
+export interface Look {
+    readonly observation: CuaObservation;
+    /** The look brought speech the participant had not heard. */
+    readonly speech: boolean;
+    /** A stopWhen rule, a pending dwell trigger or a newly completed task matched the look. */
+    readonly condition: boolean;
+}
 /** What a checkpoint found: a stop, or the observation the next turn reacts to. */
 export type Checkpoint = {
     readonly stop: Stop;
@@ -18,15 +26,26 @@ export declare class DesktopObserver {
     constructor(session: LoopSession);
     /**
      * Observe, persist the frame, advance the task funnel, run a due dwell window, and check the
-     * harness stop conditions. Turn 0 is the initial observation, before any model turn.
+     * harness stop conditions. Turn 0 is the initial observation, before any model turn. `looked` is
+     * the look that ended a watch: it stands in for the observation, so the frame that woke the
+     * participant is the one it sees, and its URL and app state were already handed on at the look.
      */
-    checkpoint(turnNumber: number): Promise<Checkpoint>;
+    checkpoint(turnNumber: number, looked?: CuaObservation): Promise<Checkpoint>;
     /**
      * Hand an observation the next turn may react to to the runtime hooks, and note whether it
      * carried app state. Runtime-only: the participant's live location.href goes back to the
      * orchestrator and is never persisted.
      */
     private handToHooks;
+    private handUrlOn;
+    private handFrameToHook;
+    /**
+     * One look at the screen while the participant waits. Speech it brings is kept for the next
+     * request, its URL goes to the URL tap (the shared-world lobby latch) and a task it completes is
+     * recorded. Detects, without acting on, the conditions the checkpoint acts on; the watch that
+     * asked hands a look that ends it to the checkpoint. Nothing is persisted.
+     */
+    look(turnNumber: number): Promise<Look>;
     /**
      * Every observation that carried app state counts, including dwell frames and the final task
      * observation, which no turn receives: they feed the task funnel, and the trace notes must say

package/dist/actors/computer-use/loop/observation.js

@@ -28,9 +28,11 @@ export class DesktopObserver {
     }
     /**
      * Observe, persist the frame, advance the task funnel, run a due dwell window, and check the
-     * harness stop conditions. Turn 0 is the initial observation, before any model turn.
+     * harness stop conditions. Turn 0 is the initial observation, before any model turn. `looked` is
+     * the look that ended a watch: it stands in for the observation, so the frame that woke the
+     * participant is the one it sees, and its URL and app state were already handed on at the look.
      */
-    async checkpoint(turnNumber) {
+    async checkpoint(turnNumber, looked) {
         const { session } = this;
         const { stopWhen } = session.settings;
         const opening = turnNumber === 0;
@@ -41,7 +43,9 @@ export class DesktopObserver {
                 throw new CuaAbortError();
             session.phase = `observing UI state after turn ${turnNumber}`;
         }
-        let observation = this.handToHooks(this.withHeardSpeech(await this.observeBounded(opening ? "initial" : `after turn ${turnNumber}`)));
+        let observation = looked === undefined
+            ? this.handToHooks(this.withHeardSpeech(await this.observeBounded(opening ? "initial" : `after turn ${turnNumber}`)))
+            : this.handFrameToHook(looked);
         // Fail closed before a vision provider is asked to reason over a missing frame.
         const frameless = missingFrame(session.provider, observation);
         if (frameless !== undefined)
@@ -78,13 +82,48 @@ export class DesktopObserver {
      * orchestrator and is never persisted.
      */
     handToHooks(observation) {
-        const { onObservedUrl, onScreenshot } = this.session.settings;
+        this.handUrlOn(observation);
+        return this.handFrameToHook(observation);
+    }
+    handUrlOn(observation) {
+        const { onObservedUrl } = this.session.settings;
         onObservedUrl?.(observation.url);
+        this.noteAppState(observation);
+    }
+    // The frame tap reads the lobby code with a model call outside the study budget, so it gets only
+    // frames a turn receives, never every look of a watch.
+    handFrameToHook(observation) {
+        const { onScreenshot } = this.session.settings;
         if (observation.screenshot !== undefined)
             onScreenshot?.(observation.screenshot);
-        this.noteAppState(observation);
         return observation;
     }
+    /**
+     * One look at the screen while the participant waits. Speech it brings is kept for the next
+     * request, its URL goes to the URL tap (the shared-world lobby latch) and a task it completes is
+     * recorded. Detects, without acting on, the conditions the checkpoint acts on; the watch that
+     * asked hands a look that ends it to the checkpoint. Nothing is persisted.
+     */
+    async look(turnNumber) {
+        const { session } = this;
+        if (session.signal?.aborted)
+            throw new CuaAbortError();
+        session.phase = `looking at the screen during turn ${turnNumber}`;
+        const heardBefore = this.pendingHeardSpeech.length;
+        const observation = this.withHeardSpeech(await this.observeBounded(`look during turn ${turnNumber}`));
+        this.handUrlOn(observation);
+        const evidence = stopObservationOf(observation);
+        const { stopWhen, dwell } = session.settings;
+        const completedTask = this.observeTasks(observation, turnNumber);
+        const dwellDue = dwell?.when !== undefined &&
+            !this.dwellDone &&
+            evaluateStopWhen(dwell.when, evidence) !== undefined;
+        return {
+            observation,
+            speech: this.pendingHeardSpeech.length > heardBefore,
+            condition: completedTask || dwellDue || evaluateStopWhen(stopWhen, evidence) !== undefined,
+        };
+    }
     /**
      * Every observation that carried app state counts, including dwell frames and the final task
      * observation, which no turn receives: they feed the task funnel, and the trace notes must say
@@ -205,16 +244,19 @@ export class DesktopObserver {
         session.trace.bump("screenshots");
     }
     // Evaluated before the stopWhen check each turn so a final task whose criterion coincides with
-    // the stop condition still lands in the funnel of the very turn that ends the session.
+    // the stop condition still lands in the funnel of the very turn that ends the session. Returns
+    // whether a task completed.
     observeTasks(observation, turnNumber) {
         const { taskTracker } = this.session;
         if (taskTracker === undefined)
-            return;
-        for (const completion of taskTracker.observe(stopObservationOf(observation), turnNumber)) {
+            return false;
+        const completions = taskTracker.observe(stopObservationOf(observation), turnNumber);
+        for (const completion of completions) {
             // The id is researcher-authored config and the kinds are rule-type names; the matched values
             // (a URL, page text) never appear here: the same discipline as the stopWhen notice.
             this.session.trace.record("notice", () => notice("matched", `task completed: ${this.session.redactNarration(completion.id)}`, this.session.redactNarration(`turn ${turnNumber}; matched rule ${completion.matchedRuleIndex} (${completion.matchedKinds.join("+")})`)));
         }
+        return completions.length > 0;
     }
     // The declared observation window. The harness holds, looks, and takes nothing back to
     // the model: no action, no turn, no tokens. It runs once, cut to whatever session budget is

package/dist/actors/computer-use/loop/trace.js

@@ -129,7 +129,11 @@ export function loopResult(session, stop, debrief) {
                         : { maxOutputTokens: provider.modelSettings.maxOutputTokens }),
                 },
             }),
-        waitSettings: { ...session.waits },
+        waitSettings: {
+            maxWaitMs: session.waits.maxWaitMs,
+            idleWaitMs: session.waits.idleWaitMs,
+            settleWaitMs: session.waits.settleWaitMs,
+        },
         counts,
         items: recorder.items,
         ...(session.finalPage === undefined ? {} : { finalPage: session.finalPage }),

package/dist/actors/computer-use/loop/types.d.ts

@@ -297,8 +297,8 @@ export interface CuaLoopOptions {
     /** Injected clock (ms). Lets tests drive deadlines deterministically. */
     now: () => number;
     /**
-     * Cancellation. Checked before each turn, each action and each dwell frame, and raced against
-     * every provider and desktop call.
+     * Cancellation. Checked before each turn, each action, each dwell frame and each look of a
+     * watched turn, and raced against every provider and desktop call and each pause of a watch.
      */
     signal?: AbortSignal;
     /** Idle streak (no material action) that trips the backstop. Default 24. */
@@ -306,19 +306,25 @@ export interface CuaLoopOptions {
     /** No-progress streak that trips the backstop. Default 20. */
     noProgressSteps?: number;
     /**
-     * The longest one `wait` action lasts, in ms: the study's actor.maxWaitMs. A longer wait is
-     * shortened to it, the trace records a `wait shortened` notice, and the participant is told on its
-     * next request. A wait longer than one desktop call carries is sent in steps. A whole number from
-     * 1000 to 600000; any other value throws a RangeError before the session starts. Default
-     * defaultMaxWaitMs: 120000, or 30000 when the executor is speechEnabled.
+     * The longest one `wait` action lasts, in ms: the study's actor.maxWaitMs. In a turn with another
+     * action a longer wait is shortened to it, the trace records a `wait shortened` notice, and the
+     * participant is told on its next request; a wait longer than one desktop call carries is sent in
+     * steps. A turn that only waits or takes screenshots is watched instead (loop/watch.ts): it first
+     * lasts its waits' lengths, the named ones together up to this or, when it is not set, 600000,
+     * and each further such turn in a row that ends with no change lasts twice as long, up to that
+     * or the turn's own length. A whole number from 1000 to 3600000; any other value throws a
+     * RangeError before the session starts. Default 120000, or 30000 when the executor is speechEnabled, for one wait sent to the
+     * desktop, and 600000 for a watch.
      */
     maxWaitMs?: number;
     /**
      * How long a `wait` that names no duration lasts, in ms, when its turn only waits or takes
-     * screenshots: the study's actor.idleWaitMs. After an action in the same turn such a wait lasts
-     * 500 ms. A whole number from 1000 to 600000 and no longer than an explicit maxWaitMs; any other
-     * value throws a RangeError before the session starts. Default 5000. The session never applies
-     * more than its maxWaitMs, and the trace records the applied lengths as `waitSettings`.
+     * screenshots: the study's actor.idleWaitMs. It is also how often the loop looks at the screen
+     * while such a turn is watched, at most every 30000 ms when the executor is speechEnabled. After
+     * an action in the same turn such a wait lasts 500 ms. A whole number from 1000 to 600000 and no
+     * longer than an explicit maxWaitMs; any other value throws a RangeError before the session
+     * starts. Default 5000. The session never applies more than its maxWaitMs, and the trace records
+     * the applied lengths as `waitSettings`.
      */
     idleWaitMs?: number;
     /**
@@ -347,9 +353,10 @@ export interface CuaLoopOptions {
     /** Persist a screenshot (raw or redacted per redactScreenshots), returning the trace ref path. */
     writeScreenshot?: (name: string, bytes: Buffer) => Promise<string>;
     /**
-     * Deterministic harness-owned success guards. Evaluated after the initial observation and after
-     * every post-action observation, before another model turn is requested. This keeps a participant from
-     * wandering after the product already reached an app-visible endpoint.
+     * Deterministic harness-owned success guards. Evaluated after the initial observation, after
+     * every post-action observation and at each look while a turn is watched, before another model
+     * turn is requested. This keeps a participant from wandering after the product already reached an
+     * app-visible endpoint.
      */
     stopWhen?: StopWhen;
     /**
@@ -359,7 +366,10 @@ export interface CuaLoopOptions {
      * most once per session and never past the session budget.
      */
     dwell?: DwellWindow;
-    /** Injected pause for the dwell window's cadence; tests advance their clock through it. */
+    /**
+     * Injected pause for the dwell window's cadence and the looks of a watched turn; tests advance
+     * their clock through it.
+     */
     sleep?: (ms: number) => Promise<void>;
     /**
      * What each turn tells the participant about the time (loop/clock.ts). With `timeZone`, a time zone
@@ -419,7 +429,7 @@ export interface CuaLoopOptions {
      * Trace snapshot for a watcher: the redacted items recorded so far, with the running
      * usage so a run can be priced in flight. Called after each checkpoint's screenshot (the initial
      * observation and every acted turn, so a flush never shows an action without the frame before
-     * it), after each dwell frame, and after a closing request. The array is a fresh copy; its items
+     * it), after each dwell frame, when a watched turn starts watching, and after a closing request. The array is a fresh copy; its items
      * are the objects the final trace persists. Not awaited, so a slow flush cannot stall a turn.
      * Default: no-op.
      */
@@ -433,8 +443,8 @@ export interface CuaLoopOptions {
 export interface LoopTaps {
     /**
      * Runtime-only URL tap: called with `observation.url` for each observation a turn may receive
-     * (the initial one, each post-action one and the one after a dwell window). The loop never
-     * persists the raw URL. Default: no-op.
+     * (the initial one, each post-action one and the one after a dwell window) and each look while a
+     * turn is watched. The loop never persists the raw URL. Default: no-op.
      *
      * Built-in caller: the external-public shared-world handoff (src/routes/shared-world/handoff.ts)
      * latches the host participant's lobby URL and checks that each follower reached the same lobby.
@@ -452,7 +462,8 @@ export interface LoopTaps {
     onMessage?: (text: string) => void;
     /**
      * Runtime-only frame tap: called with each raw frame a turn may receive and each dwell-window
-     * frame, before any redaction. The buffer stays in memory; screenshot persistence is separate and
+     * frame, before any redaction. Of a watched turn's looks, only the one that ended the watch is a
+     * frame a turn receives. The buffer stays in memory; screenshot persistence is separate and
      * follows redactScreenshots. Not awaited. Default: no-op.
      *
      * Built-in caller: the external-public handoff reads the lobby code off each participant's own

package/dist/actors/computer-use/loop/watch.d.ts

@@ -0,0 +1,42 @@
+import type { ActorWatch } from "../../contract.js";
+import { type WatchedWaits } from "./actions.js";
+import { type Stop } from "./ending.js";
+import type { DesktopObserver } from "./observation.js";
+import type { LoopSession } from "./session.js";
+import type { CuaAction, CuaObservation } from "./types.js";
+/** A watch that ran for one idle turn: how it ended, and what the loop does next. */
+export interface Watched {
+    readonly wake: ActorWatch["wake"];
+    /**
+     * The look that ended the watch, which the checkpoint takes in place of a fresh observe. Absent
+     * when no look ended it: a stalled look, or the participant's end.
+     */
+    readonly observation?: CuaObservation;
+    /** The next request's note: how long the participant waited and why it woke. */
+    readonly hint: string | undefined;
+    /**
+     * Set when the session ends here: the study budget or the session's time ran out, or the
+     * participant's end came.
+     */
+    readonly stop?: Stop;
+    /** Each wait's part of the watch, for the trace. */
+    readonly waits: WatchedWaits;
+}
+/**
+ * The watches of one participant's idle turns. Between turns it keeps the run of idle turns the
+ * doubling counts: a watch that ended at its limit or on a small change makes the next one in the
+ * run longer, and a turn with another action ends the run.
+ */
+export declare class IdleWatcher {
+    private readonly session;
+    private readonly observer;
+    /** The limit of the run's last watch, while that watch ended at its limit or on a small change. */
+    private grownFromMs;
+    constructor(session: LoopSession, observer: DesktopObserver);
+    /**
+     * Watch the screen in place of this turn's waits. Undefined, with nothing run, for a turn with
+     * another action, which ends the run, or with no wait time (screenshots only, or waits of 0 ms),
+     * which leaves the run as it is. `reference` is the observation the participant chose to wait on.
+     */
+    watch(turnNumber: number, reference: CuaObservation, actions: readonly CuaAction[]): Promise<Watched | undefined>;
+}