humanish 0.130.0 → 0.131.0

package/dist/actors/computer-use/loop/watch.js

@@ -0,0 +1,260 @@
+import { watchFrames, } from "../../../evidence/frame-change.js";
+import { plural } from "../../../run/text.js";
+import { isIdleTurn } from "./actions.js";
+import { progressKeyOf } from "./backstop.js";
+import { clockEndStop } from "./clock.js";
+import { timeLimit } from "./ending.js";
+import { CuaAbortError, CuaStallError, raceSessionDeadline } from "./race.js";
+import { spendStop } from "./spend.js";
+/**
+ * The watches of one participant's idle turns. Between turns it keeps the run of idle turns the
+ * doubling counts: a watch that ended at its limit or on a small change makes the next one in the
+ * run longer, and a turn with another action ends the run.
+ */
+export class IdleWatcher {
+    session;
+    observer;
+    /** The limit of the run's last watch, while that watch ended at its limit or on a small change. */
+    grownFromMs;
+    constructor(session, observer) {
+        this.session = session;
+        this.observer = observer;
+    }
+    /**
+     * Watch the screen in place of this turn's waits. Undefined, with nothing run, for a turn with
+     * another action, which ends the run, or with no wait time (screenshots only, or waits of 0 ms),
+     * which leaves the run as it is. `reference` is the observation the participant chose to wait on.
+     */
+    async watch(turnNumber, reference, actions) {
+        const { session } = this;
+        if (!isIdleTurn(actions)) {
+            this.grownFromMs = undefined;
+            return undefined;
+        }
+        // A named wait past maxWaitMs, the longest one wait sent to the desktop lasts, is watched for
+        // its length: a participant told to wait 10 minutes waits 10 minutes. A turn's named waits
+        // together last at most watchMaxMs, in order, so a Codex call of 4 long waits stays open no
+        // longer than one. A wait that names no length lasts idleWaitMs.
+        const asked = actions.flatMap((action) => action.kind === "wait" ? [Math.max(0, action.ms ?? session.waits.idleWaitMs)] : []);
+        let namedMs = 0;
+        const lengths = actions.flatMap((action) => {
+            if (action.kind !== "wait")
+                return [];
+            if (action.ms === undefined)
+                return [session.waits.idleWaitMs];
+            const ms = Math.min(Math.max(0, action.ms), session.waits.watchMaxMs - namedMs);
+            namedMs += ms;
+            return [ms];
+        });
+        const plannedMs = lengths.reduce((sum, ms) => sum + ms, 0);
+        if (plannedMs === 0)
+            return undefined;
+        const capMs = Math.max(plannedMs, session.waits.watchMaxMs);
+        // Never shorter than the turn's own planned length, so a long named wait is not cut short.
+        const limitMs = this.grownFromMs === undefined
+            ? plannedMs
+            : Math.min(capMs, Math.max(plannedMs, 2 * this.grownFromMs));
+        const ending = await watchScreen(session, this.observer, turnNumber, reference, limitMs);
+        // A small change that held still (a status label, or a counter the rule did not hold back)
+        // wakes the participant and keeps the doubling, so a repeating small change cannot keep every
+        // watch short.
+        const grows = ending.wake === "limit" || (ending.seen?.verdict === "settled" && ending.seen.small);
+        this.grownFromMs = grows ? limitMs : undefined;
+        return {
+            wake: ending.wake,
+            ...(ending.observation === undefined ? {} : { observation: ending.observation }),
+            ...(ending.stop === undefined ? {} : { stop: ending.stop }),
+            hint: hintFor(ending, grows && limitMs < capMs ? capMs : undefined),
+            waits: {
+                ...splitWatch(lengths, limitMs, ending.ms),
+                sessionEnded: ending.stop !== undefined,
+                watch: {
+                    limitMs,
+                    askedMs: asked.reduce((sum, ms) => sum + ms, 0),
+                    wake: ending.wake,
+                    looks: ending.looks,
+                    ...(ending.area === undefined ? {} : { area: { ...ending.area } }),
+                },
+            },
+        };
+    }
+}
+async function watchScreen(session, observer, turnNumber, reference, limitMs) {
+    const { lookEveryMs, settleWaitMs } = session.waits;
+    const startedAt = session.now();
+    const limitAt = startedAt + limitMs;
+    const deadlineAt = session.startedAtMs + session.timeoutMs;
+    const end = session.settings.clock?.end;
+    const endAt = end?.atMs;
+    // A wind-down already under way bounds no watch: the participant was told when it started.
+    const windDownAt = end !== undefined && end.atMs - end.windDownMs > startedAt
+        ? end.atMs - end.windDownMs
+        : undefined;
+    // A reply that came back past the end or the deadline starts no watch.
+    if (session.signal?.aborted)
+        throw new CuaAbortError();
+    if (endAt !== undefined && startedAt >= endAt)
+        return {
+            wake: "clock",
+            ms: 0,
+            looks: 0,
+            ...stopAt(clockEndStop(session, reference, startedAt)),
+        };
+    if (startedAt >= deadlineAt)
+        return { wake: "stopped", ms: 0, looks: 0, stop: timeLimit(session) };
+    session.trace.record("notice", () => ({
+        lifecycle: "started",
+        title: "watching the screen",
+        text: `until the screen changes or ${limitMs}ms pass, looking every ${lookEveryMs}ms and at the end; no model request until then`,
+    }));
+    session.flush();
+    const frames = reference.screenshot === undefined ? undefined : watchFrames(reference.screenshot);
+    const referenceKey = progressKeyOf(reference);
+    let looks = 0;
+    let quickLooks = 0;
+    let lastLook;
+    let nextLookAt = startedAt + lookEveryMs;
+    for (;;) {
+        if (session.signal?.aborted)
+            throw new CuaAbortError();
+        session.phase = `watching the screen during turn ${turnNumber}`;
+        const target = Math.min(nextLookAt, limitAt, windDownAt ?? limitAt, endAt ?? limitAt, deadlineAt);
+        const sleptFrom = session.now();
+        await raceSessionDeadline(session.sleep(Math.max(0, target - sleptFrom)), deadlineAt - sleptFrom, session.signal);
+        // An injected clock need not move with the injected sleep; the watch's schedule still does.
+        const at = Math.max(session.now(), target);
+        // The wait lasts until the watch returns, so it includes the look that ended it, and a stalled
+        // look's observe and retry.
+        const lasted = () => Math.max(session.now(), at) - startedAt;
+        // No observe after the end or the deadline: one started with no time left would fail, and at
+        // the end the session would end as timed out. The stop is read at `at`: a host timer can fire
+        // a millisecond before the clock reaches the end.
+        if (endAt !== undefined && at >= endAt)
+            return {
+                wake: "clock",
+                ms: lasted(),
+                looks,
+                ...stopAt(clockEndStop(session, lastLook ?? reference, at)),
+            };
+        if (at >= deadlineAt)
+            return { wake: "stopped", ms: lasted(), looks, stop: timeLimit(session) };
+        let look;
+        try {
+            look = await observer.look(turnNumber);
+        }
+        catch (error) {
+            if (error instanceof CuaStallError)
+                return { wake: "stalled", ms: lasted(), looks };
+            throw error;
+        }
+        looks += 1;
+        lastLook = look.observation;
+        const ms = lasted();
+        const ended = (wake, seen) => ({
+            wake,
+            ms,
+            looks,
+            observation: look.observation,
+            ...(seen === undefined ? {} : { seen }),
+            ...(seen?.verdict === "settled" ? { area: seen.area } : {}),
+        });
+        // A condition comes first: the checkpoint persists the look that matched it (a task completed
+        // at it is counted already), and the budget then stops the session before the next request.
+        if (look.condition)
+            return ended("condition");
+        const overBudget = spendStop(session);
+        if (overBudget !== undefined)
+            return { ...ended("stopped"), stop: overBudget };
+        if (look.speech)
+            return ended("speech");
+        const seen = compareLook(frames, referenceKey, look.observation, at);
+        if (seen.verdict === "settled")
+            return ended("changed", seen);
+        const windingDown = windDownAt !== undefined && at >= windDownAt;
+        // A change still moving at the last look, such as a video, ended nothing: the participant
+        // sees it in that look's frame, the hint says a large part was changing, and the next watch
+        // grows as after a still screen.
+        if (at >= limitAt || windingDown)
+            return ended(windingDown ? "clock" : "limit", seen);
+        const largeMove = seen.verdict === "changing" && seen.lookSoon;
+        quickLooks = largeMove && quickLooks < 2 ? quickLooks + 1 : 0;
+        nextLookAt = at + (quickLooks > 0 ? settleWaitMs : lookEveryMs);
+    }
+}
+const stopAt = (stop) => (stop === undefined ? {} : { stop });
+/**
+ * Whether a look shows a change that wakes the participant. A frame is compared with the reference
+ * frame tile by tile; an observation without one (a state executor) by its progress key.
+ */
+function compareLook(frames, referenceKey, observation, atMs) {
+    if (frames === undefined || observation.screenshot === undefined)
+        return progressKeyOf(observation) === referenceKey
+            ? { verdict: "unchanged" }
+            : { verdict: "settled", area: { x: 0, y: 0, width: 0, height: 0 }, small: false };
+    return frames.next(observation.screenshot, atMs);
+}
+/**
+ * Each wait's part of a watch that lasted `ms`: the limit is shared among the waits in proportion
+ * to their planned lengths, and the watch covers them in order. Later waits get 0.
+ */
+function splitWatch(lengths, limitMs, ms) {
+    const plannedMs = lengths.reduce((sum, length) => sum + length, 0);
+    const starts = [];
+    const ends = [];
+    let planned = 0;
+    for (const length of lengths) {
+        starts.push(Math.floor((limitMs * planned) / plannedMs));
+        planned += length;
+        ends.push(Math.floor((limitMs * planned) / plannedMs));
+    }
+    const found = ends.findIndex((shareEnd, index) => shareEnd > starts[index] && ms <= shareEnd);
+    const endedOn = found === -1 ? lengths.length - 1 : found;
+    return {
+        ms: lengths.map((_, index) => index < endedOn ? ends[index] - starts[index] : index === endedOn ? ms - starts[index] : 0),
+        endedOn,
+    };
+}
+/**
+ * The next request's note. A clock ending says nothing: the clock text that follows says it. A
+ * small change is named by its place, so a participant is not told the screen changed when what
+ * changed was a counter or a timestamp. `growsToMs` adds the growth sentence when the next watch
+ * doubles.
+ */
+function hintFor(ending, growsToMs) {
+    const lasted = `Your wait lasted ${duration(ending.ms)}`;
+    const grows = growsToMs === undefined
+        ? ""
+        : ` A wait right after it can last twice as long, at most ${duration(growsToMs)}.`;
+    const { seen } = ending;
+    switch (ending.wake) {
+        case "changed":
+            return seen?.verdict === "settled" && seen.small
+                ? `${lasted} and ended when a small part of the screen changed and then stayed the same for over a minute, ${place(seen.area)}.${grows}`
+                : `${lasted} and ended when the screen changed.`;
+        case "speech":
+            return `${lasted} and ended when you heard speech.`;
+        case "condition":
+            return `${lasted} and ended when the page changed.`;
+        // A small change seen less than 65 s before the limit has not held still yet, so the hint
+        // never says the screen stayed the same.
+        case "limit":
+            if (seen?.verdict === "changing" && seen.lookSoon)
+                return `${lasted}; a large part of the screen was changing at its last check.${grows}`;
+            if (seen?.verdict !== "settled" && seen?.held !== undefined)
+                return `${lasted}; a small part of the screen changed and stayed the same, ${place(seen.held)}. A small change does not end a wait.${grows}`;
+            return `${lasted}; no change on the screen held still long enough to end it.${grows}`;
+        case "stalled":
+            return `${lasted}; the screen could not be checked during it.`;
+        case "clock":
+        case "stopped":
+            return undefined;
+    }
+}
+const place = (area) => `at x ${area.x}, y ${area.y} (${area.width} x ${area.height} pixels)`;
+function duration(ms) {
+    if (ms < 1_000)
+        return plural(ms, "millisecond");
+    if (ms < 120_000)
+        return plural(Math.round(ms / 1_000), "second");
+    return plural(Math.round(ms / 60_000), "minute");
+}

package/dist/actors/computer-use/loop.js

@@ -14,6 +14,7 @@ import { LoopSession } from "./loop/session.js";
 import { spendStop, unknownSpendStop } from "./loop/spend.js";
 import { accountBillingConflicts } from "./loop/usage.js";
 import { loopResult, notice } from "./loop/trace.js";
+import { IdleWatcher } from "./loop/watch.js";
 export { describeCuaAction } from "./loop/actions.js";
 export { stableProgressKey } from "./loop/backstop.js";
 export { IMPRESSIONS_ASK } from "./loop/impressions.js";
@@ -75,6 +76,7 @@ export async function runComputerUseLoopWithTaps(options) {
  */
 async function runTurns(session, conversation) {
     const observer = new DesktopObserver(session);
+    const watcher = new IdleWatcher(session, observer);
     const opening = await observer.checkpoint(0);
     if ("stop" in opening)
         return opening.stop;
@@ -142,9 +144,13 @@ async function runTurns(session, conversation) {
                 ? { ...ended, debriefTrigger: { kind: "participant_end", observation } }
                 : ended;
         }
-        const batch = await runActionBatch(session, turn.actions);
+        const watched = await watcher.watch(turnNumber, observation, turn.actions);
+        const batch = await runActionBatch(session, turn.actions, watched?.waits);
         conversation.previousExecution = batch.execution;
-        const checkpoint = await observer.checkpoint(turnNumber);
+        // An earlier reply's unknown usage still stops the session before its closing request.
+        if (watched?.stop !== undefined)
+            return unknownSpendStop(session) ?? watched.stop;
+        const checkpoint = await observer.checkpoint(turnNumber, watched?.observation);
         if ("stop" in checkpoint)
             return checkpoint.stop;
         observation = checkpoint.observation;
@@ -153,11 +159,19 @@ async function runTurns(session, conversation) {
             observation,
             heardNewSpeech: observer.heardNewSpeech,
             waitingIsTheTask: heldToScheduledEnd(session),
+            screenChanged: watched?.wake === "changed" ||
+                watched?.wake === "speech" ||
+                watched?.wake === "condition",
         }, { idleSteps: session.idleSteps, noProgressSteps: session.noProgressSteps });
         backstop = step.backstop;
-        const stalled = applyBackstop(session, conversation, step, [checkpoint.hint, batch.hint]);
+        const stalled = applyBackstop(session, conversation, step, [
+            watched?.hint,
+            checkpoint.hint,
+            batch.hint,
+        ]);
+        // A turn that reached the participant's end is that end, never the backstop's.
         if (stalled !== undefined)
-            return stalled;
+            return clockEndStop(session, observation) ?? stalled;
         const unknownSpend = unknownSpendStop(session);
         if (unknownSpend !== undefined)
             return unknownSpend;

package/dist/actors/computer-use/wait.d.ts

@@ -7,31 +7,54 @@ export declare const CUA_WAIT_LIMITS: Readonly<{
      */
     stepMs: 30000;
     /**
-     * The longest one wait action lasts when a study does not set actor.maxWaitMs and the desktop
-     * has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
+     * The longest one wait action sent to the desktop lasts when a study does not set
+     * actor.maxWaitMs and the desktop has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
      * arrive, and the participant still sees a fresh screenshot at least that often.
      */
     defaultMaxMs: 120000;
+    /**
+     * The longest a watch over an unchanged screen grows to, and the longest the named waits of a
+     * turn that only waits are watched for together, when a study does not set actor.maxWaitMs. Each further watch
+     * in a row that ends with no change lasts twice as long as the one before, so on a static screen
+     * a participant settles at one model request per 10 minutes.
+     */
+    defaultWatchMaxMs: 600000;
     /**
      * How long a wait that names no duration lasts in a turn that only waits or takes screenshots,
      * when a study does not set actor.idleWaitMs. OpenAI's computer-use `wait` names no duration. A
      * participant told to wait about a minute sent ten in one turn, which lasts 50 s at this length;
-     * one wait a turn makes a one-minute wait about six turns.
+     * one wait a turn makes a one-minute wait about six turns. It is also how often a watch looks.
      */
     defaultIdleMs: 5000;
     /**
      * How long a wait that names no duration lasts after the participant acts in the same turn, so
      * the screen can settle before the next screenshot. Nearly every OpenAI computer-use wait follows
-     * a click or keypress this way, so a longer one would slow every run.
+     * a click or keypress this way, so a longer one would slow every run. A watch looks again this
+     * soon after a large part of the screen moved.
      */
     settleMs: 500;
+    /**
+     * The longest gap between two looks of a watch on a desktop with speech. Heard speech reaches the
+     * participant with a look, and the desktop session ends when more utterances arrive between two
+     * observations than it holds; 30 s is the spacing the speech default kept before watches.
+     */
+    speechLookMs: 30000;
     /** The smallest actor.maxWaitMs or actor.idleWaitMs a study may set. */
     leastMs: 1000;
-    /** The largest actor.maxWaitMs or actor.idleWaitMs a study may set. */
-    mostMs: 600000;
+    /** The largest actor.idleWaitMs a study may set. */
+    mostIdleMs: 600000;
+    /** The largest actor.maxWaitMs a study may set: a watch may grow to an hour. */
+    mostMaxMs: 3600000;
 }>;
-/** The wait lengths one session applies; the trace records them as `waitSettings`. */
-export type WaitSettings = Readonly<ActorWaitSettings>;
+/**
+ * The wait lengths one session applies. The trace records the first three as `waitSettings`.
+ * `watchMaxMs` is the longest a watch grows to over an unchanged screen and the longest a watched
+ * turn's named waits last together, and `lookEveryMs` how often a watch looks.
+ */
+export type WaitSettings = Readonly<ActorWaitSettings> & {
+    readonly watchMaxMs: number;
+    readonly lookEveryMs: number;
+};
 /** One wait as the desktop runs it. */
 export interface PlannedWait {
     /** How long the wait lasts. */
@@ -41,13 +64,6 @@ export interface PlannedWait {
     /** What the participant asked for, when that was longer than maxWaitMs. */
     readonly shortenedFromMs?: number;
 }
-/**
- * The longest one wait action lasts when a study does not set actor.maxWaitMs. On a desktop with
- * speech, heard speech reaches the participant only with a screenshot, and the desktop session ends
- * when more utterances arrive between two screenshots than it holds. A wait takes no screenshot, so
- * there the default keeps one wait to one desktop call.
- */
-export declare function defaultMaxWaitMs(speechEnabled: boolean): number;
 /**
  * Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
  * in a study file, empty for loop options. `maxPrefix` names the owner of maxWaitMs when it is
@@ -59,8 +75,9 @@ export declare function waitFieldsReason(fields: {
 }, prefix: string, maxPrefix?: string): string | null;
 /**
  * The wait lengths a session applies: the study's values, or their defaults. The idle wait never
- * outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
- * for fields waitFieldsReason refuses.
+ * outlasts the longest wait, which on a desktop with speech defaults to 30 s. A watch grows to
+ * actor.maxWaitMs, else 10 minutes, and looks every idle wait, at most every 30 s on a desktop with
+ * speech. Throws a RangeError for fields waitFieldsReason refuses.
  */
 export declare function resolveWaitSettings(options: {
     readonly maxWaitMs?: number | undefined;
@@ -70,6 +87,7 @@ export declare function resolveWaitSettings(options: {
 /**
  * How long one wait lasts and the desktop calls it takes. A wait that names no duration lasts
  * idleWaitMs when its turn only waits or takes screenshots (`idleTurn`), and settleWaitMs after an
- * action. A longer wait than maxWaitMs is shortened to it. 70 s is sent as 30 s, 30 s and 10 s.
+ * action. A longer wait than maxWaitMs is shortened to it, and a negative one lasts 0 ms. 70 s is
+ * sent as 30 s, 30 s and 10 s.
  */
 export declare function planWait(settings: WaitSettings, requestedMs: number | undefined, idleTurn: boolean): PlannedWait;

package/dist/actors/computer-use/wait.js

@@ -1,7 +1,9 @@
 // How long one participant wait action lasts, for every provider and desktop. A study sets the
 // longest (actor.maxWaitMs) and how long a wait that names no duration lasts while the participant
-// only waits (actor.idleWaitMs). The loop asks planWait for each wait and sends it to the desktop
-// in steps one desktop call carries.
+// only waits (actor.idleWaitMs). The loop asks planWait for each wait. A turn with another action
+// sends its waits to the desktop in steps one desktop call carries; a turn that only waits watches
+// the screen instead (loop/watch.ts), for its waits' lengths, the named ones together up to
+// watchMaxMs, or longer as the watches of idle turns in a row grow.
 export const CUA_WAIT_LIMITS = Object.freeze({
     /**
      * The longest wait one desktop call is given. A browser-control request must be answered within
@@ -10,35 +12,51 @@ export const CUA_WAIT_LIMITS = Object.freeze({
      */
     stepMs: 30_000,
     /**
-     * The longest one wait action lasts when a study does not set actor.maxWaitMs and the desktop
-     * has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
+     * The longest one wait action sent to the desktop lasts when a study does not set
+     * actor.maxWaitMs and the desktop has no speech (defaultMaxWaitMs). Two minutes covers waiting in a lobby or for an email to
      * arrive, and the participant still sees a fresh screenshot at least that often.
      */
     defaultMaxMs: 120_000,
+    /**
+     * The longest a watch over an unchanged screen grows to, and the longest the named waits of a
+     * turn that only waits are watched for together, when a study does not set actor.maxWaitMs. Each further watch
+     * in a row that ends with no change lasts twice as long as the one before, so on a static screen
+     * a participant settles at one model request per 10 minutes.
+     */
+    defaultWatchMaxMs: 600_000,
     /**
      * How long a wait that names no duration lasts in a turn that only waits or takes screenshots,
      * when a study does not set actor.idleWaitMs. OpenAI's computer-use `wait` names no duration. A
      * participant told to wait about a minute sent ten in one turn, which lasts 50 s at this length;
-     * one wait a turn makes a one-minute wait about six turns.
+     * one wait a turn makes a one-minute wait about six turns. It is also how often a watch looks.
      */
     defaultIdleMs: 5_000,
     /**
      * How long a wait that names no duration lasts after the participant acts in the same turn, so
      * the screen can settle before the next screenshot. Nearly every OpenAI computer-use wait follows
-     * a click or keypress this way, so a longer one would slow every run.
+     * a click or keypress this way, so a longer one would slow every run. A watch looks again this
+     * soon after a large part of the screen moved.
      */
     settleMs: 500,
+    /**
+     * The longest gap between two looks of a watch on a desktop with speech. Heard speech reaches the
+     * participant with a look, and the desktop session ends when more utterances arrive between two
+     * observations than it holds; 30 s is the spacing the speech default kept before watches.
+     */
+    speechLookMs: 30_000,
     /** The smallest actor.maxWaitMs or actor.idleWaitMs a study may set. */
     leastMs: 1_000,
-    /** The largest actor.maxWaitMs or actor.idleWaitMs a study may set. */
-    mostMs: 600_000,
+    /** The largest actor.idleWaitMs a study may set. */
+    mostIdleMs: 600_000,
+    /** The largest actor.maxWaitMs a study may set: a watch may grow to an hour. */
+    mostMaxMs: 3_600_000,
 });
-/** A whole number of milliseconds within the range actor.maxWaitMs and actor.idleWaitMs accept. */
-function isWaitMs(value) {
+/** A whole number of milliseconds from CUA_WAIT_LIMITS.leastMs to `mostMs`. */
+function isWaitMs(value, mostMs) {
     return (typeof value === "number" &&
         Number.isSafeInteger(value) &&
         value >= CUA_WAIT_LIMITS.leastMs &&
-        value <= CUA_WAIT_LIMITS.mostMs);
+        value <= mostMs);
 }
 /**
  * The longest one wait action lasts when a study does not set actor.maxWaitMs. On a desktop with
@@ -46,12 +64,16 @@ function isWaitMs(value) {
  * when more utterances arrive between two screenshots than it holds. A wait takes no screenshot, so
  * there the default keeps one wait to one desktop call.
  */
-export function defaultMaxWaitMs(speechEnabled) {
+function defaultMaxWaitMs(speechEnabled) {
     return speechEnabled ? CUA_WAIT_LIMITS.stepMs : CUA_WAIT_LIMITS.defaultMaxMs;
 }
 const FIELDS = [
-    ["maxWaitMs", "the longest one wait action lasts"],
-    ["idleWaitMs", "how long a wait with no duration lasts while the participant only waits"],
+    ["maxWaitMs", "the longest one wait action lasts", CUA_WAIT_LIMITS.mostMaxMs],
+    [
+        "idleWaitMs",
+        "how long a wait with no duration lasts while the participant only waits",
+        CUA_WAIT_LIMITS.mostIdleMs,
+    ],
 ];
 /**
  * Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
@@ -59,9 +81,9 @@ const FIELDS = [
  * another, as for a participant's own idleWaitMs under actor.maxWaitMs.
  */
 export function waitFieldsReason(fields, prefix, maxPrefix = prefix) {
-    for (const [name, meaning] of FIELDS) {
-        if (fields[name] !== undefined && !isWaitMs(fields[name]))
-            return `${name === "maxWaitMs" ? maxPrefix : prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
+    for (const [name, meaning, mostMs] of FIELDS) {
+        if (fields[name] !== undefined && !isWaitMs(fields[name], mostMs))
+            return `${name === "maxWaitMs" ? maxPrefix : prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${mostMs}.`;
     }
     const { maxWaitMs, idleWaitMs } = fields;
     if (typeof maxWaitMs === "number" && typeof idleWaitMs === "number" && idleWaitMs > maxWaitMs)
@@ -70,27 +92,34 @@ export function waitFieldsReason(fields, prefix, maxPrefix = prefix) {
 }
 /**
  * The wait lengths a session applies: the study's values, or their defaults. The idle wait never
- * outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
- * for fields waitFieldsReason refuses.
+ * outlasts the longest wait, which on a desktop with speech defaults to 30 s. A watch grows to
+ * actor.maxWaitMs, else 10 minutes, and looks every idle wait, at most every 30 s on a desktop with
+ * speech. Throws a RangeError for fields waitFieldsReason refuses.
  */
 export function resolveWaitSettings(options) {
     const reason = waitFieldsReason(options, "");
     if (reason !== null)
         throw new RangeError(reason);
     const maxWaitMs = options.maxWaitMs ?? defaultMaxWaitMs(options.speechEnabled);
+    const idleWaitMs = Math.min(options.idleWaitMs ?? CUA_WAIT_LIMITS.defaultIdleMs, maxWaitMs);
     return {
         maxWaitMs,
-        idleWaitMs: Math.min(options.idleWaitMs ?? CUA_WAIT_LIMITS.defaultIdleMs, maxWaitMs),
+        idleWaitMs,
         settleWaitMs: CUA_WAIT_LIMITS.settleMs,
+        watchMaxMs: options.maxWaitMs ?? CUA_WAIT_LIMITS.defaultWatchMaxMs,
+        lookEveryMs: options.speechEnabled
+            ? Math.min(idleWaitMs, CUA_WAIT_LIMITS.speechLookMs)
+            : idleWaitMs,
     };
 }
 /**
  * How long one wait lasts and the desktop calls it takes. A wait that names no duration lasts
  * idleWaitMs when its turn only waits or takes screenshots (`idleTurn`), and settleWaitMs after an
- * action. A longer wait than maxWaitMs is shortened to it. 70 s is sent as 30 s, 30 s and 10 s.
+ * action. A longer wait than maxWaitMs is shortened to it, and a negative one lasts 0 ms. 70 s is
+ * sent as 30 s, 30 s and 10 s.
  */
 export function planWait(settings, requestedMs, idleTurn) {
-    const askedMs = requestedMs ?? (idleTurn ? settings.idleWaitMs : settings.settleWaitMs);
+    const askedMs = Math.max(0, requestedMs ?? (idleTurn ? settings.idleWaitMs : settings.settleWaitMs));
     const ms = Math.min(askedMs, settings.maxWaitMs);
     const { stepMs } = CUA_WAIT_LIMITS;
     const steps = [];

package/dist/actors/contract.d.ts

@@ -87,6 +87,40 @@ export interface ActorTraceItem {
     /** On a computer-use `screenshot` item: the page that observation was on. Additive: absent when
      *  the observation read no URL, on other producers and in older bundles. */
     page?: TracePage;
+    /** On the computer-use `ui_action` of the wait during which a watch ended: the watch the loop
+     *  ran in place of a turn that only waited. Additive: absent on plain waits, on other items and
+     *  producers, and in older bundles. */
+    watch?: ActorWatch;
+}
+/**
+ * A watch over the screen while a computer-use participant only waited: the loop looked every
+ * idle wait and asked the model again when the screen changed and held still, or at the limit.
+ */
+export interface ActorWatch {
+    /** How long the watch could last, in milliseconds. */
+    limitMs: number;
+    /**
+     * How long the turn's waits asked for, in milliseconds: the lengths they named, and the idle
+     * wait for each that named none. Above limitMs when the named waits together passed the longest
+     * a watch grows to (actor.maxWaitMs, else 10 minutes).
+     */
+    askedMs: number;
+    /**
+     * Why it ended: `changed` (a change held still), `speech` (the participant heard speech),
+     * `condition` (a study stop condition, dwell trigger or task matched a look), `limit` (its time
+     * passed with no change holding still), `clock` (the participant's wind-down or end arrived),
+     * `stalled` (a look got no frame), `stopped` (the study budget or the session's time ran out).
+     */
+    wake: "changed" | "speech" | "condition" | "limit" | "clock" | "stalled" | "stopped";
+    /** How many times it looked at the screen. */
+    looks: number;
+    /** The region of change that woke it, in pixels from the frame's top-left; coordinates only. */
+    area?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
 }
 export interface ActorCapabilities {
     headless: boolean;

package/dist/actors/local-agent/cli.js

@@ -190,6 +190,11 @@ export function promptFor(request, screenshotPath) {
             "giving up, and put your closing words in message. When done=true, set outcome: reached if " +
             "the task is finished, blocked if something in the app stopped you, not_reached if you are " +
             "stopping for another reason. Otherwise outcome is null.",
+        "A turn whose actions are only waits ends when the screen changes and holds still, or when its " +
+            "waits' ms pass. While no change on the screen holds still, each further turn in a row that " +
+            "only waits lasts twice as long as the one before, up to the study's longest wait, and the note " +
+            "from the harness says how long you waited and why it ended. A wait among other actions lasts " +
+            "its ms.",
         `Reply with ONLY a JSON object of this shape: ${shape}`,
         hint,
     ].join("\n");

package/dist/evidence/frame-change.d.ts

@@ -0,0 +1,39 @@
+/** A rectangle of the frame, in pixels from its top-left corner. */
+export interface FrameRegion {
+    readonly x: number;
+    readonly y: number;
+    readonly width: number;
+    readonly height: number;
+}
+/**
+ * What one look found, compared with the frame the participant last saw. `settled`: a region of
+ * change held still long enough to wake the participant; `area` bounds it, and `small` is true
+ * when it is at most LARGE_REGION_TILES tiles, which the participant's hint names. `changing`: tiles moved
+ * since the previous look and nothing settled; `lookSoon` when more than LARGE_REGION_TILES moved,
+ * so a large change is confirmed by a quick look. `unchanged`: nothing moved and nothing settled.
+ * `held` on the last two bounds a small region that held still past the hold and is held back by a
+ * ticking tile or a large moving region near it.
+ */
+export type FrameVerdict = {
+    readonly verdict: "unchanged";
+    readonly held?: FrameRegion;
+} | {
+    readonly verdict: "changing";
+    readonly lookSoon: boolean;
+    readonly held?: FrameRegion;
+} | {
+    readonly verdict: "settled";
+    readonly area: FrameRegion;
+    readonly small: boolean;
+};
+/**
+ * Compare later looks at one screen with `reference`, the frame the participant last saw. Call
+ * `next` once per look, in order, with the look's time. Each tile that differs from the reference
+ * keeps the time it last moved, so a moving tile (a spinner) never joins a still region beside
+ * it. Nothing is masked: a large change that stops changing always settles, and a small one does
+ * unless a tile near it keeps ticking. Stillness is sampled at looks, so a blink seen in the same
+ * phase at every look reads as still.
+ */
+export declare function watchFrames(reference: Buffer): {
+    next(frame: Buffer, atMs: number): FrameVerdict;
+};