humanish 0.130.0 → 0.131.1

package/dist/evidence/frame-change.js

@@ -0,0 +1,293 @@
+import { PNG } from "pngjs";
+// Whether a screen changed while a participant waited, and whether the change has stopped
+// changing. The loop's watch (src/actors/computer-use/loop/watch.ts) asks after each look.
+//
+// The constants were chosen on 323 idle screenshot pairs from 203 retained hosted-desktop runs.
+// Regions that changed on their own were the panel clock (2 to 6 tiles, every 60 s), a text caret
+// (2 tiles, about every 0.5 s), Chrome's tab spinner (2 to 4 tiles) and one landing page's
+// rotating tagline (10 to 36 tiles, median 18). Page loads, navigations and app updates changed
+// 94 to 2,598 tiles. A moved list selection changed 35 tiles and a re-rendered
+// block of timestamps 8 to 12, both missed by the frame hash in frame-signature.ts.
+//
+// The rule for small regions was chosen on the wave 9 look corpus: 3 pages recorded for 40 minutes
+// each on hosted desktops, replayed over 84 to 168 look timelines a page. There a small region
+// that held still woke the participant falsely beside a panel clock, a relative timestamp
+// ("Opened 3 minutes ago", whose digit ticks each minute and whose tail shifts when the text
+// grows) and a moving chart. With the conditions below it woke none falsely and missed no event.
+/** The side of a square tile, in pixels: one tile is the unit of change. */
+const TILE_PX = 16;
+/**
+ * A pixel changed when one of its channels differs by more than this (0 to 255). On the wave 9
+ * corpus 32 added a false chart wake, and 48 split a new row's light fill from its text, so a row
+ * read as small regions and was seen 69.9 s late.
+ */
+const PIXEL_TOLERANCE = 24;
+/**
+ * A tile changed when at least this many of its pixels changed. One or two pixels are an
+ * antialiased edge: a spinner edge 2 pixels over the tolerance woke a 900 s watch in wave 7 live
+ * run 3c. On the wave 9 corpus every tile of its status and export labels changed 10 to 69 pixels,
+ * and its new rows (509 and 545 tiles) lost 7 or 8 tiles at 3. 1 and 3 gave the same wakes there.
+ */
+const TILE_MIN_PIXELS = 3;
+/**
+ * A still region of more than this many tiles wakes the participant at once: it is above every
+ * self-changing region measured except 11 of 110 tagline swaps, and below every page change. On
+ * the wave 9 corpus the status label (30 tiles) and new rows were large, and the export label (8
+ * tiles) small.
+ */
+const LARGE_REGION_TILES = 24;
+/**
+ * A still region of at most LARGE_REGION_TILES tiles wakes the participant once every tile in it
+ * has held still for more than this, and no tile within NEAR_TILES of it ticked or moved with a
+ * region of more than MOVING_REGION_TILES tiles in this time. Minute updates (a clock's minutes
+ * digit, "2 min ago") move again within it. On the wave 9 corpus a one-time 8-tile text change was
+ * seen 72.4 s after it landed (median, 5 s looks).
+ */
+const SMALL_REGION_HOLD_MS = 65_000;
+/**
+ * How near, in tiles, a ticking tile or a large moving region holds a small region back. The
+ * relative timestamp's tail ("1 minute ago" to "2 minutes ago") sat 4 tiles from its digit in
+ * wave 7 live run 2a and on the wave 9 corpus, where a distance of 1 left 4 tail wakes in 168
+ * timelines and 4 left none.
+ */
+const NEAR_TILES = 4;
+/**
+ * A tile ticks when its last move came at least this long after the move before it, or when it
+ * has moved once. A minute counter ticks. A spinner tile that moves every few looks does not, so a
+ * label beside a spinner still wakes the participant, at 90 s on the wave 9 synthetic screens.
+ */
+const TICK_GAP_MS = 30_000;
+/**
+ * A tile that moves with a connected moving region of more than this many tiles (a chart, a video)
+ * holds a small region within NEAR_TILES back. A chart's tiles move every few looks, so they never
+ * tick. On the wave 9 corpus, without this condition a chart's tiles woke 3 watches falsely in its
+ * page's 40 minutes, and 259 in its 84 replayed timelines; with it, none. It was not tuned apart
+ * from LARGE_REGION_TILES and has the same value. A spinner of 1 to 4 tiles stays under it.
+ */
+const MOVING_REGION_TILES = 24;
+const UNCHANGED = { verdict: "unchanged" };
+function decode(frame) {
+    try {
+        const image = PNG.sync.read(frame);
+        return image.width > 0 && image.height > 0 ? image : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Compare later looks at one screen with `reference`, the frame the participant last saw. Call
+ * `next` once per look, in order, with the look's time. Each tile that differs from the reference
+ * keeps the time it last moved, so a moving tile (a spinner) never joins a still region beside
+ * it. Nothing is masked: a large change that stops changing always settles, and a small one does
+ * unless a tile near it keeps ticking. Stillness is sampled at looks, so a blink seen in the same
+ * phase at every look reads as still.
+ */
+export function watchFrames(reference) {
+    const base = decode(reference);
+    // Each tile that differs from the reference, with its pixels at the look it last moved at. Only
+    // these are kept from the previous look, so a watch holds one decoded frame and not two.
+    const differing = new Map();
+    let moves;
+    return {
+        next(frame, atMs) {
+            const current = decode(frame);
+            // A frame that cannot be decoded is compared by its bytes; a frame of another size is a new
+            // screen, one change over the whole frame.
+            if (base === undefined || current === undefined)
+                return frame.equals(reference) ? UNCHANGED : wholeFrame(current ?? base);
+            if (current.width !== base.width || current.height !== base.height)
+                return wholeFrame(current);
+            moves ??= tileMoves(base);
+            return compare(base, current, atMs, differing, moves);
+        },
+    };
+}
+function tileMoves(image) {
+    const columns = Math.ceil(image.width / TILE_PX);
+    const rows = Math.ceil(image.height / TILE_PX);
+    const never = () => new Float64Array(columns * rows).fill(Number.NaN);
+    return { columns, rows, moved: never(), movedBefore: never(), movedLarge: never() };
+}
+function wholeFrame(image) {
+    return {
+        verdict: "settled",
+        area: { x: 0, y: 0, width: image?.width ?? 0, height: image?.height ?? 0 },
+        small: false,
+    };
+}
+function compare(base, current, atMs, differing, moves) {
+    const { columns, rows } = moves;
+    const still = new Set();
+    const movedNow = new Set();
+    let moved = 0;
+    const move = (tile) => {
+        moves.movedBefore[tile] = moves.moved[tile];
+        moves.moved[tile] = atMs;
+        movedNow.add(tile);
+    };
+    for (let row = 0; row < rows; row += 1)
+        for (let column = 0; column < columns; column += 1) {
+            const tile = row * columns + column;
+            const before = differing.get(tile);
+            if (!framesDiffer(base, current, column, row)) {
+                if (before === undefined)
+                    continue;
+                differing.delete(tile);
+                // A move back to the reference is a move: a ticking digit can tick back to its old shape.
+                if (tileDiffers(before, current, column, row))
+                    move(tile);
+                continue;
+            }
+            // A tile that starts differing has moved, whatever the previous look's pixels were.
+            if (before === undefined || tileDiffers(before, current, column, row)) {
+                differing.set(tile, tilePixels(current, column, row));
+                move(tile);
+                moved += 1;
+            }
+            else
+                still.add(tile);
+        }
+    for (const region of connectedRegions(movedNow, columns))
+        if (region.length > MOVING_REGION_TILES)
+            for (const tile of region)
+                moves.movedLarge[tile] = atMs;
+    const regions = connectedRegions(still, columns);
+    const area = (region) => areaOf(region, columns, base.width, base.height);
+    const large = largest(regions.filter((region) => region.length > LARGE_REGION_TILES));
+    if (large !== undefined)
+        return { verdict: "settled", area: area(large), small: false };
+    const heldSmall = regions.filter((region) => region.every((tile) => atMs - moves.moved[tile] > SMALL_REGION_HOLD_MS));
+    const settled = largest(heldSmall.filter((region) => !heldBack(region, moves, atMs)));
+    if (settled !== undefined)
+        return { verdict: "settled", area: area(settled), small: true };
+    const waiting = largest(heldSmall);
+    const held = waiting === undefined ? {} : { held: area(waiting) };
+    if (moved > 0)
+        return { verdict: "changing", lookSoon: moved > LARGE_REGION_TILES, ...held };
+    return { verdict: "unchanged", ...held };
+}
+/**
+ * Whether a tile within NEAR_TILES of the region, outside it, ticked or moved with a large region
+ * within the hold.
+ */
+function heldBack(region, moves, atMs) {
+    const inRegion = new Set(region);
+    // NaN, a tile that never moved, is never recent.
+    const recent = (ms) => atMs - ms <= SMALL_REGION_HOLD_MS;
+    for (const tile of region)
+        for (let dy = -NEAR_TILES; dy <= NEAR_TILES; dy += 1)
+            for (let dx = -NEAR_TILES; dx <= NEAR_TILES; dx += 1) {
+                const column = (tile % moves.columns) + dx;
+                const row = Math.floor(tile / moves.columns) + dy;
+                if (column < 0 || column >= moves.columns || row < 0 || row >= moves.rows)
+                    continue;
+                const near = row * moves.columns + column;
+                const moved = moves.moved[near];
+                if (inRegion.has(near) || !recent(moved))
+                    continue;
+                const before = moves.movedBefore[near];
+                if (Number.isNaN(before) ||
+                    moved - before >= TICK_GAP_MS ||
+                    recent(moves.movedLarge[near]))
+                    return true;
+            }
+    return false;
+}
+/** Whether at least TILE_MIN_PIXELS pixels of one tile differ between two frames of one size. */
+function framesDiffer(a, b, column, row) {
+    const { width, height } = a;
+    const xEnd = Math.min(width, (column + 1) * TILE_PX);
+    const yEnd = Math.min(height, (row + 1) * TILE_PX);
+    let changed = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1)
+        for (let x = column * TILE_PX; x < xEnd; x += 1) {
+            const at = (y * width + x) * 4;
+            if ((Math.abs(a.data[at] - b.data[at]) > PIXEL_TOLERANCE ||
+                Math.abs(a.data[at + 1] - b.data[at + 1]) > PIXEL_TOLERANCE ||
+                Math.abs(a.data[at + 2] - b.data[at + 2]) > PIXEL_TOLERANCE) &&
+                ++changed >= TILE_MIN_PIXELS)
+                return true;
+        }
+    return false;
+}
+/** A copy of one tile's RGBA pixels, row by row. */
+function tilePixels(image, column, row) {
+    const xStart = column * TILE_PX;
+    const xEnd = Math.min(image.width, xStart + TILE_PX);
+    const yEnd = Math.min(image.height, (row + 1) * TILE_PX);
+    const pixels = new Uint8Array((xEnd - xStart) * (yEnd - row * TILE_PX) * 4);
+    let to = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1) {
+        const from = (y * image.width + xStart) * 4;
+        pixels.set(image.data.subarray(from, from + (xEnd - xStart) * 4), to);
+        to += (xEnd - xStart) * 4;
+    }
+    return pixels;
+}
+/**
+ * Whether at least TILE_MIN_PIXELS pixels of one tile of `image` differ from `pixels`, that tile's
+ * earlier pixels.
+ */
+function tileDiffers(pixels, image, column, row) {
+    const xStart = column * TILE_PX;
+    const xEnd = Math.min(image.width, xStart + TILE_PX);
+    const yEnd = Math.min(image.height, (row + 1) * TILE_PX);
+    const { data } = image;
+    let at = 0;
+    let changed = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1)
+        for (let x = xStart; x < xEnd; x += 1, at += 4) {
+            const from = (y * image.width + x) * 4;
+            if ((Math.abs(pixels[at] - data[from]) > PIXEL_TOLERANCE ||
+                Math.abs(pixels[at + 1] - data[from + 1]) > PIXEL_TOLERANCE ||
+                Math.abs(pixels[at + 2] - data[from + 2]) > PIXEL_TOLERANCE) &&
+                ++changed >= TILE_MIN_PIXELS)
+                return true;
+        }
+    return false;
+}
+/** Tiles grouped with every tile of the set next to them, diagonals included. */
+function connectedRegions(tiles, columns) {
+    const seen = new Set();
+    const regions = [];
+    for (const start of tiles) {
+        if (seen.has(start))
+            continue;
+        seen.add(start);
+        const region = [];
+        const queue = [start];
+        for (let tile = queue.pop(); tile !== undefined; tile = queue.pop()) {
+            region.push(tile);
+            const column = tile % columns;
+            for (const dy of [-1, 0, 1])
+                for (const dx of [-1, 0, 1]) {
+                    const neighbourColumn = column + dx;
+                    const neighbour = tile + dy * columns + dx;
+                    if (neighbourColumn < 0 || neighbourColumn >= columns)
+                        continue;
+                    if (!tiles.has(neighbour) || seen.has(neighbour))
+                        continue;
+                    seen.add(neighbour);
+                    queue.push(neighbour);
+                }
+        }
+        regions.push(region);
+    }
+    return regions;
+}
+function largest(regions) {
+    return regions.reduce((best, region) => (best === undefined || region.length > best.length ? region : best), undefined);
+}
+function areaOf(tiles, columns, width, height) {
+    const columnsOf = tiles.map((tile) => tile % columns);
+    const rowsOf = tiles.map((tile) => Math.floor(tile / columns));
+    const x = Math.min(...columnsOf) * TILE_PX;
+    const y = Math.min(...rowsOf) * TILE_PX;
+    return {
+        x,
+        y,
+        width: Math.min(width, (Math.max(...columnsOf) + 1) * TILE_PX) - x,
+        height: Math.min(height, (Math.max(...rowsOf) + 1) * TILE_PX) - y,
+    };
+}

package/dist/routes/computer-use/e2b-desktop/desktop.js

@@ -2,7 +2,7 @@ import { createE2BDesktopExecutor } from "../../../substrates/e2b/desktop-execut
 import { participantInbox, planParticipantComms } from "./comms.js";
 import { participantBrowserStateObserver } from "./fidelity.js";
 import { acquireParticipantDesktop, placeParticipantFiles, provisionParticipantSubject, verifyParticipantScreen, } from "./prepare.js";
-import { openParticipantSurface, startParticipantMedia, startParticipantStream } from "./start.js";
+import { keepParticipantScreenOn, openParticipantSurface, startParticipantMedia, startParticipantStream, } from "./start.js";
 import { desktopEvidenceOf, newParticipantState } from "./state.js";
 import { finishE2BDesktop } from "./teardown.js";
 import { participantServeUrl } from "../types.js";
@@ -44,6 +44,7 @@ export function createE2BParticipantDesktop(spec, deps, warnings) {
         await provisionParticipantSubject(ctx, state, desktop);
         await startParticipantMedia(ctx, state, desktop);
         await openParticipantSurface(ctx, state, desktop);
+        await keepParticipantScreenOn(ctx, desktop);
         prepared = true;
     }
     async function openSession() {

package/dist/routes/computer-use/e2b-desktop/start.d.ts

@@ -8,6 +8,15 @@ export declare function startParticipantMedia(ctx: E2BParticipantContext, state:
  * desktop's panel clock in it; a clock that cannot be set is a warning, and it shows UTC.
  */
 export declare function openParticipantSurface(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
+/**
+ * Turn the desktop's screen saver off, so the screen stays on while the participant waits without
+ * input. The XFCE session sets the screen saver's timeout in its first seconds, and the last write
+ * wins. Nothing waits for those writes: this runs after the browser or terminal has opened and the
+ * desktop has waited DESKTOP_SETTLE_MS, and on 16 fresh desktops the last write came within 2.5 s
+ * of a command sent as soon as the desktop was created. A screen saver that cannot be turned off
+ * is a warning.
+ */
+export declare function keepParticipantScreenOn(ctx: E2BParticipantContext, desktop: E2BDesktopSandbox): Promise<void>;
 /**
  * Measure the browser window and start the live stream. A stream that cannot start is a warning;
  * unusable browser geometry fails the participant before any action.

package/dist/routes/computer-use/e2b-desktop/start.js

@@ -1,12 +1,13 @@
 // The second half of an E2B desktop participant's preparation, and the start of its session: speech and
-// recording, then the browser or terminal the participant sees, then the live stream. Each step
-// fills the participant's state.
+// recording, then the browser or terminal the participant sees, then the screen saver turned off,
+// then the live stream. Each step fills the participant's state.
 import { redactText, toErrorMessage } from "../../../evidence/redaction.js";
 import { CHROME_DEVTOOLS_PORT, DESKTOP_SETTLE_MS, openDesktopBrowserTarget, openDesktopTerminal, startDesktopStream, } from "../../../substrates/e2b/desktop-browser.js";
 import { setPanelClockZone } from "../../../substrates/e2b/desktop-clock.js";
 import { captureDesktopBrowserGeometry } from "../../../substrates/e2b/desktop-geometry.js";
 import { prepareDesktopMedia, startE2BDesktopMedia, } from "../../../substrates/e2b/desktop-media.js";
 import { startE2BDesktopRecording } from "../../../substrates/e2b/desktop-recording.js";
+import { turnOffScreenSaver } from "../../../substrates/e2b/desktop-screen.js";
 import { applyParticipantMobileFidelity, mobileLaunchFlags } from "./fidelity.js";
 /** Start the declared speech worker and screen recording. A recording that cannot start is a warning. */
 export async function startParticipantMedia(ctx, state, desktop) {
@@ -101,6 +102,20 @@ export async function openParticipantSurface(ctx, state, desktop) {
         warnings,
     });
 }
+/**
+ * Turn the desktop's screen saver off, so the screen stays on while the participant waits without
+ * input. The XFCE session sets the screen saver's timeout in its first seconds, and the last write
+ * wins. Nothing waits for those writes: this runs after the browser or terminal has opened and the
+ * desktop has waited DESKTOP_SETTLE_MS, and on 16 fresh desktops the last write came within 2.5 s
+ * of a command sent as soon as the desktop was created. A screen saver that cannot be turned off
+ * is a warning.
+ */
+export async function keepParticipantScreenOn(ctx, desktop) {
+    const { spec, deps, warnings } = ctx;
+    await turnOffScreenSaver(desktop, deps.requestTimeoutMs).catch((error) => {
+        warnings.push(`The desktop's screen saver for participant ${spec.planned.id} could not be turned off, so its screen may go black after 5 minutes without input: ${redactText(deps.scrubKnownValues(toErrorMessage(error)))}`);
+    });
+}
 /**
  * DevTools answering later than this after launch is recorded as a warning on the participant. Six live
  * launches answered in 4.8-7.7 s; without this wait the reads it guards give up at about 11 s.

package/dist/study/plan-participants.d.ts

@@ -37,11 +37,11 @@ export interface DesktopParticipant extends Participant {
         readonly dwell?: DwellWindow;
         readonly reasoningEffort?: ReasoningEffort;
         readonly maxOutputTokens?: number;
-        /** actor.maxWaitMs: the longest one wait action lasts. */
+        /** actor.maxWaitMs: the longest one wait or one turn of only waits lasts. */
         readonly maxWaitMs?: number;
         /**
          * The participant's idleWaitMs, else actor.idleWaitMs: how long a wait with no duration lasts
-         * while the participant only waits.
+         * while the participant only waits, and how often the loop looks at its screen then.
          */
         readonly idleWaitMs?: number;
         /**

package/dist/study/types.d.ts

@@ -361,7 +361,8 @@ export interface StudyParticipantEntry {
     windDownMs?: number;
     /**
      * Computer use and shared world: this participant's actor.idleWaitMs, how long a wait that names
-     * no duration lasts while it only waits. Same range, and no longer than actor.maxWaitMs.
+     * no duration lasts while it only waits and how often the loop looks at its screen then. Same
+     * range, and no longer than actor.maxWaitMs.
      */
     idleWaitMs?: number;
     /**
@@ -404,16 +405,19 @@ export interface StudyActor {
     maxOutputTokens?: number;
     /**
      * Computer-use and shared-world: the longest one `wait` action lasts, in whole milliseconds from
-     * 1000 to 600000. Default 120000, or 30000 on a desktop with speech. A longer wait is shortened
-     * to it, the trace records a `wait shortened` notice, and the participant is told on its next
-     * turn.
+     * 1000 to 3600000. Default 120000, or 30000 on a desktop with speech. In a turn with another
+     * action a longer wait is shortened to it, the trace records a `wait shortened` notice, and the
+     * participant is told on its next turn. A turn that only waits is watched: its named waits last
+     * their lengths, together up to this or to 600000 when it is not set, and on a screen that stays
+     * the same its watches in a row grow to the same cap.
      */
     maxWaitMs?: number;
     /**
      * Computer-use and shared-world: how long a `wait` that names no duration lasts when the
-     * participant's turn only waits or takes screenshots, in whole milliseconds from 1000 to 600000
-     * and no longer than maxWaitMs. Default 5000. After an action in the same turn such a wait lasts
-     * 500 ms. OpenAI's computer-use `wait` never names a duration.
+     * participant's turn only waits or takes screenshots, and how often the loop looks at the screen
+     * while it watches such a turn, in whole milliseconds from 1000 to 600000 and no longer than
+     * maxWaitMs. Default 5000. After an action in the same turn such a wait lasts 500 ms. OpenAI's
+     * computer-use `wait` never names a duration.
      */
     idleWaitMs?: number;
     /**

package/dist/study/warnings.js

@@ -66,7 +66,7 @@ const ACTOR_ROWS = [
     },
     {
         field: "actor.idleWaitMs",
-        reason: "how long a computer-use participant's wait with no duration lasts; needs route: computer-use or shared-world",
+        reason: "how long a computer-use participant's wait with no duration lasts and how often it looks while it waits; needs route: computer-use or shared-world",
         applies: (config, routes) => config.actor?.idleWaitMs !== undefined && !routes.cua && !routes.shared,
     },
 ];

package/dist/substrates/e2b/desktop-screen.d.ts

@@ -0,0 +1,10 @@
+import type { E2BDesktopSandbox } from "./sdk.js";
+/**
+ * Turn the desktop's X screen saver off with `xset s off`. Nothing here waits for the three
+ * writers, so a call holds only when it comes after their last write: on 16 fresh desktops that
+ * write came 1.6 to 2.5 s after a command sent as soon as the desktop was created, and none came
+ * in the following 27 s. After that, xfce4-screensaver and xfce4-power-manager write the timeout
+ * again only when the xfconf settings of either one change, which setting the panel clock does not
+ * do. A desktop without xset is left as it is. Throws with xset's error output when xset fails.
+ */
+export declare function turnOffScreenSaver(desktop: E2BDesktopSandbox, requestTimeoutMs: number): Promise<void>;

package/dist/substrates/e2b/desktop-screen.js

@@ -0,0 +1,25 @@
+// The hosted desktop's X screen saver. The stock template's XFCE session has three writers of its
+// timeout, each acting in the session's first seconds: xfce4-screensaver sets its idle delay
+// (300 s), xfce4-power-manager sets its blank-on-ac time (600 s), and an autostart entry runs
+// `xset s off`. The last write wins. A timeout left at 300 s turns the screen black after 5 minutes
+// without input, so a participant waiting on a page sees a black screen. The desktop runs on Xvfb,
+// which has no display power management extension, so the screen saver is the only thing that
+// blanks it.
+import { e2bShell } from "./shell.js";
+/**
+ * Turn the desktop's X screen saver off with `xset s off`. Nothing here waits for the three
+ * writers, so a call holds only when it comes after their last write: on 16 fresh desktops that
+ * write came 1.6 to 2.5 s after a command sent as soon as the desktop was created, and none came
+ * in the following 27 s. After that, xfce4-screensaver and xfce4-power-manager write the timeout
+ * again only when the xfconf settings of either one change, which setting the panel clock does not
+ * do. A desktop without xset is left as it is. Throws with xset's error output when xset fails.
+ */
+export async function turnOffScreenSaver(desktop, requestTimeoutMs) {
+    const result = await e2bShell(desktop).run([
+        'export DISPLAY="${DISPLAY:-:0}"',
+        "command -v xset >/dev/null 2>&1 || exit 0",
+        "xset s off",
+    ].join("\n"), { requestTimeoutMs, timeoutMs: 15_000 });
+    if (result.exitCode !== 0)
+        throw new Error(result.stderr.trim() || `xset exited ${result.exitCode}`);
+}

package/docs/architecture/actor-contract.md

@@ -168,6 +168,14 @@ export interface ActorTraceItem {
   screenshotRef?: { path: string; redaction: "blurred" | "ocr_scrubbed" | "none" };
   text?: string; // redacted
   page?: { origin: string; path: string }; // computer-use screenshot items; redacted, no query
+  // computer-use: the wait ui_action during which a watch of a turn that only waited ended
+  watch?: {
+    limitMs: number;
+    askedMs: number; // what the turn's waits asked for; above limitMs past the watch's growth cap
+    wake: "changed" | "speech" | "condition" | "limit" | "clock" | "stalled" | "stopped";
+    looks: number;
+    area?: { x: number; y: number; width: number; height: number }; // pixels, coordinates only
+  };
 }
 
 export interface ActorCapabilities {

package/docs/architecture/browser-control.md

@@ -104,15 +104,17 @@ title, text, fractional scroll position and heard speech; those remain
 runtime-only under the existing loop contract. Arbitrary `appState` is refused
 because v1 has no closed schema for it. The protocol does not truncate strings,
 round coordinates, or silently drop unsupported state. A participant may ask for a
-longer wait than one request carries. The computer-use loop shortens it to the study's
-`actor.maxWaitMs` (by default two minutes, or 30 seconds on a desktop with speech, whose heard
-speech reaches the participant only with a screenshot), records a `wait shortened` notice with both
-durations when it does, and sends the rest as consecutive waits of at most 30 seconds. The Codex
-participant's `humanish_ui` schema leaves a wait's length open for that reason; the wire still
-refuses a single wait over 30 seconds. A wait that names no duration, as every OpenAI computer-use
-wait does, reaches the desktop with a length too: the study's `actor.idleWaitMs` (5 seconds by
-default) when the participant's turn only waits or takes screenshots, and 500 ms after an action in
-the same turn.
+longer wait than one request carries. In a turn with another action, the computer-use loop
+shortens it to the study's `actor.maxWaitMs` (by default two minutes, or 30 seconds on a desktop
+with speech, whose heard speech reaches the participant only with a screenshot), records a `wait
+shortened` notice with both durations when it does, and sends the rest as consecutive waits of at
+most 30 seconds. The Codex participant's `humanish_ui` schema leaves a wait's length open for that
+reason; the wire still refuses a single wait over 30 seconds. A wait that names no duration, as
+every OpenAI computer-use wait does, reaches the desktop with a length too: 500 ms after an action
+in the same turn. A turn that only waits or takes screenshots sends no wait to the desktop: the
+loop observes the screen every `actor.idleWaitMs` until it changes or the wait's time passes
+(`src/actors/computer-use/loop/watch.ts`), so a waiting participant costs one observe request per
+look.
 
 Click, double-click, move, scroll and drag may carry `heldKeys`, which the
 OpenAI provider maps from its computer tool's `keys`. The Codex participant's

package/docs/contracts/schemas.md

@@ -263,8 +263,10 @@ The rest of this section names keys by their v3 spelling:
   the end, a turn that requests no action does not end the session: the trace records a
   `participant ended before its scheduled end` notice, the next request says the time left, and
   the turn counts toward the idle backstop. Before the end, a turn that only waits or takes
-  screenshots counts toward neither backstop streak (`advanceBackstop`'s `waitingIsTheTask`). The
-  plan entry carries `endAfterMs`, and `study check` prints a `windows` row;
+  screenshots counts toward neither backstop streak (`advanceBackstop`'s `waitingIsTheTask`). A
+  watch that reaches the end stops there with no further look, and a backstop streak that trips on
+  a turn reaching the end gives way to the end. The plan entry carries `endAfterMs`, and `study
+  check` prints a `windows` row;
 - `participants[].maxSessionMs` (computer-use only, also on `count:` groups): the longest the
   participant's session lasts, in whole milliseconds from 1 to 86,400,000, counted from the start
   of its own session. `windDownMs` applies to it as to `endAfterMs`. Refused: a value outside that
@@ -282,7 +284,9 @@ The rest of this section names keys by their v3 spelling:
   `cap=<duration>`;
 - `participants[].idleWaitMs` (computer-use and shared-world, also on `count:` groups): this
   participant's `actor.idleWaitMs`, in the same range and refused above an explicit
-  `actor.maxWaitMs` (`waitLimitValidationReason`). Its trace records it in `waitSettings`;
+  `actor.maxWaitMs` (`waitLimitValidationReason`): how long its wait with no duration lasts in a
+  turn that only waits, and how often the loop looks at its screen while it watches such a turn.
+  Its trace records it in `waitSettings`;
 - `actor.clock.timeZone` (computer-use and shared-world): a tz database time zone name, such as `Europe/London`. Each turn's context hint carries
   the local time in it, and a scheduled end or session cap is given in it (UTC without it). On a hosted desktop the
   participant's browser launches with `TZ` set to it (`openParticipantSurface` in
@@ -1067,6 +1071,35 @@ Core-owned fields:
   click-like `ui_action` items, structured `coord` (`x`/`y`). Both are additive:
   absent on older bundles and non-stamping producers, and absence means
   "timing/position unrecorded", never zero
+- optional `items[].watch` (`ActorWatch` in `src/actors/contract.ts`), on the
+  computer-use `ui_action` of the wait during which a watch ended. A turn whose
+  actions are only waits and screenshots, and that asks for wait time, sends no
+  wait to the desktop: the loop looks at the screen every `waitSettings.idleWaitMs`
+  and requests the next turn when a change holds still, speech arrives, a
+  `stopWhen` rule, `dwell.when` or task `success` rule matches a look, the
+  participant's wind-down or end arrives, or the watch's time passes
+  (`src/actors/computer-use/loop/watch.ts`). A change of more than 24 tiles of
+  16 pixels counts once it holds still at a look; one of 24 tiles or fewer once
+  it has held still for more than 65 s with no tile within 4 tiles of it ticking
+  or moving with a region of more than 24 tiles
+  (`src/evidence/frame-change.ts`). The first watch lasts the turn's planned
+  length: the named waits' lengths, together up to `actor.maxWaitMs` (else
+  600000), and `idleWaitMs` for each wait that names none. Each further one in a row that ended
+  at its limit, or that a change of 24 tiles or fewer woke, lasts twice as long,
+  up to the larger of its planned length and that cap. `limitMs` is how long it
+  could last; `askedMs` is what the turn's waits asked for, the same sum with no
+  cap, so it is above `limitMs` when the named waits passed the cap and below it
+  when the watch grew; `wake` is `changed`, `speech`,
+  `condition`, `limit`, `clock` (wind-down or end), `stalled` (a look got no
+  frame; the next observation is a fresh one), or `stopped` (the study budget or
+  the session's time ran out); `looks` counts its looks; `area` (`x`, `y`,
+  `width`, `height` in pixels) bounds the change that woke it and is absent for
+  other wakes and for a change still moving at the last look. A `notice` titled
+  `watching the screen`, `lifecycle: started`, opens each watch. The turn's
+  waits share the watch in order, in proportion to their planned lengths, and
+  waits after the one carrying `watch` are titled `wait 0ms`. The look that
+  ended a watch is the turn's `turn-NN` screenshot; other looks are not kept.
+  Additive: absent on plain waits, other producers and older bundles
 - optional `items[].page` and `finalPage` (`{ origin, path }`, `TracePage` in
   `src/actors/contract.ts`): where a computer-use participant's browser was. The loop puts `page`
   on the `screenshot` item of each observation that read a URL: `turn-00-start` is the page before
@@ -1121,13 +1154,20 @@ Core-owned fields:
   `stopCause: provider_output_limit`.
 - optional `waitSettings`: the wait lengths a computer-use participant's session
   applied, in milliseconds, defaults included. `maxWaitMs` is the longest one
-  wait action lasts (`actor.maxWaitMs`); a longer wait is shortened to it.
+  wait action lasts (`actor.maxWaitMs`, else 120000, or 30000 on a desktop with
+  speech); in a turn with another action a longer wait is shortened to it.
   `idleWaitMs` is how long a wait that names no duration lasts in a turn that
   only waits or takes screenshots (`participants[].idleWaitMs`, else
-  `actor.idleWaitMs`), and `settleWaitMs` how
-  long such a wait lasts after an action in the same turn. Each `ui_action` for
-  a wait is titled with the length it lasted. Absent on other routes and on
-  earlier bundles
+  `actor.idleWaitMs`), and the time between looks at the screen while the loop
+  watches such a turn, at most 30000 on a desktop with speech. `settleWaitMs` is
+  how long a wait that names no duration lasts after an action in the same turn,
+  and the time to a watch's quick look after a large change. A watch's length
+  grows to `actor.maxWaitMs`, else 600000, which `waitSettings` does not record;
+  each watch's `watch.limitMs` does. The named waits of a turn that only waits
+  are watched for their lengths, together up to that cap, past `maxWaitMs`. A wait sent to the desktop has a
+  `ui_action` titled with its planned length, which a stalled wait keeps. A
+  watched wait's `ui_action` is titled with the part of the watch it covered.
+  Absent on other routes and on earlier bundles
 - optional `conversation`: how the first-party OpenAI computer-use provider
   carried the conversation between requests. `mode` is `threaded` (the server
   kept it through `previous_response_id`) or `explicit_context` (the server kept

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "humanish",
-  "version": "0.130.0",
+  "version": "0.131.1",
   "description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
   "keywords": [
     "cli",