humanish 0.145.0 → 0.147.0

package/dist/routes/computer-use/e2b-desktop/start.js

@@ -1,43 +1,55 @@
-// The second half of an E2B desktop participant's preparation, and the start of its session: speech and
-// recording, then the browser or terminal the participant sees, then the screen saver turned off,
-// then the live stream. Each step fills the participant's state.
+// The second half of an E2B desktop participant's preparation, and the start of its session: speech,
+// recording, the browser or terminal the participant sees, the wait for its page, the screen saver
+// turned off, then the live stream. Each step fills the participant's state.
 import { redactText, toErrorMessage } from "../../../evidence/redaction.js";
 import { CHROME_DEVTOOLS_PORT, DESKTOP_SETTLE_MS, openDesktopBrowserTarget, openDesktopTerminal, startDesktopStream, } from "../../../substrates/e2b/desktop-browser.js";
+import { waitForChromePageLoad } from "../../../substrates/e2b/desktop-cdp.js";
 import { setPanelClockZone } from "../../../substrates/e2b/desktop-clock.js";
 import { captureDesktopBrowserGeometry } from "../../../substrates/e2b/desktop-geometry.js";
 import { prepareDesktopMedia, startE2BDesktopMedia, } from "../../../substrates/e2b/desktop-media.js";
 import { startE2BDesktopRecording } from "../../../substrates/e2b/desktop-recording.js";
 import { turnOffScreenSaver } from "../../../substrates/e2b/desktop-screen.js";
-import { applyParticipantMobileFidelity, mobileLaunchFlags } from "./fidelity.js";
+import { applyParticipantMobileFidelity, cdpEndpoint, mobileLaunchFlags } from "./fidelity.js";
 import { noteNetworkStart, readParticipantNetworkStart, startParticipantNetwork, } from "./network.js";
-/** Start the declared speech worker and screen recording. A recording that cannot start is a warning. */
-export async function startParticipantMedia(ctx, state, desktop) {
+/** Start the declared speech worker. Chrome is launched in its sound env, so it starts first. */
+export async function startParticipantSpeech(ctx, state, desktop) {
+    const requestedMedia = ctx.deps.residual.execution?.desktop?.media;
+    if (ctx.desktopCliRoute || requestedMedia?.microphone?.source !== "speech")
+        return;
+    state.speech = await startE2BDesktopMedia({
+        desktop,
+        media: requestedMedia,
+        signal: state.mediaStop.signal,
+        onTerminal: () => state.mediaStop.abort(),
+        requestTimeoutMs: ctx.deps.requestTimeoutMs,
+    });
+}
+/**
+ * Whether the study records the desktop's sound. An audible recording runs its own sound server
+ * unless speech already did, and Chrome is launched in that server's env, so the recording starts
+ * before Chrome. A silent recording needs nothing from Chrome and starts beside it.
+ */
+export function recordsSound(ctx) {
+    return ctx.deps.residual.execution?.desktop?.recording?.audio === true;
+}
+/** Start the declared screen recording. A recording that cannot start is a warning. */
+export async function startParticipantRecording(ctx, state, desktop) {
     const { spec, deps, warnings } = ctx;
-    const requestedMedia = deps.residual.execution?.desktop?.media;
-    if (!ctx.desktopCliRoute && requestedMedia?.microphone?.source === "speech") {
-        state.speech = await startE2BDesktopMedia({
+    const requestedRecording = deps.residual.execution?.desktop?.recording;
+    if (!requestedRecording)
+        return;
+    try {
+        state.recording = await startE2BDesktopRecording({
             desktop,
-            media: requestedMedia,
-            signal: state.mediaStop.signal,
-            onTerminal: () => state.mediaStop.abort(),
+            width: spec.planned.device.resolution[0],
+            height: spec.planned.device.resolution[1],
+            audio: requestedRecording.audio,
+            ...(state.speech === undefined ? {} : { pulseEnv: state.speech.env }),
             requestTimeoutMs: deps.requestTimeoutMs,
         });
     }
-    const requestedRecording = deps.residual.execution?.desktop?.recording;
-    if (requestedRecording) {
-        try {
-            state.recording = await startE2BDesktopRecording({
-                desktop,
-                width: spec.planned.device.resolution[0],
-                height: spec.planned.device.resolution[1],
-                audio: requestedRecording.audio,
-                ...(state.speech === undefined ? {} : { pulseEnv: state.speech.env }),
-                requestTimeoutMs: deps.requestTimeoutMs,
-            });
-        }
-        catch (error) {
-            warnings.push(`Desktop recording startup failed; the study continues without video: ${redactText(deps.scrubKnownValues(toErrorMessage(error)))}`);
-        }
+    catch (error) {
+        warnings.push(`Desktop recording startup failed; the study continues without video: ${redactText(deps.scrubKnownValues(toErrorMessage(error)))}`);
     }
 }
 /**
@@ -72,7 +84,8 @@ export async function openParticipantSurface(ctx, state, desktop) {
     // The participant's time zone is TZ on the launch: the browser process and every renderer it
     // starts read it at startup, so every tab runs in the zone from the first page load. A CDP
     // timezone override holds only on the one tab whose DevTools session sets it, from after its load.
-    const mediaEnv = state.speech?.env ?? state.recording?.env;
+    // A silent recording may still be starting, and Chrome needs nothing from it.
+    const mediaEnv = state.speech?.env ?? (recordsSound(ctx) ? state.recording?.env : undefined);
     const browserLaunch = await openDesktopBrowserTarget(desktop, targetUrl, deps.requestTimeoutMs, residual.execution?.desktop?.browser, [...emulationFlags, ...(mediaEvidence?.flags ?? [])], timeZone === undefined ? mediaEnv : { ...mediaEnv, TZ: timeZone });
     state.desktopBrowser =
         mediaEvidence === undefined
@@ -91,10 +104,10 @@ export async function openParticipantSurface(ctx, state, desktop) {
     const emulated = emulationFlags.length > 0;
     noteDevToolsReadiness(ctx, browserLaunch.devTools, emulated);
     // A mobile-emulated participant's holder captures too; any other gets a capture-only holder,
-    // started here so the settle wait covers its start.
+    // started here so it records the launch page's load while the page wait runs.
     if (!emulated)
         await startParticipantNetwork(ctx, state, desktop);
-    await desktop.wait(DESKTOP_SETTLE_MS).catch(() => undefined);
+    await waitForParticipantPage(ctx, state, desktop, browserLaunch.devTools);
     if (!emulated)
         await readParticipantNetworkStart(ctx, state, desktop);
     const network = state.network;
@@ -119,13 +132,38 @@ export async function openParticipantSurface(ctx, state, desktop) {
             }),
     });
 }
+/**
+ * Hold the participant's first screenshot until its page has loaded and painted, so the participant
+ * does not start on a blank or half-drawn desktop. A Chromium page whose DevTools answered is read over DevTools for at most DESKTOP_SETTLE_MS, so no
+ * participant waits longer than the fixed settle this replaced, and the result is a timed phase in
+ * the bundle. Any other browser, and a page wait that cannot run, waits DESKTOP_SETTLE_MS.
+ */
+async function waitForParticipantPage(ctx, state, desktop, devTools) {
+    const { deps } = ctx;
+    const load = state.launchedBrowserFamily === "chromium" && devTools?.state === "ready"
+        ? await waitForChromePageLoad(desktop, deps.requestTimeoutMs, cdpEndpoint(state.browserLaunchIdentity, ctx.targetUrl), DESKTOP_SETTLE_MS)
+        : undefined;
+    if (load === undefined) {
+        await desktop.wait(DESKTOP_SETTLE_MS).catch(() => undefined);
+        return;
+    }
+    ctx.onSubjectPhase({
+        at: new Date(deps.now()).toISOString(),
+        type: "cua-lab.browser.page.completed",
+        ok: load.loaded,
+        durationMs: load.waitedMs,
+        message: load.loaded
+            ? "The page loaded and painted"
+            : `The page had not loaded and painted when the wait ended (${load.seen}); the participant started anyway`,
+    });
+}
 /**
  * Turn the desktop's screen saver off, so the screen stays on while the participant waits without
  * input. The XFCE session sets the screen saver's timeout in its first seconds, and the last write
- * wins. Nothing waits for those writes: this runs after the browser or terminal has opened and the
- * desktop has waited DESKTOP_SETTLE_MS, and on 16 fresh desktops the last write came within 2.5 s
- * of a command sent as soon as the desktop was created. A screen saver that cannot be turned off
- * is a warning.
+ * wins. Nothing waits for those writes: this runs after a Chromium page has loaded, which needs
+ * DevTools to answer first (4.8 s or more after launch on live desktops), or after the fixed 8 s
+ * wait, and on 16 fresh desktops the last write came within 2.5 s of a command sent as soon as the
+ * desktop was created. A screen saver that cannot be turned off is a warning.
  */
 export async function keepParticipantScreenOn(ctx, desktop) {
     const { spec, deps, warnings } = ctx;

package/dist/routes/computer-use/participant-execution.d.ts

@@ -1,12 +1,5 @@
-import { type PreparedOutputRoot } from "../../run/contained-output.js";
 import type { CuaParticipantDeps, CuaParticipantPlan, CuaParticipantResult, DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
 import type { RunSubjectProvenance } from "../../run/bundle.js";
-/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
- *  the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
- *  at N>1). */
-export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOutputRoot, spec: {
-    screenshotDir: string;
-}, screenshots: string[]): (name: string, bytes: Buffer) => Promise<string>;
 /** Run one participant against a prepared desktop. The adapter owns provisioning, final
  * evidence and cleanup; this runner owns the model, trace and participant outcome. */
 export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;

package/dist/routes/computer-use/participant-execution.js

@@ -2,36 +2,12 @@ import { scheduledParticipantDeps } from "./participant-runs.js";
 import { judgeParticipantRecords } from "../../run/judge.js";
 import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
 import { createInProcessDesktop } from "./in-process-desktop.js";
-import path from "node:path";
 import { cuaParticipantDiagnostics } from "./diagnostics.js";
 import { pastEndReason, runOnSchedule } from "../../study/arrivals.js";
-import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
 import { redactText, toErrorMessage } from "../../evidence/redaction.js";
-import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
+import { participantFrames } from "./participant-frames.js";
 import { participantOutcomeOk, participantFactsOf } from "./participant-facts.js";
 import { closeParticipantModel, judgeParticipantSession, participantSessionOptions, recordParticipantTrace, startParticipantModel, } from "./participant-model.js";
-/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
- *  the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
- *  at N>1). */
-export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots) {
-    if (spec.screenshotDir) {
-        assertSafeOutputPathSegment(spec.screenshotDir, "Screenshot participant id");
-    }
-    const dirParts = spec.screenshotDir ? ["screenshots", spec.screenshotDir] : ["screenshots"];
-    const relPrefix = spec.screenshotDir
-        ? path.posix.join("screenshots", spec.screenshotDir)
-        : "screenshots";
-    return async (name, bytes) => {
-        assertSafeOutputPathSegment(name, "Screenshot name");
-        const rel = path.posix.join(relPrefix, name);
-        // Evidence holds image data only, whatever the executor's frames carried.
-        const frame = stripPngMetadataChunks(bytes);
-        await assertScreenshotEvidence(rel, frame);
-        await writeContainedOutputFile(artifactRoot, path.join(...dirParts, name), frame);
-        screenshots.push(rel);
-        return rel;
-    };
-}
 /** A blocked participant outcome: the pipeline gate or the study budget skipped it. */
 function skippedOutcome(spec, reason, arrival) {
     return {
@@ -65,8 +41,8 @@ export async function runCuaParticipant(spec, deps) {
         return skippedOutcome(spec, pastEnd);
     let model = {};
     const warnings = [];
-    const screenshots = [];
-    const writeScreenshot = makeParticipantWriteScreenshot(deps.artifactRoot, spec, screenshots);
+    const frames = participantFrames(deps.artifactRoot, spec, { blurred: deps.redactScreenshots });
+    const writeScreenshot = frames.write;
     let session;
     let sessionError;
     let providerCleanupError;
@@ -112,6 +88,7 @@ export async function runCuaParticipant(spec, deps) {
         else
             sessionError = desktopFailure;
     }
+    const replacedFrames = await frames.finish(session?.trace);
     if (session)
         await recordParticipantTrace(spec, deps, session, warnings);
     const { noEngagement, selfReportedBlocker, reportedFriction } = judgeParticipantSession(session, deps, warnings);
@@ -129,7 +106,8 @@ export async function runCuaParticipant(spec, deps) {
         ...(providerPolicyError === undefined ? {} : { providerPolicyError }),
         ...desktopEvidence,
         killed: released,
-        screenshots,
+        screenshots: frames.screenshots,
+        replacedFrames,
         warnings,
         noEngagement,
         selfReportedBlocker,

package/dist/routes/computer-use/participant-frames.d.ts

@@ -0,0 +1,32 @@
+import type { ActorTrace } from "../../actors/contract.js";
+import { type WebpTranscoder } from "../../evidence/webp-transcoder.js";
+import { type PreparedOutputRoot } from "../../run/contained-output.js";
+/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
+ *  the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
+ *  at N>1). */
+export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOutputRoot, spec: {
+    screenshotDir: string;
+}, screenshots: string[]): (name: string, bytes: Buffer) => Promise<string>;
+export interface ParticipantFrames {
+    /** The session's writeScreenshot: writes the PNG, then queues its WebP. */
+    write(name: string, bytes: Buffer): Promise<string>;
+    /** Every frame written, by the path its trace references names. */
+    readonly screenshots: string[];
+    /**
+     * Waits up to FINISH_WAIT_MS for the frames still queued, then points `trace` and `screenshots`
+     * at each frame's WebP. Returns the PNGs those WebP replace, which the run removes once its final
+     * bundle is written: until then the live trace in run.json still names them.
+     */
+    finish(trace: ActorTrace | undefined): Promise<string[]>;
+}
+/**
+ * `blurred` is the study's policies.redactScreenshots: its frames are the redactor's PNG
+ * thumbnails, which stay PNG, since only that shape counts as blurred evidence.
+ */
+export declare function participantFrames(artifactRoot: PreparedOutputRoot, spec: {
+    screenshotDir: string;
+}, options: {
+    blurred: boolean;
+    transcoder?: WebpTranscoder;
+    finishWaitMs?: number;
+}): ParticipantFrames;

package/dist/routes/computer-use/participant-frames.js

@@ -0,0 +1,93 @@
+// A computer-use participant's screenshots: each frame is written and referenced as PNG during the
+// session, as the live Observer and a mid-run copy read it, and queued for a lossless WebP copy.
+// When the session ends, each frame whose WebP is written is referenced by its WebP instead and
+// its PNG removed, so at no point does a reference name a file that is not there.
+import path from "node:path";
+import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
+import { webpTranscoder } from "../../evidence/webp-transcoder.js";
+import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
+/**
+ * How long a finished session waits for its frames still queued. Encoding takes about 330 ms a
+ * frame, so this covers a short backlog; a frame not done by then stays PNG.
+ */
+const FINISH_WAIT_MS = 5_000;
+/** Build a participant's writeScreenshot closure: writes under screenshots/<screenshotDir>/ and records
+ *  the relative path the trace references (screenshots/<name> at N=1; screenshots/<laneId>/<name>
+ *  at N>1). */
+export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots) {
+    if (spec.screenshotDir) {
+        assertSafeOutputPathSegment(spec.screenshotDir, "Screenshot participant id");
+    }
+    const dirParts = spec.screenshotDir ? ["screenshots", spec.screenshotDir] : ["screenshots"];
+    const relPrefix = spec.screenshotDir
+        ? path.posix.join("screenshots", spec.screenshotDir)
+        : "screenshots";
+    return async (name, bytes) => {
+        assertSafeOutputPathSegment(name, "Screenshot name");
+        const rel = path.posix.join(relPrefix, name);
+        // Evidence holds image data only, whatever the executor's frames carried.
+        const frame = stripPngMetadataChunks(bytes);
+        await assertScreenshotEvidence(rel, frame);
+        await writeContainedOutputFile(artifactRoot, path.join(...dirParts, name), frame);
+        screenshots.push(rel);
+        return rel;
+    };
+}
+/**
+ * `blurred` is the study's policies.redactScreenshots: its frames are the redactor's PNG
+ * thumbnails, which stay PNG, since only that shape counts as blurred evidence.
+ */
+export function participantFrames(artifactRoot, spec, options) {
+    const transcoder = options.blurred ? undefined : (options.transcoder ?? webpTranscoder());
+    const finishWaitMs = options.finishWaitMs ?? FINISH_WAIT_MS;
+    const screenshots = [];
+    const writePng = makeParticipantWriteScreenshot(artifactRoot, spec, screenshots);
+    const queued = [];
+    /** Each frame whose WebP is written: its PNG path and its WebP path. */
+    const done = new Map();
+    // Once the session's references are settled, a late WebP is not written: nothing would name it.
+    let finished = false;
+    return {
+        screenshots,
+        async write(name, bytes) {
+            const rel = await writePng(name, bytes);
+            const webpRel = rel.replace(/\.png$/, ".webp");
+            const job = webpRel === rel ? undefined : transcoder?.transcode(bytes);
+            // A frame the worker cannot take, or fails to encode or write, stays PNG.
+            if (job !== undefined)
+                queued.push({
+                    done: job.webp
+                        .then(async (webp) => {
+                        if (finished)
+                            return;
+                        await writeContainedOutputFile(artifactRoot, webpRel, webp);
+                        done.set(rel, webpRel);
+                    })
+                        .catch(() => undefined),
+                    abandon: job.abandon,
+                });
+            return rel;
+        },
+        async finish(trace) {
+            let timer;
+            await Promise.race([
+                Promise.all(queued.map((job) => job.done)),
+                new Promise((resolve) => (timer = setTimeout(resolve, finishWaitMs))),
+            ]);
+            clearTimeout(timer);
+            finished = true;
+            const replaced = new Map(done);
+            if (trace !== undefined)
+                trace.items = trace.items.map((item) => {
+                    const webp = item.screenshotRef && replaced.get(item.screenshotRef.path);
+                    return webp === undefined || item.screenshotRef === undefined
+                        ? item
+                        : { ...item, screenshotRef: { ...item.screenshotRef, path: webp } };
+                });
+            screenshots.splice(0, screenshots.length, ...screenshots.map((rel) => replaced.get(rel) ?? rel));
+            for (const job of queued)
+                job.abandon();
+            return [...replaced.keys()];
+        },
+    };
+}

package/dist/routes/computer-use/participant-model.js

@@ -118,6 +118,9 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
             ...(spec.planned.limits.maxOutputTokens === undefined
                 ? {}
                 : { maxOutputTokens: spec.planned.limits.maxOutputTokens }),
+            ...(spec.planned.limits.serviceTier === undefined
+                ? {}
+                : { serviceTier: spec.planned.limits.serviceTier }),
         },
         ...(maxUsd === undefined
             ? {}

package/dist/routes/computer-use/participant-runs.js

@@ -152,6 +152,9 @@ function planEntry(spec) {
         ...(spec.planned.limits.maxOutputTokens === undefined
             ? {}
             : { maxOutputTokens: spec.planned.limits.maxOutputTokens }),
+        ...(spec.planned.limits.serviceTier === undefined
+            ? {}
+            : { serviceTier: spec.planned.limits.serviceTier }),
         ...(spec.planned.targetUrl === undefined
             ? {}
             : { targetDigest: digestUrl(spec.planned.targetUrl) }),
@@ -179,6 +182,7 @@ export function formatParticipantPlanEntry(entry) {
         entry.surface ? `surface=${entry.surface}` : undefined,
         entry.caseGroup ? `case=${entry.caseGroup}` : undefined,
         entry.reasoningEffort ? `effort=${entry.reasoningEffort}` : undefined,
+        entry.serviceTier ? `tier=${entry.serviceTier}` : undefined,
         entry.startAfterMs === undefined ? undefined : `start=+${formatDuration(entry.startAfterMs)}`,
         entry.endAfterMs === undefined ? undefined : `end=+${formatDuration(entry.endAfterMs)}`,
         entry.maxSessionMs === undefined ? undefined : `cap=${formatDuration(entry.maxSessionMs)}`,

package/dist/routes/computer-use/plan.js

@@ -13,7 +13,7 @@ import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provis
 import { computerUseParticipants, declaredTargets, } from "../../study/plan-participants.js";
 import { planStarts, windowDeadlineRefusal } from "../../study/arrivals.js";
 import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
-import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
+import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
 import { desktopCliProductReason } from "../../study/composition-rules.js";
 import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
 import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
@@ -89,12 +89,16 @@ function unsupportedDeclarationReason(config, hasRunSession, driving, { inProces
     const reason = desktopMediaValidationReason(config) ||
         networkCaptureValidationReason(config, "computer-use") ||
         outputTokenLimitValidationReason(config) ||
+        serviceTierValidationReason(config) ||
         waitLimitValidationReason(config);
     if (reason)
         return invalid(reason);
     if (config.actor?.maxOutputTokens !== undefined &&
         (hasRunSession || driving.createProvider || driving.inProcess))
         return invalid("maxOutputTokens cannot be enforced by a custom runSession/provider/executor route.");
+    if (config.actor?.serviceTier !== undefined &&
+        (hasRunSession || driving.createProvider || driving.inProcess))
+        return invalid("serviceTier is not sent by a custom runSession/provider/executor route.");
     const receivingReason = receivingEmailValidationReason(config);
     if (receivingReason)
         return invalid(receivingReason);

package/dist/routes/computer-use/result.js

@@ -1,3 +1,4 @@
+import { removeContainedOutputFile } from "../../run/contained-output.js";
 import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
 import { redactText } from "../../evidence/redaction.js";
 import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
@@ -260,6 +261,11 @@ export async function finishCuaRun(setup, finish, ran) {
         execution,
         policy,
     });
+    // The final bundle names each replaced frame's WebP, so its PNG goes before the Observer
+    // verifies the run directory.
+    for (const outcome of outcomes ?? [])
+        for (const png of outcome.replacedFrames ?? [])
+            await removeContainedOutputFile(finished.paths, png);
     const observer = await finished.renderObserver();
     // A caller's renderer runs with this process's file access, so after one that reports success
     // the run directory is checked again before the result points anyone at it. A failed render is

package/dist/routes/computer-use/types.d.ts

@@ -112,6 +112,8 @@ export interface CuaParticipantPlanEntry {
      *  you read before spending money, so a declared per-participant difference has to be visible there. */
     reasoningEffort?: string;
     maxOutputTokens?: number;
+    /** The declared OpenAI service tier, which bills at its own rate. */
+    serviceTier?: string;
     /** Present only when a participant overrides subject.appUrl; digest avoids leaking preview hosts in plan logs. */
     targetDigest?: string;
     /** The participant's declared start, in ms after the run starts its participants. */
@@ -432,6 +434,8 @@ export interface SandboxReleaseFact {
 /** One participant's end-to-end run outcome (internal; projected into CuaParticipantResult + the bundle). */
 export interface ParticipantRunOutcome {
     spec: DesktopParticipantRun;
+    /** The PNG frames its WebP replaced, which the run removes once its final bundle is written. */
+    replacedFrames?: string[];
     session?: CuaLoopResult;
     /** The harness failed before the session reached a terminal status. */
     sessionError?: string;

package/dist/routes/shared-world/plan.js

@@ -12,7 +12,7 @@ import { DESKTOP_FILES_ROUTE_REASON } from "../../study/desktop-files.js";
 import { participantList } from "../../study/study-fields.js";
 import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
 import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
-import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
+import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
 import { unpricedCapCheck } from "../../study/requirements.js";
 import { participantsAtOnce, sandboxDeadlineRefusal, sandboxHeadroomMs, } from "../../substrates/e2b/lifetime.js";
 import { boundedConcurrency } from "../computer-use/participant-runs.js";
@@ -62,6 +62,7 @@ export function planSharedWorldStudy(config, input) {
     // A library caller's config skips the parser, which refuses these first.
     studyUrlCredentialReason(config) ??
         outputTokenLimitValidationReason(config) ??
+        serviceTierValidationReason(config) ??
         waitLimitValidationReason(config) ??
         (externalPublic
             ? externalPublicSharedWorldValidationReason(config)
@@ -70,6 +71,8 @@ export function planSharedWorldStudy(config, input) {
         return refuse(invalid, invalidReason, actor);
     if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
         return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
+    if (config.actor?.serviceTier !== undefined && input.hasRunSession === true)
+        return refuse(invalid, "serviceTier is not sent by a custom runSession.", actor);
     // A live cap is priced here, at plan time, so its refusal wins over the route's key checks.
     const unpriced = input.dryRun
         ? undefined

package/dist/run/pricing.d.ts

@@ -30,6 +30,11 @@ export interface ModelRate {
     until?: string;
     /** The rates the sheet publishes from the day after `until`. */
     after?: Pick<ModelRate, "inputUsdPerToken" | "outputUsdPerToken" | "cachedInputUsdPerToken">;
+    /** Fast mode rates (OpenAI `service_tier: "fast"`, served as `fast` or `priority`), when the
+     *  sheet publishes them. The long-context multipliers apply on top. */
+    fast?: Pick<ModelRate, "inputUsdPerToken" | "outputUsdPerToken" | "cachedInputUsdPerToken" | "cacheWriteUsdPerToken"> & {
+        source?: string;
+    };
     /** The `YYYY-MM-DD` date the entry was last checked against `source`. */
     asOf: string;
     /** Public pricing page the number came from (a comment/URL, never a secret). */
@@ -101,6 +106,8 @@ export interface ActorEstimatedCost {
         /** How many requests crossed the long-context threshold and were re-tiered. Present only
          *  when per-request `turns` records made exact tiering possible. */
         longContextTurns?: number;
+        /** How many requests the provider served in Fast mode and were priced at its rates. */
+        fastTurns?: number;
     };
 }
 /** The desktop-minute cost estimate (host-side create->teardown span * a per-minute rate). Same