humanish 0.121.0 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/routes/computer-use/bundle.d.ts +0 -1
- package/dist/routes/computer-use/bundle.js +0 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +2 -12
- package/dist/routes/computer-use/live-phase.d.ts +0 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-execution.d.ts +9 -15
- package/dist/routes/computer-use/participant-execution.js +22 -31
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +1 -2
- package/dist/routes/computer-use/types.d.ts +17 -5
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/host-suspension.d.ts +2 -2
- package/dist/run/host-suspension.js +2 -2
- package/dist/run/judge.d.ts +1 -1
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/preflight.js +11 -1
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/docs/contracts/run-bundle.md +2 -2
- package/docs/contracts/schemas.md +24 -4
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,21 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.122.0: Participants end on the clock, a harness error ends only its participant (2026-10-12)
|
|
12
|
+
|
|
13
|
+
humanish 0.122.0 lets a participant end at a scheduled time and keeps one participant's harness
|
|
14
|
+
error from stopping the rest of a study. `participants[].endAfterMs`, on the same clock as
|
|
15
|
+
`startAfterMs`, ends a participant's session at that time with stop cause `scheduled_end`,
|
|
16
|
+
counted as reaching the goal; from `endAfterMs - windDownMs` (10 minutes by default) each turn
|
|
17
|
+
tells the participant the end time and the minutes left, its turns that only wait no longer count
|
|
18
|
+
toward the idle limits before then, and its sandbox is sized from its own window. `actor.clock {
|
|
19
|
+
timeZone }` adds the local time to each turn, and `participants[].idleWaitMs` sets one
|
|
20
|
+
participant's wait length, so staff on a long shift can check the screen less often than patients.
|
|
21
|
+
A participant's harness error now ends only that participant: the study-wide fail-fast is removed,
|
|
22
|
+
and every later participant starts on its schedule.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.122.0)
|
|
25
|
+
|
|
11
26
|
## 0.121.0: One failed desktop fails alone, host sleep named as the cause (2026-10-10)
|
|
12
27
|
|
|
13
28
|
humanish 0.121.0 keeps one participant's failed desktop from costing a whole hosted study, and
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ActorPersonaRef, ActorTokenUsage, ActorTraceItem } from "../contract.js";
|
|
2
2
|
import { type CuaExecutor, type CuaLiveMetadata, type CuaLoopResult, type CuaProvider, type CuaSafetyCheck } from "./loop.js";
|
|
3
|
+
import type { ParticipantClock } from "./loop/types.js";
|
|
3
4
|
import { type E2BDesktopExecutorOptions, type E2BDesktopLike } from "../../substrates/e2b/desktop-executor.js";
|
|
4
5
|
import { type OpenAiResponsesProviderOptions } from "./openai-provider.js";
|
|
5
6
|
import { type RedactionHooks } from "../../evidence/redaction.js";
|
|
@@ -42,6 +43,8 @@ export interface CuaActorSessionOptions {
|
|
|
42
43
|
maxWaitMs?: number;
|
|
43
44
|
/** How long a wait with no duration lasts in an idle turn. See LoopRunOptions.idleWaitMs. */
|
|
44
45
|
idleWaitMs?: number;
|
|
46
|
+
/** The participant's time zone and scheduled end, forwarded to the loop. See CuaLoopOptions.clock. */
|
|
47
|
+
clock?: ParticipantClock;
|
|
45
48
|
/**
|
|
46
49
|
* Redact persisted screenshots (blur+downscale). Default `false`: full fidelity for local use.
|
|
47
50
|
* Set true for unowned subjects or share-as-is bundles. The provider always sees raw frames.
|
|
@@ -43,6 +43,7 @@ export async function runCuaActorSession(options) {
|
|
|
43
43
|
...(options.writeScreenshot === undefined ? {} : { writeScreenshot: options.writeScreenshot }),
|
|
44
44
|
...(options.stopWhen === undefined ? {} : { stopWhen: options.stopWhen }),
|
|
45
45
|
...(options.dwell === undefined ? {} : { dwell: options.dwell }),
|
|
46
|
+
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
46
47
|
...(options.tasks === undefined ? {} : { tasks: options.tasks }),
|
|
47
48
|
...(options.maxUsd === undefined ? {} : { maxUsd: options.maxUsd }),
|
|
48
49
|
...(options.estimateTurnCostUsd === undefined
|
|
@@ -34,9 +34,14 @@ export interface BackstopStep {
|
|
|
34
34
|
/** The gave_up reason when a streak tripped. */
|
|
35
35
|
readonly tripReason: string | undefined;
|
|
36
36
|
}
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
|
|
39
|
+
* participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
|
|
40
|
+
* toward neither streak and earns no nudge. A turn that acts is folded as usual.
|
|
41
|
+
*/
|
|
38
42
|
export declare function advanceBackstop(previous: Backstop, turn: {
|
|
39
43
|
actions: readonly CuaAction[];
|
|
40
44
|
observation: CuaObservation;
|
|
41
45
|
heardNewSpeech: boolean;
|
|
46
|
+
waitingIsTheTask?: boolean;
|
|
42
47
|
}, limits: BackstopLimits): BackstopStep;
|
|
@@ -122,9 +122,21 @@ export function startBackstop(observation) {
|
|
|
122
122
|
idleProgressForgivenessUsed: 0,
|
|
123
123
|
};
|
|
124
124
|
}
|
|
125
|
-
/**
|
|
125
|
+
/**
|
|
126
|
+
* Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
|
|
127
|
+
* participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
|
|
128
|
+
* toward neither streak and earns no nudge. A turn that acts is folded as usual.
|
|
129
|
+
*/
|
|
126
130
|
export function advanceBackstop(previous, turn, limits) {
|
|
127
131
|
const idle = isIdleTurn(turn.actions);
|
|
132
|
+
if (idle && turn.waitingIsTheTask === true)
|
|
133
|
+
return {
|
|
134
|
+
backstop: { ...previous, progressKey: progressKeyOf(turn.observation), consecutiveIdle: 0 },
|
|
135
|
+
idle,
|
|
136
|
+
progressed: true,
|
|
137
|
+
hints: [],
|
|
138
|
+
tripReason: undefined,
|
|
139
|
+
};
|
|
128
140
|
const progressKey = progressKeyOf(turn.observation);
|
|
129
141
|
const frameChanged = progressKey !== previous.progressKey;
|
|
130
142
|
// Corroboration. A stale frame alone is not evidence of a stuck agent. The frame hash is
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Stop } from "./ending.js";
|
|
2
|
+
import type { LoopSession } from "./session.js";
|
|
3
|
+
import type { CuaObservation, ParticipantClock } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* The time text for the request the loop is about to send: the local time when the participant
|
|
6
|
+
* has a time zone, and the end time and the minutes left from the start of its wind-down, or
|
|
7
|
+
* after it tried to end its session before its end (`askedToEnd`). Undefined when there is
|
|
8
|
+
* nothing to say.
|
|
9
|
+
*/
|
|
10
|
+
export declare function clockHint(clock: ParticipantClock | undefined, now: () => number, askedToEnd?: boolean): string | undefined;
|
|
11
|
+
/**
|
|
12
|
+
* Whether the participant has a scheduled end still to come. Its own end before then does not end
|
|
13
|
+
* the session: the end is what the study declared for it.
|
|
14
|
+
*/
|
|
15
|
+
export declare function beforeScheduledEnd(session: LoopSession): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* The stop at the participant's scheduled end, or undefined before it. The session passes as a
|
|
18
|
+
* planned end and the participant is asked for its closing report on the last observation.
|
|
19
|
+
*/
|
|
20
|
+
export declare function scheduledEndStop(session: LoopSession, observation: CuaObservation): Stop | undefined;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { plural } from "../../../run/text.js";
|
|
2
|
+
// What the participant knows about the time. A hosted desktop shows a maximized browser with no
|
|
3
|
+
// OS clock, so a participant told "your shift ends at 11:00" cannot relate it to now unless each
|
|
4
|
+
// turn says so. The text rides the request's context hint, which every provider sends with the
|
|
5
|
+
// turn (OpenAI as a user message, Codex and the local agents as the harness note).
|
|
6
|
+
/**
|
|
7
|
+
* The time text for the request the loop is about to send: the local time when the participant
|
|
8
|
+
* has a time zone, and the end time and the minutes left from the start of its wind-down, or
|
|
9
|
+
* after it tried to end its session before its end (`askedToEnd`). Undefined when there is
|
|
10
|
+
* nothing to say.
|
|
11
|
+
*/
|
|
12
|
+
export function clockHint(clock, now, askedToEnd = false) {
|
|
13
|
+
if (clock === undefined)
|
|
14
|
+
return undefined;
|
|
15
|
+
const nowMs = now();
|
|
16
|
+
const parts = [];
|
|
17
|
+
if (clock.timeZone !== undefined)
|
|
18
|
+
parts.push(`The local time is ${hourMinute(nowMs, clock.timeZone)} (${clock.timeZone}).`);
|
|
19
|
+
const { end } = clock;
|
|
20
|
+
if (end !== undefined && (askedToEnd || nowMs >= end.atMs - end.windDownMs)) {
|
|
21
|
+
const minutesLeft = Math.max(1, Math.ceil((end.atMs - nowMs) / 60_000));
|
|
22
|
+
parts.push(`${askedToEnd ? "Your session is not over. " : ""}Your session ends at ${endTime(clock, end.atMs)}, in ${plural(minutesLeft, "minute")}. Keep working until then. The session ends by itself at that time, and you will then be asked for your closing account.`);
|
|
23
|
+
}
|
|
24
|
+
return parts.length === 0 ? undefined : parts.join(" ");
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Whether the participant has a scheduled end still to come. Its own end before then does not end
|
|
28
|
+
* the session: the end is what the study declared for it.
|
|
29
|
+
*/
|
|
30
|
+
export function beforeScheduledEnd(session) {
|
|
31
|
+
const end = session.settings.clock?.end;
|
|
32
|
+
return end !== undefined && session.now() < end.atMs;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The stop at the participant's scheduled end, or undefined before it. The session passes as a
|
|
36
|
+
* planned end and the participant is asked for its closing report on the last observation.
|
|
37
|
+
*/
|
|
38
|
+
export function scheduledEndStop(session, observation) {
|
|
39
|
+
const end = session.settings.clock?.end;
|
|
40
|
+
if (end === undefined || session.now() < end.atMs)
|
|
41
|
+
return undefined;
|
|
42
|
+
return {
|
|
43
|
+
completionReason: "goal_satisfied",
|
|
44
|
+
reason: `the participant's scheduled end at ${endTime(session.settings.clock, end.atMs)} was reached`,
|
|
45
|
+
stopCause: "scheduled_end",
|
|
46
|
+
debriefTrigger: { kind: "scheduled_end", observation },
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// The end to the nearest minute: a run starts at no particular second, and "ends at 00:09" read
|
|
50
|
+
// at 00:09:06 looks past.
|
|
51
|
+
function endTime(clock, atMs) {
|
|
52
|
+
const minute = Math.round(atMs / 60_000) * 60_000;
|
|
53
|
+
return clock.timeZone === undefined
|
|
54
|
+
? `${hourMinute(minute, "UTC")} UTC`
|
|
55
|
+
: hourMinute(minute, clock.timeZone);
|
|
56
|
+
}
|
|
57
|
+
function hourMinute(ms, timeZone) {
|
|
58
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
59
|
+
timeZone,
|
|
60
|
+
hour: "2-digit",
|
|
61
|
+
minute: "2-digit",
|
|
62
|
+
hourCycle: "h23",
|
|
63
|
+
}).format(ms);
|
|
64
|
+
}
|
|
@@ -8,7 +8,7 @@ import type { CuaObservation, CuaProvider, CuaTurn } from "./types.js";
|
|
|
8
8
|
* the participant's own ending when only its impressions are missing.
|
|
9
9
|
*/
|
|
10
10
|
export interface DebriefTrigger {
|
|
11
|
-
readonly kind: "stop_when" | "dwell" | "participant_end";
|
|
11
|
+
readonly kind: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
12
12
|
/** The final observation, already captured; the debrief request sends it without observing. */
|
|
13
13
|
readonly observation: CuaObservation;
|
|
14
14
|
}
|
|
@@ -12,7 +12,7 @@ export type ScreenshotRef = NonNullable<ActorTraceItem["screenshotRef"]>;
|
|
|
12
12
|
* wrapped callbacks are fields on LoopSession instead. Callers destructure hooks to call them.
|
|
13
13
|
*/
|
|
14
14
|
export type LoopSettings = {
|
|
15
|
-
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]: LoopRunOptions[K];
|
|
15
|
+
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "clock" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]: LoopRunOptions[K];
|
|
16
16
|
};
|
|
17
17
|
/** The participant's dispatched actions, as the backstop and failure notices describe them. */
|
|
18
18
|
export interface ActionHistory {
|
|
@@ -70,13 +70,14 @@ export class LoopSession {
|
|
|
70
70
|
scrubText;
|
|
71
71
|
constructor(options) {
|
|
72
72
|
// Read once, in this order: a caller mutating its options object mid-run changes nothing.
|
|
73
|
-
const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, maxWaitMs, idleWaitMs, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace, } = options;
|
|
73
|
+
const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, maxWaitMs, idleWaitMs, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, clock, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace, } = options;
|
|
74
74
|
this.settings = {
|
|
75
75
|
instructions,
|
|
76
76
|
persona,
|
|
77
77
|
redaction,
|
|
78
78
|
stopWhen,
|
|
79
79
|
dwell,
|
|
80
|
+
clock,
|
|
80
81
|
tasks,
|
|
81
82
|
maxUsd,
|
|
82
83
|
overRunBudget,
|
|
@@ -4,6 +4,14 @@ import type { RedactionHooks } from "../../../evidence/redaction.js";
|
|
|
4
4
|
import type { DwellWindow, StopWhen } from "../../stop-conditions.js";
|
|
5
5
|
import type { StudyTask } from "../../../study/tasks.js";
|
|
6
6
|
import type { ReasoningEffort } from "../../reasoning-effort.js";
|
|
7
|
+
/** The participant's clock: its local time zone and its scheduled end, on the loop's clock. */
|
|
8
|
+
export interface ParticipantClock {
|
|
9
|
+
readonly timeZone?: string;
|
|
10
|
+
readonly end?: {
|
|
11
|
+
readonly atMs: number;
|
|
12
|
+
readonly windDownMs: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
7
15
|
/**
|
|
8
16
|
* One desktop action. `heldKeys` on a pointer action are held down for the whole action and
|
|
9
17
|
* released after it, named as the provider sent them (for example `SHIFT` for a shift-click). An
|
|
@@ -346,6 +354,14 @@ export interface CuaLoopOptions {
|
|
|
346
354
|
dwell?: DwellWindow;
|
|
347
355
|
/** Injected pause for the dwell window's cadence; tests advance their clock through it. */
|
|
348
356
|
sleep?: (ms: number) => Promise<void>;
|
|
357
|
+
/**
|
|
358
|
+
* What each turn tells the participant about the time (loop/clock.ts). With `timeZone`, a time zone
|
|
359
|
+
* name, every request carries the local time. With `end`, from `end.atMs - end.windDownMs`
|
|
360
|
+
* every request names the end time (in `timeZone`, else UTC) and the minutes left, and at
|
|
361
|
+
* `end.atMs`, read on `now`, the loop asks for the closing report and ends the session with
|
|
362
|
+
* stop cause `scheduled_end`.
|
|
363
|
+
*/
|
|
364
|
+
clock?: ParticipantClock;
|
|
349
365
|
/**
|
|
350
366
|
* The study's declared protocol: discrete tasks whose completion is corroborated by the
|
|
351
367
|
* same observations stopWhen reads, on the same cadence. The tracker never influences the loop's
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CuaLoopOptions, CuaLoopResult, LoopRunOptions } from "./loop/types.js";
|
|
2
|
-
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, } from "./loop/types.js";
|
|
2
|
+
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, ParticipantClock, } from "./loop/types.js";
|
|
3
3
|
export { describeCuaAction } from "./loop/actions.js";
|
|
4
4
|
export { stableProgressKey } from "./loop/backstop.js";
|
|
5
5
|
export { IMPRESSIONS_ASK } from "./loop/impressions.js";
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { closingReportSchema } from "../closing-report.js";
|
|
2
2
|
import { ComputerUseProviderError } from "./provider-error.js";
|
|
3
3
|
import { runActionBatch } from "./loop/actions.js";
|
|
4
|
-
import { advanceBackstop, startBackstop } from "./loop/backstop.js";
|
|
4
|
+
import { advanceBackstop, startBackstop, } from "./loop/backstop.js";
|
|
5
|
+
import { beforeScheduledEnd, clockHint, scheduledEndStop } from "./loop/clock.js";
|
|
5
6
|
import { requestDebrief } from "./loop/debrief.js";
|
|
6
7
|
import * as stops from "./loop/ending.js";
|
|
7
8
|
import { declaredOutcomeOf } from "./loop/ending.js";
|
|
@@ -51,6 +52,7 @@ export async function runComputerUseLoopWithTaps(options) {
|
|
|
51
52
|
previousExecution: undefined,
|
|
52
53
|
acknowledgedSafetyChecks: undefined,
|
|
53
54
|
contextHint: undefined,
|
|
55
|
+
askedToEnd: false,
|
|
54
56
|
};
|
|
55
57
|
let stop;
|
|
56
58
|
try {
|
|
@@ -84,7 +86,7 @@ async function runTurns(session, conversation) {
|
|
|
84
86
|
// A request whose reply was cut off by the output limit, to be sent once more.
|
|
85
87
|
let resend;
|
|
86
88
|
for (;;) {
|
|
87
|
-
const halt = haltBeforeTurn(session);
|
|
89
|
+
const halt = haltBeforeTurn(session, observation);
|
|
88
90
|
if (halt !== undefined)
|
|
89
91
|
return halt;
|
|
90
92
|
turnNumber += 1;
|
|
@@ -116,6 +118,13 @@ async function runTurns(session, conversation) {
|
|
|
116
118
|
const blocked = reviewSafetyChecks(session, conversation, turn);
|
|
117
119
|
if (blocked !== undefined)
|
|
118
120
|
return blocked;
|
|
121
|
+
if ((turn.done || turn.actions.length === 0) && beforeScheduledEnd(session)) {
|
|
122
|
+
const held = keepWorking(session, conversation, backstop, observation);
|
|
123
|
+
if ("stop" in held)
|
|
124
|
+
return held.stop;
|
|
125
|
+
backstop = held.backstop;
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
119
128
|
if (turn.done || turn.actions.length === 0) {
|
|
120
129
|
// The declared outcome is kept even if redacting the participant's summary fails.
|
|
121
130
|
session.declaredOutcome = declaredOutcomeOf(turn);
|
|
@@ -139,7 +148,12 @@ async function runTurns(session, conversation) {
|
|
|
139
148
|
if ("stop" in checkpoint)
|
|
140
149
|
return checkpoint.stop;
|
|
141
150
|
observation = checkpoint.observation;
|
|
142
|
-
const step = advanceBackstop(backstop, {
|
|
151
|
+
const step = advanceBackstop(backstop, {
|
|
152
|
+
actions: turn.actions,
|
|
153
|
+
observation,
|
|
154
|
+
heardNewSpeech: observer.heardNewSpeech,
|
|
155
|
+
waitingIsTheTask: beforeScheduledEnd(session),
|
|
156
|
+
}, { idleSteps: session.idleSteps, noProgressSteps: session.noProgressSteps });
|
|
143
157
|
backstop = step.backstop;
|
|
144
158
|
const stalled = applyBackstop(session, conversation, step, [checkpoint.hint, batch.hint]);
|
|
145
159
|
if (stalled !== undefined)
|
|
@@ -149,6 +163,20 @@ async function runTurns(session, conversation) {
|
|
|
149
163
|
return unknownSpend;
|
|
150
164
|
}
|
|
151
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* A participant with a scheduled end that ended its own session before the end keeps going: the
|
|
168
|
+
* next request tells it the time left. The turn took no action, so it counts toward the idle
|
|
169
|
+
* backstop, which ends a participant that only keeps trying to stop.
|
|
170
|
+
*/
|
|
171
|
+
function keepWorking(session, conversation, backstop, observation) {
|
|
172
|
+
session.trace.record("notice", () => notice("warn", "participant ended before its scheduled end", "The participant ended its session before its scheduled end, so the session continues and the participant is told the time left."));
|
|
173
|
+
conversation.askedToEnd = true;
|
|
174
|
+
// The turn executed nothing, so the next request acknowledges nothing.
|
|
175
|
+
conversation.previousExecution = undefined;
|
|
176
|
+
const step = advanceBackstop(backstop, { actions: [], observation, heardNewSpeech: false }, { idleSteps: session.idleSteps, noProgressSteps: session.noProgressSteps });
|
|
177
|
+
const stalled = applyBackstop(session, conversation, step, []);
|
|
178
|
+
return stalled === undefined ? { backstop: step.backstop } : { stop: stalled };
|
|
179
|
+
}
|
|
152
180
|
/**
|
|
153
181
|
* Count the turn's idle and no-progress flags, stage every hint for the next request, and stop
|
|
154
182
|
* when a streak tripped. The
|
|
@@ -173,15 +201,22 @@ function refuseAccountBilledCaps(options) {
|
|
|
173
201
|
});
|
|
174
202
|
}
|
|
175
203
|
}
|
|
176
|
-
function haltBeforeTurn(session) {
|
|
204
|
+
function haltBeforeTurn(session, observation) {
|
|
177
205
|
if (session.signal?.aborted)
|
|
178
206
|
return stops.harnessAborted;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
return undefined;
|
|
207
|
+
return (scheduledEndStop(session, observation) ??
|
|
208
|
+
(session.remaining() < 0 ? stops.timeLimit(session) : undefined));
|
|
182
209
|
}
|
|
183
|
-
/**
|
|
210
|
+
/**
|
|
211
|
+
* Build the next request; the context hint and safety acknowledgements ride it once. The
|
|
212
|
+
* participant's clock text rides every request, after any other hint.
|
|
213
|
+
*/
|
|
184
214
|
function nextRequest(session, conversation, observation) {
|
|
215
|
+
const clock = clockHint(session.settings.clock, session.now, conversation.askedToEnd);
|
|
216
|
+
conversation.askedToEnd = false;
|
|
217
|
+
if (clock !== undefined)
|
|
218
|
+
conversation.contextHint =
|
|
219
|
+
conversation.contextHint === undefined ? clock : `${conversation.contextHint} ${clock}`;
|
|
185
220
|
const request = {
|
|
186
221
|
instructions: session.settings.instructions,
|
|
187
222
|
observation,
|
|
@@ -50,12 +50,13 @@ export interface PlannedWait {
|
|
|
50
50
|
export declare function defaultMaxWaitMs(speechEnabled: boolean): number;
|
|
51
51
|
/**
|
|
52
52
|
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
53
|
-
* in a study file, empty for loop options.
|
|
53
|
+
* in a study file, empty for loop options. `maxPrefix` names the owner of maxWaitMs when it is
|
|
54
|
+
* another, as for a participant's own idleWaitMs under actor.maxWaitMs.
|
|
54
55
|
*/
|
|
55
56
|
export declare function waitFieldsReason(fields: {
|
|
56
57
|
readonly maxWaitMs?: unknown;
|
|
57
58
|
readonly idleWaitMs?: unknown;
|
|
58
|
-
}, prefix: string): string | null;
|
|
59
|
+
}, prefix: string, maxPrefix?: string): string | null;
|
|
59
60
|
/**
|
|
60
61
|
* The wait lengths a session applies: the study's values, or their defaults. The idle wait never
|
|
61
62
|
* outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
|
|
@@ -55,16 +55,17 @@ const FIELDS = [
|
|
|
55
55
|
];
|
|
56
56
|
/**
|
|
57
57
|
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
58
|
-
* in a study file, empty for loop options.
|
|
58
|
+
* in a study file, empty for loop options. `maxPrefix` names the owner of maxWaitMs when it is
|
|
59
|
+
* another, as for a participant's own idleWaitMs under actor.maxWaitMs.
|
|
59
60
|
*/
|
|
60
|
-
export function waitFieldsReason(fields, prefix) {
|
|
61
|
+
export function waitFieldsReason(fields, prefix, maxPrefix = prefix) {
|
|
61
62
|
for (const [name, meaning] of FIELDS) {
|
|
62
63
|
if (fields[name] !== undefined && !isWaitMs(fields[name]))
|
|
63
|
-
return `${prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
|
|
64
|
+
return `${name === "maxWaitMs" ? maxPrefix : prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
|
|
64
65
|
}
|
|
65
66
|
const { maxWaitMs, idleWaitMs } = fields;
|
|
66
67
|
if (typeof maxWaitMs === "number" && typeof idleWaitMs === "number" && idleWaitMs > maxWaitMs)
|
|
67
|
-
return `${prefix}idleWaitMs (${idleWaitMs}) is longer than ${
|
|
68
|
+
return `${prefix}idleWaitMs (${idleWaitMs}) is longer than ${maxPrefix}maxWaitMs (${maxWaitMs}), the longest one wait action lasts. Lower ${prefix}idleWaitMs or raise ${maxPrefix}maxWaitMs.`;
|
|
68
69
|
return null;
|
|
69
70
|
}
|
|
70
71
|
/**
|
|
@@ -37,7 +37,7 @@ export type ParticipantDeclaredOutcome = "reached" | "not_reached" | "blocked";
|
|
|
37
37
|
export declare const PARTICIPANT_OUTCOME_STATUSES: readonly ActorStatus[];
|
|
38
38
|
/** Optional precise interruption cause; completionReason and status retain their original meaning.
|
|
39
39
|
* One list, so the analysis schema and the diagnostics projection cannot drift from the trace. */
|
|
40
|
-
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported"];
|
|
40
|
+
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end"];
|
|
41
41
|
export type ActorStopCause = (typeof ACTOR_STOP_CAUSES)[number];
|
|
42
42
|
export type ActorCompletionReason = "goal_satisfied" | "turn_completed" | "gave_up" | "blocked_approval" | "timed_out" | "budget_reached" | "actor_error" | "step_failed" | "harness_error";
|
|
43
43
|
type ActorRunKind = "code" | "computer-use" | "scripted-browser" | "terminal";
|
|
@@ -346,11 +346,11 @@ export interface ActorTrace {
|
|
|
346
346
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
347
347
|
/**
|
|
348
348
|
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
349
|
-
* `dwell`), or impressions only after the participant ended the session itself
|
|
349
|
+
* `dwell`, `scheduled_end`), or impressions only after the participant ended the session itself
|
|
350
350
|
* (`participant_end`, never with `report`). Does not change task outcomes or permit actions.
|
|
351
351
|
*/
|
|
352
352
|
debrief?: {
|
|
353
|
-
trigger: "stop_when" | "dwell" | "participant_end";
|
|
353
|
+
trigger: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
354
354
|
status: "completed" | "skipped" | "failed";
|
|
355
355
|
reason: string;
|
|
356
356
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
package/dist/actors/contract.js
CHANGED
|
@@ -53,6 +53,9 @@ export const ACTOR_STOP_CAUSES = [
|
|
|
53
53
|
"provider_refused_prompt",
|
|
54
54
|
"harness_aborted",
|
|
55
55
|
"usage_unreported",
|
|
56
|
+
// The participant's scheduled end (participants[].endAfterMs) came. A planned end: the session
|
|
57
|
+
// is goal_satisfied, and the closing report follows.
|
|
58
|
+
"scheduled_end",
|
|
56
59
|
];
|
|
57
60
|
/**
|
|
58
61
|
* The oldest Codex CLI release a recorded account profile may name. Readers accept every stable
|
|
@@ -10,6 +10,7 @@ const labels = {
|
|
|
10
10
|
provider_status: "unexpected provider status",
|
|
11
11
|
provider_refused_prompt: "provider refused the prompt under its usage policy",
|
|
12
12
|
harness_aborted: "stopped by harness",
|
|
13
|
+
scheduled_end: "scheduled end",
|
|
13
14
|
unspecified_limit: "limit reached",
|
|
14
15
|
};
|
|
15
16
|
/** Shared wording for consumers that already hold a validated, finite stop cause. */
|
|
@@ -35,7 +35,6 @@ interface CuaRunBundleState {
|
|
|
35
35
|
outcomes: ParticipantRunOutcome[] | undefined;
|
|
36
36
|
subjects: RunSubjectProvenance[];
|
|
37
37
|
aggregateSubject: RunSubjectProvenance;
|
|
38
|
-
failFastReason?: string;
|
|
39
38
|
/** The final bundle's reading of the run's host suspensions. */
|
|
40
39
|
hostSuspension?: HostSuspensionReading;
|
|
41
40
|
inProgress?: true;
|
|
@@ -60,7 +60,6 @@ export function buildCuaRunBundle(base, state) {
|
|
|
60
60
|
source: base.source,
|
|
61
61
|
participantPlan: base.participantPlan,
|
|
62
62
|
...(base.rerun === undefined ? {} : { rerun: base.rerun }),
|
|
63
|
-
...(state.failFastReason === undefined ? {} : { failFastReason: state.failFastReason }),
|
|
64
63
|
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
65
64
|
...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
|
|
66
65
|
...inProgress,
|
|
@@ -3,7 +3,7 @@ import { type ActorCompletionReason, type ActorStatus, type ActorStopCause } fro
|
|
|
3
3
|
declare const CUA_DIAGNOSTIC_CATEGORIES: readonly ["preview", "participant_outcome", "session_interrupted", "execution_error", "evidence_invalid", "mixed", "unknown"];
|
|
4
4
|
export type CuaDiagnosticCategory = (typeof CUA_DIAGNOSTIC_CATEGORIES)[number];
|
|
5
5
|
/** Closed at both projection and telemetry boundaries. No provider messages are classified. */
|
|
6
|
-
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "unspecified_limit", "mixed", "unknown"];
|
|
6
|
+
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "unspecified_limit", "mixed", "unknown"];
|
|
7
7
|
export type CuaDiagnosticStopCause = (typeof CUA_DIAGNOSTIC_STOP_CAUSES)[number];
|
|
8
8
|
export interface CuaDiagnostics {
|
|
9
9
|
category: CuaDiagnosticCategory;
|
|
@@ -30,6 +30,10 @@ export function cuaParticipantDiagnostics(input) {
|
|
|
30
30
|
const session = input.session;
|
|
31
31
|
if (!session)
|
|
32
32
|
return { category: "unknown" };
|
|
33
|
+
// A scheduled end is the participant's planned ending, so a run whose other participants ended
|
|
34
|
+
// themselves reads as one kind of ending.
|
|
35
|
+
if (session.stopCause === "scheduled_end")
|
|
36
|
+
return { category: "participant_outcome" };
|
|
33
37
|
if (session.stopCause !== undefined) {
|
|
34
38
|
return {
|
|
35
39
|
category: "session_interrupted",
|
|
@@ -3,9 +3,8 @@ import type { CuaFanoutBundleArgs } from "./types.js";
|
|
|
3
3
|
/**
|
|
4
4
|
* Project N>1 fan-out participants into a humanish.run-bundle.v1 (the evidence schema is unchanged; this
|
|
5
5
|
* is a new producer for the multi-stream shape). One sim + one stream per participant; per-participant
|
|
6
|
-
* provenance/session events; a recorded `cua-lab.fanout.plan` event
|
|
7
|
-
*
|
|
8
|
-
* streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
6
|
+
* provenance/session events; a recorded `cua-lab.fanout.plan` event. N-ary verify/Observer already
|
|
7
|
+
* handle multiple streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
9
8
|
* (bundle.ts) sends it to buildSingleParticipantBundle (single-bundle.ts).
|
|
10
9
|
*/
|
|
11
10
|
export declare function buildCuaFanoutBundle(args: CuaFanoutBundleArgs): RunBundle;
|
|
@@ -181,9 +181,8 @@ function fanoutFeedbackCandidates(args) {
|
|
|
181
181
|
/**
|
|
182
182
|
* Project N>1 fan-out participants into a humanish.run-bundle.v1 (the evidence schema is unchanged; this
|
|
183
183
|
* is a new producer for the multi-stream shape). One sim + one stream per participant; per-participant
|
|
184
|
-
* provenance/session events; a recorded `cua-lab.fanout.plan` event
|
|
185
|
-
*
|
|
186
|
-
* streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
184
|
+
* provenance/session events; a recorded `cua-lab.fanout.plan` event. N-ary verify/Observer already
|
|
185
|
+
* handle multiple streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
187
186
|
* (bundle.ts) sends it to buildSingleParticipantBundle (single-bundle.ts).
|
|
188
187
|
*/
|
|
189
188
|
export function buildCuaFanoutBundle(args) {
|
|
@@ -215,15 +214,6 @@ export function buildCuaFanoutBundle(args) {
|
|
|
215
214
|
streams.push(records.stream);
|
|
216
215
|
events.push(...records.events);
|
|
217
216
|
});
|
|
218
|
-
if (args.failFastReason) {
|
|
219
|
-
events.push({
|
|
220
|
-
id: nextEventId("fanout-fail-fast"),
|
|
221
|
-
at: args.run.createdAt,
|
|
222
|
-
level: "warn",
|
|
223
|
-
type: "cua-lab.fanout.fail-fast",
|
|
224
|
-
message: `Fan-out fail-fast: ${args.failFastReason}. In-flight participants finished; queued participants were skipped (blocked); completed evidence is retained.`,
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
217
|
const review = fanoutReview(args, streams, judgment);
|
|
228
218
|
const anyRaw = (outcomes ?? []).some((outcome) => outcome.session?.trace.redaction.screenshots === "raw");
|
|
229
219
|
const ranLive = (outcomes ?? []).some((outcome) => outcome.session !== undefined || outcome.sessionError !== undefined);
|
|
@@ -47,7 +47,6 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
47
47
|
runId: string;
|
|
48
48
|
};
|
|
49
49
|
outcomes?: never;
|
|
50
|
-
failFastReason?: never;
|
|
51
50
|
receiving?: never;
|
|
52
51
|
receivingWarnings?: never;
|
|
53
52
|
externalCommsWarnings?: never;
|
|
@@ -55,7 +54,6 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
55
54
|
result?: never;
|
|
56
55
|
ok: true;
|
|
57
56
|
outcomes: ParticipantRunOutcome[] | undefined;
|
|
58
|
-
failFastReason: string | undefined;
|
|
59
57
|
receiving: CommsReceivingRun | undefined;
|
|
60
58
|
receivingWarnings: string[];
|
|
61
59
|
externalCommsWarnings: string[];
|
|
@@ -94,17 +94,16 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
94
94
|
}
|
|
95
95
|
// Run the participants (dry-run runs none). In-process is always one.
|
|
96
96
|
let outcomes;
|
|
97
|
-
let failFastReason;
|
|
98
97
|
try {
|
|
99
98
|
if (!dryRun)
|
|
100
|
-
|
|
99
|
+
outcomes = await runAllCuaParticipants(participantRuns, {
|
|
101
100
|
...deps,
|
|
102
101
|
// A session that throws returns no trace; the run still records that it started.
|
|
103
102
|
runSession: (options) => {
|
|
104
103
|
run.participantStarted();
|
|
105
104
|
return deps.runSession(options);
|
|
106
105
|
},
|
|
107
|
-
}, participantPlan, inProcess)
|
|
106
|
+
}, participantPlan, inProcess);
|
|
108
107
|
}
|
|
109
108
|
finally {
|
|
110
109
|
try {
|
|
@@ -130,7 +129,6 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
130
129
|
return {
|
|
131
130
|
ok: true,
|
|
132
131
|
outcomes,
|
|
133
|
-
failFastReason,
|
|
134
132
|
receiving,
|
|
135
133
|
receivingWarnings,
|
|
136
134
|
externalCommsWarnings,
|
|
@@ -12,28 +12,22 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
14
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
-
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* ever enumerates.
|
|
15
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox.
|
|
16
|
+
* A harness error ends only its own participant: one desktop's stall or one provider failure does
|
|
17
|
+
* not predict the next participant's, so the schedule goes on. Only a closed gate ends every wait
|
|
18
|
+
* for a later start at once, and the participants it skips are `blocked` with its reason. Each
|
|
19
|
+
* participant tears down its own sandbox by id; nothing here ever enumerates.
|
|
21
20
|
*
|
|
22
21
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
23
22
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
24
23
|
* the default.
|
|
25
24
|
*/
|
|
26
|
-
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<
|
|
27
|
-
outcomes: ParticipantRunOutcome[];
|
|
28
|
-
failFastReason?: string;
|
|
29
|
-
}>;
|
|
25
|
+
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<ParticipantRunOutcome[]>;
|
|
30
26
|
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
31
27
|
export declare function toParticipantResult(spec: DesktopParticipantRun, outcome: ParticipantRunOutcome | undefined, subject: RunSubjectProvenance, dryRun: boolean): CuaParticipantResult;
|
|
32
28
|
/**
|
|
33
29
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
34
|
-
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and
|
|
30
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency, and stops
|
|
31
|
+
* early only when the pipeline gate closes.
|
|
35
32
|
*/
|
|
36
|
-
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<
|
|
37
|
-
outcomes: ParticipantRunOutcome[];
|
|
38
|
-
failFastReason: string | undefined;
|
|
39
|
-
}>;
|
|
33
|
+
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<ParticipantRunOutcome[]>;
|