humanish 0.120.1 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/cli/commands/study-format.js +25 -4
- package/dist/routes/computer-use/bundle.d.ts +3 -1
- package/dist/routes/computer-use/bundle.js +2 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +22 -14
- package/dist/routes/computer-use/live-phase.d.ts +1 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-deps.js +3 -1
- package/dist/routes/computer-use/participant-execution.d.ts +9 -14
- package/dist/routes/computer-use/participant-execution.js +102 -89
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +57 -9
- package/dist/routes/computer-use/setup.d.ts +1 -1
- package/dist/routes/computer-use/single-bundle.d.ts +3 -0
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +33 -7
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/host-suspension.d.ts +56 -0
- package/dist/run/host-suspension.js +106 -0
- package/dist/run/judge.d.ts +1 -1
- package/dist/run/outcomes.js +3 -1
- package/dist/run/run.d.ts +15 -2
- package/dist/run/run.js +15 -8
- package/dist/run/status.d.ts +8 -2
- package/dist/run/status.js +11 -8
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/preflight.js +11 -1
- package/dist/study/study-deps.d.ts +6 -0
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/dist/tui-app.js +35 -35
- package/docs/contracts/run-bundle.md +20 -1
- package/docs/contracts/schemas.md +24 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,36 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.122.0: Participants end on the clock, a harness error ends only its participant (2026-10-12)
|
|
12
|
+
|
|
13
|
+
humanish 0.122.0 lets a participant end at a scheduled time and keeps one participant's harness
|
|
14
|
+
error from stopping the rest of a study. `participants[].endAfterMs`, on the same clock as
|
|
15
|
+
`startAfterMs`, ends a participant's session at that time with stop cause `scheduled_end`,
|
|
16
|
+
counted as reaching the goal; from `endAfterMs - windDownMs` (10 minutes by default) each turn
|
|
17
|
+
tells the participant the end time and the minutes left, its turns that only wait no longer count
|
|
18
|
+
toward the idle limits before then, and its sandbox is sized from its own window. `actor.clock {
|
|
19
|
+
timeZone }` adds the local time to each turn, and `participants[].idleWaitMs` sets one
|
|
20
|
+
participant's wait length, so staff on a long shift can check the screen less often than patients.
|
|
21
|
+
A participant's harness error now ends only that participant: the study-wide fail-fast is removed,
|
|
22
|
+
and every later participant starts on its schedule.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.122.0)
|
|
25
|
+
|
|
26
|
+
## 0.121.0: One failed desktop fails alone, host sleep named as the cause (2026-10-10)
|
|
27
|
+
|
|
28
|
+
humanish 0.121.0 keeps one participant's failed desktop from costing a whole hosted study, and
|
|
29
|
+
names a host that slept mid-run as the likely cause of the failures after it. A computer-use study
|
|
30
|
+
on an `app-url` subject no longer holds its participants behind the first one's desktop, and a
|
|
31
|
+
participant that fails before its session no longer stops the participants queued behind it; a
|
|
32
|
+
study whose subject is built in each sandbox still lets one participant set it up first, and the
|
|
33
|
+
next one takes over when the first never got a desktop. When the machine running a study sleeps,
|
|
34
|
+
the run records the suspension as a `host.suspended` event, and the run outcome, `humanish run`,
|
|
35
|
+
the review and the `--json` result say the failures of participants running through it are
|
|
36
|
+
likely its effect, while each participant's record keeps its own error. The review's participant
|
|
37
|
+
line lists interruption causes in a fixed order.
|
|
38
|
+
|
|
39
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.121.0)
|
|
40
|
+
|
|
11
41
|
## 0.120.1: Analysis requests under Bun run past 5 minutes (2026-10-09)
|
|
12
42
|
|
|
13
43
|
humanish 0.120.1 lets an analysis request from a Bun process run longer than 5 minutes. Under Bun,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ActorPersonaRef, ActorTokenUsage, ActorTraceItem } from "../contract.js";
|
|
2
2
|
import { type CuaExecutor, type CuaLiveMetadata, type CuaLoopResult, type CuaProvider, type CuaSafetyCheck } from "./loop.js";
|
|
3
|
+
import type { ParticipantClock } from "./loop/types.js";
|
|
3
4
|
import { type E2BDesktopExecutorOptions, type E2BDesktopLike } from "../../substrates/e2b/desktop-executor.js";
|
|
4
5
|
import { type OpenAiResponsesProviderOptions } from "./openai-provider.js";
|
|
5
6
|
import { type RedactionHooks } from "../../evidence/redaction.js";
|
|
@@ -42,6 +43,8 @@ export interface CuaActorSessionOptions {
|
|
|
42
43
|
maxWaitMs?: number;
|
|
43
44
|
/** How long a wait with no duration lasts in an idle turn. See LoopRunOptions.idleWaitMs. */
|
|
44
45
|
idleWaitMs?: number;
|
|
46
|
+
/** The participant's time zone and scheduled end, forwarded to the loop. See CuaLoopOptions.clock. */
|
|
47
|
+
clock?: ParticipantClock;
|
|
45
48
|
/**
|
|
46
49
|
* Redact persisted screenshots (blur+downscale). Default `false`: full fidelity for local use.
|
|
47
50
|
* Set true for unowned subjects or share-as-is bundles. The provider always sees raw frames.
|
|
@@ -43,6 +43,7 @@ export async function runCuaActorSession(options) {
|
|
|
43
43
|
...(options.writeScreenshot === undefined ? {} : { writeScreenshot: options.writeScreenshot }),
|
|
44
44
|
...(options.stopWhen === undefined ? {} : { stopWhen: options.stopWhen }),
|
|
45
45
|
...(options.dwell === undefined ? {} : { dwell: options.dwell }),
|
|
46
|
+
...(options.clock === undefined ? {} : { clock: options.clock }),
|
|
46
47
|
...(options.tasks === undefined ? {} : { tasks: options.tasks }),
|
|
47
48
|
...(options.maxUsd === undefined ? {} : { maxUsd: options.maxUsd }),
|
|
48
49
|
...(options.estimateTurnCostUsd === undefined
|
|
@@ -34,9 +34,14 @@ export interface BackstopStep {
|
|
|
34
34
|
/** The gave_up reason when a streak tripped. */
|
|
35
35
|
readonly tripReason: string | undefined;
|
|
36
36
|
}
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
|
|
39
|
+
* participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
|
|
40
|
+
* toward neither streak and earns no nudge. A turn that acts is folded as usual.
|
|
41
|
+
*/
|
|
38
42
|
export declare function advanceBackstop(previous: Backstop, turn: {
|
|
39
43
|
actions: readonly CuaAction[];
|
|
40
44
|
observation: CuaObservation;
|
|
41
45
|
heardNewSpeech: boolean;
|
|
46
|
+
waitingIsTheTask?: boolean;
|
|
42
47
|
}, limits: BackstopLimits): BackstopStep;
|
|
@@ -122,9 +122,21 @@ export function startBackstop(observation) {
|
|
|
122
122
|
idleProgressForgivenessUsed: 0,
|
|
123
123
|
};
|
|
124
124
|
}
|
|
125
|
-
/**
|
|
125
|
+
/**
|
|
126
|
+
* Fold one acted turn and the observation after it into the backstop. `waitingIsTheTask` says the
|
|
127
|
+
* participant has a scheduled end still to come: waiting for it is its job, so an idle turn counts
|
|
128
|
+
* toward neither streak and earns no nudge. A turn that acts is folded as usual.
|
|
129
|
+
*/
|
|
126
130
|
export function advanceBackstop(previous, turn, limits) {
|
|
127
131
|
const idle = isIdleTurn(turn.actions);
|
|
132
|
+
if (idle && turn.waitingIsTheTask === true)
|
|
133
|
+
return {
|
|
134
|
+
backstop: { ...previous, progressKey: progressKeyOf(turn.observation), consecutiveIdle: 0 },
|
|
135
|
+
idle,
|
|
136
|
+
progressed: true,
|
|
137
|
+
hints: [],
|
|
138
|
+
tripReason: undefined,
|
|
139
|
+
};
|
|
128
140
|
const progressKey = progressKeyOf(turn.observation);
|
|
129
141
|
const frameChanged = progressKey !== previous.progressKey;
|
|
130
142
|
// Corroboration. A stale frame alone is not evidence of a stuck agent. The frame hash is
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { Stop } from "./ending.js";
|
|
2
|
+
import type { LoopSession } from "./session.js";
|
|
3
|
+
import type { CuaObservation, ParticipantClock } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* The time text for the request the loop is about to send: the local time when the participant
|
|
6
|
+
* has a time zone, and the end time and the minutes left from the start of its wind-down, or
|
|
7
|
+
* after it tried to end its session before its end (`askedToEnd`). Undefined when there is
|
|
8
|
+
* nothing to say.
|
|
9
|
+
*/
|
|
10
|
+
export declare function clockHint(clock: ParticipantClock | undefined, now: () => number, askedToEnd?: boolean): string | undefined;
|
|
11
|
+
/**
|
|
12
|
+
* Whether the participant has a scheduled end still to come. Its own end before then does not end
|
|
13
|
+
* the session: the end is what the study declared for it.
|
|
14
|
+
*/
|
|
15
|
+
export declare function beforeScheduledEnd(session: LoopSession): boolean;
|
|
16
|
+
/**
|
|
17
|
+
* The stop at the participant's scheduled end, or undefined before it. The session passes as a
|
|
18
|
+
* planned end and the participant is asked for its closing report on the last observation.
|
|
19
|
+
*/
|
|
20
|
+
export declare function scheduledEndStop(session: LoopSession, observation: CuaObservation): Stop | undefined;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { plural } from "../../../run/text.js";
|
|
2
|
+
// What the participant knows about the time. A hosted desktop shows a maximized browser with no
|
|
3
|
+
// OS clock, so a participant told "your shift ends at 11:00" cannot relate it to now unless each
|
|
4
|
+
// turn says so. The text rides the request's context hint, which every provider sends with the
|
|
5
|
+
// turn (OpenAI as a user message, Codex and the local agents as the harness note).
|
|
6
|
+
/**
|
|
7
|
+
* The time text for the request the loop is about to send: the local time when the participant
|
|
8
|
+
* has a time zone, and the end time and the minutes left from the start of its wind-down, or
|
|
9
|
+
* after it tried to end its session before its end (`askedToEnd`). Undefined when there is
|
|
10
|
+
* nothing to say.
|
|
11
|
+
*/
|
|
12
|
+
export function clockHint(clock, now, askedToEnd = false) {
|
|
13
|
+
if (clock === undefined)
|
|
14
|
+
return undefined;
|
|
15
|
+
const nowMs = now();
|
|
16
|
+
const parts = [];
|
|
17
|
+
if (clock.timeZone !== undefined)
|
|
18
|
+
parts.push(`The local time is ${hourMinute(nowMs, clock.timeZone)} (${clock.timeZone}).`);
|
|
19
|
+
const { end } = clock;
|
|
20
|
+
if (end !== undefined && (askedToEnd || nowMs >= end.atMs - end.windDownMs)) {
|
|
21
|
+
const minutesLeft = Math.max(1, Math.ceil((end.atMs - nowMs) / 60_000));
|
|
22
|
+
parts.push(`${askedToEnd ? "Your session is not over. " : ""}Your session ends at ${endTime(clock, end.atMs)}, in ${plural(minutesLeft, "minute")}. Keep working until then. The session ends by itself at that time, and you will then be asked for your closing account.`);
|
|
23
|
+
}
|
|
24
|
+
return parts.length === 0 ? undefined : parts.join(" ");
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Whether the participant has a scheduled end still to come. Its own end before then does not end
|
|
28
|
+
* the session: the end is what the study declared for it.
|
|
29
|
+
*/
|
|
30
|
+
export function beforeScheduledEnd(session) {
|
|
31
|
+
const end = session.settings.clock?.end;
|
|
32
|
+
return end !== undefined && session.now() < end.atMs;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The stop at the participant's scheduled end, or undefined before it. The session passes as a
|
|
36
|
+
* planned end and the participant is asked for its closing report on the last observation.
|
|
37
|
+
*/
|
|
38
|
+
export function scheduledEndStop(session, observation) {
|
|
39
|
+
const end = session.settings.clock?.end;
|
|
40
|
+
if (end === undefined || session.now() < end.atMs)
|
|
41
|
+
return undefined;
|
|
42
|
+
return {
|
|
43
|
+
completionReason: "goal_satisfied",
|
|
44
|
+
reason: `the participant's scheduled end at ${endTime(session.settings.clock, end.atMs)} was reached`,
|
|
45
|
+
stopCause: "scheduled_end",
|
|
46
|
+
debriefTrigger: { kind: "scheduled_end", observation },
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
// The end to the nearest minute: a run starts at no particular second, and "ends at 00:09" read
|
|
50
|
+
// at 00:09:06 looks past.
|
|
51
|
+
function endTime(clock, atMs) {
|
|
52
|
+
const minute = Math.round(atMs / 60_000) * 60_000;
|
|
53
|
+
return clock.timeZone === undefined
|
|
54
|
+
? `${hourMinute(minute, "UTC")} UTC`
|
|
55
|
+
: hourMinute(minute, clock.timeZone);
|
|
56
|
+
}
|
|
57
|
+
function hourMinute(ms, timeZone) {
|
|
58
|
+
return new Intl.DateTimeFormat("en-GB", {
|
|
59
|
+
timeZone,
|
|
60
|
+
hour: "2-digit",
|
|
61
|
+
minute: "2-digit",
|
|
62
|
+
hourCycle: "h23",
|
|
63
|
+
}).format(ms);
|
|
64
|
+
}
|
|
@@ -8,7 +8,7 @@ import type { CuaObservation, CuaProvider, CuaTurn } from "./types.js";
|
|
|
8
8
|
* the participant's own ending when only its impressions are missing.
|
|
9
9
|
*/
|
|
10
10
|
export interface DebriefTrigger {
|
|
11
|
-
readonly kind: "stop_when" | "dwell" | "participant_end";
|
|
11
|
+
readonly kind: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
12
12
|
/** The final observation, already captured; the debrief request sends it without observing. */
|
|
13
13
|
readonly observation: CuaObservation;
|
|
14
14
|
}
|
|
@@ -12,7 +12,7 @@ export type ScreenshotRef = NonNullable<ActorTraceItem["screenshotRef"]>;
|
|
|
12
12
|
* wrapped callbacks are fields on LoopSession instead. Callers destructure hooks to call them.
|
|
13
13
|
*/
|
|
14
14
|
export type LoopSettings = {
|
|
15
|
-
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]: LoopRunOptions[K];
|
|
15
|
+
readonly [K in "instructions" | "persona" | "redaction" | "stopWhen" | "dwell" | "clock" | "tasks" | "maxUsd" | "overRunBudget" | "estimateTurnCostUsd" | "onObservedUrl" | "onMessage" | "onScreenshot" | "onTrace"]: LoopRunOptions[K];
|
|
16
16
|
};
|
|
17
17
|
/** The participant's dispatched actions, as the backstop and failure notices describe them. */
|
|
18
18
|
export interface ActionHistory {
|
|
@@ -70,13 +70,14 @@ export class LoopSession {
|
|
|
70
70
|
scrubText;
|
|
71
71
|
constructor(options) {
|
|
72
72
|
// Read once, in this order: a caller mutating its options object mid-run changes nothing.
|
|
73
|
-
const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, maxWaitMs, idleWaitMs, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace, } = options;
|
|
73
|
+
const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, maxWaitMs, idleWaitMs, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, clock, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace, } = options;
|
|
74
74
|
this.settings = {
|
|
75
75
|
instructions,
|
|
76
76
|
persona,
|
|
77
77
|
redaction,
|
|
78
78
|
stopWhen,
|
|
79
79
|
dwell,
|
|
80
|
+
clock,
|
|
80
81
|
tasks,
|
|
81
82
|
maxUsd,
|
|
82
83
|
overRunBudget,
|
|
@@ -4,6 +4,14 @@ import type { RedactionHooks } from "../../../evidence/redaction.js";
|
|
|
4
4
|
import type { DwellWindow, StopWhen } from "../../stop-conditions.js";
|
|
5
5
|
import type { StudyTask } from "../../../study/tasks.js";
|
|
6
6
|
import type { ReasoningEffort } from "../../reasoning-effort.js";
|
|
7
|
+
/** The participant's clock: its local time zone and its scheduled end, on the loop's clock. */
|
|
8
|
+
export interface ParticipantClock {
|
|
9
|
+
readonly timeZone?: string;
|
|
10
|
+
readonly end?: {
|
|
11
|
+
readonly atMs: number;
|
|
12
|
+
readonly windDownMs: number;
|
|
13
|
+
};
|
|
14
|
+
}
|
|
7
15
|
/**
|
|
8
16
|
* One desktop action. `heldKeys` on a pointer action are held down for the whole action and
|
|
9
17
|
* released after it, named as the provider sent them (for example `SHIFT` for a shift-click). An
|
|
@@ -346,6 +354,14 @@ export interface CuaLoopOptions {
|
|
|
346
354
|
dwell?: DwellWindow;
|
|
347
355
|
/** Injected pause for the dwell window's cadence; tests advance their clock through it. */
|
|
348
356
|
sleep?: (ms: number) => Promise<void>;
|
|
357
|
+
/**
|
|
358
|
+
* What each turn tells the participant about the time (loop/clock.ts). With `timeZone`, a time zone
|
|
359
|
+
* name, every request carries the local time. With `end`, from `end.atMs - end.windDownMs`
|
|
360
|
+
* every request names the end time (in `timeZone`, else UTC) and the minutes left, and at
|
|
361
|
+
* `end.atMs`, read on `now`, the loop asks for the closing report and ends the session with
|
|
362
|
+
* stop cause `scheduled_end`.
|
|
363
|
+
*/
|
|
364
|
+
clock?: ParticipantClock;
|
|
349
365
|
/**
|
|
350
366
|
* The study's declared protocol: discrete tasks whose completion is corroborated by the
|
|
351
367
|
* same observations stopWhen reads, on the same cadence. The tracker never influences the loop's
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CuaLoopOptions, CuaLoopResult, LoopRunOptions } from "./loop/types.js";
|
|
2
|
-
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, } from "./loop/types.js";
|
|
2
|
+
export type { CuaAction, CuaExecutor, CuaLiveMetadata, CuaLoopOptions, CuaLoopResult, CuaObservation, CuaProvider, CuaSafetyCheck, CuaSpendGate, CuaTurn, CuaTurnRequest, ParticipantClock, } from "./loop/types.js";
|
|
3
3
|
export { describeCuaAction } from "./loop/actions.js";
|
|
4
4
|
export { stableProgressKey } from "./loop/backstop.js";
|
|
5
5
|
export { IMPRESSIONS_ASK } from "./loop/impressions.js";
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { closingReportSchema } from "../closing-report.js";
|
|
2
2
|
import { ComputerUseProviderError } from "./provider-error.js";
|
|
3
3
|
import { runActionBatch } from "./loop/actions.js";
|
|
4
|
-
import { advanceBackstop, startBackstop } from "./loop/backstop.js";
|
|
4
|
+
import { advanceBackstop, startBackstop, } from "./loop/backstop.js";
|
|
5
|
+
import { beforeScheduledEnd, clockHint, scheduledEndStop } from "./loop/clock.js";
|
|
5
6
|
import { requestDebrief } from "./loop/debrief.js";
|
|
6
7
|
import * as stops from "./loop/ending.js";
|
|
7
8
|
import { declaredOutcomeOf } from "./loop/ending.js";
|
|
@@ -51,6 +52,7 @@ export async function runComputerUseLoopWithTaps(options) {
|
|
|
51
52
|
previousExecution: undefined,
|
|
52
53
|
acknowledgedSafetyChecks: undefined,
|
|
53
54
|
contextHint: undefined,
|
|
55
|
+
askedToEnd: false,
|
|
54
56
|
};
|
|
55
57
|
let stop;
|
|
56
58
|
try {
|
|
@@ -84,7 +86,7 @@ async function runTurns(session, conversation) {
|
|
|
84
86
|
// A request whose reply was cut off by the output limit, to be sent once more.
|
|
85
87
|
let resend;
|
|
86
88
|
for (;;) {
|
|
87
|
-
const halt = haltBeforeTurn(session);
|
|
89
|
+
const halt = haltBeforeTurn(session, observation);
|
|
88
90
|
if (halt !== undefined)
|
|
89
91
|
return halt;
|
|
90
92
|
turnNumber += 1;
|
|
@@ -116,6 +118,13 @@ async function runTurns(session, conversation) {
|
|
|
116
118
|
const blocked = reviewSafetyChecks(session, conversation, turn);
|
|
117
119
|
if (blocked !== undefined)
|
|
118
120
|
return blocked;
|
|
121
|
+
if ((turn.done || turn.actions.length === 0) && beforeScheduledEnd(session)) {
|
|
122
|
+
const held = keepWorking(session, conversation, backstop, observation);
|
|
123
|
+
if ("stop" in held)
|
|
124
|
+
return held.stop;
|
|
125
|
+
backstop = held.backstop;
|
|
126
|
+
continue;
|
|
127
|
+
}
|
|
119
128
|
if (turn.done || turn.actions.length === 0) {
|
|
120
129
|
// The declared outcome is kept even if redacting the participant's summary fails.
|
|
121
130
|
session.declaredOutcome = declaredOutcomeOf(turn);
|
|
@@ -139,7 +148,12 @@ async function runTurns(session, conversation) {
|
|
|
139
148
|
if ("stop" in checkpoint)
|
|
140
149
|
return checkpoint.stop;
|
|
141
150
|
observation = checkpoint.observation;
|
|
142
|
-
const step = advanceBackstop(backstop, {
|
|
151
|
+
const step = advanceBackstop(backstop, {
|
|
152
|
+
actions: turn.actions,
|
|
153
|
+
observation,
|
|
154
|
+
heardNewSpeech: observer.heardNewSpeech,
|
|
155
|
+
waitingIsTheTask: beforeScheduledEnd(session),
|
|
156
|
+
}, { idleSteps: session.idleSteps, noProgressSteps: session.noProgressSteps });
|
|
143
157
|
backstop = step.backstop;
|
|
144
158
|
const stalled = applyBackstop(session, conversation, step, [checkpoint.hint, batch.hint]);
|
|
145
159
|
if (stalled !== undefined)
|
|
@@ -149,6 +163,20 @@ async function runTurns(session, conversation) {
|
|
|
149
163
|
return unknownSpend;
|
|
150
164
|
}
|
|
151
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* A participant with a scheduled end that ended its own session before the end keeps going: the
|
|
168
|
+
* next request tells it the time left. The turn took no action, so it counts toward the idle
|
|
169
|
+
* backstop, which ends a participant that only keeps trying to stop.
|
|
170
|
+
*/
|
|
171
|
+
function keepWorking(session, conversation, backstop, observation) {
|
|
172
|
+
session.trace.record("notice", () => notice("warn", "participant ended before its scheduled end", "The participant ended its session before its scheduled end, so the session continues and the participant is told the time left."));
|
|
173
|
+
conversation.askedToEnd = true;
|
|
174
|
+
// The turn executed nothing, so the next request acknowledges nothing.
|
|
175
|
+
conversation.previousExecution = undefined;
|
|
176
|
+
const step = advanceBackstop(backstop, { actions: [], observation, heardNewSpeech: false }, { idleSteps: session.idleSteps, noProgressSteps: session.noProgressSteps });
|
|
177
|
+
const stalled = applyBackstop(session, conversation, step, []);
|
|
178
|
+
return stalled === undefined ? { backstop: step.backstop } : { stop: stalled };
|
|
179
|
+
}
|
|
152
180
|
/**
|
|
153
181
|
* Count the turn's idle and no-progress flags, stage every hint for the next request, and stop
|
|
154
182
|
* when a streak tripped. The
|
|
@@ -173,15 +201,22 @@ function refuseAccountBilledCaps(options) {
|
|
|
173
201
|
});
|
|
174
202
|
}
|
|
175
203
|
}
|
|
176
|
-
function haltBeforeTurn(session) {
|
|
204
|
+
function haltBeforeTurn(session, observation) {
|
|
177
205
|
if (session.signal?.aborted)
|
|
178
206
|
return stops.harnessAborted;
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
return undefined;
|
|
207
|
+
return (scheduledEndStop(session, observation) ??
|
|
208
|
+
(session.remaining() < 0 ? stops.timeLimit(session) : undefined));
|
|
182
209
|
}
|
|
183
|
-
/**
|
|
210
|
+
/**
|
|
211
|
+
* Build the next request; the context hint and safety acknowledgements ride it once. The
|
|
212
|
+
* participant's clock text rides every request, after any other hint.
|
|
213
|
+
*/
|
|
184
214
|
function nextRequest(session, conversation, observation) {
|
|
215
|
+
const clock = clockHint(session.settings.clock, session.now, conversation.askedToEnd);
|
|
216
|
+
conversation.askedToEnd = false;
|
|
217
|
+
if (clock !== undefined)
|
|
218
|
+
conversation.contextHint =
|
|
219
|
+
conversation.contextHint === undefined ? clock : `${conversation.contextHint} ${clock}`;
|
|
185
220
|
const request = {
|
|
186
221
|
instructions: session.settings.instructions,
|
|
187
222
|
observation,
|
|
@@ -50,12 +50,13 @@ export interface PlannedWait {
|
|
|
50
50
|
export declare function defaultMaxWaitMs(speechEnabled: boolean): number;
|
|
51
51
|
/**
|
|
52
52
|
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
53
|
-
* in a study file, empty for loop options.
|
|
53
|
+
* in a study file, empty for loop options. `maxPrefix` names the owner of maxWaitMs when it is
|
|
54
|
+
* another, as for a participant's own idleWaitMs under actor.maxWaitMs.
|
|
54
55
|
*/
|
|
55
56
|
export declare function waitFieldsReason(fields: {
|
|
56
57
|
readonly maxWaitMs?: unknown;
|
|
57
58
|
readonly idleWaitMs?: unknown;
|
|
58
|
-
}, prefix: string): string | null;
|
|
59
|
+
}, prefix: string, maxPrefix?: string): string | null;
|
|
59
60
|
/**
|
|
60
61
|
* The wait lengths a session applies: the study's values, or their defaults. The idle wait never
|
|
61
62
|
* outlasts the longest wait, which on a desktop with speech defaults to 30 s. Throws a RangeError
|
|
@@ -55,16 +55,17 @@ const FIELDS = [
|
|
|
55
55
|
];
|
|
56
56
|
/**
|
|
57
57
|
* Why these wait fields cannot run, or null. `prefix` names their owner in the message: `actor.`
|
|
58
|
-
* in a study file, empty for loop options.
|
|
58
|
+
* in a study file, empty for loop options. `maxPrefix` names the owner of maxWaitMs when it is
|
|
59
|
+
* another, as for a participant's own idleWaitMs under actor.maxWaitMs.
|
|
59
60
|
*/
|
|
60
|
-
export function waitFieldsReason(fields, prefix) {
|
|
61
|
+
export function waitFieldsReason(fields, prefix, maxPrefix = prefix) {
|
|
61
62
|
for (const [name, meaning] of FIELDS) {
|
|
62
63
|
if (fields[name] !== undefined && !isWaitMs(fields[name]))
|
|
63
|
-
return `${prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
|
|
64
|
+
return `${name === "maxWaitMs" ? maxPrefix : prefix}${name}, ${meaning}, must be a whole number of milliseconds from ${CUA_WAIT_LIMITS.leastMs} to ${CUA_WAIT_LIMITS.mostMs}.`;
|
|
64
65
|
}
|
|
65
66
|
const { maxWaitMs, idleWaitMs } = fields;
|
|
66
67
|
if (typeof maxWaitMs === "number" && typeof idleWaitMs === "number" && idleWaitMs > maxWaitMs)
|
|
67
|
-
return `${prefix}idleWaitMs (${idleWaitMs}) is longer than ${
|
|
68
|
+
return `${prefix}idleWaitMs (${idleWaitMs}) is longer than ${maxPrefix}maxWaitMs (${maxWaitMs}), the longest one wait action lasts. Lower ${prefix}idleWaitMs or raise ${maxPrefix}maxWaitMs.`;
|
|
68
69
|
return null;
|
|
69
70
|
}
|
|
70
71
|
/**
|
|
@@ -37,7 +37,7 @@ export type ParticipantDeclaredOutcome = "reached" | "not_reached" | "blocked";
|
|
|
37
37
|
export declare const PARTICIPANT_OUTCOME_STATUSES: readonly ActorStatus[];
|
|
38
38
|
/** Optional precise interruption cause; completionReason and status retain their original meaning.
|
|
39
39
|
* One list, so the analysis schema and the diagnostics projection cannot drift from the trace. */
|
|
40
|
-
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported"];
|
|
40
|
+
export declare const ACTOR_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end"];
|
|
41
41
|
export type ActorStopCause = (typeof ACTOR_STOP_CAUSES)[number];
|
|
42
42
|
export type ActorCompletionReason = "goal_satisfied" | "turn_completed" | "gave_up" | "blocked_approval" | "timed_out" | "budget_reached" | "actor_error" | "step_failed" | "harness_error";
|
|
43
43
|
type ActorRunKind = "code" | "computer-use" | "scripted-browser" | "terminal";
|
|
@@ -346,11 +346,11 @@ export interface ActorTrace {
|
|
|
346
346
|
declaredOutcome?: ParticipantDeclaredOutcome;
|
|
347
347
|
/**
|
|
348
348
|
* The read-only closing request: a closing report after a harness-owned stop (`stop_when`,
|
|
349
|
-
* `dwell`), or impressions only after the participant ended the session itself
|
|
349
|
+
* `dwell`, `scheduled_end`), or impressions only after the participant ended the session itself
|
|
350
350
|
* (`participant_end`, never with `report`). Does not change task outcomes or permit actions.
|
|
351
351
|
*/
|
|
352
352
|
debrief?: {
|
|
353
|
-
trigger: "stop_when" | "dwell" | "participant_end";
|
|
353
|
+
trigger: "stop_when" | "dwell" | "scheduled_end" | "participant_end";
|
|
354
354
|
status: "completed" | "skipped" | "failed";
|
|
355
355
|
reason: string;
|
|
356
356
|
/** Absent if no request was made; false means token accounting is incomplete. */
|
package/dist/actors/contract.js
CHANGED
|
@@ -53,6 +53,9 @@ export const ACTOR_STOP_CAUSES = [
|
|
|
53
53
|
"provider_refused_prompt",
|
|
54
54
|
"harness_aborted",
|
|
55
55
|
"usage_unreported",
|
|
56
|
+
// The participant's scheduled end (participants[].endAfterMs) came. A planned end: the session
|
|
57
|
+
// is goal_satisfied, and the closing report follows.
|
|
58
|
+
"scheduled_end",
|
|
56
59
|
];
|
|
57
60
|
/**
|
|
58
61
|
* The oldest Codex CLI release a recorded account profile may name. Readers accept every stable
|
|
@@ -10,6 +10,7 @@ const labels = {
|
|
|
10
10
|
provider_status: "unexpected provider status",
|
|
11
11
|
provider_refused_prompt: "provider refused the prompt under its usage policy",
|
|
12
12
|
harness_aborted: "stopped by harness",
|
|
13
|
+
scheduled_end: "scheduled end",
|
|
13
14
|
unspecified_limit: "limit reached",
|
|
14
15
|
};
|
|
15
16
|
/** Shared wording for consumers that already hold a validated, finite stop cause. */
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
3
|
import { subjectName } from "../../run/subject-name.js";
|
|
4
|
+
import { cli } from "../invocation.js";
|
|
4
5
|
/**
|
|
5
6
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
6
7
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
7
|
-
* complete turns `ok` false. That run finished, and the line says so.
|
|
8
|
+
* complete turns `ok` false. That run finished, and the line says so. A host suspension follows,
|
|
9
|
+
* since it may explain every failure listed after it.
|
|
8
10
|
*/
|
|
9
11
|
function runHeader(result, route) {
|
|
10
12
|
const kind = result.dryRun === true ? "dry run" : result.dryRun === false ? "live run" : "run";
|
|
@@ -13,7 +15,24 @@ function runHeader(result, route) {
|
|
|
13
15
|
: result.runOk === true
|
|
14
16
|
? "finished; the analysis did not complete"
|
|
15
17
|
: "failed";
|
|
16
|
-
return [
|
|
18
|
+
return [
|
|
19
|
+
`humanish run ${result.studyId}: ${kind} ${ending}`,
|
|
20
|
+
`route: ${route}`,
|
|
21
|
+
...(result.hostSuspension === undefined ? [] : [result.hostSuspension.summary]),
|
|
22
|
+
];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* What to run after a host suspension failed participants: the rerun of those that did not pass,
|
|
26
|
+
* or the study again for a run with one participant.
|
|
27
|
+
*/
|
|
28
|
+
function suspensionNextStep(result, participants) {
|
|
29
|
+
if ((result.hostSuspension?.participantIds.length ?? 0) === 0)
|
|
30
|
+
return [];
|
|
31
|
+
return participants > 1
|
|
32
|
+
? [
|
|
33
|
+
`Rerun the participants that did not pass: ${cli(`run ${result.studyId} --rerun-failed-from ${result.runId}`)}`,
|
|
34
|
+
]
|
|
35
|
+
: [`Run it again: ${cli(`run ${result.studyId}`)}`];
|
|
17
36
|
}
|
|
18
37
|
/**
|
|
19
38
|
* A run's human output: its lines on stdout and, when it failed with an error, that error on
|
|
@@ -167,8 +186,10 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
167
186
|
}
|
|
168
187
|
/** `subject` is the study's declared subject, which names it when the result has no URL. */
|
|
169
188
|
export function formatCuaStudyHuman(result, subject) {
|
|
189
|
+
const participants = result.lanes ?? [];
|
|
170
190
|
return withError(result.error, [
|
|
171
191
|
...runHeader(result, "computer-use"),
|
|
192
|
+
...suspensionNextStep(result, participants.length),
|
|
172
193
|
`run: ${result.runId}`,
|
|
173
194
|
`actor: ${result.actor}`,
|
|
174
195
|
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
@@ -181,8 +202,8 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
181
202
|
? [`rerun: ${result.rerun.selectedLaneIds.join(", ")} from ${result.rerun.sourceRunId}`]
|
|
182
203
|
: []),
|
|
183
204
|
...(result.diagnostics ? [`diagnostic: ${formatCuaDiagnostics(result.diagnostics)}`] : []),
|
|
184
|
-
...participantLines(
|
|
185
|
-
...(result.session &&
|
|
205
|
+
...participantLines(participants),
|
|
206
|
+
...(result.session && participants.length <= 1
|
|
186
207
|
? [
|
|
187
208
|
`session: ${participantStatus(result.session.status)} (${result.session.completionReason})${result.session.stopCause ? ` · ${formatCuaStopCause(result.session.stopCause)}` : ""} · ${result.session.reason}`,
|
|
188
209
|
`screenshots: ${result.session.screenshots}`,
|
|
@@ -2,6 +2,7 @@ import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
|
2
2
|
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { BundleRun, RunBundle, RunRerunLineage, RunSubjectProvenance } from "../../run/bundle.js";
|
|
4
4
|
import { type Judgment } from "../../run/judge.js";
|
|
5
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
5
6
|
import type { CuaParticipantPlan, DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
6
7
|
/** What every bundle of one run shares, in progress or final. */
|
|
7
8
|
export interface CuaRunBundleBase {
|
|
@@ -34,7 +35,8 @@ interface CuaRunBundleState {
|
|
|
34
35
|
outcomes: ParticipantRunOutcome[] | undefined;
|
|
35
36
|
subjects: RunSubjectProvenance[];
|
|
36
37
|
aggregateSubject: RunSubjectProvenance;
|
|
37
|
-
|
|
38
|
+
/** The final bundle's reading of the run's host suspensions. */
|
|
39
|
+
hostSuspension?: HostSuspensionReading;
|
|
38
40
|
inProgress?: true;
|
|
39
41
|
}
|
|
40
42
|
/**
|
|
@@ -60,7 +60,7 @@ export function buildCuaRunBundle(base, state) {
|
|
|
60
60
|
source: base.source,
|
|
61
61
|
participantPlan: base.participantPlan,
|
|
62
62
|
...(base.rerun === undefined ? {} : { rerun: base.rerun }),
|
|
63
|
-
...(state.
|
|
63
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
64
64
|
...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
|
|
65
65
|
...inProgress,
|
|
66
66
|
});
|
|
@@ -117,6 +117,7 @@ function singleParticipantArgs(base, state) {
|
|
|
117
117
|
}),
|
|
118
118
|
source: base.source,
|
|
119
119
|
...(state.inProgress === undefined ? {} : { inProgress: state.inProgress }),
|
|
120
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
120
121
|
subject: state.aggregateSubject,
|
|
121
122
|
...(desktopTemplate === undefined ? {} : { desktopTemplate }),
|
|
122
123
|
...(outcome?.desktopBrowser === undefined ? {} : { desktopBrowser: outcome.desktopBrowser }),
|
|
@@ -3,7 +3,7 @@ import { type ActorCompletionReason, type ActorStatus, type ActorStopCause } fro
|
|
|
3
3
|
declare const CUA_DIAGNOSTIC_CATEGORIES: readonly ["preview", "participant_outcome", "session_interrupted", "execution_error", "evidence_invalid", "mixed", "unknown"];
|
|
4
4
|
export type CuaDiagnosticCategory = (typeof CUA_DIAGNOSTIC_CATEGORIES)[number];
|
|
5
5
|
/** Closed at both projection and telemetry boundaries. No provider messages are classified. */
|
|
6
|
-
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "unspecified_limit", "mixed", "unknown"];
|
|
6
|
+
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "unspecified_limit", "mixed", "unknown"];
|
|
7
7
|
export type CuaDiagnosticStopCause = (typeof CUA_DIAGNOSTIC_STOP_CAUSES)[number];
|
|
8
8
|
export interface CuaDiagnostics {
|
|
9
9
|
category: CuaDiagnosticCategory;
|
|
@@ -30,6 +30,10 @@ export function cuaParticipantDiagnostics(input) {
|
|
|
30
30
|
const session = input.session;
|
|
31
31
|
if (!session)
|
|
32
32
|
return { category: "unknown" };
|
|
33
|
+
// A scheduled end is the participant's planned ending, so a run whose other participants ended
|
|
34
|
+
// themselves reads as one kind of ending.
|
|
35
|
+
if (session.stopCause === "scheduled_end")
|
|
36
|
+
return { category: "participant_outcome" };
|
|
33
37
|
if (session.stopCause !== undefined) {
|
|
34
38
|
return {
|
|
35
39
|
category: "session_interrupted",
|
|
@@ -3,9 +3,8 @@ import type { CuaFanoutBundleArgs } from "./types.js";
|
|
|
3
3
|
/**
|
|
4
4
|
* Project N>1 fan-out participants into a humanish.run-bundle.v1 (the evidence schema is unchanged; this
|
|
5
5
|
* is a new producer for the multi-stream shape). One sim + one stream per participant; per-participant
|
|
6
|
-
* provenance/session events; a recorded `cua-lab.fanout.plan` event
|
|
7
|
-
*
|
|
8
|
-
* streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
6
|
+
* provenance/session events; a recorded `cua-lab.fanout.plan` event. N-ary verify/Observer already
|
|
7
|
+
* handle multiple streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
9
8
|
* (bundle.ts) sends it to buildSingleParticipantBundle (single-bundle.ts).
|
|
10
9
|
*/
|
|
11
10
|
export declare function buildCuaFanoutBundle(args: CuaFanoutBundleArgs): RunBundle;
|