humanish 0.121.0 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/routes/computer-use/bundle.d.ts +0 -1
- package/dist/routes/computer-use/bundle.js +0 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +2 -12
- package/dist/routes/computer-use/live-phase.d.ts +0 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-execution.d.ts +9 -15
- package/dist/routes/computer-use/participant-execution.js +22 -31
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +1 -2
- package/dist/routes/computer-use/types.d.ts +17 -5
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/host-suspension.d.ts +2 -2
- package/dist/run/host-suspension.js +2 -2
- package/dist/run/judge.d.ts +1 -1
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/preflight.js +11 -1
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/docs/contracts/run-bundle.md +2 -2
- package/docs/contracts/schemas.md +24 -4
- package/package.json +1 -1
package/dist/study/arrivals.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
// When each participant of a computer-use or shared-world run starts
|
|
2
|
-
// participant an offset from the moment the run starts its participants
|
|
3
|
-
// (`participants[].startAfterMs`)
|
|
1
|
+
// When each participant of a computer-use or shared-world run starts, and on computer use when it
|
|
2
|
+
// ends. A study may give each participant an offset from the moment the run starts its participants
|
|
3
|
+
// (`participants[].startAfterMs`) and, on computer use, an end on the same clock
|
|
4
|
+
// (`participants[].endAfterMs`), with a wind-down before it in which each turn tells the
|
|
5
|
+
// participant the time left (`windDownMs`). The plan simulates the start queue to report the schedule and
|
|
4
6
|
// when the last session ends at the latest; the routes start their participants through
|
|
5
7
|
// runOnSchedule, which waits for each one's time and then for a free slot. A participant's desktop
|
|
6
8
|
// is created when it starts, so a late start holds no sandbox before then.
|
|
@@ -20,6 +22,92 @@ export function isStartAfterMs(value) {
|
|
|
20
22
|
value >= 0 &&
|
|
21
23
|
value <= LATEST_START_AFTER_MS);
|
|
22
24
|
}
|
|
25
|
+
/** How long before its end a participant is told the time left, unless it declares `windDownMs`. */
|
|
26
|
+
export const DEFAULT_WIND_DOWN_MS = 10 * 60_000;
|
|
27
|
+
/**
|
|
28
|
+
* Room a scheduled participant's session keeps past its end for the turn in flight and the closing
|
|
29
|
+
* report. The loop ends the session at the end; this keeps the session's own time limit, and the
|
|
30
|
+
* sandbox sized from it, from cutting the closing report short.
|
|
31
|
+
*/
|
|
32
|
+
const SCHEDULED_END_ROOM_MS = 5 * 60_000;
|
|
33
|
+
/**
|
|
34
|
+
* Why a participant's declared end cannot run, or undefined: an end that is not a whole number of
|
|
35
|
+
* milliseconds up to 24 hours, one at or before its start, or a wind-down that is not a whole
|
|
36
|
+
* number of milliseconds or has no end. `path` names the entry, as in "participants[1]".
|
|
37
|
+
*/
|
|
38
|
+
export function windowReason(entry, path) {
|
|
39
|
+
const { endAfterMs, windDownMs } = entry;
|
|
40
|
+
if (windDownMs !== undefined && endAfterMs === undefined)
|
|
41
|
+
return `${path}.windDownMs is how long before its endAfterMs a participant is told the time left, so it needs ${path}.endAfterMs. Add the end, or remove windDownMs.`;
|
|
42
|
+
if (windDownMs !== undefined && !isStartAfterMs(windDownMs))
|
|
43
|
+
return `${path}.windDownMs must be a whole number of milliseconds from 0 to ${LATEST_START_AFTER_MS} (24 hours): how long before its end the participant is told the time left.`;
|
|
44
|
+
if (endAfterMs === undefined)
|
|
45
|
+
return undefined;
|
|
46
|
+
if (!isStartAfterMs(endAfterMs))
|
|
47
|
+
return `${path}.endAfterMs must be a whole number of milliseconds up to ${LATEST_START_AFTER_MS} (24 hours): when the participant's session ends, after the run starts its participants.`;
|
|
48
|
+
const startAfterMs = entry.startAfterMs ?? 0;
|
|
49
|
+
if (endAfterMs <= startAfterMs) {
|
|
50
|
+
const who = entry.id === undefined ? "" : ` ("${entry.id}")`;
|
|
51
|
+
return `${path}.endAfterMs${who} is ${endAfterMs} ms and its startAfterMs is ${startAfterMs} ms, so it would end before it starts. Set endAfterMs after startAfterMs.`;
|
|
52
|
+
}
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A participant's session budget: with a declared end, its window plus SCHEDULED_END_ROOM_MS in
|
|
57
|
+
* place of the study's `sessionMs` (execution.timeoutMs or its default); else `sessionMs`.
|
|
58
|
+
*/
|
|
59
|
+
export function participantSessionMs(participant, sessionMs) {
|
|
60
|
+
if (participant.endAfterMs === undefined)
|
|
61
|
+
return sessionMs;
|
|
62
|
+
return participant.endAfterMs - (participant.startAfterMs ?? 0) + SCHEDULED_END_ROOM_MS;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* When a participant that was due at `scheduledAt` (runOnSchedule's ScheduledStart) ends, on the
|
|
66
|
+
* same clock, or undefined when it declares no end. A participant that waited for a slot still ends
|
|
67
|
+
* at its declared time.
|
|
68
|
+
*/
|
|
69
|
+
export function scheduledEndAt(participant, scheduledAt) {
|
|
70
|
+
if (participant.endAfterMs === undefined)
|
|
71
|
+
return undefined;
|
|
72
|
+
return {
|
|
73
|
+
atMs: scheduledAt - (participant.startAfterMs ?? 0) + participant.endAfterMs,
|
|
74
|
+
windDownMs: participant.windDownMs ?? DEFAULT_WIND_DOWN_MS,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Why the sandbox of the first participant whose window passes `ceilingMs` cannot live that long,
|
|
79
|
+
* or undefined. Its deadline is its session (participantSessionMs) plus `headroomMs`, what the
|
|
80
|
+
* route adds to a session for setup and teardown (sandboxHeadroomMs).
|
|
81
|
+
*/
|
|
82
|
+
export function windowDeadlineRefusal(participants, sandbox, ceilingMs) {
|
|
83
|
+
const inMinutes = (ms) => Math.round(ms / 60_000);
|
|
84
|
+
for (const participant of participants) {
|
|
85
|
+
if (participant.endAfterMs === undefined)
|
|
86
|
+
continue;
|
|
87
|
+
const deadlineMs = participantSessionMs(participant, sandbox.sessionMs) + sandbox.headroomMs;
|
|
88
|
+
if (deadlineMs <= ceilingMs)
|
|
89
|
+
continue;
|
|
90
|
+
const name = participant.id === undefined ? "A participant" : `Participant "${participant.id}"`;
|
|
91
|
+
return `${name} works from +${formatDuration(participant.startAfterMs ?? 0)} to +${formatDuration(participant.endAfterMs)}, so its sandbox needs a ${inMinutes(deadlineMs)}m deadline: the window, ${inMinutes(SCHEDULED_END_ROOM_MS)}m for its closing report and ${inMinutes(sandbox.headroomMs)}m to set up and tear down. A sandbox may not live longer than ${ceilingMs / 60_000}m. Shorten its window.${ceilingAdvice(deadlineMs)}`;
|
|
92
|
+
}
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Each participant's window for `study check`, when any participant declares an end: when it starts
|
|
97
|
+
* and ends and when it is first told the time left, or when it starts and its session budget.
|
|
98
|
+
*/
|
|
99
|
+
export function describeWindows(participants, sessionMs) {
|
|
100
|
+
if (!participants.some((participant) => participant.endAfterMs !== undefined))
|
|
101
|
+
return undefined;
|
|
102
|
+
const windows = participants.map((participant) => {
|
|
103
|
+
const start = `${participant.id ?? "participant"} from +${formatDuration(participant.startAfterMs ?? 0)}`;
|
|
104
|
+
if (participant.endAfterMs === undefined)
|
|
105
|
+
return `${start} for up to ${formatDuration(sessionMs)}`;
|
|
106
|
+
const told = Math.max(participant.startAfterMs ?? 0, participant.endAfterMs - (participant.windDownMs ?? DEFAULT_WIND_DOWN_MS));
|
|
107
|
+
return `${start} to +${formatDuration(participant.endAfterMs)}, told the time left from +${formatDuration(told)}`;
|
|
108
|
+
});
|
|
109
|
+
return `${windows.join("; ")}.`;
|
|
110
|
+
}
|
|
23
111
|
/** The refusal of a start offset that is not one, naming its `field`. */
|
|
24
112
|
export function startAfterMessage(field) {
|
|
25
113
|
return `${field} must be a whole number of milliseconds from 0 to ${LATEST_START_AFTER_MS} (24 hours): how long after the run starts its participants this one starts.`;
|
|
@@ -41,11 +129,13 @@ export function rosterStartsReason(roster) {
|
|
|
41
129
|
* The schedule a roster's declared offsets give under the queue (`undefined` is 0: the participant
|
|
42
130
|
* starts with the run). Participants start in schedule order, by offset and then roster position,
|
|
43
131
|
* each at its time or when a slot frees, whichever is later; this is the rule runOnSchedule
|
|
44
|
-
* follows.
|
|
132
|
+
* follows. `sessionsMs` gives a participant with a declared end its own session budget
|
|
133
|
+
* (participantSessionMs); the others hold their slot for `queue.sessionMs`.
|
|
45
134
|
*/
|
|
46
|
-
export function planArrivals(offsets, queue) {
|
|
135
|
+
export function planArrivals(offsets, queue, sessionsMs = []) {
|
|
47
136
|
const due = offsets.map((offset) => offset ?? 0);
|
|
48
|
-
const
|
|
137
|
+
const session = (index) => sessionsMs[index] ?? queue.sessionMs;
|
|
138
|
+
const starts = queueStarts(due, queue, session);
|
|
49
139
|
let waiting = 0;
|
|
50
140
|
let longestWaitMs = 0;
|
|
51
141
|
for (const [index, start] of starts.entries()) {
|
|
@@ -58,10 +148,10 @@ export function planArrivals(offsets, queue) {
|
|
|
58
148
|
declared: offsets.some((offset) => offset !== undefined),
|
|
59
149
|
firstStartMs: Math.min(...due),
|
|
60
150
|
lastStartMs: Math.max(...due),
|
|
61
|
-
peak: mostAtOnce(starts,
|
|
151
|
+
peak: mostAtOnce(starts, session),
|
|
62
152
|
waiting,
|
|
63
153
|
longestWaitMs,
|
|
64
|
-
lastEndMs: Math.max(...starts) +
|
|
154
|
+
lastEndMs: Math.max(...starts.map((start, index) => start + session(index))),
|
|
65
155
|
sessionMs: queue.sessionMs,
|
|
66
156
|
};
|
|
67
157
|
}
|
|
@@ -83,7 +173,7 @@ export function describeArrivals(arrivals) {
|
|
|
83
173
|
* roster in waves.
|
|
84
174
|
*/
|
|
85
175
|
export function planStarts(participants, queue, lowered) {
|
|
86
|
-
const arrivals = planArrivals(participants.map((participant) => participant.startAfterMs), queue);
|
|
176
|
+
const arrivals = planArrivals(participants.map((participant) => participant.startAfterMs), queue, participants.map((participant) => participantSessionMs(participant, queue.sessionMs)));
|
|
87
177
|
return { arrivals, warnings: startQueueWarnings(arrivals, queue, lowered) };
|
|
88
178
|
}
|
|
89
179
|
// A roster that starts together keeps `lowered`. A declared schedule spreads its participants
|
|
@@ -126,8 +216,8 @@ export function servedScheduleRefusal(arrivals, sandbox, ceilingMs) {
|
|
|
126
216
|
function scheduleOrder(due) {
|
|
127
217
|
return due.map((_, index) => index).sort((a, b) => due[a] - due[b] || a - b);
|
|
128
218
|
}
|
|
129
|
-
/** Each participant's start in roster order, when
|
|
130
|
-
function queueStarts(due, queue) {
|
|
219
|
+
/** Each participant's start in roster order, when each session holds its slot for `session(index)`. */
|
|
220
|
+
function queueStarts(due, queue, session) {
|
|
131
221
|
const starts = [...due];
|
|
132
222
|
const queued = scheduleOrder(due).filter((index) => index !== queue.ownSlot);
|
|
133
223
|
const pooled = queue.ownSlot === undefined ? queue.slots : queue.slots - 1;
|
|
@@ -137,16 +227,16 @@ function queueStarts(due, queue) {
|
|
|
137
227
|
const slot = free.indexOf(Math.min(...free));
|
|
138
228
|
const start = Math.max(due[index], free[slot]);
|
|
139
229
|
starts[index] = start;
|
|
140
|
-
free[slot] = start +
|
|
230
|
+
free[slot] = start + session(index);
|
|
141
231
|
}
|
|
142
232
|
return starts;
|
|
143
233
|
}
|
|
144
|
-
function mostAtOnce(starts,
|
|
234
|
+
function mostAtOnce(starts, session) {
|
|
145
235
|
// An end sorts before a start at the same instant: that slot is free again.
|
|
146
236
|
const changes = starts
|
|
147
|
-
.flatMap((start) => [
|
|
237
|
+
.flatMap((start, index) => [
|
|
148
238
|
{ at: start, by: 1 },
|
|
149
|
-
{ at: start +
|
|
239
|
+
{ at: start + session(index), by: -1 },
|
|
150
240
|
])
|
|
151
241
|
.sort((a, b) => a.at - b.at || a.by - b.by);
|
|
152
242
|
let live = 0;
|
package/dist/study/keys.d.ts
CHANGED
|
@@ -19,6 +19,9 @@ export declare const PARTICIPANT_ENTRY: {
|
|
|
19
19
|
entry: true;
|
|
20
20
|
host: true;
|
|
21
21
|
startAfterMs: true;
|
|
22
|
+
endAfterMs: true;
|
|
23
|
+
windDownMs: true;
|
|
24
|
+
idleWaitMs: true;
|
|
22
25
|
reasoningEffort: true;
|
|
23
26
|
stopWhen: {
|
|
24
27
|
any: {
|
|
@@ -110,6 +113,9 @@ export declare const ACTOR: {
|
|
|
110
113
|
maxOutputTokens: true;
|
|
111
114
|
maxWaitMs: true;
|
|
112
115
|
idleWaitMs: true;
|
|
116
|
+
clock: {
|
|
117
|
+
timeZone: true;
|
|
118
|
+
};
|
|
113
119
|
localAgent: true;
|
|
114
120
|
reasoningEffort: true;
|
|
115
121
|
stopWhen: {
|
package/dist/study/keys.js
CHANGED
|
@@ -40,6 +40,9 @@ export const PARTICIPANT_ENTRY = {
|
|
|
40
40
|
entry: true,
|
|
41
41
|
host: true,
|
|
42
42
|
startAfterMs: true,
|
|
43
|
+
endAfterMs: true,
|
|
44
|
+
windDownMs: true,
|
|
45
|
+
idleWaitMs: true,
|
|
43
46
|
reasoningEffort: true,
|
|
44
47
|
stopWhen: STOP_WHEN,
|
|
45
48
|
dwell: DWELL,
|
|
@@ -97,6 +100,7 @@ export const ACTOR = {
|
|
|
97
100
|
maxOutputTokens: true,
|
|
98
101
|
maxWaitMs: true,
|
|
99
102
|
idleWaitMs: true,
|
|
103
|
+
clock: { timeZone: true },
|
|
100
104
|
localAgent: true,
|
|
101
105
|
reasoningEffort: true,
|
|
102
106
|
stopWhen: STOP_WHEN,
|
package/dist/study/migrate/v2.js
CHANGED
|
@@ -92,12 +92,15 @@ const V2_KEYS = {
|
|
|
92
92
|
actors: {
|
|
93
93
|
type: true,
|
|
94
94
|
count: true,
|
|
95
|
-
//
|
|
96
|
-
lanes: without(PARTICIPANT_ENTRY, "startAfterMs"),
|
|
95
|
+
// The participant's window and idle wait arrived after v2, so a v2 participant cannot carry them.
|
|
96
|
+
lanes: without(PARTICIPANT_ENTRY, "startAfterMs", "endAfterMs", "windDownMs", "idleWaitMs"),
|
|
97
97
|
// Roster groups are participants with a count; the parser expanded them into `lanes[]`.
|
|
98
|
-
roster: {
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
roster: {
|
|
99
|
+
...without(PARTICIPANT_ENTRY, "startAfterMs", "endAfterMs", "windDownMs", "idleWaitMs"),
|
|
100
|
+
count: true,
|
|
101
|
+
},
|
|
102
|
+
// maxWaitMs, idleWaitMs and clock arrived after v2, so a v2 file cannot carry them.
|
|
103
|
+
...without(ACTOR, "type", "maxWaitMs", "idleWaitMs", "clock"),
|
|
101
104
|
laneFocus: { id: true, label: true, instruction: true },
|
|
102
105
|
},
|
|
103
106
|
execution: withAfter(EXECUTION, "egressAllow", { caps: CAPS }),
|
|
@@ -25,7 +25,7 @@ export interface ParticipantPaths {
|
|
|
25
25
|
* Parse a participant list into a fan-out roster (computer-use E2B route): `participants` in a
|
|
26
26
|
* study, `actors[0].lanes` in the v2 file migrate reads, each named in messages by `paths`.
|
|
27
27
|
* Structural only: each entry is `{ id?, actorType?, surface?, caseGroup?, persona?, device?,
|
|
28
|
-
* instruction?, target?, entry?, host?, startAfterMs?, reasoningEffort?, stopWhen?, dwell? }`.
|
|
28
|
+
* instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, reasoningEffort?, stopWhen?, dwell? }`.
|
|
29
29
|
* Participant ids (when declared) must be public-safe path tokens and unique; grouping metadata
|
|
30
30
|
* must be public-safe tokens; a device must be a known preset name. The
|
|
31
31
|
* route-scoped cross-validation (device XOR raw resolution, targets, the participant bound) runs in parseStudy
|
|
@@ -7,7 +7,7 @@ import { entryCredentialReason, urlCredentialReason } from "./url-credentials.js
|
|
|
7
7
|
import { registeredComputerUseActors, registeredScriptedBrowserActors, registeredTerminalActors, } from "../routing.js";
|
|
8
8
|
import { invalid, str } from "./values.js";
|
|
9
9
|
import { isRecord } from "../../run/type-guards.js";
|
|
10
|
-
import { isStartAfterMs, startAfterMessage } from "../arrivals.js";
|
|
10
|
+
import { isStartAfterMs, startAfterMessage, windowReason } from "../arrivals.js";
|
|
11
11
|
// A participant id interpolates into its evidence paths (screenshots/<id>/, actors/<id>.json), so
|
|
12
12
|
// it must be a public-safe path token, same shape as a study id.
|
|
13
13
|
export const PARTICIPANT_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
@@ -63,6 +63,12 @@ export function parseActorFields(entry, path) {
|
|
|
63
63
|
actor.maxWaitMs = entry.maxWaitMs;
|
|
64
64
|
if (typeof entry.idleWaitMs === "number")
|
|
65
65
|
actor.idleWaitMs = entry.idleWaitMs;
|
|
66
|
+
if (entry.clock !== undefined) {
|
|
67
|
+
const timeZone = isRecord(entry.clock) ? entry.clock.timeZone : undefined;
|
|
68
|
+
if (typeof timeZone !== "string" || !isTimeZone(timeZone))
|
|
69
|
+
return invalid(`${path}.clock must be { timeZone }, with timeZone a time zone name such as "Europe/London": the participant's local time, which each turn tells it.`);
|
|
70
|
+
actor.clock = { timeZone };
|
|
71
|
+
}
|
|
66
72
|
const localAgent = str(entry.localAgent);
|
|
67
73
|
if (entry.localAgent !== undefined) {
|
|
68
74
|
if (localAgent !== "codex" && localAgent !== "claude") {
|
|
@@ -97,7 +103,7 @@ export function parseActorFields(entry, path) {
|
|
|
97
103
|
* Parse a participant list into a fan-out roster (computer-use E2B route): `participants` in a
|
|
98
104
|
* study, `actors[0].lanes` in the v2 file migrate reads, each named in messages by `paths`.
|
|
99
105
|
* Structural only: each entry is `{ id?, actorType?, surface?, caseGroup?, persona?, device?,
|
|
100
|
-
* instruction?, target?, entry?, host?, startAfterMs?, reasoningEffort?, stopWhen?, dwell? }`.
|
|
106
|
+
* instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, reasoningEffort?, stopWhen?, dwell? }`.
|
|
101
107
|
* Participant ids (when declared) must be public-safe path tokens and unique; grouping metadata
|
|
102
108
|
* must be public-safe tokens; a device must be a known preset name. The
|
|
103
109
|
* route-scoped cross-validation (device XOR raw resolution, targets, the participant bound) runs in parseStudy
|
|
@@ -108,13 +114,13 @@ export function parseParticipantEntries(raw, paths) {
|
|
|
108
114
|
return { ok: true, value: undefined };
|
|
109
115
|
}
|
|
110
116
|
if (!Array.isArray(raw) || raw.length === 0) {
|
|
111
|
-
return invalid(`${paths.list} must be a non-empty array of participant objects ({ id?, actorType?, surface?, caseGroup?, persona?, device?, instruction?, target?, entry?, host?, startAfterMs?, reasoningEffort?, stopWhen?, dwell? }) when set.`);
|
|
117
|
+
return invalid(`${paths.list} must be a non-empty array of participant objects ({ id?, actorType?, surface?, caseGroup?, persona?, device?, instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, reasoningEffort?, stopWhen?, dwell? }) when set.`);
|
|
112
118
|
}
|
|
113
119
|
const entries = [];
|
|
114
120
|
const seenIds = new Set();
|
|
115
121
|
for (const [entryIndex, entry] of raw.entries()) {
|
|
116
122
|
if (!isRecord(entry)) {
|
|
117
|
-
return invalid(`${paths.entry(entryIndex)} must be an object ({ id?, actorType?, surface?, caseGroup?, persona?, device?, instruction?, target?, entry?, host?, startAfterMs?, reasoningEffort?, stopWhen?, dwell? }).`);
|
|
123
|
+
return invalid(`${paths.entry(entryIndex)} must be an object ({ id?, actorType?, surface?, caseGroup?, persona?, device?, instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, reasoningEffort?, stopWhen?, dwell? }).`);
|
|
118
124
|
}
|
|
119
125
|
const parsedEntry = {};
|
|
120
126
|
const id = str(entry.id);
|
|
@@ -205,6 +211,19 @@ export function parseParticipantEntries(raw, paths) {
|
|
|
205
211
|
return invalid(startAfterMessage(`${paths.entry(entryIndex)}.startAfterMs`));
|
|
206
212
|
parsedEntry.startAfterMs = entry.startAfterMs;
|
|
207
213
|
}
|
|
214
|
+
const window = windowReason({ ...parsedEntry, endAfterMs: entry.endAfterMs, windDownMs: entry.windDownMs }, paths.entry(entryIndex));
|
|
215
|
+
if (window !== undefined)
|
|
216
|
+
return invalid(window);
|
|
217
|
+
if (typeof entry.endAfterMs === "number")
|
|
218
|
+
parsedEntry.endAfterMs = entry.endAfterMs;
|
|
219
|
+
if (typeof entry.windDownMs === "number")
|
|
220
|
+
parsedEntry.windDownMs = entry.windDownMs;
|
|
221
|
+
// Checked against actor.maxWaitMs once the actor is read (waitLimitValidationReason).
|
|
222
|
+
const idleWait = waitFieldsReason({ idleWaitMs: entry.idleWaitMs }, `${paths.entry(entryIndex)}.`);
|
|
223
|
+
if (idleWait)
|
|
224
|
+
return invalid(idleWait);
|
|
225
|
+
if (typeof entry.idleWaitMs === "number")
|
|
226
|
+
parsedEntry.idleWaitMs = entry.idleWaitMs;
|
|
208
227
|
entries.push(parsedEntry);
|
|
209
228
|
}
|
|
210
229
|
return { ok: true, value: entries };
|
|
@@ -376,3 +395,13 @@ function parseEntryMetadata(raw, field) {
|
|
|
376
395
|
}
|
|
377
396
|
return { ok: true, value };
|
|
378
397
|
}
|
|
398
|
+
/** Whether Intl knows `name` as a time zone. */
|
|
399
|
+
function isTimeZone(name) {
|
|
400
|
+
try {
|
|
401
|
+
new Intl.DateTimeFormat("en-GB", { timeZone: name });
|
|
402
|
+
return true;
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
@@ -38,11 +38,20 @@ interface DesktopParticipant extends Participant {
|
|
|
38
38
|
readonly maxOutputTokens?: number;
|
|
39
39
|
/** actor.maxWaitMs: the longest one wait action lasts. */
|
|
40
40
|
readonly maxWaitMs?: number;
|
|
41
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* The participant's idleWaitMs, else actor.idleWaitMs: how long a wait with no duration lasts
|
|
43
|
+
* while the participant only waits.
|
|
44
|
+
*/
|
|
42
45
|
readonly idleWaitMs?: number;
|
|
46
|
+
/** actor.clock.timeZone: the participant's local time zone, which each turn tells it the time in. */
|
|
47
|
+
readonly timeZone?: string;
|
|
43
48
|
};
|
|
44
49
|
/** When it starts, in ms after the run starts its participants; absent, with the run. */
|
|
45
50
|
readonly startAfterMs?: number;
|
|
51
|
+
/** Computer use: when it ends, on the same clock; absent, it runs for the study's session budget. */
|
|
52
|
+
readonly endAfterMs?: number;
|
|
53
|
+
/** With endAfterMs: how long before the end each turn tells it the time left (default applied). */
|
|
54
|
+
readonly windDownMs?: number;
|
|
46
55
|
}
|
|
47
56
|
/** Only independent computer-use participants consume a task protocol and their own target. */
|
|
48
57
|
export interface ComputerUseParticipant extends DesktopParticipant {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// Who takes part in a study: one record per participant, built from the declared roster or a count.
|
|
2
2
|
// Computer-use and shared-world participants share the desktop fields; each kind adds its own.
|
|
3
|
+
import { DEFAULT_WIND_DOWN_MS } from "./arrivals.js";
|
|
3
4
|
import { resolveParticipantDevice } from "./device-presets.js";
|
|
4
5
|
import { computerUseParticipantCount, isSharedWorldComposition, participantIdAt, } from "./routing.js";
|
|
5
6
|
import { participantList, declaredParticipantCount, participantInstruction, } from "./study-fields.js";
|
|
@@ -13,7 +14,8 @@ function desktopParticipant(config, entry, index, kind, focus) {
|
|
|
13
14
|
const reasoningEffort = entry?.reasoningEffort ?? actor?.reasoningEffort;
|
|
14
15
|
const maxOutputTokens = actor?.maxOutputTokens;
|
|
15
16
|
const maxWaitMs = actor?.maxWaitMs;
|
|
16
|
-
const idleWaitMs = actor?.idleWaitMs;
|
|
17
|
+
const idleWaitMs = entry?.idleWaitMs ?? actor?.idleWaitMs;
|
|
18
|
+
const timeZone = actor?.clock?.timeZone;
|
|
17
19
|
return {
|
|
18
20
|
id: participantIdAt(index, entry?.id, kind),
|
|
19
21
|
index,
|
|
@@ -35,8 +37,15 @@ function desktopParticipant(config, entry, index, kind, focus) {
|
|
|
35
37
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
36
38
|
...(maxWaitMs === undefined ? {} : { maxWaitMs }),
|
|
37
39
|
...(idleWaitMs === undefined ? {} : { idleWaitMs }),
|
|
40
|
+
...(timeZone === undefined ? {} : { timeZone }),
|
|
38
41
|
},
|
|
39
42
|
...(entry?.startAfterMs === undefined ? {} : { startAfterMs: entry.startAfterMs }),
|
|
43
|
+
...(entry?.endAfterMs === undefined
|
|
44
|
+
? {}
|
|
45
|
+
: {
|
|
46
|
+
endAfterMs: entry.endAfterMs,
|
|
47
|
+
windDownMs: entry.windDownMs ?? DEFAULT_WIND_DOWN_MS,
|
|
48
|
+
}),
|
|
40
49
|
};
|
|
41
50
|
}
|
|
42
51
|
/**
|
package/dist/study/preflight.js
CHANGED
|
@@ -6,7 +6,7 @@ import {} from "./types.js";
|
|
|
6
6
|
import { runPublicPreviewPreflight, runSandboxLoopbackPreflight } from "./preflight-probes.js";
|
|
7
7
|
import { digest, fail, finalize, STUDY_CHECK_SCHEMA } from "./preflight-result.js";
|
|
8
8
|
import { planStudy, routeOf } from "./plan.js";
|
|
9
|
-
import { describeArrivals } from "./arrivals.js";
|
|
9
|
+
import { describeArrivals, describeWindows } from "./arrivals.js";
|
|
10
10
|
import { resolveStudyManifest } from "./discover.js";
|
|
11
11
|
import { participantList } from "./study-fields.js";
|
|
12
12
|
import { isLocalBrowserStudy } from "../substrates/local/runtime-config.js";
|
|
@@ -78,6 +78,9 @@ export async function runStudyPreflight(options) {
|
|
|
78
78
|
const schedule = scheduleCheck(planned.planned.plan);
|
|
79
79
|
if (schedule !== undefined)
|
|
80
80
|
ctx.checks.push(schedule);
|
|
81
|
+
const windows = windowsCheck(planned.planned.plan);
|
|
82
|
+
if (windows !== undefined)
|
|
83
|
+
ctx.checks.push(windows);
|
|
81
84
|
const machine = machineCheck(ctx.config);
|
|
82
85
|
// A local study's app and desktops run on this machine, so the hosted probes do not apply.
|
|
83
86
|
if (reachability !== "metadata" && isLocalBrowserStudy(ctx.config))
|
|
@@ -130,6 +133,13 @@ function scheduleCheck(plan) {
|
|
|
130
133
|
return undefined;
|
|
131
134
|
return { name: "schedule", ok: true, message: describeArrivals(plan.arrivals) };
|
|
132
135
|
}
|
|
136
|
+
/** Each participant's window, for a computer-use study in which a participant declares an end. */
|
|
137
|
+
function windowsCheck(plan) {
|
|
138
|
+
if (plan.route !== "computer-use")
|
|
139
|
+
return undefined;
|
|
140
|
+
const message = describeWindows(plan.runner.participants, plan.sessionBudgetMs);
|
|
141
|
+
return message === undefined ? undefined : { name: "windows", ok: true, message };
|
|
142
|
+
}
|
|
133
143
|
/** What study check leaves to doctor: this machine's setup for the study, and the command. */
|
|
134
144
|
function machineCheck(config) {
|
|
135
145
|
const command = cli(`doctor --study ${config.id}`);
|
package/dist/study/types.d.ts
CHANGED
|
@@ -325,6 +325,24 @@ export interface StudyParticipantEntry {
|
|
|
325
325
|
* Absent, the participant starts with the run. src/study/arrivals.ts schedules the start.
|
|
326
326
|
*/
|
|
327
327
|
startAfterMs?: number;
|
|
328
|
+
/**
|
|
329
|
+
* Computer use: when this participant's session ends, in milliseconds after the run starts its
|
|
330
|
+
* participants, on the same clock as startAfterMs and after it. At the end the participant is
|
|
331
|
+
* asked for its closing report and the session ends as passed, with stop cause `scheduled_end`.
|
|
332
|
+
* Its session lasts the window plus 5 minutes in place of execution.timeoutMs, and its sandbox
|
|
333
|
+
* is sized from that.
|
|
334
|
+
*/
|
|
335
|
+
endAfterMs?: number;
|
|
336
|
+
/**
|
|
337
|
+
* With endAfterMs: how long before the end each turn tells the participant the end time and the
|
|
338
|
+
* minutes left. Default 600000 (10 minutes).
|
|
339
|
+
*/
|
|
340
|
+
windDownMs?: number;
|
|
341
|
+
/**
|
|
342
|
+
* Computer use and shared world: this participant's actor.idleWaitMs, how long a wait that names
|
|
343
|
+
* no duration lasts while it only waits. Same range, and no longer than actor.maxWaitMs.
|
|
344
|
+
*/
|
|
345
|
+
idleWaitMs?: number;
|
|
328
346
|
}
|
|
329
347
|
export interface StudyActor {
|
|
330
348
|
/**
|
|
@@ -369,6 +387,14 @@ export interface StudyActor {
|
|
|
369
387
|
* 500 ms. OpenAI's computer-use `wait` never names a duration.
|
|
370
388
|
*/
|
|
371
389
|
idleWaitMs?: number;
|
|
390
|
+
/**
|
|
391
|
+
* Computer use: the participant's clock. `timeZone`, a time zone name such as "Europe/London", adds
|
|
392
|
+
* the local time to each turn, for missions that refer to appointment times, and gives a
|
|
393
|
+
* scheduled end (participants[].endAfterMs) in that zone. A hosted desktop shows no clock.
|
|
394
|
+
*/
|
|
395
|
+
clock?: {
|
|
396
|
+
timeZone: string;
|
|
397
|
+
};
|
|
372
398
|
/**
|
|
373
399
|
* `local-agent` only: which locally signed-in coding agent is the brain. Absent = codex.
|
|
374
400
|
*
|
|
@@ -37,8 +37,9 @@ export declare function cloneTargetValidationReason(config: StudyConfig): string
|
|
|
37
37
|
/** Refuse a claimed output bound when the route cannot pass it to the first-party provider. */
|
|
38
38
|
export declare function outputTokenLimitValidationReason(config: StudyConfig): string | null;
|
|
39
39
|
/**
|
|
40
|
-
* Refuse a wait field out of range, or an idle wait longer than the longest wait
|
|
41
|
-
*
|
|
40
|
+
* Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
|
|
41
|
+
* each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
|
|
42
|
+
* wait actions the fields are inert (warnings.ts).
|
|
42
43
|
*/
|
|
43
44
|
export declare function waitLimitValidationReason(config: StudyConfig): string | null;
|
|
44
45
|
/**
|
package/dist/study/validation.js
CHANGED
|
@@ -6,7 +6,7 @@ import { isHttpUrl, isLoopbackUrl } from "./parse/subject.js";
|
|
|
6
6
|
import { declaredTargets } from "./plan-participants.js";
|
|
7
7
|
import { actorResolvesToComputerUse, computerUseParticipantCount, participantCountReason, registeredComputerUseActors, resolveEntryUrl, isComputerUseComposition, isScriptedBrowserComposition, isSharedWorldComposition, isTerminalProductComposition, } from "./routing.js";
|
|
8
8
|
import { participantList } from "./study-fields.js";
|
|
9
|
-
import { isStartAfterMs, rosterStartsReason, startAfterMessage } from "./arrivals.js";
|
|
9
|
+
import { isStartAfterMs, rosterStartsReason, startAfterMessage, windowReason } from "./arrivals.js";
|
|
10
10
|
/**
|
|
11
11
|
* Cross-validate the computer-use fan-out declaration (`per-lane-worlds`). Returns the failure
|
|
12
12
|
* message, or null when valid. The parser enforces it, and runStudyWith checks it again for a config
|
|
@@ -82,6 +82,9 @@ function rosterStructuralValidationReason(config) {
|
|
|
82
82
|
if (entry.startAfterMs !== undefined && !isStartAfterMs(entry.startAfterMs)) {
|
|
83
83
|
return startAfterMessage(`participants[${index}].startAfterMs`);
|
|
84
84
|
}
|
|
85
|
+
const window = windowReason(entry, `participants[${index}]`);
|
|
86
|
+
if (window !== undefined)
|
|
87
|
+
return window;
|
|
85
88
|
const id = entry.id;
|
|
86
89
|
if (id === undefined) {
|
|
87
90
|
continue;
|
|
@@ -107,6 +110,10 @@ function rosterStructuralValidationReason(config) {
|
|
|
107
110
|
* runStudyWith takes a config that skipped the parser.
|
|
108
111
|
*/
|
|
109
112
|
export function sharedWorldValidationReason(config) {
|
|
113
|
+
// The subject sandbox's deadline assumes every participant runs for execution.timeoutMs.
|
|
114
|
+
const ending = participantList(config)?.findIndex((entry) => entry?.endAfterMs !== undefined);
|
|
115
|
+
if (ending !== undefined && ending >= 0)
|
|
116
|
+
return `participants[${ending}].endAfterMs ends a participant at a set time on route: computer-use only. A shared-world participant runs for execution.timeoutMs; remove endAfterMs and windDownMs.`;
|
|
110
117
|
return config.subject.source === "app-url"
|
|
111
118
|
? externalPublicSharedWorldValidationReason(config)
|
|
112
119
|
: concurrentSharedWorldValidationReason(config);
|
|
@@ -250,11 +257,22 @@ export function outputTokenLimitValidationReason(config) {
|
|
|
250
257
|
return null;
|
|
251
258
|
}
|
|
252
259
|
/**
|
|
253
|
-
* Refuse a wait field out of range, or an idle wait longer than the longest wait
|
|
254
|
-
*
|
|
260
|
+
* Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
|
|
261
|
+
* each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
|
|
262
|
+
* wait actions the fields are inert (warnings.ts).
|
|
255
263
|
*/
|
|
256
264
|
export function waitLimitValidationReason(config) {
|
|
257
|
-
|
|
265
|
+
const actorReason = config.actor === undefined ? null : waitFieldsReason(config.actor, "actor.");
|
|
266
|
+
if (actorReason !== null)
|
|
267
|
+
return actorReason;
|
|
268
|
+
for (const [index, entry] of (participantList(config) ?? []).entries()) {
|
|
269
|
+
if (entry?.idleWaitMs === undefined)
|
|
270
|
+
continue;
|
|
271
|
+
const reason = waitFieldsReason({ idleWaitMs: entry.idleWaitMs, maxWaitMs: config.actor?.maxWaitMs }, `participants[${index}].`, "actor.");
|
|
272
|
+
if (reason !== null)
|
|
273
|
+
return reason;
|
|
274
|
+
}
|
|
275
|
+
return null;
|
|
258
276
|
}
|
|
259
277
|
/**
|
|
260
278
|
* Shared-world participants share one live app, so at least two must be live at once. The
|
|
@@ -183,7 +183,7 @@ whose message names its start, end, length and offset into the run (`src/run/hos
|
|
|
183
183
|
|
|
184
184
|
On a computer-use run, a participant that failed in the harness or at its session time limit, and
|
|
185
185
|
was in flight across a suspension, likely failed because of it. So did a participant skipped after
|
|
186
|
-
such a failure (the pipeline gate
|
|
186
|
+
such a failure (the pipeline gate). When any did, the outcome's first failure has
|
|
187
187
|
`kind: run` and one sentence, such as "The host was suspended for 6m 5s at +1m; the 34
|
|
188
188
|
participant failures after it are likely its effect.", ahead of each participant's own failure.
|
|
189
189
|
`review.summary` starts with the same sentence, the attributed participants share one gap line,
|
|
@@ -197,7 +197,7 @@ An actor trace may include `stopCause` alongside its unchanged `status`,
|
|
|
197
197
|
`completionReason` and verbatim `reason`. Computer-use sessions distinguish
|
|
198
198
|
`provider_output_limit`, `provider_token_limit`, `time_limit`, `spend_limit`,
|
|
199
199
|
`study_spend_limit`, `adapter_limit`, `provider_incomplete`, `provider_status`,
|
|
200
|
-
`provider_refused_prompt`, `harness_aborted`, and `
|
|
200
|
+
`provider_refused_prompt`, `harness_aborted`, `usage_unreported`, and `scheduled_end`. Absence means the route or
|
|
201
201
|
recording did not retain this precise field; it does not mean the participant
|
|
202
202
|
finished.
|
|
203
203
|
|
|
@@ -227,6 +227,25 @@ The rest of this section names keys by their v3 spelling:
|
|
|
227
227
|
and the most participants at once with every session at its full budget, and the provisioned
|
|
228
228
|
shared world's subject sandbox serves until the last session can end, which the plan checks
|
|
229
229
|
against the sandbox ceiling. A rerun starts its selected participants together;
|
|
230
|
+
- `participants[].endAfterMs` and `windDownMs` (computer-use only): when the participant's
|
|
231
|
+
session ends, in whole milliseconds on the `startAfterMs` clock and after its start, and how
|
|
232
|
+
long before that each turn tells it the end time and the minutes left (default 600,000).
|
|
233
|
+
Refused: an end at or before the start, a `windDownMs` without `endAfterMs`, an end on
|
|
234
|
+
`shared-world`, and a window whose sandbox deadline passes the sandbox ceiling. The
|
|
235
|
+
participant's session budget is its window plus 5 minutes (`participantSessionMs` in
|
|
236
|
+
`src/study/arrivals.ts`) in place of `execution.timeoutMs`, and its sandbox deadline adds the
|
|
237
|
+
route's headroom to that. At the end the loop requests the closing report (`debrief.trigger:
|
|
238
|
+
scheduled_end`) and the session ends `goal_satisfied` with `stopCause: scheduled_end`. Before
|
|
239
|
+
the end, a turn that requests no action does not end the session: the trace records a
|
|
240
|
+
`participant ended before its scheduled end` notice, the next request says the time left, and
|
|
241
|
+
the turn counts toward the idle backstop. Before the end, a turn that only waits or takes
|
|
242
|
+
screenshots counts toward neither backstop streak (`advanceBackstop`'s `waitingIsTheTask`). The
|
|
243
|
+
plan entry carries `endAfterMs`, and `study check` prints a `windows` row;
|
|
244
|
+
- `participants[].idleWaitMs` (computer-use and shared-world, also on `count:` groups): this
|
|
245
|
+
participant's `actor.idleWaitMs`, in the same range and refused above an explicit
|
|
246
|
+
`actor.maxWaitMs` (`waitLimitValidationReason`). Its trace records it in `waitSettings`;
|
|
247
|
+
- `actor.clock.timeZone` (computer-use): a tz database time zone name, such as `Europe/London`. Each turn's context hint carries
|
|
248
|
+
the local time in it, and a scheduled end is given in it (UTC without it);
|
|
230
249
|
- `execution.concurrency` (computer-use E2B routes, including shared-world): a
|
|
231
250
|
cap on participants in flight at once. When omitted, every participant runs
|
|
232
251
|
simultaneously: both routes resolve it when they plan, from the final participant count
|
|
@@ -460,9 +479,9 @@ length 1 at N=1; per-participant status/session/sandbox/subject), and `laneSumma
|
|
|
460
479
|
mirror the first participant and `subject.commit` is unanimity-gated across participants
|
|
461
480
|
(omitted with a divergence warning when participants resolve different commits). At N=1
|
|
462
481
|
the run bundle is byte-stable with the pre-fan-out output; only the result
|
|
463
|
-
projection changed. A fan-out run records a `cua-lab.fanout.plan` bundle event
|
|
464
|
-
|
|
465
|
-
|
|
482
|
+
projection changed. A fan-out run records a `cua-lab.fanout.plan` bundle event, and
|
|
483
|
+
each participant's outcome event (`cua-lab.session.<completionReason>`, `cua-lab.session.error`
|
|
484
|
+
or `cua-lab.session.blocked`) carries its reason;
|
|
466
485
|
`ok = observer.ok ∧ every participant ok ∧ no failed adapter or declared-scorer verdict`,
|
|
467
486
|
where a dry-run participant is ok as synthetic and a live participant must pass
|
|
468
487
|
`participantPassed` (`src/run/judge.ts`): not skipped, status `passed`, no harness
|
|
@@ -1014,7 +1033,8 @@ Core-owned fields:
|
|
|
1014
1033
|
applied, in milliseconds, defaults included. `maxWaitMs` is the longest one
|
|
1015
1034
|
wait action lasts (`actor.maxWaitMs`); a longer wait is shortened to it.
|
|
1016
1035
|
`idleWaitMs` is how long a wait that names no duration lasts in a turn that
|
|
1017
|
-
only waits or takes screenshots (`
|
|
1036
|
+
only waits or takes screenshots (`participants[].idleWaitMs`, else
|
|
1037
|
+
`actor.idleWaitMs`), and `settleWaitMs` how
|
|
1018
1038
|
long such a wait lasts after an action in the same turn. Each `ui_action` for
|
|
1019
1039
|
a wait is titled with the length it lasted. Absent on other routes and on
|
|
1020
1040
|
earlier bundles
|