humanish 0.126.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/actors/computer-use/loop/ending.d.ts +3 -1
- package/dist/actors/computer-use/loop/ending.js +5 -3
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/observer/data.js +8 -5
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-runs.d.ts +9 -9
- package/dist/routes/computer-use/participant-runs.js +52 -86
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +26 -16
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.js +1 -1
- package/dist/routes/computer-use/single-bundle.js +9 -1
- package/dist/routes/computer-use/types.d.ts +2 -3
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +1 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +3 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/judge.d.ts +7 -0
- package/dist/run/judge.js +4 -4
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/study/arrivals.d.ts +84 -31
- package/dist/study/arrivals.js +165 -86
- package/dist/study/bundle-estimate.d.ts +8 -7
- package/dist/study/bundle-estimate.js +131 -70
- package/dist/study/desktop-files.js +4 -4
- package/dist/study/plan-participants.d.ts +4 -9
- package/dist/study/plan-participants.js +2 -14
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +6 -4
- package/dist/study/types.d.ts +2 -1
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +24 -11
- package/package.json +1 -1
|
@@ -1,46 +1,85 @@
|
|
|
1
1
|
// How large a computer-use run's bundle grows, estimated from its plan before anything runs, and
|
|
2
|
-
// whether it would pass a limit a later command holds it to: analysis reads
|
|
3
|
-
// EVIDENCE_LIMITS.sourceBytes, verify reads no file larger than
|
|
4
|
-
// public-safety scan visits at most runEntryLimit(participants)
|
|
2
|
+
// whether it would pass a limit a later command holds it to: analysis reads run.json and each
|
|
3
|
+
// participant's trace up to EVIDENCE_LIMITS.sourceBytes each, verify reads no file larger than
|
|
4
|
+
// RUN_ARTIFACT_MAX_BYTES, and the public-safety scan visits at most runEntryLimit(participants)
|
|
5
|
+
// files and folders.
|
|
6
|
+
//
|
|
7
|
+
// observer/index.html carries every participant's items gzipped, so with busy participants it can
|
|
8
|
+
// pass verify's 32 MiB before run.json passes analysis's 16 MiB: at these rates, 17 participants
|
|
9
|
+
// working 8 hours. observer-data.json carries the same per-request lists as run.json and was within
|
|
10
|
+
// 7 % of its size on every live run measured, so it passes 32 MiB only after run.json passes
|
|
11
|
+
// 16 MiB, and the estimate does not check it.
|
|
5
12
|
import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
|
|
6
13
|
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
7
14
|
import { formatDuration } from "../run/projection.js";
|
|
8
15
|
import { plural } from "../run/text.js";
|
|
9
16
|
import { runEntryLimit } from "../verify/scan-limit.js";
|
|
10
|
-
// Measured on 2026-10-
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// working the whole window; these rates give 12.6 KB, so the estimate errs high, and a limit it
|
|
17
|
-
// passes is a warning.
|
|
17
|
+
// Measured on 2026-10-13 on run bundle v3. Live: the 7 finished hosted computer-use runs kept from
|
|
18
|
+
// the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
|
|
19
|
+
// model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
|
|
20
|
+
// own files. Pace, session length and screenshots: the live computer-use runs kept on the
|
|
21
|
+
// maintainer's machine. A trace's rate a turn on the public site/public/runs/lobby-0927 run
|
|
22
|
+
// (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the page, under the live fit.
|
|
18
23
|
const RATES = Object.freeze({
|
|
19
|
-
/** run.json bytes
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
/** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
|
|
25
|
+
runBytes: 4_400,
|
|
26
|
+
/**
|
|
27
|
+
* run.json bytes for each participant before its turns: its stream, actor summary and closing
|
|
28
|
+
* report request. Live fit: 16,239, plus one request.
|
|
29
|
+
*/
|
|
30
|
+
participantBytes: 16_500,
|
|
31
|
+
/**
|
|
32
|
+
* run.json bytes a model turn adds: the request's entries in `tokenUsage.turns` and
|
|
33
|
+
* `conversation.requests`. Live: 30,623 bytes over 137 requests, 223.5 each.
|
|
34
|
+
*/
|
|
35
|
+
turnRunBytes: 225,
|
|
36
|
+
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
|
|
37
|
+
participantTraceBytes: 5_400,
|
|
38
|
+
/**
|
|
39
|
+
* Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
|
|
40
|
+
* for the `page` each screenshot item now carries. Live fit: 1,965 a turn. Participants with 4
|
|
41
|
+
* or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15 turns, most of them
|
|
42
|
+
* UI actions. Analysis refuses the whole run when one trace passes its limit, so each trace is
|
|
43
|
+
* counted at the busiest participant's rate.
|
|
44
|
+
*/
|
|
45
|
+
turnTraceBytes: 4_700,
|
|
46
|
+
/**
|
|
47
|
+
* Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
|
|
48
|
+
* plus 91 for the page.
|
|
49
|
+
*/
|
|
50
|
+
frameTraceBytes: 500,
|
|
51
|
+
/**
|
|
52
|
+
* Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
|
|
53
|
+
* minutes or more was 10.4.
|
|
54
|
+
*/
|
|
30
55
|
secondsPerTurn: 10,
|
|
31
56
|
/**
|
|
32
57
|
* How long a participant with no declared end is counted, at most its session budget. It ends
|
|
33
|
-
* when it finishes its mission: 90 % of 974
|
|
58
|
+
* when it finishes its mission: 90 % of 974 kept sessions ended within 184 s on 2026-10-12; of
|
|
59
|
+
* the 197 kept on 2026-10-13, 64 % ended within 180 s and 90 % within 325 s. Counting 325 s would
|
|
60
|
+
* add under 0.4 MiB to the run.json of 100 participants.
|
|
34
61
|
*/
|
|
35
62
|
openSessionMs: 3 * 60_000,
|
|
36
|
-
/** observer
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
63
|
+
/** observer/index.html's app before any run data: 1,199,831 bytes on the dry runs. */
|
|
64
|
+
observerAppBytes: 1_200_000,
|
|
65
|
+
/**
|
|
66
|
+
* The page's compact copy of observer-data.json, as a share of run.json: 0.64 to 0.81 whole on
|
|
67
|
+
* the live runs, 0.49 for a participant's part, and 0.42 for the per-request entries.
|
|
68
|
+
*/
|
|
69
|
+
observerPageDataShare: 0.5,
|
|
70
|
+
/**
|
|
71
|
+
* The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114 on the
|
|
72
|
+
* live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes a
|
|
73
|
+
* turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
74
|
+
* synthetic 8-hour run.
|
|
75
|
+
*/
|
|
76
|
+
observerPageItemsShare: 0.12,
|
|
77
|
+
/** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
|
|
42
78
|
screenshotBytes: 110_000,
|
|
43
|
-
/**
|
|
79
|
+
/**
|
|
80
|
+
* Run files and folders besides the participants': 10 on a dry run, and live the sandbox
|
|
81
|
+
* receipts, `actors/` and an analysis's 9.
|
|
82
|
+
*/
|
|
44
83
|
runEntries: 20,
|
|
45
84
|
/** A participant's entries besides its turn screenshots: its trace file, its screenshot folder, its first frame. */
|
|
46
85
|
participantEntries: 3,
|
|
@@ -56,45 +95,53 @@ const MIB = 1024 * 1024;
|
|
|
56
95
|
export function estimateBundle(participants, sessionMs) {
|
|
57
96
|
let turns = 0;
|
|
58
97
|
let frames = 0;
|
|
98
|
+
let traces = 0;
|
|
99
|
+
let largestTrace = 0;
|
|
59
100
|
let longestWindowMs = 0;
|
|
60
101
|
// The longest any participant without an end is counted, for the messages.
|
|
61
102
|
let openMs = Math.min(sessionMs, RATES.openSessionMs);
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
103
|
+
let scheduled = 0;
|
|
104
|
+
for (const { schedule, limits } of participants) {
|
|
105
|
+
const { end } = schedule;
|
|
106
|
+
const budgetMs = end?.windowMs ?? sessionMs;
|
|
107
|
+
let countedMs;
|
|
108
|
+
if (end?.kind === "scheduled_end") {
|
|
109
|
+
scheduled += 1;
|
|
110
|
+
countedMs = end.windowMs;
|
|
111
|
+
longestWindowMs = Math.max(longestWindowMs, countedMs);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
countedMs = Math.min(budgetMs, RATES.openSessionMs);
|
|
71
115
|
openMs = Math.max(openMs, countedMs);
|
|
72
|
-
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
116
|
+
}
|
|
117
|
+
const own = Math.ceil(countedMs / (RATES.secondsPerTurn * 1000));
|
|
118
|
+
const dwell = limits.dwell;
|
|
119
|
+
const ownFrames = dwell === undefined ? 0 : Math.floor(Math.min(dwell.ms, budgetMs) / dwell.everyMs) + 1;
|
|
120
|
+
const trace = RATES.participantTraceBytes + own * RATES.turnTraceBytes + ownFrames * RATES.frameTraceBytes;
|
|
121
|
+
turns += own;
|
|
122
|
+
frames += ownFrames;
|
|
123
|
+
traces += trace;
|
|
124
|
+
largestTrace = Math.max(largestTrace, trace);
|
|
76
125
|
}
|
|
77
126
|
const count = participants.length;
|
|
78
|
-
const runFileBytes = RATES.runBytes +
|
|
79
|
-
count * RATES.participantBytes +
|
|
80
|
-
turns * RATES.turnBytes +
|
|
81
|
-
frames * RATES.frameBytes;
|
|
82
|
-
const observerDataBytes = runFileBytes + count * RATES.observerDataParticipantBytes;
|
|
83
|
-
const pageBytes = RATES.observerPageBytes + Math.round(RATES.observerPageShare * runFileBytes);
|
|
127
|
+
const runFileBytes = RATES.runBytes + count * RATES.participantBytes + turns * RATES.turnRunBytes;
|
|
84
128
|
const screenshots = count + turns + frames;
|
|
85
|
-
const
|
|
129
|
+
const observerPageBytes = Math.round(RATES.observerAppBytes +
|
|
130
|
+
RATES.observerPageDataShare * runFileBytes +
|
|
131
|
+
RATES.observerPageItemsShare * traces);
|
|
86
132
|
return {
|
|
87
133
|
runFileBytes,
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
134
|
+
largestTraceBytes: largestTrace,
|
|
135
|
+
observerPageBytes,
|
|
136
|
+
// observer-data.json is about run.json's size.
|
|
137
|
+
totalBytes: observerPageBytes + screenshots * RATES.screenshotBytes + traces + 2 * runFileBytes,
|
|
91
138
|
entries: RATES.runEntries + count * RATES.participantEntries + turns + frames,
|
|
92
139
|
entryLimit: runEntryLimit(count),
|
|
93
140
|
basis: {
|
|
94
141
|
participants: count,
|
|
95
|
-
scheduled
|
|
142
|
+
scheduled,
|
|
96
143
|
longestWindowMs,
|
|
97
|
-
open: count - scheduled
|
|
144
|
+
open: count - scheduled,
|
|
98
145
|
openMs,
|
|
99
146
|
dwell: frames > 0,
|
|
100
147
|
},
|
|
@@ -102,23 +149,31 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
102
149
|
}
|
|
103
150
|
/**
|
|
104
151
|
* The warning for a run whose estimate passes a limit, or undefined: each limit it passes, what the
|
|
105
|
-
* estimate counted,
|
|
152
|
+
* estimate counted, and what to change.
|
|
106
153
|
*/
|
|
107
154
|
export function bundleLimitReason(estimate) {
|
|
155
|
+
const runFile = estimate.runFileBytes > EVIDENCE_LIMITS.sourceBytes;
|
|
156
|
+
const page = estimate.observerPageBytes > RUN_ARTIFACT_MAX_BYTES;
|
|
157
|
+
const entries = estimate.entries > estimate.entryLimit;
|
|
108
158
|
const passed = [
|
|
109
|
-
...(
|
|
159
|
+
...(runFile
|
|
110
160
|
? [
|
|
111
161
|
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads.`,
|
|
112
162
|
]
|
|
113
163
|
: []),
|
|
114
|
-
...(estimate.
|
|
164
|
+
...(estimate.largestTraceBytes > EVIDENCE_LIMITS.sourceBytes
|
|
165
|
+
? [
|
|
166
|
+
`One participant's trace may grow to about ${mib(estimate.largestTraceBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads of each trace.`,
|
|
167
|
+
]
|
|
168
|
+
: []),
|
|
169
|
+
...(page
|
|
115
170
|
? [
|
|
116
|
-
`
|
|
171
|
+
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file.`,
|
|
117
172
|
]
|
|
118
173
|
: []),
|
|
119
|
-
...(
|
|
174
|
+
...(entries
|
|
120
175
|
? [
|
|
121
|
-
`
|
|
176
|
+
`The run may hold about ${count(estimate.entries)} files and folders, past the ${count(estimate.entryLimit)} the public-safety scan reads in a run of ${plural(estimate.basis.participants, "participant")}.`,
|
|
122
177
|
]
|
|
123
178
|
: []),
|
|
124
179
|
];
|
|
@@ -128,7 +183,7 @@ export function bundleLimitReason(estimate) {
|
|
|
128
183
|
const counted = [
|
|
129
184
|
...(scheduled > 0
|
|
130
185
|
? [
|
|
131
|
-
`${plural(scheduled, "participant")} working
|
|
186
|
+
`${plural(scheduled, "participant")} working ${scheduled === 1 ? "a declared window" : "declared windows"} of up to ${formatDuration(longestWindowMs)}`,
|
|
132
187
|
]
|
|
133
188
|
: []),
|
|
134
189
|
...(open > 0
|
|
@@ -141,23 +196,29 @@ export function bundleLimitReason(estimate) {
|
|
|
141
196
|
...(scheduled > 0 ? ["shorten the longest windows (participants[].endAfterMs)"] : []),
|
|
142
197
|
...(dwell ? ["shorten the dwell window (dwell.ms) or capture less often (dwell.everyMs)"] : []),
|
|
143
198
|
];
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
199
|
+
// A trace is one participant's, so splitting the study helps only the run's own files.
|
|
200
|
+
if (runFile || page || entries || changes.length === 0)
|
|
201
|
+
changes.push("split the participants across studies");
|
|
202
|
+
const advice = changes.length < 2
|
|
203
|
+
? changes.join("")
|
|
204
|
+
: `${changes.slice(0, -1).join(", ")}, or ${changes.at(-1)}`;
|
|
147
205
|
return [
|
|
148
206
|
...passed,
|
|
149
|
-
`The estimate counts ${counted.join(" and ")}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)} that adds about ${(RATES.
|
|
150
|
-
|
|
151
|
-
`To stay under the limit, ${advice}.`,
|
|
207
|
+
`The estimate counts ${counted.join(" and ")}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)} that adds about ${RATES.turnRunBytes} bytes to run.json and up to ${(RATES.turnTraceBytes / 1000).toFixed(1)} KB to the participant's trace.`,
|
|
208
|
+
"The run goes ahead.",
|
|
209
|
+
`To stay under ${passed.length === 1 ? "the limit" : "the limits"}, ${advice}.`,
|
|
152
210
|
].join(" ");
|
|
153
211
|
}
|
|
154
212
|
/** The estimate beside the limits, for `study check`. */
|
|
155
213
|
export function describeBundle(estimate) {
|
|
156
|
-
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest
|
|
214
|
+
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest trace about ${mib(estimate.largestTraceBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads of each; Observer page about ${mib(estimate.observerPageBytes)} of the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads; about ${count(estimate.entries)} files and folders of the ${count(estimate.entryLimit)} the public-safety scan reads; about ${mib(estimate.totalBytes, 0)} in all. Participants with a declared end are counted for their whole window and the others for up to ${formatDuration(estimate.basis.openMs)}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)}.`;
|
|
157
215
|
}
|
|
216
|
+
/** Bytes in MiB to `digits` places, or two under 0.1 MiB, so a short study's trace does not read 0.0. */
|
|
158
217
|
function mib(bytes, digits = 1) {
|
|
159
218
|
const value = bytes / MIB;
|
|
160
|
-
|
|
219
|
+
if (Number.isInteger(value))
|
|
220
|
+
return `${value} MiB`;
|
|
221
|
+
return `${value.toFixed(value < 0.1 ? 2 : digits)} MiB`;
|
|
161
222
|
}
|
|
162
223
|
function count(value) {
|
|
163
224
|
return value.toLocaleString("en-US");
|
|
@@ -32,18 +32,18 @@ export function parseParticipantFiles(raw, field) {
|
|
|
32
32
|
if (from === undefined || as === undefined)
|
|
33
33
|
return invalid(listReason);
|
|
34
34
|
if (!isPlainRelativePath(from))
|
|
35
|
-
return invalid(`\`${at}.from\` must be a path inside the folder that holds the study file, written relative to it, such as fixtures/
|
|
35
|
+
return invalid(`\`${at}.from\` must be a path inside the folder that holds the study file, written relative to it, such as fixtures/photo.png, with / between folders and no empty, . or .. part.`);
|
|
36
36
|
if (isLocalTreeDenylisted(from))
|
|
37
37
|
return invalid(`\`${at}.from\` names a file humanish keeps on this machine: a .env file, a key, a credential file, or one under .git, node_modules or .humanish. The local-tree archive leaves the same files out. Copy a synthetic file instead.`);
|
|
38
38
|
if (!isPlainRelativePath(as) || as.startsWith("~"))
|
|
39
|
-
return invalid(`\`${at}.as\` must be a path relative to the desktop user's home folder, such as Downloads/
|
|
39
|
+
return invalid(`\`${at}.as\` must be a path relative to the desktop user's home folder, such as Downloads/photo.png, with / between folders, no empty, . or .. part, and no leading ~.`);
|
|
40
40
|
if (isHumanishDesktopPath(as))
|
|
41
|
-
return invalid(`\`${at}.as\` is a path humanish uses on the desktop: ${SUBJECT_FOLDER}/ holds the cloned or unpacked app, and names that start with .humanish are humanish's own files. Pick another path, such as Downloads/
|
|
41
|
+
return invalid(`\`${at}.as\` is a path humanish uses on the desktop: ${SUBJECT_FOLDER}/ holds the cloned or unpacked app, and names that start with .humanish are humanish's own files. Pick another path, such as Downloads/photo.png.`);
|
|
42
42
|
// The run records `as`, and redacting it would record a different path. Encoded text the
|
|
43
43
|
// public-safety scan cannot read would leave run.json unscanned.
|
|
44
44
|
const scanned = scanEncodedText(as);
|
|
45
45
|
if (scanned.sensitive || scanned.opaque)
|
|
46
|
-
return invalid(`\`${at}.as\` looks like a secret, a local path or encoded text, and the run records it. Name the file plainly, such as Downloads/
|
|
46
|
+
return invalid(`\`${at}.as\` looks like a secret, a local path or encoded text, and the run records it. Name the file plainly, such as Downloads/photo.png.`);
|
|
47
47
|
const clash = files.find((earlier) => earlier.as === as || as.startsWith(`${earlier.as}/`) || earlier.as.startsWith(`${as}/`));
|
|
48
48
|
if (clash !== undefined)
|
|
49
49
|
return invalid(clash.as === as
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ParticipantSchedule } from "./arrivals.js";
|
|
1
2
|
import type { DwellWindow, StopWhen } from "../actors/stop-conditions.js";
|
|
2
3
|
import type { ReasoningEffort } from "../actors/reasoning-effort.js";
|
|
3
4
|
import { type DevicePreset } from "./device-presets.js";
|
|
@@ -49,17 +50,11 @@ export interface DesktopParticipant extends Participant {
|
|
|
49
50
|
*/
|
|
50
51
|
readonly timeZone?: string;
|
|
51
52
|
};
|
|
52
|
-
/** When it starts, in ms after the run starts its participants; absent, with the run. */
|
|
53
|
-
readonly startAfterMs?: number;
|
|
54
|
-
/** Computer use: when it ends, on the same clock; absent, it runs for the study's session budget. */
|
|
55
|
-
readonly endAfterMs?: number;
|
|
56
|
-
/** Computer use: the longest its session lasts, from its start; it may end the session sooner. */
|
|
57
|
-
readonly maxSessionMs?: number;
|
|
58
53
|
/**
|
|
59
|
-
*
|
|
60
|
-
*
|
|
54
|
+
* When it starts and, on computer use, how its session ends: its startAfterMs, and its endAfterMs
|
|
55
|
+
* or maxSessionMs with its wind-down.
|
|
61
56
|
*/
|
|
62
|
-
readonly
|
|
57
|
+
readonly schedule: ParticipantSchedule;
|
|
63
58
|
}
|
|
64
59
|
/** Only independent computer-use participants consume a task protocol and their own target. */
|
|
65
60
|
export interface ComputerUseParticipant extends DesktopParticipant {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Who takes part in a study: one record per participant, built from the declared roster or a count.
|
|
2
2
|
// Computer-use and shared-world participants share the desktop fields; each kind adds its own.
|
|
3
|
-
import {
|
|
3
|
+
import { participantSchedule } from "./arrivals.js";
|
|
4
4
|
import { resolveParticipantDevice } from "./device-presets.js";
|
|
5
5
|
import { computerUseParticipantCount, isSharedWorldComposition, participantIdAt, } from "./routing.js";
|
|
6
6
|
import { participantList, declaredParticipantCount, participantInstruction, } from "./study-fields.js";
|
|
@@ -39,19 +39,7 @@ function desktopParticipant(config, entry, index, kind, focus) {
|
|
|
39
39
|
...(idleWaitMs === undefined ? {} : { idleWaitMs }),
|
|
40
40
|
...(timeZone === undefined ? {} : { timeZone }),
|
|
41
41
|
},
|
|
42
|
-
|
|
43
|
-
...(entry?.endAfterMs === undefined
|
|
44
|
-
? {}
|
|
45
|
-
: {
|
|
46
|
-
endAfterMs: entry.endAfterMs,
|
|
47
|
-
windDownMs: entry.windDownMs ?? DEFAULT_WIND_DOWN_MS,
|
|
48
|
-
}),
|
|
49
|
-
...(entry?.maxSessionMs === undefined
|
|
50
|
-
? {}
|
|
51
|
-
: {
|
|
52
|
-
maxSessionMs: entry.maxSessionMs,
|
|
53
|
-
windDownMs: entry.windDownMs ?? DEFAULT_WIND_DOWN_MS,
|
|
54
|
-
}),
|
|
42
|
+
schedule: participantSchedule(entry ?? {}),
|
|
55
43
|
};
|
|
56
44
|
}
|
|
57
45
|
/**
|
|
@@ -163,8 +163,13 @@ export interface ComputerUsePlan extends PlanBase {
|
|
|
163
163
|
/** The registered computer-use actor id. */
|
|
164
164
|
readonly actor: string;
|
|
165
165
|
readonly runner: ComputerUseRunner;
|
|
166
|
-
/**
|
|
166
|
+
/**
|
|
167
|
+
* How many participants run at once: the declared cap clamped to [1, participants], else all of
|
|
168
|
+
* them, lowered to what the E2B plan and HUMANISH_CUA_MAX_CONCURRENCY allow (participantsAtOnce).
|
|
169
|
+
*/
|
|
167
170
|
readonly concurrency: number;
|
|
171
|
+
/** The count before HUMANISH_CUA_MAX_CONCURRENCY lowered it, when it did. */
|
|
172
|
+
readonly envLoweredConcurrencyFrom?: number;
|
|
168
173
|
/** When the participants start and how many run at once, with every session at its budget. */
|
|
169
174
|
readonly arrivals: ArrivalPlan;
|
|
170
175
|
readonly sessionBudgetMs: number;
|
package/dist/study/plan.d.ts
CHANGED
|
@@ -8,7 +8,8 @@ export declare function resolveStudyDryRun(config: StudyConfig, override: boolea
|
|
|
8
8
|
/**
|
|
9
9
|
* The plan a study runs under, built without reading files or the network. From the environment
|
|
10
10
|
* (options.env, else process.env, as the routes read it) it reads the operator's E2B plan limits,
|
|
11
|
-
* the sandbox ceiling and the concurrent sandboxes, which no
|
|
11
|
+
* the sandbox ceiling and the concurrent sandboxes, and HUMANISH_CUA_MAX_CONCURRENCY, which no
|
|
12
|
+
* study file can know. Each route's
|
|
12
13
|
* planner makes every refusal that route makes, in the route's order and with its codes and
|
|
13
14
|
* messages; the route's exported runner calls the same planner.
|
|
14
15
|
*/
|
package/dist/study/plan.js
CHANGED
|
@@ -6,7 +6,7 @@ import { callerDrivingOf, planComputerUseStudy } from "../routes/computer-use/pl
|
|
|
6
6
|
import { injectedBrowser, planScriptedStudy } from "../routes/scripted/plan.js";
|
|
7
7
|
import { planSharedWorldStudy } from "../routes/shared-world/plan.js";
|
|
8
8
|
import { planTerminalStudy } from "../routes/terminal/plan.js";
|
|
9
|
-
import { concurrentSandboxes, sandboxCeiling, } from "../substrates/e2b/lifetime.js";
|
|
9
|
+
import { concurrencyCap, concurrentSandboxes, sandboxCeiling, } from "../substrates/e2b/lifetime.js";
|
|
10
10
|
import { localBrowserDefaults } from "../substrates/local/runtime-config.js";
|
|
11
11
|
import { THIS_REPO_DRY_RUN_ONLY } from "./composition-rules.js";
|
|
12
12
|
import { bundleLimitReason, estimateBundle } from "./bundle-estimate.js";
|
|
@@ -68,7 +68,8 @@ function planPreview(config, options, input) {
|
|
|
68
68
|
/**
|
|
69
69
|
* The plan a study runs under, built without reading files or the network. From the environment
|
|
70
70
|
* (options.env, else process.env, as the routes read it) it reads the operator's E2B plan limits,
|
|
71
|
-
* the sandbox ceiling and the concurrent sandboxes, which no
|
|
71
|
+
* the sandbox ceiling and the concurrent sandboxes, and HUMANISH_CUA_MAX_CONCURRENCY, which no
|
|
72
|
+
* study file can know. Each route's
|
|
72
73
|
* planner makes every refusal that route makes, in the route's order and with its codes and
|
|
73
74
|
* messages; the route's exported runner calls the same planner.
|
|
74
75
|
*/
|
|
@@ -79,6 +80,7 @@ export function planStudy(config, options, deps = {}) {
|
|
|
79
80
|
const e2b = {
|
|
80
81
|
sandboxCeiling: sandboxCeiling(env),
|
|
81
82
|
concurrentSandboxes: concurrentSandboxes(env),
|
|
83
|
+
concurrencyCap: concurrencyCap(env),
|
|
82
84
|
};
|
|
83
85
|
const result = planRoute(routeOf(config), study, options, input, deps, e2b);
|
|
84
86
|
if (!result.ok)
|
|
@@ -125,8 +127,8 @@ function closingReportWarnings(plan) {
|
|
|
125
127
|
if (brain.kind !== "local-agent" || brain.agent !== "claude")
|
|
126
128
|
return [];
|
|
127
129
|
const { participants } = plan.runner;
|
|
128
|
-
const scheduled = participants.filter((
|
|
129
|
-
const capped = participants.filter((
|
|
130
|
+
const scheduled = participants.filter(({ schedule }) => schedule.end?.kind === "scheduled_end");
|
|
131
|
+
const capped = participants.filter(({ schedule }) => schedule.end?.kind === "session_cap");
|
|
130
132
|
const claude = "Claude as a local-agent participant model cannot be asked for a closing report";
|
|
131
133
|
const writers = "A computer-use model (actor.type: openai-computer-use) or Codex (actor.localAgent: codex) writes one.";
|
|
132
134
|
return [
|
package/dist/study/types.d.ts
CHANGED
|
@@ -355,7 +355,8 @@ export interface StudyParticipantEntry {
|
|
|
355
355
|
maxSessionMs?: number;
|
|
356
356
|
/**
|
|
357
357
|
* With endAfterMs or maxSessionMs: how long before the end each turn tells the participant the
|
|
358
|
-
* end time and the minutes left. Default
|
|
358
|
+
* end time and the minutes left. Default: a quarter of the participant's window or cap, at most
|
|
359
|
+
* 600000 (10 minutes).
|
|
359
360
|
*/
|
|
360
361
|
windDownMs?: number;
|
|
361
362
|
/**
|
|
@@ -45,14 +45,24 @@ export type ConcurrentSandboxes = {
|
|
|
45
45
|
export declare function concurrentSandboxes(env: Readonly<Record<string, string | undefined>>): ConcurrentSandboxes;
|
|
46
46
|
/** The limit in use and where it came from, or why the setting cannot be read (doctor's row). */
|
|
47
47
|
export declare function describeConcurrentSandboxes(limit: ConcurrentSandboxes): string;
|
|
48
|
+
/** The setting that lowers how many participants a study runs at once, on any desktop. */
|
|
49
|
+
export declare const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
48
50
|
/**
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
*
|
|
51
|
+
* HUMANISH_CUA_MAX_CONCURRENCY when it is a whole number of 1 or more, else undefined: the
|
|
52
|
+
* operator's own bound on participants at once on this machine, which may only lower a study's.
|
|
53
|
+
* Any other value is ignored.
|
|
54
|
+
*/
|
|
55
|
+
export declare function concurrencyCap(env: Readonly<Record<string, string | undefined>>): number | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* How many participants a study runs at once: its `concurrency` (the declared
|
|
58
|
+
* execution.concurrency clamped to the roster, else every participant), lowered to `cap`
|
|
59
|
+
* (concurrencyCap) and then to what the E2B plan runs (`sandboxes`, on a hosted desktop).
|
|
60
|
+
* `declared` says whether the study set it. `alongside` counts the sandboxes the study keeps for
|
|
61
|
+
* the whole run besides its participants' desktops, such as a shared world's app, and `minimum` is
|
|
62
|
+
* the fewest at once the route can run. A declared value the E2B plan cannot run, or a cap under
|
|
63
|
+
* `minimum`, is a refusal naming the setting. `lowered` is the warning a plan records when the cap
|
|
64
|
+
* or the E2B limit lowers the count; when both do, it is the E2B limit's, since that sets the
|
|
65
|
+
* count. `loweredFrom` is the count the cap lowered, when the cap sets the count.
|
|
56
66
|
*/
|
|
57
67
|
export declare function participantsAtOnce(study: {
|
|
58
68
|
readonly concurrency: number;
|
|
@@ -60,10 +70,14 @@ export declare function participantsAtOnce(study: {
|
|
|
60
70
|
readonly participants: number;
|
|
61
71
|
readonly alongside?: number;
|
|
62
72
|
readonly minimum?: number;
|
|
63
|
-
},
|
|
73
|
+
}, limits: {
|
|
74
|
+
readonly sandboxes?: ConcurrentSandboxes;
|
|
75
|
+
readonly cap?: number | undefined;
|
|
76
|
+
}): {
|
|
64
77
|
readonly ok: true;
|
|
65
78
|
readonly count: number;
|
|
66
79
|
readonly lowered?: string;
|
|
80
|
+
readonly loweredFrom?: number;
|
|
67
81
|
} | {
|
|
68
82
|
readonly ok: false;
|
|
69
83
|
readonly message: string;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// The operator's E2B plan limits humanish plans against: how long a sandbox may live and how many
|
|
2
|
-
// run at once
|
|
2
|
+
// run at once, with the operator's own bound on participants at once (HUMANISH_CUA_MAX_CONCURRENCY).
|
|
3
|
+
// Also the budgets the desktop routes add to a participant's own deadline when they set
|
|
3
4
|
// a sandbox's server-side timeout. A dead host process can never orphan a sandbox past that
|
|
4
5
|
// timeout, so every route derives it from these values.
|
|
5
6
|
import { plural } from "../../run/text.js";
|
|
@@ -88,19 +89,62 @@ export function describeConcurrentSandboxes(limit) {
|
|
|
88
89
|
if (!limit.ok)
|
|
89
90
|
return limit.message;
|
|
90
91
|
return limit.source === "setting"
|
|
91
|
-
? `${limit.count} at once, from ${CONCURRENCY_SETTING}. A study that runs more desktops at once runs them in waves, and an execution.concurrency above it is refused before any sandbox is created.`
|
|
92
|
+
? `${limit.count} at once, from ${CONCURRENCY_SETTING}. A study that runs more desktops at once runs them in waves, and an execution.concurrency above it is refused before any sandbox is created, unless ${CUA_MAX_CONCURRENCY_ENV} lowers it to the limit or below.`
|
|
92
93
|
: `${limit.count} at once, E2B's Hobby limit. On a plan that runs more (Pro runs ${PRO_CONCURRENT_SANDBOXES}), set ${CONCURRENCY_SETTING} to that plan's limit.`;
|
|
93
94
|
}
|
|
95
|
+
/** The setting that lowers how many participants a study runs at once, on any desktop. */
|
|
96
|
+
export const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
94
97
|
/**
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
* for the whole run besides its participants' desktops, such as a shared world's app, and
|
|
99
|
-
* `minimum` is the fewest at once the route can run. A declared value that does not fit is a
|
|
100
|
-
* refusal naming the setting value that admits it. `lowered` is the warning a plan records when
|
|
101
|
-
* the limit holds an undeclared default to waves.
|
|
98
|
+
* HUMANISH_CUA_MAX_CONCURRENCY when it is a whole number of 1 or more, else undefined: the
|
|
99
|
+
* operator's own bound on participants at once on this machine, which may only lower a study's.
|
|
100
|
+
* Any other value is ignored.
|
|
102
101
|
*/
|
|
103
|
-
export function
|
|
102
|
+
export function concurrencyCap(env) {
|
|
103
|
+
const cap = readPositiveInt(env[CUA_MAX_CONCURRENCY_ENV], 0);
|
|
104
|
+
return cap > 0 ? cap : undefined;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* How many participants a study runs at once: its `concurrency` (the declared
|
|
108
|
+
* execution.concurrency clamped to the roster, else every participant), lowered to `cap`
|
|
109
|
+
* (concurrencyCap) and then to what the E2B plan runs (`sandboxes`, on a hosted desktop).
|
|
110
|
+
* `declared` says whether the study set it. `alongside` counts the sandboxes the study keeps for
|
|
111
|
+
* the whole run besides its participants' desktops, such as a shared world's app, and `minimum` is
|
|
112
|
+
* the fewest at once the route can run. A declared value the E2B plan cannot run, or a cap under
|
|
113
|
+
* `minimum`, is a refusal naming the setting. `lowered` is the warning a plan records when the cap
|
|
114
|
+
* or the E2B limit lowers the count; when both do, it is the E2B limit's, since that sets the
|
|
115
|
+
* count. `loweredFrom` is the count the cap lowered, when the cap sets the count.
|
|
116
|
+
*/
|
|
117
|
+
export function participantsAtOnce(study, limits) {
|
|
118
|
+
const capped = cappedAtOnce(study, limits.cap);
|
|
119
|
+
if (!capped.ok || limits.sandboxes === undefined)
|
|
120
|
+
return capped;
|
|
121
|
+
// A count the cap lowered is the operator's bound and no longer the study's declared value, so
|
|
122
|
+
// the E2B limit runs it in waves as it would an undeclared default.
|
|
123
|
+
const left = capped.loweredFrom === undefined
|
|
124
|
+
? study
|
|
125
|
+
: { ...study, concurrency: capped.count, declared: false };
|
|
126
|
+
const planned = sandboxesAtOnce(left, limits.sandboxes);
|
|
127
|
+
return !planned.ok || planned.count < capped.count ? planned : capped;
|
|
128
|
+
}
|
|
129
|
+
/** participantsAtOnce under the cap alone. */
|
|
130
|
+
function cappedAtOnce(study, cap) {
|
|
131
|
+
if (cap === undefined || cap >= study.concurrency)
|
|
132
|
+
return { ok: true, count: study.concurrency };
|
|
133
|
+
const minimum = study.minimum ?? 1;
|
|
134
|
+
if (cap < minimum)
|
|
135
|
+
return {
|
|
136
|
+
ok: false,
|
|
137
|
+
message: `This study runs at least ${minimum} participants at once, and ${CUA_MAX_CONCURRENCY_ENV} allows ${cap}. Set it to ${minimum} or more, or unset it.`,
|
|
138
|
+
};
|
|
139
|
+
return {
|
|
140
|
+
ok: true,
|
|
141
|
+
count: cap,
|
|
142
|
+
loweredFrom: study.concurrency,
|
|
143
|
+
lowered: `This study's ${study.participants} participants run ${cap} at a time, because ${CUA_MAX_CONCURRENCY_ENV} is ${cap}.`,
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/** participantsAtOnce under the E2B plan's limit alone. */
|
|
147
|
+
function sandboxesAtOnce(study, limit) {
|
|
104
148
|
if (!limit.ok)
|
|
105
149
|
return limit;
|
|
106
150
|
const alongside = study.alongside ?? 0;
|