humanish 0.125.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/actors/computer-use/actor.d.ts +2 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/clock.d.ts +9 -8
- package/dist/actors/computer-use/loop/clock.js +27 -17
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -1
- package/dist/actors/computer-use/loop/debrief.js +15 -5
- package/dist/actors/computer-use/loop/ending.d.ts +10 -5
- package/dist/actors/computer-use/loop/ending.js +12 -7
- package/dist/actors/computer-use/loop/observation.d.ts +6 -0
- package/dist/actors/computer-use/loop/observation.js +19 -1
- package/dist/actors/computer-use/loop/session.d.ts +4 -2
- package/dist/actors/computer-use/loop/session.js +4 -1
- package/dist/actors/computer-use/loop/trace.js +4 -2
- package/dist/actors/computer-use/loop/types.d.ts +22 -8
- package/dist/actors/computer-use/loop.js +5 -5
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/contract.d.ts +31 -8
- package/dist/actors/contract.js +3 -0
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/cli/commands/study.js +8 -0
- package/dist/evidence/redaction.d.ts +1 -0
- package/dist/evidence/redaction.js +1 -0
- package/dist/evidence/trace-page.d.ts +6 -0
- package/dist/evidence/trace-page.js +90 -0
- package/dist/observer/data.js +8 -5
- package/dist/observer/feed-cursors.d.ts +14 -0
- package/dist/observer/feed-cursors.js +61 -0
- package/dist/observer/participant-items.d.ts +7 -0
- package/dist/observer/pinned-files.d.ts +2 -1
- package/dist/observer/pinned-files.js +2 -2
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +38 -17
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle.js +4 -0
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/desktop.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +6 -0
- package/dist/routes/computer-use/e2b-desktop/prepare.js +47 -1
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +2 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +15 -3
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -1
- package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +5 -1
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-desktop.d.ts +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-model.js +16 -6
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +54 -84
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +65 -17
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.d.ts +1 -0
- package/dist/routes/computer-use/setup.js +29 -1
- package/dist/routes/computer-use/single-bundle.d.ts +5 -1
- package/dist/routes/computer-use/single-bundle.js +11 -1
- package/dist/routes/computer-use/types.d.ts +18 -13
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +4 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/participant-specs.js +3 -2
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +8 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/contained-output.d.ts +7 -3
- package/dist/run/contained-output.js +11 -7
- package/dist/run/judge.d.ts +14 -4
- package/dist/run/judge.js +17 -12
- package/dist/run/live-items.d.ts +19 -0
- package/dist/run/live-items.js +28 -8
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/run/stream-trace.d.ts +25 -4
- package/dist/run/stream-trace.js +55 -17
- package/dist/run/streams.d.ts +16 -0
- package/dist/study/arrivals.d.ts +105 -34
- package/dist/study/arrivals.js +179 -74
- package/dist/study/bundle-estimate.d.ts +12 -10
- package/dist/study/bundle-estimate.js +138 -72
- package/dist/study/desktop-files.d.ts +37 -0
- package/dist/study/desktop-files.js +174 -0
- package/dist/study/keys.d.ts +8 -0
- package/dist/study/keys.js +3 -0
- package/dist/study/migrate/v2.js +12 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +28 -20
- package/dist/study/parse/clock.d.ts +18 -0
- package/dist/study/parse/clock.js +36 -0
- package/dist/study/plan-participants.d.ts +19 -10
- package/dist/study/plan-participants.js +13 -9
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +28 -15
- package/dist/study/preflight.js +4 -0
- package/dist/study/types.d.ts +35 -5
- package/dist/study/validation.js +3 -0
- package/dist/study/warnings.d.ts +2 -2
- package/dist/study/warnings.js +7 -0
- package/dist/subject/local-tree-archive.d.ts +7 -0
- package/dist/subject/local-tree-archive.js +12 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -1
- package/dist/substrates/e2b/desktop-browser.js +6 -2
- package/dist/substrates/e2b/desktop-clock.d.ts +10 -0
- package/dist/substrates/e2b/desktop-clock.js +29 -0
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/dist/substrates/e2b/sdk.d.ts +6 -0
- package/dist/verify/pages.d.ts +7 -0
- package/dist/verify/pages.js +50 -0
- package/dist/verify/verify.js +10 -3
- package/docs/architecture/actor-contract.md +2 -0
- package/docs/architecture/external-public-shared-world.md +4 -1
- package/docs/architecture/observer.md +10 -1
- package/docs/contracts/policy.md +7 -0
- package/docs/contracts/run-bundle.md +31 -1
- package/docs/contracts/schemas.md +93 -11
- package/package.json +2 -2
|
@@ -1,46 +1,85 @@
|
|
|
1
1
|
// How large a computer-use run's bundle grows, estimated from its plan before anything runs, and
|
|
2
|
-
// whether it would pass a limit a later command holds it to: analysis reads
|
|
3
|
-
// EVIDENCE_LIMITS.sourceBytes, verify reads no file larger than
|
|
4
|
-
// public-safety scan visits at most runEntryLimit(participants)
|
|
2
|
+
// whether it would pass a limit a later command holds it to: analysis reads run.json and each
|
|
3
|
+
// participant's trace up to EVIDENCE_LIMITS.sourceBytes each, verify reads no file larger than
|
|
4
|
+
// RUN_ARTIFACT_MAX_BYTES, and the public-safety scan visits at most runEntryLimit(participants)
|
|
5
|
+
// files and folders.
|
|
6
|
+
//
|
|
7
|
+
// observer/index.html carries every participant's items gzipped, so with busy participants it can
|
|
8
|
+
// pass verify's 32 MiB before run.json passes analysis's 16 MiB: at these rates, 17 participants
|
|
9
|
+
// working 8 hours. observer-data.json carries the same per-request lists as run.json and was within
|
|
10
|
+
// 7 % of its size on every live run measured, so it passes 32 MiB only after run.json passes
|
|
11
|
+
// 16 MiB, and the estimate does not check it.
|
|
5
12
|
import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
|
|
6
13
|
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
7
14
|
import { formatDuration } from "../run/projection.js";
|
|
8
15
|
import { plural } from "../run/text.js";
|
|
9
16
|
import { runEntryLimit } from "../verify/scan-limit.js";
|
|
10
|
-
// Measured on 2026-10-
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// working the whole window; these rates give 12.6 KB, so the estimate errs high, and a limit it
|
|
17
|
-
// passes is a warning.
|
|
17
|
+
// Measured on 2026-10-13 on run bundle v3. Live: the 7 finished hosted computer-use runs kept from
|
|
18
|
+
// the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
|
|
19
|
+
// model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
|
|
20
|
+
// own files. Pace, session length and screenshots: the live computer-use runs kept on the
|
|
21
|
+
// maintainer's machine. A trace's rate a turn on the public site/public/runs/lobby-0927 run
|
|
22
|
+
// (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the page, under the live fit.
|
|
18
23
|
const RATES = Object.freeze({
|
|
19
|
-
/** run.json bytes
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
/** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
|
|
25
|
+
runBytes: 4_400,
|
|
26
|
+
/**
|
|
27
|
+
* run.json bytes for each participant before its turns: its stream, actor summary and closing
|
|
28
|
+
* report request. Live fit: 16,239, plus one request.
|
|
29
|
+
*/
|
|
30
|
+
participantBytes: 16_500,
|
|
31
|
+
/**
|
|
32
|
+
* run.json bytes a model turn adds: the request's entries in `tokenUsage.turns` and
|
|
33
|
+
* `conversation.requests`. Live: 30,623 bytes over 137 requests, 223.5 each.
|
|
34
|
+
*/
|
|
35
|
+
turnRunBytes: 225,
|
|
36
|
+
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
|
|
37
|
+
participantTraceBytes: 5_400,
|
|
38
|
+
/**
|
|
39
|
+
* Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
|
|
40
|
+
* for the `page` each screenshot item now carries. Live fit: 1,965 a turn. Participants with 4
|
|
41
|
+
* or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15 turns, most of them
|
|
42
|
+
* UI actions. Analysis refuses the whole run when one trace passes its limit, so each trace is
|
|
43
|
+
* counted at the busiest participant's rate.
|
|
44
|
+
*/
|
|
45
|
+
turnTraceBytes: 4_700,
|
|
46
|
+
/**
|
|
47
|
+
* Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
|
|
48
|
+
* plus 91 for the page.
|
|
49
|
+
*/
|
|
50
|
+
frameTraceBytes: 500,
|
|
51
|
+
/**
|
|
52
|
+
* Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
|
|
53
|
+
* minutes or more was 10.4.
|
|
54
|
+
*/
|
|
30
55
|
secondsPerTurn: 10,
|
|
31
56
|
/**
|
|
32
57
|
* How long a participant with no declared end is counted, at most its session budget. It ends
|
|
33
|
-
* when it finishes its mission: 90 % of 974
|
|
58
|
+
* when it finishes its mission: 90 % of 974 kept sessions ended within 184 s on 2026-10-12; of
|
|
59
|
+
* the 197 kept on 2026-10-13, 64 % ended within 180 s and 90 % within 325 s. Counting 325 s would
|
|
60
|
+
* add under 0.4 MiB to the run.json of 100 participants.
|
|
34
61
|
*/
|
|
35
62
|
openSessionMs: 3 * 60_000,
|
|
36
|
-
/** observer
|
|
37
|
-
|
|
38
|
-
/**
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
63
|
+
/** observer/index.html's app before any run data: 1,199,831 bytes on the dry runs. */
|
|
64
|
+
observerAppBytes: 1_200_000,
|
|
65
|
+
/**
|
|
66
|
+
* The page's compact copy of observer-data.json, as a share of run.json: 0.64 to 0.81 whole on
|
|
67
|
+
* the live runs, 0.49 for a participant's part, and 0.42 for the per-request entries.
|
|
68
|
+
*/
|
|
69
|
+
observerPageDataShare: 0.5,
|
|
70
|
+
/**
|
|
71
|
+
* The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114 on the
|
|
72
|
+
* live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes a
|
|
73
|
+
* turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
74
|
+
* synthetic 8-hour run.
|
|
75
|
+
*/
|
|
76
|
+
observerPageItemsShare: 0.12,
|
|
77
|
+
/** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
|
|
42
78
|
screenshotBytes: 110_000,
|
|
43
|
-
/**
|
|
79
|
+
/**
|
|
80
|
+
* Run files and folders besides the participants': 10 on a dry run, and live the sandbox
|
|
81
|
+
* receipts, `actors/` and an analysis's 9.
|
|
82
|
+
*/
|
|
44
83
|
runEntries: 20,
|
|
45
84
|
/** A participant's entries besides its turn screenshots: its trace file, its screenshot folder, its first frame. */
|
|
46
85
|
participantEntries: 3,
|
|
@@ -48,48 +87,61 @@ const RATES = Object.freeze({
|
|
|
48
87
|
const MIB = 1024 * 1024;
|
|
49
88
|
/**
|
|
50
89
|
* The bundle a live run of `participants` leaves, each in its session of `sessionMs` unless it
|
|
51
|
-
* declares an end. A participant with an end works its whole window; one without
|
|
52
|
-
* the measured open session above, or its budget
|
|
53
|
-
*
|
|
54
|
-
* session, and the frame after it.
|
|
90
|
+
* declares an end or a cap. A participant with an end works its whole window; one without ends
|
|
91
|
+
* itself, so it is counted for the measured open session above, or its budget (its cap, else
|
|
92
|
+
* `sessionMs`) when that is shorter. Each takes a model turn at the measured pace, and a dwell
|
|
93
|
+
* window adds a frame every `everyMs` for its length, cut to the session, and the frame after it.
|
|
55
94
|
*/
|
|
56
95
|
export function estimateBundle(participants, sessionMs) {
|
|
57
96
|
let turns = 0;
|
|
58
97
|
let frames = 0;
|
|
98
|
+
let traces = 0;
|
|
99
|
+
let largestTrace = 0;
|
|
59
100
|
let longestWindowMs = 0;
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
101
|
+
// The longest any participant without an end is counted, for the messages.
|
|
102
|
+
let openMs = Math.min(sessionMs, RATES.openSessionMs);
|
|
103
|
+
let scheduled = 0;
|
|
104
|
+
for (const { schedule, limits } of participants) {
|
|
105
|
+
const { end } = schedule;
|
|
106
|
+
const budgetMs = end?.windowMs ?? sessionMs;
|
|
107
|
+
let countedMs;
|
|
108
|
+
if (end?.kind === "scheduled_end") {
|
|
109
|
+
scheduled += 1;
|
|
110
|
+
countedMs = end.windowMs;
|
|
111
|
+
longestWindowMs = Math.max(longestWindowMs, countedMs);
|
|
112
|
+
}
|
|
113
|
+
else {
|
|
114
|
+
countedMs = Math.min(budgetMs, RATES.openSessionMs);
|
|
115
|
+
openMs = Math.max(openMs, countedMs);
|
|
116
|
+
}
|
|
117
|
+
const own = Math.ceil(countedMs / (RATES.secondsPerTurn * 1000));
|
|
118
|
+
const dwell = limits.dwell;
|
|
119
|
+
const ownFrames = dwell === undefined ? 0 : Math.floor(Math.min(dwell.ms, budgetMs) / dwell.everyMs) + 1;
|
|
120
|
+
const trace = RATES.participantTraceBytes + own * RATES.turnTraceBytes + ownFrames * RATES.frameTraceBytes;
|
|
121
|
+
turns += own;
|
|
122
|
+
frames += ownFrames;
|
|
123
|
+
traces += trace;
|
|
124
|
+
largestTrace = Math.max(largestTrace, trace);
|
|
71
125
|
}
|
|
72
126
|
const count = participants.length;
|
|
73
|
-
const runFileBytes = RATES.runBytes +
|
|
74
|
-
count * RATES.participantBytes +
|
|
75
|
-
turns * RATES.turnBytes +
|
|
76
|
-
frames * RATES.frameBytes;
|
|
77
|
-
const observerDataBytes = runFileBytes + count * RATES.observerDataParticipantBytes;
|
|
78
|
-
const pageBytes = RATES.observerPageBytes + Math.round(RATES.observerPageShare * runFileBytes);
|
|
127
|
+
const runFileBytes = RATES.runBytes + count * RATES.participantBytes + turns * RATES.turnRunBytes;
|
|
79
128
|
const screenshots = count + turns + frames;
|
|
80
|
-
const
|
|
129
|
+
const observerPageBytes = Math.round(RATES.observerAppBytes +
|
|
130
|
+
RATES.observerPageDataShare * runFileBytes +
|
|
131
|
+
RATES.observerPageItemsShare * traces);
|
|
81
132
|
return {
|
|
82
133
|
runFileBytes,
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
134
|
+
largestTraceBytes: largestTrace,
|
|
135
|
+
observerPageBytes,
|
|
136
|
+
// observer-data.json is about run.json's size.
|
|
137
|
+
totalBytes: observerPageBytes + screenshots * RATES.screenshotBytes + traces + 2 * runFileBytes,
|
|
86
138
|
entries: RATES.runEntries + count * RATES.participantEntries + turns + frames,
|
|
87
139
|
entryLimit: runEntryLimit(count),
|
|
88
140
|
basis: {
|
|
89
141
|
participants: count,
|
|
90
|
-
scheduled
|
|
142
|
+
scheduled,
|
|
91
143
|
longestWindowMs,
|
|
92
|
-
open: count - scheduled
|
|
144
|
+
open: count - scheduled,
|
|
93
145
|
openMs,
|
|
94
146
|
dwell: frames > 0,
|
|
95
147
|
},
|
|
@@ -97,23 +149,31 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
97
149
|
}
|
|
98
150
|
/**
|
|
99
151
|
* The warning for a run whose estimate passes a limit, or undefined: each limit it passes, what the
|
|
100
|
-
* estimate counted,
|
|
152
|
+
* estimate counted, and what to change.
|
|
101
153
|
*/
|
|
102
154
|
export function bundleLimitReason(estimate) {
|
|
155
|
+
const runFile = estimate.runFileBytes > EVIDENCE_LIMITS.sourceBytes;
|
|
156
|
+
const page = estimate.observerPageBytes > RUN_ARTIFACT_MAX_BYTES;
|
|
157
|
+
const entries = estimate.entries > estimate.entryLimit;
|
|
103
158
|
const passed = [
|
|
104
|
-
...(
|
|
159
|
+
...(runFile
|
|
105
160
|
? [
|
|
106
161
|
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads.`,
|
|
107
162
|
]
|
|
108
163
|
: []),
|
|
109
|
-
...(estimate.
|
|
164
|
+
...(estimate.largestTraceBytes > EVIDENCE_LIMITS.sourceBytes
|
|
165
|
+
? [
|
|
166
|
+
`One participant's trace may grow to about ${mib(estimate.largestTraceBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads of each trace.`,
|
|
167
|
+
]
|
|
168
|
+
: []),
|
|
169
|
+
...(page
|
|
110
170
|
? [
|
|
111
|
-
`
|
|
171
|
+
`The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file.`,
|
|
112
172
|
]
|
|
113
173
|
: []),
|
|
114
|
-
...(
|
|
174
|
+
...(entries
|
|
115
175
|
? [
|
|
116
|
-
`
|
|
176
|
+
`The run may hold about ${count(estimate.entries)} files and folders, past the ${count(estimate.entryLimit)} the public-safety scan reads in a run of ${plural(estimate.basis.participants, "participant")}.`,
|
|
117
177
|
]
|
|
118
178
|
: []),
|
|
119
179
|
];
|
|
@@ -123,7 +183,7 @@ export function bundleLimitReason(estimate) {
|
|
|
123
183
|
const counted = [
|
|
124
184
|
...(scheduled > 0
|
|
125
185
|
? [
|
|
126
|
-
`${plural(scheduled, "participant")} working
|
|
186
|
+
`${plural(scheduled, "participant")} working ${scheduled === 1 ? "a declared window" : "declared windows"} of up to ${formatDuration(longestWindowMs)}`,
|
|
127
187
|
]
|
|
128
188
|
: []),
|
|
129
189
|
...(open > 0
|
|
@@ -136,23 +196,29 @@ export function bundleLimitReason(estimate) {
|
|
|
136
196
|
...(scheduled > 0 ? ["shorten the longest windows (participants[].endAfterMs)"] : []),
|
|
137
197
|
...(dwell ? ["shorten the dwell window (dwell.ms) or capture less often (dwell.everyMs)"] : []),
|
|
138
198
|
];
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
199
|
+
// A trace is one participant's, so splitting the study helps only the run's own files.
|
|
200
|
+
if (runFile || page || entries || changes.length === 0)
|
|
201
|
+
changes.push("split the participants across studies");
|
|
202
|
+
const advice = changes.length < 2
|
|
203
|
+
? changes.join("")
|
|
204
|
+
: `${changes.slice(0, -1).join(", ")}, or ${changes.at(-1)}`;
|
|
142
205
|
return [
|
|
143
206
|
...passed,
|
|
144
|
-
`The estimate counts ${counted.join(" and ")}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)} that adds about ${(RATES.
|
|
145
|
-
|
|
146
|
-
`To stay under the limit, ${advice}.`,
|
|
207
|
+
`The estimate counts ${counted.join(" and ")}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)} that adds about ${RATES.turnRunBytes} bytes to run.json and up to ${(RATES.turnTraceBytes / 1000).toFixed(1)} KB to the participant's trace.`,
|
|
208
|
+
"The run goes ahead.",
|
|
209
|
+
`To stay under ${passed.length === 1 ? "the limit" : "the limits"}, ${advice}.`,
|
|
147
210
|
].join(" ");
|
|
148
211
|
}
|
|
149
212
|
/** The estimate beside the limits, for `study check`. */
|
|
150
213
|
export function describeBundle(estimate) {
|
|
151
|
-
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest
|
|
214
|
+
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest trace about ${mib(estimate.largestTraceBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads of each; Observer page about ${mib(estimate.observerPageBytes)} of the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads; about ${count(estimate.entries)} files and folders of the ${count(estimate.entryLimit)} the public-safety scan reads; about ${mib(estimate.totalBytes, 0)} in all. Participants with a declared end are counted for their whole window and the others for up to ${formatDuration(estimate.basis.openMs)}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)}.`;
|
|
152
215
|
}
|
|
216
|
+
/** Bytes in MiB to `digits` places, or two under 0.1 MiB, so a short study's trace does not read 0.0. */
|
|
153
217
|
function mib(bytes, digits = 1) {
|
|
154
218
|
const value = bytes / MIB;
|
|
155
|
-
|
|
219
|
+
if (Number.isInteger(value))
|
|
220
|
+
return `${value} MiB`;
|
|
221
|
+
return `${value.toFixed(value < 0.1 ? 2 : digits)} MiB`;
|
|
156
222
|
}
|
|
157
223
|
function count(value) {
|
|
158
224
|
return value.toLocaleString("en-US");
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { StudyParseFailure, StudyParticipantFile } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Where `participants[].files` act: the hosted desktop of an independent computer-use participant.
|
|
4
|
+
* The parser's unread-field check and the route planners give this reason.
|
|
5
|
+
*/
|
|
6
|
+
export declare const DESKTOP_FILES_ROUTE_REASON = "files are copied onto a hosted computer-use desktop; needs route: computer-use with execution.target: e2b-desktop";
|
|
7
|
+
/** A participant's `files` list: each entry's `from` and `as`. */
|
|
8
|
+
export declare function parseParticipantFiles(raw: unknown, field: string): {
|
|
9
|
+
ok: true;
|
|
10
|
+
value: StudyParticipantFile[] | undefined;
|
|
11
|
+
} | StudyParseFailure;
|
|
12
|
+
/** One file read on this machine for a participant's desktop. */
|
|
13
|
+
export interface DesktopFile {
|
|
14
|
+
/** Where it lands, relative to the desktop user's home folder. */
|
|
15
|
+
readonly as: string;
|
|
16
|
+
/** The file's bytes, shared by every participant that receives it and uploaded as they are. */
|
|
17
|
+
readonly data: ArrayBuffer;
|
|
18
|
+
readonly size: number;
|
|
19
|
+
/** The sha256 of `data`, 64 hex characters. */
|
|
20
|
+
readonly sha256: string;
|
|
21
|
+
}
|
|
22
|
+
type DesktopFilesRead = {
|
|
23
|
+
readonly ok: true;
|
|
24
|
+
readonly files: ReadonlyMap<string, readonly DesktopFile[]>;
|
|
25
|
+
} | {
|
|
26
|
+
readonly ok: false;
|
|
27
|
+
readonly message: string;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Read every participant's declared files from `folder`, the folder that holds the study file, and
|
|
31
|
+
* return them by participant id. Runs before any sandbox exists, so a refusal costs nothing.
|
|
32
|
+
*/
|
|
33
|
+
export declare function readDesktopFiles(folder: string, participants: readonly {
|
|
34
|
+
readonly id: string;
|
|
35
|
+
readonly files?: readonly StudyParticipantFile[];
|
|
36
|
+
}[]): Promise<DesktopFilesRead>;
|
|
37
|
+
export {};
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
// Files a participant's hosted desktop receives before it starts (`participants[].files`), so a
|
|
2
|
+
// browser upload dialog can find them. The parser checks the declared paths; readDesktopFiles reads
|
|
3
|
+
// the files from the study's folder on this machine before any sandbox exists.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { stat } from "node:fs/promises";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
8
|
+
import { prepareSelectedOutputDirectory, readContainedRegularFile, } from "../run/contained-output.js";
|
|
9
|
+
import { isRecord } from "../run/type-guards.js";
|
|
10
|
+
import { isLocalTreeDenylisted } from "../subject/local-tree-archive.js";
|
|
11
|
+
import { SUBJECT_DIR } from "../subject/steps.js";
|
|
12
|
+
import { invalid, str } from "./parse/values.js";
|
|
13
|
+
/**
|
|
14
|
+
* Where `participants[].files` act: the hosted desktop of an independent computer-use participant.
|
|
15
|
+
* The parser's unread-field check and the route planners give this reason.
|
|
16
|
+
*/
|
|
17
|
+
export const DESKTOP_FILES_ROUTE_REASON = "files are copied onto a hosted computer-use desktop; needs route: computer-use with execution.target: e2b-desktop";
|
|
18
|
+
/** The most files one participant's desktop receives. */
|
|
19
|
+
const MAX_FILES = 20;
|
|
20
|
+
/** A participant's `files` list: each entry's `from` and `as`. */
|
|
21
|
+
export function parseParticipantFiles(raw, field) {
|
|
22
|
+
if (raw === undefined)
|
|
23
|
+
return { ok: true, value: undefined };
|
|
24
|
+
const listReason = `\`${field}\` must be a list of 1 to ${MAX_FILES} { from, as } entries, one per file the participant's desktop receives.`;
|
|
25
|
+
if (!Array.isArray(raw) || raw.length === 0 || raw.length > MAX_FILES)
|
|
26
|
+
return invalid(listReason);
|
|
27
|
+
const files = [];
|
|
28
|
+
for (const [index, entry] of raw.entries()) {
|
|
29
|
+
const at = `${field}[${index}]`;
|
|
30
|
+
const from = isRecord(entry) ? str(entry.from) : undefined;
|
|
31
|
+
const as = isRecord(entry) ? str(entry.as) : undefined;
|
|
32
|
+
if (from === undefined || as === undefined)
|
|
33
|
+
return invalid(listReason);
|
|
34
|
+
if (!isPlainRelativePath(from))
|
|
35
|
+
return invalid(`\`${at}.from\` must be a path inside the folder that holds the study file, written relative to it, such as fixtures/photo.png, with / between folders and no empty, . or .. part.`);
|
|
36
|
+
if (isLocalTreeDenylisted(from))
|
|
37
|
+
return invalid(`\`${at}.from\` names a file humanish keeps on this machine: a .env file, a key, a credential file, or one under .git, node_modules or .humanish. The local-tree archive leaves the same files out. Copy a synthetic file instead.`);
|
|
38
|
+
if (!isPlainRelativePath(as) || as.startsWith("~"))
|
|
39
|
+
return invalid(`\`${at}.as\` must be a path relative to the desktop user's home folder, such as Downloads/photo.png, with / between folders, no empty, . or .. part, and no leading ~.`);
|
|
40
|
+
if (isHumanishDesktopPath(as))
|
|
41
|
+
return invalid(`\`${at}.as\` is a path humanish uses on the desktop: ${SUBJECT_FOLDER}/ holds the cloned or unpacked app, and names that start with .humanish are humanish's own files. Pick another path, such as Downloads/photo.png.`);
|
|
42
|
+
// The run records `as`, and redacting it would record a different path. Encoded text the
|
|
43
|
+
// public-safety scan cannot read would leave run.json unscanned.
|
|
44
|
+
const scanned = scanEncodedText(as);
|
|
45
|
+
if (scanned.sensitive || scanned.opaque)
|
|
46
|
+
return invalid(`\`${at}.as\` looks like a secret, a local path or encoded text, and the run records it. Name the file plainly, such as Downloads/photo.png.`);
|
|
47
|
+
const clash = files.find((earlier) => earlier.as === as || as.startsWith(`${earlier.as}/`) || earlier.as.startsWith(`${as}/`));
|
|
48
|
+
if (clash !== undefined)
|
|
49
|
+
return invalid(clash.as === as
|
|
50
|
+
? `\`${at}.as\` names ${as}, which another file in this list already uses. Give each file its own path.`
|
|
51
|
+
: `\`${at}.as\` names ${as}, and another file in this list uses ${clash.as}, so one would have to be both a file and a folder. Give each file its own path.`);
|
|
52
|
+
files.push({ from, as });
|
|
53
|
+
}
|
|
54
|
+
return { ok: true, value: files };
|
|
55
|
+
}
|
|
56
|
+
/** The subject's folder under the sandbox user's home, where a clone or local tree is put. */
|
|
57
|
+
const SUBJECT_FOLDER = path.posix.relative("/home/user", SUBJECT_DIR);
|
|
58
|
+
/**
|
|
59
|
+
* Whether `as` lands where humanish puts its own files: the subject's folder, which provisioning
|
|
60
|
+
* replaces, or a name starting with .humanish, such as the local-tree archive.
|
|
61
|
+
*/
|
|
62
|
+
function isHumanishDesktopPath(as) {
|
|
63
|
+
const first = as.split("/")[0] ?? "";
|
|
64
|
+
return first === SUBJECT_FOLDER || first.startsWith(".humanish");
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* A relative path of plain names joined by `/`. A backslash is refused because the desktop is
|
|
68
|
+
* Linux, where it is part of a name, and on Windows it would be a separator.
|
|
69
|
+
*/
|
|
70
|
+
function isPlainRelativePath(value) {
|
|
71
|
+
if (value.includes("\0") || value.includes("\\") || value.startsWith("/"))
|
|
72
|
+
return false;
|
|
73
|
+
return value.split("/").every((part) => part !== "" && part !== "." && part !== "..");
|
|
74
|
+
}
|
|
75
|
+
const MIB = 1024 * 1024;
|
|
76
|
+
/** The most bytes humanish copies for one file. */
|
|
77
|
+
const FILE_MAX_BYTES = 10 * MIB;
|
|
78
|
+
/** The most bytes one participant's desktop receives, a file counted each time it is placed. */
|
|
79
|
+
const PARTICIPANT_MAX_BYTES = 25 * MIB;
|
|
80
|
+
/** The most bytes of distinct files a study names: humanish holds them in memory for the run. */
|
|
81
|
+
const STUDY_MAX_BYTES = 100 * MIB;
|
|
82
|
+
/**
|
|
83
|
+
* Read every participant's declared files from `folder`, the folder that holds the study file, and
|
|
84
|
+
* return them by participant id. Runs before any sandbox exists, so a refusal costs nothing.
|
|
85
|
+
*/
|
|
86
|
+
export async function readDesktopFiles(folder, participants) {
|
|
87
|
+
// A library caller's config skips the parser, so its rules run again here.
|
|
88
|
+
for (const participant of participants) {
|
|
89
|
+
const checked = parseParticipantFiles(participant.files, `Participant ${participant.id}'s files`);
|
|
90
|
+
if (!checked.ok)
|
|
91
|
+
return { ok: false, message: checked.error.message };
|
|
92
|
+
}
|
|
93
|
+
const files = new Map();
|
|
94
|
+
if (!participants.some((participant) => participant.files !== undefined))
|
|
95
|
+
return { ok: true, files };
|
|
96
|
+
const root = await bindFolder(folder);
|
|
97
|
+
if (root === undefined)
|
|
98
|
+
return {
|
|
99
|
+
ok: false,
|
|
100
|
+
message: "The folder that holds the study file could not be read as a folder, so its participants' files cannot be copied. Run the study from its file, with the folder in place.",
|
|
101
|
+
};
|
|
102
|
+
// One read per distinct `from`: a group of participants shares one file's bytes.
|
|
103
|
+
const sources = new Map();
|
|
104
|
+
let studyBytes = 0;
|
|
105
|
+
for (const participant of participants) {
|
|
106
|
+
const placed = [];
|
|
107
|
+
let participantBytes = 0;
|
|
108
|
+
for (const [index, file] of (participant.files ?? []).entries()) {
|
|
109
|
+
let source = sources.get(file.from);
|
|
110
|
+
if (source === undefined) {
|
|
111
|
+
// A file past what the study has left is refused unread, so memory stays under the total.
|
|
112
|
+
const left = STUDY_MAX_BYTES - studyBytes;
|
|
113
|
+
const read = await readContainedRegularFile(root, file.from, Math.min(FILE_MAX_BYTES, left));
|
|
114
|
+
if (read.status === "refused" && read.reason === "too-large" && left < FILE_MAX_BYTES)
|
|
115
|
+
return {
|
|
116
|
+
ok: false,
|
|
117
|
+
message: `The files this study's participants receive total more than ${STUDY_MAX_BYTES / MIB} MiB, the most humanish holds in memory for one run. Use smaller files, or give more participants the same file.`,
|
|
118
|
+
};
|
|
119
|
+
if (read.status !== "read")
|
|
120
|
+
return {
|
|
121
|
+
ok: false,
|
|
122
|
+
message: `Participant ${participant.id}'s files[${index}].from, ${file.from}, ${unreadReason(read)}. Paths in from are relative to the folder that holds the study file.`,
|
|
123
|
+
};
|
|
124
|
+
const sha256 = createHash("sha256").update(read.bytes).digest("hex");
|
|
125
|
+
source = { data: new Uint8Array(read.bytes).buffer, size: read.bytes.length, sha256 };
|
|
126
|
+
sources.set(file.from, source);
|
|
127
|
+
studyBytes += source.size;
|
|
128
|
+
}
|
|
129
|
+
participantBytes += source.size;
|
|
130
|
+
if (participantBytes > PARTICIPANT_MAX_BYTES)
|
|
131
|
+
return {
|
|
132
|
+
ok: false,
|
|
133
|
+
message: `Participant ${participant.id}'s files total more than ${PARTICIPANT_MAX_BYTES / MIB} MiB, the most humanish copies onto one desktop. Use smaller or fewer files.`,
|
|
134
|
+
};
|
|
135
|
+
placed.push({ as: file.as, ...source });
|
|
136
|
+
}
|
|
137
|
+
if (placed.length > 0)
|
|
138
|
+
files.set(participant.id, placed);
|
|
139
|
+
}
|
|
140
|
+
return { ok: true, files };
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* The study's folder as a contained-read root, or undefined when it is not an existing folder.
|
|
144
|
+
* prepareSelectedOutputDirectory would create a missing one.
|
|
145
|
+
*/
|
|
146
|
+
async function bindFolder(folder) {
|
|
147
|
+
try {
|
|
148
|
+
if (!(await stat(folder)).isDirectory())
|
|
149
|
+
return undefined;
|
|
150
|
+
return await prepareSelectedOutputDirectory(folder, folder);
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
/** Why a `from` could not be read, as a clause after its path. */
|
|
157
|
+
function unreadReason(read) {
|
|
158
|
+
if (read.status === "missing")
|
|
159
|
+
return "does not exist in the folder that holds the study file";
|
|
160
|
+
switch (read.reason) {
|
|
161
|
+
case "too-large":
|
|
162
|
+
return `is larger than ${FILE_MAX_BYTES / MIB} MiB, the most humanish copies for one file`;
|
|
163
|
+
case "directory":
|
|
164
|
+
return "is a folder; name a file";
|
|
165
|
+
case "not-regular":
|
|
166
|
+
return "is a symbolic link, a hard link or another kind of file humanish does not copy; put a plain copy of the file in the folder";
|
|
167
|
+
case "unsafe-path":
|
|
168
|
+
return "leaves the folder that holds the study file or passes through a symbolic link; put a plain copy of the file in the folder";
|
|
169
|
+
case "changed":
|
|
170
|
+
return "changed while humanish read it; run the study again";
|
|
171
|
+
case "unreadable":
|
|
172
|
+
return "could not be read; check its permissions";
|
|
173
|
+
}
|
|
174
|
+
}
|
package/dist/study/keys.d.ts
CHANGED
|
@@ -16,12 +16,20 @@ export declare const PARTICIPANT_ENTRY: {
|
|
|
16
16
|
device: true;
|
|
17
17
|
instruction: true;
|
|
18
18
|
target: true;
|
|
19
|
+
files: {
|
|
20
|
+
from: true;
|
|
21
|
+
as: true;
|
|
22
|
+
};
|
|
19
23
|
entry: true;
|
|
20
24
|
host: true;
|
|
21
25
|
startAfterMs: true;
|
|
22
26
|
endAfterMs: true;
|
|
27
|
+
maxSessionMs: true;
|
|
23
28
|
windDownMs: true;
|
|
24
29
|
idleWaitMs: true;
|
|
30
|
+
clock: {
|
|
31
|
+
timeZone: true;
|
|
32
|
+
};
|
|
25
33
|
reasoningEffort: true;
|
|
26
34
|
stopWhen: {
|
|
27
35
|
any: {
|
package/dist/study/keys.js
CHANGED
|
@@ -37,12 +37,15 @@ export const PARTICIPANT_ENTRY = {
|
|
|
37
37
|
device: true,
|
|
38
38
|
instruction: true,
|
|
39
39
|
target: true,
|
|
40
|
+
files: { from: true, as: true },
|
|
40
41
|
entry: true,
|
|
41
42
|
host: true,
|
|
42
43
|
startAfterMs: true,
|
|
43
44
|
endAfterMs: true,
|
|
45
|
+
maxSessionMs: true,
|
|
44
46
|
windDownMs: true,
|
|
45
47
|
idleWaitMs: true,
|
|
48
|
+
clock: { timeZone: true },
|
|
46
49
|
reasoningEffort: true,
|
|
47
50
|
stopWhen: STOP_WHEN,
|
|
48
51
|
dwell: DWELL,
|
package/dist/study/migrate/v2.js
CHANGED
|
@@ -80,6 +80,16 @@ export function movedV2KeyReason(raw) {
|
|
|
80
80
|
}
|
|
81
81
|
return undefined;
|
|
82
82
|
}
|
|
83
|
+
// Participant keys that arrived after v2, so a v2 participant cannot carry them.
|
|
84
|
+
const AFTER_V2_PARTICIPANT_KEYS = [
|
|
85
|
+
"startAfterMs",
|
|
86
|
+
"endAfterMs",
|
|
87
|
+
"windDownMs",
|
|
88
|
+
"idleWaitMs",
|
|
89
|
+
"files",
|
|
90
|
+
"maxSessionMs",
|
|
91
|
+
"clock",
|
|
92
|
+
];
|
|
83
93
|
// The keys of a humanish.lab.v2 file, in the order its parser listed them: the v3 sections with
|
|
84
94
|
// `subject.topology`, the actor list with its participant keys, `execution.caps`, `personas` and
|
|
85
95
|
// the `scenario` object.
|
|
@@ -92,11 +102,10 @@ const V2_KEYS = {
|
|
|
92
102
|
actors: {
|
|
93
103
|
type: true,
|
|
94
104
|
count: true,
|
|
95
|
-
|
|
96
|
-
lanes: without(PARTICIPANT_ENTRY, "startAfterMs", "endAfterMs", "windDownMs", "idleWaitMs"),
|
|
105
|
+
lanes: without(PARTICIPANT_ENTRY, ...AFTER_V2_PARTICIPANT_KEYS),
|
|
97
106
|
// Roster groups are participants with a count; the parser expanded them into `lanes[]`.
|
|
98
107
|
roster: {
|
|
99
|
-
...without(PARTICIPANT_ENTRY,
|
|
108
|
+
...without(PARTICIPANT_ENTRY, ...AFTER_V2_PARTICIPANT_KEYS),
|
|
100
109
|
count: true,
|
|
101
110
|
},
|
|
102
111
|
// maxWaitMs, idleWaitMs and clock arrived after v2, so a v2 file cannot carry them.
|
|
@@ -25,7 +25,7 @@ export interface ParticipantPaths {
|
|
|
25
25
|
* Parse a participant list into a fan-out roster (computer-use E2B route): `participants` in a
|
|
26
26
|
* study, `actors[0].lanes` in the v2 file migrate reads, each named in messages by `paths`.
|
|
27
27
|
* Structural only: each entry is `{ id?, actorType?, surface?, caseGroup?, persona?, device?,
|
|
28
|
-
* instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, reasoningEffort?, stopWhen?, dwell? }`.
|
|
28
|
+
* instruction?, target?, entry?, host?, startAfterMs?, endAfterMs?, windDownMs?, idleWaitMs?, maxSessionMs?, clock?, reasoningEffort?, stopWhen?, dwell?, files? }`.
|
|
29
29
|
* Participant ids (when declared) must be public-safe path tokens and unique; grouping metadata
|
|
30
30
|
* must be public-safe tokens; a device must be a known preset name. The
|
|
31
31
|
* route-scoped cross-validation (device XOR raw resolution, targets, the participant bound) runs in parseStudy
|