humanish 0.121.0 → 0.123.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/CONTEXT.md +1 -1
- package/README.md +33 -6
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/automatic.js +2 -1
- package/dist/analysis/evidence.js +41 -21
- package/dist/cli/commands/study-run.js +9 -3
- package/dist/feedback/derivative-text.d.ts +12 -0
- package/dist/feedback/derivative-text.js +130 -0
- package/dist/feedback/export-bundle.js +38 -131
- package/dist/feedback/export.js +9 -1
- package/dist/observer/run-routes.js +61 -18
- package/dist/routes/computer-use/bundle.d.ts +0 -1
- package/dist/routes/computer-use/bundle.js +0 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +2 -12
- package/dist/routes/computer-use/live-phase.d.ts +0 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-execution.d.ts +9 -15
- package/dist/routes/computer-use/participant-execution.js +22 -31
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +1 -2
- package/dist/routes/computer-use/types.d.ts +17 -5
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/bundle-shape.js +2 -2
- package/dist/run/bundle.d.ts +5 -2
- package/dist/run/bundle.js +4 -1
- package/dist/run/detail.d.ts +6 -0
- package/dist/run/detail.js +14 -1
- package/dist/run/host-suspension.d.ts +2 -2
- package/dist/run/host-suspension.js +2 -2
- package/dist/run/judge.d.ts +1 -1
- package/dist/run/locate.js +3 -1
- package/dist/run/projection.d.ts +7 -0
- package/dist/run/projection.js +10 -0
- package/dist/run/run.js +7 -1
- package/dist/run/stored-runs.js +10 -4
- package/dist/run/stream-shape.d.ts +11 -1
- package/dist/run/stream-shape.js +26 -2
- package/dist/run/stream-trace.d.ts +68 -0
- package/dist/run/stream-trace.js +260 -0
- package/dist/run/streams.d.ts +20 -0
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/bundle-estimate.d.ts +48 -0
- package/dist/study/bundle-estimate.js +159 -0
- package/dist/study/doctor.js +4 -0
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/plan-types.d.ts +3 -0
- package/dist/study/plan.js +22 -2
- package/dist/study/preflight.js +19 -1
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/dist/tui-app.js +87 -87
- package/dist/verify/artifacts.d.ts +17 -10
- package/dist/verify/artifacts.js +22 -13
- package/dist/verify/scan-limit.d.ts +6 -0
- package/dist/verify/scan-limit.js +25 -0
- package/dist/verify/verify.d.ts +1 -1
- package/dist/verify/verify.js +68 -18
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +89 -8
- package/docs/contracts/schemas.md +38 -11
- package/package.json +6 -3
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
// Where a participant's trace lives. A v1 run.json holds every stream's trace items and screenshot
|
|
2
|
+
// entries inline. From v2 the run writer moves them into one file per stream and leaves an index
|
|
3
|
+
// in run.json, so run.json stays a summary whatever the length of the sessions. Every reader that
|
|
4
|
+
// needs the items loads them here, which accepts both versions and checks a file against its index.
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { RUN_BUNDLE_SCHEMA } from "./bundle.js";
|
|
7
|
+
import { readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
|
|
8
|
+
import { isRunStreamArtifact, isStreamTraceIndex, isStreamTracePath } from "./stream-shape.js";
|
|
9
|
+
import { isRecord } from "./type-guards.js";
|
|
10
|
+
/** A trace that does not match its index, or cannot be read. The message names the stream. */
|
|
11
|
+
export class StreamTraceRefusedError extends Error {
|
|
12
|
+
streamId;
|
|
13
|
+
constructor(streamId, detail) {
|
|
14
|
+
super(`Participant ${streamId}'s trace was not read: ${detail}. A trace file must stay as the run wrote it; restore the run folder from a copy.`);
|
|
15
|
+
this.streamId = streamId;
|
|
16
|
+
this.name = "StreamTraceRefusedError";
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
20
|
+
const isScreenshot = (artifact) => artifact.kind === "screenshot";
|
|
21
|
+
/**
|
|
22
|
+
* The file a trace path names on any file system. Trace file names are ASCII (isStreamTracePath),
|
|
23
|
+
* so lower case folds them as a case-insensitive file system does: actors/Case.json and
|
|
24
|
+
* actors/case.json are one file there.
|
|
25
|
+
*/
|
|
26
|
+
const traceFileKey = (tracePath) => tracePath.toLowerCase();
|
|
27
|
+
/** Names of the form actors/<n>.json, which the splitter gives by position and never reuses. */
|
|
28
|
+
const POSITION_TRACE_FILE = /^actors\/\d+\.json$/;
|
|
29
|
+
/**
|
|
30
|
+
* The stream's actor trace artifact (actor.json, actors/<stream>.json, actor-<surface>.json) when
|
|
31
|
+
* no earlier stream claimed it (compared by traceFileKey), else actors/<position>.json. The
|
|
32
|
+
* position names are a namespace of their own: an artifact named that way is not reused, so two
|
|
33
|
+
* streams never get one file. Another trace artifact (the scripted route's traces/<surface>.json
|
|
34
|
+
* holds its native step trace) keeps its own file.
|
|
35
|
+
*/
|
|
36
|
+
function tracePath(stream, index, claimed) {
|
|
37
|
+
const own = stream.artifacts.find((artifact) => artifact.kind === "trace" &&
|
|
38
|
+
isStreamTracePath(artifact.path) &&
|
|
39
|
+
!POSITION_TRACE_FILE.test(artifact.path))?.path;
|
|
40
|
+
return own !== undefined && !claimed.has(traceFileKey(own)) ? own : `actors/${index + 1}.json`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The run.json the run writes, and the trace files it writes first. Each stream with a finished
|
|
44
|
+
* actor keeps that actor without `items`, its artifacts other than screenshots, and an index of its
|
|
45
|
+
* trace file; the file is the actor trace with the stream's screenshot entries as `artifacts`. A
|
|
46
|
+
* stream already split (its actor read back from a v2 run.json without its items) keeps its index,
|
|
47
|
+
* since its file is on disk.
|
|
48
|
+
*/
|
|
49
|
+
export function splitStreamTraces(bundle) {
|
|
50
|
+
const files = [];
|
|
51
|
+
const claimed = new Set();
|
|
52
|
+
for (const stream of bundle.streams)
|
|
53
|
+
if (stream.trace)
|
|
54
|
+
claimed.add(traceFileKey(stream.trace.path));
|
|
55
|
+
const streams = bundle.streams.map((stream, index) => {
|
|
56
|
+
const actor = stream.actor;
|
|
57
|
+
if (actor === undefined || !Array.isArray(actor.items))
|
|
58
|
+
return stream;
|
|
59
|
+
const { trace: _previous, ...rest } = stream;
|
|
60
|
+
const filePath = tracePath(rest, index, claimed);
|
|
61
|
+
// Only a stream already split that moved position can hold this stream's position name.
|
|
62
|
+
if (claimed.has(traceFileKey(filePath)))
|
|
63
|
+
throw new Error(`Two participants' traces would both be written to ${filePath}.`);
|
|
64
|
+
claimed.add(traceFileKey(filePath));
|
|
65
|
+
const screenshots = stream.artifacts.filter(isScreenshot);
|
|
66
|
+
const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots }, null, 2)}\n`);
|
|
67
|
+
files.push({ path: filePath, bytes });
|
|
68
|
+
const { items, ...summary } = actor;
|
|
69
|
+
return {
|
|
70
|
+
...rest,
|
|
71
|
+
// run.json's actor is the summary: every field of the trace except its items.
|
|
72
|
+
actor: summary,
|
|
73
|
+
artifacts: stream.artifacts.filter((artifact) => !isScreenshot(artifact)),
|
|
74
|
+
trace: {
|
|
75
|
+
path: filePath,
|
|
76
|
+
bytes: bytes.length,
|
|
77
|
+
sha256: sha256(bytes),
|
|
78
|
+
items: items.length,
|
|
79
|
+
screenshots: screenshots.length,
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
});
|
|
83
|
+
return { bundle: { ...bundle, schema: RUN_BUNDLE_SCHEMA, streams }, files };
|
|
84
|
+
}
|
|
85
|
+
function containedReader(runPaths) {
|
|
86
|
+
return async (relativePath, maxBytes) => {
|
|
87
|
+
const read = await readContainedRegularFile(runPaths, relativePath, maxBytes);
|
|
88
|
+
if (read.status === "missing")
|
|
89
|
+
return { refused: `${relativePath} is missing` };
|
|
90
|
+
if (read.status === "refused")
|
|
91
|
+
return { refused: refusalText(relativePath, read) };
|
|
92
|
+
return { bytes: read.bytes };
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The stream from the bytes of its trace file: its actor with the file's items, and its artifacts
|
|
97
|
+
* followed by the file's screenshot entries, without the index.
|
|
98
|
+
*/
|
|
99
|
+
function joinTrace(stream, bytes) {
|
|
100
|
+
const index = stream.trace;
|
|
101
|
+
const refuse = (detail) => {
|
|
102
|
+
throw new StreamTraceRefusedError(stream.id, detail);
|
|
103
|
+
};
|
|
104
|
+
if (bytes.length !== index.bytes)
|
|
105
|
+
refuse(`${index.path} holds ${bytes.length} bytes and run.json records ${index.bytes} bytes`);
|
|
106
|
+
if (sha256(bytes) !== index.sha256)
|
|
107
|
+
refuse(`${index.path} does not match the sha256 run.json records for it`);
|
|
108
|
+
let value;
|
|
109
|
+
try {
|
|
110
|
+
value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
refuse(`${index.path} is not JSON`);
|
|
114
|
+
}
|
|
115
|
+
if (!isRecord(value) || !Array.isArray(value.items) || !Array.isArray(value.artifacts))
|
|
116
|
+
return refuse(`${index.path} has no items or artifacts list`);
|
|
117
|
+
const artifacts = value.artifacts;
|
|
118
|
+
if (!artifacts.every((artifact) => isRunStreamArtifact(artifact) && isScreenshot(artifact)))
|
|
119
|
+
refuse(`${index.path} lists an artifact that is not a screenshot entry`);
|
|
120
|
+
if (value.items.length !== index.items || artifacts.length !== index.screenshots)
|
|
121
|
+
refuse(`${index.path} holds ${value.items.length} items and ${artifacts.length} screenshots and run.json records ${index.items} and ${index.screenshots}`);
|
|
122
|
+
const { trace: _index, ...rest } = stream;
|
|
123
|
+
return {
|
|
124
|
+
...rest,
|
|
125
|
+
actor: { ...stream.actor, items: value.items },
|
|
126
|
+
artifacts: [...stream.artifacts, ...artifacts],
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
/** Refuses an index the shape check would refuse, naming a file that is not a trace file name. */
|
|
130
|
+
function assertIndex(stream) {
|
|
131
|
+
const index = stream.trace;
|
|
132
|
+
if (isRecord(index) && typeof index.path === "string" && !isStreamTracePath(index.path))
|
|
133
|
+
throw new StreamTraceRefusedError(stream.id, `run.json indexes ${index.path}, which is not a trace file name (actor.json, actors/<stream>.json or actor-<surface>.json)`);
|
|
134
|
+
if (!isStreamTraceIndex(index) || stream.actor === undefined)
|
|
135
|
+
throw new StreamTraceRefusedError(stream.id, "run.json records a malformed trace index");
|
|
136
|
+
if (stream.actor.items !== undefined)
|
|
137
|
+
throw new StreamTraceRefusedError(stream.id, "run.json holds items beside its trace index");
|
|
138
|
+
}
|
|
139
|
+
/** The stream with its trace joined, and the bytes it was joined from when it has an index. */
|
|
140
|
+
async function readStreamTrace(read, stream, maxBytes) {
|
|
141
|
+
if (stream.trace === undefined)
|
|
142
|
+
return { stream };
|
|
143
|
+
assertIndex(stream);
|
|
144
|
+
const file = await read(stream.trace.path, maxBytes);
|
|
145
|
+
if ("refused" in file)
|
|
146
|
+
throw new StreamTraceRefusedError(stream.id, file.refused);
|
|
147
|
+
return { stream: joinTrace(stream, file.bytes), bytes: file.bytes };
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* The stream as a v1 run.json held it: its actor with every item and every artifact entry. A
|
|
151
|
+
* stream without an index is returned as it is (a v1 stream, or one with no actor). An indexed
|
|
152
|
+
* stream's file must have a trace file name (isStreamTracePath); it is read under `maxBytes` and
|
|
153
|
+
* checked against the index's byte count, sha256, item count and screenshot count, and any
|
|
154
|
+
* difference throws StreamTraceRefusedError. The rules that need the whole bundle (an unindexed v2
|
|
155
|
+
* actor, two participants indexing one file) are loadStreamTraces'.
|
|
156
|
+
*/
|
|
157
|
+
export async function loadStreamTrace(runPaths, stream, maxBytes = RUN_ARTIFACT_MAX_BYTES) {
|
|
158
|
+
return (await readStreamTrace(containedReader(runPaths), stream, maxBytes)).stream;
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Why a stream of this bundle cannot be read, before any file is. A trace file belongs to one
|
|
162
|
+
* participant, so every stream indexing a path another stream indexes, compared as a
|
|
163
|
+
* case-insensitive file system compares them, is refused. In v2 the writer
|
|
164
|
+
* indexes every actor on every write, mid-run flushes included, so a v2 actor without an index lost
|
|
165
|
+
* its trace file, and inline items never come from the writer. A run still writing carries its
|
|
166
|
+
* participants' items so far as `liveActor` (humanish.live-actor.v1), which no index binds; a
|
|
167
|
+
* run.json whose outcome says the run finished never does, since the final write replaces it.
|
|
168
|
+
*/
|
|
169
|
+
function bundleRefusal(bundle, stream, owners) {
|
|
170
|
+
const shared = isRecord(stream.trace)
|
|
171
|
+
? owners.get(traceFileKey(String(stream.trace.path)))
|
|
172
|
+
: undefined;
|
|
173
|
+
if (shared !== undefined && shared.length > 1)
|
|
174
|
+
return `run.json indexes ${String(stream.trace.path)} for participants ${shared.join(", ")} (paths that differ only in case name one file on some file systems), and a trace file belongs to one participant`;
|
|
175
|
+
if (bundle.schema !== RUN_BUNDLE_SCHEMA)
|
|
176
|
+
return undefined;
|
|
177
|
+
if (stream.actor !== undefined && stream.trace === undefined)
|
|
178
|
+
return "run.json holds its actor and names no trace file";
|
|
179
|
+
if (stream.liveActor !== undefined && bundle.outcome?.state === "finished")
|
|
180
|
+
return "run.json records a finished run and still holds in-progress items";
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Every stream of the bundle loaded through loadStreamTrace, one file at a time, and the refusal
|
|
185
|
+
* of each stream that could not be (see bundleRefusal for the rules that need the whole bundle).
|
|
186
|
+
* Each refused stream is kept as run.json holds it; a reader that shows the run passes the result
|
|
187
|
+
* to displayedTraces. `source` is the run's prepared paths, or a TraceFileReader. `onTrace`
|
|
188
|
+
* receives each trace file's path and the bytes the stream was joined from, after they matched
|
|
189
|
+
* the index, so a caller scans exactly what it reads.
|
|
190
|
+
*/
|
|
191
|
+
export async function loadStreamTraces(source, bundle, options = {}) {
|
|
192
|
+
const read = typeof source === "function" ? source : containedReader(source);
|
|
193
|
+
const owners = new Map();
|
|
194
|
+
for (const stream of bundle.streams)
|
|
195
|
+
if (isRecord(stream.trace)) {
|
|
196
|
+
const key = traceFileKey(String(stream.trace.path));
|
|
197
|
+
owners.set(key, [...(owners.get(key) ?? []), stream.id]);
|
|
198
|
+
}
|
|
199
|
+
const streams = [];
|
|
200
|
+
const refused = [];
|
|
201
|
+
for (const stream of bundle.streams) {
|
|
202
|
+
try {
|
|
203
|
+
const reason = bundleRefusal(bundle, stream, owners);
|
|
204
|
+
if (reason !== undefined)
|
|
205
|
+
throw new StreamTraceRefusedError(stream.id, reason);
|
|
206
|
+
const loaded = await readStreamTrace(read, stream, options.maxBytes ?? RUN_ARTIFACT_MAX_BYTES);
|
|
207
|
+
if (loaded.bytes !== undefined)
|
|
208
|
+
options.onTrace?.(stream.trace.path, loaded.bytes);
|
|
209
|
+
streams.push(loaded.stream);
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
if (!(error instanceof StreamTraceRefusedError))
|
|
213
|
+
throw error;
|
|
214
|
+
refused.push(error);
|
|
215
|
+
streams.push(stream);
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
return { bundle: { ...bundle, streams }, refused };
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* A loaded bundle as a reader that shows a run presents it. A refused stream keeps run.json's
|
|
222
|
+
* summary, shows none of the items run.json or a file supplied for it (its actor's items, its
|
|
223
|
+
* in-progress items, its screenshot entries), and carries `traceUnavailable` with the reason; a
|
|
224
|
+
* warn event on its timeline says the same.
|
|
225
|
+
*/
|
|
226
|
+
export function displayedTraces(loaded) {
|
|
227
|
+
const reasons = new Map(loaded.refused.map((error) => [error.streamId, error.message]));
|
|
228
|
+
if (reasons.size === 0)
|
|
229
|
+
return loaded.bundle;
|
|
230
|
+
const streams = loaded.bundle.streams.map((stream) => {
|
|
231
|
+
const reason = reasons.get(stream.id);
|
|
232
|
+
if (reason === undefined)
|
|
233
|
+
return stream;
|
|
234
|
+
const { liveActor: _live, trace: _index, actor, ...rest } = stream;
|
|
235
|
+
return {
|
|
236
|
+
...rest,
|
|
237
|
+
...(actor === undefined ? {} : { actor: { ...actor, items: [] } }),
|
|
238
|
+
artifacts: stream.artifacts.filter((artifact) => !isScreenshot(artifact)),
|
|
239
|
+
traceUnavailable: reason,
|
|
240
|
+
};
|
|
241
|
+
});
|
|
242
|
+
const events = loaded.bundle.streams
|
|
243
|
+
.filter((stream) => reasons.has(stream.id))
|
|
244
|
+
.map((stream) => ({
|
|
245
|
+
id: `event-trace-unavailable-${stream.id}`,
|
|
246
|
+
at: stream.updatedAt,
|
|
247
|
+
level: "warn",
|
|
248
|
+
type: "trace.unavailable",
|
|
249
|
+
message: reasons.get(stream.id),
|
|
250
|
+
streamId: stream.id,
|
|
251
|
+
}));
|
|
252
|
+
return { ...loaded.bundle, streams, events: [...loaded.bundle.events, ...events] };
|
|
253
|
+
}
|
|
254
|
+
/** The bundle with every stream loaded; the first refusal throws. */
|
|
255
|
+
export async function loadRunTraces(source, bundle, maxBytes) {
|
|
256
|
+
const loaded = await loadStreamTraces(source, bundle, maxBytes === undefined ? {} : { maxBytes });
|
|
257
|
+
if (loaded.refused[0])
|
|
258
|
+
throw loaded.refused[0];
|
|
259
|
+
return loaded.bundle;
|
|
260
|
+
}
|
package/dist/run/streams.d.ts
CHANGED
|
@@ -142,6 +142,18 @@ export interface RunDesktopGeometry {
|
|
|
142
142
|
/** Public-safe geometry measurement/fill warnings retained with the stream evidence. */
|
|
143
143
|
warnings?: string[];
|
|
144
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* A stream's trace file in a v2 run.json: the actor trace with its items and the stream's
|
|
147
|
+
* screenshot entries as `artifacts`. `bytes` and `sha256` bind the file to run.json; `items` and
|
|
148
|
+
* `screenshots` are what it holds, so a reader can list them without opening it.
|
|
149
|
+
*/
|
|
150
|
+
export interface StreamTraceIndex {
|
|
151
|
+
path: string;
|
|
152
|
+
bytes: number;
|
|
153
|
+
sha256: string;
|
|
154
|
+
items: number;
|
|
155
|
+
screenshots: number;
|
|
156
|
+
}
|
|
145
157
|
/** The original participant-facing assignment, before runtime access/coordination details. */
|
|
146
158
|
export interface RunParticipantAssignment {
|
|
147
159
|
mission: string;
|
|
@@ -189,6 +201,12 @@ export interface RunStream {
|
|
|
189
201
|
* and never persisted into bundles; declared here because the served observer-data carries it
|
|
190
202
|
* and the client is typed against this contract. */
|
|
191
203
|
liveEnded?: boolean;
|
|
204
|
+
/**
|
|
205
|
+
* Set by a reader that shows the run (displayedTraces in src/run/stream-trace.ts) when the loader
|
|
206
|
+
* refused this participant's trace: the reason. Runtime-only and never persisted; the stream then
|
|
207
|
+
* carries none of the refused items.
|
|
208
|
+
*/
|
|
209
|
+
traceUnavailable?: string;
|
|
192
210
|
/**
|
|
193
211
|
* Browser CSS layout viewport. Deterministic browser adapters may declare and render this
|
|
194
212
|
* exactly; hosted-desktop CUA producers set it only from a runtime measurement. Historical
|
|
@@ -234,6 +252,8 @@ export interface RunStream {
|
|
|
234
252
|
turnId?: string;
|
|
235
253
|
};
|
|
236
254
|
actor?: ActorTrace;
|
|
255
|
+
/** Present in a v2 run.json on each stream with an actor: where its items and screenshots are. */
|
|
256
|
+
trace?: StreamTraceIndex;
|
|
237
257
|
/**
|
|
238
258
|
* Mid-run partial actor evidence: the redacted trace items recorded so far,
|
|
239
259
|
* flushed while a live participant is still running so the attached Observer's timeline can
|
package/dist/study/arrivals.d.ts
CHANGED
|
@@ -6,6 +6,54 @@ import type { RunParticipantArrival } from "../run/bundle.js";
|
|
|
6
6
|
export declare const LATEST_START_AFTER_MS: number;
|
|
7
7
|
/** Whether `value` is a start offset humanish runs: whole milliseconds from 0 to 24 hours. */
|
|
8
8
|
export declare function isStartAfterMs(value: unknown): value is number;
|
|
9
|
+
/** How long before its end a participant is told the time left, unless it declares `windDownMs`. */
|
|
10
|
+
export declare const DEFAULT_WIND_DOWN_MS: number;
|
|
11
|
+
/** A participant's declared window: when it starts and when it ends, on the run's clock. */
|
|
12
|
+
export interface ParticipantWindow {
|
|
13
|
+
readonly id?: string;
|
|
14
|
+
readonly startAfterMs?: number;
|
|
15
|
+
readonly endAfterMs?: number;
|
|
16
|
+
readonly windDownMs?: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Why a participant's declared end cannot run, or undefined: an end that is not a whole number of
|
|
20
|
+
* milliseconds up to 24 hours, one at or before its start, or a wind-down that is not a whole
|
|
21
|
+
* number of milliseconds or has no end. `path` names the entry, as in "participants[1]".
|
|
22
|
+
*/
|
|
23
|
+
export declare function windowReason(entry: {
|
|
24
|
+
readonly id?: string;
|
|
25
|
+
readonly startAfterMs?: number;
|
|
26
|
+
readonly endAfterMs?: unknown;
|
|
27
|
+
readonly windDownMs?: unknown;
|
|
28
|
+
}, path: string): string | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* A participant's session budget: with a declared end, its window plus SCHEDULED_END_ROOM_MS in
|
|
31
|
+
* place of the study's `sessionMs` (execution.timeoutMs or its default); else `sessionMs`.
|
|
32
|
+
*/
|
|
33
|
+
export declare function participantSessionMs(participant: ParticipantWindow, sessionMs: number): number;
|
|
34
|
+
/**
|
|
35
|
+
* When a participant that was due at `scheduledAt` (runOnSchedule's ScheduledStart) ends, on the
|
|
36
|
+
* same clock, or undefined when it declares no end. A participant that waited for a slot still ends
|
|
37
|
+
* at its declared time.
|
|
38
|
+
*/
|
|
39
|
+
export declare function scheduledEndAt(participant: ParticipantWindow, scheduledAt: number): {
|
|
40
|
+
readonly atMs: number;
|
|
41
|
+
readonly windDownMs: number;
|
|
42
|
+
} | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Why the sandbox of the first participant whose window passes `ceilingMs` cannot live that long,
|
|
45
|
+
* or undefined. Its deadline is its session (participantSessionMs) plus `headroomMs`, what the
|
|
46
|
+
* route adds to a session for setup and teardown (sandboxHeadroomMs).
|
|
47
|
+
*/
|
|
48
|
+
export declare function windowDeadlineRefusal(participants: readonly ParticipantWindow[], sandbox: {
|
|
49
|
+
readonly sessionMs: number;
|
|
50
|
+
readonly headroomMs: number;
|
|
51
|
+
}, ceilingMs: number): string | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Each participant's window for `study check`, when any participant declares an end: when it starts
|
|
54
|
+
* and ends and when it is first told the time left, or when it starts and its session budget.
|
|
55
|
+
*/
|
|
56
|
+
export declare function describeWindows(participants: readonly ParticipantWindow[], sessionMs: number): string | undefined;
|
|
9
57
|
/** The refusal of a start offset that is not one, naming its `field`. */
|
|
10
58
|
export declare function startAfterMessage(field: string): string;
|
|
11
59
|
/**
|
|
@@ -49,9 +97,10 @@ export interface StartQueue {
|
|
|
49
97
|
* The schedule a roster's declared offsets give under the queue (`undefined` is 0: the participant
|
|
50
98
|
* starts with the run). Participants start in schedule order, by offset and then roster position,
|
|
51
99
|
* each at its time or when a slot frees, whichever is later; this is the rule runOnSchedule
|
|
52
|
-
* follows.
|
|
100
|
+
* follows. `sessionsMs` gives a participant with a declared end its own session budget
|
|
101
|
+
* (participantSessionMs); the others hold their slot for `queue.sessionMs`.
|
|
53
102
|
*/
|
|
54
|
-
export declare function planArrivals(offsets: readonly (number | undefined)[], queue: StartQueue): ArrivalPlan;
|
|
103
|
+
export declare function planArrivals(offsets: readonly (number | undefined)[], queue: StartQueue, sessionsMs?: readonly number[]): ArrivalPlan;
|
|
55
104
|
/**
|
|
56
105
|
* The plan's schedule, after a "schedule:" label: the first and last start, and how many run at
|
|
57
106
|
* once at the most.
|
|
@@ -64,9 +113,7 @@ export declare function describeArrivals(arrivals: ArrivalPlan): string;
|
|
|
64
113
|
* sandbox with servedScheduleRefusal. `lowered` is the warning that the E2B plan's limit runs the
|
|
65
114
|
* roster in waves.
|
|
66
115
|
*/
|
|
67
|
-
export declare function planStarts(participants: readonly {
|
|
68
|
-
readonly startAfterMs?: number;
|
|
69
|
-
}[], queue: StartQueue, lowered: string | undefined): {
|
|
116
|
+
export declare function planStarts(participants: readonly ParticipantWindow[], queue: StartQueue, lowered: string | undefined): {
|
|
70
117
|
arrivals: ArrivalPlan;
|
|
71
118
|
warnings: string[];
|
|
72
119
|
};
|
package/dist/study/arrivals.js
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
// When each participant of a computer-use or shared-world run starts
|
|
2
|
-
// participant an offset from the moment the run starts its participants
|
|
3
|
-
// (`participants[].startAfterMs`)
|
|
1
|
+
// When each participant of a computer-use or shared-world run starts, and on computer use when it
|
|
2
|
+
// ends. A study may give each participant an offset from the moment the run starts its participants
|
|
3
|
+
// (`participants[].startAfterMs`) and, on computer use, an end on the same clock
|
|
4
|
+
// (`participants[].endAfterMs`), with a wind-down before it in which each turn tells the
|
|
5
|
+
// participant the time left (`windDownMs`). The plan simulates the start queue to report the schedule and
|
|
4
6
|
// when the last session ends at the latest; the routes start their participants through
|
|
5
7
|
// runOnSchedule, which waits for each one's time and then for a free slot. A participant's desktop
|
|
6
8
|
// is created when it starts, so a late start holds no sandbox before then.
|
|
@@ -20,6 +22,92 @@ export function isStartAfterMs(value) {
|
|
|
20
22
|
value >= 0 &&
|
|
21
23
|
value <= LATEST_START_AFTER_MS);
|
|
22
24
|
}
|
|
25
|
+
/** How long before its end a participant is told the time left, unless it declares `windDownMs`. */
|
|
26
|
+
export const DEFAULT_WIND_DOWN_MS = 10 * 60_000;
|
|
27
|
+
/**
|
|
28
|
+
* Room a scheduled participant's session keeps past its end for the turn in flight and the closing
|
|
29
|
+
* report. The loop ends the session at the end; this keeps the session's own time limit, and the
|
|
30
|
+
* sandbox sized from it, from cutting the closing report short.
|
|
31
|
+
*/
|
|
32
|
+
const SCHEDULED_END_ROOM_MS = 5 * 60_000;
|
|
33
|
+
/**
|
|
34
|
+
* Why a participant's declared end cannot run, or undefined: an end that is not a whole number of
|
|
35
|
+
* milliseconds up to 24 hours, one at or before its start, or a wind-down that is not a whole
|
|
36
|
+
* number of milliseconds or has no end. `path` names the entry, as in "participants[1]".
|
|
37
|
+
*/
|
|
38
|
+
export function windowReason(entry, path) {
|
|
39
|
+
const { endAfterMs, windDownMs } = entry;
|
|
40
|
+
if (windDownMs !== undefined && endAfterMs === undefined)
|
|
41
|
+
return `${path}.windDownMs is how long before its endAfterMs a participant is told the time left, so it needs ${path}.endAfterMs. Add the end, or remove windDownMs.`;
|
|
42
|
+
if (windDownMs !== undefined && !isStartAfterMs(windDownMs))
|
|
43
|
+
return `${path}.windDownMs must be a whole number of milliseconds from 0 to ${LATEST_START_AFTER_MS} (24 hours): how long before its end the participant is told the time left.`;
|
|
44
|
+
if (endAfterMs === undefined)
|
|
45
|
+
return undefined;
|
|
46
|
+
if (!isStartAfterMs(endAfterMs))
|
|
47
|
+
return `${path}.endAfterMs must be a whole number of milliseconds up to ${LATEST_START_AFTER_MS} (24 hours): when the participant's session ends, after the run starts its participants.`;
|
|
48
|
+
const startAfterMs = entry.startAfterMs ?? 0;
|
|
49
|
+
if (endAfterMs <= startAfterMs) {
|
|
50
|
+
const who = entry.id === undefined ? "" : ` ("${entry.id}")`;
|
|
51
|
+
return `${path}.endAfterMs${who} is ${endAfterMs} ms and its startAfterMs is ${startAfterMs} ms, so it would end before it starts. Set endAfterMs after startAfterMs.`;
|
|
52
|
+
}
|
|
53
|
+
return undefined;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* A participant's session budget: with a declared end, its window plus SCHEDULED_END_ROOM_MS in
|
|
57
|
+
* place of the study's `sessionMs` (execution.timeoutMs or its default); else `sessionMs`.
|
|
58
|
+
*/
|
|
59
|
+
export function participantSessionMs(participant, sessionMs) {
|
|
60
|
+
if (participant.endAfterMs === undefined)
|
|
61
|
+
return sessionMs;
|
|
62
|
+
return participant.endAfterMs - (participant.startAfterMs ?? 0) + SCHEDULED_END_ROOM_MS;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* When a participant that was due at `scheduledAt` (runOnSchedule's ScheduledStart) ends, on the
|
|
66
|
+
* same clock, or undefined when it declares no end. A participant that waited for a slot still ends
|
|
67
|
+
* at its declared time.
|
|
68
|
+
*/
|
|
69
|
+
export function scheduledEndAt(participant, scheduledAt) {
|
|
70
|
+
if (participant.endAfterMs === undefined)
|
|
71
|
+
return undefined;
|
|
72
|
+
return {
|
|
73
|
+
atMs: scheduledAt - (participant.startAfterMs ?? 0) + participant.endAfterMs,
|
|
74
|
+
windDownMs: participant.windDownMs ?? DEFAULT_WIND_DOWN_MS,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Why the sandbox of the first participant whose window passes `ceilingMs` cannot live that long,
|
|
79
|
+
* or undefined. Its deadline is its session (participantSessionMs) plus `headroomMs`, what the
|
|
80
|
+
* route adds to a session for setup and teardown (sandboxHeadroomMs).
|
|
81
|
+
*/
|
|
82
|
+
export function windowDeadlineRefusal(participants, sandbox, ceilingMs) {
|
|
83
|
+
const inMinutes = (ms) => Math.round(ms / 60_000);
|
|
84
|
+
for (const participant of participants) {
|
|
85
|
+
if (participant.endAfterMs === undefined)
|
|
86
|
+
continue;
|
|
87
|
+
const deadlineMs = participantSessionMs(participant, sandbox.sessionMs) + sandbox.headroomMs;
|
|
88
|
+
if (deadlineMs <= ceilingMs)
|
|
89
|
+
continue;
|
|
90
|
+
const name = participant.id === undefined ? "A participant" : `Participant "${participant.id}"`;
|
|
91
|
+
return `${name} works from +${formatDuration(participant.startAfterMs ?? 0)} to +${formatDuration(participant.endAfterMs)}, so its sandbox needs a ${inMinutes(deadlineMs)}m deadline: the window, ${inMinutes(SCHEDULED_END_ROOM_MS)}m for its closing report and ${inMinutes(sandbox.headroomMs)}m to set up and tear down. A sandbox may not live longer than ${ceilingMs / 60_000}m. Shorten its window.${ceilingAdvice(deadlineMs)}`;
|
|
92
|
+
}
|
|
93
|
+
return undefined;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Each participant's window for `study check`, when any participant declares an end: when it starts
|
|
97
|
+
* and ends and when it is first told the time left, or when it starts and its session budget.
|
|
98
|
+
*/
|
|
99
|
+
export function describeWindows(participants, sessionMs) {
|
|
100
|
+
if (!participants.some((participant) => participant.endAfterMs !== undefined))
|
|
101
|
+
return undefined;
|
|
102
|
+
const windows = participants.map((participant) => {
|
|
103
|
+
const start = `${participant.id ?? "participant"} from +${formatDuration(participant.startAfterMs ?? 0)}`;
|
|
104
|
+
if (participant.endAfterMs === undefined)
|
|
105
|
+
return `${start} for up to ${formatDuration(sessionMs)}`;
|
|
106
|
+
const told = Math.max(participant.startAfterMs ?? 0, participant.endAfterMs - (participant.windDownMs ?? DEFAULT_WIND_DOWN_MS));
|
|
107
|
+
return `${start} to +${formatDuration(participant.endAfterMs)}, told the time left from +${formatDuration(told)}`;
|
|
108
|
+
});
|
|
109
|
+
return `${windows.join("; ")}.`;
|
|
110
|
+
}
|
|
23
111
|
/** The refusal of a start offset that is not one, naming its `field`. */
|
|
24
112
|
export function startAfterMessage(field) {
|
|
25
113
|
return `${field} must be a whole number of milliseconds from 0 to ${LATEST_START_AFTER_MS} (24 hours): how long after the run starts its participants this one starts.`;
|
|
@@ -41,11 +129,13 @@ export function rosterStartsReason(roster) {
|
|
|
41
129
|
* The schedule a roster's declared offsets give under the queue (`undefined` is 0: the participant
|
|
42
130
|
* starts with the run). Participants start in schedule order, by offset and then roster position,
|
|
43
131
|
* each at its time or when a slot frees, whichever is later; this is the rule runOnSchedule
|
|
44
|
-
* follows.
|
|
132
|
+
* follows. `sessionsMs` gives a participant with a declared end its own session budget
|
|
133
|
+
* (participantSessionMs); the others hold their slot for `queue.sessionMs`.
|
|
45
134
|
*/
|
|
46
|
-
export function planArrivals(offsets, queue) {
|
|
135
|
+
export function planArrivals(offsets, queue, sessionsMs = []) {
|
|
47
136
|
const due = offsets.map((offset) => offset ?? 0);
|
|
48
|
-
const
|
|
137
|
+
const session = (index) => sessionsMs[index] ?? queue.sessionMs;
|
|
138
|
+
const starts = queueStarts(due, queue, session);
|
|
49
139
|
let waiting = 0;
|
|
50
140
|
let longestWaitMs = 0;
|
|
51
141
|
for (const [index, start] of starts.entries()) {
|
|
@@ -58,10 +148,10 @@ export function planArrivals(offsets, queue) {
|
|
|
58
148
|
declared: offsets.some((offset) => offset !== undefined),
|
|
59
149
|
firstStartMs: Math.min(...due),
|
|
60
150
|
lastStartMs: Math.max(...due),
|
|
61
|
-
peak: mostAtOnce(starts,
|
|
151
|
+
peak: mostAtOnce(starts, session),
|
|
62
152
|
waiting,
|
|
63
153
|
longestWaitMs,
|
|
64
|
-
lastEndMs: Math.max(...starts) +
|
|
154
|
+
lastEndMs: Math.max(...starts.map((start, index) => start + session(index))),
|
|
65
155
|
sessionMs: queue.sessionMs,
|
|
66
156
|
};
|
|
67
157
|
}
|
|
@@ -83,7 +173,7 @@ export function describeArrivals(arrivals) {
|
|
|
83
173
|
* roster in waves.
|
|
84
174
|
*/
|
|
85
175
|
export function planStarts(participants, queue, lowered) {
|
|
86
|
-
const arrivals = planArrivals(participants.map((participant) => participant.startAfterMs), queue);
|
|
176
|
+
const arrivals = planArrivals(participants.map((participant) => participant.startAfterMs), queue, participants.map((participant) => participantSessionMs(participant, queue.sessionMs)));
|
|
87
177
|
return { arrivals, warnings: startQueueWarnings(arrivals, queue, lowered) };
|
|
88
178
|
}
|
|
89
179
|
// A roster that starts together keeps `lowered`. A declared schedule spreads its participants
|
|
@@ -126,8 +216,8 @@ export function servedScheduleRefusal(arrivals, sandbox, ceilingMs) {
|
|
|
126
216
|
function scheduleOrder(due) {
|
|
127
217
|
return due.map((_, index) => index).sort((a, b) => due[a] - due[b] || a - b);
|
|
128
218
|
}
|
|
129
|
-
/** Each participant's start in roster order, when
|
|
130
|
-
function queueStarts(due, queue) {
|
|
219
|
+
/** Each participant's start in roster order, when each session holds its slot for `session(index)`. */
|
|
220
|
+
function queueStarts(due, queue, session) {
|
|
131
221
|
const starts = [...due];
|
|
132
222
|
const queued = scheduleOrder(due).filter((index) => index !== queue.ownSlot);
|
|
133
223
|
const pooled = queue.ownSlot === undefined ? queue.slots : queue.slots - 1;
|
|
@@ -137,16 +227,16 @@ function queueStarts(due, queue) {
|
|
|
137
227
|
const slot = free.indexOf(Math.min(...free));
|
|
138
228
|
const start = Math.max(due[index], free[slot]);
|
|
139
229
|
starts[index] = start;
|
|
140
|
-
free[slot] = start +
|
|
230
|
+
free[slot] = start + session(index);
|
|
141
231
|
}
|
|
142
232
|
return starts;
|
|
143
233
|
}
|
|
144
|
-
function mostAtOnce(starts,
|
|
234
|
+
function mostAtOnce(starts, session) {
|
|
145
235
|
// An end sorts before a start at the same instant: that slot is free again.
|
|
146
236
|
const changes = starts
|
|
147
|
-
.flatMap((start) => [
|
|
237
|
+
.flatMap((start, index) => [
|
|
148
238
|
{ at: start, by: 1 },
|
|
149
|
-
{ at: start +
|
|
239
|
+
{ at: start + session(index), by: -1 },
|
|
150
240
|
])
|
|
151
241
|
.sort((a, b) => a.at - b.at || a.by - b.by);
|
|
152
242
|
let live = 0;
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { DwellWindow } from "../actors/stop-conditions.js";
|
|
2
|
+
/** One participant as the plan has it: when it starts and ends, and its dwell window. */
|
|
3
|
+
export interface EstimatedParticipant {
|
|
4
|
+
readonly startAfterMs?: number;
|
|
5
|
+
readonly endAfterMs?: number;
|
|
6
|
+
readonly limits: {
|
|
7
|
+
readonly dwell?: DwellWindow;
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
/** A run's estimated bundle, and the scan's limit for its participants. */
|
|
11
|
+
export interface BundleEstimate {
|
|
12
|
+
/** run.json, which analysis reads whole. */
|
|
13
|
+
readonly runFileBytes: number;
|
|
14
|
+
/** The largest file: observer/observer-data.json, or the Observer page for a small run. */
|
|
15
|
+
readonly largestFileBytes: number;
|
|
16
|
+
/** Every file, screenshots included. */
|
|
17
|
+
readonly totalBytes: number;
|
|
18
|
+
/** Files and folders in the run folder, as the public-safety scan counts them. */
|
|
19
|
+
readonly entries: number;
|
|
20
|
+
/** The most entries the scan reads in a run of these participants (runEntryLimit). */
|
|
21
|
+
readonly entryLimit: number;
|
|
22
|
+
/** What the estimate assumed, for its messages. */
|
|
23
|
+
readonly basis: {
|
|
24
|
+
readonly participants: number;
|
|
25
|
+
/** Participants with a declared end, counted for their whole window, and the longest window. */
|
|
26
|
+
readonly scheduled: number;
|
|
27
|
+
readonly longestWindowMs: number;
|
|
28
|
+
/** Participants without one, and how long each is counted at most. */
|
|
29
|
+
readonly open: number;
|
|
30
|
+
readonly openMs: number;
|
|
31
|
+
readonly dwell: boolean;
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The bundle a live run of `participants` leaves, each in its session of `sessionMs` unless it
|
|
36
|
+
* declares an end. A participant with an end works its whole window; one without is counted for
|
|
37
|
+
* the measured open session above, or its budget when that is shorter. Each takes a model turn at
|
|
38
|
+
* the measured pace, and a dwell window adds a frame every `everyMs` for its length, cut to the
|
|
39
|
+
* session, and the frame after it.
|
|
40
|
+
*/
|
|
41
|
+
export declare function estimateBundle(participants: readonly EstimatedParticipant[], sessionMs: number): BundleEstimate;
|
|
42
|
+
/**
|
|
43
|
+
* The warning for a run whose estimate passes a limit, or undefined: each limit it passes, what the
|
|
44
|
+
* estimate counted, how high it runs, and what to change.
|
|
45
|
+
*/
|
|
46
|
+
export declare function bundleLimitReason(estimate: BundleEstimate): string | undefined;
|
|
47
|
+
/** The estimate beside the limits, for `study check`. */
|
|
48
|
+
export declare function describeBundle(estimate: BundleEstimate): string;
|