humanish 0.129.0 → 0.131.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/actors/codex/restricted-participant.d.ts +2 -2
- package/dist/actors/codex/restricted-participant.js +3 -3
- package/dist/actors/computer-use/loop/actions.d.ts +17 -2
- package/dist/actors/computer-use/loop/actions.js +24 -5
- package/dist/actors/computer-use/loop/backstop.d.ts +9 -1
- package/dist/actors/computer-use/loop/backstop.js +4 -3
- package/dist/actors/computer-use/loop/clock.d.ts +4 -4
- package/dist/actors/computer-use/loop/clock.js +5 -5
- package/dist/actors/computer-use/loop/observation.d.ts +21 -2
- package/dist/actors/computer-use/loop/observation.js +50 -8
- package/dist/actors/computer-use/loop/trace.js +5 -1
- package/dist/actors/computer-use/loop/types.d.ts +30 -19
- package/dist/actors/computer-use/loop/watch.d.ts +42 -0
- package/dist/actors/computer-use/loop/watch.js +260 -0
- package/dist/actors/computer-use/loop.js +18 -4
- package/dist/actors/computer-use/wait.d.ts +36 -18
- package/dist/actors/computer-use/wait.js +51 -22
- package/dist/actors/contract.d.ts +34 -0
- package/dist/actors/local-agent/cli.js +5 -0
- package/dist/actors/stop-cause.d.ts +1 -1
- package/dist/analysis/automatic.js +0 -1
- package/dist/analysis/evidence-sources.d.ts +44 -7
- package/dist/analysis/evidence-sources.js +40 -6
- package/dist/analysis/evidence.js +96 -122
- package/dist/analysis/service.js +12 -6
- package/dist/analysis/source.d.ts +28 -0
- package/dist/analysis/source.js +118 -0
- package/dist/evidence/frame-change.d.ts +39 -0
- package/dist/evidence/frame-change.js +293 -0
- package/dist/feedback/export.js +0 -1
- package/dist/observer/data.d.ts +2 -2
- package/dist/observer-app.html +1 -1
- package/dist/run/bundle-shape.d.ts +2 -2
- package/dist/run/bundle.d.ts +8 -1
- package/dist/run/detail.js +4 -1
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/stream-shape.d.ts +2 -2
- package/dist/run/stream-trace.d.ts +21 -7
- package/dist/run/stream-trace.js +42 -8
- package/dist/run/streams.d.ts +18 -0
- package/dist/study/bundle-estimate.d.ts +4 -1
- package/dist/study/bundle-estimate.js +40 -15
- package/dist/study/plan-participants.d.ts +1 -1
- package/dist/study/types.d.ts +11 -7
- package/dist/study/warnings.js +1 -1
- package/dist/verify/verify.js +28 -21
- package/docs/architecture/actor-contract.md +8 -0
- package/docs/architecture/browser-control.md +11 -9
- package/docs/contracts/run-bundle.md +2 -2
- package/docs/contracts/schemas.md +48 -8
- package/docs/contracts/study-analysis.md +13 -1
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { type ReviewSummary, type RunAdapterScore, type
|
|
1
|
+
import { type ReviewSummary, type RunAdapterScore, type StoredRunBundle, type RunOutcome } from "./bundle.js";
|
|
2
2
|
import { type StoredCleanupResult, type RunPointer } from "./results.js";
|
|
3
|
-
export declare function isRunBundle(value: unknown): value is
|
|
3
|
+
export declare function isRunBundle(value: unknown): value is StoredRunBundle;
|
|
4
4
|
/** run.json's `outcome`: a finished run's ok and execution outcome, or the signal that stopped it. */
|
|
5
5
|
export declare function isRunOutcome(value: unknown): value is RunOutcome;
|
|
6
6
|
export declare function isCleanupResult(value: unknown): value is StoredCleanupResult;
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { ExecutionOutcome } from "./judge.js";
|
|
|
4
4
|
import type { SharedWorldEvidence } from "./shared-world-evidence.js";
|
|
5
5
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
6
6
|
import type { RunInterruptSignal } from "./status.js";
|
|
7
|
-
import type { RunSimulationStatus, RunStream, RunStreamKind } from "./streams.js";
|
|
7
|
+
import type { RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
|
|
8
8
|
/**
|
|
9
9
|
* The run bundle version this release writes: each stream's trace, and each running participant's
|
|
10
10
|
* items so far, are files run.json indexes, and each participant's per-request lists are only in
|
|
@@ -388,6 +388,13 @@ export interface RunBundle {
|
|
|
388
388
|
*/
|
|
389
389
|
cost?: RunCostSummary;
|
|
390
390
|
}
|
|
391
|
+
/**
|
|
392
|
+
* run.json as a reader parses it, in any version: each stream a StoredRunStream, without the
|
|
393
|
+
* per-request lists that from v4 only its trace file holds. loadStreamTraces gives the RunBundle.
|
|
394
|
+
*/
|
|
395
|
+
export type StoredRunBundle = Omit<RunBundle, "streams"> & {
|
|
396
|
+
streams: StoredRunStream[];
|
|
397
|
+
};
|
|
391
398
|
/**
|
|
392
399
|
* How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
|
|
393
400
|
* and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
|
package/dist/run/detail.js
CHANGED
|
@@ -119,7 +119,10 @@ function costOf(trace, rowsLoaded) {
|
|
|
119
119
|
* items run.json or a file supplied for it.
|
|
120
120
|
*/
|
|
121
121
|
async function displayedStreams(runPaths, streams, bundle) {
|
|
122
|
-
const loaded = await loadStreamTraces(runPaths, {
|
|
122
|
+
const loaded = await loadStreamTraces(runPaths, {
|
|
123
|
+
...bundle,
|
|
124
|
+
streams,
|
|
125
|
+
});
|
|
123
126
|
return displayedTraces(loaded).streams;
|
|
124
127
|
}
|
|
125
128
|
/**
|
|
@@ -21,7 +21,7 @@ export declare function participantEvent(ids: ParticipantIds, event: Omit<RunEve
|
|
|
21
21
|
/** The ids a provider resource saves for the participant whose sandbox it is. */
|
|
22
22
|
export declare function participantResourceIds(ids: ParticipantIds, participantId: string): Required<Pick<RunProviderResource, "simId" | "streamId" | "laneId">>;
|
|
23
23
|
/** The record id a saved stream belongs to. */
|
|
24
|
-
export declare function recordIdOf(stream: RunStream): string;
|
|
24
|
+
export declare function recordIdOf(stream: Pick<RunStream, "simId">): string;
|
|
25
25
|
/** The record id a saved event names, when it names one. */
|
|
26
26
|
export declare function eventRecordIdOf(event: RunEvent): string | undefined;
|
|
27
27
|
/** The participant id an adapter recorded on a saved stream (a fan-out `lane-NN` id), when it has one. */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { type RunSimulationStatus, type RunStream, type RunStreamKind, type StreamTraceIndex } from "./streams.js";
|
|
2
|
-
export declare function isRunStream(value: unknown): value is
|
|
1
|
+
import { type RunSimulationStatus, type RunStream, type RunStreamKind, type StoredRunStream, type StreamTraceIndex } from "./streams.js";
|
|
2
|
+
export declare function isRunStream(value: unknown): value is StoredRunStream;
|
|
3
3
|
export declare function isRunStreamArtifact(value: unknown): value is RunStream["artifacts"][number];
|
|
4
4
|
/**
|
|
5
5
|
* The trace file names the run writer produces: actor.json for one participant,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import { type RunBundle } from "./bundle.js";
|
|
1
|
+
import { type RunBundle, type StoredRunBundle } from "./bundle.js";
|
|
2
2
|
import { type PreparedOutputRoot } from "./contained-output.js";
|
|
3
3
|
import { type LiveItemsCursor } from "./live-items.js";
|
|
4
|
-
import type { LiveItemsIndex, RunStream } from "./streams.js";
|
|
4
|
+
import type { LiveItemsIndex, RunStream, StoredRunStream, WithoutRequestLists } from "./streams.js";
|
|
5
5
|
/**
|
|
6
6
|
* A trace that does not match its index, or cannot be read. The message names the stream, and ends
|
|
7
7
|
* with `advice`: by default, restore the run folder from a copy.
|
|
@@ -10,8 +10,14 @@ export declare class StreamTraceRefusedError extends Error {
|
|
|
10
10
|
readonly streamId: string;
|
|
11
11
|
constructor(streamId: string, detail: string, advice?: string);
|
|
12
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* A trace or live file whose index records more than the loader reads of one file. The loader reads
|
|
15
|
+
* no file past the limit, so a copy of the file is refused too.
|
|
16
|
+
*/
|
|
17
|
+
export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
18
|
+
}
|
|
13
19
|
/** `trace` without the per-request lists; every other field as it is. */
|
|
14
|
-
export declare function withoutRequestLists<T extends object>(trace: T): T
|
|
20
|
+
export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
|
|
15
21
|
/**
|
|
16
22
|
* The run.json the run writes, and the trace files it writes first. Each stream with a finished
|
|
17
23
|
* actor keeps that actor without `items` and the per-request lists, its artifacts other than
|
|
@@ -21,7 +27,7 @@ export declare function withoutRequestLists<T extends object>(trace: T): T;
|
|
|
21
27
|
* run.json held beside it. A running participant's live actor is written by liveSummary.
|
|
22
28
|
*/
|
|
23
29
|
export declare function splitStreamTraces(bundle: RunBundle): {
|
|
24
|
-
bundle:
|
|
30
|
+
bundle: StoredRunBundle;
|
|
25
31
|
files: {
|
|
26
32
|
path: string;
|
|
27
33
|
bytes: Buffer;
|
|
@@ -69,7 +75,7 @@ type LiveAfter = (index: LiveItemsIndex) => LiveItemsCursor | undefined;
|
|
|
69
75
|
* before the file is read. The rules that need the whole bundle (an unindexed v2
|
|
70
76
|
* actor, two participants indexing one file) are loadStreamTraces'.
|
|
71
77
|
*/
|
|
72
|
-
export declare function loadStreamTrace(runPaths: PreparedOutputRoot, stream:
|
|
78
|
+
export declare function loadStreamTrace(runPaths: PreparedOutputRoot, stream: StoredRunStream, maxBytes?: number): Promise<RunStream>;
|
|
73
79
|
/**
|
|
74
80
|
* Whether a run.json of `schema` holds the per-request lists (REQUEST_LISTS) the writer kept in
|
|
75
81
|
* it: v1 to v3, and a schema WRITER_RULES does not name, which is read as v1. From v4 they are
|
|
@@ -102,7 +108,7 @@ interface LoadedTraces {
|
|
|
102
108
|
* run finished. The loader then reads run.json again, once, through the same reader, and loads
|
|
103
109
|
* that bundle instead when it is a run bundle of the same run.
|
|
104
110
|
*/
|
|
105
|
-
export declare function loadStreamTraces(source: PreparedOutputRoot | TraceFileReader, bundle:
|
|
111
|
+
export declare function loadStreamTraces(source: PreparedOutputRoot | TraceFileReader, bundle: StoredRunBundle, options?: {
|
|
106
112
|
maxBytes?: number;
|
|
107
113
|
onTrace?: (path: string, bytes: Buffer) => void;
|
|
108
114
|
liveAfter?: LiveAfter;
|
|
@@ -120,6 +126,14 @@ export declare function displayedTraces(loaded: {
|
|
|
120
126
|
refused: readonly StreamTraceRefusedError[];
|
|
121
127
|
live?: readonly LiveItemsRead[];
|
|
122
128
|
}): RunBundle;
|
|
129
|
+
/**
|
|
130
|
+
* What `reduce` returns for each stream of the bundle, loaded as loadRunTraces loads it, in stream
|
|
131
|
+
* order; the first refusal throws. Streams are loaded one at a time and nothing of a stream's files
|
|
132
|
+
* is kept past its `reduce` call, so a caller that keeps part of each trace holds one trace at a
|
|
133
|
+
* time where loadRunTraces holds them all. Unlike loadStreamTraces it does not read run.json again
|
|
134
|
+
* when a live file cannot be read, so it suits a run whose participants have finished.
|
|
135
|
+
*/
|
|
136
|
+
export declare function reduceRunTraces<T>(runPaths: PreparedOutputRoot, bundle: StoredRunBundle, reduce: (stream: RunStream, index: number) => T): Promise<T[]>;
|
|
123
137
|
/** The bundle with every stream loaded; the first refusal throws. */
|
|
124
|
-
export declare function loadRunTraces(source: PreparedOutputRoot | TraceFileReader, bundle:
|
|
138
|
+
export declare function loadRunTraces(source: PreparedOutputRoot | TraceFileReader, bundle: StoredRunBundle, maxBytes?: number): Promise<RunBundle>;
|
|
125
139
|
export {};
|
package/dist/run/stream-trace.js
CHANGED
|
@@ -28,10 +28,13 @@ export class StreamTraceRefusedError extends Error {
|
|
|
28
28
|
/** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
|
|
29
29
|
const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
|
|
30
30
|
/**
|
|
31
|
-
*
|
|
31
|
+
* A trace or live file whose index records more than the loader reads of one file. The loader reads
|
|
32
32
|
* no file past the limit, so a copy of the file is refused too.
|
|
33
33
|
*/
|
|
34
|
-
|
|
34
|
+
export class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
35
|
+
}
|
|
36
|
+
/** The refusal of a trace or live file whose index records more than `maxBytes`. */
|
|
37
|
+
const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
35
38
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
36
39
|
const isScreenshot = (artifact) => artifact.kind === "screenshot";
|
|
37
40
|
/**
|
|
@@ -236,6 +239,13 @@ function assertIndex(stream) {
|
|
|
236
239
|
if (stream.actor.items !== undefined)
|
|
237
240
|
throw new StreamTraceRefusedError(stream.id, "run.json holds items beside its trace index");
|
|
238
241
|
}
|
|
242
|
+
/**
|
|
243
|
+
* A stream of run.json typed as a loaded one, for the streams the loader joins no trace into: one
|
|
244
|
+
* without an index, which v1 wrote with its actor's lists, and a refused one, kept as run.json
|
|
245
|
+
* holds it. From v4 a refused stream's actor has none of the lists, and its `conversation` lacks
|
|
246
|
+
* the `requests` ActorTrace declares; the loader's `refused` names each such stream.
|
|
247
|
+
*/
|
|
248
|
+
const unjoined = (stream) => stream;
|
|
239
249
|
/** A live actor whose items run.json holds inline: humanish.live-actor.v1, or one with no schema. */
|
|
240
250
|
const isInlineLiveActor = (live) => live.schema === LIVE_ACTOR_V1_SCHEMA || live.schema === undefined;
|
|
241
251
|
/** A live file that could not be read, which a writer that moved on may have removed. */
|
|
@@ -329,7 +339,7 @@ async function readLiveTail(read, stream, index, after) {
|
|
|
329
339
|
*/
|
|
330
340
|
async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
331
341
|
const files = [];
|
|
332
|
-
let joined = stream;
|
|
342
|
+
let joined = unjoined(stream);
|
|
333
343
|
if (stream.trace !== undefined) {
|
|
334
344
|
assertIndex(stream);
|
|
335
345
|
if (stream.trace.bytes > maxBytes)
|
|
@@ -453,20 +463,25 @@ async function rereadBundle(read, runId) {
|
|
|
453
463
|
return undefined;
|
|
454
464
|
}
|
|
455
465
|
}
|
|
456
|
-
|
|
466
|
+
/** bundleRefusal for each stream of `bundle`, with the streams that index each file counted once. */
|
|
467
|
+
function bundleRefusals(bundle) {
|
|
457
468
|
const owners = new Map();
|
|
458
469
|
for (const stream of bundle.streams)
|
|
459
470
|
if (isRecord(stream.trace)) {
|
|
460
471
|
const key = traceFileKey(String(stream.trace.path));
|
|
461
472
|
owners.set(key, [...(owners.get(key) ?? []), stream.id]);
|
|
462
473
|
}
|
|
474
|
+
return (stream) => bundleRefusal(bundle, stream, owners);
|
|
475
|
+
}
|
|
476
|
+
async function loadEveryStream(read, bundle, maxBytes, liveAfter) {
|
|
477
|
+
const refusal = bundleRefusals(bundle);
|
|
463
478
|
const streams = [];
|
|
464
479
|
const refused = [];
|
|
465
480
|
const files = [];
|
|
466
481
|
const live = [];
|
|
467
482
|
for (const stream of bundle.streams) {
|
|
468
483
|
try {
|
|
469
|
-
const reason =
|
|
484
|
+
const reason = refusal(stream);
|
|
470
485
|
if (reason !== undefined)
|
|
471
486
|
throw new StreamTraceRefusedError(stream.id, reason);
|
|
472
487
|
const loaded = await readStreamTrace(read, stream, maxBytes, liveAfter);
|
|
@@ -479,7 +494,7 @@ async function loadEveryStream(read, bundle, maxBytes, liveAfter) {
|
|
|
479
494
|
if (!(error instanceof StreamTraceRefusedError))
|
|
480
495
|
throw error;
|
|
481
496
|
refused.push(error);
|
|
482
|
-
streams.push(stream);
|
|
497
|
+
streams.push(unjoined(stream));
|
|
483
498
|
}
|
|
484
499
|
}
|
|
485
500
|
return { bundle: { ...bundle, streams }, refused, files, live };
|
|
@@ -502,12 +517,12 @@ export function displayedTraces(loaded) {
|
|
|
502
517
|
if (reason === undefined)
|
|
503
518
|
return stream;
|
|
504
519
|
const { liveActor: _live, trace: _index, actor, ...rest } = stream;
|
|
505
|
-
return {
|
|
520
|
+
return unjoined({
|
|
506
521
|
...rest,
|
|
507
522
|
...(actor === undefined ? {} : { actor: { ...withoutRequestLists(actor), items: [] } }),
|
|
508
523
|
artifacts: stream.artifacts.filter((artifact) => !isScreenshot(artifact)),
|
|
509
524
|
traceUnavailable: reason,
|
|
510
|
-
};
|
|
525
|
+
});
|
|
511
526
|
});
|
|
512
527
|
const events = loaded.bundle.streams
|
|
513
528
|
.filter((stream) => reasons.has(stream.id))
|
|
@@ -533,6 +548,25 @@ export function displayedTraces(loaded) {
|
|
|
533
548
|
events: [...loaded.bundle.events, ...events, ...notRead],
|
|
534
549
|
};
|
|
535
550
|
}
|
|
551
|
+
/**
|
|
552
|
+
* What `reduce` returns for each stream of the bundle, loaded as loadRunTraces loads it, in stream
|
|
553
|
+
* order; the first refusal throws. Streams are loaded one at a time and nothing of a stream's files
|
|
554
|
+
* is kept past its `reduce` call, so a caller that keeps part of each trace holds one trace at a
|
|
555
|
+
* time where loadRunTraces holds them all. Unlike loadStreamTraces it does not read run.json again
|
|
556
|
+
* when a live file cannot be read, so it suits a run whose participants have finished.
|
|
557
|
+
*/
|
|
558
|
+
export async function reduceRunTraces(runPaths, bundle, reduce) {
|
|
559
|
+
const read = containedReader(runPaths);
|
|
560
|
+
const refusal = bundleRefusals(bundle);
|
|
561
|
+
const kept = [];
|
|
562
|
+
for (const [index, stream] of bundle.streams.entries()) {
|
|
563
|
+
const reason = refusal(stream);
|
|
564
|
+
if (reason !== undefined)
|
|
565
|
+
throw new StreamTraceRefusedError(stream.id, reason);
|
|
566
|
+
kept.push(reduce((await readStreamTrace(read, stream, RUN_ARTIFACT_MAX_BYTES)).stream, index));
|
|
567
|
+
}
|
|
568
|
+
return kept;
|
|
569
|
+
}
|
|
536
570
|
/** The bundle with every stream loaded; the first refusal throws. */
|
|
537
571
|
export async function loadRunTraces(source, bundle, maxBytes) {
|
|
538
572
|
const loaded = await loadStreamTraces(source, bundle, maxBytes === undefined ? {} : { maxBytes });
|
package/dist/run/streams.d.ts
CHANGED
|
@@ -316,4 +316,22 @@ export interface RunStream {
|
|
|
316
316
|
kind: "bundle" | "review" | "observer" | "events" | "screenshot" | "trace" | "log" | "filesystem" | "recording";
|
|
317
317
|
}>;
|
|
318
318
|
}
|
|
319
|
+
/**
|
|
320
|
+
* `T` without a participant's per-request lists (REQUEST_LISTS in src/run/stream-trace.ts):
|
|
321
|
+
* `tokenUsage.turns`, `conversation.requests` and `providerRequests`. From run bundle v4 only the
|
|
322
|
+
* trace file holds them, so a reader of run.json or of the Observer's data has none to read.
|
|
323
|
+
*/
|
|
324
|
+
export type WithoutRequestLists<T> = Omit<T, "providerRequests" | "tokenUsage" | "conversation"> & {
|
|
325
|
+
[K in "tokenUsage" & keyof T]?: Omit<NonNullable<T[K]>, "turns"> | undefined;
|
|
326
|
+
} & {
|
|
327
|
+
[K in "conversation" & keyof T]?: Omit<NonNullable<T[K]>, "requests"> | undefined;
|
|
328
|
+
};
|
|
329
|
+
/**
|
|
330
|
+
* A stream as run.json holds it, in any version: its actor and live actor WithoutRequestLists.
|
|
331
|
+
* loadStreamTraces joins the lists from the trace file.
|
|
332
|
+
*/
|
|
333
|
+
export type StoredRunStream = Omit<RunStream, "actor" | "liveActor"> & {
|
|
334
|
+
actor?: WithoutRequestLists<ActorTrace>;
|
|
335
|
+
liveActor?: WithoutRequestLists<NonNullable<RunStream["liveActor"]>>;
|
|
336
|
+
};
|
|
319
337
|
export {};
|
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import type { DwellWindow } from "../actors/stop-conditions.js";
|
|
2
2
|
import type { ParticipantSchedule } from "./arrivals.js";
|
|
3
|
-
/** One participant as the plan has it: its schedule
|
|
3
|
+
/** One participant as the plan has it: its schedule, its dwell window and its mission. */
|
|
4
4
|
export interface EstimatedParticipant {
|
|
5
5
|
readonly schedule: ParticipantSchedule;
|
|
6
6
|
readonly limits: {
|
|
7
7
|
readonly dwell?: DwellWindow;
|
|
8
8
|
};
|
|
9
|
+
readonly assignment: {
|
|
10
|
+
readonly mission?: string;
|
|
11
|
+
};
|
|
9
12
|
}
|
|
10
13
|
/** A run's estimated bundle, and the scan's limit for its participants. */
|
|
11
14
|
export interface BundleEstimate {
|
|
@@ -1,14 +1,18 @@
|
|
|
1
1
|
// How large a computer-use run's bundle grows, estimated from its plan before anything runs, and
|
|
2
|
-
// whether it would pass a limit a later command holds it to: analysis reads run.json
|
|
3
|
-
//
|
|
4
|
-
// RUN_ARTIFACT_MAX_BYTES,
|
|
5
|
-
// files and folders.
|
|
2
|
+
// whether it would pass a limit a later command holds it to: analysis reads run.json up to
|
|
3
|
+
// EVIDENCE_LIMITS.sourceBytes, analysis, verify and the Observer read each participant's trace up to
|
|
4
|
+
// RUN_ARTIFACT_MAX_BYTES, verify reads no file larger than RUN_ARTIFACT_MAX_BYTES, and the
|
|
5
|
+
// public-safety scan visits at most runEntryLimit(participants) files and folders.
|
|
6
6
|
//
|
|
7
7
|
// observer/index.html carries every participant's items gzipped, so with busy participants it
|
|
8
8
|
// passes verify's 32 MiB long before run.json passes analysis's 16 MiB: at these rates, 20
|
|
9
9
|
// participants working 8 hours. From run bundle v4 run.json grows by participant, not by model
|
|
10
|
-
// request: each participant's per-request lists are only in its trace. run.json
|
|
11
|
-
//
|
|
10
|
+
// request: each participant's per-request lists are only in its trace. run.json holds the mission
|
|
11
|
+
// once for the run, once in each participant's stream and once more in the feedback candidate of
|
|
12
|
+
// each participant that reports friction or stops. The mission is not capped, and the estimate
|
|
13
|
+
// counts that last copy for every participant, so it warns at 100 participants with a mission of
|
|
14
|
+
// 75,238 bytes. On 2026-10-13 the longest mission on the live computer-use runs kept on the
|
|
15
|
+
// maintainer's machine was 807 characters.
|
|
12
16
|
// observer-data.json leaves the lists out too and was 0.94 to 1.12 of run.json's size on the live
|
|
13
17
|
// runs measured, so it passes 32 MiB only after run.json passes 16 MiB, and the estimate does not
|
|
14
18
|
// check it.
|
|
@@ -63,8 +67,12 @@ const RATES = Object.freeze({
|
|
|
63
67
|
/** observer/index.html's app before any run data: 1,199,831 bytes on the dry runs. */
|
|
64
68
|
observerAppBytes: 1_200_000,
|
|
65
69
|
/**
|
|
66
|
-
* The page's compact copy of observer-data.json, as a share of run.json: 0.71
|
|
67
|
-
* live runs written as v4, where neither carries the per-request lists.
|
|
70
|
+
* The page's compact copy of observer-data.json, as a share of run.json before the mission: 0.71
|
|
71
|
+
* to 0.84 on the live runs written as v4, where neither carries the per-request lists. The page
|
|
72
|
+
* holds the mission at its full size once for the run and once for each participant: dry runs of
|
|
73
|
+
* 1 and 4 participants grew the page by 2 and 5 copies, as run.json grew, and a 0.128.0 run of 2
|
|
74
|
+
* participants with 2 feedback candidates held 5 copies in run.json and 3 on the page. An
|
|
75
|
+
* analysis adds each participant's assignment, cut to 8,000 bytes.
|
|
68
76
|
*/
|
|
69
77
|
observerPageDataShare: 0.85,
|
|
70
78
|
/**
|
|
@@ -97,11 +105,13 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
97
105
|
let frames = 0;
|
|
98
106
|
let traces = 0;
|
|
99
107
|
let largestTrace = 0;
|
|
108
|
+
let missionBytes = 0;
|
|
109
|
+
let longestMission = 0;
|
|
100
110
|
let longestWindowMs = 0;
|
|
101
111
|
// The longest any participant without an end is counted, for the messages.
|
|
102
112
|
let openMs = Math.min(sessionMs, RATES.openSessionMs);
|
|
103
113
|
let scheduled = 0;
|
|
104
|
-
for (const { schedule, limits } of participants) {
|
|
114
|
+
for (const { schedule, limits, assignment } of participants) {
|
|
105
115
|
const { end } = schedule;
|
|
106
116
|
const budgetMs = end?.windowMs ?? sessionMs;
|
|
107
117
|
let countedMs;
|
|
@@ -122,12 +132,26 @@ export function estimateBundle(participants, sessionMs) {
|
|
|
122
132
|
frames += ownFrames;
|
|
123
133
|
traces += trace;
|
|
124
134
|
largestTrace = Math.max(largestTrace, trace);
|
|
135
|
+
// The mission's bytes as run.json stores them, a JSON string without its quotes.
|
|
136
|
+
const mission = assignment.mission === undefined
|
|
137
|
+
? 0
|
|
138
|
+
: Buffer.byteLength(JSON.stringify(assignment.mission)) - 2;
|
|
139
|
+
missionBytes += mission;
|
|
140
|
+
longestMission = Math.max(longestMission, mission);
|
|
125
141
|
}
|
|
126
142
|
const count = participants.length;
|
|
127
|
-
const
|
|
143
|
+
const dataBytes = RATES.runBytes + count * RATES.participantBytes;
|
|
144
|
+
// run.json holds the mission in the run's scenario goal, in each participant's stream and in the
|
|
145
|
+
// feedback candidate of each participant that reports friction or stops, counted for every
|
|
146
|
+
// participant. The rates were fit on runs with short missions, so a short mission is counted
|
|
147
|
+
// twice.
|
|
148
|
+
const runFileBytes = dataBytes + longestMission + 2 * missionBytes;
|
|
128
149
|
const screenshots = count + turns + frames;
|
|
150
|
+
// The page holds no feedback candidate's copy.
|
|
129
151
|
const observerPageBytes = Math.round(RATES.observerAppBytes +
|
|
130
|
-
RATES.observerPageDataShare *
|
|
152
|
+
RATES.observerPageDataShare * dataBytes +
|
|
153
|
+
longestMission +
|
|
154
|
+
missionBytes +
|
|
131
155
|
RATES.observerPageItemsShare * traces);
|
|
132
156
|
return {
|
|
133
157
|
runFileBytes,
|
|
@@ -158,12 +182,12 @@ export function bundleLimitReason(estimate) {
|
|
|
158
182
|
const passed = [
|
|
159
183
|
...(runFile
|
|
160
184
|
? [
|
|
161
|
-
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads.`,
|
|
185
|
+
`This study's run.json may grow to about ${mib(estimate.runFileBytes)}, past the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads. It holds the mission once for the run, once in each participant's stream and once more for each participant that reports friction or stops, and the estimate counts the last copy for every participant.`,
|
|
162
186
|
]
|
|
163
187
|
: []),
|
|
164
|
-
...(estimate.largestTraceBytes >
|
|
188
|
+
...(estimate.largestTraceBytes > RUN_ARTIFACT_MAX_BYTES
|
|
165
189
|
? [
|
|
166
|
-
`One participant's trace may grow to about ${mib(estimate.largestTraceBytes)}, past the ${mib(
|
|
190
|
+
`One participant's trace may grow to about ${mib(estimate.largestTraceBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} humanish reads of each trace.`,
|
|
167
191
|
]
|
|
168
192
|
: []),
|
|
169
193
|
...(page
|
|
@@ -195,6 +219,7 @@ export function bundleLimitReason(estimate) {
|
|
|
195
219
|
const changes = [
|
|
196
220
|
...(scheduled > 0 ? ["shorten the longest windows (participants[].endAfterMs)"] : []),
|
|
197
221
|
...(dwell ? ["shorten the dwell window (dwell.ms) or capture less often (dwell.everyMs)"] : []),
|
|
222
|
+
...(runFile ? ["shorten the mission (actor.mission)"] : []),
|
|
198
223
|
];
|
|
199
224
|
// A trace is one participant's, so splitting the study helps only the run's own files.
|
|
200
225
|
if (runFile || page || entries || changes.length === 0)
|
|
@@ -211,7 +236,7 @@ export function bundleLimitReason(estimate) {
|
|
|
211
236
|
}
|
|
212
237
|
/** The estimate beside the limits, for `study check`. */
|
|
213
238
|
export function describeBundle(estimate) {
|
|
214
|
-
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest trace about ${mib(estimate.largestTraceBytes)} of the ${mib(
|
|
239
|
+
return `run.json about ${mib(estimate.runFileBytes)} of the ${mib(EVIDENCE_LIMITS.sourceBytes)} analysis reads; largest trace about ${mib(estimate.largestTraceBytes)} of the ${mib(RUN_ARTIFACT_MAX_BYTES)} humanish reads of each; Observer page about ${mib(estimate.observerPageBytes)} of the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads; about ${count(estimate.entries)} files and folders of the ${count(estimate.entryLimit)} the public-safety scan reads; about ${mib(estimate.totalBytes, 0)} in all. Participants with a declared end are counted for their whole window and the others for up to ${formatDuration(estimate.basis.openMs)}, at a model turn every ${formatDuration(RATES.secondsPerTurn * 1000)}.`;
|
|
215
240
|
}
|
|
216
241
|
/** Bytes in MiB to `digits` places, or two under 0.1 MiB, so a short study's trace does not read 0.0. */
|
|
217
242
|
function mib(bytes, digits = 1) {
|
|
@@ -41,7 +41,7 @@ export interface DesktopParticipant extends Participant {
|
|
|
41
41
|
readonly maxWaitMs?: number;
|
|
42
42
|
/**
|
|
43
43
|
* The participant's idleWaitMs, else actor.idleWaitMs: how long a wait with no duration lasts
|
|
44
|
-
* while the participant only waits.
|
|
44
|
+
* while the participant only waits, and how often the loop looks at its screen then.
|
|
45
45
|
*/
|
|
46
46
|
readonly idleWaitMs?: number;
|
|
47
47
|
/**
|
package/dist/study/types.d.ts
CHANGED
|
@@ -361,7 +361,8 @@ export interface StudyParticipantEntry {
|
|
|
361
361
|
windDownMs?: number;
|
|
362
362
|
/**
|
|
363
363
|
* Computer use and shared world: this participant's actor.idleWaitMs, how long a wait that names
|
|
364
|
-
* no duration lasts while it only waits
|
|
364
|
+
* no duration lasts while it only waits and how often the loop looks at its screen then. Same
|
|
365
|
+
* range, and no longer than actor.maxWaitMs.
|
|
365
366
|
*/
|
|
366
367
|
idleWaitMs?: number;
|
|
367
368
|
/**
|
|
@@ -404,16 +405,19 @@ export interface StudyActor {
|
|
|
404
405
|
maxOutputTokens?: number;
|
|
405
406
|
/**
|
|
406
407
|
* Computer-use and shared-world: the longest one `wait` action lasts, in whole milliseconds from
|
|
407
|
-
* 1000 to
|
|
408
|
-
* to it, the trace records a `wait shortened` notice, and the
|
|
409
|
-
* turn.
|
|
408
|
+
* 1000 to 3600000. Default 120000, or 30000 on a desktop with speech. In a turn with another
|
|
409
|
+
* action a longer wait is shortened to it, the trace records a `wait shortened` notice, and the
|
|
410
|
+
* participant is told on its next turn. A turn that only waits is watched: its named waits last
|
|
411
|
+
* their lengths, together up to this or to 600000 when it is not set, and on a screen that stays
|
|
412
|
+
* the same its watches in a row grow to the same cap.
|
|
410
413
|
*/
|
|
411
414
|
maxWaitMs?: number;
|
|
412
415
|
/**
|
|
413
416
|
* Computer-use and shared-world: how long a `wait` that names no duration lasts when the
|
|
414
|
-
* participant's turn only waits or takes screenshots,
|
|
415
|
-
*
|
|
416
|
-
*
|
|
417
|
+
* participant's turn only waits or takes screenshots, and how often the loop looks at the screen
|
|
418
|
+
* while it watches such a turn, in whole milliseconds from 1000 to 600000 and no longer than
|
|
419
|
+
* maxWaitMs. Default 5000. After an action in the same turn such a wait lasts 500 ms. OpenAI's
|
|
420
|
+
* computer-use `wait` never names a duration.
|
|
417
421
|
*/
|
|
418
422
|
idleWaitMs?: number;
|
|
419
423
|
/**
|
package/dist/study/warnings.js
CHANGED
|
@@ -66,7 +66,7 @@ const ACTOR_ROWS = [
|
|
|
66
66
|
},
|
|
67
67
|
{
|
|
68
68
|
field: "actor.idleWaitMs",
|
|
69
|
-
reason: "how long a computer-use participant's wait with no duration lasts; needs route: computer-use or shared-world",
|
|
69
|
+
reason: "how long a computer-use participant's wait with no duration lasts and how often it looks while it waits; needs route: computer-use or shared-world",
|
|
70
70
|
applies: (config, routes) => config.actor?.idleWaitMs !== undefined && !routes.cua && !routes.shared,
|
|
71
71
|
},
|
|
72
72
|
];
|
package/dist/verify/verify.js
CHANGED
|
@@ -80,7 +80,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
80
80
|
// Every check below reads the streams with their traces loaded; a trace that does not match its
|
|
81
81
|
// index fails the evidence-artifacts check and its stream is checked without its items.
|
|
82
82
|
const traces = await loadTraces(runPaths, runJsonValue(bundleRead));
|
|
83
|
-
const { bundle, refusedTraces } = traces;
|
|
83
|
+
const { bundle, recorded, refusedTraces } = traces;
|
|
84
84
|
const cleanupRead = await readRunJsonIfExists(runPaths, "cleanup.json");
|
|
85
85
|
const review = {
|
|
86
86
|
json: await readRunJsonIfExists(runPaths, "review.json"),
|
|
@@ -101,20 +101,20 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
101
101
|
unscannedArtifacts.files.push(SANDBOX_RECEIPTS_ARTIFACT);
|
|
102
102
|
}
|
|
103
103
|
const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
|
|
104
|
-
recordingPaths: new Set(
|
|
104
|
+
recordingPaths: new Set(bundle
|
|
105
105
|
? bundle.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : []))
|
|
106
106
|
: []),
|
|
107
|
-
screenshotPaths:
|
|
107
|
+
screenshotPaths: bundle ? streamScreenshotPaths(bundle) : new Set(),
|
|
108
108
|
// A run from before 0.110 with no receipts still names its ids in run.json.
|
|
109
|
-
sandboxIds: [...new Set([...journaledSandboxIds, ...keyedSandboxIds(
|
|
109
|
+
sandboxIds: [...new Set([...journaledSandboxIds, ...keyedSandboxIds(recorded)])],
|
|
110
110
|
sandboxIdFiles,
|
|
111
|
-
}, unscannedArtifacts,
|
|
111
|
+
}, unscannedArtifacts, bundle ? bundle.simulations.length : 0);
|
|
112
112
|
addTraceScan(traces, publicSafetyFindings, unscannedArtifacts);
|
|
113
113
|
// JSON escapes can hide a sensitive value from the byte scan while the
|
|
114
114
|
// decoded recording exposes it to Observer, feedback, or analysis input.
|
|
115
115
|
if (publicSafetyFindings.length < MAX_REPORTED_FINDINGS &&
|
|
116
|
-
|
|
117
|
-
containsSensitive(JSON.stringify(
|
|
116
|
+
recorded !== null &&
|
|
117
|
+
containsSensitive(JSON.stringify(recorded))) {
|
|
118
118
|
publicSafetyFindings.push("sensitive decoded run.json");
|
|
119
119
|
}
|
|
120
120
|
checks.push({
|
|
@@ -124,14 +124,14 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
124
124
|
? "no run text artifact or public-proof path matches a known secret or browser-profile pattern"
|
|
125
125
|
: `public-safety findings: ${publicSafetyFindings.slice(0, 5).join(", ")}`,
|
|
126
126
|
});
|
|
127
|
-
const missingEvidenceArtifacts =
|
|
127
|
+
const missingEvidenceArtifacts = bundle
|
|
128
128
|
? await missingLocalEvidenceArtifacts(runPaths, bundle)
|
|
129
129
|
: [];
|
|
130
|
-
const redactedShapeFrames =
|
|
130
|
+
const redactedShapeFrames = bundle
|
|
131
131
|
? await redactedShapeFramePaths(runPaths, bundle)
|
|
132
132
|
: new Set();
|
|
133
|
-
const invalidEvidenceReferences =
|
|
134
|
-
checks.push(evidenceArtifactsCheck(
|
|
133
|
+
const invalidEvidenceReferences = bundle ? invalidRunEvidenceReferences(bundle) : [];
|
|
134
|
+
checks.push(evidenceArtifactsCheck(bundle !== undefined, [
|
|
135
135
|
...refusedTraces,
|
|
136
136
|
...(invalidEvidenceReferences.length > 0
|
|
137
137
|
? [`invalid evidence artifact references: ${invalidEvidenceReferences.join(", ")}`]
|
|
@@ -148,10 +148,10 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
148
148
|
message: "Derived analysis contains sensitive text or unsafe artifact paths; sharing is blocked, original recording remains independently verifiable.",
|
|
149
149
|
});
|
|
150
150
|
const ok = checks.every((check) => check.ok);
|
|
151
|
-
const notFinished =
|
|
151
|
+
const notFinished = bundle
|
|
152
152
|
? await runNotFinished(runPaths, bundle)
|
|
153
153
|
: { warnings: [], unfinished: undefined };
|
|
154
|
-
const warnings =
|
|
154
|
+
const warnings = bundle
|
|
155
155
|
? [
|
|
156
156
|
...rawScreenshotPostureWarnings(bundle, redactedShapeFrames),
|
|
157
157
|
...undeclaredSubjectStateWarnings(bundle),
|
|
@@ -160,7 +160,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
160
160
|
...notFinished.warnings,
|
|
161
161
|
]
|
|
162
162
|
: [];
|
|
163
|
-
const shareSafety =
|
|
163
|
+
const shareSafety = bundle
|
|
164
164
|
? buildShareSafety({
|
|
165
165
|
ok,
|
|
166
166
|
bundle,
|
|
@@ -226,12 +226,19 @@ function evidenceArtifactsCheck(shaped, problems) {
|
|
|
226
226
|
/**
|
|
227
227
|
* run.json with each stream's trace loaded, one file at a time, and the refusal of each trace that
|
|
228
228
|
* does not match its index, with the ids of those streams. A run.json that fails the shape guard
|
|
229
|
-
* is
|
|
229
|
+
* gives no bundle. `recorded` is the loaded bundle, or run.json as parsed when it has none.
|
|
230
230
|
*/
|
|
231
231
|
async function loadTraces(runPaths, stored) {
|
|
232
232
|
const scanned = { sensitive: [], opaque: [] };
|
|
233
233
|
if (!isRunBundle(stored))
|
|
234
|
-
return {
|
|
234
|
+
return {
|
|
235
|
+
bundle: undefined,
|
|
236
|
+
recorded: stored,
|
|
237
|
+
refusedTraces: [],
|
|
238
|
+
refusedStreams: new Set(),
|
|
239
|
+
...scanned,
|
|
240
|
+
pages: [],
|
|
241
|
+
};
|
|
235
242
|
// Each trace is source evidence: scanned by the bytes it was joined from, apart from the
|
|
236
243
|
// directory walk and any path exemption the walk makes.
|
|
237
244
|
const { bundle, refused } = await loadStreamTraces(runPaths, stored, {
|
|
@@ -243,6 +250,7 @@ async function loadTraces(runPaths, stored) {
|
|
|
243
250
|
});
|
|
244
251
|
return {
|
|
245
252
|
bundle,
|
|
253
|
+
recorded: bundle,
|
|
246
254
|
refusedTraces: refused.map((error) => error.message),
|
|
247
255
|
refusedStreams: new Set(refused.map((error) => error.streamId)),
|
|
248
256
|
...scanned,
|
|
@@ -381,13 +389,12 @@ function findingsCheck(name, findings, okMessage, failLabel) {
|
|
|
381
389
|
*/
|
|
382
390
|
const SHAPE_UNCHECKED = "not checked, because run.json failed the shape check";
|
|
383
391
|
/**
|
|
384
|
-
* The bundle-content checks
|
|
385
|
-
* `refusedStreams` names the streams whose traces
|
|
386
|
-
* check reports.
|
|
392
|
+
* The bundle-content checks of the loaded bundle `valid`, in report order. Without one (run.json
|
|
393
|
+
* failed its shape guard) no check has findings. `refusedStreams` names the streams whose traces
|
|
394
|
+
* the loader refused, which the evidence-artifacts check reports.
|
|
387
395
|
*/
|
|
388
|
-
async function evidenceChecks(runPaths,
|
|
396
|
+
async function evidenceChecks(runPaths, valid, cleanupRead, refusedStreams) {
|
|
389
397
|
const cleanupJson = runJsonValue(cleanupRead);
|
|
390
|
-
const valid = isRunBundle(bundle) ? bundle : null;
|
|
391
398
|
const pass = (message) => (valid ? message : SHAPE_UNCHECKED);
|
|
392
399
|
const checks = [
|
|
393
400
|
findingsCheck("terminal-product evidence", valid ? await validateTerminalProductEvidence(runPaths, valid) : [], pass("live terminal-product evidence is complete (ledgers, redacted output, proven teardown, known spend within the cap), or the run has none"), "terminal-product findings"),
|
|
@@ -168,6 +168,14 @@ export interface ActorTraceItem {
|
|
|
168
168
|
screenshotRef?: { path: string; redaction: "blurred" | "ocr_scrubbed" | "none" };
|
|
169
169
|
text?: string; // redacted
|
|
170
170
|
page?: { origin: string; path: string }; // computer-use screenshot items; redacted, no query
|
|
171
|
+
// computer-use: the wait ui_action during which a watch of a turn that only waited ended
|
|
172
|
+
watch?: {
|
|
173
|
+
limitMs: number;
|
|
174
|
+
askedMs: number; // what the turn's waits asked for; above limitMs past the watch's growth cap
|
|
175
|
+
wake: "changed" | "speech" | "condition" | "limit" | "clock" | "stalled" | "stopped";
|
|
176
|
+
looks: number;
|
|
177
|
+
area?: { x: number; y: number; width: number; height: number }; // pixels, coordinates only
|
|
178
|
+
};
|
|
171
179
|
}
|
|
172
180
|
|
|
173
181
|
export interface ActorCapabilities {
|