humanish 0.127.0 → 0.128.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/CONTEXT.md +2 -2
- package/README.md +1 -1
- package/dist/actors/closing-report.d.ts +8 -1
- package/dist/actors/closing-report.js +4 -1
- package/dist/actors/codex/restricted-participant-policy.js +2 -5
- package/dist/actors/computer-use/loop/debrief.js +6 -5
- package/dist/actors/computer-use/loop/ending.d.ts +5 -4
- package/dist/actors/computer-use/loop/ending.js +5 -4
- package/dist/actors/computer-use/loop/output-limit.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -10
- package/dist/actors/computer-use/loop.js +8 -3
- package/dist/actors/contract.d.ts +10 -6
- package/dist/cli/commands/study-format.js +13 -2
- package/dist/feedback/export.js +3 -2
- package/dist/observer/data.d.ts +13 -6
- package/dist/observer/data.js +40 -10
- package/dist/observer-app.html +8 -8
- package/dist/routes/computer-use/fanout-bundle.js +25 -6
- package/dist/routes/computer-use/participant-model.js +1 -1
- package/dist/run/bundle.d.ts +7 -4
- package/dist/run/bundle.js +10 -4
- package/dist/run/detail.js +8 -2
- package/dist/run/locate.d.ts +2 -2
- package/dist/run/locate.js +2 -2
- package/dist/run/run.js +11 -3
- package/dist/run/stream-trace.d.ts +17 -7
- package/dist/run/stream-trace.js +135 -29
- package/dist/run/streams.d.ts +4 -1
- package/dist/study/bundle-estimate.js +17 -17
- package/dist/verify/costs.d.ts +7 -3
- package/dist/verify/costs.js +27 -12
- package/dist/verify/verify.js +14 -8
- package/docs/architecture/actor-contract.md +7 -3
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/observer.md +19 -10
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +93 -15
- package/docs/contracts/schemas.md +55 -45
- package/docs/contracts/study-analysis.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +1 -1
- package/package.json +1 -1
package/dist/verify/costs.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// totals match their known lines, and account-billed participants never acquire a price.
|
|
3
3
|
import { validActorExecutionProfile, validActorProviderRequests } from "../actors/contract.js";
|
|
4
4
|
import { round6 } from "../run/pricing.js";
|
|
5
|
+
import { keepsRequestListsInRunJson } from "../run/stream-trace.js";
|
|
5
6
|
/** Account-billed participants must never acquire a price through an aggregate or ambiguous model line. */
|
|
6
7
|
export function contradictsAccountBilling(streams, cost) {
|
|
7
8
|
const account = (stream) => (stream.liveActor?.executionProfile ?? stream.actor?.executionProfile)?.billing ===
|
|
@@ -28,18 +29,23 @@ export function contradictsAccountBilling(streams, cost) {
|
|
|
28
29
|
* a null estimate, or a participant without estimatedCost is fine. A non-null figure must carry its
|
|
29
30
|
* ratesAsOf date + source; a number total must equal round6(sum of only the non-null lines) and a
|
|
30
31
|
* null line may never be coerced to 0. A null estimate must be declared (a reason + null
|
|
31
|
-
* ratesAsOf), mirroring the terminal no-spend proof's null-discipline.
|
|
32
|
-
*
|
|
32
|
+
* ratesAsOf), mirroring the terminal no-spend proof's null-discipline. A running participant's
|
|
33
|
+
* estimate is checked the same way. Account-billed participants also have their execution profile
|
|
34
|
+
* and provider request receipts checked here. From v4 a running participant's receipts are written
|
|
35
|
+
* only in its trace when it finishes, so its live actor holds none. The receipts of a stream in
|
|
36
|
+
* `refused` (its trace the loader refused) are not checked: from v4 they are only in that trace,
|
|
37
|
+
* and verify reports the refusal.
|
|
33
38
|
*/
|
|
34
|
-
export function costAndReceiptFindings(bundle) {
|
|
39
|
+
export function costAndReceiptFindings(bundle, refused) {
|
|
35
40
|
const findings = [];
|
|
36
41
|
if (contradictsAccountBilling(bundle.streams, bundle.cost))
|
|
37
42
|
findings.push("Run cost lines contradict account billing identity");
|
|
38
43
|
findings.push(...costSummaryFindings(bundle.cost));
|
|
44
|
+
const liveReceipts = keepsRequestListsInRunJson(bundle.schema);
|
|
39
45
|
for (const stream of bundle.streams) {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
46
|
+
const loaded = !refused.has(stream.id);
|
|
47
|
+
findings.push(...executionReceiptFindings(stream.actor, loaded));
|
|
48
|
+
findings.push(...executionReceiptFindings(stream.liveActor, liveReceipts && loaded));
|
|
43
49
|
findings.push(...participantEstimateFindings(stream));
|
|
44
50
|
}
|
|
45
51
|
return findings;
|
|
@@ -81,14 +87,17 @@ function costSummaryFindings(cost) {
|
|
|
81
87
|
}
|
|
82
88
|
return findings;
|
|
83
89
|
}
|
|
84
|
-
/**
|
|
85
|
-
|
|
90
|
+
/**
|
|
91
|
+
* An account-billed participant's execution profile and, when `receipts` says the trace holds
|
|
92
|
+
* them, its provider request receipts.
|
|
93
|
+
*/
|
|
94
|
+
function executionReceiptFindings(actor, receipts) {
|
|
86
95
|
const findings = [];
|
|
87
96
|
if (actor?.executionProfile === undefined)
|
|
88
97
|
return findings;
|
|
89
98
|
if (!validActorExecutionProfile(actor.executionProfile))
|
|
90
99
|
findings.push("Invalid actor execution profile");
|
|
91
|
-
if (!validActorProviderRequests(actor.providerRequests))
|
|
100
|
+
if (receipts && !validActorProviderRequests(actor.providerRequests))
|
|
92
101
|
findings.push("Invalid account participant request receipts");
|
|
93
102
|
if (actor.historyTurnsOmitted !== undefined &&
|
|
94
103
|
(!Number.isSafeInteger(actor.historyTurnsOmitted) || actor.historyTurnsOmitted < 0))
|
|
@@ -99,14 +108,20 @@ function executionReceiptFindings(actor) {
|
|
|
99
108
|
findings.push("Account participant dollars must remain unknown");
|
|
100
109
|
return findings;
|
|
101
110
|
}
|
|
111
|
+
/** The labeling of a participant's estimate: its finished actor's, and its live actor's while it runs. */
|
|
102
112
|
function participantEstimateFindings(stream) {
|
|
113
|
+
const participantLabel = stream.laneId ?? stream.id;
|
|
114
|
+
return [
|
|
115
|
+
...estimateFindings(stream.actor?.estimatedCost, participantLabel, "actor"),
|
|
116
|
+
...estimateFindings(stream.liveActor?.estimatedCost, participantLabel, "live actor"),
|
|
117
|
+
];
|
|
118
|
+
}
|
|
119
|
+
function estimateFindings(estimate, participantLabel, field) {
|
|
103
120
|
const findings = [];
|
|
104
|
-
const estimate = stream.actor?.estimatedCost;
|
|
105
121
|
if (!estimate)
|
|
106
122
|
return findings;
|
|
107
|
-
const participantLabel = stream.laneId ?? stream.id;
|
|
108
123
|
if (estimate.schema !== "humanish.actor-estimated-cost.v1") {
|
|
109
|
-
findings.push(`participant ${participantLabel}
|
|
124
|
+
findings.push(`participant ${participantLabel} ${field} estimatedCost schema is ${String(estimate.schema)}, expected humanish.actor-estimated-cost.v1`);
|
|
110
125
|
}
|
|
111
126
|
if (estimate.estimatedCostUsd !== null) {
|
|
112
127
|
if (typeof estimate.ratesAsOf !== "string" || estimate.ratesAsOf.length === 0) {
|
package/dist/verify/verify.js
CHANGED
|
@@ -5,7 +5,7 @@ import { analysisSharingProblems } from "../analysis/sharing.js";
|
|
|
5
5
|
import { containsSensitive, REDACTED_SANDBOX_ID } from "../evidence/redaction.js";
|
|
6
6
|
import { keyedSandboxIds, readRunSandboxIds } from "../run/sandbox-ids.js";
|
|
7
7
|
import { validatePreparedRunArtifactPaths } from "../run/paths.js";
|
|
8
|
-
import { isReadRunBundleSchema, RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA, RUN_BUNDLE_V1_SCHEMA, RUN_BUNDLE_V2_SCHEMA, } from "../run/bundle.js";
|
|
8
|
+
import { isReadRunBundleSchema, RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA, RUN_BUNDLE_V1_SCHEMA, RUN_BUNDLE_V2_SCHEMA, RUN_BUNDLE_V3_SCHEMA, } from "../run/bundle.js";
|
|
9
9
|
import { ContainedReadRefusedError, refusalText } from "../run/contained-output.js";
|
|
10
10
|
import { isCleanupResult, isRunBundle } from "../run/bundle-shape.js";
|
|
11
11
|
import { loadStreamTraces } from "../run/stream-trace.js";
|
|
@@ -139,7 +139,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
139
139
|
? [`missing local evidence artifacts: ${missingEvidenceArtifacts.join(", ")}`]
|
|
140
140
|
: []),
|
|
141
141
|
]));
|
|
142
|
-
checks.push(...(await evidenceChecks(runPaths, bundle, cleanupRead)));
|
|
142
|
+
checks.push(...(await evidenceChecks(runPaths, bundle, cleanupRead, traces.refusedStreams)));
|
|
143
143
|
const recordingOk = checks.every((check) => check.ok);
|
|
144
144
|
if (derivedPublicSafetyFindings.length > 0)
|
|
145
145
|
checks.push({
|
|
@@ -225,12 +225,13 @@ function evidenceArtifactsCheck(shaped, problems) {
|
|
|
225
225
|
}
|
|
226
226
|
/**
|
|
227
227
|
* run.json with each stream's trace loaded, one file at a time, and the refusal of each trace that
|
|
228
|
-
* does not match its index. A run.json that fails the shape guard
|
|
228
|
+
* does not match its index, with the ids of those streams. A run.json that fails the shape guard
|
|
229
|
+
* is returned as it is.
|
|
229
230
|
*/
|
|
230
231
|
async function loadTraces(runPaths, stored) {
|
|
231
232
|
const scanned = { sensitive: [], opaque: [] };
|
|
232
233
|
if (!isRunBundle(stored))
|
|
233
|
-
return { bundle: stored, refusedTraces: [], ...scanned, pages: [] };
|
|
234
|
+
return { bundle: stored, refusedTraces: [], refusedStreams: new Set(), ...scanned, pages: [] };
|
|
234
235
|
// Each trace is source evidence: scanned by the bytes it was joined from, apart from the
|
|
235
236
|
// directory walk and any path exemption the walk makes.
|
|
236
237
|
const { bundle, refused } = await loadStreamTraces(runPaths, stored, {
|
|
@@ -243,6 +244,7 @@ async function loadTraces(runPaths, stored) {
|
|
|
243
244
|
return {
|
|
244
245
|
bundle,
|
|
245
246
|
refusedTraces: refused.map((error) => error.message),
|
|
247
|
+
refusedStreams: new Set(refused.map((error) => error.streamId)),
|
|
246
248
|
...scanned,
|
|
247
249
|
pages: pageFindings(bundle),
|
|
248
250
|
};
|
|
@@ -312,7 +314,7 @@ function bundlePresenceChecks(bundleRead, review) {
|
|
|
312
314
|
? `run.json declares ${String(schema)}`
|
|
313
315
|
: record === undefined
|
|
314
316
|
? "no readable run.json, so no schema"
|
|
315
|
-
: `run.json declares ${typeof schema === "string" ? schema : "no schema"}; verify reads ${RUN_BUNDLE_SCHEMA}, ${RUN_BUNDLE_V2_SCHEMA} and ${RUN_BUNDLE_V1_SCHEMA}`,
|
|
317
|
+
: `run.json declares ${typeof schema === "string" ? schema : "no schema"}; verify reads ${RUN_BUNDLE_SCHEMA}, ${RUN_BUNDLE_V3_SCHEMA}, ${RUN_BUNDLE_V2_SCHEMA} and ${RUN_BUNDLE_V1_SCHEMA}`,
|
|
316
318
|
},
|
|
317
319
|
{
|
|
318
320
|
name: "run bundle shape",
|
|
@@ -378,8 +380,12 @@ function findingsCheck(name, findings, okMessage, failLabel) {
|
|
|
378
380
|
* check has no findings and stays ok; the shape check is the failure.
|
|
379
381
|
*/
|
|
380
382
|
const SHAPE_UNCHECKED = "not checked, because run.json failed the shape check";
|
|
381
|
-
/**
|
|
382
|
-
|
|
383
|
+
/**
|
|
384
|
+
* The bundle-content checks, in report order. A bundle that fails its shape guard has no findings.
|
|
385
|
+
* `refusedStreams` names the streams whose traces the loader refused, which the evidence-artifacts
|
|
386
|
+
* check reports.
|
|
387
|
+
*/
|
|
388
|
+
async function evidenceChecks(runPaths, bundle, cleanupRead, refusedStreams) {
|
|
383
389
|
const cleanupJson = runJsonValue(cleanupRead);
|
|
384
390
|
const valid = isRunBundle(bundle) ? bundle : null;
|
|
385
391
|
const pass = (message) => (valid ? message : SHAPE_UNCHECKED);
|
|
@@ -409,7 +415,7 @@ async function evidenceChecks(runPaths, bundle, cleanupRead) {
|
|
|
409
415
|
// Absence passes (fail-open on display); a claimed dollar figure without its ratesAsOf date +
|
|
410
416
|
// source, or a total that does not match its known lines, fails. Magnitude is never inspected:
|
|
411
417
|
// a correctly-labeled huge estimate still passes.
|
|
412
|
-
findingsCheck("cost estimate labeling", valid ? costAndReceiptFindings(valid) : [], pass("every cost estimate carries its ratesAsOf date and source and the total matches its lines, or the run claims no cost"), "cost labeling findings"),
|
|
418
|
+
findingsCheck("cost estimate labeling", valid ? costAndReceiptFindings(valid, refusedStreams) : [], pass("every cost estimate carries its ratesAsOf date and source and the total matches its lines, or the run claims no cost"), "cost labeling findings"),
|
|
413
419
|
];
|
|
414
420
|
return checks;
|
|
415
421
|
}
|
|
@@ -352,14 +352,18 @@ this paragraph is the record of that decision.
|
|
|
352
352
|
|
|
353
353
|
`ComputerUseProvider.debrief` is an optional read-only request after a structured
|
|
354
354
|
`stopWhen` or dwell stop. It returns a `ComputerUseTurn` with `closingReport` containing
|
|
355
|
-
`summary`, `frictionReports` and `impressions`; an empty friction or impressions list
|
|
356
|
-
and a report without `impressions` (an older provider) is still accepted.
|
|
355
|
+
`outcome`, `summary`, `frictionReports` and `impressions`; an empty friction or impressions list
|
|
356
|
+
is valid, and a report without `outcome` or `impressions` (an older provider) is still accepted.
|
|
357
|
+
After a scheduled end or session cap, `outcome` (`reached`, `not_reached` or `blocked`) is the
|
|
358
|
+
trace's `declaredOutcome`, read before the first line of `summary`. The loop rejects
|
|
357
359
|
actions, pending safety checks, and invalid report shapes. It redacts accepted
|
|
358
360
|
reports, records them in `ActorTrace.debrief`, and projects one readable message
|
|
359
361
|
without invoking action or communication callbacks. Each impression becomes its own message
|
|
360
362
|
and goes to `ActorTrace.impressions`; a session without them records why in the same field.
|
|
361
363
|
A turn that ends the session itself may carry `closingReport` too, and the loop keeps its
|
|
362
|
-
impressions the same way without a second request.
|
|
364
|
+
impressions the same way without a second request. Its `declaredOutcome` is the report's
|
|
365
|
+
`outcome`, then the turn's `outcome`, then the first line of its message, the order a closing
|
|
366
|
+
report at a scheduled end is read in.
|
|
363
367
|
|
|
364
368
|
`ComputerUseProvider.requestImpressions` is the optional request after the participant ended
|
|
365
369
|
the session itself without impressions. It asks for impressions only, returns them as
|
|
@@ -129,7 +129,7 @@ humanish_swarm:
|
|
|
129
129
|
- .env*
|
|
130
130
|
- .github/workflows/**
|
|
131
131
|
- infra/**
|
|
132
|
-
artifact_schema_version: humanish.run-bundle.
|
|
132
|
+
artifact_schema_version: humanish.run-bundle.v4
|
|
133
133
|
credential_manifest: []
|
|
134
134
|
network_policy: no_network
|
|
135
135
|
spend_policy: no_spend
|
|
@@ -25,12 +25,17 @@ Observer commands may add derived artifacts:
|
|
|
25
25
|
```
|
|
26
26
|
|
|
27
27
|
`run.json` remains the source bundle. `observer/observer-data.json`
|
|
28
|
-
(`humanish.observer-data.
|
|
29
|
-
with every participant except their trace items
|
|
30
|
-
artifact list
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
28
|
+
(`humanish.observer-data.v3`) is the normalized view model consumed by the Observer,
|
|
29
|
+
with every participant except their trace items, the screenshot entries of their
|
|
30
|
+
artifact list, and their per-request lists (`tokenUsage.turns`, `conversation.requests`
|
|
31
|
+
and `providerRequests`, `REQUEST_LISTS` in `src/run/stream-trace.ts`), which no view
|
|
32
|
+
shows. A participant working for hours records thousands of items and a screenshot each
|
|
33
|
+
turn and makes a model request every few seconds, so these are most of a long run's
|
|
34
|
+
bytes; apart, the saved file stays small enough for `verify` to read, and a flush does
|
|
35
|
+
not rewrite them. The items and lists stay in the run's traces (`actors/<stream>.json`).
|
|
36
|
+
`buildObserverData` leaves the lists out of every projection: observer-data.json, the
|
|
37
|
+
page's data slot, the served feed and the HTML export. The page's validator admits an
|
|
38
|
+
account participant's `executionProfile` without its receipts. `events.ndjson` is the appendable event stream
|
|
34
39
|
contract that live adapters update while a run is active.
|
|
35
40
|
|
|
36
41
|
`observer/index.html` carries both parts: the data slot holds observer-data.json's
|
|
@@ -41,9 +46,13 @@ trace items compress about eight times. The page joins the two before it renders
|
|
|
41
46
|
(`joinObserverData` in `src/observer/participant-items.ts`), so every view reads the
|
|
42
47
|
same data as before. A stream whose screenshot entries moved records how many in
|
|
43
48
|
`screenshotArtifacts`, and the page renders nothing from items that are another
|
|
44
|
-
run's or miss a participant's items or screenshot entries. The
|
|
45
|
-
|
|
46
|
-
|
|
49
|
+
run's or miss a participant's items or screenshot entries. The page reads only v3
|
|
50
|
+
data: it checks the schema in the data slot and the feed before it validates
|
|
51
|
+
(`observer/lib/data.ts`). A saved observer-data.json is read as v3
|
|
52
|
+
(`observerDataFromSaved`): a v1 file, which holds the items, without its lists, and
|
|
53
|
+
a v2 or v3 file only when no participant has items to join; the server answers one
|
|
54
|
+
that leaves items out, or one of another schema, as unavailable. `humanish export` rebuilds a page saved as v2
|
|
55
|
+
from the run and its traces, as it rebuilds a current one.
|
|
47
56
|
|
|
48
57
|
## Stream Model
|
|
49
58
|
|
|
@@ -281,7 +290,7 @@ These remain adapter work:
|
|
|
281
290
|
## The rendering layer
|
|
282
291
|
|
|
283
292
|
The renderer is the `observer/` workspace: a Vite single-file build on the
|
|
284
|
-
`@humanish` registry tokens, frozen against `humanish.observer-data.
|
|
293
|
+
`@humanish` registry tokens, frozen against `humanish.observer-data.v3`
|
|
285
294
|
(`tests/observer/data-contract.test.ts`). The root build copies the workspace
|
|
286
295
|
artifact to `dist/observer-app.html`; in a repo checkout a missing or stale
|
|
287
296
|
artifact auto-builds, and an unconditional preflight at CLI startup makes a
|
package/docs/contracts/README.md
CHANGED
|
@@ -8,7 +8,7 @@ disagree.
|
|
|
8
8
|
(`RunStudyOptions`) and the exports removed in 0.107.0.
|
|
9
9
|
- [core.md](core.md): the records a run writes under `.humanish/runs/<id>/` and how each route
|
|
10
10
|
starts a run.
|
|
11
|
-
- [run-bundle.md](run-bundle.md): `humanish.run-bundle.
|
|
11
|
+
- [run-bundle.md](run-bundle.md): `humanish.run-bundle.v4`, the evidence contract that `verify`
|
|
12
12
|
checks.
|
|
13
13
|
- [policy.md](policy.md): credential, network, spend, redaction, sharing and assisted-run
|
|
14
14
|
boundaries.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Run Bundle Contract
|
|
2
2
|
|
|
3
|
-
`humanish.run-bundle.
|
|
3
|
+
`humanish.run-bundle.v4` is the evidence contract. The TypeScript shape in `src/run/bundle.ts`
|
|
4
4
|
(with `streams[]` in `src/run/streams.ts` and `sharedWorld` as `SharedWorldEvidence` in
|
|
5
5
|
`src/run/shared-world-evidence.ts`) and the fail-closed checks in `src/verify/verify.ts` are the
|
|
6
6
|
source of truth. This page explains the stable public fields and the extension rules; it does not
|
|
@@ -39,7 +39,8 @@ trace:
|
|
|
39
39
|
|
|
40
40
|
run.json keeps every field readers list runs by: each stream's status, judged status,
|
|
41
41
|
assignment and geometry, and the actor's status, completion reason, counts, persona, debrief,
|
|
42
|
-
impressions, token usage and estimated cost. Each trace is written once.
|
|
42
|
+
impressions, token usage totals and estimated cost. Each trace is written once. From v4 the
|
|
43
|
+
per-request lists are only in the trace file ([Per-request lists](#per-request-lists)).
|
|
43
44
|
|
|
44
45
|
`trace.path` must be one of those trace file names (`actor.json`, `actors/<name>.json`,
|
|
45
46
|
`actor-<name>.json`, with `<name>` a plain file name); `isStreamTracePath` in
|
|
@@ -48,7 +49,7 @@ it. Any other path, such as `observer/index.html`, fails the shape check.
|
|
|
48
49
|
|
|
49
50
|
A reader outside humanish loads a run through the package's `loadRunBundle(cwd, run)`, with `run`
|
|
50
51
|
a run id or `latest`: it returns run.json with every trace file and live file joined and checked
|
|
51
|
-
as below, for v1, v2 and
|
|
52
|
+
as below, for v1, v2, v3 and v4 bundles, with each participant's per-request lists, and throws an Error named `StreamTraceRefusedError` for a
|
|
52
53
|
file that does not match its index. Inside humanish, readers load a stream through
|
|
53
54
|
`loadStreamTrace(runPaths, stream)`. A v1 stream returns as it is.
|
|
54
55
|
An indexed stream's file is read under the 32 MiB per-file limit (analysis uses its 16 MiB
|
|
@@ -82,8 +83,8 @@ stream with a `liveActor`:
|
|
|
82
83
|
`streams[]` (from 1). Each line is one trace item as JSON, scrubbed of the run's raw sandbox ids
|
|
83
84
|
as it is appended. A live file is never truncated or rewritten;
|
|
84
85
|
- in run.json, `liveActor` as `humanish.live-actor.v2`: its summary fields (`updatedAt`,
|
|
85
|
-
`persona`, `tokenUsage`, `ids`, `executionProfile`, `
|
|
86
|
-
`
|
|
86
|
+
`persona`, `tokenUsage`, `ids`, `executionProfile`, `historyTurnsOmitted`, `estimatedCost`, and
|
|
87
|
+
in v3 `providerRequests`) and, in place of `items`, the index of the prefix written so far:
|
|
87
88
|
|
|
88
89
|
```yaml
|
|
89
90
|
liveActor:
|
|
@@ -122,7 +123,7 @@ Readers load live items through the same loader as traces: `loadStreamTraces` re
|
|
|
122
123
|
indexed bytes of the live file (bytes appended after run.json was written are not read), refuses
|
|
123
124
|
them unless their size, sha256 and line count match the index and each line is a JSON object,
|
|
124
125
|
and returns the stream with `liveActor.items` and without `itemsFile`. It refuses a `liveActor`
|
|
125
|
-
schema it does not know, and inline live items (`humanish.live-actor.v1`) in a v3 run.json. A
|
|
126
|
+
schema it does not know, and inline live items (`humanish.live-actor.v1`) in a v3 or v4 run.json. A
|
|
126
127
|
reader that cannot read a live file reads run.json again once and loads that bundle instead, since
|
|
127
128
|
the writer may have removed a superseded generation, or every live file at finish, after the
|
|
128
129
|
reader read run.json. v1 and v2 run.json keep `liveActor.items` inline (`humanish.live-actor.v1`)
|
|
@@ -138,15 +139,92 @@ used as a bundle answers 404 with the reason, never the saved projection. In saf
|
|
|
138
139
|
(`observe --all --safe`) every read is checked against the files admitted with the run's
|
|
139
140
|
share_ready verification, so a run.json changed after admission is refused and the admitted
|
|
140
141
|
projection is served.
|
|
141
|
-
observer-data.json is built from the loaded bundle. From 0.124.0 it
|
|
142
|
-
|
|
143
|
-
|
|
142
|
+
observer-data.json is built from the loaded bundle. From 0.124.0 it leaves out each participant's
|
|
143
|
+
trace items, which stay in the traces, and from this release it is `humanish.observer-data.v3`,
|
|
144
|
+
which also leaves out each participant's per-request lists (docs/architecture/observer.md).
|
|
145
|
+
|
|
146
|
+
## Per-request lists
|
|
147
|
+
|
|
148
|
+
From v4, three lists that gain one entry per model request are only in each participant's trace
|
|
149
|
+
file: `tokenUsage.turns` (each request's usage), `conversation.requests` (each request's mode and,
|
|
150
|
+
in explicit_context, what it carried) and `providerRequests` (an account participant's receipt for
|
|
151
|
+
each dispatch). Before v4 run.json held a copy of each beside the trace file's, and a flush wrote
|
|
152
|
+
every running participant's usage rows into run.json and observer-data.json. A synthetic run of
|
|
153
|
+
30 participants making 2,880 requests each, written through the run writer, leaves a run.json of
|
|
154
|
+
about 0.1 MB, where the same run with the lists was 16 MB with threaded conversations and 42 MB
|
|
155
|
+
with account receipts. `REQUEST_LISTS` in `src/run/stream-trace.ts` names the three paths, and the
|
|
156
|
+
writer, the loader and the Observer projection read it.
|
|
157
|
+
|
|
158
|
+
The writer (`splitStreamTraces`) writes a finished participant's trace file as before, lists
|
|
159
|
+
included, and run.json's `streams[].actor` without `items` and without the three lists:
|
|
160
|
+
`tokenUsage` keeps its totals and `conversation` its `mode` and `summarizedTurns`.
|
|
161
|
+
|
|
162
|
+
For a running participant, run.json's `liveActor` holds no `tokenUsage.turns` and no
|
|
163
|
+
`providerRequests`. When it has `tokenUsage` and a model id in `ids.model`, the writer prices its
|
|
164
|
+
rows before leaving them out and records the figure as `liveActor.estimatedCost`
|
|
165
|
+
(`humanish.actor-estimated-cost.v1`, from `estimateActorCostForExecution` in
|
|
166
|
+
`src/run/pricing.ts`), in place of any estimate the producer passed. The long-context tier prices
|
|
167
|
+
a request by its own input size, so the totals alone can price a long session at under half its
|
|
168
|
+
cost. An account participant's estimate is a null figure with the reason
|
|
169
|
+
`account_billing_unknown`.
|
|
144
170
|
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
171
|
+
```yaml
|
|
172
|
+
liveActor:
|
|
173
|
+
schema: humanish.live-actor.v2
|
|
174
|
+
updatedAt: "<ISO timestamp>"
|
|
175
|
+
tokenUsage: { input: 412000, output: 9100, cachedInput: 380000, cacheWriteInput: 21000 }
|
|
176
|
+
ids: { model: gpt-5.6-sol }
|
|
177
|
+
estimatedCost:
|
|
178
|
+
schema: humanish.actor-estimated-cost.v1
|
|
179
|
+
estimatedCostUsd: 0.4123
|
|
180
|
+
ratesAsOf: "2026-09-03"
|
|
181
|
+
source: "<rate source>"
|
|
182
|
+
itemsFile: { path: live/1.1.ndjson, generation: 1, items: 4210, bytes: 1048211, sha256: "<hex>" }
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
The rows and receipts are written once, in the trace file, when the participant finishes. A run
|
|
186
|
+
interrupted before then keeps each running participant's totals, estimate and live file. No file
|
|
187
|
+
holds that participant's rows or receipts, including a receipt with `usageComplete: false` or
|
|
188
|
+
`cleanup: "unconfirmed"`, so neither the trace path nor `loadRunBundle` recovers them.
|
|
189
|
+
|
|
190
|
+
The loader joins the lists back for every indexed stream, whatever its version: each list the
|
|
191
|
+
trace file holds goes into run.json's `tokenUsage` or `conversation` when run.json has that
|
|
192
|
+
object, and `providerRequests` at the actor's top level. A v2 or v3 writer wrote run.json's copy
|
|
193
|
+
and the file's from one actor, so their result does not change. A trace file whose `conversation`
|
|
194
|
+
has no `requests` list, or that holds one of the three as something other than a list, is
|
|
195
|
+
refused.
|
|
196
|
+
|
|
197
|
+
Which writer rules the loader applies depends on the version run.json declares. A run.json that
|
|
198
|
+
breaks one did not come from that version's writer, and the stream is refused before any file is
|
|
199
|
+
read:
|
|
200
|
+
|
|
201
|
+
| Rule | v1 | v2 | v3 | v4 |
|
|
202
|
+
| ----------------------------------------------------------- | ------- | ------- | ------- | ------- |
|
|
203
|
+
| Two streams index one trace file | refused | refused | refused | refused |
|
|
204
|
+
| An actor without a trace index | read | refused | refused | refused |
|
|
205
|
+
| `liveActor` in a run whose `outcome.state` is `finished` | read | refused | refused | refused |
|
|
206
|
+
| Inline live items (`humanish.live-actor.v1`) | read | read | refused | refused |
|
|
207
|
+
| One of the three lists in run.json's `actor` or `liveActor` | read | read | read | refused |
|
|
208
|
+
|
|
209
|
+
A reader that shows a run (`displayedTraces`) shows a refused stream without the lists run.json held
|
|
210
|
+
for it, so run detail never prices rows the loader refused. Run detail shows a refused participant's
|
|
211
|
+
cost only when run.json records an `estimatedCost` for it, since its totals alone could price under
|
|
212
|
+
half of what its rows give. verify checks a running participant's `liveActor.estimatedCost` as it
|
|
213
|
+
checks a finished one's: its schema, a `ratesAsOf` date and `source` beside a figure, and a reason
|
|
214
|
+
beside a null figure. In a v4 run.json it no longer asks a running account participant for receipts;
|
|
215
|
+
the check that account dollars stay unknown still applies, and the run still grades `local_only`
|
|
216
|
+
with `RUN_NOT_FINISHED`. verify does not check the receipts of a participant whose trace the loader
|
|
217
|
+
refused, in any version: the refusal fails `local evidence artifacts exist`, and from v4 the
|
|
218
|
+
receipts are only in that trace.
|
|
219
|
+
|
|
220
|
+
Compatibility: humanish 0.122.0 and earlier read only v1 and refuse a later run.json by its schema
|
|
221
|
+
(`run schema` fails in verify, analysis refuses the source); 0.123.0 and 0.124.0 read v1 and v2
|
|
222
|
+
and refuse v3 and v4 the same way. 0.125.0 and 0.126.0 read v1, v2 and v3: their verify fails
|
|
223
|
+
`run schema` on a v4 run.json, their analysis refuses it, and their `loadRunBundle` returns null.
|
|
224
|
+
Their Observer server and run detail parse run.json without checking its schema and show a v4
|
|
225
|
+
run without the lists; the 0.126.0 page refuses an account participant's data without receipts.
|
|
226
|
+
This release reads v1, v2 and v3 bundles as they are, with no migration. An exported copy
|
|
227
|
+
(`humanish export --format bundle`) is written as v4 whatever the source version.
|
|
150
228
|
|
|
151
229
|
## Purpose
|
|
152
230
|
|
|
@@ -157,7 +235,7 @@ feedback drafts and future public issues.
|
|
|
157
235
|
## Minimum Bundle Shape
|
|
158
236
|
|
|
159
237
|
```yaml
|
|
160
|
-
schema: humanish.run-bundle.
|
|
238
|
+
schema: humanish.run-bundle.v4 # v3, v2 and v1 bundles are still read
|
|
161
239
|
runId: "<core run id>"
|
|
162
240
|
mode: "dry-run|live"
|
|
163
241
|
simCount: 1
|