humanish 0.127.1 → 0.129.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/CONTEXT.md +2 -2
- package/README.md +1 -1
- package/dist/actors/computer-use/loop/output-limit.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -10
- package/dist/actors/computer-use/loop.js +6 -1
- package/dist/analysis/evidence.js +1 -4
- package/dist/analysis/validation.js +1 -1
- package/dist/cli/commands/study-format.js +2 -1
- package/dist/feedback/export.js +3 -2
- package/dist/observer/data.d.ts +13 -6
- package/dist/observer/data.js +40 -10
- package/dist/observer/feed-cursors.d.ts +3 -1
- package/dist/observer/feed-cursors.js +5 -2
- package/dist/observer/run-routes.js +40 -7
- package/dist/observer-app.html +8 -8
- package/dist/routes/computer-use/fanout-bundle.js +4 -7
- package/dist/routes/computer-use/participant-model.js +1 -1
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.d.ts +8 -0
- package/dist/routes/shared-world/bundle.js +33 -13
- package/dist/routes/shared-world/participant-specs.d.ts +2 -1
- package/dist/routes/shared-world/participant-specs.js +5 -2
- package/dist/routes/shared-world/result.js +13 -7
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle.d.ts +7 -4
- package/dist/run/bundle.js +10 -4
- package/dist/run/detail.js +8 -2
- package/dist/run/locate.d.ts +2 -2
- package/dist/run/locate.js +2 -2
- package/dist/run/outcomes.d.ts +7 -0
- package/dist/run/outcomes.js +14 -0
- package/dist/run/run.js +11 -3
- package/dist/run/stream-trace.d.ts +41 -14
- package/dist/run/stream-trace.js +195 -42
- package/dist/run/streams.d.ts +4 -1
- package/dist/study/bundle-estimate.js +17 -17
- package/dist/verify/costs.d.ts +7 -3
- package/dist/verify/costs.js +27 -12
- package/dist/verify/verify.js +14 -8
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/observer.md +32 -10
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +102 -18
- package/docs/contracts/schemas.md +55 -45
- package/docs/contracts/study-analysis.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +1 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,32 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.129.0: Analysis reads long runs past 100,000 items; traces past 32 MiB get their own advice; shared-world skips counted apart (2026-10-13)
|
|
12
|
+
|
|
13
|
+
humanish 0.129.0 fixes three things long studies reach. Analysis no longer refuses a run by how
|
|
14
|
+
many events, trace items or screenshot frames it holds; it still reads each file only up to 16
|
|
15
|
+
MiB. A trace or live file past 32 MiB, the most the trace loader reads, is refused with advice
|
|
16
|
+
that fits, a running participant whose live file passes 32 MiB keeps the items the Observer page
|
|
17
|
+
already shows, and the Observer server reads run.json under that limit. Shared-world runs count a
|
|
18
|
+
participant skipped before its session apart from those that used the app, and no longer count a
|
|
19
|
+
skip as overlap. This is a minor release because a shared-world `skippedReason` is now a sentence
|
|
20
|
+
where it was `handoff-timeout` or `host-ended-before-handoff`.
|
|
21
|
+
|
|
22
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.129.0)
|
|
23
|
+
|
|
24
|
+
## 0.128.0: Run bundle v4 keeps per-request lists in trace files; the study budget is checked before each request (2026-10-13)
|
|
25
|
+
|
|
26
|
+
humanish 0.128.0 keeps run.json small in long studies. Run bundles are now
|
|
27
|
+
`humanish.run-bundle.v4`: each participant's per-request lists (`tokenUsage.turns`,
|
|
28
|
+
`conversation.requests` and `providerRequests`) are only in its trace file, and run.json keeps the
|
|
29
|
+
totals. A synthetic run of 30 participants making 2,880 requests each wrote run.json at 93 to 106
|
|
30
|
+
KB, where 0.127.1 wrote 16.1, 27.2 and 41.8 MB; analysis reads run.json only up to 16 MiB. This
|
|
31
|
+
release reads v1 to v4 bundles, and `loadRunBundle` joins the lists back. 0.127.1 and earlier fail
|
|
32
|
+
verify on a v4 run. A computer-use participant now checks the study budget before each model
|
|
33
|
+
request, so one whose sibling crossed `caps.maxTotalUsd` no longer sends one more request.
|
|
34
|
+
|
|
35
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.128.0)
|
|
36
|
+
|
|
11
37
|
## 0.127.1: Clock-end closing reports declare their outcome; skipped participants named as skipped (2026-10-13)
|
|
12
38
|
|
|
13
39
|
humanish 0.127.1 fixes two things the 0.127.0 release smoke found in scheduled studies. An OpenAI
|
package/CONTEXT.md
CHANGED
|
@@ -12,7 +12,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
12
12
|
| Term | Meaning | Where it lives in the code |
|
|
13
13
|
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
14
14
|
| study | What a user designs and runs: one YAML file that declares the subject, the actor, the participants and the caps. The file lives at `humanish/studies/<id>.yaml` with schema `humanish.study.v3`, read by `humanish study` and `--study`. humanish refuses a 0.107 `humanish.lab.v2` file, and `humanish migrate` converts it (second table). | `src/study/`; `StudyConfig` in `src/study/types.ts`, parsed by `parseStudy` in `src/study/config.ts` |
|
|
15
|
-
| run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.
|
|
15
|
+
| run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.v4`) as the source of truth and each participant's trace in its own file. | `src/run/`; `RunBundle` in `src/run/bundle.ts` |
|
|
16
16
|
| dry run | A run with no live participant, from `--dry-run` or `mode: dry-run`. humanish writes a synthetic run bundle, review and Observer with no browser, model, keys or provider spend, and tests no product behavior. The `preview` route only dry-runs; "preview" names that route and nothing else. | `runDryRun` in `src/run/dry-run.ts`; `resolveStudyDryRun` in `src/study/plan.ts` |
|
|
17
17
|
| participant | One simulated person, or agent, taking part in a study. The planner builds each one's record (id, persona id, assignment, labels and the route's own fields). Shared-world participants resolve the plan's records; computer-use participants rebuild the same records from the study config with `computerUseParticipants` (`src/study/plan-participants.ts`) and resolve those. `ResolvedParticipant` adds the bundle ids, persona and prompt. `run.json` records each one as a `simulations[]` entry with one or more `streams[]`. | `ResolvedParticipant` in `src/run/participant.ts`; the planned records in `src/study/plan-participants.ts`. Computer-use and shared-world participants run a `DesktopParticipantRun` (`src/routes/computer-use/types.ts`). The terminal session (`src/routes/terminal/session.ts`), scripted surfaces (`src/routes/scripted/plan.ts`) and fixture simulations (`src/run/dry-run.ts`) still build their own. |
|
|
18
18
|
| persona | Who a participant is: traits, background, device. `humanish/personas/<id>.yaml`. | `src/study/persona.ts`, `src/study/persona-resolve.ts` |
|
|
@@ -36,7 +36,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
36
36
|
| topology | Whether participants share one instance of the subject. `route: shared-world` gives one for all; every other route gives each participant its own (`per-lane-worlds`). `run.json` records it as `sharedWorld.topology`. | `route` in `StudyConfig` (`src/study/types.ts`); `isSharedWorldComposition` in `src/study/routing.ts` |
|
|
37
37
|
| plane | The one instance of the subject that a shared-world study's participants use together. A `provisioned` plane is a `clone` or `local-tree` subject that humanish serves in a sandbox, with a declared subject state. An `external-public` plane is a deployment already running at `subject.appUrl`, whose owner the study names in `subject.publicTarget.owner`. Each plane has at least two participants. | `SharedWorldPlane` in `src/study/plan-types.ts`; `src/routes/shared-world/` |
|
|
38
38
|
| stream | One live or recorded view of a participant, such as a browser or a terminal, in `run.json` and the Observer. | `RunStream` in `src/run/streams.ts` |
|
|
39
|
-
| Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.
|
|
39
|
+
| Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.v3`). | `observer/` (the page), `src/observer/` (data, render, serve) |
|
|
40
40
|
| evidence | Files a run keeps: screenshots, traces, transcripts, recordings. `verify` checks them and grades share-safety. | `src/evidence/`; `verifyRun` in `src/verify/verify.ts` |
|
|
41
41
|
| analysis | An independent model reading of a finished run's retained evidence, kept apart from participant feedback. Supported live runs request it automatically, and `humanish analyze` runs it on demand. It records findings and observations as `humanish.study-analysis.v1`. | `src/analysis/`; `runAutomaticAnalysis` in `src/analysis/automatic.ts`; `analyzeRun` in `src/analysis/service.ts`; `ANALYSIS_SCHEMA` in `src/analysis/types.ts` |
|
|
42
42
|
| finding | A result of analysis, with citations to evidence in the run. Each finding groups the observations that support it. | `src/analysis/` |
|
package/README.md
CHANGED
|
@@ -335,7 +335,7 @@ and the library exports.
|
|
|
335
335
|
- Breaking changes ship together in one minor release. Its release notes carry a migration
|
|
336
336
|
table: each old form, its new form, and the change to make.
|
|
337
337
|
- The study file and the run bundle carry versioned schemas, `humanish.study.v3` and
|
|
338
|
-
`humanish.run-bundle.
|
|
338
|
+
`humanish.run-bundle.v4`. A breaking change to either one bumps its version.
|
|
339
339
|
|
|
340
340
|
Pin the exact version in `package.json` and read [CHANGELOG.md](CHANGELOG.md) before upgrading.
|
|
341
341
|
|
|
@@ -17,5 +17,8 @@ export function retryAfterOutputLimit(session, turn, turnNumber, alreadyRetried)
|
|
|
17
17
|
session.usage.record(turn, "interaction");
|
|
18
18
|
const output = turn.usage?.output;
|
|
19
19
|
session.trace.record("notice", () => notice("warn", "provider reply cut off by the output limit; asking again", `provider turn ${turnNumber} reached the output-token limit${output === undefined ? "" : ` after ${output} output tokens`} before any action; its actions were not run, and the same request is sent once more`));
|
|
20
|
+
// haltBeforeTurn checks spend again before the resend. Checking here as well keeps a reply that
|
|
21
|
+
// crossed a cap out of the trace's narration, as after a complete reply, and keeps a crossed
|
|
22
|
+
// cap ahead of unknown usage.
|
|
20
23
|
return spendStop(session) ?? unknownSpendStop(session) ?? "retry";
|
|
21
24
|
}
|
|
@@ -386,19 +386,25 @@ export interface CuaLoopOptions {
|
|
|
386
386
|
*/
|
|
387
387
|
maxUsd?: number;
|
|
388
388
|
/**
|
|
389
|
-
* Injected pure
|
|
390
|
-
*
|
|
391
|
-
* estimated USD, or null when unpriceable. Only consulted when `maxUsd` is set.
|
|
392
|
-
*
|
|
393
|
-
*
|
|
389
|
+
* Injected pure cost estimator (keeps the loop free of the operator rate table and makes the
|
|
390
|
+
* cap deterministic in tests). Given running (input, output) token totals, returns the
|
|
391
|
+
* estimated USD, or null when unpriceable. Only consulted when `maxUsd` is set. The loop calls it
|
|
392
|
+
* after each reply and again before each provider request, the first included, when every total
|
|
393
|
+
* is 0 and the usage has no `turns`. An estimate of zero usage above `maxUsd` therefore stops the
|
|
394
|
+
* session before its first request. A null estimate mid-run cannot trip the cap; preflight
|
|
395
|
+
* already guaranteed a rate exists, so a null here is a vanished-rate harness condition, not a
|
|
396
|
+
* silent uncapped pass.
|
|
394
397
|
*/
|
|
395
398
|
estimateTurnCostUsd?: (usage: ActorTokenUsage) => number | null;
|
|
396
399
|
/**
|
|
397
|
-
* Run-level spend guard: called with this participant's running usage each
|
|
398
|
-
*
|
|
399
|
-
*
|
|
400
|
-
*
|
|
401
|
-
*
|
|
400
|
+
* Run-level spend guard: called with this participant's running usage at each point the
|
|
401
|
+
* per-participant cap is checked, after each reply and before each provider request. The first
|
|
402
|
+
* call comes before the first request, with every total at 0 and no `turns`. How many calls a
|
|
403
|
+
* turn makes depends on the loop's checks, so a guard should decide from the usage it is given
|
|
404
|
+
* and the ledger it reads. Returns a human-readable reason when the study's shared budget is
|
|
405
|
+
* exhausted, else null. On a non-null return the loop stops with `budget_reached` regardless of
|
|
406
|
+
* material progress: a study-level stop is a recruiting decision hitting its limit, not this
|
|
407
|
+
* participant's runaway, so it never reads as `gave_up`.
|
|
402
408
|
*/
|
|
403
409
|
overRunBudget?: (usage: ActorTokenUsage) => string | null;
|
|
404
410
|
/**
|
|
@@ -201,11 +201,16 @@ function refuseAccountBilledCaps(options) {
|
|
|
201
201
|
});
|
|
202
202
|
}
|
|
203
203
|
}
|
|
204
|
+
/**
|
|
205
|
+
* The stops checked before every provider request. The spend caps are checked here as well as
|
|
206
|
+
* after each reply, because a sibling participant can exhaust the study budget while this one acts.
|
|
207
|
+
*/
|
|
204
208
|
function haltBeforeTurn(session, observation) {
|
|
205
209
|
if (session.signal?.aborted)
|
|
206
210
|
return stops.harnessAborted;
|
|
207
211
|
return (clockEndStop(session, observation) ??
|
|
208
|
-
(session.remaining() < 0 ? stops.timeLimit(session) : undefined)
|
|
212
|
+
(session.remaining() < 0 ? stops.timeLimit(session) : undefined) ??
|
|
213
|
+
spendStop(session));
|
|
209
214
|
}
|
|
210
215
|
/**
|
|
211
216
|
* Build the next request; the context hint and safety acknowledgements ride it once. The
|
|
@@ -35,7 +35,6 @@ function parseSource(prepared, bytes) {
|
|
|
35
35
|
!Array.isArray(value.streams) ||
|
|
36
36
|
value.streams.length > EVIDENCE_LIMITS.participants ||
|
|
37
37
|
!Array.isArray(value.events) ||
|
|
38
|
-
value.events.length > 100000 ||
|
|
39
38
|
(value.commsReceiving !== undefined && !isCommsReceivingEvidence(value.commsReceiving)))
|
|
40
39
|
throw new Error("ANALYSIS_SOURCE_INVALID");
|
|
41
40
|
const ids = new Set();
|
|
@@ -79,10 +78,8 @@ async function readSource(prepared, bytes, traceBytes) {
|
|
|
79
78
|
return loaded.bundle;
|
|
80
79
|
}
|
|
81
80
|
function validateTraceItems(actor) {
|
|
82
|
-
if (actor !== undefined &&
|
|
83
|
-
(!isRecord(actor) || !Array.isArray(actor.items) || actor.items.length > 100000)) {
|
|
81
|
+
if (actor !== undefined && (!isRecord(actor) || !Array.isArray(actor.items)))
|
|
84
82
|
throw new Error("ANALYSIS_SOURCE_INVALID");
|
|
85
|
-
}
|
|
86
83
|
const eventIds = new Set();
|
|
87
84
|
for (const item of isRecord(actor) ? actor.items : []) {
|
|
88
85
|
if (!isRecord(item) ||
|
|
@@ -150,7 +150,7 @@ const analysisEvidenceSchema = z
|
|
|
150
150
|
quoteEligible: z.boolean(),
|
|
151
151
|
at: timestamp.nullable(),
|
|
152
152
|
elapsedMs: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
|
|
153
|
-
frame: z.number().int().min(0).max(
|
|
153
|
+
frame: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable(),
|
|
154
154
|
capture: z
|
|
155
155
|
.object({
|
|
156
156
|
eventId: sourceId,
|
|
@@ -138,7 +138,8 @@ export function formatConcurrentSharedWorldStudyHuman(result) {
|
|
|
138
138
|
? "Participants used the app at the same time (observed in this run only)."
|
|
139
139
|
: "Participants were not observed using the app at the same time.",
|
|
140
140
|
]),
|
|
141
|
-
|
|
141
|
+
// A skipped participant's line ends on the reason the run recorded, as on computer use.
|
|
142
|
+
...result.roles.map((participant) => `${participantCaption({ id: participant.id, personaId: participant.persona })}: ${participantStatus(participant.status)}${participant.ok ? "" : ", not ok"}${participant.skippedReason === undefined ? "" : ` · ${participant.skippedReason}`}`),
|
|
142
143
|
...(result.subjectSandbox
|
|
143
144
|
? [
|
|
144
145
|
`subject sandbox: ${sandboxName(result.subjectSandbox)} killed=${result.subjectSandbox.killed ? "yes" : "no"}`,
|
package/dist/feedback/export.js
CHANGED
|
@@ -180,7 +180,8 @@ async function readObserverSlot(cwd, runInput, runRoot, runId, verified, deps, w
|
|
|
180
180
|
* even if an older saved HTML was never refreshed; old recording-only exports keep the saved slot.
|
|
181
181
|
* A page saved since observer-data v2 holds no participant's items in its slot, and verify does not
|
|
182
182
|
* read the page's compressed copy, so its export is the current projection of the run and its
|
|
183
|
-
* traces. Undefined when run.json no longer matches the analysis, or
|
|
183
|
+
* traces, for a page saved as v2 too. Undefined when run.json no longer matches the analysis, or
|
|
184
|
+
* such a page's run cannot be read.
|
|
184
185
|
*/
|
|
185
186
|
async function currentObserverData(runPaths, analysis, slot) {
|
|
186
187
|
const saved = JSON.parse(slot);
|
|
@@ -189,7 +190,7 @@ async function currentObserverData(runPaths, analysis, slot) {
|
|
|
189
190
|
const schema = saved?.schema;
|
|
190
191
|
if (schema === "humanish.observer-data.v1")
|
|
191
192
|
return { value: observerDataFromSaved(saved) };
|
|
192
|
-
if (schema !== OBSERVER_DATA_SCHEMA)
|
|
193
|
+
if (schema !== OBSERVER_DATA_SCHEMA && schema !== "humanish.observer-data.v2")
|
|
193
194
|
return { value: saved };
|
|
194
195
|
const bundle = await sourceTraces(runPaths, await source());
|
|
195
196
|
return bundle === undefined ? undefined : { value: buildObserverData(bundle) };
|
package/dist/observer/data.d.ts
CHANGED
|
@@ -2,7 +2,12 @@ import { type ActorEnding } from "../actors/stop-cause.js";
|
|
|
2
2
|
import type { RunBundle, RunCostSummary, RunEvent, RunSimulation } from "../run/bundle.js";
|
|
3
3
|
import { type RunDisplay } from "../run/display.js";
|
|
4
4
|
import type { RunStream, RunStreamKind } from "../run/streams.js";
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* The Observer's data. From v2 observer-data.json leaves out each participant's items, and from v3
|
|
7
|
+
* each participant's per-request lists (REQUEST_LISTS in src/run/stream-trace.ts), which no part
|
|
8
|
+
* of the page shows.
|
|
9
|
+
*/
|
|
10
|
+
export declare const OBSERVER_DATA_SCHEMA = "humanish.observer-data.v3";
|
|
6
11
|
interface ObserverArtifactLink {
|
|
7
12
|
label: string;
|
|
8
13
|
href: string;
|
|
@@ -10,7 +15,7 @@ interface ObserverArtifactLink {
|
|
|
10
15
|
}
|
|
11
16
|
export interface ObserverData {
|
|
12
17
|
schema: typeof OBSERVER_DATA_SCHEMA;
|
|
13
|
-
schemaVersion:
|
|
18
|
+
schemaVersion: 3;
|
|
14
19
|
generatedAt: string;
|
|
15
20
|
/** Served-only liveness observation. Never a replacement for the recorded evidence verdict. */
|
|
16
21
|
runtime?: {
|
|
@@ -124,10 +129,12 @@ interface ObserverLaneGroup {
|
|
|
124
129
|
*/
|
|
125
130
|
export declare function buildObserverData(bundle: RunBundle, generatedAt?: string, statusRecord?: unknown): ObserverData;
|
|
126
131
|
/**
|
|
127
|
-
* A saved Observer projection as this version reads it, unchecked as before
|
|
128
|
-
* written before v2 holds every participant's items
|
|
129
|
-
*
|
|
130
|
-
*
|
|
132
|
+
* A saved Observer projection as this version reads it, as v3 and unchecked as before. An
|
|
133
|
+
* observer-data.json written before v2 holds every participant's items, and its per-request lists,
|
|
134
|
+
* which are left out. A v2 or v3 one holds no items, so it is read only when no participant has
|
|
135
|
+
* any: showing participants with no steps would say they did nothing, and the items are in the
|
|
136
|
+
* run's traces. Null for a projection that leaves items out, and for a schema other than v1, v2
|
|
137
|
+
* and v3.
|
|
131
138
|
*/
|
|
132
139
|
export declare function observerDataFromSaved(value: unknown): ObserverData | null;
|
|
133
140
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
package/dist/observer/data.js
CHANGED
|
@@ -3,8 +3,15 @@ import { isCommsReceivingEvidence, receivingAnalysisContext } from "../comms/rec
|
|
|
3
3
|
import { formatParticipantOutcomes, formatRunTaskFunnel, participantOutcomeDetails, withCuaReviewProvenance, } from "../run/outcomes.js";
|
|
4
4
|
import { cuaGoalSource, CUA_COMPLETION_NOTE } from "../actors/goal-source.js";
|
|
5
5
|
import { bundleDisplayFacts, runDisplay } from "../run/display.js";
|
|
6
|
+
import { withoutRequestLists } from "../run/stream-trace.js";
|
|
7
|
+
import { isRecord } from "../run/type-guards.js";
|
|
6
8
|
import { joinObserverData, shellNeedsItems } from "./participant-items.js";
|
|
7
|
-
|
|
9
|
+
/**
|
|
10
|
+
* The Observer's data. From v2 observer-data.json leaves out each participant's items, and from v3
|
|
11
|
+
* each participant's per-request lists (REQUEST_LISTS in src/run/stream-trace.ts), which no part
|
|
12
|
+
* of the page shows.
|
|
13
|
+
*/
|
|
14
|
+
export const OBSERVER_DATA_SCHEMA = "humanish.observer-data.v3";
|
|
8
15
|
/** Discard a forged or stale runtime grant before projecting persisted evidence. */
|
|
9
16
|
export function recordedStreamEmbed(embed) {
|
|
10
17
|
if (!embed)
|
|
@@ -52,7 +59,7 @@ export function buildObserverData(bundle, generatedAt = new Date().toISOString()
|
|
|
52
59
|
: stream.status);
|
|
53
60
|
byKind[stream.kind] += 1;
|
|
54
61
|
return {
|
|
55
|
-
...stream,
|
|
62
|
+
...withoutStreamLists(stream),
|
|
56
63
|
...(stream.embed === undefined ? {} : { embed: recordedStreamEmbed(stream.embed) }),
|
|
57
64
|
status,
|
|
58
65
|
sim,
|
|
@@ -81,7 +88,7 @@ export function buildObserverData(bundle, generatedAt = new Date().toISOString()
|
|
|
81
88
|
const active = streams.filter((stream) => stream.status === "running" || stream.status === "preparing").length;
|
|
82
89
|
return withObserverEndings({
|
|
83
90
|
schema: OBSERVER_DATA_SCHEMA,
|
|
84
|
-
schemaVersion:
|
|
91
|
+
schemaVersion: 3,
|
|
85
92
|
generatedAt,
|
|
86
93
|
run: {
|
|
87
94
|
runId: bundle.runId,
|
|
@@ -142,19 +149,42 @@ export function buildObserverData(bundle, generatedAt = new Date().toISOString()
|
|
|
142
149
|
},
|
|
143
150
|
});
|
|
144
151
|
}
|
|
152
|
+
/** The stream without its participant's per-request lists; a saved stream that is not an object as it is. */
|
|
153
|
+
function withoutStreamLists(stream) {
|
|
154
|
+
if (!isRecord(stream))
|
|
155
|
+
return stream;
|
|
156
|
+
return {
|
|
157
|
+
...stream,
|
|
158
|
+
...(isRecord(stream.actor) ? { actor: withoutRequestLists(stream.actor) } : {}),
|
|
159
|
+
...(isRecord(stream.liveActor) ? { liveActor: withoutRequestLists(stream.liveActor) } : {}),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
145
162
|
/**
|
|
146
|
-
* A saved Observer projection as this version reads it, unchecked as before
|
|
147
|
-
* written before v2 holds every participant's items
|
|
148
|
-
*
|
|
149
|
-
*
|
|
163
|
+
* A saved Observer projection as this version reads it, as v3 and unchecked as before. An
|
|
164
|
+
* observer-data.json written before v2 holds every participant's items, and its per-request lists,
|
|
165
|
+
* which are left out. A v2 or v3 one holds no items, so it is read only when no participant has
|
|
166
|
+
* any: showing participants with no steps would say they did nothing, and the items are in the
|
|
167
|
+
* run's traces. Null for a projection that leaves items out, and for a schema other than v1, v2
|
|
168
|
+
* and v3.
|
|
150
169
|
*/
|
|
151
170
|
export function observerDataFromSaved(value) {
|
|
152
171
|
const saved = value;
|
|
153
|
-
|
|
154
|
-
|
|
172
|
+
const schema = saved.schema;
|
|
173
|
+
if (schema === "humanish.observer-data.v1") {
|
|
174
|
+
const streams = Array.isArray(saved.streams)
|
|
175
|
+
? saved.streams.map(withoutStreamLists)
|
|
176
|
+
: saved.streams;
|
|
177
|
+
return { ...saved, streams, schema: OBSERVER_DATA_SCHEMA, schemaVersion: 3 };
|
|
178
|
+
}
|
|
179
|
+
if (schema !== "humanish.observer-data.v2" && schema !== OBSERVER_DATA_SCHEMA)
|
|
180
|
+
return null;
|
|
155
181
|
return shellNeedsItems(saved)
|
|
156
182
|
? null
|
|
157
|
-
:
|
|
183
|
+
: {
|
|
184
|
+
...joinObserverData(saved, null),
|
|
185
|
+
schema: OBSERVER_DATA_SCHEMA,
|
|
186
|
+
schemaVersion: 3,
|
|
187
|
+
};
|
|
158
188
|
}
|
|
159
189
|
const GOAL_SOURCE_LABELS = {
|
|
160
190
|
participant_report: "Reported complete",
|
|
@@ -9,6 +9,8 @@ import type { ObserverItems } from "./participant-items.js";
|
|
|
9
9
|
export declare function parseFeedCursors(after: string | null): ReadonlyMap<string, LiveItemsCursor>;
|
|
10
10
|
/**
|
|
11
11
|
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
12
|
-
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
12
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it. A
|
|
13
|
+
* live file the loader stopped reading (LiveItemsRead's `stopped`) keeps the page's cursor as its
|
|
14
|
+
* own, so the page holds its items and sends that cursor again.
|
|
13
15
|
*/
|
|
14
16
|
export declare function withFeedCursors(items: ObserverItems, live: readonly LiveItemsRead[]): ObserverItems;
|
|
@@ -43,7 +43,9 @@ function cursorText(path, position) {
|
|
|
43
43
|
}
|
|
44
44
|
/**
|
|
45
45
|
* `items` with each live file's cursor on the entry of the stream it was joined into, and
|
|
46
|
-
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it.
|
|
46
|
+
* `liveAfter`, the page's cursor, where the entry's `liveActor` holds only the items past it. A
|
|
47
|
+
* live file the loader stopped reading (LiveItemsRead's `stopped`) keeps the page's cursor as its
|
|
48
|
+
* own, so the page holds its items and sends that cursor again.
|
|
47
49
|
*/
|
|
48
50
|
export function withFeedCursors(items, live) {
|
|
49
51
|
const streams = Object.assign(Object.create(null), items.streams);
|
|
@@ -51,9 +53,10 @@ export function withFeedCursors(items, live) {
|
|
|
51
53
|
const entry = Object.hasOwn(streams, read.streamId) ? streams[read.streamId] : undefined;
|
|
52
54
|
if (entry === undefined || !Array.isArray(entry.liveActor))
|
|
53
55
|
continue;
|
|
56
|
+
const end = read.stopped === undefined ? read.index : read.after;
|
|
54
57
|
streams[read.streamId] = {
|
|
55
58
|
...entry,
|
|
56
|
-
live: cursorText(read.index.path,
|
|
59
|
+
live: cursorText(read.index.path, end),
|
|
57
60
|
...(read.after === undefined ? {} : { liveAfter: cursorText(read.index.path, read.after) }),
|
|
58
61
|
};
|
|
59
62
|
}
|
|
@@ -10,6 +10,7 @@ import { listRuns } from "../run/stored-runs.js";
|
|
|
10
10
|
import { readAnalysisAccounting } from "../run/costs.js";
|
|
11
11
|
import { bindExistingRunArtifactPaths, isPathInside, isSafeRunIdSegment, } from "../run/paths.js";
|
|
12
12
|
import { isRunOutcome } from "../run/bundle-shape.js";
|
|
13
|
+
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
13
14
|
import { displayedTraces, loadStreamTraces, } from "../run/stream-trace.js";
|
|
14
15
|
import { isNodeError, isRecord } from "../run/type-guards.js";
|
|
15
16
|
import { MAX_NOTE_FILE_BYTES, readRunNotes, RUN_NOTES_DIR, } from "../run/note-files.js";
|
|
@@ -150,10 +151,17 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
150
151
|
writeResponse(response, 404, "Not found", "text/plain; charset=utf-8");
|
|
151
152
|
return;
|
|
152
153
|
}
|
|
154
|
+
// run.json is read under the limit every other reader of it uses, by name as through the feed.
|
|
155
|
+
const runJson = cleanedRelativePath === "run.json";
|
|
153
156
|
try {
|
|
154
|
-
const body = await readContainedFile(runRoot, filePath, notePath
|
|
157
|
+
const body = await readContainedFile(runRoot, filePath, notePath
|
|
158
|
+
? { maxBytes: MAX_NOTE_FILE_BYTES }
|
|
159
|
+
: runJson
|
|
160
|
+
? { maxBytes: RUN_ARTIFACT_MAX_BYTES }
|
|
161
|
+
: {});
|
|
155
162
|
if (!body) {
|
|
156
|
-
|
|
163
|
+
const tooLarge = runJson ? await pastReadLimit(runRoot, "run.json") : undefined;
|
|
164
|
+
writeResponse(response, 404, tooLarge === undefined ? "Not found" : `Not found: ${tooLarge}`, "text/plain; charset=utf-8");
|
|
157
165
|
return;
|
|
158
166
|
}
|
|
159
167
|
response.writeHead(200, {
|
|
@@ -198,12 +206,15 @@ async function readObserverData(runRoot, runtimeStreamUrls = []) {
|
|
|
198
206
|
* fallback. The saved observer-data.json is read only while the run has no run.json at all, or in
|
|
199
207
|
* safe serving, where only its admitted bytes can be read. Otherwise a run.json that is there and
|
|
200
208
|
* cannot be read, parsed or used as a bundle (which a browser poll can meet mid-write) gives the
|
|
201
|
-
* reason, a 404 the poller retries, never the saved projection.
|
|
209
|
+
* reason, a 404 the poller retries, never the saved projection. Both files are read under
|
|
210
|
+
* RUN_ARTIFACT_MAX_BYTES, as every other reader of a run file reads them.
|
|
202
211
|
*/
|
|
203
212
|
async function observerDataOrReason(runRoot, runtimeStreamUrls, cursors) {
|
|
204
213
|
const record = await readLocalRunStatus(runRoot);
|
|
205
214
|
const runJson = path.join(runRoot.physicalPath, "run.json");
|
|
206
|
-
const bundleBytes = await readContainedFile(runRoot, runJson
|
|
215
|
+
const bundleBytes = await readContainedFile(runRoot, runJson, {
|
|
216
|
+
maxBytes: RUN_ARTIFACT_MAX_BYTES,
|
|
217
|
+
}).catch(() => null);
|
|
207
218
|
const absent = bundleBytes === null &&
|
|
208
219
|
(await lstat(runJson).then(() => false, (error) => isNodeError(error) && error.code === "ENOENT"));
|
|
209
220
|
// A root admitted for safe serving checks every read against the identity and digest admitted
|
|
@@ -214,7 +225,10 @@ async function observerDataOrReason(runRoot, runtimeStreamUrls, cursors) {
|
|
|
214
225
|
return "unavailable" in saved ? saved : { data: saved, live: [] };
|
|
215
226
|
}
|
|
216
227
|
if (bundleBytes === null)
|
|
217
|
-
return {
|
|
228
|
+
return {
|
|
229
|
+
unavailable: (await pastReadLimit(runRoot, "run.json")) ??
|
|
230
|
+
"run.json is there but could not be read: it is a link or changed.",
|
|
231
|
+
};
|
|
218
232
|
let stored;
|
|
219
233
|
try {
|
|
220
234
|
const parsed = JSON.parse(bundleBytes.toString("utf8"));
|
|
@@ -237,12 +251,31 @@ async function observerDataOrReason(runRoot, runtimeStreamUrls, cursors) {
|
|
|
237
251
|
return { unavailable: "run.json could not be shown as a run." };
|
|
238
252
|
}
|
|
239
253
|
}
|
|
254
|
+
/**
|
|
255
|
+
* Why run.json or a saved observer-data.json was not read, when the file at `relativePath` passes
|
|
256
|
+
* the root's checks and is larger than RUN_ARTIFACT_MAX_BYTES, the most the server reads of either;
|
|
257
|
+
* otherwise undefined. The size comes from the opened file, so no link or file outside the run
|
|
258
|
+
* folder is measured.
|
|
259
|
+
*/
|
|
260
|
+
async function pastReadLimit(runRoot, relativePath) {
|
|
261
|
+
const opened = await openContainedFile(runRoot, path.join(runRoot.physicalPath, relativePath));
|
|
262
|
+
if (opened === null)
|
|
263
|
+
return undefined;
|
|
264
|
+
await opened.handle.close().catch(() => undefined);
|
|
265
|
+
return opened.size > RUN_ARTIFACT_MAX_BYTES
|
|
266
|
+
? `${relativePath} is larger than ${RUN_ARTIFACT_MAX_BYTES} bytes, the most humanish reads of it.`
|
|
267
|
+
: undefined;
|
|
268
|
+
}
|
|
240
269
|
/** The saved observer-data.json of a run that has written no run.json yet. */
|
|
241
270
|
async function savedObserverData(runRoot, record, runtimeStreamUrls) {
|
|
242
271
|
try {
|
|
243
|
-
const observerBytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, "observer", "observer-data.json"));
|
|
244
|
-
if (!observerBytes)
|
|
272
|
+
const observerBytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, "observer", "observer-data.json"), { maxBytes: RUN_ARTIFACT_MAX_BYTES });
|
|
273
|
+
if (!observerBytes) {
|
|
274
|
+
const tooLarge = await pastReadLimit(runRoot, "observer/observer-data.json");
|
|
275
|
+
if (tooLarge !== undefined)
|
|
276
|
+
return { unavailable: tooLarge };
|
|
245
277
|
throw new Error("observer-data.json unavailable");
|
|
278
|
+
}
|
|
246
279
|
const saved = observerDataFromSaved(JSON.parse(observerBytes.toString("utf8")));
|
|
247
280
|
if (saved === null)
|
|
248
281
|
throw new Error("observer-data.json holds no participant's items");
|