humanish 0.123.0 → 0.125.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/CONTEXT.md +2 -2
- package/README.md +2 -2
- package/dist/actors/computer-use/loop/trace.js +6 -1
- package/dist/actors/computer-use/openai-conversation.d.ts +0 -6
- package/dist/actors/computer-use/openai-conversation.js +0 -8
- package/dist/actors/computer-use/openai-provider.js +5 -8
- package/dist/analysis/evidence.js +2 -2
- package/dist/evidence/image.js +82 -3
- package/dist/feedback/export-bundle.js +1 -5
- package/dist/feedback/export.js +30 -11
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/observer/artifact.js +2 -9
- package/dist/observer/data.d.ts +9 -2
- package/dist/observer/data.js +17 -2
- package/dist/observer/page-data.d.ts +5 -0
- package/dist/observer/page-data.js +21 -0
- package/dist/observer/participant-items.d.ts +45 -0
- package/dist/observer/participant-items.js +85 -0
- package/dist/observer/pinned-files.d.ts +3 -1
- package/dist/observer/pinned-files.js +10 -7
- package/dist/observer/render.js +2 -1
- package/dist/observer/run-routes.js +16 -5
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/live-flush.d.ts +9 -4
- package/dist/routes/computer-use/live-flush.js +16 -3
- package/dist/run/bundle-shape.js +2 -2
- package/dist/run/bundle.d.ts +10 -3
- package/dist/run/bundle.js +11 -2
- package/dist/run/contained-output.d.ts +22 -0
- package/dist/run/contained-output.js +97 -1
- package/dist/run/evidence-files.d.ts +12 -1
- package/dist/run/evidence-files.js +70 -40
- package/dist/run/live-items.d.ts +48 -0
- package/dist/run/live-items.js +242 -0
- package/dist/run/locate.d.ts +10 -0
- package/dist/run/locate.js +10 -0
- package/dist/run/run.d.ts +3 -2
- package/dist/run/run.js +31 -9
- package/dist/run/sandbox-ids.d.ts +0 -5
- package/dist/run/sandbox-ids.js +0 -7
- package/dist/run/stream-trace.d.ts +18 -9
- package/dist/run/stream-trace.js +113 -22
- package/dist/run/streams.d.ts +17 -2
- package/dist/study/parse/actors.js +23 -6
- package/dist/study/parse/execution.js +6 -0
- package/dist/study/plan-base.js +4 -1
- package/dist/study/plan.js +20 -0
- package/dist/verify/artifacts.js +39 -51
- package/dist/verify/screenshot-reads.d.ts +21 -0
- package/dist/verify/screenshot-reads.js +46 -0
- package/dist/verify/verify.js +7 -6
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/architecture/observer.md +24 -5
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +78 -14
- package/docs/contracts/schemas.md +38 -37
- package/docs/contracts/study-analysis.md +1 -1
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -1
- package/package.json +4 -2
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,34 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.125.0: Run bundle v3 with live items in append-only files, loadRunBundle, faster repeated verify (2026-10-12)
|
|
12
|
+
|
|
13
|
+
humanish 0.125.0 keeps each live flush of a long study small, exports the run bundle loader, and
|
|
14
|
+
verifies a finished study faster. Run bundles are now `humanish.run-bundle.v3`: a running
|
|
15
|
+
participant's items so far live in an append-only file under `live/`, and each flush appends only
|
|
16
|
+
the items recorded since the last one. On a synthetic run of 24 participants working 8 hours, a
|
|
17
|
+
flush wrote about 0.23 MB in about 0.24 s, where 0.124.0 wrote up to 120 MB in 3.0 s. A finished
|
|
18
|
+
v3 bundle differs from v2 only in its schema name; 0.124.0 and earlier refuse it by name.
|
|
19
|
+
`loadRunBundle(cwd, run)` returns run.json with every trace and live file joined and checked, for
|
|
20
|
+
v1, v2 and v3 bundles. The five verifies of a finished study read each screenshot once: 10 s in
|
|
21
|
+
all for 1,203 screenshots, down from 35 s. A key in a study's title is redacted from the bundle,
|
|
22
|
+
and one in a persona id or desktop template fails at parse instead of blocking the run.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.125.0)
|
|
25
|
+
|
|
26
|
+
## 0.124.0: Small Observer files for long studies, faster verify, closing reports under zero data retention (2026-10-12)
|
|
27
|
+
|
|
28
|
+
humanish 0.124.0 keeps a long study's Observer files small, checks screenshots about eight times
|
|
29
|
+
faster, and asks an `openai-computer-use` participant for its closing report at a scheduled end
|
|
30
|
+
under zero data retention. `observer/observer-data.json` is now `humanish.observer-data.v2`: the
|
|
31
|
+
run and each participant without their trace items and screenshot entries, which the Observer
|
|
32
|
+
page carries compressed. On a synthetic run of 24 participants working 8 hours, observer-data.json
|
|
33
|
+
fell from 155 MiB to 0.33 MiB and index.html from 94 MiB to 16 MiB, so verify reads both. A
|
|
34
|
+
reader of observer-data.json that needs a participant's items reads the run's traces. verify of a
|
|
35
|
+
synthetic 8-hour run with 8,895 real-size screenshots took 51 s, down from 282 s.
|
|
36
|
+
|
|
37
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.124.0)
|
|
38
|
+
|
|
11
39
|
## 0.123.0: Run bundle v2 writes each trace once, bundle size estimate (2026-10-12)
|
|
12
40
|
|
|
13
41
|
humanish 0.123.0 writes each participant's trace once, outside run.json, so long studies stay
|
package/CONTEXT.md
CHANGED
|
@@ -12,7 +12,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
12
12
|
| Term | Meaning | Where it lives in the code |
|
|
13
13
|
| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
14
14
|
| study | What a user designs and runs: one YAML file that declares the subject, the actor, the participants and the caps. The file lives at `humanish/studies/<id>.yaml` with schema `humanish.study.v3`, read by `humanish study` and `--study`. humanish refuses a 0.107 `humanish.lab.v2` file, and `humanish migrate` converts it (second table). | `src/study/`; `StudyConfig` in `src/study/types.ts`, parsed by `parseStudy` in `src/study/config.ts` |
|
|
15
|
-
| run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.
|
|
15
|
+
| run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.v3`) as the source of truth and each participant's trace in its own file. | `src/run/`; `RunBundle` in `src/run/bundle.ts` |
|
|
16
16
|
| dry run | A run with no live participant, from `--dry-run` or `mode: dry-run`. humanish writes a synthetic run bundle, review and Observer with no browser, model, keys or provider spend, and tests no product behavior. The `preview` route only dry-runs; "preview" names that route and nothing else. | `runDryRun` in `src/run/dry-run.ts`; `resolveStudyDryRun` in `src/study/plan.ts` |
|
|
17
17
|
| participant | One simulated person, or agent, taking part in a study. The planner builds each one's record (id, persona id, assignment, labels and the route's own fields). Shared-world participants resolve the plan's records; computer-use participants rebuild the same records from the study config with `computerUseParticipants` (`src/study/plan-participants.ts`) and resolve those. `ResolvedParticipant` adds the bundle ids, persona and prompt. `run.json` records each one as a `simulations[]` entry with one or more `streams[]`. | `ResolvedParticipant` in `src/run/participant.ts`; the planned records in `src/study/plan-participants.ts`. Computer-use and shared-world participants run a `DesktopParticipantRun` (`src/routes/computer-use/types.ts`). The terminal session (`src/routes/terminal/session.ts`), scripted surfaces (`src/routes/scripted/plan.ts`) and fixture simulations (`src/run/dry-run.ts`) still build their own. |
|
|
18
18
|
| persona | Who a participant is: traits, background, device. `humanish/personas/<id>.yaml`. | `src/study/persona.ts`, `src/study/persona-resolve.ts` |
|
|
@@ -36,7 +36,7 @@ bundle and output fields keep older spellings; the second table maps them.
|
|
|
36
36
|
| topology | Whether participants share one instance of the subject. `route: shared-world` gives one for all; every other route gives each participant its own (`per-lane-worlds`). `run.json` records it as `sharedWorld.topology`. | `route` in `StudyConfig` (`src/study/types.ts`); `isSharedWorldComposition` in `src/study/routing.ts` |
|
|
37
37
|
| plane | The one instance of the subject that a shared-world study's participants use together. A `provisioned` plane is a `clone` or `local-tree` subject that humanish serves in a sandbox, with a declared subject state. An `external-public` plane is a deployment already running at `subject.appUrl`, whose owner the study names in `subject.publicTarget.owner`. Each plane has at least two participants. | `SharedWorldPlane` in `src/study/plan-types.ts`; `src/routes/shared-world/` |
|
|
38
38
|
| stream | One live or recorded view of a participant, such as a browser or a terminal, in `run.json` and the Observer. | `RunStream` in `src/run/streams.ts` |
|
|
39
|
-
| Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.
|
|
39
|
+
| Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.v2`). | `observer/` (the page), `src/observer/` (data, render, serve) |
|
|
40
40
|
| evidence | Files a run keeps: screenshots, traces, transcripts, recordings. `verify` checks them and grades share-safety. | `src/evidence/`; `verifyRun` in `src/verify/verify.ts` |
|
|
41
41
|
| analysis | An independent model reading of a finished run's retained evidence, kept apart from participant feedback. Supported live runs request it automatically, and `humanish analyze` runs it on demand. It records findings and observations as `humanish.study-analysis.v1`. | `src/analysis/`; `runAutomaticAnalysis` in `src/analysis/automatic.ts`; `analyzeRun` in `src/analysis/service.ts`; `ANALYSIS_SCHEMA` in `src/analysis/types.ts` |
|
|
42
42
|
| finding | A result of analysis, with citations to evidence in the run. Each finding groups the observations that support it. | `src/analysis/` |
|
package/README.md
CHANGED
|
@@ -335,7 +335,7 @@ and the library exports.
|
|
|
335
335
|
- Breaking changes ship together in one minor release. Its release notes carry a migration
|
|
336
336
|
table: each old form, its new form, and the change to make.
|
|
337
337
|
- The study file and the run bundle carry versioned schemas, `humanish.study.v3` and
|
|
338
|
-
`humanish.run-bundle.
|
|
338
|
+
`humanish.run-bundle.v3`. A breaking change to either one bumps its version.
|
|
339
339
|
|
|
340
340
|
Pin the exact version in `package.json` and read [CHANGELOG.md](CHANGELOG.md) before upgrading.
|
|
341
341
|
|
|
@@ -483,7 +483,7 @@ for the supported library seam.
|
|
|
483
483
|
|
|
484
484
|
- run a study: `runStudy`, `RunStudyOptions`, `StudyOutcome`, `StudyResult`, `StudyEvent`, `routeOf`,
|
|
485
485
|
`parseStudy`, `STUDY_SCHEMA`;
|
|
486
|
-
- read a run: `verifyRun`, `renderObserver`, `RunBundle`, `ActorTrace`;
|
|
486
|
+
- read a run: `verifyRun`, `loadRunBundle`, `renderObserver`, `RunBundle`, `ActorTrace`;
|
|
487
487
|
- bring a participant: `ComputerUseProvider`, `ComputerUseExecutor`, `ProviderContext`,
|
|
488
488
|
`createOpenAiResponsesProvider`, `runComputerUseLoop`, `defaultRedactionHooks`;
|
|
489
489
|
- score a run: `AdapterScorerModule` and its scoring contexts.
|
|
@@ -65,7 +65,12 @@ export function statusForCompletionReason(reason) {
|
|
|
65
65
|
}
|
|
66
66
|
export function loopResult(session, stop, debrief) {
|
|
67
67
|
const { provider, settings, trace: recorder, usage } = session;
|
|
68
|
-
const { completionReason,
|
|
68
|
+
const { completionReason, stopCause } = stop;
|
|
69
|
+
// A participant whose session ended on the clock has its closing account as its reason, as one
|
|
70
|
+
// that ended the session itself has its last message; the stop cause keeps the scheduled end.
|
|
71
|
+
const reason = stopCause === "scheduled_end" && debrief?.report !== undefined
|
|
72
|
+
? debrief.report.summary
|
|
73
|
+
: stop.reason;
|
|
69
74
|
const completedAtMs = session.now();
|
|
70
75
|
const status = statusForCompletionReason(completionReason);
|
|
71
76
|
const ids = {};
|
|
@@ -65,12 +65,6 @@ export declare class OpenAiConversation {
|
|
|
65
65
|
* from the request being sent. False when none are asked for, so the rejection stands.
|
|
66
66
|
*/
|
|
67
67
|
dropReasoningSummaries(): boolean;
|
|
68
|
-
/**
|
|
69
|
-
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
70
|
-
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
71
|
-
* or switched to at runtime.
|
|
72
|
-
*/
|
|
73
|
-
get serverHoldsSession(): boolean;
|
|
74
68
|
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
75
69
|
record(): ActorConversation;
|
|
76
70
|
/**
|
|
@@ -72,14 +72,6 @@ export class OpenAiConversation {
|
|
|
72
72
|
this.reasoningSummary = undefined;
|
|
73
73
|
return true;
|
|
74
74
|
}
|
|
75
|
-
/**
|
|
76
|
-
* Whether the server holds the whole session, so a closing report threaded on it reads every
|
|
77
|
-
* turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
|
|
78
|
-
* or switched to at runtime.
|
|
79
|
-
*/
|
|
80
|
-
get serverHoldsSession() {
|
|
81
|
-
return this.mode === "threaded" && this.lastResponseId !== undefined;
|
|
82
|
-
}
|
|
83
75
|
/** The trace's account of how the conversation was carried (ActorConversation). */
|
|
84
76
|
record() {
|
|
85
77
|
return this.evidence.snapshot(this.mode);
|
|
@@ -509,14 +509,11 @@ export function createOpenAiResponsesProvider(options) {
|
|
|
509
509
|
get conversation() {
|
|
510
510
|
return conversation.record();
|
|
511
511
|
},
|
|
512
|
-
// A
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
},
|
|
518
|
-
// Offered in both modes. An explicit_context conversation carries its newest screens whole and
|
|
519
|
-
// earlier turns as text, which serves impressions; a closing report needs the whole session.
|
|
512
|
+
// Both are offered in both modes. A threaded request reads the session the server holds; an
|
|
513
|
+
// explicit_context one carries its newest screens whole and its oldest turns summarized past
|
|
514
|
+
// the budget, which the trace's conversation record counts. A participant that worked a long
|
|
515
|
+
// shift writes its closing account from what it carries, as a person writes one from memory.
|
|
516
|
+
debrief: (req, signal) => requestTurn(req, signal, "report"),
|
|
520
517
|
requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
|
|
521
518
|
};
|
|
522
519
|
}
|
|
@@ -3,7 +3,7 @@ import { createHash } from "node:crypto";
|
|
|
3
3
|
import { screenshotEvidenceError } from "../evidence/image.js";
|
|
4
4
|
import { runIdOf } from "../run/paths.js";
|
|
5
5
|
import { isRecord } from "../run/type-guards.js";
|
|
6
|
-
import {
|
|
6
|
+
import { isReadRunBundleSchema, RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
7
7
|
import { loadStreamTraces } from "../run/stream-trace.js";
|
|
8
8
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
9
9
|
import { digestAnalysisInput, hashAnalysisValue, validateAnalysisInputMetadata, } from "./validation.js";
|
|
@@ -30,7 +30,7 @@ function parseSource(prepared, bytes) {
|
|
|
30
30
|
throw new Error("ANALYSIS_SOURCE_TOO_LARGE");
|
|
31
31
|
const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
|
|
32
32
|
if (!isRecord(value) ||
|
|
33
|
-
(value.schema
|
|
33
|
+
!isReadRunBundleSchema(value.schema) ||
|
|
34
34
|
value.runId !== runIdOf(prepared) ||
|
|
35
35
|
!Array.isArray(value.streams) ||
|
|
36
36
|
value.streams.length > EVIDENCE_LIMITS.participants ||
|
package/dist/evidence/image.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { crc32, inflateSync } from "node:zlib";
|
|
1
2
|
import { PNG } from "pngjs";
|
|
2
3
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
3
4
|
const PNG_IHDR_LENGTH = 13;
|
|
@@ -145,12 +146,90 @@ export function screenshotEvidenceError(relativePath, bytes) {
|
|
|
145
146
|
if (chunks?.some((chunk) => chunk.type === "IEND" && chunk.end - chunk.start !== 12)) {
|
|
146
147
|
return "PNG IEND chunk must be empty";
|
|
147
148
|
}
|
|
149
|
+
return chunks === null || !pngStructureHolds(bytes, chunks)
|
|
150
|
+
? "could not decode PNG evidence"
|
|
151
|
+
: null;
|
|
152
|
+
}
|
|
153
|
+
/** The bit depths PNG allows for each color type. */
|
|
154
|
+
const PNG_BIT_DEPTHS = {
|
|
155
|
+
0: [1, 2, 4, 8, 16],
|
|
156
|
+
2: [8, 16],
|
|
157
|
+
3: [1, 2, 4, 8],
|
|
158
|
+
4: [8, 16],
|
|
159
|
+
6: [8, 16],
|
|
160
|
+
};
|
|
161
|
+
const PNG_CHANNELS = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 };
|
|
162
|
+
/** The fewest bytes the decoder reads from a chunk; a shorter one fails to decode. */
|
|
163
|
+
const PNG_MIN_CHUNK_BYTES = { gAMA: 4 };
|
|
164
|
+
/**
|
|
165
|
+
* Whether a PNG that passed pngDecodeRefusal decodes, read from its structure: the file ends at an
|
|
166
|
+
* empty IEND, every chunk's CRC matches, IHDR comes once with fields PNG defines, the image data is
|
|
167
|
+
* one run of IDAT chunks, and it inflates, with zlib's native code, to exactly the
|
|
168
|
+
* bytes its rows need with nothing after the stream, each row starting with a filter type from 0
|
|
169
|
+
* to 4. For an image without a palette that is all a decode establishes, at about an eighth of
|
|
170
|
+
* the time of pngjs, which unfilters every pixel in JavaScript; a run holds a screenshot for each
|
|
171
|
+
* turn. A palette image is also decoded, since each pixel must index a palette that comes before
|
|
172
|
+
* the image data; in any other image a palette chunk is only a suggestion, wherever it is.
|
|
173
|
+
*/
|
|
174
|
+
function pngStructureHolds(bytes, chunks) {
|
|
175
|
+
const last = chunks.at(-1);
|
|
176
|
+
if (last?.type !== "IEND" || last.end !== bytes.length)
|
|
177
|
+
return false;
|
|
178
|
+
const crcHolds = (chunk) => crc32(bytes.subarray(chunk.start + 4, chunk.end - 4)) === bytes.readUInt32BE(chunk.end - 4);
|
|
179
|
+
if (!chunks.every(crcHolds))
|
|
180
|
+
return false;
|
|
181
|
+
const types = chunks.map((chunk) => chunk.type);
|
|
182
|
+
const firstIdat = types.indexOf("IDAT");
|
|
183
|
+
const idats = chunks.filter((chunk) => chunk.type === "IDAT");
|
|
184
|
+
if (types.lastIndexOf("IHDR") !== 0 ||
|
|
185
|
+
firstIdat < 0 ||
|
|
186
|
+
types.lastIndexOf("IDAT") !== firstIdat + idats.length - 1)
|
|
187
|
+
return false;
|
|
188
|
+
const [width, height] = [bytes.readUInt32BE(16), bytes.readUInt32BE(20)];
|
|
189
|
+
const [depth, colorType, compression, filter] = [bytes[24], bytes[25], bytes[26], bytes[27]];
|
|
190
|
+
if (!PNG_BIT_DEPTHS[colorType]?.includes(depth) || compression !== 0 || filter !== 0)
|
|
191
|
+
return false;
|
|
192
|
+
// The decoder sizes a row in 32-bit arithmetic and refuses one wider; no screenshot is.
|
|
193
|
+
const rowBits = width * PNG_CHANNELS[colorType] * depth;
|
|
194
|
+
if (rowBits + 7 > 0x7fffffff)
|
|
195
|
+
return false;
|
|
196
|
+
const trnsBytes = colorType === 0 ? 2 : colorType === 2 ? 6 : 0;
|
|
197
|
+
const tooShort = (chunk) => chunk.end - chunk.start - 12 <
|
|
198
|
+
(chunk.type === "tRNS" ? trnsBytes : (PNG_MIN_CHUNK_BYTES[chunk.type] ?? 0));
|
|
199
|
+
if (chunks.some(tooShort))
|
|
200
|
+
return false;
|
|
201
|
+
const rowBytes = 1 + Math.ceil(rowBits / 8);
|
|
202
|
+
const data = Buffer.concat(idats.map((chunk) => bytes.subarray(chunk.start + 8, chunk.end - 4)));
|
|
203
|
+
let rows;
|
|
148
204
|
try {
|
|
149
|
-
|
|
150
|
-
|
|
205
|
+
// A stream that would inflate past the rows the image needs stops there, so a small file
|
|
206
|
+
// cannot claim gigabytes.
|
|
207
|
+
const options = {
|
|
208
|
+
maxOutputLength: rowBytes * height,
|
|
209
|
+
info: true,
|
|
210
|
+
};
|
|
211
|
+
const inflated = inflateSync(data, options);
|
|
212
|
+
if (inflated.engine.bytesWritten !== data.length)
|
|
213
|
+
return false;
|
|
214
|
+
rows = inflated.buffer;
|
|
151
215
|
}
|
|
152
216
|
catch {
|
|
153
|
-
return
|
|
217
|
+
return false;
|
|
218
|
+
}
|
|
219
|
+
if (rows.length !== rowBytes * height)
|
|
220
|
+
return false;
|
|
221
|
+
for (let row = 0; row < rows.length; row += rowBytes)
|
|
222
|
+
if (rows[row] > 4)
|
|
223
|
+
return false;
|
|
224
|
+
return colorType !== 3 || pngDecodes(bytes);
|
|
225
|
+
}
|
|
226
|
+
function pngDecodes(bytes) {
|
|
227
|
+
try {
|
|
228
|
+
PNG.sync.read(bytes, { checkCRC: true });
|
|
229
|
+
return true;
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return false;
|
|
154
233
|
}
|
|
155
234
|
}
|
|
156
235
|
export function assertScreenshotEvidence(relativePath, bytes) {
|
|
@@ -5,7 +5,6 @@ import { lstat, mkdir, mkdtemp, readdir, realpath, rename, rm, rmdir } from "nod
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { tmpdir } from "node:os";
|
|
7
7
|
import { renderObserver } from "../observer/render.js";
|
|
8
|
-
import { buildObserverData } from "../observer/data.js";
|
|
9
8
|
import { inventorySandboxIds, publicSandboxView, scrubSandboxIdBytes } from "../run/sandbox-ids.js";
|
|
10
9
|
import { splitStreamTraces } from "../run/stream-trace.js";
|
|
11
10
|
import { redactScreenshot, redactText } from "../evidence/redaction.js";
|
|
@@ -441,10 +440,7 @@ async function buildDerivative(stage, source, runId, bundle, maxBytes) {
|
|
|
441
440
|
note: "A redacted copy of the same study, not a new attempt. Original pixels are not present. Text and findings still require human review before sharing.",
|
|
442
441
|
files: transformed.entries,
|
|
443
442
|
};
|
|
444
|
-
|
|
445
|
-
if (!transformedBundle)
|
|
446
|
-
throw new Error("Derivative bundle could not be loaded.");
|
|
447
|
-
await writeContainedOutputFile(stagePaths, "observer/observer-data.json", jsonBytes(buildObserverData(transformedBundle.bundle)));
|
|
443
|
+
// Rendering writes observer-data.json and index.html from the derivative's run.json.
|
|
448
444
|
const rendered = await renderObserver(stage.physicalPath, runId, { open: false });
|
|
449
445
|
if (!rendered.ok)
|
|
450
446
|
throw new Error("Derivative Observer could not be rebuilt.");
|
package/dist/feedback/export.js
CHANGED
|
@@ -13,7 +13,7 @@ import { createHash } from "node:crypto";
|
|
|
13
13
|
import path from "node:path";
|
|
14
14
|
import { renderObserver } from "../observer/render.js";
|
|
15
15
|
import { renderObserverHtml } from "../observer/artifact.js";
|
|
16
|
-
import { buildObserverData } from "../observer/data.js";
|
|
16
|
+
import { buildObserverData, OBSERVER_DATA_SCHEMA, observerDataFromSaved, } from "../observer/data.js";
|
|
17
17
|
import { resolveRunPath } from "../run/locate.js";
|
|
18
18
|
import { loadRunTraces, StreamTraceRefusedError } from "../run/stream-trace.js";
|
|
19
19
|
import { verifyRun } from "../verify/verify.js";
|
|
@@ -25,6 +25,7 @@ import { cli } from "../cli/invocation.js";
|
|
|
25
25
|
import { analysisSharingProblems } from "../analysis/sharing.js";
|
|
26
26
|
import { EVIDENCE_LIMITS, validateAnalysisEvidence } from "../analysis/evidence.js";
|
|
27
27
|
import { readBoundedFile } from "../run/evidence-files.js";
|
|
28
|
+
import { isRecord } from "../run/type-guards.js";
|
|
28
29
|
import { shellQuote } from "../substrates/shell.js";
|
|
29
30
|
import { plural } from "../run/text.js";
|
|
30
31
|
const EXPORT_SCHEMA = "humanish.export-result.v1";
|
|
@@ -80,7 +81,7 @@ export async function exportRun(cwdInput, runInput, options = {}, deps = {}) {
|
|
|
80
81
|
const analysis = await loadAnalysis(runPaths);
|
|
81
82
|
const data = await currentObserverData(runPaths, analysis, slot);
|
|
82
83
|
if (data === undefined)
|
|
83
|
-
return exportFailure(cwd, runInput, "HUMANISH_EXPORT_VERIFY_FAILED", "
|
|
84
|
+
return exportFailure(cwd, runInput, "HUMANISH_EXPORT_VERIFY_FAILED", "The run's evidence changed, or a participant's trace could not be read, while preparing the export.");
|
|
84
85
|
const omittedRecording = omitRecordings(data.value);
|
|
85
86
|
if (omittedRecording)
|
|
86
87
|
warnings.push(MEDIA_OMISSION);
|
|
@@ -177,24 +178,42 @@ async function readObserverSlot(cwd, runInput, runRoot, runId, verified, deps, w
|
|
|
177
178
|
/**
|
|
178
179
|
* The Observer data to export. A current report must travel with the current source projection,
|
|
179
180
|
* even if an older saved HTML was never refreshed; old recording-only exports keep the saved slot.
|
|
180
|
-
*
|
|
181
|
+
* A page saved since observer-data v2 holds no participant's items in its slot, and verify does not
|
|
182
|
+
* read the page's compressed copy, so its export is the current projection of the run and its
|
|
183
|
+
* traces. Undefined when run.json no longer matches the analysis, or a v2 page's run cannot be read.
|
|
181
184
|
*/
|
|
182
185
|
async function currentObserverData(runPaths, analysis, slot) {
|
|
183
186
|
const saved = JSON.parse(slot);
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
187
|
+
const source = async () => readBoundedFile(runPaths, "run.json", EVIDENCE_LIMITS.sourceBytes);
|
|
188
|
+
if (analysis.state !== "ready" || !analysis.analysis) {
|
|
189
|
+
const schema = saved?.schema;
|
|
190
|
+
if (schema === "humanish.observer-data.v1")
|
|
191
|
+
return { value: observerDataFromSaved(saved) };
|
|
192
|
+
if (schema !== OBSERVER_DATA_SCHEMA)
|
|
193
|
+
return { value: saved };
|
|
194
|
+
const bundle = await sourceTraces(runPaths, await source());
|
|
195
|
+
return bundle === undefined ? undefined : { value: buildObserverData(bundle) };
|
|
196
|
+
}
|
|
197
|
+
const bytes = await source();
|
|
198
|
+
if (!bytes ||
|
|
199
|
+
createHash("sha256").update(bytes).digest("hex") !== analysis.analysis.sourceRunSha256)
|
|
200
|
+
return undefined;
|
|
201
|
+
const bundle = await sourceTraces(runPaths, bytes);
|
|
202
|
+
return bundle === undefined ? undefined : { value: buildObserverData(bundle) };
|
|
203
|
+
}
|
|
204
|
+
/** The run with its traces read, or undefined when run.json is missing or a trace is refused. */
|
|
205
|
+
async function sourceTraces(runPaths, bytes) {
|
|
206
|
+
if (!bytes)
|
|
207
|
+
return undefined;
|
|
208
|
+
const stored = JSON.parse(bytes.toString("utf8"));
|
|
209
|
+
if (!isRecord(stored) || !Array.isArray(stored.streams))
|
|
189
210
|
return undefined;
|
|
190
|
-
const stored = JSON.parse(source.toString("utf8"));
|
|
191
211
|
// A trace that no longer matches run.json's index is a source that no longer matches.
|
|
192
|
-
|
|
212
|
+
return loadRunTraces(runPaths, stored).catch((error) => {
|
|
193
213
|
if (error instanceof StreamTraceRefusedError)
|
|
194
214
|
return undefined;
|
|
195
215
|
throw error;
|
|
196
216
|
});
|
|
197
|
-
return bundle === undefined ? undefined : { value: buildObserverData(bundle) };
|
|
198
217
|
}
|
|
199
218
|
/** Portable HTML keeps the lightweight evidence; media remains in the source run. */
|
|
200
219
|
function omitRecordings(data) {
|
package/dist/index.d.ts
CHANGED
|
@@ -8,6 +8,7 @@ export { parseStudy } from "./study/config.js";
|
|
|
8
8
|
export { STUDY_SCHEMA } from "./study/types.js";
|
|
9
9
|
export type { StudyConfig } from "./study/types.js";
|
|
10
10
|
export { verifyRun } from "./verify/verify.js";
|
|
11
|
+
export { loadRunBundle } from "./run/locate.js";
|
|
11
12
|
export type { VerifyResult } from "./verify/verify.js";
|
|
12
13
|
export { renderObserver } from "./observer/render.js";
|
|
13
14
|
export type { ObserverResult } from "./observer/render.js";
|
package/dist/index.js
CHANGED
|
@@ -8,6 +8,7 @@ export { parseStudy } from "./study/config.js";
|
|
|
8
8
|
export { STUDY_SCHEMA } from "./study/types.js";
|
|
9
9
|
// Read a run.
|
|
10
10
|
export { verifyRun } from "./verify/verify.js";
|
|
11
|
+
export { loadRunBundle } from "./run/locate.js";
|
|
11
12
|
export { renderObserver } from "./observer/render.js";
|
|
12
13
|
// Bring a participant.
|
|
13
14
|
export { runComputerUseLoop, stableProgressKey } from "./actors/computer-use/loop.js";
|
|
@@ -7,6 +7,7 @@ import path from "node:path";
|
|
|
7
7
|
import { fileURLToPath } from "node:url";
|
|
8
8
|
import { projectShareCheckedAnalysis, analysisSharingProblems } from "../analysis/sharing.js";
|
|
9
9
|
import { withObserverEndings } from "./data.js";
|
|
10
|
+
import { escapeJsonScript, observerDataElements } from "./page-data.js";
|
|
10
11
|
const OBSERVER_DATA_PLACEHOLDER = "__HUMANISH_OBSERVER_DATA__";
|
|
11
12
|
const OBSERVER_DATA_SLOT = `<script id="observer-data" type="application/json">${OBSERVER_DATA_PLACEHOLDER}</script>`;
|
|
12
13
|
let cachedObserverArtifact = null;
|
|
@@ -166,7 +167,7 @@ function renderObserverAppHtml(data, snapshot, analysis, notes, assets) {
|
|
|
166
167
|
return (snapshot
|
|
167
168
|
? artifact.replace("</head>", '<meta name="humanish-observer-mode" content="snapshot"></head>')
|
|
168
169
|
: artifact)
|
|
169
|
-
.replace(OBSERVER_DATA_SLOT, () =>
|
|
170
|
+
.replace(OBSERVER_DATA_SLOT, () => observerDataElements(data))
|
|
170
171
|
.replace(/<script id="study-analysis" type="application\/json">[\s\S]*?<\/script>/, () => `<script id="study-analysis" type="application/json">${escapeJsonScript(analysis)}</script>`)
|
|
171
172
|
.replace(/<script id="run-notes" type="application\/json">[\s\S]*?<\/script>/, () => `<script id="run-notes" type="application/json">${escapeJsonScript(notes)}</script>`)
|
|
172
173
|
.replace("</body>", () => `${renderExportAssets(assets)}</body>`)
|
|
@@ -241,11 +242,3 @@ function escapeHtml(value) {
|
|
|
241
242
|
}
|
|
242
243
|
});
|
|
243
244
|
}
|
|
244
|
-
function escapeJsonScript(value) {
|
|
245
|
-
return JSON.stringify(value)
|
|
246
|
-
.replace(/</g, "\\u003c")
|
|
247
|
-
.replace(/>/g, "\\u003e")
|
|
248
|
-
.replace(/&/g, "\\u0026")
|
|
249
|
-
.replace(/\u2028/g, "\\u2028")
|
|
250
|
-
.replace(/\u2029/g, "\\u2029");
|
|
251
|
-
}
|
package/dist/observer/data.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type ActorEnding } from "../actors/stop-cause.js";
|
|
|
2
2
|
import type { RunBundle, RunCostSummary, RunEvent, RunSimulation } from "../run/bundle.js";
|
|
3
3
|
import { type RunDisplay } from "../run/display.js";
|
|
4
4
|
import type { RunStream, RunStreamKind } from "../run/streams.js";
|
|
5
|
-
export declare const OBSERVER_DATA_SCHEMA = "humanish.observer-data.
|
|
5
|
+
export declare const OBSERVER_DATA_SCHEMA = "humanish.observer-data.v2";
|
|
6
6
|
interface ObserverArtifactLink {
|
|
7
7
|
label: string;
|
|
8
8
|
href: string;
|
|
@@ -10,7 +10,7 @@ interface ObserverArtifactLink {
|
|
|
10
10
|
}
|
|
11
11
|
export interface ObserverData {
|
|
12
12
|
schema: typeof OBSERVER_DATA_SCHEMA;
|
|
13
|
-
schemaVersion:
|
|
13
|
+
schemaVersion: 2;
|
|
14
14
|
generatedAt: string;
|
|
15
15
|
/** Served-only liveness observation. Never a replacement for the recorded evidence verdict. */
|
|
16
16
|
runtime?: {
|
|
@@ -123,6 +123,13 @@ interface ObserverLaneGroup {
|
|
|
123
123
|
* carried its outcome.
|
|
124
124
|
*/
|
|
125
125
|
export declare function buildObserverData(bundle: RunBundle, generatedAt?: string, statusRecord?: unknown): ObserverData;
|
|
126
|
+
/**
|
|
127
|
+
* A saved Observer projection as this version reads it, unchecked as before: an observer-data.json
|
|
128
|
+
* written before v2 holds every participant's items. A v2 one holds none, so it is read only when
|
|
129
|
+
* no participant has any: showing participants with no steps would say they did nothing, and the
|
|
130
|
+
* items are in the run's traces. Null for a v2 projection that leaves items out.
|
|
131
|
+
*/
|
|
132
|
+
export declare function observerDataFromSaved(value: unknown): ObserverData | null;
|
|
126
133
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
|
127
134
|
export declare function withObserverEndings(data: ObserverData): ObserverData;
|
|
128
135
|
export {};
|
package/dist/observer/data.js
CHANGED
|
@@ -3,7 +3,8 @@ import { isCommsReceivingEvidence, receivingAnalysisContext } from "../comms/rec
|
|
|
3
3
|
import { formatParticipantOutcomes, formatRunTaskFunnel, participantOutcomeDetails, withCuaReviewProvenance, } from "../run/outcomes.js";
|
|
4
4
|
import { cuaGoalSource, CUA_COMPLETION_NOTE } from "../actors/goal-source.js";
|
|
5
5
|
import { bundleDisplayFacts, runDisplay } from "../run/display.js";
|
|
6
|
-
|
|
6
|
+
import { joinObserverData, shellNeedsItems } from "./participant-items.js";
|
|
7
|
+
export const OBSERVER_DATA_SCHEMA = "humanish.observer-data.v2";
|
|
7
8
|
/** Discard a forged or stale runtime grant before projecting persisted evidence. */
|
|
8
9
|
export function recordedStreamEmbed(embed) {
|
|
9
10
|
if (!embed)
|
|
@@ -80,7 +81,7 @@ export function buildObserverData(bundle, generatedAt = new Date().toISOString()
|
|
|
80
81
|
const active = streams.filter((stream) => stream.status === "running" || stream.status === "preparing").length;
|
|
81
82
|
return withObserverEndings({
|
|
82
83
|
schema: OBSERVER_DATA_SCHEMA,
|
|
83
|
-
schemaVersion:
|
|
84
|
+
schemaVersion: 2,
|
|
84
85
|
generatedAt,
|
|
85
86
|
run: {
|
|
86
87
|
runId: bundle.runId,
|
|
@@ -141,6 +142,20 @@ export function buildObserverData(bundle, generatedAt = new Date().toISOString()
|
|
|
141
142
|
},
|
|
142
143
|
});
|
|
143
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* A saved Observer projection as this version reads it, unchecked as before: an observer-data.json
|
|
147
|
+
* written before v2 holds every participant's items. A v2 one holds none, so it is read only when
|
|
148
|
+
* no participant has any: showing participants with no steps would say they did nothing, and the
|
|
149
|
+
* items are in the run's traces. Null for a v2 projection that leaves items out.
|
|
150
|
+
*/
|
|
151
|
+
export function observerDataFromSaved(value) {
|
|
152
|
+
const saved = value;
|
|
153
|
+
if (saved.schema === "humanish.observer-data.v1")
|
|
154
|
+
return { ...saved, schema: OBSERVER_DATA_SCHEMA, schemaVersion: 2 };
|
|
155
|
+
return shellNeedsItems(saved)
|
|
156
|
+
? null
|
|
157
|
+
: joinObserverData(saved, null);
|
|
158
|
+
}
|
|
144
159
|
/** Refresh presentation from recorded actor evidence, including older exported snapshots. */
|
|
145
160
|
export function withObserverEndings(data) {
|
|
146
161
|
const streams = (data.streams ?? []).map(({ ending: _previousEnding, ...stream }) => {
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ObserverData } from "./data.js";
|
|
2
|
+
/** The page's data elements for `data`, in place of the built page's data slot. */
|
|
3
|
+
export declare function observerDataElements(data: ObserverData): string;
|
|
4
|
+
/** JSON that stays inert inside a script element: no `</script>` breakout, no HTML entities. */
|
|
5
|
+
export declare function escapeJsonScript(value: unknown): string;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// How an Observer page carries a run's data: observer-data.json's content as JSON, and every
|
|
2
|
+
// participant's trace items gzip-compressed in base64, which a file:// page decodes without a
|
|
3
|
+
// fetch. The items are most of a long run's bytes and compress about eight times.
|
|
4
|
+
import { gzipSync } from "node:zlib";
|
|
5
|
+
import { splitObserverData } from "./participant-items.js";
|
|
6
|
+
/** The page's data elements for `data`, in place of the built page's data slot. */
|
|
7
|
+
export function observerDataElements(data) {
|
|
8
|
+
const { shell, items } = splitObserverData(data);
|
|
9
|
+
const packed = gzipSync(JSON.stringify(items)).toString("base64");
|
|
10
|
+
return (`<script id="observer-data" type="application/json">${escapeJsonScript(shell)}</script>` +
|
|
11
|
+
`<script id="observer-items" type="application/octet-stream" data-encoding="gzip">${packed}</script>`);
|
|
12
|
+
}
|
|
13
|
+
/** JSON that stays inert inside a script element: no `</script>` breakout, no HTML entities. */
|
|
14
|
+
export function escapeJsonScript(value) {
|
|
15
|
+
return JSON.stringify(value)
|
|
16
|
+
.replace(/</g, "\\u003c")
|
|
17
|
+
.replace(/>/g, "\\u003e")
|
|
18
|
+
.replace(/&/g, "\\u0026")
|
|
19
|
+
.replace(/\u2028/g, "\\u2028")
|
|
20
|
+
.replace(/\u2029/g, "\\u2029");
|
|
21
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { ActorTraceItem } from "../actors/contract.js";
|
|
2
|
+
import type { ObserverData, ObserverStream } from "./data.js";
|
|
3
|
+
export declare const OBSERVER_ITEMS_SCHEMA = "humanish.observer-items.v1";
|
|
4
|
+
/** A stream's actor and live actor, each without its items. */
|
|
5
|
+
type ObserverShellStream = Omit<ObserverStream, "actor" | "liveActor"> & {
|
|
6
|
+
actor?: Omit<NonNullable<ObserverStream["actor"]>, "items">;
|
|
7
|
+
liveActor?: Omit<NonNullable<ObserverStream["liveActor"]>, "items">;
|
|
8
|
+
/** How many screenshot entries of `artifacts` travel with the items, when any do. */
|
|
9
|
+
screenshotArtifacts?: number;
|
|
10
|
+
};
|
|
11
|
+
/** observer-data.json: the Observer's data with no participant's trace items. */
|
|
12
|
+
export type ObserverShell = Omit<ObserverData, "streams"> & {
|
|
13
|
+
streams: ObserverShellStream[];
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* Each participant's trace items by stream id, with the stream's artifact list when it names
|
|
17
|
+
* screenshots: one per turn, so it grows with the items.
|
|
18
|
+
*/
|
|
19
|
+
export interface ObserverItems {
|
|
20
|
+
schema: typeof OBSERVER_ITEMS_SCHEMA;
|
|
21
|
+
runId: string;
|
|
22
|
+
streams: Record<string, {
|
|
23
|
+
actor?: ActorTraceItem[];
|
|
24
|
+
liveActor?: ActorTraceItem[];
|
|
25
|
+
artifacts?: ObserverStream["artifacts"];
|
|
26
|
+
}>;
|
|
27
|
+
}
|
|
28
|
+
/** The data as observer-data.json and the page's items carry it. */
|
|
29
|
+
export declare function splitObserverData(data: ObserverData): {
|
|
30
|
+
shell: ObserverShell;
|
|
31
|
+
items: ObserverItems;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Whether `items` are this run's and hold every participant's items that the data leaves out. The
|
|
35
|
+
* page reads nothing short of that: participants shown with no steps would say they did nothing.
|
|
36
|
+
*/
|
|
37
|
+
export declare function itemsCoverShell(shell: ObserverShell, items: ObserverItems): boolean;
|
|
38
|
+
/** Whether the data leaves out items or screenshot entries that only an items document holds. */
|
|
39
|
+
export declare function shellNeedsItems(shell: ObserverShell): boolean;
|
|
40
|
+
/**
|
|
41
|
+
* The data with each participant's items back in place. A stream the items do not cover keeps an
|
|
42
|
+
* empty list, so the page shows the participant with no recorded steps.
|
|
43
|
+
*/
|
|
44
|
+
export declare function joinObserverData(shell: ObserverShell, items: ObserverItems | null): ObserverData;
|
|
45
|
+
export {};
|