humanish 0.111.4 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +71 -22
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { redactText } from "../../evidence/redaction.js";
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import {} from "../../run/judge.js";
|
|
5
|
+
import { renderReviewMarkdown } from "../../run/review-markdown.js";
|
|
6
6
|
import {} from "../../run/streams.js";
|
|
7
7
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
8
8
|
import { TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGERS_ARTIFACT, TERMINAL_TRANSCRIPT_ARTIFACT, } from "../../run/terminal-contract.js";
|
|
9
9
|
import { DEFAULT_RUNTIME_AUTH } from "./runtime-auth.js";
|
|
10
|
+
import { participantCaption } from "../../run/participant-caption.js";
|
|
11
|
+
import { formatParticipantOutcomes, tallyParticipantOutcomes } from "../../run/outcomes.js";
|
|
10
12
|
import { describeMeasuredSpend, noSpendLineMeasured, noSpendNotEstablished } from "./ledger.js";
|
|
11
13
|
/**
|
|
12
14
|
* Project the terminal-product run into a humanish.run-bundle.v1 (no schema change: a new
|
|
@@ -80,7 +82,7 @@ export function buildTerminalProductBundle(args) {
|
|
|
80
82
|
const review = {
|
|
81
83
|
schema: REVIEW_SCHEMA,
|
|
82
84
|
verdict: args.verdict,
|
|
83
|
-
summary:
|
|
85
|
+
summary: `Dry run: 1 participant would try ${args.productName} from a terminal. No sandbox was created, no key was used and $0 was spent.`,
|
|
84
86
|
gaps: [
|
|
85
87
|
"This dry run did not execute the live in-sandbox agent route; it checks the evidence shape only, not live behavior, scale, or adoption.",
|
|
86
88
|
"No exec-stream, transcript, substrate, cost, or cleanup artifacts were produced because no live session ran; live verification requires those artifacts.",
|
|
@@ -164,7 +166,7 @@ export function buildLiveTerminalProductBundle(args) {
|
|
|
164
166
|
const review = {
|
|
165
167
|
schema: REVIEW_SCHEMA,
|
|
166
168
|
verdict: args.verdict,
|
|
167
|
-
summary: args.sessionReason
|
|
169
|
+
summary: `1 participant took part: ${formatParticipantOutcomes(tallyParticipantOutcomes([args.trace.status]))}. How it ended: ${args.sessionReason}`,
|
|
168
170
|
gaps: [
|
|
169
171
|
...(args.trace.status === "passed"
|
|
170
172
|
? []
|
|
@@ -212,7 +214,7 @@ function terminalParticipant(args, session) {
|
|
|
212
214
|
const stream = participantStream(TERMINAL_IDS, {
|
|
213
215
|
assignment: participantAssignment({ mission: args.mission }),
|
|
214
216
|
kind: "terminal",
|
|
215
|
-
label:
|
|
217
|
+
label: participantCaption({ personaId: args.persona.id }),
|
|
216
218
|
status: session.status,
|
|
217
219
|
transport: "snapshot",
|
|
218
220
|
updatedAt: session.updatedAt,
|
|
@@ -275,25 +277,14 @@ function describeCaps(caps) {
|
|
|
275
277
|
parts.push(`maxMinutes=${caps.maxMinutes}`);
|
|
276
278
|
return parts.length > 0 ? parts.join(", ") : "empty";
|
|
277
279
|
}
|
|
278
|
-
export function renderTerminalReviewMarkdown(bundle) {
|
|
280
|
+
export function renderTerminalReviewMarkdown(bundle, status) {
|
|
279
281
|
const subject = bundle.events.find((event) => event.type === "terminal-lab.subject.declared");
|
|
280
282
|
const credentials = bundle.events.find((event) => event.type === "terminal-lab.credentials.declared");
|
|
281
283
|
const caps = bundle.events.find((event) => event.type === "terminal-lab.caps.declared");
|
|
282
|
-
return [
|
|
283
|
-
`# ${bundle.scenario.title}`,
|
|
284
|
-
"",
|
|
285
|
-
`- run: ${bundle.runId}`,
|
|
286
|
-
`- mode: ${bundle.mode}`,
|
|
287
|
-
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
288
|
-
`- outcome: ${reviewOutcome(bundle)}`,
|
|
289
|
-
`- summary: ${bundle.review.summary}`,
|
|
284
|
+
return renderReviewMarkdown(bundle, [
|
|
290
285
|
`- mission: ${bundle.scenario.goal}`,
|
|
291
286
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
292
287
|
...(credentials ? [`- credentials: ${credentials.message}`] : []),
|
|
293
288
|
...(caps ? [`- caps: ${caps.message}`] : []),
|
|
294
|
-
|
|
295
|
-
? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
|
|
296
|
-
: []),
|
|
297
|
-
"",
|
|
298
|
-
].join("\n");
|
|
289
|
+
], { status });
|
|
299
290
|
}
|
|
@@ -124,6 +124,7 @@ function runtimeDeclaredEvent(runtime, createdAt) {
|
|
|
124
124
|
level: "info",
|
|
125
125
|
type: "terminal-lab.runtime.declared",
|
|
126
126
|
message: redactText(JSON.stringify(declaredRuntimeProvenance({
|
|
127
|
+
doNotTrack: runtime.doNotTrack,
|
|
127
128
|
...(version === undefined ? {} : { version }),
|
|
128
129
|
model,
|
|
129
130
|
modelSource,
|
|
@@ -261,7 +261,10 @@ export class LiveTerminalSandbox {
|
|
|
261
261
|
// The install step may run the product itself (release:dogfood's does: `humanish init
|
|
262
262
|
// --yes`); on the 0.67.0 dogfood that one command arrived unmarked while the participant's
|
|
263
263
|
// nine others carried the marker.
|
|
264
|
-
envs: {
|
|
264
|
+
envs: {
|
|
265
|
+
HUMANISH_STUDY_PARTICIPANT: "1",
|
|
266
|
+
...(plan.runtime.doNotTrack ? { DO_NOT_TRACK: "1" } : {}),
|
|
267
|
+
},
|
|
265
268
|
requestTimeoutMs,
|
|
266
269
|
timeoutMs: PRODUCT_SETUP_TIMEOUT_MS,
|
|
267
270
|
});
|
|
@@ -314,7 +317,11 @@ export class LiveTerminalSandbox {
|
|
|
314
317
|
// The selected command env (raw key or inert placeholder). The participant
|
|
315
318
|
// marker rides the same command: humanish telemetry from inside a study reads as a new
|
|
316
319
|
// adopter otherwise.
|
|
317
|
-
envs: {
|
|
320
|
+
envs: {
|
|
321
|
+
...runtimeEnv.envs,
|
|
322
|
+
HUMANISH_STUDY_PARTICIPANT: "1",
|
|
323
|
+
...(plan.runtime.doNotTrack ? { DO_NOT_TRACK: "1" } : {}),
|
|
324
|
+
},
|
|
318
325
|
requestTimeoutMs,
|
|
319
326
|
timeoutMs: wallClockMs,
|
|
320
327
|
onStdout: (data) => recordStreamedTerminalChunk("stdout", data),
|
|
@@ -69,6 +69,7 @@ export function planTerminalStudy(config, input) {
|
|
|
69
69
|
...(actor?.persona === undefined ? {} : { personaId: actor.persona }),
|
|
70
70
|
...(actor?.mission === undefined ? {} : { mission: actor.mission }),
|
|
71
71
|
runtime: {
|
|
72
|
+
doNotTrack: config.execution?.terminal?.doNotTrack !== false,
|
|
72
73
|
...(runtimeVersion === undefined ? {} : { version: runtimeVersion }),
|
|
73
74
|
// Codex's own default can change with any release and its JSON stream does not name it, so
|
|
74
75
|
// the route always passes a model: the declared one, else humanish's participant default.
|
|
@@ -8,6 +8,7 @@ export declare function parseTerminalRuntimeVersion(stdout: string): string | un
|
|
|
8
8
|
export declare function buildRuntimeVersionCommand(requestedVersion?: string): string;
|
|
9
9
|
export declare function buildRuntimeExecPrefix(version: string, model: string, reasoningEffort?: ReasoningEffort): string;
|
|
10
10
|
export declare function declaredRuntimeProvenance(args: {
|
|
11
|
+
doNotTrack: boolean;
|
|
11
12
|
version?: string;
|
|
12
13
|
model: string;
|
|
13
14
|
modelSource: "declared" | "humanish_default";
|
|
@@ -35,6 +35,7 @@ export function declaredRuntimeProvenance(args) {
|
|
|
35
35
|
return {
|
|
36
36
|
schema: "humanish.actor-runtime.v1",
|
|
37
37
|
package: TERMINAL_RUNTIME_PACKAGE,
|
|
38
|
+
doNotTrack: args.doNotTrack,
|
|
38
39
|
requestedVersion: args.version ?? "latest",
|
|
39
40
|
versionStatus: "unobserved",
|
|
40
41
|
requestedModel: args.model,
|
|
@@ -75,6 +75,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
75
75
|
});
|
|
76
76
|
const { version, model, modelSource, reasoningEffort } = plan.runtime;
|
|
77
77
|
const runtime = declaredRuntimeProvenance({
|
|
78
|
+
doNotTrack: plan.runtime.doNotTrack,
|
|
78
79
|
...(version === undefined ? {} : { version }),
|
|
79
80
|
model: sanitize(model),
|
|
80
81
|
modelSource,
|
package/dist/run/costs.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { runEstimateUsd, sumEstimatedUsd } from "./run-cost.js";
|
|
2
2
|
import { bindExistingRunArtifactPaths } from "./paths.js";
|
|
3
3
|
import { contradictsAccountBilling } from "../verify/costs.js";
|
|
4
4
|
import { EVIDENCE_LIMITS } from "../analysis/evidence.js";
|
|
@@ -20,13 +20,9 @@ export function emptyCostTotals() {
|
|
|
20
20
|
analysisHistoryUncertainRuns: 0,
|
|
21
21
|
};
|
|
22
22
|
}
|
|
23
|
-
const isKnownUsd = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
24
|
-
function sumKnown(a, b) {
|
|
25
|
-
return a === null && b === null ? null : round6((a ?? 0) + (b ?? 0));
|
|
26
|
-
}
|
|
27
23
|
export function addCostTotals(into, next) {
|
|
28
24
|
for (const key of ["estimatedTotalUsd", "runEstimatedUsd", "analysisEstimatedUsd"]) {
|
|
29
|
-
into[key] =
|
|
25
|
+
into[key] = sumEstimatedUsd(into[key], next[key]);
|
|
30
26
|
}
|
|
31
27
|
for (const key of [
|
|
32
28
|
"incompleteRunEstimates",
|
|
@@ -49,11 +45,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
49
45
|
const analysisWarnings = [];
|
|
50
46
|
// A dry run makes no model request and creates no desktop, so with no figure it costs $0. A
|
|
51
47
|
// recorded unknown (null) figure stays unknown.
|
|
52
|
-
costs.runEstimatedUsd =
|
|
53
|
-
? entry.estimatedCostUsd
|
|
54
|
-
: entry.estimatedCostUsd === undefined && entry.mode === "dry-run"
|
|
55
|
-
? 0
|
|
56
|
-
: null;
|
|
48
|
+
costs.runEstimatedUsd = runEstimateUsd(entry);
|
|
57
49
|
try {
|
|
58
50
|
const prepared = await bindExistingRunArtifactPaths(cwd, entry.runId);
|
|
59
51
|
const bytes = await readBoundedFile(prepared, RUN_BUNDLE_FILE, EVIDENCE_LIMITS.sourceBytes);
|
|
@@ -70,9 +62,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
70
62
|
runWarnings.push("RUN_COST_ID_MISMATCH");
|
|
71
63
|
}
|
|
72
64
|
if (bundle?.cost !== undefined) {
|
|
73
|
-
costs.runEstimatedUsd =
|
|
74
|
-
? bundle.cost.estimatedTotalUsd
|
|
75
|
-
: null;
|
|
65
|
+
costs.runEstimatedUsd = runEstimateUsd({ estimatedCostUsd: bundle.cost?.estimatedTotalUsd });
|
|
76
66
|
if (bundle.cost?.fullyEstimated !== true || costs.runEstimatedUsd === null) {
|
|
77
67
|
costs.incompleteRunEstimates = 1;
|
|
78
68
|
runWarnings.push("RUN_COST_PARTIAL_OR_UNKNOWN");
|
|
@@ -102,7 +92,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
102
92
|
costs.analysisHistoryUncertainRuns = 1;
|
|
103
93
|
analysisWarnings.push("STUDY_COST_ACCOUNTING_UNAVAILABLE");
|
|
104
94
|
}
|
|
105
|
-
costs.estimatedTotalUsd =
|
|
95
|
+
costs.estimatedTotalUsd = sumEstimatedUsd(costs.runEstimatedUsd, costs.analysisEstimatedUsd);
|
|
106
96
|
return {
|
|
107
97
|
runId: entry.runId,
|
|
108
98
|
costs,
|
|
@@ -153,16 +143,15 @@ export async function readAnalysisAccounting(prepared) {
|
|
|
153
143
|
}
|
|
154
144
|
if (!usage.dispatched) {
|
|
155
145
|
accounting.notDispatched += 1;
|
|
156
|
-
accounting.estimatedUsd =
|
|
146
|
+
accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, 0);
|
|
157
147
|
continue;
|
|
158
148
|
}
|
|
159
149
|
accounting.dispatched += 1;
|
|
160
150
|
if (record.receipt)
|
|
161
151
|
providers.add(record.receipt.provider);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
if (!isKnownUsd(usage.estimatedCostUsd) || !usage.usageComplete)
|
|
152
|
+
const estimatedUsd = runEstimateUsd({ estimatedCostUsd: usage.estimatedCostUsd });
|
|
153
|
+
accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, estimatedUsd);
|
|
154
|
+
if (estimatedUsd === null || !usage.usageComplete)
|
|
166
155
|
accounting.unpriced += 1;
|
|
167
156
|
}
|
|
168
157
|
// A skipped or queued automatic job says nothing about earlier manual requests. Only final
|
package/dist/run/detail.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ interface RunThought {
|
|
|
11
11
|
export interface RunParticipant {
|
|
12
12
|
/** Stream id: stable within a run, and what distinguishes participants of one study. */
|
|
13
13
|
id: string;
|
|
14
|
-
/** What to call them on screen: the
|
|
14
|
+
/** What to call them on screen: the caption the Observer shows, else the persona id. */
|
|
15
15
|
label: string;
|
|
16
16
|
/** The persona id, when the trace recorded one. */
|
|
17
17
|
personaId?: string;
|
package/dist/run/detail.js
CHANGED
|
@@ -12,6 +12,8 @@ import path from "node:path";
|
|
|
12
12
|
import { estimateActorCostForExecution } from "./pricing.js";
|
|
13
13
|
import { resolveRunPath } from "./locate.js";
|
|
14
14
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
15
|
+
import { savedCaption } from "./participant-caption.js";
|
|
16
|
+
import { streamParticipantIdOf } from "./participant-records.js";
|
|
15
17
|
import { readContainedRegularFile } from "./contained-output.js";
|
|
16
18
|
import { isPathInside, resolvePhysicalCwd } from "./paths.js";
|
|
17
19
|
const RUN_DETAIL_SCHEMA = "humanish.run-detail.v1";
|
|
@@ -47,13 +49,20 @@ function participantFrom(stream, index) {
|
|
|
47
49
|
const trace = stream.liveActor ?? stream.actor ?? {};
|
|
48
50
|
const id = stream.id ?? `stream-${index + 1}`;
|
|
49
51
|
const personaId = trace.persona?.id;
|
|
52
|
+
const participantId = streamParticipantIdOf(stream);
|
|
50
53
|
// Computed once: calling it twice to test-then-use reads as though the two could differ.
|
|
51
54
|
const thought = latestThought(trace);
|
|
52
55
|
const status = trace.status ?? stream.status;
|
|
53
56
|
const thoughts = (trace.items ?? []).filter((item) => item?.kind === "reasoning").length;
|
|
54
57
|
return {
|
|
55
58
|
id,
|
|
56
|
-
label: stream.label
|
|
59
|
+
label: stream.label === undefined
|
|
60
|
+
? (personaId ?? id)
|
|
61
|
+
: savedCaption({
|
|
62
|
+
label: stream.label,
|
|
63
|
+
...(participantId === undefined ? {} : { participantId }),
|
|
64
|
+
...(personaId === undefined ? {} : { personaId }),
|
|
65
|
+
}),
|
|
57
66
|
...(personaId === undefined ? {} : { personaId }),
|
|
58
67
|
traits: Array.isArray(trace.persona?.traitsApplied) ? trace.persona.traitsApplied : [],
|
|
59
68
|
...(status === undefined ? {} : { status }),
|
package/dist/run/display.d.ts
CHANGED
|
@@ -49,14 +49,14 @@ export interface DisplayedBundle {
|
|
|
49
49
|
* The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
|
|
50
50
|
* without one is in progress, or was recorded before the field existed: its status record, when
|
|
51
51
|
* given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
|
|
52
|
-
* With no record, a bundle whose simulations are still running is
|
|
52
|
+
* With no record, a bundle whose simulations are still running is interrupted.
|
|
53
53
|
*/
|
|
54
54
|
export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unknown, nowMs?: number): RunDisplayFacts;
|
|
55
55
|
/**
|
|
56
56
|
* How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
|
|
57
57
|
* execution failed never reads as a pass beside its participants' verdict.
|
|
58
58
|
*/
|
|
59
|
-
export declare function reviewOutcome(bundle: DisplayedBundle): string;
|
|
59
|
+
export declare function reviewOutcome(bundle: DisplayedBundle, status?: unknown): string;
|
|
60
60
|
/**
|
|
61
61
|
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
|
62
62
|
* was still running keeps the review its last live flush wrote, which describes that session as
|
package/dist/run/display.js
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// library, `humanish runs`, `humanish stats` and review.md. Each of them calls runDisplay with the
|
|
3
3
|
// same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
|
|
4
4
|
// cannot disagree about one run.
|
|
5
|
-
import {
|
|
5
|
+
import {} from "./status.js";
|
|
6
|
+
import { runLiveness } from "./liveness.js";
|
|
6
7
|
import { plural } from "./text.js";
|
|
7
8
|
const LABELS = {
|
|
8
9
|
running: "running",
|
|
@@ -72,38 +73,34 @@ function runDisplayLine(display) {
|
|
|
72
73
|
* The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
|
|
73
74
|
* without one is in progress, or was recorded before the field existed: its status record, when
|
|
74
75
|
* given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
|
|
75
|
-
* With no record, a bundle whose simulations are still running is
|
|
76
|
+
* With no record, a bundle whose simulations are still running is interrupted.
|
|
76
77
|
*/
|
|
77
78
|
export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
|
|
78
|
-
const
|
|
79
|
+
const { liveness, record } = runLiveness(bundle.runId, status, bundle, nowMs);
|
|
80
|
+
const base = { liveness, mode: bundle.mode, verdict: bundle.review?.verdict };
|
|
79
81
|
const outcome = bundle.outcome;
|
|
80
82
|
if (outcome?.state === "interrupted")
|
|
81
|
-
return
|
|
83
|
+
return base;
|
|
82
84
|
if (outcome?.state === "finished")
|
|
83
85
|
return {
|
|
84
86
|
...base,
|
|
85
|
-
liveness: "finished",
|
|
86
87
|
ok: outcome.ok,
|
|
87
88
|
failure: outcome.execution.failures[0],
|
|
88
89
|
};
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
: { failure: status.outcome.execution.failures[0] }),
|
|
97
|
-
};
|
|
98
|
-
const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
|
|
99
|
-
return { ...base, liveness: inProgress ? "running" : "finished" };
|
|
90
|
+
return {
|
|
91
|
+
...base,
|
|
92
|
+
...(typeof record?.outcome?.ok === "boolean" ? { ok: record.outcome.ok } : {}),
|
|
93
|
+
...(record?.outcome?.execution?.failures[0] === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { failure: record.outcome.execution.failures[0] }),
|
|
96
|
+
};
|
|
100
97
|
}
|
|
101
98
|
/**
|
|
102
99
|
* How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
|
|
103
100
|
* execution failed never reads as a pass beside its participants' verdict.
|
|
104
101
|
*/
|
|
105
|
-
export function reviewOutcome(bundle) {
|
|
106
|
-
return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
|
|
102
|
+
export function reviewOutcome(bundle, status) {
|
|
103
|
+
return runDisplayLine(runDisplay(bundleDisplayFacts(bundle, status)));
|
|
107
104
|
}
|
|
108
105
|
/**
|
|
109
106
|
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
package/dist/run/dry-run.js
CHANGED
|
@@ -9,8 +9,8 @@ import {} from "./streams.js";
|
|
|
9
9
|
import { participantEvent, participantIdsOf, participantRecord, participantStream, } from "./participant-records.js";
|
|
10
10
|
import { implicitProjectDirectoryExists, readPackageName, validateCwd } from "./project.js";
|
|
11
11
|
import { loadDryRunInputs } from "./dry-run-inputs.js";
|
|
12
|
-
import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk
|
|
13
|
-
import {
|
|
12
|
+
import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk } from "./judge.js";
|
|
13
|
+
import { renderReviewMarkdown } from "./review-markdown.js";
|
|
14
14
|
import { cli } from "../cli/invocation.js";
|
|
15
15
|
/**
|
|
16
16
|
* The preview route's run. The run scope closes the run it started on every exit, including
|
|
@@ -68,7 +68,7 @@ async function runDryRunInScope(options, scope) {
|
|
|
68
68
|
mode: options.dryRun ? "dry-run" : "live",
|
|
69
69
|
study: options.study,
|
|
70
70
|
warnings: options.warnings,
|
|
71
|
-
renderReview:
|
|
71
|
+
renderReview: renderPreviewReview,
|
|
72
72
|
// The run starts at the time its id and source were stamped with.
|
|
73
73
|
now: () => Date.parse(createdAt),
|
|
74
74
|
...(options.observer === undefined ? {} : { observer: { open: options.observer.open } }),
|
|
@@ -170,28 +170,13 @@ function createReviewSummary(verdict) {
|
|
|
170
170
|
};
|
|
171
171
|
}
|
|
172
172
|
/** The preview's review.md. */
|
|
173
|
-
function
|
|
174
|
-
return
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
Verdict: ${verdictText(bundle.review.verdict, bundle.mode)}
|
|
181
|
-
|
|
182
|
-
Outcome: ${reviewOutcome(bundle)}
|
|
183
|
-
|
|
184
|
-
${bundle.review.summary}
|
|
185
|
-
|
|
186
|
-
## Public-Safety
|
|
187
|
-
|
|
188
|
-
- Redaction: ${bundle.redaction.status}
|
|
189
|
-
- Notes: ${bundle.redaction.notes}
|
|
190
|
-
|
|
191
|
-
## Gaps
|
|
192
|
-
|
|
193
|
-
${bundle.review.gaps.map((gap) => `- ${gap}`).join("\n")}
|
|
194
|
-
`;
|
|
173
|
+
function renderPreviewReview(bundle, status) {
|
|
174
|
+
return renderReviewMarkdown(bundle, [
|
|
175
|
+
"## Public-Safety",
|
|
176
|
+
"",
|
|
177
|
+
`- Redaction: ${bundle.redaction.status}`,
|
|
178
|
+
`- Notes: ${bundle.redaction.notes}`,
|
|
179
|
+
], { style: "preview", status });
|
|
195
180
|
}
|
|
196
181
|
/**
|
|
197
182
|
* Render through the finished run, so a directory swapped in under the same id is never rendered.
|
package/dist/run/judge.d.ts
CHANGED
|
@@ -41,6 +41,30 @@ export interface ParticipantFacts {
|
|
|
41
41
|
/** selfReportedBlocker held for its session. */
|
|
42
42
|
selfReportedBlocker: boolean;
|
|
43
43
|
}
|
|
44
|
+
/** The participant facts its bundle and result can name. */
|
|
45
|
+
export interface ParticipantRecordFacts extends ParticipantFacts {
|
|
46
|
+
id?: string;
|
|
47
|
+
reason?: string;
|
|
48
|
+
skippedReason?: string;
|
|
49
|
+
reportedFriction?: boolean;
|
|
50
|
+
inProgress?: boolean;
|
|
51
|
+
}
|
|
52
|
+
/** Judge participant evidence for the bundle and the result in the same order. */
|
|
53
|
+
export declare function judgeParticipantRecords(participants: readonly ParticipantRecordFacts[], wording?: {
|
|
54
|
+
sessionLabel?: string;
|
|
55
|
+
missingSessionMessage?: string;
|
|
56
|
+
runningReason?: string;
|
|
57
|
+
contractReason?: string;
|
|
58
|
+
}): {
|
|
59
|
+
participants: {
|
|
60
|
+
status: "abandoned" | "blocked" | "contract_proof_only" | "failed" | "incomplete" | "passed" | "running" | "timed_out";
|
|
61
|
+
judgedStatus: "abandoned" | "blocked" | "failed" | "incomplete" | "passed" | "timed_out" | undefined;
|
|
62
|
+
reason: string;
|
|
63
|
+
notPassedMessage: string;
|
|
64
|
+
gapLine: string | undefined;
|
|
65
|
+
}[];
|
|
66
|
+
tally: import("./bundle.js").ParticipantOutcomes;
|
|
67
|
+
};
|
|
44
68
|
/**
|
|
45
69
|
* The status a participant is judged under. A harness failure is failed, and a skipped participant
|
|
46
70
|
* is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
|
package/dist/run/judge.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// 4. foldScorerFailures, which every route with a scorer applies before the run finishes;
|
|
13
13
|
// 5. the outcome: judgeExecution applies the route's policy to its execution failures, and
|
|
14
14
|
// resultOk gives the result's ok.
|
|
15
|
+
import { tallyParticipantOutcomes } from "./outcomes.js";
|
|
15
16
|
import { cli } from "../cli/invocation.js";
|
|
16
17
|
/**
|
|
17
18
|
* The participant said it reached the goal having taken no action and said nothing, and no stop
|
|
@@ -35,6 +36,59 @@ export function selfReportedBlocker(ending) {
|
|
|
35
36
|
ending.closingReportReadsBlocked &&
|
|
36
37
|
!ending.stopConditionMatched);
|
|
37
38
|
}
|
|
39
|
+
/** Judge participant evidence for the bundle and the result in the same order. */
|
|
40
|
+
export function judgeParticipantRecords(participants, wording = {}) {
|
|
41
|
+
const records = participants.map((participant) => {
|
|
42
|
+
const status = participant.inProgress
|
|
43
|
+
? "running"
|
|
44
|
+
: participant.skipped
|
|
45
|
+
? "blocked"
|
|
46
|
+
: (participant.status ??
|
|
47
|
+
(participant.sessionError === undefined ? "contract_proof_only" : "failed"));
|
|
48
|
+
const judged = participant.inProgress ? undefined : judgedStatus(participant);
|
|
49
|
+
return {
|
|
50
|
+
status,
|
|
51
|
+
judgedStatus: judged,
|
|
52
|
+
reason: participant.inProgress
|
|
53
|
+
? (wording.runningReason ??
|
|
54
|
+
"Live computer-use session is running; stream auth URL is available only through the attached Observer server.")
|
|
55
|
+
: (participant.skippedReason ??
|
|
56
|
+
participant.reason ??
|
|
57
|
+
participant.sessionError ??
|
|
58
|
+
wording.contractReason ??
|
|
59
|
+
"Dry run: the evidence shape was written without launching a desktop or spending provider tokens."),
|
|
60
|
+
notPassedMessage: participantNotPassedMessage(participant, wording),
|
|
61
|
+
gapLine: participant.inProgress || judged === "passed"
|
|
62
|
+
? undefined
|
|
63
|
+
: `${participant.id ?? "Participant"}: ${participant.skippedReason ?? participant.sessionError ?? participant.reason ?? "did not pass"}`,
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
const terminal = records.filter((_, index) => participants[index].status !== undefined && !participants[index].inProgress);
|
|
67
|
+
return {
|
|
68
|
+
participants: records,
|
|
69
|
+
tally: tallyParticipantOutcomes(terminal.map((participant) => participant.judgedStatus), participants
|
|
70
|
+
.filter((participant) => participant.status !== undefined && !participant.inProgress)
|
|
71
|
+
.map((participant) => participant.reportedFriction === true)),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function participantNotPassedMessage(participant, wording) {
|
|
75
|
+
const label = wording.sessionLabel ?? "Computer-use session";
|
|
76
|
+
if (participant.sessionError !== undefined)
|
|
77
|
+
return participant.sessionError;
|
|
78
|
+
if (participant.skippedReason !== undefined)
|
|
79
|
+
return participant.skippedReason;
|
|
80
|
+
if (participant.noEngagement)
|
|
81
|
+
return "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied.";
|
|
82
|
+
if (participant.selfReportedBlocker)
|
|
83
|
+
return "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass.";
|
|
84
|
+
if (participant.completionReason === "harness_error")
|
|
85
|
+
return `${label} ended with a harness error: ${participant.reason}`;
|
|
86
|
+
if (participant.status === undefined && wording.missingSessionMessage !== undefined)
|
|
87
|
+
return wording.missingSessionMessage;
|
|
88
|
+
if (participant.status !== "passed")
|
|
89
|
+
return `${label} ended with ${participant.status ?? "unknown"}: ${participant.reason ?? "no terminal reason"}`;
|
|
90
|
+
return (wording.missingSessionMessage ?? "The computer-use run did not produce a terminal session.");
|
|
91
|
+
}
|
|
38
92
|
/**
|
|
39
93
|
* The status a participant is judged under. A harness failure is failed, and a skipped participant
|
|
40
94
|
* is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { RunOutcome } from "./bundle.js";
|
|
2
|
+
import { type RunLiveness, type RunStatusRecord } from "./status.js";
|
|
3
|
+
interface LivenessBundle {
|
|
4
|
+
simulations?: ReadonlyArray<{
|
|
5
|
+
status?: string | undefined;
|
|
6
|
+
} | undefined> | undefined;
|
|
7
|
+
outcome?: RunOutcome | undefined;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
|
|
11
|
+
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
12
|
+
* is the status record the liveness came from, when there was one.
|
|
13
|
+
*/
|
|
14
|
+
export declare function runLiveness(runId: string, status: unknown, bundle: LivenessBundle, nowMs: number): {
|
|
15
|
+
liveness: RunLiveness;
|
|
16
|
+
record?: RunStatusRecord;
|
|
17
|
+
};
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { classifyRunStatus, isRunStatusRecord, } from "./status.js";
|
|
2
|
+
/**
|
|
3
|
+
* The bundle-only reading, for a run with no usable status record. A bundle on disk usually means
|
|
4
|
+
* the run reached its final write. But a live run now flushes an in-progress bundle as it goes (so
|
|
5
|
+
* anything asking what a participant is doing has something to read), and that bundle marks its
|
|
6
|
+
* simulations `running`. With no status record there is no freshness to judge, and the
|
|
7
|
+
* reading of "it started, and nothing here says it finished" is interrupted, not finished.
|
|
8
|
+
*/
|
|
9
|
+
function bundleLiveness(bundle) {
|
|
10
|
+
const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
|
|
11
|
+
return inProgress ? "interrupted" : "finished";
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
|
|
15
|
+
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
16
|
+
* is the status record the liveness came from, when there was one.
|
|
17
|
+
*/
|
|
18
|
+
export function runLiveness(runId, status, bundle, nowMs) {
|
|
19
|
+
if (bundle.outcome !== undefined)
|
|
20
|
+
return { liveness: bundle.outcome.state };
|
|
21
|
+
return isRunStatusRecord(status) && status.runId === runId
|
|
22
|
+
? { liveness: classifyRunStatus(status, nowMs), record: status }
|
|
23
|
+
: { liveness: bundleLiveness(bundle) };
|
|
24
|
+
}
|
package/dist/run/outcomes.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type ActorStatus } from "../actors/contract.js";
|
|
1
|
+
import { type ActorStatus, type ActorTrace } from "../actors/contract.js";
|
|
2
2
|
import { type CuaGoalSource } from "../actors/goal-source.js";
|
|
3
3
|
import type { TaskFunnel } from "../study/tasks.js";
|
|
4
4
|
import type { ParticipantOutcomes, ReviewSummary, RunTaskFunnel } from "./bundle.js";
|
|
@@ -16,6 +16,13 @@ type ParticipantOutcomeDetail = {
|
|
|
16
16
|
label?: string;
|
|
17
17
|
goalSource?: CuaGoalSource;
|
|
18
18
|
};
|
|
19
|
+
/**
|
|
20
|
+
* How one participant's session ended, in words a participant summary can end on. A computer-use
|
|
21
|
+
* goal says where the claim came from, as the run's participant line does.
|
|
22
|
+
*/
|
|
23
|
+
export declare function sessionEndingInWords(trace: ActorTrace): string;
|
|
24
|
+
/** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
|
|
25
|
+
export declare function gapsListClause(listed: number, total: number): string;
|
|
19
26
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
20
27
|
export declare function formatParticipantOutcomes(outcomes: ParticipantOutcomes, terminalCauses?: readonly ParticipantOutcomeDetail[]): string;
|
|
21
28
|
/** Presentation details tolerate optional legacy actor payloads without changing their tallies. */
|