humanish 0.111.3 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +49 -4
- package/dist/cli/findings.js +179 -23
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/route.js +1 -0
- package/dist/routes/scripted/setup.js +53 -10
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +7 -4
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/secrets.d.ts +15 -2
- package/dist/run/secrets.js +134 -9
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/requirements.js +2 -1
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +76 -23
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -4,50 +4,45 @@ import { TAIL_CHARS } from "./types.js";
|
|
|
4
4
|
/**
|
|
5
5
|
* Per-chunk sanitization cannot recognize a value split across deliveries. Redact those complete
|
|
6
6
|
* known values before persistence without collapsing events or changing stdout/stderr ordering.
|
|
7
|
-
*
|
|
7
|
+
* `spansOf` finds them in the joined text: RunSecrets.spans, which also finds a value in an
|
|
8
|
+
* encoded form or split by a terminal escape sequence. Work backwards through the spans so edits
|
|
9
|
+
* to later text leave earlier offsets valid.
|
|
8
10
|
*/
|
|
9
|
-
export function scrubSplitKnownValues(events,
|
|
11
|
+
export function scrubSplitKnownValues(events, spansOf, discardedPrefixes) {
|
|
10
12
|
for (const order of ["stdout", "stderr", "combined"]) {
|
|
11
13
|
const chunks = order === "combined" ? [...events] : events.filter((event) => event.stream === order);
|
|
12
14
|
// A virtual final chunk makes a key crossing the capture cap recognizable. Edits to retained
|
|
13
15
|
// events redact evidence; the raw overlap and this virtual chunk are never persisted.
|
|
14
16
|
if (discardedPrefixes[order])
|
|
15
17
|
chunks.push({ chunk: discardedPrefixes[order] });
|
|
16
|
-
|
|
17
|
-
|
|
18
|
+
let offset = 0;
|
|
19
|
+
const starts = chunks.map((event) => {
|
|
20
|
+
const start = offset;
|
|
21
|
+
offset += event.chunk.length;
|
|
22
|
+
return start;
|
|
23
|
+
});
|
|
24
|
+
const text = chunks.map((event) => event.chunk).join("");
|
|
25
|
+
for (const [at, end] of [...spansOf(text)].reverse()) {
|
|
26
|
+
let first = 0;
|
|
27
|
+
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
28
|
+
first += 1;
|
|
29
|
+
let last = first;
|
|
30
|
+
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < end)
|
|
31
|
+
last += 1;
|
|
32
|
+
const firstChunk = chunks[first];
|
|
33
|
+
const lastChunk = chunks[last];
|
|
34
|
+
if (!firstChunk || !lastChunk)
|
|
18
35
|
continue;
|
|
19
|
-
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
const matches = [];
|
|
27
|
-
for (let at = text.indexOf(value); at !== -1; at = text.indexOf(value, at + value.length))
|
|
28
|
-
matches.push(at);
|
|
29
|
-
for (const at of matches.reverse()) {
|
|
30
|
-
let first = 0;
|
|
31
|
-
while (first + 1 < starts.length && (starts[first + 1] ?? Infinity) <= at)
|
|
32
|
-
first += 1;
|
|
33
|
-
let last = first;
|
|
34
|
-
while (last + 1 < starts.length && (starts[last + 1] ?? Infinity) < at + value.length)
|
|
35
|
-
last += 1;
|
|
36
|
-
const firstChunk = chunks[first];
|
|
37
|
-
const lastChunk = chunks[last];
|
|
38
|
-
if (!firstChunk || !lastChunk)
|
|
39
|
-
continue;
|
|
40
|
-
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
41
|
-
const after = lastChunk.chunk.slice(at + value.length - (starts[last] ?? 0));
|
|
42
|
-
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
43
|
-
for (let index = first + 1; index < last; index += 1) {
|
|
44
|
-
const middle = chunks[index];
|
|
45
|
-
if (middle)
|
|
46
|
-
middle.chunk = "";
|
|
47
|
-
}
|
|
48
|
-
if (first !== last)
|
|
49
|
-
lastChunk.chunk = after;
|
|
36
|
+
const before = firstChunk.chunk.slice(0, at - (starts[first] ?? 0));
|
|
37
|
+
const after = lastChunk.chunk.slice(end - (starts[last] ?? 0));
|
|
38
|
+
firstChunk.chunk = `${before}[REDACTED_SECRET]${first === last ? after : ""}`;
|
|
39
|
+
for (let index = first + 1; index < last; index += 1) {
|
|
40
|
+
const middle = chunks[index];
|
|
41
|
+
if (middle)
|
|
42
|
+
middle.chunk = "";
|
|
50
43
|
}
|
|
44
|
+
if (first !== last)
|
|
45
|
+
lastChunk.chunk = after;
|
|
51
46
|
}
|
|
52
47
|
}
|
|
53
48
|
}
|
package/dist/run/costs.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { runEstimateUsd, sumEstimatedUsd } from "./run-cost.js";
|
|
2
2
|
import { bindExistingRunArtifactPaths } from "./paths.js";
|
|
3
3
|
import { contradictsAccountBilling } from "../verify/costs.js";
|
|
4
4
|
import { EVIDENCE_LIMITS } from "../analysis/evidence.js";
|
|
@@ -20,13 +20,9 @@ export function emptyCostTotals() {
|
|
|
20
20
|
analysisHistoryUncertainRuns: 0,
|
|
21
21
|
};
|
|
22
22
|
}
|
|
23
|
-
const isKnownUsd = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
24
|
-
function sumKnown(a, b) {
|
|
25
|
-
return a === null && b === null ? null : round6((a ?? 0) + (b ?? 0));
|
|
26
|
-
}
|
|
27
23
|
export function addCostTotals(into, next) {
|
|
28
24
|
for (const key of ["estimatedTotalUsd", "runEstimatedUsd", "analysisEstimatedUsd"]) {
|
|
29
|
-
into[key] =
|
|
25
|
+
into[key] = sumEstimatedUsd(into[key], next[key]);
|
|
30
26
|
}
|
|
31
27
|
for (const key of [
|
|
32
28
|
"incompleteRunEstimates",
|
|
@@ -49,11 +45,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
49
45
|
const analysisWarnings = [];
|
|
50
46
|
// A dry run makes no model request and creates no desktop, so with no figure it costs $0. A
|
|
51
47
|
// recorded unknown (null) figure stays unknown.
|
|
52
|
-
costs.runEstimatedUsd =
|
|
53
|
-
? entry.estimatedCostUsd
|
|
54
|
-
: entry.estimatedCostUsd === undefined && entry.mode === "dry-run"
|
|
55
|
-
? 0
|
|
56
|
-
: null;
|
|
48
|
+
costs.runEstimatedUsd = runEstimateUsd(entry);
|
|
57
49
|
try {
|
|
58
50
|
const prepared = await bindExistingRunArtifactPaths(cwd, entry.runId);
|
|
59
51
|
const bytes = await readBoundedFile(prepared, RUN_BUNDLE_FILE, EVIDENCE_LIMITS.sourceBytes);
|
|
@@ -70,9 +62,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
70
62
|
runWarnings.push("RUN_COST_ID_MISMATCH");
|
|
71
63
|
}
|
|
72
64
|
if (bundle?.cost !== undefined) {
|
|
73
|
-
costs.runEstimatedUsd =
|
|
74
|
-
? bundle.cost.estimatedTotalUsd
|
|
75
|
-
: null;
|
|
65
|
+
costs.runEstimatedUsd = runEstimateUsd({ estimatedCostUsd: bundle.cost?.estimatedTotalUsd });
|
|
76
66
|
if (bundle.cost?.fullyEstimated !== true || costs.runEstimatedUsd === null) {
|
|
77
67
|
costs.incompleteRunEstimates = 1;
|
|
78
68
|
runWarnings.push("RUN_COST_PARTIAL_OR_UNKNOWN");
|
|
@@ -102,7 +92,7 @@ export async function readCostTotals(cwd, entry) {
|
|
|
102
92
|
costs.analysisHistoryUncertainRuns = 1;
|
|
103
93
|
analysisWarnings.push("STUDY_COST_ACCOUNTING_UNAVAILABLE");
|
|
104
94
|
}
|
|
105
|
-
costs.estimatedTotalUsd =
|
|
95
|
+
costs.estimatedTotalUsd = sumEstimatedUsd(costs.runEstimatedUsd, costs.analysisEstimatedUsd);
|
|
106
96
|
return {
|
|
107
97
|
runId: entry.runId,
|
|
108
98
|
costs,
|
|
@@ -153,16 +143,15 @@ export async function readAnalysisAccounting(prepared) {
|
|
|
153
143
|
}
|
|
154
144
|
if (!usage.dispatched) {
|
|
155
145
|
accounting.notDispatched += 1;
|
|
156
|
-
accounting.estimatedUsd =
|
|
146
|
+
accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, 0);
|
|
157
147
|
continue;
|
|
158
148
|
}
|
|
159
149
|
accounting.dispatched += 1;
|
|
160
150
|
if (record.receipt)
|
|
161
151
|
providers.add(record.receipt.provider);
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
if (!isKnownUsd(usage.estimatedCostUsd) || !usage.usageComplete)
|
|
152
|
+
const estimatedUsd = runEstimateUsd({ estimatedCostUsd: usage.estimatedCostUsd });
|
|
153
|
+
accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, estimatedUsd);
|
|
154
|
+
if (estimatedUsd === null || !usage.usageComplete)
|
|
166
155
|
accounting.unpriced += 1;
|
|
167
156
|
}
|
|
168
157
|
// A skipped or queued automatic job says nothing about earlier manual requests. Only final
|
package/dist/run/detail.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ interface RunThought {
|
|
|
11
11
|
export interface RunParticipant {
|
|
12
12
|
/** Stream id: stable within a run, and what distinguishes participants of one study. */
|
|
13
13
|
id: string;
|
|
14
|
-
/** What to call them on screen: the
|
|
14
|
+
/** What to call them on screen: the caption the Observer shows, else the persona id. */
|
|
15
15
|
label: string;
|
|
16
16
|
/** The persona id, when the trace recorded one. */
|
|
17
17
|
personaId?: string;
|
package/dist/run/detail.js
CHANGED
|
@@ -12,6 +12,8 @@ import path from "node:path";
|
|
|
12
12
|
import { estimateActorCostForExecution } from "./pricing.js";
|
|
13
13
|
import { resolveRunPath } from "./locate.js";
|
|
14
14
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
15
|
+
import { savedCaption } from "./participant-caption.js";
|
|
16
|
+
import { streamParticipantIdOf } from "./participant-records.js";
|
|
15
17
|
import { readContainedRegularFile } from "./contained-output.js";
|
|
16
18
|
import { isPathInside, resolvePhysicalCwd } from "./paths.js";
|
|
17
19
|
const RUN_DETAIL_SCHEMA = "humanish.run-detail.v1";
|
|
@@ -47,13 +49,20 @@ function participantFrom(stream, index) {
|
|
|
47
49
|
const trace = stream.liveActor ?? stream.actor ?? {};
|
|
48
50
|
const id = stream.id ?? `stream-${index + 1}`;
|
|
49
51
|
const personaId = trace.persona?.id;
|
|
52
|
+
const participantId = streamParticipantIdOf(stream);
|
|
50
53
|
// Computed once: calling it twice to test-then-use reads as though the two could differ.
|
|
51
54
|
const thought = latestThought(trace);
|
|
52
55
|
const status = trace.status ?? stream.status;
|
|
53
56
|
const thoughts = (trace.items ?? []).filter((item) => item?.kind === "reasoning").length;
|
|
54
57
|
return {
|
|
55
58
|
id,
|
|
56
|
-
label: stream.label
|
|
59
|
+
label: stream.label === undefined
|
|
60
|
+
? (personaId ?? id)
|
|
61
|
+
: savedCaption({
|
|
62
|
+
label: stream.label,
|
|
63
|
+
...(participantId === undefined ? {} : { participantId }),
|
|
64
|
+
...(personaId === undefined ? {} : { personaId }),
|
|
65
|
+
}),
|
|
57
66
|
...(personaId === undefined ? {} : { personaId }),
|
|
58
67
|
traits: Array.isArray(trace.persona?.traitsApplied) ? trace.persona.traitsApplied : [],
|
|
59
68
|
...(status === undefined ? {} : { status }),
|
package/dist/run/display.d.ts
CHANGED
|
@@ -49,14 +49,14 @@ export interface DisplayedBundle {
|
|
|
49
49
|
* The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
|
|
50
50
|
* without one is in progress, or was recorded before the field existed: its status record, when
|
|
51
51
|
* given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
|
|
52
|
-
* With no record, a bundle whose simulations are still running is
|
|
52
|
+
* With no record, a bundle whose simulations are still running is interrupted.
|
|
53
53
|
*/
|
|
54
54
|
export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unknown, nowMs?: number): RunDisplayFacts;
|
|
55
55
|
/**
|
|
56
56
|
* How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
|
|
57
57
|
* execution failed never reads as a pass beside its participants' verdict.
|
|
58
58
|
*/
|
|
59
|
-
export declare function reviewOutcome(bundle: DisplayedBundle): string;
|
|
59
|
+
export declare function reviewOutcome(bundle: DisplayedBundle, status?: unknown): string;
|
|
60
60
|
/**
|
|
61
61
|
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
|
62
62
|
* was still running keeps the review its last live flush wrote, which describes that session as
|
package/dist/run/display.js
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// library, `humanish runs`, `humanish stats` and review.md. Each of them calls runDisplay with the
|
|
3
3
|
// same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
|
|
4
4
|
// cannot disagree about one run.
|
|
5
|
-
import {
|
|
5
|
+
import {} from "./status.js";
|
|
6
|
+
import { runLiveness } from "./liveness.js";
|
|
6
7
|
import { plural } from "./text.js";
|
|
7
8
|
const LABELS = {
|
|
8
9
|
running: "running",
|
|
@@ -72,38 +73,34 @@ function runDisplayLine(display) {
|
|
|
72
73
|
* The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
|
|
73
74
|
* without one is in progress, or was recorded before the field existed: its status record, when
|
|
74
75
|
* given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
|
|
75
|
-
* With no record, a bundle whose simulations are still running is
|
|
76
|
+
* With no record, a bundle whose simulations are still running is interrupted.
|
|
76
77
|
*/
|
|
77
78
|
export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
|
|
78
|
-
const
|
|
79
|
+
const { liveness, record } = runLiveness(bundle.runId, status, bundle, nowMs);
|
|
80
|
+
const base = { liveness, mode: bundle.mode, verdict: bundle.review?.verdict };
|
|
79
81
|
const outcome = bundle.outcome;
|
|
80
82
|
if (outcome?.state === "interrupted")
|
|
81
|
-
return
|
|
83
|
+
return base;
|
|
82
84
|
if (outcome?.state === "finished")
|
|
83
85
|
return {
|
|
84
86
|
...base,
|
|
85
|
-
liveness: "finished",
|
|
86
87
|
ok: outcome.ok,
|
|
87
88
|
failure: outcome.execution.failures[0],
|
|
88
89
|
};
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
: { failure: status.outcome.execution.failures[0] }),
|
|
97
|
-
};
|
|
98
|
-
const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
|
|
99
|
-
return { ...base, liveness: inProgress ? "running" : "finished" };
|
|
90
|
+
return {
|
|
91
|
+
...base,
|
|
92
|
+
...(typeof record?.outcome?.ok === "boolean" ? { ok: record.outcome.ok } : {}),
|
|
93
|
+
...(record?.outcome?.execution?.failures[0] === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { failure: record.outcome.execution.failures[0] }),
|
|
96
|
+
};
|
|
100
97
|
}
|
|
101
98
|
/**
|
|
102
99
|
* How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
|
|
103
100
|
* execution failed never reads as a pass beside its participants' verdict.
|
|
104
101
|
*/
|
|
105
|
-
export function reviewOutcome(bundle) {
|
|
106
|
-
return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
|
|
102
|
+
export function reviewOutcome(bundle, status) {
|
|
103
|
+
return runDisplayLine(runDisplay(bundleDisplayFacts(bundle, status)));
|
|
107
104
|
}
|
|
108
105
|
/**
|
|
109
106
|
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
package/dist/run/dry-run.js
CHANGED
|
@@ -9,8 +9,8 @@ import {} from "./streams.js";
|
|
|
9
9
|
import { participantEvent, participantIdsOf, participantRecord, participantStream, } from "./participant-records.js";
|
|
10
10
|
import { implicitProjectDirectoryExists, readPackageName, validateCwd } from "./project.js";
|
|
11
11
|
import { loadDryRunInputs } from "./dry-run-inputs.js";
|
|
12
|
-
import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk
|
|
13
|
-
import {
|
|
12
|
+
import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk } from "./judge.js";
|
|
13
|
+
import { renderReviewMarkdown } from "./review-markdown.js";
|
|
14
14
|
import { cli } from "../cli/invocation.js";
|
|
15
15
|
/**
|
|
16
16
|
* The preview route's run. The run scope closes the run it started on every exit, including
|
|
@@ -68,7 +68,7 @@ async function runDryRunInScope(options, scope) {
|
|
|
68
68
|
mode: options.dryRun ? "dry-run" : "live",
|
|
69
69
|
study: options.study,
|
|
70
70
|
warnings: options.warnings,
|
|
71
|
-
renderReview:
|
|
71
|
+
renderReview: renderPreviewReview,
|
|
72
72
|
// The run starts at the time its id and source were stamped with.
|
|
73
73
|
now: () => Date.parse(createdAt),
|
|
74
74
|
...(options.observer === undefined ? {} : { observer: { open: options.observer.open } }),
|
|
@@ -170,28 +170,13 @@ function createReviewSummary(verdict) {
|
|
|
170
170
|
};
|
|
171
171
|
}
|
|
172
172
|
/** The preview's review.md. */
|
|
173
|
-
function
|
|
174
|
-
return
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
Verdict: ${verdictText(bundle.review.verdict, bundle.mode)}
|
|
181
|
-
|
|
182
|
-
Outcome: ${reviewOutcome(bundle)}
|
|
183
|
-
|
|
184
|
-
${bundle.review.summary}
|
|
185
|
-
|
|
186
|
-
## Public-Safety
|
|
187
|
-
|
|
188
|
-
- Redaction: ${bundle.redaction.status}
|
|
189
|
-
- Notes: ${bundle.redaction.notes}
|
|
190
|
-
|
|
191
|
-
## Gaps
|
|
192
|
-
|
|
193
|
-
${bundle.review.gaps.map((gap) => `- ${gap}`).join("\n")}
|
|
194
|
-
`;
|
|
173
|
+
function renderPreviewReview(bundle, status) {
|
|
174
|
+
return renderReviewMarkdown(bundle, [
|
|
175
|
+
"## Public-Safety",
|
|
176
|
+
"",
|
|
177
|
+
`- Redaction: ${bundle.redaction.status}`,
|
|
178
|
+
`- Notes: ${bundle.redaction.notes}`,
|
|
179
|
+
], { style: "preview", status });
|
|
195
180
|
}
|
|
196
181
|
/**
|
|
197
182
|
* Render through the finished run, so a directory swapped in under the same id is never rendered.
|
package/dist/run/judge.d.ts
CHANGED
|
@@ -41,6 +41,30 @@ export interface ParticipantFacts {
|
|
|
41
41
|
/** selfReportedBlocker held for its session. */
|
|
42
42
|
selfReportedBlocker: boolean;
|
|
43
43
|
}
|
|
44
|
+
/** The participant facts its bundle and result can name. */
|
|
45
|
+
export interface ParticipantRecordFacts extends ParticipantFacts {
|
|
46
|
+
id?: string;
|
|
47
|
+
reason?: string;
|
|
48
|
+
skippedReason?: string;
|
|
49
|
+
reportedFriction?: boolean;
|
|
50
|
+
inProgress?: boolean;
|
|
51
|
+
}
|
|
52
|
+
/** Judge participant evidence for the bundle and the result in the same order. */
|
|
53
|
+
export declare function judgeParticipantRecords(participants: readonly ParticipantRecordFacts[], wording?: {
|
|
54
|
+
sessionLabel?: string;
|
|
55
|
+
missingSessionMessage?: string;
|
|
56
|
+
runningReason?: string;
|
|
57
|
+
contractReason?: string;
|
|
58
|
+
}): {
|
|
59
|
+
participants: {
|
|
60
|
+
status: "abandoned" | "blocked" | "contract_proof_only" | "failed" | "incomplete" | "passed" | "running" | "timed_out";
|
|
61
|
+
judgedStatus: "abandoned" | "blocked" | "failed" | "incomplete" | "passed" | "timed_out" | undefined;
|
|
62
|
+
reason: string;
|
|
63
|
+
notPassedMessage: string;
|
|
64
|
+
gapLine: string | undefined;
|
|
65
|
+
}[];
|
|
66
|
+
tally: import("./bundle.js").ParticipantOutcomes;
|
|
67
|
+
};
|
|
44
68
|
/**
|
|
45
69
|
* The status a participant is judged under. A harness failure is failed, and a skipped participant
|
|
46
70
|
* is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
|
package/dist/run/judge.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// 4. foldScorerFailures, which every route with a scorer applies before the run finishes;
|
|
13
13
|
// 5. the outcome: judgeExecution applies the route's policy to its execution failures, and
|
|
14
14
|
// resultOk gives the result's ok.
|
|
15
|
+
import { tallyParticipantOutcomes } from "./outcomes.js";
|
|
15
16
|
import { cli } from "../cli/invocation.js";
|
|
16
17
|
/**
|
|
17
18
|
* The participant said it reached the goal having taken no action and said nothing, and no stop
|
|
@@ -35,6 +36,59 @@ export function selfReportedBlocker(ending) {
|
|
|
35
36
|
ending.closingReportReadsBlocked &&
|
|
36
37
|
!ending.stopConditionMatched);
|
|
37
38
|
}
|
|
39
|
+
/** Judge participant evidence for the bundle and the result in the same order. */
|
|
40
|
+
export function judgeParticipantRecords(participants, wording = {}) {
|
|
41
|
+
const records = participants.map((participant) => {
|
|
42
|
+
const status = participant.inProgress
|
|
43
|
+
? "running"
|
|
44
|
+
: participant.skipped
|
|
45
|
+
? "blocked"
|
|
46
|
+
: (participant.status ??
|
|
47
|
+
(participant.sessionError === undefined ? "contract_proof_only" : "failed"));
|
|
48
|
+
const judged = participant.inProgress ? undefined : judgedStatus(participant);
|
|
49
|
+
return {
|
|
50
|
+
status,
|
|
51
|
+
judgedStatus: judged,
|
|
52
|
+
reason: participant.inProgress
|
|
53
|
+
? (wording.runningReason ??
|
|
54
|
+
"Live computer-use session is running; stream auth URL is available only through the attached Observer server.")
|
|
55
|
+
: (participant.skippedReason ??
|
|
56
|
+
participant.reason ??
|
|
57
|
+
participant.sessionError ??
|
|
58
|
+
wording.contractReason ??
|
|
59
|
+
"Dry run: the evidence shape was written without launching a desktop or spending provider tokens."),
|
|
60
|
+
notPassedMessage: participantNotPassedMessage(participant, wording),
|
|
61
|
+
gapLine: participant.inProgress || judged === "passed"
|
|
62
|
+
? undefined
|
|
63
|
+
: `${participant.id ?? "Participant"}: ${participant.skippedReason ?? participant.sessionError ?? participant.reason ?? "did not pass"}`,
|
|
64
|
+
};
|
|
65
|
+
});
|
|
66
|
+
const terminal = records.filter((_, index) => participants[index].status !== undefined && !participants[index].inProgress);
|
|
67
|
+
return {
|
|
68
|
+
participants: records,
|
|
69
|
+
tally: tallyParticipantOutcomes(terminal.map((participant) => participant.judgedStatus), participants
|
|
70
|
+
.filter((participant) => participant.status !== undefined && !participant.inProgress)
|
|
71
|
+
.map((participant) => participant.reportedFriction === true)),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
function participantNotPassedMessage(participant, wording) {
|
|
75
|
+
const label = wording.sessionLabel ?? "Computer-use session";
|
|
76
|
+
if (participant.sessionError !== undefined)
|
|
77
|
+
return participant.sessionError;
|
|
78
|
+
if (participant.skippedReason !== undefined)
|
|
79
|
+
return participant.skippedReason;
|
|
80
|
+
if (participant.noEngagement)
|
|
81
|
+
return "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied.";
|
|
82
|
+
if (participant.selfReportedBlocker)
|
|
83
|
+
return "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass.";
|
|
84
|
+
if (participant.completionReason === "harness_error")
|
|
85
|
+
return `${label} ended with a harness error: ${participant.reason}`;
|
|
86
|
+
if (participant.status === undefined && wording.missingSessionMessage !== undefined)
|
|
87
|
+
return wording.missingSessionMessage;
|
|
88
|
+
if (participant.status !== "passed")
|
|
89
|
+
return `${label} ended with ${participant.status ?? "unknown"}: ${participant.reason ?? "no terminal reason"}`;
|
|
90
|
+
return (wording.missingSessionMessage ?? "The computer-use run did not produce a terminal session.");
|
|
91
|
+
}
|
|
38
92
|
/**
|
|
39
93
|
* The status a participant is judged under. A harness failure is failed, and a skipped participant
|
|
40
94
|
* is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { RunOutcome } from "./bundle.js";
|
|
2
|
+
import { type RunLiveness, type RunStatusRecord } from "./status.js";
|
|
3
|
+
interface LivenessBundle {
|
|
4
|
+
simulations?: ReadonlyArray<{
|
|
5
|
+
status?: string | undefined;
|
|
6
|
+
} | undefined> | undefined;
|
|
7
|
+
outcome?: RunOutcome | undefined;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
|
|
11
|
+
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
12
|
+
* is the status record the liveness came from, when there was one.
|
|
13
|
+
*/
|
|
14
|
+
export declare function runLiveness(runId: string, status: unknown, bundle: LivenessBundle, nowMs: number): {
|
|
15
|
+
liveness: RunLiveness;
|
|
16
|
+
record?: RunStatusRecord;
|
|
17
|
+
};
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { classifyRunStatus, isRunStatusRecord, } from "./status.js";
|
|
2
|
+
/**
|
|
3
|
+
* The bundle-only reading, for a run with no usable status record. A bundle on disk usually means
|
|
4
|
+
* the run reached its final write. But a live run now flushes an in-progress bundle as it goes (so
|
|
5
|
+
* anything asking what a participant is doing has something to read), and that bundle marks its
|
|
6
|
+
* simulations `running`. With no status record there is no freshness to judge, and the
|
|
7
|
+
* reading of "it started, and nothing here says it finished" is interrupted, not finished.
|
|
8
|
+
*/
|
|
9
|
+
function bundleLiveness(bundle) {
|
|
10
|
+
const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
|
|
11
|
+
return inProgress ? "interrupted" : "finished";
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
|
|
15
|
+
* `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
|
|
16
|
+
* is the status record the liveness came from, when there was one.
|
|
17
|
+
*/
|
|
18
|
+
export function runLiveness(runId, status, bundle, nowMs) {
|
|
19
|
+
if (bundle.outcome !== undefined)
|
|
20
|
+
return { liveness: bundle.outcome.state };
|
|
21
|
+
return isRunStatusRecord(status) && status.runId === runId
|
|
22
|
+
? { liveness: classifyRunStatus(status, nowMs), record: status }
|
|
23
|
+
: { liveness: bundleLiveness(bundle) };
|
|
24
|
+
}
|
package/dist/run/outcomes.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type ActorStatus } from "../actors/contract.js";
|
|
1
|
+
import { type ActorStatus, type ActorTrace } from "../actors/contract.js";
|
|
2
2
|
import { type CuaGoalSource } from "../actors/goal-source.js";
|
|
3
3
|
import type { TaskFunnel } from "../study/tasks.js";
|
|
4
4
|
import type { ParticipantOutcomes, ReviewSummary, RunTaskFunnel } from "./bundle.js";
|
|
@@ -16,6 +16,13 @@ type ParticipantOutcomeDetail = {
|
|
|
16
16
|
label?: string;
|
|
17
17
|
goalSource?: CuaGoalSource;
|
|
18
18
|
};
|
|
19
|
+
/**
|
|
20
|
+
* How one participant's session ended, in words a participant summary can end on. A computer-use
|
|
21
|
+
* goal says where the claim came from, as the run's participant line does.
|
|
22
|
+
*/
|
|
23
|
+
export declare function sessionEndingInWords(trace: ActorTrace): string;
|
|
24
|
+
/** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
|
|
25
|
+
export declare function gapsListClause(listed: number, total: number): string;
|
|
19
26
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
20
27
|
export declare function formatParticipantOutcomes(outcomes: ParticipantOutcomes, terminalCauses?: readonly ParticipantOutcomeDetail[]): string;
|
|
21
28
|
/** Presentation details tolerate optional legacy actor payloads without changing their tallies. */
|
package/dist/run/outcomes.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { ACTOR_STATUSES } from "../actors/contract.js";
|
|
2
2
|
import { cuaGoalSource, isCuaTrace, CUA_COMPLETION_NOTE, } from "../actors/goal-source.js";
|
|
3
3
|
import { actorEnding } from "../actors/stop-cause.js";
|
|
4
|
+
import { plural } from "./text.js";
|
|
4
5
|
import { isNonNegativeSafeInteger, isRecord } from "./type-guards.js";
|
|
5
6
|
/** Tally participant outcomes from actor statuses. Statuses this does not recognise are counted in
|
|
6
7
|
* `total` but nowhere else, so the parts can never exceed the whole. */
|
|
@@ -113,6 +114,46 @@ function participantCompletionLine(outcomes, terminalCauses) {
|
|
|
113
114
|
}
|
|
114
115
|
return goalLine;
|
|
115
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* How one participant's session ended, in words a participant summary can end on. A computer-use
|
|
119
|
+
* goal says where the claim came from, as the run's participant line does.
|
|
120
|
+
*/
|
|
121
|
+
export function sessionEndingInWords(trace) {
|
|
122
|
+
switch (trace.completionReason) {
|
|
123
|
+
case "goal_satisfied": {
|
|
124
|
+
const source = cuaGoalSource(trace);
|
|
125
|
+
if (source === "participant_report")
|
|
126
|
+
return "reported reaching the goal";
|
|
127
|
+
return source === "condition_matched" ? "met its completion condition" : "reached the goal";
|
|
128
|
+
}
|
|
129
|
+
case "turn_completed":
|
|
130
|
+
return "said it was done";
|
|
131
|
+
case "gave_up":
|
|
132
|
+
return "gave up";
|
|
133
|
+
case "blocked_approval":
|
|
134
|
+
return "was blocked waiting for an approval";
|
|
135
|
+
case "timed_out":
|
|
136
|
+
return "ran out of time";
|
|
137
|
+
case "budget_reached":
|
|
138
|
+
return "was stopped by a time or spend limit";
|
|
139
|
+
case "actor_error":
|
|
140
|
+
return "stopped on an error";
|
|
141
|
+
case "step_failed":
|
|
142
|
+
return "failed a scripted step";
|
|
143
|
+
case "harness_error":
|
|
144
|
+
return "was lost to a harness failure";
|
|
145
|
+
default:
|
|
146
|
+
return "ended";
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
|
|
150
|
+
export function gapsListClause(listed, total) {
|
|
151
|
+
if (listed <= 0)
|
|
152
|
+
return "";
|
|
153
|
+
if (listed < total)
|
|
154
|
+
return `; the gaps list the other ${listed}`;
|
|
155
|
+
return total === 1 ? "; the gaps say why" : `; the gaps list all ${total}`;
|
|
156
|
+
}
|
|
116
157
|
/** One line a stakeholder can read, with the denominator attached to every number. */
|
|
117
158
|
export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
|
|
118
159
|
if (outcomes.total === 0)
|
|
@@ -189,20 +230,25 @@ export function withCuaReviewProvenance(review, streams) {
|
|
|
189
230
|
: !details.some((entry) => entry.goalSource !== undefined)))
|
|
190
231
|
return review;
|
|
191
232
|
const outcomes = formatParticipantOutcomes(review.participants, details);
|
|
192
|
-
|
|
193
|
-
//
|
|
194
|
-
|
|
195
|
-
const
|
|
196
|
-
const
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
const
|
|
200
|
-
const
|
|
201
|
-
|
|
202
|
-
.
|
|
233
|
+
const tasks = review.tasks ? ` Tasks: ${formatRunTaskFunnel(review.tasks)}.` : "";
|
|
234
|
+
// A summary that states the bare count gets it qualified where it stands, keeping rerun context,
|
|
235
|
+
// the participant's own words and adapter findings around it.
|
|
236
|
+
const { reachedGoal, total } = review.participants;
|
|
237
|
+
const bareLine = `${reachedGoal}/${total} reached the goal`;
|
|
238
|
+
const qualifiedLine = participantCompletionLine(review.participants, details);
|
|
239
|
+
// A summary with no count gets it in a header. Earlier releases wrote a header in other words.
|
|
240
|
+
const header = `${plural(total, "participant")} took part: ${outcomes}.${tasks} `;
|
|
241
|
+
const olderHeader = `Run gate: ${review.verdict}. Participants: ${outcomes}.${tasks} Recorded summary: `;
|
|
242
|
+
const summary = review.summary.startsWith(olderHeader)
|
|
243
|
+
? `${header}${review.summary.slice(olderHeader.length)}`
|
|
244
|
+
: review.summary.includes(bareLine)
|
|
245
|
+
? review.summary.split(bareLine).join(qualifiedLine)
|
|
246
|
+
: review.summary.includes(qualifiedLine)
|
|
247
|
+
? review.summary
|
|
248
|
+
: `${header}${review.summary}`;
|
|
203
249
|
return {
|
|
204
250
|
...review,
|
|
205
|
-
summary
|
|
251
|
+
summary,
|
|
206
252
|
gaps: review.participants.reachedGoal === 0
|
|
207
253
|
? review.gaps
|
|
208
254
|
: [...review.gaps.filter((gap) => gap !== CUA_COMPLETION_NOTE), CUA_COMPLETION_NOTE],
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/** A participant, as much of it as a caption reads. */
|
|
2
|
+
export interface CaptionFacts {
|
|
3
|
+
/** The roster id, on a route that has a roster. */
|
|
4
|
+
readonly id?: string;
|
|
5
|
+
/** The persona, which names the participant when there is no roster id or it is only a number. */
|
|
6
|
+
readonly personaId: string;
|
|
7
|
+
/** The participant's device, when it has one. */
|
|
8
|
+
readonly device?: {
|
|
9
|
+
readonly name: string;
|
|
10
|
+
readonly preset: {
|
|
11
|
+
readonly isMobile: boolean;
|
|
12
|
+
};
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
/** "Lobby host", or "Player two, phone" for a participant on a phone. */
|
|
16
|
+
export declare function participantCaption(participant: CaptionFacts): string;
|
|
17
|
+
/** The fields of a saved stream that name its participant. */
|
|
18
|
+
export interface SavedCaptionFacts {
|
|
19
|
+
readonly label: string;
|
|
20
|
+
/** The participant id a computer-use stream records. */
|
|
21
|
+
readonly participantId?: string;
|
|
22
|
+
readonly personaId?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The caption a saved stream shows: its label, which a route writes with `participantCaption`.
|
|
26
|
+
* Earlier releases wrote ids and taxonomy into the label (`lane-01 · browser`, `CUA participant
|
|
27
|
+
* <id>: <study>`, `Concurrent persona <id> (type:<t> / ...) · <study>`), so a stream saved that way
|
|
28
|
+
* is captioned again from the id it names, without a device.
|
|
29
|
+
*/
|
|
30
|
+
export declare function savedCaption(stream: SavedCaptionFacts): string;
|