humanish 0.111.3 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +49 -4
- package/dist/cli/findings.js +179 -23
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/route.js +1 -0
- package/dist/routes/scripted/setup.js +53 -10
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +7 -4
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/secrets.d.ts +15 -2
- package/dist/run/secrets.js +134 -9
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/requirements.js +2 -1
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +76 -23
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
package/dist/cli/findings.d.ts
CHANGED
|
@@ -4,14 +4,13 @@ import type { AnalysisArtifact, LoadedAnalysis } from "../analysis/types.js";
|
|
|
4
4
|
import { type CliIo } from "./io.js";
|
|
5
5
|
export declare const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
6
6
|
type Finding = NonNullable<AnalysisArtifact["result"]>["findings"][number];
|
|
7
|
+
type DesignFinding = NonNullable<NonNullable<AnalysisArtifact["result"]>["designFindings"]>[number];
|
|
7
8
|
/** `ready` and `stale` carry findings; every other state carries none and says why in `message`. */
|
|
8
9
|
type FindingsState = "ready" | "stale" | "running" | "none" | "skipped" | "failed" | "dry_run" | "unavailable";
|
|
9
|
-
interface
|
|
10
|
+
interface CitedEvidence {
|
|
10
11
|
id: string;
|
|
11
12
|
streamId: string;
|
|
12
13
|
kind: string;
|
|
13
|
-
/** How the finding's observations used this item: visual, action, participant_statement, inference. */
|
|
14
|
-
bases: string[];
|
|
15
14
|
/** Index of the latest retained capture at this item, from 0; null before any capture. */
|
|
16
15
|
frame: number | null;
|
|
17
16
|
/** Time since the first retained capture, not a video offset. */
|
|
@@ -20,8 +19,19 @@ interface FindingEvidence {
|
|
|
20
19
|
/** The capture file relative to the project directory, or null for nonvisual evidence. */
|
|
21
20
|
capture: string | null;
|
|
22
21
|
}
|
|
22
|
+
interface FindingEvidence extends CitedEvidence {
|
|
23
|
+
/** How the finding's observations used this item: visual, action, participant_statement, inference. */
|
|
24
|
+
bases: string[];
|
|
25
|
+
}
|
|
23
26
|
interface FindingView {
|
|
24
27
|
id: string;
|
|
28
|
+
/**
|
|
29
|
+
* One plain sentence about what happened, and one to three about what the person tried and what
|
|
30
|
+
* got in the way. Null in analyses written before them. An amended finding keeps the original
|
|
31
|
+
* text here; `correction.replacementClaim` replaces both.
|
|
32
|
+
*/
|
|
33
|
+
headline: string | null;
|
|
34
|
+
experience: string | null;
|
|
25
35
|
title: string;
|
|
26
36
|
summary: string;
|
|
27
37
|
impact: Finding["impact"];
|
|
@@ -41,6 +51,22 @@ interface FindingView {
|
|
|
41
51
|
replacementClaim: string | null;
|
|
42
52
|
} | null;
|
|
43
53
|
}
|
|
54
|
+
interface DesignFindingView {
|
|
55
|
+
id: string;
|
|
56
|
+
headline: string;
|
|
57
|
+
screen: string;
|
|
58
|
+
notice: string;
|
|
59
|
+
whyItMatters: string;
|
|
60
|
+
suggestion: string;
|
|
61
|
+
severity: DesignFinding["severity"];
|
|
62
|
+
confidence: DesignFinding["confidence"];
|
|
63
|
+
seenBy: {
|
|
64
|
+
streamId: string;
|
|
65
|
+
label: string;
|
|
66
|
+
}[];
|
|
67
|
+
/** Each cited item once; at least one has a capture. */
|
|
68
|
+
evidence: CitedEvidence[];
|
|
69
|
+
}
|
|
44
70
|
export interface AnalysisFindings {
|
|
45
71
|
schema: typeof ANALYSIS_FINDINGS_SCHEMA;
|
|
46
72
|
runId: string;
|
|
@@ -48,7 +74,10 @@ export interface AnalysisFindings {
|
|
|
48
74
|
/** The stable code behind a state other than ready, when one is recorded. */
|
|
49
75
|
reason: string | null;
|
|
50
76
|
message: string;
|
|
51
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* The command that gets findings for this run, or for a dry run the command that starts a live
|
|
79
|
+
* one. Null when none can.
|
|
80
|
+
*/
|
|
52
81
|
next: string | null;
|
|
53
82
|
analysisId: string | null;
|
|
54
83
|
status: AnalysisArtifact["status"] | null;
|
|
@@ -63,6 +92,11 @@ export interface AnalysisFindings {
|
|
|
63
92
|
summary: string | null;
|
|
64
93
|
/** Highest priority first, as the analysis ranked them. */
|
|
65
94
|
findings: FindingView[];
|
|
95
|
+
/**
|
|
96
|
+
* Problems a designer would see in the captures, most severe first. Null when the view carries
|
|
97
|
+
* no analysis result, or the analysis was written before design findings.
|
|
98
|
+
*/
|
|
99
|
+
designFindings: DesignFindingView[] | null;
|
|
66
100
|
limitations: string[];
|
|
67
101
|
warnings: string[];
|
|
68
102
|
}
|
|
@@ -80,6 +114,17 @@ interface FindingsSource {
|
|
|
80
114
|
* false`, read when the view is built. Undefined when it does not, or cannot be read.
|
|
81
115
|
*/
|
|
82
116
|
analysisOffIn?: string;
|
|
117
|
+
/** For a dry run, the study a live run can start from, read when the view is built. */
|
|
118
|
+
liveStudy?: LiveStudy;
|
|
119
|
+
}
|
|
120
|
+
/** A study `run` can start live after a dry run. */
|
|
121
|
+
interface LiveStudy {
|
|
122
|
+
/** The study argument for `run`: its id, or its path when the run named it by path. */
|
|
123
|
+
run: string;
|
|
124
|
+
/** Its id, for the message. */
|
|
125
|
+
id: string;
|
|
126
|
+
/** The study file to set `mode: live` in first, when the file still starts as a dry run. */
|
|
127
|
+
setLiveIn?: string;
|
|
83
128
|
}
|
|
84
129
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
85
130
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
package/dist/cli/findings.js
CHANGED
|
@@ -9,6 +9,7 @@ import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
|
9
9
|
import { resolvePhysicalCwd, runIdOf } from "../run/paths.js";
|
|
10
10
|
import { studyProvenanceOf } from "../run/study-provenance.js";
|
|
11
11
|
import { resolveStudyManifest } from "../study/discover.js";
|
|
12
|
+
import { routeOf } from "../study/routing.js";
|
|
12
13
|
import { plural } from "../run/text.js";
|
|
13
14
|
import { shellQuote } from "../substrates/shell.js";
|
|
14
15
|
import { analysisOutcomeText, wantsJson } from "./io.js";
|
|
@@ -26,16 +27,10 @@ function analyzeCommand(source, provider) {
|
|
|
26
27
|
? `${run} --provider codex`
|
|
27
28
|
: `${run} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
28
29
|
}
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
for (const observation of finding.observations)
|
|
32
|
-
for (const id of observation.evidenceIds) {
|
|
33
|
-
const set = bases.get(id) ?? new Set();
|
|
34
|
-
set.add(observation.basis);
|
|
35
|
-
bases.set(id, set);
|
|
36
|
-
}
|
|
30
|
+
/** The cited items the analysis evidence resolves, each once, in citation order. */
|
|
31
|
+
function citedEvidence(ids, analysis, runRoot) {
|
|
37
32
|
const byId = new Map(analysis.evidence.map((entry) => [entry.id, entry]));
|
|
38
|
-
return [...
|
|
33
|
+
return [...new Set(ids)].flatMap((id) => {
|
|
39
34
|
const entry = byId.get(id);
|
|
40
35
|
if (!entry)
|
|
41
36
|
return [];
|
|
@@ -44,7 +39,6 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
44
39
|
id,
|
|
45
40
|
streamId: entry.streamId,
|
|
46
41
|
kind: entry.kind,
|
|
47
|
-
bases: [...used],
|
|
48
42
|
frame: entry.frame,
|
|
49
43
|
elapsedMs: entry.elapsedMs,
|
|
50
44
|
at: entry.at,
|
|
@@ -53,11 +47,58 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
53
47
|
];
|
|
54
48
|
});
|
|
55
49
|
}
|
|
50
|
+
function evidenceOf(finding, analysis, runRoot) {
|
|
51
|
+
const bases = new Map();
|
|
52
|
+
for (const observation of finding.observations)
|
|
53
|
+
for (const id of observation.evidenceIds) {
|
|
54
|
+
const set = bases.get(id) ?? new Set();
|
|
55
|
+
set.add(observation.basis);
|
|
56
|
+
bases.set(id, set);
|
|
57
|
+
}
|
|
58
|
+
return citedEvidence(bases.keys(), analysis, runRoot).map(({ id, streamId, kind, ...entry }) => ({
|
|
59
|
+
id,
|
|
60
|
+
streamId,
|
|
61
|
+
kind,
|
|
62
|
+
bases: [...bases.get(id)],
|
|
63
|
+
...entry,
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
const SEVERITY_RANK = {
|
|
67
|
+
major: 0,
|
|
68
|
+
moderate: 1,
|
|
69
|
+
minor: 2,
|
|
70
|
+
};
|
|
71
|
+
/** Most severe first; equal severity keeps the analysis order. */
|
|
72
|
+
function designFindingViews(analysis, runRoot) {
|
|
73
|
+
const designFindings = analysis.result?.designFindings;
|
|
74
|
+
if (designFindings === undefined)
|
|
75
|
+
return null;
|
|
76
|
+
const labels = new Map(analysis.participants.map((p) => [p.streamId, p.label]));
|
|
77
|
+
return [...designFindings]
|
|
78
|
+
.sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity])
|
|
79
|
+
.map((finding) => ({
|
|
80
|
+
id: finding.id,
|
|
81
|
+
headline: finding.headline,
|
|
82
|
+
screen: finding.screen,
|
|
83
|
+
notice: finding.notice,
|
|
84
|
+
whyItMatters: finding.whyItMatters,
|
|
85
|
+
suggestion: finding.suggestion,
|
|
86
|
+
severity: finding.severity,
|
|
87
|
+
confidence: finding.confidence,
|
|
88
|
+
seenBy: finding.seenByStreamIds.map((streamId) => ({
|
|
89
|
+
streamId,
|
|
90
|
+
label: labels.get(streamId) ?? streamId,
|
|
91
|
+
})),
|
|
92
|
+
evidence: citedEvidence(finding.evidenceIds, analysis, runRoot),
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
56
95
|
function findingView(finding, loaded, runRoot) {
|
|
57
96
|
const labels = new Map(loaded.analysis.participants.map((p) => [p.streamId, p.label]));
|
|
58
97
|
const correction = loaded.corrections.filter((entry) => entry.findingId === finding.id).at(-1);
|
|
59
98
|
return {
|
|
60
99
|
id: finding.id,
|
|
100
|
+
headline: finding.headline ?? null,
|
|
101
|
+
experience: finding.experience ?? null,
|
|
61
102
|
title: finding.title,
|
|
62
103
|
summary: finding.summary,
|
|
63
104
|
impact: finding.impact,
|
|
@@ -98,14 +139,26 @@ function withoutFindings(source, state, reason, message, next) {
|
|
|
98
139
|
path: null,
|
|
99
140
|
summary: null,
|
|
100
141
|
findings: [],
|
|
142
|
+
designFindings: null,
|
|
101
143
|
limitations: [],
|
|
102
144
|
warnings: source.loaded.warnings,
|
|
103
145
|
};
|
|
104
146
|
}
|
|
147
|
+
/** A dry run's view: nothing to analyze, and the command that starts a live run. */
|
|
148
|
+
function dryRunFindings(source) {
|
|
149
|
+
const opening = "This is a dry run: no participant used the product, so there is nothing to analyze.";
|
|
150
|
+
const live = source.liveStudy;
|
|
151
|
+
if (live === undefined)
|
|
152
|
+
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs. The next command lists this project's studies; a study with mode: live runs participants.`, cli(`study list${source.cwdFlag}`));
|
|
153
|
+
const howLive = live.setLiveIn === undefined
|
|
154
|
+
? `the next command runs ${live.id} live.`
|
|
155
|
+
: `set mode: live in ${live.setLiveIn}, then run the next command.`;
|
|
156
|
+
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs: ${howLive}`, cli(`run ${live.run}${source.cwdFlag}`));
|
|
157
|
+
}
|
|
105
158
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
106
159
|
export function analysisFindings(source) {
|
|
107
160
|
if (source.mode !== undefined && source.mode !== "live")
|
|
108
|
-
return
|
|
161
|
+
return dryRunFindings(source);
|
|
109
162
|
const loaded = projectShareCheckedAnalysis(source.loaded);
|
|
110
163
|
const analysis = loaded.analysis;
|
|
111
164
|
if (analysis?.result && (loaded.state === "ready" || loaded.state === "stale")) {
|
|
@@ -130,6 +183,7 @@ export function analysisFindings(source) {
|
|
|
130
183
|
path: path.join(source.runRoot, "analysis", analysis.id, "analysis.json"),
|
|
131
184
|
summary: result.summary,
|
|
132
185
|
findings: result.findings.map((finding) => findingView(finding, { ...loaded, analysis }, source.runRoot)),
|
|
186
|
+
designFindings: designFindingViews(analysis, source.runRoot),
|
|
133
187
|
limitations: result.limitations,
|
|
134
188
|
warnings: loaded.warnings,
|
|
135
189
|
};
|
|
@@ -170,6 +224,30 @@ async function studyAnalysisOff(cwd, study) {
|
|
|
170
224
|
return undefined;
|
|
171
225
|
return resolved.config.review?.analysis === false ? study.path : undefined;
|
|
172
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* The study a live run can start from after a dry run: the run's own study when its file runs
|
|
229
|
+
* live, else the starter study try-live when this project has one that runs live, else the run's
|
|
230
|
+
* own study once its file is set live. A preview study only dry-runs. Undefined when none applies.
|
|
231
|
+
*/
|
|
232
|
+
async function liveStudyAfterDryRun(cwd, study) {
|
|
233
|
+
const resolve = async (ref) => {
|
|
234
|
+
const resolved = await resolveStudyManifest(cwd, ref).catch(() => null);
|
|
235
|
+
return resolved?.ok ? resolved.config : undefined;
|
|
236
|
+
};
|
|
237
|
+
const ownConfig = study === undefined
|
|
238
|
+
? undefined
|
|
239
|
+
: await resolve(study.path ?? study.id).then((config) => config?.id === study.id ? config : undefined);
|
|
240
|
+
const own = study === undefined
|
|
241
|
+
? undefined
|
|
242
|
+
: { run: study.origin === "explicit" ? (study.path ?? study.id) : study.id, id: study.id };
|
|
243
|
+
if (own !== undefined && ownConfig?.mode === "live")
|
|
244
|
+
return own;
|
|
245
|
+
if ((await resolve("try-live"))?.mode === "live")
|
|
246
|
+
return { run: "try-live", id: "try-live" };
|
|
247
|
+
if (own !== undefined && ownConfig !== undefined && study?.path !== undefined)
|
|
248
|
+
return routeOf(ownConfig) === "preview" ? undefined : { ...own, setLiveIn: study.path };
|
|
249
|
+
return undefined;
|
|
250
|
+
}
|
|
173
251
|
/**
|
|
174
252
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
175
253
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
@@ -180,16 +258,18 @@ export async function readRunAnalysis(cwd, run, analysisId) {
|
|
|
180
258
|
if (!prepared)
|
|
181
259
|
return null;
|
|
182
260
|
const bundle = await loadRunBundlePrepared(physical, prepared).catch(() => null);
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
261
|
+
const mode = bundle?.bundle.mode;
|
|
262
|
+
const study = bundle === null ? undefined : studyProvenanceOf(bundle.bundle);
|
|
263
|
+
const analysisOffIn = mode === "live" ? await studyAnalysisOff(physical, study) : undefined;
|
|
264
|
+
const liveStudy = mode !== undefined && mode !== "live" ? await liveStudyAfterDryRun(physical, study) : undefined;
|
|
186
265
|
return {
|
|
187
266
|
runId: runIdOf(prepared),
|
|
188
|
-
mode
|
|
267
|
+
mode,
|
|
189
268
|
runRoot: prepared.relativeRunRoot,
|
|
190
269
|
loaded: await loadAnalysis(prepared, analysisId),
|
|
191
270
|
cwdFlag: cwdFlag(cwd),
|
|
192
271
|
...(analysisOffIn === undefined ? {} : { analysisOffIn }),
|
|
272
|
+
...(liveStudy === undefined ? {} : { liveStudy }),
|
|
193
273
|
};
|
|
194
274
|
}
|
|
195
275
|
/** The view of a run that could not be found. */
|
|
@@ -250,20 +330,49 @@ function momentsText(evidence) {
|
|
|
250
330
|
parts.push(`${plural(unframed, "item")} with no frame`);
|
|
251
331
|
return parts.join(", ");
|
|
252
332
|
}
|
|
333
|
+
/** The capture files, relative to the run directory, in frame order. */
|
|
334
|
+
function captureFiles(evidence, runRoot) {
|
|
335
|
+
return [
|
|
336
|
+
...new Set(evidence
|
|
337
|
+
.filter((entry) => entry.capture !== null)
|
|
338
|
+
.sort((a, b) => (a.frame ?? 0) - (b.frame ?? 0))
|
|
339
|
+
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
340
|
+
];
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* What a reader sees first: the headline, or the reviewer's claim when a review amended the finding,
|
|
344
|
+
* since the headline and experience describe the claim it replaced. Null before headlines existed.
|
|
345
|
+
*/
|
|
346
|
+
function leadOf(finding) {
|
|
347
|
+
const amended = finding.correction?.status === "amended" ? finding.correction : null;
|
|
348
|
+
if (finding.headline === null)
|
|
349
|
+
return null;
|
|
350
|
+
return amended?.replacementClaim
|
|
351
|
+
? { headline: amended.replacementClaim, corrected: true }
|
|
352
|
+
: { headline: finding.headline, corrected: false };
|
|
353
|
+
}
|
|
253
354
|
function findingLines(finding, runRoot) {
|
|
254
355
|
const cited = [...new Set(finding.evidence.map((entry) => entry.streamId))];
|
|
255
356
|
const labels = new Map(finding.affected.map((p) => [p.streamId, p.label]));
|
|
256
357
|
const affectedIds = finding.affected.map((p) => p.streamId);
|
|
257
358
|
const streams = [...affectedIds, ...cited.filter((id) => !labels.has(id))];
|
|
258
|
-
const captures =
|
|
259
|
-
...new Set(finding.evidence
|
|
260
|
-
.filter((entry) => entry.capture !== null)
|
|
261
|
-
.sort((a, b) => (a.frame ?? 0) - (b.frame ?? 0))
|
|
262
|
-
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
263
|
-
];
|
|
359
|
+
const captures = captureFiles(finding.evidence, runRoot);
|
|
264
360
|
const correction = finding.correction;
|
|
361
|
+
const lead = leadOf(finding);
|
|
265
362
|
return [
|
|
266
|
-
|
|
363
|
+
...(lead === null
|
|
364
|
+
? [`${finding.id} ${finding.title}`]
|
|
365
|
+
: [
|
|
366
|
+
`${finding.id} ${lead.headline}`,
|
|
367
|
+
...(lead.corrected
|
|
368
|
+
? [
|
|
369
|
+
" Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
370
|
+
]
|
|
371
|
+
: finding.experience === null
|
|
372
|
+
? []
|
|
373
|
+
: [` ${finding.experience}`]),
|
|
374
|
+
` evidence: ${finding.title}`,
|
|
375
|
+
]),
|
|
267
376
|
` impact: ${IMPACT_TEXT[finding.impact]} · confidence: ${finding.confidence} · recovery: ${RECOVERY_TEXT[finding.recovery]}`,
|
|
268
377
|
` ${finding.summary}`,
|
|
269
378
|
` affected: ${finding.affected.length} of ${plural(finding.exposedCount, "exposed participant")}`,
|
|
@@ -281,6 +390,34 @@ function findingLines(finding, runRoot) {
|
|
|
281
390
|
]),
|
|
282
391
|
];
|
|
283
392
|
}
|
|
393
|
+
function designFindingLines(finding, runRoot) {
|
|
394
|
+
const captures = captureFiles(finding.evidence, runRoot);
|
|
395
|
+
return [
|
|
396
|
+
`${finding.id} ${finding.severity}: ${finding.headline}`,
|
|
397
|
+
` screen: ${finding.screen}`,
|
|
398
|
+
` notice: ${finding.notice}`,
|
|
399
|
+
` why it matters: ${finding.whyItMatters}`,
|
|
400
|
+
` suggestion: ${finding.suggestion}`,
|
|
401
|
+
` seen by: ${finding.seenBy.map((p) => p.label).join(", ")} · confidence: ${finding.confidence}`,
|
|
402
|
+
...(captures.length === 0 ? [] : [` captures: ${captures.join(", ")}`]),
|
|
403
|
+
];
|
|
404
|
+
}
|
|
405
|
+
/** Nothing for an analysis written before design findings. */
|
|
406
|
+
function designSection(view) {
|
|
407
|
+
const design = view.designFindings;
|
|
408
|
+
if (design === null)
|
|
409
|
+
return [];
|
|
410
|
+
if (design.length === 0)
|
|
411
|
+
return ["", "design findings: none in the reviewed captures"];
|
|
412
|
+
return [
|
|
413
|
+
"",
|
|
414
|
+
`design findings: ${design.length}, most severe first`,
|
|
415
|
+
...design.flatMap((finding, index) => [
|
|
416
|
+
...(index === 0 ? [] : [""]),
|
|
417
|
+
...designFindingLines(finding, view.runPath ?? ""),
|
|
418
|
+
]),
|
|
419
|
+
];
|
|
420
|
+
}
|
|
284
421
|
/** The full findings block: the analysis, every finding with its evidence, and its limitations. */
|
|
285
422
|
export function formatFindings(view) {
|
|
286
423
|
if (view.state !== "ready" && view.state !== "stale")
|
|
@@ -292,6 +429,7 @@ export function formatFindings(view) {
|
|
|
292
429
|
...(view.state === "stale" ? [`warning: ${view.message}`, `next: ${view.next}`] : []),
|
|
293
430
|
...(view.summary ? [view.summary] : []),
|
|
294
431
|
...view.findings.flatMap((finding) => ["", ...findingLines(finding, view.runPath ?? "")]),
|
|
432
|
+
...designSection(view),
|
|
295
433
|
...(view.limitations.length === 0
|
|
296
434
|
? []
|
|
297
435
|
: ["", "limitations:", ...view.limitations.map((limitation) => `- ${limitation}`)]),
|
|
@@ -305,12 +443,30 @@ export function formatFindingsSummary(view, fullCommand, limit = 3) {
|
|
|
305
443
|
if (view.state !== "ready" && view.state !== "stale")
|
|
306
444
|
return [`findings: ${view.message}`, ...(view.next ? [`next: ${view.next}`] : [])];
|
|
307
445
|
const shown = view.findings.slice(0, limit);
|
|
446
|
+
const design = view.designFindings;
|
|
308
447
|
return [
|
|
309
448
|
`findings: ${view.findings.length === 0 ? "none" : view.findings.length}${view.state === "stale" ? " (stale)" : ""}`,
|
|
310
|
-
...shown.map((finding) =>
|
|
449
|
+
...shown.map((finding) => {
|
|
450
|
+
const qualities = `${IMPACT_TEXT[finding.impact]}, ${finding.confidence} confidence, ${RECOVERY_TEXT[finding.recovery]}`;
|
|
451
|
+
const lead = leadOf(finding);
|
|
452
|
+
if (lead === null)
|
|
453
|
+
return `- ${finding.id} ${qualities}: ${finding.title}`;
|
|
454
|
+
return `- ${finding.id} ${lead.headline} (${lead.corrected ? "corrected in human review" : qualities})`;
|
|
455
|
+
}),
|
|
311
456
|
...(view.findings.length > limit
|
|
312
457
|
? [`- and ${plural(view.findings.length - limit, "more finding")}`]
|
|
313
458
|
: []),
|
|
459
|
+
...(design === null
|
|
460
|
+
? []
|
|
461
|
+
: [
|
|
462
|
+
`design findings: ${design.length === 0 ? "none" : design.length}`,
|
|
463
|
+
...design
|
|
464
|
+
.slice(0, limit)
|
|
465
|
+
.map((finding) => `- ${finding.id} ${finding.severity}: ${finding.headline}`),
|
|
466
|
+
...(design.length > limit
|
|
467
|
+
? [`- and ${plural(design.length - limit, "more design finding")}`]
|
|
468
|
+
: []),
|
|
469
|
+
]),
|
|
314
470
|
`all findings: ${fullCommand}`,
|
|
315
471
|
];
|
|
316
472
|
}
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -18,7 +18,8 @@ export declare const DOTENV_OPTION_DESCRIPTION = "Load unset variables from this
|
|
|
18
18
|
* The hidden `--env-file <path>` that `--dotenv` replaced. Node scans the whole argv for
|
|
19
19
|
* `--env-file` before a script runs, and stops only at `--` (src/node_dotenv.cc), so a missing file
|
|
20
20
|
* exits 9 through Node and never reaches humanish. `--dotenv` does not collide.
|
|
21
|
-
* Removed in
|
|
21
|
+
* Removed in the first minor release on or after 2026-11-03: deprecated in 0.110.0 on 2026-10-03, it stays at least 30 days and one
|
|
22
|
+
* minor release.
|
|
22
23
|
*/
|
|
23
24
|
export declare function envFileAliasOption(): Option;
|
|
24
25
|
/** The dotenv path a command was given: `--dotenv`, or the `--env-file` alias with its warning. */
|
package/dist/cli/io.js
CHANGED
|
@@ -15,12 +15,13 @@ export const JSON_OPTION_DESCRIPTION = "Print a machine-readable JSON response."
|
|
|
15
15
|
export const CWD_OPTION_DESCRIPTION = "Project directory.";
|
|
16
16
|
export const RUN_OPTION_DESCRIPTION = "Run id, or latest.";
|
|
17
17
|
export const DOTENV_OPTION_DESCRIPTION = "Load unset variables from this dotenv file. Values are never printed or saved.";
|
|
18
|
-
const ENV_FILE_RENAMED = "`--env-file` is now `--dotenv`; `--env-file` is removed in
|
|
18
|
+
const ENV_FILE_RENAMED = "`--env-file` is now `--dotenv`; `--env-file` is removed in the first minor release on or after 2026-11-03.";
|
|
19
19
|
/**
|
|
20
20
|
* The hidden `--env-file <path>` that `--dotenv` replaced. Node scans the whole argv for
|
|
21
21
|
* `--env-file` before a script runs, and stops only at `--` (src/node_dotenv.cc), so a missing file
|
|
22
22
|
* exits 9 through Node and never reaches humanish. `--dotenv` does not collide.
|
|
23
|
-
* Removed in
|
|
23
|
+
* Removed in the first minor release on or after 2026-11-03: deprecated in 0.110.0 on 2026-10-03, it stays at least 30 days and one
|
|
24
|
+
* minor release.
|
|
24
25
|
*/
|
|
25
26
|
export function envFileAliasOption() {
|
|
26
27
|
return new Option("--env-file <path>").hideHelp().conflicts("dotenv");
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
3
|
+
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
4
|
+
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
5
|
+
* is not ordinary text.
|
|
6
|
+
*/
|
|
7
|
+
export declare function encodedForms(value: string): string[];
|
|
1
8
|
/**
|
|
2
9
|
* A scrub that replaces each non-empty value and its encoded forms with `[REDACTED_SECRET]`. Every
|
|
3
10
|
* occurrence of every form is found on its own, and overlapping finds are merged into one span, so
|
|
@@ -25,25 +25,34 @@ function base64Middles(bytes, encoding) {
|
|
|
25
25
|
return encoded.slice(start, end);
|
|
26
26
|
});
|
|
27
27
|
}
|
|
28
|
-
/**
|
|
29
|
-
|
|
28
|
+
/**
|
|
29
|
+
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
30
|
+
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
31
|
+
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
32
|
+
* is not ordinary text.
|
|
33
|
+
*/
|
|
34
|
+
export function encodedForms(value) {
|
|
30
35
|
const bytes = Buffer.from(value, "utf8");
|
|
31
|
-
const
|
|
32
|
-
|
|
36
|
+
const json = JSON.stringify(value).slice(1, -1);
|
|
37
|
+
// A serializer that writes ASCII only, such as Python's json.dumps, spells é as \u00e9.
|
|
38
|
+
const asciiJson = json.replace(/[^\x00-\x7f]/g, (char) => `\\u${char.charCodeAt(0).toString(16).padStart(4, "0")}`);
|
|
39
|
+
const escaped = [json, asciiJson, JSON.stringify(json).slice(1, -1)];
|
|
40
|
+
// The encoders throw on a lone surrogate. Such a value is still found as written. encodeURI
|
|
41
|
+
// keeps a `/` or a `:` as written, as a URL path does.
|
|
42
|
+
try {
|
|
43
|
+
escaped.push(encodeURIComponent(value), encodeURI(value));
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// no percent-encoded form
|
|
47
|
+
}
|
|
48
|
+
const binary = [
|
|
33
49
|
bytes.toString("base64"),
|
|
34
50
|
bytes.toString("base64url"),
|
|
35
51
|
bytes.toString("hex"),
|
|
36
52
|
...base64Middles(bytes, "base64"),
|
|
37
53
|
...base64Middles(bytes, "base64url"),
|
|
38
54
|
];
|
|
39
|
-
|
|
40
|
-
try {
|
|
41
|
-
encoded.push(encodeURIComponent(value));
|
|
42
|
-
}
|
|
43
|
-
catch {
|
|
44
|
-
// no percent-encoded form
|
|
45
|
-
}
|
|
46
|
-
return [value, ...encoded.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
55
|
+
return [value, ...escaped, ...binary.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
47
56
|
}
|
|
48
57
|
/** Every occurrence of every form, outside the markers, merged where they overlap or touch. */
|
|
49
58
|
function secretSpans(text, forms) {
|
package/dist/feedback/draft.js
CHANGED
|
@@ -120,7 +120,8 @@ export async function buildAnalysisDraft(context, options) {
|
|
|
120
120
|
const root = path.dirname(context.loaded.bundlePath);
|
|
121
121
|
const evidenceIds = new Set(finding.observations.flatMap((item) => item.evidenceIds));
|
|
122
122
|
const evidence = analysis.evidence.filter((item) => evidenceIds.has(item.id));
|
|
123
|
-
const
|
|
123
|
+
const amended = correction?.status === "amended";
|
|
124
|
+
const claim = amended ? correction.replacementClaim : (finding.headline ?? finding.title);
|
|
124
125
|
const firstLine = claim.trim().split(/\r?\n/)[0] || `Reviewed finding ${finding.id}`;
|
|
125
126
|
const summary = Array.from(firstLine).length > 160
|
|
126
127
|
? Array.from(firstLine).slice(0, 159).join("") + "…"
|
|
@@ -144,10 +145,19 @@ export async function buildAnalysisDraft(context, options) {
|
|
|
144
145
|
summary,
|
|
145
146
|
expected: "Review the cited behavior against the participant assignment and confirm the expected product behavior.",
|
|
146
147
|
actual: [
|
|
148
|
+
// The plain account leads; an amendment makes it stale, so the draft says so in its place.
|
|
149
|
+
...(finding.headline === undefined
|
|
150
|
+
? []
|
|
151
|
+
: amended
|
|
152
|
+
? [
|
|
153
|
+
"This finding was corrected in human review. The reviewer's claim above replaces its original headline and account.",
|
|
154
|
+
"",
|
|
155
|
+
]
|
|
156
|
+
: finding.experience === undefined
|
|
157
|
+
? []
|
|
158
|
+
: [finding.experience, ""]),
|
|
147
159
|
"Independent study analysis; does not replace participant feedback or recorded completion outcomes.",
|
|
148
|
-
|
|
149
|
-
? `Amended claim: ${claim}. Original analysis: ${finding.summary}`
|
|
150
|
-
: finding.summary,
|
|
160
|
+
amended ? `Amended claim: ${claim}. Original analysis: ${finding.summary}` : finding.summary,
|
|
151
161
|
`Impact: ${finding.impact}. ${finding.affectedStreamIds.length} affected / ${finding.exposedStreamIds.length} observed exposed participants. ${finding.exposureReason}`,
|
|
152
162
|
`Recovery: ${finding.recovery}. Confidence: ${finding.confidence}.`,
|
|
153
163
|
...finding.observations.map((item) => `${item.basis}: ${item.claim}${item.limitation ? ` Limitation: ${item.limitation}` : ""}`),
|
|
@@ -8,6 +8,7 @@ import { isLocalOnlyRunFile, isLocalOnlyRunFileIdentity } from "../run/local-onl
|
|
|
8
8
|
import { listRuns } from "../run/stored-runs.js";
|
|
9
9
|
import { readAnalysisAccounting } from "../run/costs.js";
|
|
10
10
|
import { bindExistingRunArtifactPaths, isPathInside, isSafeRunIdSegment, } from "../run/paths.js";
|
|
11
|
+
import { isRunOutcome } from "../run/bundle-shape.js";
|
|
11
12
|
import { analysisCostOf, runCost, runCostLabel } from "../run/run-cost.js";
|
|
12
13
|
import { RUN_STATUS_FILE, RUN_STATUS_STALE_MS, isRunStatusRecord, } from "../run/status.js";
|
|
13
14
|
import { renderObserverHtml } from "./artifact.js";
|
|
@@ -138,7 +139,7 @@ async function readObserverData(runRoot, runtimeStreamUrls = []) {
|
|
|
138
139
|
if (!bundleBytes)
|
|
139
140
|
throw new Error("run.json unavailable");
|
|
140
141
|
const bundle = JSON.parse(bundleBytes.toString("utf8"));
|
|
141
|
-
return withRuntimeStreamUrls(withLocalRunStatus(record, buildObserverData(bundle, undefined, record)), runtimeStreamUrls);
|
|
142
|
+
return withRuntimeStreamUrls(withLocalRunStatus(record, buildObserverData(bundle, undefined, record), bundle.outcome), runtimeStreamUrls);
|
|
142
143
|
}
|
|
143
144
|
catch { }
|
|
144
145
|
try {
|
|
@@ -166,12 +167,12 @@ async function readLocalRunStatus(runRoot) {
|
|
|
166
167
|
}
|
|
167
168
|
}
|
|
168
169
|
/**
|
|
169
|
-
*
|
|
170
|
+
* A recorded bundle outcome wins; otherwise runtime status reads the contained local heartbeat.
|
|
170
171
|
* A stale heartbeat means unknown: neither an old timestamp nor a persisted PID proves that a
|
|
171
172
|
* process died (the evidence may have been copied from another machine). A record its stopped
|
|
172
173
|
* process wrote is interrupted. No PID is served/probed.
|
|
173
174
|
*/
|
|
174
|
-
function withLocalRunStatus(record, input) {
|
|
175
|
+
function withLocalRunStatus(record, input, outcome) {
|
|
175
176
|
// A served observation must never be inherited from a persisted projection or export.
|
|
176
177
|
const { runtime: _persistedRuntime, ...data } = input;
|
|
177
178
|
if (record === undefined || record.runId !== data.run.runId || record.mode !== data.run.mode)
|
|
@@ -181,7 +182,10 @@ function withLocalRunStatus(record, input) {
|
|
|
181
182
|
const updated = Date.parse(record.updatedAt);
|
|
182
183
|
const timestampsValid = Number.isFinite(started) && Number.isFinite(updated) && started <= updated && updated <= now;
|
|
183
184
|
let state = "unknown";
|
|
184
|
-
if (
|
|
185
|
+
if (isRunOutcome(outcome)) {
|
|
186
|
+
state = outcome.state;
|
|
187
|
+
}
|
|
188
|
+
else if (timestampsValid) {
|
|
185
189
|
if (record.state === "finished") {
|
|
186
190
|
state = "finished";
|
|
187
191
|
}
|