humanish 0.111.4 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +34 -3
- package/dist/cli/findings.js +136 -18
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +1 -0
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +71 -22
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
package/dist/analysis/types.d.ts
CHANGED
|
@@ -104,6 +104,11 @@ export interface AnalysisObservation {
|
|
|
104
104
|
interface AnalysisFinding {
|
|
105
105
|
id: string;
|
|
106
106
|
title: string;
|
|
107
|
+
/** One plain sentence for a reader who did not watch the session. Absent in older reports. */
|
|
108
|
+
headline?: string;
|
|
109
|
+
/** One to three plain sentences: what the person tried, what got in the way, how it seemed to
|
|
110
|
+
* feel. Absent in older reports. */
|
|
111
|
+
experience?: string;
|
|
107
112
|
summary: string;
|
|
108
113
|
impact: "blocked_task" | "friction" | "recovery" | "uncertain";
|
|
109
114
|
affectedStreamIds: string[];
|
|
@@ -115,6 +120,22 @@ interface AnalysisFinding {
|
|
|
115
120
|
nextStep: string;
|
|
116
121
|
priorityReason: string;
|
|
117
122
|
}
|
|
123
|
+
/** A problem a product designer would see in the cited captures, whether or not a participant
|
|
124
|
+
* mentioned it. It cites at least one capture. */
|
|
125
|
+
interface AnalysisDesignFinding {
|
|
126
|
+
id: string;
|
|
127
|
+
headline: string;
|
|
128
|
+
/** The screen, in plain words. */
|
|
129
|
+
screen: string;
|
|
130
|
+
notice: string;
|
|
131
|
+
whyItMatters: string;
|
|
132
|
+
suggestion: string;
|
|
133
|
+
severity: "minor" | "moderate" | "major";
|
|
134
|
+
confidence: "low" | "medium" | "high";
|
|
135
|
+
/** Participants whose cited evidence shows the problem. */
|
|
136
|
+
seenByStreamIds: string[];
|
|
137
|
+
evidenceIds: string[];
|
|
138
|
+
}
|
|
118
139
|
/** Evidence-grounded disposition of a material concern, not model reasoning. */
|
|
119
140
|
interface AnalysisConcernReview extends AnalysisObservation {
|
|
120
141
|
disposition: "finding" | "context" | "unsupported";
|
|
@@ -128,6 +149,8 @@ export interface AnalysisResult {
|
|
|
128
149
|
concernReviews?: AnalysisConcernReview[];
|
|
129
150
|
/** Highest priority first; counts are derived from distinct stream sets. */
|
|
130
151
|
findings: AnalysisFinding[];
|
|
152
|
+
/** Absent in older reports. New analyses supply it, empty when the captures show no problem. */
|
|
153
|
+
designFindings?: AnalysisDesignFinding[];
|
|
131
154
|
limitations: string[];
|
|
132
155
|
}
|
|
133
156
|
interface OpenAIAnalysisConfig {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { type AnalysisArtifact, type AnalysisCorrection, type AnalysisInput, type AnalysisResult } from "./types.js";
|
|
3
|
-
/** Historical artifacts may omit concernReviews
|
|
3
|
+
/** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
|
|
4
|
+
* new provider response must supply them. */
|
|
4
5
|
export declare const analysisResponseSchema: z.ZodObject<{
|
|
5
6
|
summary: z.ZodString;
|
|
6
7
|
participants: z.ZodArray<z.ZodObject<{
|
|
@@ -22,9 +23,30 @@ export declare const analysisResponseSchema: z.ZodObject<{
|
|
|
22
23
|
}, z.core.$strict>>;
|
|
23
24
|
limitations: z.ZodArray<z.ZodString>;
|
|
24
25
|
}, z.core.$strict>>;
|
|
26
|
+
concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
27
|
+
claim: z.ZodString;
|
|
28
|
+
basis: z.ZodEnum<{
|
|
29
|
+
action: "action";
|
|
30
|
+
inference: "inference";
|
|
31
|
+
participant_statement: "participant_statement";
|
|
32
|
+
visual: "visual";
|
|
33
|
+
}>;
|
|
34
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
35
|
+
limitation: z.ZodString;
|
|
36
|
+
disposition: z.ZodEnum<{
|
|
37
|
+
context: "context";
|
|
38
|
+
finding: "finding";
|
|
39
|
+
unsupported: "unsupported";
|
|
40
|
+
}>;
|
|
41
|
+
findingId: z.ZodNullable<z.ZodString>;
|
|
42
|
+
reason: z.ZodString;
|
|
43
|
+
}, z.core.$strict>>>>;
|
|
44
|
+
limitations: z.ZodArray<z.ZodString>;
|
|
25
45
|
findings: z.ZodArray<z.ZodObject<{
|
|
26
46
|
id: z.ZodString;
|
|
27
47
|
title: z.ZodString;
|
|
48
|
+
headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
49
|
+
experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
28
50
|
summary: z.ZodString;
|
|
29
51
|
impact: z.ZodEnum<{
|
|
30
52
|
blocked_task: "blocked_task";
|
|
@@ -59,25 +81,26 @@ export declare const analysisResponseSchema: z.ZodObject<{
|
|
|
59
81
|
nextStep: z.ZodString;
|
|
60
82
|
priorityReason: z.ZodString;
|
|
61
83
|
}, z.core.$strict>>;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
84
|
+
designFindings: z.ZodArray<z.ZodObject<{
|
|
85
|
+
id: z.ZodString;
|
|
86
|
+
headline: z.ZodString;
|
|
87
|
+
screen: z.ZodString;
|
|
88
|
+
notice: z.ZodString;
|
|
89
|
+
whyItMatters: z.ZodString;
|
|
90
|
+
suggestion: z.ZodString;
|
|
91
|
+
severity: z.ZodEnum<{
|
|
92
|
+
major: "major";
|
|
93
|
+
minor: "minor";
|
|
94
|
+
moderate: "moderate";
|
|
69
95
|
}>;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
finding: "finding";
|
|
75
|
-
unsupported: "unsupported";
|
|
96
|
+
confidence: z.ZodEnum<{
|
|
97
|
+
high: "high";
|
|
98
|
+
low: "low";
|
|
99
|
+
medium: "medium";
|
|
76
100
|
}>;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}, z.core.$strict
|
|
80
|
-
limitations: z.ZodArray<z.ZodString>;
|
|
101
|
+
seenByStreamIds: z.ZodArray<z.ZodString>;
|
|
102
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
103
|
+
}, z.core.$strict>>;
|
|
81
104
|
}, z.core.$strict>;
|
|
82
105
|
export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
83
106
|
summary: z.ZodString;
|
|
@@ -100,9 +123,30 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
|
|
|
100
123
|
}, z.core.$strict>>;
|
|
101
124
|
limitations: z.ZodArray<z.ZodString>;
|
|
102
125
|
}, z.core.$strict>>;
|
|
126
|
+
concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
127
|
+
claim: z.ZodString;
|
|
128
|
+
basis: z.ZodEnum<{
|
|
129
|
+
action: "action";
|
|
130
|
+
inference: "inference";
|
|
131
|
+
participant_statement: "participant_statement";
|
|
132
|
+
visual: "visual";
|
|
133
|
+
}>;
|
|
134
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
135
|
+
limitation: z.ZodString;
|
|
136
|
+
disposition: z.ZodEnum<{
|
|
137
|
+
context: "context";
|
|
138
|
+
finding: "finding";
|
|
139
|
+
unsupported: "unsupported";
|
|
140
|
+
}>;
|
|
141
|
+
findingId: z.ZodNullable<z.ZodString>;
|
|
142
|
+
reason: z.ZodString;
|
|
143
|
+
}, z.core.$strict>>>>;
|
|
144
|
+
limitations: z.ZodArray<z.ZodString>;
|
|
103
145
|
findings: z.ZodArray<z.ZodObject<{
|
|
104
146
|
id: z.ZodString;
|
|
105
147
|
title: z.ZodString;
|
|
148
|
+
headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
149
|
+
experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
106
150
|
summary: z.ZodString;
|
|
107
151
|
impact: z.ZodEnum<{
|
|
108
152
|
blocked_task: "blocked_task";
|
|
@@ -137,25 +181,26 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
|
|
|
137
181
|
nextStep: z.ZodString;
|
|
138
182
|
priorityReason: z.ZodString;
|
|
139
183
|
}, z.core.$strict>>;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
184
|
+
designFindings: z.ZodArray<z.ZodObject<{
|
|
185
|
+
id: z.ZodString;
|
|
186
|
+
headline: z.ZodString;
|
|
187
|
+
screen: z.ZodString;
|
|
188
|
+
notice: z.ZodString;
|
|
189
|
+
whyItMatters: z.ZodString;
|
|
190
|
+
suggestion: z.ZodString;
|
|
191
|
+
severity: z.ZodEnum<{
|
|
192
|
+
major: "major";
|
|
193
|
+
minor: "minor";
|
|
194
|
+
moderate: "moderate";
|
|
147
195
|
}>;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
finding: "finding";
|
|
153
|
-
unsupported: "unsupported";
|
|
196
|
+
confidence: z.ZodEnum<{
|
|
197
|
+
high: "high";
|
|
198
|
+
low: "low";
|
|
199
|
+
medium: "medium";
|
|
154
200
|
}>;
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
}, z.core.$strict
|
|
158
|
-
limitations: z.ZodArray<z.ZodString>;
|
|
201
|
+
seenByStreamIds: z.ZodArray<z.ZodString>;
|
|
202
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
203
|
+
}, z.core.$strict>>;
|
|
159
204
|
}, z.core.$strict>>;
|
|
160
205
|
/** Stable hashing is independent of object insertion order and excludes no fields implicitly. */
|
|
161
206
|
export declare function hashAnalysisValue(value: unknown): string;
|
|
@@ -27,6 +27,20 @@ const observationSchema = z
|
|
|
27
27
|
limitation: text(2000),
|
|
28
28
|
})
|
|
29
29
|
.strict();
|
|
30
|
+
const designFindingSchema = z
|
|
31
|
+
.object({
|
|
32
|
+
id,
|
|
33
|
+
headline: label,
|
|
34
|
+
screen: label,
|
|
35
|
+
notice: text(1500).min(1),
|
|
36
|
+
whyItMatters: text(1000).min(1),
|
|
37
|
+
suggestion: text(1000).min(1),
|
|
38
|
+
severity: z.enum(["minor", "moderate", "major"]),
|
|
39
|
+
confidence: z.enum(["low", "medium", "high"]),
|
|
40
|
+
seenByStreamIds: sourceIds.min(1),
|
|
41
|
+
evidenceIds: refs,
|
|
42
|
+
})
|
|
43
|
+
.strict();
|
|
30
44
|
const analysisResultSchema = z
|
|
31
45
|
.object({
|
|
32
46
|
summary: text(4000).min(1),
|
|
@@ -51,6 +65,8 @@ const analysisResultSchema = z
|
|
|
51
65
|
.object({
|
|
52
66
|
id,
|
|
53
67
|
title: label,
|
|
68
|
+
headline: label.optional(),
|
|
69
|
+
experience: text(1200).min(1).optional(),
|
|
54
70
|
summary: text(4000).min(1),
|
|
55
71
|
impact: z.enum(["blocked_task", "friction", "recovery", "uncertain"]),
|
|
56
72
|
affectedStreamIds: sourceIds.min(1),
|
|
@@ -64,6 +80,7 @@ const analysisResultSchema = z
|
|
|
64
80
|
})
|
|
65
81
|
.strict())
|
|
66
82
|
.max(100),
|
|
83
|
+
designFindings: z.array(designFindingSchema).max(40).optional(),
|
|
67
84
|
concernReviews: z
|
|
68
85
|
.array(observationSchema
|
|
69
86
|
.extend({
|
|
@@ -77,13 +94,25 @@ const analysisResultSchema = z
|
|
|
77
94
|
limitations,
|
|
78
95
|
})
|
|
79
96
|
.strict();
|
|
80
|
-
/** Historical artifacts may omit concernReviews
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
97
|
+
/** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
|
|
98
|
+
* new provider response must supply them. */
|
|
99
|
+
export const analysisResponseSchema = analysisResultSchema
|
|
100
|
+
.extend({
|
|
101
|
+
findings: z
|
|
102
|
+
.array(analysisResultSchema.shape.findings.element.required({ headline: true, experience: true }))
|
|
103
|
+
.max(100),
|
|
104
|
+
designFindings: z.array(designFindingSchema).max(40),
|
|
105
|
+
})
|
|
106
|
+
.required({ concernReviews: true });
|
|
84
107
|
export const analysisResultJsonSchema = z.toJSONSchema(analysisResponseSchema);
|
|
85
|
-
const normalizedResult = ({ concernReviews, ...result }) => ({
|
|
108
|
+
const normalizedResult = ({ concernReviews, designFindings, findings, ...result }) => ({
|
|
86
109
|
...result,
|
|
110
|
+
findings: findings.map(({ headline, experience, ...finding }) => ({
|
|
111
|
+
...finding,
|
|
112
|
+
...(headline === undefined ? {} : { headline }),
|
|
113
|
+
...(experience === undefined ? {} : { experience }),
|
|
114
|
+
})),
|
|
115
|
+
...(designFindings === undefined ? {} : { designFindings }),
|
|
87
116
|
...(concernReviews === undefined ? {} : { concernReviews }),
|
|
88
117
|
});
|
|
89
118
|
const analysisCoverageSchema = z
|
|
@@ -349,6 +378,21 @@ export function checkAnalysisResult(input, value) {
|
|
|
349
378
|
if (finding.affectedStreamIds.some((stream) => !citedStreams.has(stream)))
|
|
350
379
|
errors.add("ANALYSIS_AFFECTED_WITHOUT_EVIDENCE");
|
|
351
380
|
}
|
|
381
|
+
const designFindings = result.designFindings ?? [];
|
|
382
|
+
if (!distinct(designFindings.map((finding) => finding.id)))
|
|
383
|
+
errors.add("ANALYSIS_DESIGN_FINDING_ID_DUPLICATE");
|
|
384
|
+
for (const finding of designFindings) {
|
|
385
|
+
const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
|
|
386
|
+
if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
387
|
+
errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
|
|
388
|
+
if (!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
|
|
389
|
+
errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
|
|
390
|
+
// Seen by means a cited capture of that participant shows the problem.
|
|
391
|
+
const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
|
|
392
|
+
if (!distinct(finding.seenByStreamIds) ||
|
|
393
|
+
finding.seenByStreamIds.some((stream) => !included.has(stream) || !captured.has(stream)))
|
|
394
|
+
errors.add("ANALYSIS_DESIGN_MEMBERSHIP_INVALID");
|
|
395
|
+
}
|
|
352
396
|
for (const review of result.concernReviews ?? []) {
|
|
353
397
|
const citedStreams = checkObservation(review);
|
|
354
398
|
const finding = result.findings.find((item) => item.id === review.findingId);
|
|
@@ -425,12 +469,15 @@ export function validateAnalysisArtifact(value) {
|
|
|
425
469
|
(artifact.status === "complete" && !coverage.complete))
|
|
426
470
|
throw new Error("ANALYSIS_STATUS_INVALID");
|
|
427
471
|
if (artifact.result !== null) {
|
|
428
|
-
// Concern accounting became required with revision 5
|
|
429
|
-
// when the prompt version advances; a later
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
472
|
+
// Concern accounting became required with revision 5, and headlines, experiences and design
|
|
473
|
+
// findings with revision 7. Keep each boundary stable when the prompt version advances; a later
|
|
474
|
+
// prompt must not regain legacy omissions.
|
|
475
|
+
const revision = Number(/^study-evidence-(\d+)$/.exec(artifact.promptVersion)?.[1] ?? 0);
|
|
476
|
+
const result = artifact.result;
|
|
477
|
+
if ((revision >= 5 && result.concernReviews === undefined) ||
|
|
478
|
+
(revision >= 7 &&
|
|
479
|
+
(result.designFindings === undefined ||
|
|
480
|
+
result.findings.some((finding) => finding.headline === undefined || finding.experience === undefined))))
|
|
434
481
|
throw new Error("ANALYSIS_RESULT_SCHEMA_INVALID");
|
|
435
482
|
validateAnalysisResult({ ...artifact, images: [] }, artifact.result);
|
|
436
483
|
}
|
|
@@ -6,6 +6,7 @@ import { DEFAULT_EXPORT_MAX_BYTES, exportRun, formatExportHuman } from "../../fe
|
|
|
6
6
|
import { listRuns, readReview } from "../../run/stored-runs.js";
|
|
7
7
|
import { verifyRun } from "../../verify/verify.js";
|
|
8
8
|
import { reclaimPreflightSandboxes, reclaimRunSandboxes, } from "../../run/reclaim.js";
|
|
9
|
+
import { runDisplay } from "../../run/display.js";
|
|
9
10
|
import { addRunOptions, handleRun } from "./run-command.js";
|
|
10
11
|
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, RUN_OPTION_DESCRIPTION, writeResult, humanError, } from "../io.js";
|
|
11
12
|
import { plural } from "../../run/text.js";
|
|
@@ -33,7 +34,7 @@ export function registerVerifyCommand(parent, io) {
|
|
|
33
34
|
io.setExitCode(result.ok ? 0 : 2);
|
|
34
35
|
});
|
|
35
36
|
}
|
|
36
|
-
const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in
|
|
37
|
+
const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in the first minor release on or after 2026-11-03.";
|
|
37
38
|
/**
|
|
38
39
|
* The deprecated `cleanup`, hidden from help: one warning, then exactly `reclaim --check`, with
|
|
39
40
|
* its output and exit code. The old command read only what computer-use runs recorded and passed
|
|
@@ -244,13 +245,6 @@ const REVIEW_VERDICTS = {
|
|
|
244
245
|
blocked: "blocked",
|
|
245
246
|
timed_out: "timed out",
|
|
246
247
|
};
|
|
247
|
-
/** The state runDisplay gives a finished run whose own ok agrees with its verdict. */
|
|
248
|
-
const VERDICT_STATES = {
|
|
249
|
-
pass: "passed",
|
|
250
|
-
fail: "failed",
|
|
251
|
-
blocked: "blocked",
|
|
252
|
-
timed_out: "timed_out",
|
|
253
|
-
};
|
|
254
248
|
/**
|
|
255
249
|
* Review's headline: the run's display label, as `humanish runs`, review.md and the Observer show
|
|
256
250
|
* it. A verdict that names another state follows in parentheses, so an interrupted run whose last
|
|
@@ -262,8 +256,8 @@ function reviewHeadline(verdict, display) {
|
|
|
262
256
|
return REVIEW_VERDICTS[verdict];
|
|
263
257
|
if (display.state === "no_verdict" || display.state === "dry_run")
|
|
264
258
|
return `${display.label}; ${NOTHING_TESTED}`;
|
|
265
|
-
const judged =
|
|
266
|
-
return
|
|
259
|
+
const judged = runDisplay({ liveness: "finished", verdict }).state;
|
|
260
|
+
return verdict === "contract_proof_only" || judged === display.state
|
|
267
261
|
? display.label
|
|
268
262
|
: `${display.label} (verdict ${REVIEW_VERDICTS[verdict]})`;
|
|
269
263
|
}
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -4,14 +4,13 @@ import type { AnalysisArtifact, LoadedAnalysis } from "../analysis/types.js";
|
|
|
4
4
|
import { type CliIo } from "./io.js";
|
|
5
5
|
export declare const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
6
6
|
type Finding = NonNullable<AnalysisArtifact["result"]>["findings"][number];
|
|
7
|
+
type DesignFinding = NonNullable<NonNullable<AnalysisArtifact["result"]>["designFindings"]>[number];
|
|
7
8
|
/** `ready` and `stale` carry findings; every other state carries none and says why in `message`. */
|
|
8
9
|
type FindingsState = "ready" | "stale" | "running" | "none" | "skipped" | "failed" | "dry_run" | "unavailable";
|
|
9
|
-
interface
|
|
10
|
+
interface CitedEvidence {
|
|
10
11
|
id: string;
|
|
11
12
|
streamId: string;
|
|
12
13
|
kind: string;
|
|
13
|
-
/** How the finding's observations used this item: visual, action, participant_statement, inference. */
|
|
14
|
-
bases: string[];
|
|
15
14
|
/** Index of the latest retained capture at this item, from 0; null before any capture. */
|
|
16
15
|
frame: number | null;
|
|
17
16
|
/** Time since the first retained capture, not a video offset. */
|
|
@@ -20,8 +19,19 @@ interface FindingEvidence {
|
|
|
20
19
|
/** The capture file relative to the project directory, or null for nonvisual evidence. */
|
|
21
20
|
capture: string | null;
|
|
22
21
|
}
|
|
22
|
+
interface FindingEvidence extends CitedEvidence {
|
|
23
|
+
/** How the finding's observations used this item: visual, action, participant_statement, inference. */
|
|
24
|
+
bases: string[];
|
|
25
|
+
}
|
|
23
26
|
interface FindingView {
|
|
24
27
|
id: string;
|
|
28
|
+
/**
|
|
29
|
+
* One plain sentence about what happened, and one to three about what the person tried and what
|
|
30
|
+
* got in the way. Null in analyses written before them. An amended finding keeps the original
|
|
31
|
+
* text here; `correction.replacementClaim` replaces both.
|
|
32
|
+
*/
|
|
33
|
+
headline: string | null;
|
|
34
|
+
experience: string | null;
|
|
25
35
|
title: string;
|
|
26
36
|
summary: string;
|
|
27
37
|
impact: Finding["impact"];
|
|
@@ -41,6 +51,22 @@ interface FindingView {
|
|
|
41
51
|
replacementClaim: string | null;
|
|
42
52
|
} | null;
|
|
43
53
|
}
|
|
54
|
+
interface DesignFindingView {
|
|
55
|
+
id: string;
|
|
56
|
+
headline: string;
|
|
57
|
+
screen: string;
|
|
58
|
+
notice: string;
|
|
59
|
+
whyItMatters: string;
|
|
60
|
+
suggestion: string;
|
|
61
|
+
severity: DesignFinding["severity"];
|
|
62
|
+
confidence: DesignFinding["confidence"];
|
|
63
|
+
seenBy: {
|
|
64
|
+
streamId: string;
|
|
65
|
+
label: string;
|
|
66
|
+
}[];
|
|
67
|
+
/** Each cited item once; at least one has a capture. */
|
|
68
|
+
evidence: CitedEvidence[];
|
|
69
|
+
}
|
|
44
70
|
export interface AnalysisFindings {
|
|
45
71
|
schema: typeof ANALYSIS_FINDINGS_SCHEMA;
|
|
46
72
|
runId: string;
|
|
@@ -66,6 +92,11 @@ export interface AnalysisFindings {
|
|
|
66
92
|
summary: string | null;
|
|
67
93
|
/** Highest priority first, as the analysis ranked them. */
|
|
68
94
|
findings: FindingView[];
|
|
95
|
+
/**
|
|
96
|
+
* Problems a designer would see in the captures, most severe first. Null when the view carries
|
|
97
|
+
* no analysis result, or the analysis was written before design findings.
|
|
98
|
+
*/
|
|
99
|
+
designFindings: DesignFindingView[] | null;
|
|
69
100
|
limitations: string[];
|
|
70
101
|
warnings: string[];
|
|
71
102
|
}
|
package/dist/cli/findings.js
CHANGED
|
@@ -27,16 +27,10 @@ function analyzeCommand(source, provider) {
|
|
|
27
27
|
? `${run} --provider codex`
|
|
28
28
|
: `${run} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
29
29
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
for (const observation of finding.observations)
|
|
33
|
-
for (const id of observation.evidenceIds) {
|
|
34
|
-
const set = bases.get(id) ?? new Set();
|
|
35
|
-
set.add(observation.basis);
|
|
36
|
-
bases.set(id, set);
|
|
37
|
-
}
|
|
30
|
+
/** The cited items the analysis evidence resolves, each once, in citation order. */
|
|
31
|
+
function citedEvidence(ids, analysis, runRoot) {
|
|
38
32
|
const byId = new Map(analysis.evidence.map((entry) => [entry.id, entry]));
|
|
39
|
-
return [...
|
|
33
|
+
return [...new Set(ids)].flatMap((id) => {
|
|
40
34
|
const entry = byId.get(id);
|
|
41
35
|
if (!entry)
|
|
42
36
|
return [];
|
|
@@ -45,7 +39,6 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
45
39
|
id,
|
|
46
40
|
streamId: entry.streamId,
|
|
47
41
|
kind: entry.kind,
|
|
48
|
-
bases: [...used],
|
|
49
42
|
frame: entry.frame,
|
|
50
43
|
elapsedMs: entry.elapsedMs,
|
|
51
44
|
at: entry.at,
|
|
@@ -54,11 +47,58 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
54
47
|
];
|
|
55
48
|
});
|
|
56
49
|
}
|
|
50
|
+
function evidenceOf(finding, analysis, runRoot) {
|
|
51
|
+
const bases = new Map();
|
|
52
|
+
for (const observation of finding.observations)
|
|
53
|
+
for (const id of observation.evidenceIds) {
|
|
54
|
+
const set = bases.get(id) ?? new Set();
|
|
55
|
+
set.add(observation.basis);
|
|
56
|
+
bases.set(id, set);
|
|
57
|
+
}
|
|
58
|
+
return citedEvidence(bases.keys(), analysis, runRoot).map(({ id, streamId, kind, ...entry }) => ({
|
|
59
|
+
id,
|
|
60
|
+
streamId,
|
|
61
|
+
kind,
|
|
62
|
+
bases: [...bases.get(id)],
|
|
63
|
+
...entry,
|
|
64
|
+
}));
|
|
65
|
+
}
|
|
66
|
+
const SEVERITY_RANK = {
|
|
67
|
+
major: 0,
|
|
68
|
+
moderate: 1,
|
|
69
|
+
minor: 2,
|
|
70
|
+
};
|
|
71
|
+
/** Most severe first; equal severity keeps the analysis order. */
|
|
72
|
+
function designFindingViews(analysis, runRoot) {
|
|
73
|
+
const designFindings = analysis.result?.designFindings;
|
|
74
|
+
if (designFindings === undefined)
|
|
75
|
+
return null;
|
|
76
|
+
const labels = new Map(analysis.participants.map((p) => [p.streamId, p.label]));
|
|
77
|
+
return [...designFindings]
|
|
78
|
+
.sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity])
|
|
79
|
+
.map((finding) => ({
|
|
80
|
+
id: finding.id,
|
|
81
|
+
headline: finding.headline,
|
|
82
|
+
screen: finding.screen,
|
|
83
|
+
notice: finding.notice,
|
|
84
|
+
whyItMatters: finding.whyItMatters,
|
|
85
|
+
suggestion: finding.suggestion,
|
|
86
|
+
severity: finding.severity,
|
|
87
|
+
confidence: finding.confidence,
|
|
88
|
+
seenBy: finding.seenByStreamIds.map((streamId) => ({
|
|
89
|
+
streamId,
|
|
90
|
+
label: labels.get(streamId) ?? streamId,
|
|
91
|
+
})),
|
|
92
|
+
evidence: citedEvidence(finding.evidenceIds, analysis, runRoot),
|
|
93
|
+
}));
|
|
94
|
+
}
|
|
57
95
|
function findingView(finding, loaded, runRoot) {
|
|
58
96
|
const labels = new Map(loaded.analysis.participants.map((p) => [p.streamId, p.label]));
|
|
59
97
|
const correction = loaded.corrections.filter((entry) => entry.findingId === finding.id).at(-1);
|
|
60
98
|
return {
|
|
61
99
|
id: finding.id,
|
|
100
|
+
headline: finding.headline ?? null,
|
|
101
|
+
experience: finding.experience ?? null,
|
|
62
102
|
title: finding.title,
|
|
63
103
|
summary: finding.summary,
|
|
64
104
|
impact: finding.impact,
|
|
@@ -99,6 +139,7 @@ function withoutFindings(source, state, reason, message, next) {
|
|
|
99
139
|
path: null,
|
|
100
140
|
summary: null,
|
|
101
141
|
findings: [],
|
|
142
|
+
designFindings: null,
|
|
102
143
|
limitations: [],
|
|
103
144
|
warnings: source.loaded.warnings,
|
|
104
145
|
};
|
|
@@ -142,6 +183,7 @@ export function analysisFindings(source) {
|
|
|
142
183
|
path: path.join(source.runRoot, "analysis", analysis.id, "analysis.json"),
|
|
143
184
|
summary: result.summary,
|
|
144
185
|
findings: result.findings.map((finding) => findingView(finding, { ...loaded, analysis }, source.runRoot)),
|
|
186
|
+
designFindings: designFindingViews(analysis, source.runRoot),
|
|
145
187
|
limitations: result.limitations,
|
|
146
188
|
warnings: loaded.warnings,
|
|
147
189
|
};
|
|
@@ -288,20 +330,49 @@ function momentsText(evidence) {
|
|
|
288
330
|
parts.push(`${plural(unframed, "item")} with no frame`);
|
|
289
331
|
return parts.join(", ");
|
|
290
332
|
}
|
|
333
|
+
/** The capture files, relative to the run directory, in frame order. */
|
|
334
|
+
function captureFiles(evidence, runRoot) {
|
|
335
|
+
return [
|
|
336
|
+
...new Set(evidence
|
|
337
|
+
.filter((entry) => entry.capture !== null)
|
|
338
|
+
.sort((a, b) => (a.frame ?? 0) - (b.frame ?? 0))
|
|
339
|
+
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
340
|
+
];
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* What a reader sees first: the headline, or the reviewer's claim when a review amended the finding,
|
|
344
|
+
* since the headline and experience describe the claim it replaced. Null before headlines existed.
|
|
345
|
+
*/
|
|
346
|
+
function leadOf(finding) {
|
|
347
|
+
const amended = finding.correction?.status === "amended" ? finding.correction : null;
|
|
348
|
+
if (finding.headline === null)
|
|
349
|
+
return null;
|
|
350
|
+
return amended?.replacementClaim
|
|
351
|
+
? { headline: amended.replacementClaim, corrected: true }
|
|
352
|
+
: { headline: finding.headline, corrected: false };
|
|
353
|
+
}
|
|
291
354
|
function findingLines(finding, runRoot) {
|
|
292
355
|
const cited = [...new Set(finding.evidence.map((entry) => entry.streamId))];
|
|
293
356
|
const labels = new Map(finding.affected.map((p) => [p.streamId, p.label]));
|
|
294
357
|
const affectedIds = finding.affected.map((p) => p.streamId);
|
|
295
358
|
const streams = [...affectedIds, ...cited.filter((id) => !labels.has(id))];
|
|
296
|
-
const captures =
|
|
297
|
-
...new Set(finding.evidence
|
|
298
|
-
.filter((entry) => entry.capture !== null)
|
|
299
|
-
.sort((a, b) => (a.frame ?? 0) - (b.frame ?? 0))
|
|
300
|
-
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
301
|
-
];
|
|
359
|
+
const captures = captureFiles(finding.evidence, runRoot);
|
|
302
360
|
const correction = finding.correction;
|
|
361
|
+
const lead = leadOf(finding);
|
|
303
362
|
return [
|
|
304
|
-
|
|
363
|
+
...(lead === null
|
|
364
|
+
? [`${finding.id} ${finding.title}`]
|
|
365
|
+
: [
|
|
366
|
+
`${finding.id} ${lead.headline}`,
|
|
367
|
+
...(lead.corrected
|
|
368
|
+
? [
|
|
369
|
+
" Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
370
|
+
]
|
|
371
|
+
: finding.experience === null
|
|
372
|
+
? []
|
|
373
|
+
: [` ${finding.experience}`]),
|
|
374
|
+
` evidence: ${finding.title}`,
|
|
375
|
+
]),
|
|
305
376
|
` impact: ${IMPACT_TEXT[finding.impact]} · confidence: ${finding.confidence} · recovery: ${RECOVERY_TEXT[finding.recovery]}`,
|
|
306
377
|
` ${finding.summary}`,
|
|
307
378
|
` affected: ${finding.affected.length} of ${plural(finding.exposedCount, "exposed participant")}`,
|
|
@@ -319,6 +390,34 @@ function findingLines(finding, runRoot) {
|
|
|
319
390
|
]),
|
|
320
391
|
];
|
|
321
392
|
}
|
|
393
|
+
function designFindingLines(finding, runRoot) {
|
|
394
|
+
const captures = captureFiles(finding.evidence, runRoot);
|
|
395
|
+
return [
|
|
396
|
+
`${finding.id} ${finding.severity}: ${finding.headline}`,
|
|
397
|
+
` screen: ${finding.screen}`,
|
|
398
|
+
` notice: ${finding.notice}`,
|
|
399
|
+
` why it matters: ${finding.whyItMatters}`,
|
|
400
|
+
` suggestion: ${finding.suggestion}`,
|
|
401
|
+
` seen by: ${finding.seenBy.map((p) => p.label).join(", ")} · confidence: ${finding.confidence}`,
|
|
402
|
+
...(captures.length === 0 ? [] : [` captures: ${captures.join(", ")}`]),
|
|
403
|
+
];
|
|
404
|
+
}
|
|
405
|
+
/** Nothing for an analysis written before design findings. */
|
|
406
|
+
function designSection(view) {
|
|
407
|
+
const design = view.designFindings;
|
|
408
|
+
if (design === null)
|
|
409
|
+
return [];
|
|
410
|
+
if (design.length === 0)
|
|
411
|
+
return ["", "design findings: none in the reviewed captures"];
|
|
412
|
+
return [
|
|
413
|
+
"",
|
|
414
|
+
`design findings: ${design.length}, most severe first`,
|
|
415
|
+
...design.flatMap((finding, index) => [
|
|
416
|
+
...(index === 0 ? [] : [""]),
|
|
417
|
+
...designFindingLines(finding, view.runPath ?? ""),
|
|
418
|
+
]),
|
|
419
|
+
];
|
|
420
|
+
}
|
|
322
421
|
/** The full findings block: the analysis, every finding with its evidence, and its limitations. */
|
|
323
422
|
export function formatFindings(view) {
|
|
324
423
|
if (view.state !== "ready" && view.state !== "stale")
|
|
@@ -330,6 +429,7 @@ export function formatFindings(view) {
|
|
|
330
429
|
...(view.state === "stale" ? [`warning: ${view.message}`, `next: ${view.next}`] : []),
|
|
331
430
|
...(view.summary ? [view.summary] : []),
|
|
332
431
|
...view.findings.flatMap((finding) => ["", ...findingLines(finding, view.runPath ?? "")]),
|
|
432
|
+
...designSection(view),
|
|
333
433
|
...(view.limitations.length === 0
|
|
334
434
|
? []
|
|
335
435
|
: ["", "limitations:", ...view.limitations.map((limitation) => `- ${limitation}`)]),
|
|
@@ -343,12 +443,30 @@ export function formatFindingsSummary(view, fullCommand, limit = 3) {
|
|
|
343
443
|
if (view.state !== "ready" && view.state !== "stale")
|
|
344
444
|
return [`findings: ${view.message}`, ...(view.next ? [`next: ${view.next}`] : [])];
|
|
345
445
|
const shown = view.findings.slice(0, limit);
|
|
446
|
+
const design = view.designFindings;
|
|
346
447
|
return [
|
|
347
448
|
`findings: ${view.findings.length === 0 ? "none" : view.findings.length}${view.state === "stale" ? " (stale)" : ""}`,
|
|
348
|
-
...shown.map((finding) =>
|
|
449
|
+
...shown.map((finding) => {
|
|
450
|
+
const qualities = `${IMPACT_TEXT[finding.impact]}, ${finding.confidence} confidence, ${RECOVERY_TEXT[finding.recovery]}`;
|
|
451
|
+
const lead = leadOf(finding);
|
|
452
|
+
if (lead === null)
|
|
453
|
+
return `- ${finding.id} ${qualities}: ${finding.title}`;
|
|
454
|
+
return `- ${finding.id} ${lead.headline} (${lead.corrected ? "corrected in human review" : qualities})`;
|
|
455
|
+
}),
|
|
349
456
|
...(view.findings.length > limit
|
|
350
457
|
? [`- and ${plural(view.findings.length - limit, "more finding")}`]
|
|
351
458
|
: []),
|
|
459
|
+
...(design === null
|
|
460
|
+
? []
|
|
461
|
+
: [
|
|
462
|
+
`design findings: ${design.length === 0 ? "none" : design.length}`,
|
|
463
|
+
...design
|
|
464
|
+
.slice(0, limit)
|
|
465
|
+
.map((finding) => `- ${finding.id} ${finding.severity}: ${finding.headline}`),
|
|
466
|
+
...(design.length > limit
|
|
467
|
+
? [`- and ${plural(design.length - limit, "more design finding")}`]
|
|
468
|
+
: []),
|
|
469
|
+
]),
|
|
352
470
|
`all findings: ${fullCommand}`,
|
|
353
471
|
];
|
|
354
472
|
}
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -18,7 +18,8 @@ export declare const DOTENV_OPTION_DESCRIPTION = "Load unset variables from this
|
|
|
18
18
|
* The hidden `--env-file <path>` that `--dotenv` replaced. Node scans the whole argv for
|
|
19
19
|
* `--env-file` before a script runs, and stops only at `--` (src/node_dotenv.cc), so a missing file
|
|
20
20
|
* exits 9 through Node and never reaches humanish. `--dotenv` does not collide.
|
|
21
|
-
* Removed in
|
|
21
|
+
* Removed in the first minor release on or after 2026-11-03: deprecated in 0.110.0 on 2026-10-03, it stays at least 30 days and one
|
|
22
|
+
* minor release.
|
|
22
23
|
*/
|
|
23
24
|
export declare function envFileAliasOption(): Option;
|
|
24
25
|
/** The dotenv path a command was given: `--dotenv`, or the `--env-file` alias with its warning. */
|