humanish 0.111.3 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
- package/dist/actors/computer-use/openai-conversation.js +209 -0
- package/dist/actors/computer-use/openai-provider.js +25 -41
- package/dist/actors/contract.d.ts +2 -0
- package/dist/actors/scripted-browser/actor.d.ts +5 -0
- package/dist/actors/scripted-browser/actor.js +40 -19
- package/dist/actors/scripted-browser/journey.d.ts +2 -0
- package/dist/actors/scripted-browser/journey.js +2 -1
- package/dist/actors/scripted-browser/steps.d.ts +2 -0
- package/dist/actors/scripted-browser/steps.js +17 -0
- package/dist/analysis/diagnostics.js +7 -12
- package/dist/analysis/execute.d.ts +1 -1
- package/dist/analysis/execute.js +35 -5
- package/dist/analysis/service.js +7 -1
- package/dist/analysis/types.d.ts +23 -0
- package/dist/analysis/validation.d.ts +80 -35
- package/dist/analysis/validation.js +58 -11
- package/dist/cli/commands/runs.js +4 -10
- package/dist/cli/findings.d.ts +49 -4
- package/dist/cli/findings.js +179 -23
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/evidence/secret-scrub.d.ts +7 -0
- package/dist/evidence/secret-scrub.js +21 -12
- package/dist/feedback/draft.js +14 -4
- package/dist/observer/run-routes.js +8 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/bundle.d.ts +1 -1
- package/dist/routes/computer-use/bundle.js +6 -16
- package/dist/routes/computer-use/fanout-bundle.js +40 -37
- package/dist/routes/computer-use/fanout-records.d.ts +2 -0
- package/dist/routes/computer-use/fanout-records.js +18 -26
- package/dist/routes/computer-use/participant-execution.js +4 -11
- package/dist/routes/computer-use/participant-facts.d.ts +2 -2
- package/dist/routes/computer-use/participant-facts.js +10 -1
- package/dist/routes/computer-use/result.js +7 -15
- package/dist/routes/computer-use/single-bundle.d.ts +2 -0
- package/dist/routes/computer-use/single-bundle.js +81 -37
- package/dist/routes/scripted/bundle.d.ts +1 -1
- package/dist/routes/scripted/bundle.js +9 -19
- package/dist/routes/scripted/route.js +1 -0
- package/dist/routes/scripted/setup.js +53 -10
- package/dist/routes/scripted/surface-records.d.ts +0 -1
- package/dist/routes/scripted/surface-records.js +27 -13
- package/dist/routes/scripted/surface-sessions.d.ts +1 -1
- package/dist/routes/scripted/surface-sessions.js +2 -0
- package/dist/routes/shared-world/bundle-records.d.ts +2 -2
- package/dist/routes/shared-world/bundle-records.js +21 -28
- package/dist/routes/shared-world/bundle.d.ts +1 -1
- package/dist/routes/shared-world/bundle.js +70 -78
- package/dist/routes/shared-world/participant-specs.d.ts +1 -2
- package/dist/routes/shared-world/participant-specs.js +0 -8
- package/dist/routes/shared-world/result.js +8 -12
- package/dist/routes/terminal/bundle.d.ts +1 -1
- package/dist/routes/terminal/bundle.js +10 -19
- package/dist/routes/terminal/credentials.js +14 -1
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +7 -4
- package/dist/routes/terminal/live-sandbox.js +9 -2
- package/dist/routes/terminal/plan.js +1 -0
- package/dist/routes/terminal/runtime.d.ts +1 -0
- package/dist/routes/terminal/runtime.js +1 -0
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/trace.d.ts +4 -2
- package/dist/routes/terminal/trace.js +30 -35
- package/dist/run/costs.js +9 -20
- package/dist/run/detail.d.ts +1 -1
- package/dist/run/detail.js +10 -1
- package/dist/run/display.d.ts +2 -2
- package/dist/run/display.js +15 -18
- package/dist/run/dry-run.js +10 -25
- package/dist/run/judge.d.ts +24 -0
- package/dist/run/judge.js +54 -0
- package/dist/run/liveness.d.ts +18 -0
- package/dist/run/liveness.js +24 -0
- package/dist/run/outcomes.d.ts +8 -1
- package/dist/run/outcomes.js +58 -12
- package/dist/run/participant-caption.d.ts +30 -0
- package/dist/run/participant-caption.js +50 -0
- package/dist/run/participant-records.d.ts +1 -1
- package/dist/run/review-markdown.d.ts +11 -0
- package/dist/run/review-markdown.js +22 -0
- package/dist/run/run-cost.d.ts +7 -0
- package/dist/run/run-cost.js +25 -5
- package/dist/run/run-index.d.ts +4 -50
- package/dist/run/run-index.js +57 -87
- package/dist/run/run.d.ts +2 -2
- package/dist/run/run.js +5 -3
- package/dist/run/secrets.d.ts +15 -2
- package/dist/run/secrets.js +134 -9
- package/dist/run/stats.js +2 -12
- package/dist/run/transient-comms-secrets.d.ts +8 -3
- package/dist/run/transient-comms-secrets.js +21 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -1
- package/dist/study/parse/execution.js +7 -1
- package/dist/study/plan-types.d.ts +1 -0
- package/dist/study/requirements.js +2 -1
- package/dist/study/types.d.ts +4 -1
- package/dist/study/warnings.d.ts +2 -1
- package/dist/study/warnings.js +3 -2
- package/dist/tui-app.js +73 -73
- package/dist/verify/liveness.d.ts +2 -2
- package/dist/verify/liveness.js +7 -4
- package/docs/architecture/terminal-product-route.md +14 -0
- package/docs/contracts/run-bundle.md +10 -0
- package/docs/contracts/schemas.md +9 -1
- package/docs/contracts/study-analysis.md +76 -23
- package/package.json +2 -1
- package/dist/actors/computer-use/openai-requests.d.ts +0 -62
- package/dist/actors/computer-use/openai-requests.js +0 -144
|
@@ -7,3 +7,5 @@ export declare function parseBrowserPersonaJourneyFromScenario(args: {
|
|
|
7
7
|
failure?: string;
|
|
8
8
|
journey?: BrowserPersonaJourney;
|
|
9
9
|
};
|
|
10
|
+
/** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
|
|
11
|
+
export declare function publicSafeToken(value: string | undefined, fallback: string): string;
|
|
@@ -172,7 +172,8 @@ function stringValue(value) {
|
|
|
172
172
|
function booleanValue(value) {
|
|
173
173
|
return typeof value === "boolean" ? value : undefined;
|
|
174
174
|
}
|
|
175
|
-
|
|
175
|
+
/** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
|
|
176
|
+
export function publicSafeToken(value, fallback) {
|
|
176
177
|
const candidate = (value ?? fallback)
|
|
177
178
|
.toLowerCase()
|
|
178
179
|
.replace(/[^a-z0-9_.-]+/g, "-")
|
|
@@ -18,6 +18,8 @@ export declare function buildBlockedBrowserPersonaSteps(args: {
|
|
|
18
18
|
timestamp: string;
|
|
19
19
|
urlPolicy?: ScriptedBrowserEvidenceUrlPolicy;
|
|
20
20
|
}): BrowserPersonaStepCapture[];
|
|
21
|
+
/** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
|
|
22
|
+
export declare function scrubRecordedStep(step: BrowserPersonaStepCapture, scrub: (text: string) => string): BrowserPersonaStepCapture;
|
|
21
23
|
export declare function screenshotPathForBrowserStep(surface: BrowserSurface, step: BrowserPersonaStepManifest | undefined): string;
|
|
22
24
|
export declare function tracePathForBrowserSurface(surface: BrowserSurface): string;
|
|
23
25
|
/**
|
|
@@ -191,6 +191,23 @@ export function buildBlockedBrowserPersonaSteps(args) {
|
|
|
191
191
|
url: sanitizeBrowserEvidenceUrl(args.currentUrl, args.urlPolicy),
|
|
192
192
|
}));
|
|
193
193
|
}
|
|
194
|
+
/** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
|
|
195
|
+
export function scrubRecordedStep(step, scrub) {
|
|
196
|
+
return {
|
|
197
|
+
...step,
|
|
198
|
+
...(step.assertions === undefined
|
|
199
|
+
? {}
|
|
200
|
+
: {
|
|
201
|
+
assertions: step.assertions.map((assertion) => ({
|
|
202
|
+
...assertion,
|
|
203
|
+
reason: scrub(assertion.reason),
|
|
204
|
+
})),
|
|
205
|
+
}),
|
|
206
|
+
label: scrub(step.label),
|
|
207
|
+
reason: scrub(step.reason),
|
|
208
|
+
url: scrub(step.url),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
194
211
|
export function screenshotPathForBrowserStep(surface, step) {
|
|
195
212
|
assertSafeOutputPathSegment(surface.id, "Browser surface id");
|
|
196
213
|
if (step) {
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { lstat, readdir, rmdir, unlink } from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import { decodeEscapes } from "../evidence/encoded-text.js";
|
|
4
3
|
import { redactText } from "../evidence/redaction.js";
|
|
5
4
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, bindExistingManagedHumanishOutputDirectory, prepareManagedHumanishOutputDirectory, writeContainedOutputFile, } from "../run/contained-output.js";
|
|
6
|
-
import {
|
|
5
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
7
6
|
import { ANALYSIS_ID_PATTERN } from "./types.js";
|
|
8
7
|
// A rejected analyst response is kept for diagnosis under .humanish/analysis-diagnostics/<run>/,
|
|
9
8
|
// outside every run directory. Export, verify and the Observer read run directories only, so the
|
|
@@ -14,19 +13,15 @@ const ANALYSIS_DIAGNOSTIC_SCHEMA = "humanish.analysis-rejected-output.v1";
|
|
|
14
13
|
/** How many rejected outputs are kept across all runs; older ones are removed on each write. */
|
|
15
14
|
export const MAX_ANALYSIS_DIAGNOSTICS = 20;
|
|
16
15
|
const REDACTED = "[REDACTED_SECRET]";
|
|
17
|
-
const markers = (text) => text.split(REDACTED).length - 1;
|
|
18
16
|
/**
|
|
19
|
-
* Scrub every string, key and scalar in the output. The
|
|
20
|
-
* percent-encoded, escaped or base64-encoded
|
|
21
|
-
*
|
|
22
|
-
*
|
|
17
|
+
* Scrub every string, key and scalar in the output. The known-value scrub finds a value written
|
|
18
|
+
* percent-encoded, escaped or base64-encoded; a string where it finds nothing keeps its original
|
|
19
|
+
* spelling, so an escape in a rejected quote stays visible. A number or boolean equal to a known
|
|
20
|
+
* value becomes the marker.
|
|
23
21
|
*/
|
|
24
22
|
function scrubber() {
|
|
25
|
-
const
|
|
26
|
-
const text = (value) =>
|
|
27
|
-
const found = encoded(value);
|
|
28
|
-
return redactText(markers(found) > markers(decodeEscapes(value)) ? found : scrubTransientCommsText(value));
|
|
29
|
-
};
|
|
23
|
+
const known = transientCommsKnownValueScrub();
|
|
24
|
+
const text = (value) => redactText(known(value));
|
|
30
25
|
const scrub = (value) => {
|
|
31
26
|
if (typeof value === "string")
|
|
32
27
|
return text(value);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
|
|
2
2
|
import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
|
|
3
3
|
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
4
|
-
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-
|
|
4
|
+
export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-7";
|
|
5
5
|
export declare const isSupportedAnalysisModel: (model: string) => boolean;
|
|
6
6
|
/** The OpenAI output limit when a study omits maxOutputTokens, and the most it may ask for. */
|
|
7
7
|
export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
|
package/dist/analysis/execute.js
CHANGED
|
@@ -3,14 +3,14 @@ import { highDetailImageTokens } from "./image-tokens.js";
|
|
|
3
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
4
|
import { estimateActorCost, MODEL_RATES } from "../run/pricing.js";
|
|
5
5
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
6
|
-
import {
|
|
6
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
7
7
|
import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
|
|
8
8
|
import { INPUT_IMAGE_DATA_URL, createAnalysisProvider, } from "./provider.js";
|
|
9
9
|
import { checkAnalysisResult, hashAnalysisValue, analysisResponseSchema, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
|
|
10
10
|
import { EVIDENCE_LIMITS } from "./evidence.js";
|
|
11
11
|
import { truncatedFrameWarning, unknownNotificationsWarning, } from "../actors/codex/restricted-notifications.js";
|
|
12
12
|
import { protocolAdditionsWarning, protocolIncompatibilityMessage, } from "../actors/codex/protocol-compat.js";
|
|
13
|
-
export const ANALYSIS_PROMPT_VERSION = "study-evidence-
|
|
13
|
+
export const ANALYSIS_PROMPT_VERSION = "study-evidence-7";
|
|
14
14
|
const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
|
|
15
15
|
"gpt-6-astra",
|
|
16
16
|
"gpt-5.5",
|
|
@@ -55,7 +55,11 @@ Harness records describe the machinery running the study. Provisioning, runtime
|
|
|
55
55
|
|
|
56
56
|
For each finding, consolidate repeated observations into one bounded problem or recovery. Each observation states one claim, labels its basis (visual, action, participant_statement, inference), cites supporting evidence, and declares its limitation. High confidence does not turn an inference into an observation. Show which distinct participants were affected and which were demonstrably exposed to the relevant interaction. Affected IDs must be a subset of exposed IDs, and both sets must be within included participants. Explain exposure; never automatically use the full panel as a denominator. Each affected participant must have supporting observation evidence.
|
|
57
57
|
|
|
58
|
-
Keep every field as qualified as its evidence. This includes
|
|
58
|
+
Keep every evidence field as qualified as its evidence. This includes titles, summaries, outcomes, impact, exposure, recovery, confidence and the premises of proposed next checks. A cautious observation cannot support an unqualified title. When a useful concern is established only as participant feedback, say it was reported wherever the concern is summarized. A dispatched action establishes an attempt; its success needs resulting evidence. Use recovered only when the improvement is supported for the participants being grouped. If recovery differs across participants, describe those differences and use unknown for the combined recovery instead of erasing an unresolved case. Keep distinct problems separate, especially an unresolved result and a different problem that was corrected. Reversible exploration and unmeasured pauses are not automatically product defects.
|
|
59
|
+
|
|
60
|
+
Write for the person who will act on the report: a designer, product manager or developer who did not watch the session and does not know this tool's vocabulary. Give every finding a headline and an experience. The headline is one plain sentence about what happened to the people involved, as a colleague would say it out loud. The experience is one to three plain sentences: what the person was trying to do, what got in their way, and how it seemed to feel, using the participant's own words where a quote exists. Do not use participant stream IDs, evidence IDs, or this report's evidence vocabulary (exposure, exposed, stream, capture, corroborated, established, basis, provenance) in the headline or experience. Plain is not unqualified: a concern known only from what participants said reads as something they said ("Two players said they lost track of which control had focus"), and a cause the evidence does not show is not stated as fact. The title, summary and observations keep their current precision underneath.
|
|
61
|
+
|
|
62
|
+
Separately, review the supplied captures as an experienced product designer would and report designFindings: problems a designer would notice in what the screens show, whether or not any participant mentioned them. Look at the size of text and controls (too small to read or to hit comfortably at this window size), layout and density (spread too wide or packed too tight for the task), visual hierarchy, legibility and contrast, wording and labels, consistency between screens, feedback after actions, and anything that makes the product look unfinished or untrustworthy. Judge the screens as the participants' personas would meet them: who they are and what they came to do. Each design finding names the screen in plain words, says what a designer notices, why it matters to a person using the product, and one concrete suggestion. Each must cite at least one supplied capture that shows the problem, and seenByStreamIds lists only participants whose cited captures show it. Rate severity by the effect on a person using the product: major when it misleads or blocks, moderate when it slows or confuses, minor when it is polish. Report only what the captures show; no capture, no design finding. Use D1, D2, and so on as IDs. An empty designFindings array is correct when the captures show no design problem worth a designer's time. Write design findings in the same plain register as headlines.
|
|
59
63
|
|
|
60
64
|
Order findings by observed task impact, replication among exposed participants, and recovery. Preserve severity and confidence as separate fields. Do not compute a numeric frustration score or universal priority score. Use F1, F2, and so on as local finding IDs. Explain ordering with concrete evidence. Provide a short, testable next check for each finding. Say when recovery was not observed rather than claiming it was impossible. Keep titles and summaries concise and specific. The overall summary must be grounded in participant reviews and observations, including successful outcomes and evidence limits.`;
|
|
61
65
|
function instructions(config) {
|
|
@@ -213,9 +217,10 @@ export function preferLargerAnalysisOutput(input, config) {
|
|
|
213
217
|
const expanded = { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS };
|
|
214
218
|
return estimateAnalysisAdmission(input, expanded).allowed ? expanded : config;
|
|
215
219
|
}
|
|
216
|
-
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact.
|
|
220
|
+
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
|
|
221
|
+
* known value is found as written and in its encoded forms. */
|
|
217
222
|
function scrubGeneratedNarrative(result) {
|
|
218
|
-
const scrub =
|
|
223
|
+
const scrub = transientCommsKnownValueScrub();
|
|
219
224
|
const observation = (value) => ({
|
|
220
225
|
...value,
|
|
221
226
|
claim: scrub(value.claim),
|
|
@@ -239,6 +244,13 @@ function scrubGeneratedNarrative(result) {
|
|
|
239
244
|
...value.exposedStreamIds,
|
|
240
245
|
...value.observations.flatMap((item) => [item.basis, ...item.evidenceIds]),
|
|
241
246
|
]),
|
|
247
|
+
...(result.designFindings ?? []).flatMap((value) => [
|
|
248
|
+
value.id,
|
|
249
|
+
value.severity,
|
|
250
|
+
value.confidence,
|
|
251
|
+
...value.seenByStreamIds,
|
|
252
|
+
...value.evidenceIds,
|
|
253
|
+
]),
|
|
242
254
|
...(result.concernReviews ?? []).flatMap((value) => [
|
|
243
255
|
value.basis,
|
|
244
256
|
value.disposition,
|
|
@@ -263,12 +275,26 @@ function scrubGeneratedNarrative(result) {
|
|
|
263
275
|
findings: result.findings.map((value) => ({
|
|
264
276
|
...value,
|
|
265
277
|
title: scrub(value.title),
|
|
278
|
+
...(value.headline === undefined ? {} : { headline: scrub(value.headline) }),
|
|
279
|
+
...(value.experience === undefined ? {} : { experience: scrub(value.experience) }),
|
|
266
280
|
summary: scrub(value.summary),
|
|
267
281
|
exposureReason: scrub(value.exposureReason),
|
|
268
282
|
nextStep: scrub(value.nextStep),
|
|
269
283
|
priorityReason: scrub(value.priorityReason),
|
|
270
284
|
observations: value.observations.map(observation),
|
|
271
285
|
})),
|
|
286
|
+
...(result.designFindings === undefined
|
|
287
|
+
? {}
|
|
288
|
+
: {
|
|
289
|
+
designFindings: result.designFindings.map((value) => ({
|
|
290
|
+
...value,
|
|
291
|
+
headline: scrub(value.headline),
|
|
292
|
+
screen: scrub(value.screen),
|
|
293
|
+
notice: scrub(value.notice),
|
|
294
|
+
whyItMatters: scrub(value.whyItMatters),
|
|
295
|
+
suggestion: scrub(value.suggestion),
|
|
296
|
+
})),
|
|
297
|
+
}),
|
|
272
298
|
...(result.concernReviews === undefined
|
|
273
299
|
? {}
|
|
274
300
|
: {
|
|
@@ -281,6 +307,10 @@ function scrubGeneratedNarrative(result) {
|
|
|
281
307
|
}
|
|
282
308
|
const VALIDATION_FAILURES = Object.freeze({
|
|
283
309
|
ANALYSIS_RESULT_SCHEMA_INVALID: "analysis_validation_failed_schema_invalid",
|
|
310
|
+
ANALYSIS_DESIGN_FINDING_ID_DUPLICATE: "analysis_validation_failed_design_finding_id_duplicate",
|
|
311
|
+
ANALYSIS_DESIGN_REFERENCE_INVALID: "analysis_validation_failed_design_reference_invalid",
|
|
312
|
+
ANALYSIS_DESIGN_WITHOUT_CAPTURE: "analysis_validation_failed_design_without_capture",
|
|
313
|
+
ANALYSIS_DESIGN_MEMBERSHIP_INVALID: "analysis_validation_failed_design_membership_invalid",
|
|
284
314
|
ANALYSIS_INPUT_DUPLICATES: "analysis_validation_failed_input_duplicates",
|
|
285
315
|
ANALYSIS_PARTICIPANT_COVERAGE_INVALID: "analysis_validation_failed_participant_coverage_invalid",
|
|
286
316
|
ANALYSIS_PARTICIPANT_REFERENCE_INVALID: "analysis_validation_failed_participant_reference_invalid",
|
package/dist/analysis/service.js
CHANGED
|
@@ -5,6 +5,7 @@ import { lstat, mkdir, realpath, rmdir } from "node:fs/promises";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { renderObserver } from "../observer/render.js";
|
|
7
7
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
8
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
8
9
|
import { verifyRunPrepared } from "../verify/verify.js";
|
|
9
10
|
import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
10
11
|
import { physicalCwdOf, runIdOf, resolvePhysicalCwd, validatePreparedRunRootIdentity, } from "../run/paths.js";
|
|
@@ -397,7 +398,12 @@ export async function correctAnalysis(cwd, run, options) {
|
|
|
397
398
|
const finding = analysis?.result?.findings.find((item) => item.id === options.findingId);
|
|
398
399
|
if (loaded.state !== "ready" || !analysis || !finding)
|
|
399
400
|
throw new Error("ANALYSIS_CORRECTION_SOURCE_UNAVAILABLE");
|
|
400
|
-
|
|
401
|
+
// A reviewer's claim replaces the finding's headline in every view, so it gets the same known
|
|
402
|
+
// value check as generated text: as written and encoded. Only a caller inside the run's secret
|
|
403
|
+
// scope holds those values; a separate `analyze correct` has only the pattern check.
|
|
404
|
+
const known = transientCommsKnownValueScrub();
|
|
405
|
+
const unsafe = (text) => containsSensitive(text) || known(text) !== text;
|
|
406
|
+
if (unsafe(options.reason) || unsafe(options.replacementClaim ?? ""))
|
|
401
407
|
throw new Error("ANALYSIS_CORRECTION_TEXT_UNSAFE");
|
|
402
408
|
const correction = {
|
|
403
409
|
schema: ANALYSIS_CORRECTION_SCHEMA,
|
package/dist/analysis/types.d.ts
CHANGED
|
@@ -104,6 +104,11 @@ export interface AnalysisObservation {
|
|
|
104
104
|
interface AnalysisFinding {
|
|
105
105
|
id: string;
|
|
106
106
|
title: string;
|
|
107
|
+
/** One plain sentence for a reader who did not watch the session. Absent in older reports. */
|
|
108
|
+
headline?: string;
|
|
109
|
+
/** One to three plain sentences: what the person tried, what got in the way, how it seemed to
|
|
110
|
+
* feel. Absent in older reports. */
|
|
111
|
+
experience?: string;
|
|
107
112
|
summary: string;
|
|
108
113
|
impact: "blocked_task" | "friction" | "recovery" | "uncertain";
|
|
109
114
|
affectedStreamIds: string[];
|
|
@@ -115,6 +120,22 @@ interface AnalysisFinding {
|
|
|
115
120
|
nextStep: string;
|
|
116
121
|
priorityReason: string;
|
|
117
122
|
}
|
|
123
|
+
/** A problem a product designer would see in the cited captures, whether or not a participant
|
|
124
|
+
* mentioned it. It cites at least one capture. */
|
|
125
|
+
interface AnalysisDesignFinding {
|
|
126
|
+
id: string;
|
|
127
|
+
headline: string;
|
|
128
|
+
/** The screen, in plain words. */
|
|
129
|
+
screen: string;
|
|
130
|
+
notice: string;
|
|
131
|
+
whyItMatters: string;
|
|
132
|
+
suggestion: string;
|
|
133
|
+
severity: "minor" | "moderate" | "major";
|
|
134
|
+
confidence: "low" | "medium" | "high";
|
|
135
|
+
/** Participants whose cited evidence shows the problem. */
|
|
136
|
+
seenByStreamIds: string[];
|
|
137
|
+
evidenceIds: string[];
|
|
138
|
+
}
|
|
118
139
|
/** Evidence-grounded disposition of a material concern, not model reasoning. */
|
|
119
140
|
interface AnalysisConcernReview extends AnalysisObservation {
|
|
120
141
|
disposition: "finding" | "context" | "unsupported";
|
|
@@ -128,6 +149,8 @@ export interface AnalysisResult {
|
|
|
128
149
|
concernReviews?: AnalysisConcernReview[];
|
|
129
150
|
/** Highest priority first; counts are derived from distinct stream sets. */
|
|
130
151
|
findings: AnalysisFinding[];
|
|
152
|
+
/** Absent in older reports. New analyses supply it, empty when the captures show no problem. */
|
|
153
|
+
designFindings?: AnalysisDesignFinding[];
|
|
131
154
|
limitations: string[];
|
|
132
155
|
}
|
|
133
156
|
interface OpenAIAnalysisConfig {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { type AnalysisArtifact, type AnalysisCorrection, type AnalysisInput, type AnalysisResult } from "./types.js";
|
|
3
|
-
/** Historical artifacts may omit concernReviews
|
|
3
|
+
/** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
|
|
4
|
+
* new provider response must supply them. */
|
|
4
5
|
export declare const analysisResponseSchema: z.ZodObject<{
|
|
5
6
|
summary: z.ZodString;
|
|
6
7
|
participants: z.ZodArray<z.ZodObject<{
|
|
@@ -22,9 +23,30 @@ export declare const analysisResponseSchema: z.ZodObject<{
|
|
|
22
23
|
}, z.core.$strict>>;
|
|
23
24
|
limitations: z.ZodArray<z.ZodString>;
|
|
24
25
|
}, z.core.$strict>>;
|
|
26
|
+
concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
27
|
+
claim: z.ZodString;
|
|
28
|
+
basis: z.ZodEnum<{
|
|
29
|
+
action: "action";
|
|
30
|
+
inference: "inference";
|
|
31
|
+
participant_statement: "participant_statement";
|
|
32
|
+
visual: "visual";
|
|
33
|
+
}>;
|
|
34
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
35
|
+
limitation: z.ZodString;
|
|
36
|
+
disposition: z.ZodEnum<{
|
|
37
|
+
context: "context";
|
|
38
|
+
finding: "finding";
|
|
39
|
+
unsupported: "unsupported";
|
|
40
|
+
}>;
|
|
41
|
+
findingId: z.ZodNullable<z.ZodString>;
|
|
42
|
+
reason: z.ZodString;
|
|
43
|
+
}, z.core.$strict>>>>;
|
|
44
|
+
limitations: z.ZodArray<z.ZodString>;
|
|
25
45
|
findings: z.ZodArray<z.ZodObject<{
|
|
26
46
|
id: z.ZodString;
|
|
27
47
|
title: z.ZodString;
|
|
48
|
+
headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
49
|
+
experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
28
50
|
summary: z.ZodString;
|
|
29
51
|
impact: z.ZodEnum<{
|
|
30
52
|
blocked_task: "blocked_task";
|
|
@@ -59,25 +81,26 @@ export declare const analysisResponseSchema: z.ZodObject<{
|
|
|
59
81
|
nextStep: z.ZodString;
|
|
60
82
|
priorityReason: z.ZodString;
|
|
61
83
|
}, z.core.$strict>>;
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
84
|
+
designFindings: z.ZodArray<z.ZodObject<{
|
|
85
|
+
id: z.ZodString;
|
|
86
|
+
headline: z.ZodString;
|
|
87
|
+
screen: z.ZodString;
|
|
88
|
+
notice: z.ZodString;
|
|
89
|
+
whyItMatters: z.ZodString;
|
|
90
|
+
suggestion: z.ZodString;
|
|
91
|
+
severity: z.ZodEnum<{
|
|
92
|
+
major: "major";
|
|
93
|
+
minor: "minor";
|
|
94
|
+
moderate: "moderate";
|
|
69
95
|
}>;
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
finding: "finding";
|
|
75
|
-
unsupported: "unsupported";
|
|
96
|
+
confidence: z.ZodEnum<{
|
|
97
|
+
high: "high";
|
|
98
|
+
low: "low";
|
|
99
|
+
medium: "medium";
|
|
76
100
|
}>;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}, z.core.$strict
|
|
80
|
-
limitations: z.ZodArray<z.ZodString>;
|
|
101
|
+
seenByStreamIds: z.ZodArray<z.ZodString>;
|
|
102
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
103
|
+
}, z.core.$strict>>;
|
|
81
104
|
}, z.core.$strict>;
|
|
82
105
|
export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
|
|
83
106
|
summary: z.ZodString;
|
|
@@ -100,9 +123,30 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
|
|
|
100
123
|
}, z.core.$strict>>;
|
|
101
124
|
limitations: z.ZodArray<z.ZodString>;
|
|
102
125
|
}, z.core.$strict>>;
|
|
126
|
+
concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
127
|
+
claim: z.ZodString;
|
|
128
|
+
basis: z.ZodEnum<{
|
|
129
|
+
action: "action";
|
|
130
|
+
inference: "inference";
|
|
131
|
+
participant_statement: "participant_statement";
|
|
132
|
+
visual: "visual";
|
|
133
|
+
}>;
|
|
134
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
135
|
+
limitation: z.ZodString;
|
|
136
|
+
disposition: z.ZodEnum<{
|
|
137
|
+
context: "context";
|
|
138
|
+
finding: "finding";
|
|
139
|
+
unsupported: "unsupported";
|
|
140
|
+
}>;
|
|
141
|
+
findingId: z.ZodNullable<z.ZodString>;
|
|
142
|
+
reason: z.ZodString;
|
|
143
|
+
}, z.core.$strict>>>>;
|
|
144
|
+
limitations: z.ZodArray<z.ZodString>;
|
|
103
145
|
findings: z.ZodArray<z.ZodObject<{
|
|
104
146
|
id: z.ZodString;
|
|
105
147
|
title: z.ZodString;
|
|
148
|
+
headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
149
|
+
experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
|
|
106
150
|
summary: z.ZodString;
|
|
107
151
|
impact: z.ZodEnum<{
|
|
108
152
|
blocked_task: "blocked_task";
|
|
@@ -137,25 +181,26 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
|
|
|
137
181
|
nextStep: z.ZodString;
|
|
138
182
|
priorityReason: z.ZodString;
|
|
139
183
|
}, z.core.$strict>>;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
184
|
+
designFindings: z.ZodArray<z.ZodObject<{
|
|
185
|
+
id: z.ZodString;
|
|
186
|
+
headline: z.ZodString;
|
|
187
|
+
screen: z.ZodString;
|
|
188
|
+
notice: z.ZodString;
|
|
189
|
+
whyItMatters: z.ZodString;
|
|
190
|
+
suggestion: z.ZodString;
|
|
191
|
+
severity: z.ZodEnum<{
|
|
192
|
+
major: "major";
|
|
193
|
+
minor: "minor";
|
|
194
|
+
moderate: "moderate";
|
|
147
195
|
}>;
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
finding: "finding";
|
|
153
|
-
unsupported: "unsupported";
|
|
196
|
+
confidence: z.ZodEnum<{
|
|
197
|
+
high: "high";
|
|
198
|
+
low: "low";
|
|
199
|
+
medium: "medium";
|
|
154
200
|
}>;
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
}, z.core.$strict
|
|
158
|
-
limitations: z.ZodArray<z.ZodString>;
|
|
201
|
+
seenByStreamIds: z.ZodArray<z.ZodString>;
|
|
202
|
+
evidenceIds: z.ZodArray<z.ZodString>;
|
|
203
|
+
}, z.core.$strict>>;
|
|
159
204
|
}, z.core.$strict>>;
|
|
160
205
|
/** Stable hashing is independent of object insertion order and excludes no fields implicitly. */
|
|
161
206
|
export declare function hashAnalysisValue(value: unknown): string;
|
|
@@ -27,6 +27,20 @@ const observationSchema = z
|
|
|
27
27
|
limitation: text(2000),
|
|
28
28
|
})
|
|
29
29
|
.strict();
|
|
30
|
+
const designFindingSchema = z
|
|
31
|
+
.object({
|
|
32
|
+
id,
|
|
33
|
+
headline: label,
|
|
34
|
+
screen: label,
|
|
35
|
+
notice: text(1500).min(1),
|
|
36
|
+
whyItMatters: text(1000).min(1),
|
|
37
|
+
suggestion: text(1000).min(1),
|
|
38
|
+
severity: z.enum(["minor", "moderate", "major"]),
|
|
39
|
+
confidence: z.enum(["low", "medium", "high"]),
|
|
40
|
+
seenByStreamIds: sourceIds.min(1),
|
|
41
|
+
evidenceIds: refs,
|
|
42
|
+
})
|
|
43
|
+
.strict();
|
|
30
44
|
const analysisResultSchema = z
|
|
31
45
|
.object({
|
|
32
46
|
summary: text(4000).min(1),
|
|
@@ -51,6 +65,8 @@ const analysisResultSchema = z
|
|
|
51
65
|
.object({
|
|
52
66
|
id,
|
|
53
67
|
title: label,
|
|
68
|
+
headline: label.optional(),
|
|
69
|
+
experience: text(1200).min(1).optional(),
|
|
54
70
|
summary: text(4000).min(1),
|
|
55
71
|
impact: z.enum(["blocked_task", "friction", "recovery", "uncertain"]),
|
|
56
72
|
affectedStreamIds: sourceIds.min(1),
|
|
@@ -64,6 +80,7 @@ const analysisResultSchema = z
|
|
|
64
80
|
})
|
|
65
81
|
.strict())
|
|
66
82
|
.max(100),
|
|
83
|
+
designFindings: z.array(designFindingSchema).max(40).optional(),
|
|
67
84
|
concernReviews: z
|
|
68
85
|
.array(observationSchema
|
|
69
86
|
.extend({
|
|
@@ -77,13 +94,25 @@ const analysisResultSchema = z
|
|
|
77
94
|
limitations,
|
|
78
95
|
})
|
|
79
96
|
.strict();
|
|
80
|
-
/** Historical artifacts may omit concernReviews
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
97
|
+
/** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
|
|
98
|
+
* new provider response must supply them. */
|
|
99
|
+
export const analysisResponseSchema = analysisResultSchema
|
|
100
|
+
.extend({
|
|
101
|
+
findings: z
|
|
102
|
+
.array(analysisResultSchema.shape.findings.element.required({ headline: true, experience: true }))
|
|
103
|
+
.max(100),
|
|
104
|
+
designFindings: z.array(designFindingSchema).max(40),
|
|
105
|
+
})
|
|
106
|
+
.required({ concernReviews: true });
|
|
84
107
|
export const analysisResultJsonSchema = z.toJSONSchema(analysisResponseSchema);
|
|
85
|
-
const normalizedResult = ({ concernReviews, ...result }) => ({
|
|
108
|
+
const normalizedResult = ({ concernReviews, designFindings, findings, ...result }) => ({
|
|
86
109
|
...result,
|
|
110
|
+
findings: findings.map(({ headline, experience, ...finding }) => ({
|
|
111
|
+
...finding,
|
|
112
|
+
...(headline === undefined ? {} : { headline }),
|
|
113
|
+
...(experience === undefined ? {} : { experience }),
|
|
114
|
+
})),
|
|
115
|
+
...(designFindings === undefined ? {} : { designFindings }),
|
|
87
116
|
...(concernReviews === undefined ? {} : { concernReviews }),
|
|
88
117
|
});
|
|
89
118
|
const analysisCoverageSchema = z
|
|
@@ -349,6 +378,21 @@ export function checkAnalysisResult(input, value) {
|
|
|
349
378
|
if (finding.affectedStreamIds.some((stream) => !citedStreams.has(stream)))
|
|
350
379
|
errors.add("ANALYSIS_AFFECTED_WITHOUT_EVIDENCE");
|
|
351
380
|
}
|
|
381
|
+
const designFindings = result.designFindings ?? [];
|
|
382
|
+
if (!distinct(designFindings.map((finding) => finding.id)))
|
|
383
|
+
errors.add("ANALYSIS_DESIGN_FINDING_ID_DUPLICATE");
|
|
384
|
+
for (const finding of designFindings) {
|
|
385
|
+
const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
|
|
386
|
+
if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
|
|
387
|
+
errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
|
|
388
|
+
if (!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
|
|
389
|
+
errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
|
|
390
|
+
// Seen by means a cited capture of that participant shows the problem.
|
|
391
|
+
const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
|
|
392
|
+
if (!distinct(finding.seenByStreamIds) ||
|
|
393
|
+
finding.seenByStreamIds.some((stream) => !included.has(stream) || !captured.has(stream)))
|
|
394
|
+
errors.add("ANALYSIS_DESIGN_MEMBERSHIP_INVALID");
|
|
395
|
+
}
|
|
352
396
|
for (const review of result.concernReviews ?? []) {
|
|
353
397
|
const citedStreams = checkObservation(review);
|
|
354
398
|
const finding = result.findings.find((item) => item.id === review.findingId);
|
|
@@ -425,12 +469,15 @@ export function validateAnalysisArtifact(value) {
|
|
|
425
469
|
(artifact.status === "complete" && !coverage.complete))
|
|
426
470
|
throw new Error("ANALYSIS_STATUS_INVALID");
|
|
427
471
|
if (artifact.result !== null) {
|
|
428
|
-
// Concern accounting became required with revision 5
|
|
429
|
-
// when the prompt version advances; a later
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
472
|
+
// Concern accounting became required with revision 5, and headlines, experiences and design
|
|
473
|
+
// findings with revision 7. Keep each boundary stable when the prompt version advances; a later
|
|
474
|
+
// prompt must not regain legacy omissions.
|
|
475
|
+
const revision = Number(/^study-evidence-(\d+)$/.exec(artifact.promptVersion)?.[1] ?? 0);
|
|
476
|
+
const result = artifact.result;
|
|
477
|
+
if ((revision >= 5 && result.concernReviews === undefined) ||
|
|
478
|
+
(revision >= 7 &&
|
|
479
|
+
(result.designFindings === undefined ||
|
|
480
|
+
result.findings.some((finding) => finding.headline === undefined || finding.experience === undefined))))
|
|
434
481
|
throw new Error("ANALYSIS_RESULT_SCHEMA_INVALID");
|
|
435
482
|
validateAnalysisResult({ ...artifact, images: [] }, artifact.result);
|
|
436
483
|
}
|
|
@@ -6,6 +6,7 @@ import { DEFAULT_EXPORT_MAX_BYTES, exportRun, formatExportHuman } from "../../fe
|
|
|
6
6
|
import { listRuns, readReview } from "../../run/stored-runs.js";
|
|
7
7
|
import { verifyRun } from "../../verify/verify.js";
|
|
8
8
|
import { reclaimPreflightSandboxes, reclaimRunSandboxes, } from "../../run/reclaim.js";
|
|
9
|
+
import { runDisplay } from "../../run/display.js";
|
|
9
10
|
import { addRunOptions, handleRun } from "./run-command.js";
|
|
10
11
|
import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, RUN_OPTION_DESCRIPTION, writeResult, humanError, } from "../io.js";
|
|
11
12
|
import { plural } from "../../run/text.js";
|
|
@@ -33,7 +34,7 @@ export function registerVerifyCommand(parent, io) {
|
|
|
33
34
|
io.setExitCode(result.ok ? 0 : 2);
|
|
34
35
|
});
|
|
35
36
|
}
|
|
36
|
-
const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in
|
|
37
|
+
const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in the first minor release on or after 2026-11-03.";
|
|
37
38
|
/**
|
|
38
39
|
* The deprecated `cleanup`, hidden from help: one warning, then exactly `reclaim --check`, with
|
|
39
40
|
* its output and exit code. The old command read only what computer-use runs recorded and passed
|
|
@@ -244,13 +245,6 @@ const REVIEW_VERDICTS = {
|
|
|
244
245
|
blocked: "blocked",
|
|
245
246
|
timed_out: "timed out",
|
|
246
247
|
};
|
|
247
|
-
/** The state runDisplay gives a finished run whose own ok agrees with its verdict. */
|
|
248
|
-
const VERDICT_STATES = {
|
|
249
|
-
pass: "passed",
|
|
250
|
-
fail: "failed",
|
|
251
|
-
blocked: "blocked",
|
|
252
|
-
timed_out: "timed_out",
|
|
253
|
-
};
|
|
254
248
|
/**
|
|
255
249
|
* Review's headline: the run's display label, as `humanish runs`, review.md and the Observer show
|
|
256
250
|
* it. A verdict that names another state follows in parentheses, so an interrupted run whose last
|
|
@@ -262,8 +256,8 @@ function reviewHeadline(verdict, display) {
|
|
|
262
256
|
return REVIEW_VERDICTS[verdict];
|
|
263
257
|
if (display.state === "no_verdict" || display.state === "dry_run")
|
|
264
258
|
return `${display.label}; ${NOTHING_TESTED}`;
|
|
265
|
-
const judged =
|
|
266
|
-
return
|
|
259
|
+
const judged = runDisplay({ liveness: "finished", verdict }).state;
|
|
260
|
+
return verdict === "contract_proof_only" || judged === display.state
|
|
267
261
|
? display.label
|
|
268
262
|
: `${display.label} (verdict ${REVIEW_VERDICTS[verdict]})`;
|
|
269
263
|
}
|