humanish 0.113.0 → 0.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/CONTEXT.md +1 -0
- package/README.md +1 -1
- package/dist/actors/closing-report-limits.d.ts +11 -0
- package/dist/actors/closing-report-limits.js +13 -0
- package/dist/actors/closing-report.d.ts +68 -0
- package/dist/actors/closing-report.js +55 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +4 -47
- package/dist/actors/codex/restricted-participant-policy.js +12 -40
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -10
- package/dist/actors/computer-use/loop/debrief.js +10 -46
- package/dist/actors/computer-use/loop/impressions.d.ts +2 -2
- package/dist/actors/computer-use/loop/impressions.js +6 -1
- package/dist/actors/computer-use/loop.d.ts +0 -1
- package/dist/actors/computer-use/loop.js +3 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +1 -1
- package/dist/actors/computer-use/openai-conversation.js +12 -36
- package/dist/actors/computer-use/openai-provider.js +11 -5
- package/dist/actors/contract.d.ts +6 -22
- package/dist/actors/contract.js +0 -13
- package/dist/analysis/admission.d.ts +47 -0
- package/dist/analysis/admission.js +65 -0
- package/dist/analysis/automatic-completion.d.ts +0 -6
- package/dist/analysis/automatic-completion.js +3 -3
- package/dist/analysis/automatic-config.d.ts +10 -2
- package/dist/analysis/automatic-config.js +29 -14
- package/dist/analysis/automatic.js +7 -0
- package/dist/analysis/execute.d.ts +24 -5
- package/dist/analysis/execute.js +61 -27
- package/dist/analysis/finding-lead.d.ts +27 -0
- package/dist/analysis/finding-lead.js +23 -0
- package/dist/analysis/image-tokens.d.ts +2 -0
- package/dist/analysis/image-tokens.js +1 -1
- package/dist/analysis/job.d.ts +9 -1
- package/dist/analysis/job.js +11 -2
- package/dist/analysis/service.js +8 -3
- package/dist/cli/commands/analyze.js +7 -2
- package/dist/cli/commands/notes.d.ts +3 -0
- package/dist/cli/commands/notes.js +145 -0
- package/dist/cli/commands/observe.js +1 -1
- package/dist/cli/commands/study-run.js +2 -1
- package/dist/cli/findings.js +11 -30
- package/dist/cli/io.d.ts +1 -0
- package/dist/cli/io.js +8 -12
- package/dist/cli/program.js +2 -0
- package/dist/cli/telemetry.js +10 -22
- package/dist/cli/update-check.js +6 -18
- package/dist/cli/user-config.d.ts +11 -0
- package/dist/cli/user-config.js +25 -0
- package/dist/comms/email-catch.js +7 -31
- package/dist/evidence/encoded-text.d.ts +17 -3
- package/dist/evidence/encoded-text.js +121 -46
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +21 -10
- package/dist/evidence/secret-scrub.d.ts +14 -1
- package/dist/evidence/secret-scrub.js +125 -28
- package/dist/feedback/draft.d.ts +19 -0
- package/dist/feedback/draft.js +62 -15
- package/dist/feedback/export-bundle.js +3 -3
- package/dist/feedback/export.js +50 -20
- package/dist/feedback/feedback.js +29 -2
- package/dist/keys/key-resolution.js +12 -17
- package/dist/observer/artifact.d.ts +5 -0
- package/dist/observer/artifact.js +18 -2
- package/dist/observer/http.d.ts +7 -0
- package/dist/observer/http.js +19 -1
- package/dist/observer/notes-route.d.ts +20 -0
- package/dist/observer/notes-route.js +120 -0
- package/dist/observer/pinned-files.d.ts +8 -1
- package/dist/observer/pinned-files.js +17 -3
- package/dist/observer/render.js +39 -14
- package/dist/observer/run-routes.d.ts +8 -2
- package/dist/observer/run-routes.js +32 -6
- package/dist/observer/serve.d.ts +3 -0
- package/dist/observer/serve.js +29 -3
- package/dist/observer-app.html +7 -6
- package/dist/run/contained-output.d.ts +8 -0
- package/dist/run/contained-output.js +27 -0
- package/dist/run/evidence-files.d.ts +7 -0
- package/dist/run/evidence-files.js +24 -14
- package/dist/run/note-files.d.ts +54 -0
- package/dist/run/note-files.js +212 -0
- package/dist/run/note-shape.d.ts +35 -0
- package/dist/run/note-shape.js +20 -0
- package/dist/run/notes.d.ts +34 -0
- package/dist/run/notes.js +192 -0
- package/dist/run/participant-caption.d.ts +9 -0
- package/dist/run/participant-caption.js +19 -0
- package/dist/run/participant-records.d.ts +2 -0
- package/dist/run/participant-records.js +17 -2
- package/dist/run/transient-comms-secrets.d.ts +6 -5
- package/dist/run/transient-comms-secrets.js +19 -24
- package/dist/study/doctor.js +1 -1
- package/dist/study/preflight-result.js +2 -1
- package/dist/study/summary.d.ts +4 -5
- package/dist/study/summary.js +6 -4
- package/dist/study/warnings.js +2 -1
- package/dist/tui-app.js +1 -1
- package/docs/architecture/actor-contract.md +3 -3
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/state-driven-executor.md +1 -1
- package/docs/contracts/errors.md +2 -2
- package/docs/contracts/run-bundle.md +61 -0
- package/docs/contracts/schemas.md +17 -14
- package/docs/contracts/study-analysis.md +35 -9
- package/docs/principles/invariants-and-defaults.md +2 -2
- package/docs/product/automatic-analysis.md +22 -16
- package/package.json +8 -7
- package/skills/humanish/SKILL.md +6 -2
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { AnalysisCorrection, AnalysisResult } from "./types.js";
|
|
2
|
+
type Severity = NonNullable<AnalysisResult["designFindings"]>[number]["severity"];
|
|
3
|
+
export interface FindingLead {
|
|
4
|
+
/** The reviewer's amended claim, or the plain headline. */
|
|
5
|
+
headline: string;
|
|
6
|
+
/**
|
|
7
|
+
* The line under the headline: the plain experience, or after an amendment a sentence saying the
|
|
8
|
+
* reviewer's claim replaced the headline and account. Null when the analysis wrote no experience.
|
|
9
|
+
*/
|
|
10
|
+
account: string | null;
|
|
11
|
+
/** The headline is a reviewer's amended claim. */
|
|
12
|
+
corrected: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The headline and account a finding leads with, given its latest human review note. An amendment
|
|
16
|
+
* replaces both, since they describe the claim the reviewer replaced. Null for an analysis written
|
|
17
|
+
* before headlines, which leads with the finding's title.
|
|
18
|
+
*/
|
|
19
|
+
export declare function findingLead(finding: {
|
|
20
|
+
headline?: string | null;
|
|
21
|
+
experience?: string | null;
|
|
22
|
+
}, correction: Pick<AnalysisCorrection, "status" | "replacementClaim"> | null | undefined): FindingLead | null;
|
|
23
|
+
/** Design findings most severe first, in a new array; equal severity keeps the analysis order. */
|
|
24
|
+
export declare function bySeverity<T extends {
|
|
25
|
+
severity: Severity;
|
|
26
|
+
}>(findings: readonly T[]): T[];
|
|
27
|
+
export {};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// How a finding opens for a reader, the same in `review`, a feedback draft and the Observer. The
|
|
2
|
+
// Observer bundles this module, so it imports types only (observer/tests/contract-lock.test.ts).
|
|
3
|
+
/**
|
|
4
|
+
* The headline and account a finding leads with, given its latest human review note. An amendment
|
|
5
|
+
* replaces both, since they describe the claim the reviewer replaced. Null for an analysis written
|
|
6
|
+
* before headlines, which leads with the finding's title.
|
|
7
|
+
*/
|
|
8
|
+
export function findingLead(finding, correction) {
|
|
9
|
+
if (finding.headline === undefined || finding.headline === null)
|
|
10
|
+
return null;
|
|
11
|
+
if (correction?.status === "amended" && correction.replacementClaim)
|
|
12
|
+
return {
|
|
13
|
+
headline: correction.replacementClaim,
|
|
14
|
+
account: "Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
15
|
+
corrected: true,
|
|
16
|
+
};
|
|
17
|
+
return { headline: finding.headline, account: finding.experience ?? null, corrected: false };
|
|
18
|
+
}
|
|
19
|
+
const SEVERITY_ORDER = ["major", "moderate", "minor"];
|
|
20
|
+
/** Design findings most severe first, in a new array; equal severity keeps the analysis order. */
|
|
21
|
+
export function bySeverity(findings) {
|
|
22
|
+
return findings.toSorted((a, b) => SEVERITY_ORDER.indexOf(a.severity) - SEVERITY_ORDER.indexOf(b.severity));
|
|
23
|
+
}
|
|
@@ -16,7 +16,7 @@ const HIGH_DETAIL_SIZING = Object.freeze({
|
|
|
16
16
|
"gpt-5.5": { patchBudget: 2500, multiplierTenths: 12 },
|
|
17
17
|
});
|
|
18
18
|
/** 2,500 patches × 1.2: what any image costs on these models when its size is unknown. */
|
|
19
|
-
const HIGH_DETAIL_IMAGE_TOKEN_CEILING = 3000;
|
|
19
|
+
export const HIGH_DETAIL_IMAGE_TOKEN_CEILING = 3000;
|
|
20
20
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
21
21
|
/** Width and height from a PNG data URL's IHDR chunk; undefined for anything else. */
|
|
22
22
|
function pngDimensions(dataUrl) {
|
package/dist/analysis/job.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AnalyzeResult } from "./service.js";
|
|
2
2
|
import { z } from "zod";
|
|
3
3
|
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
4
|
+
import type { RefusedAnalysisCost } from "./admission.js";
|
|
4
5
|
export declare const AUTOMATIC_ANALYSIS_DIRECTORY = "analysis-automatic";
|
|
5
6
|
export declare const AUTOMATIC_ANALYSIS_STALE_MS = 15000;
|
|
6
7
|
/** Execution metadata only. Never a participant outcome or permission to dispatch. */
|
|
@@ -11,6 +12,8 @@ export interface AutomaticAnalysisView {
|
|
|
11
12
|
/** Safe stable code, not provider text. */
|
|
12
13
|
reason: string | null;
|
|
13
14
|
updatedAt: string;
|
|
15
|
+
/** What the analysis would have cost, when admission refused it for its cost. */
|
|
16
|
+
admission?: RefusedAnalysisCost;
|
|
14
17
|
}
|
|
15
18
|
export interface AutomaticAnalysisOutcome {
|
|
16
19
|
state: "complete" | "partial" | "failed" | "cancelled" | "skipped" | "unknown";
|
|
@@ -69,6 +72,11 @@ declare const jobSchema: z.ZodObject<{
|
|
|
69
72
|
receiptSha256: z.ZodNullable<z.ZodString>;
|
|
70
73
|
startedAt: z.ZodNullable<z.ZodISODateTime>;
|
|
71
74
|
finishedAt: z.ZodNullable<z.ZodISODateTime>;
|
|
75
|
+
admission: z.ZodOptional<z.ZodObject<{
|
|
76
|
+
expectedCostUsd: z.ZodNumber;
|
|
77
|
+
worstCaseCostUsd: z.ZodNumber;
|
|
78
|
+
maxCostUsd: z.ZodNumber;
|
|
79
|
+
}, z.core.$strict>>;
|
|
72
80
|
}, z.core.$strict>;
|
|
73
81
|
type JobRecord = z.infer<typeof jobSchema>;
|
|
74
82
|
/** Also guards direct HTML projection callers, not only the contained-file reader. */
|
|
@@ -86,7 +94,7 @@ export declare function readAutomaticAnalysisPrepared(prepared: PreparedRunArtif
|
|
|
86
94
|
export interface AutomaticAnalysisJob {
|
|
87
95
|
readonly attemptId: string;
|
|
88
96
|
/** Serialized, awaited durability boundary. A failed write must prevent dispatch. */
|
|
89
|
-
update(update: Partial<Pick<JobRecord, "state" | "analysisId" | "reason" | "sourceRunSha256" | "inputDigest" | "startedAt" | "analysisSha256" | "receiptSha256">>): Promise<void>;
|
|
97
|
+
update(update: Partial<Pick<JobRecord, "state" | "analysisId" | "reason" | "sourceRunSha256" | "inputDigest" | "startedAt" | "analysisSha256" | "receiptSha256" | "admission">>): Promise<void>;
|
|
90
98
|
touch(): Promise<void>;
|
|
91
99
|
cancellationRequested(): Promise<boolean>;
|
|
92
100
|
}
|
package/dist/analysis/job.js
CHANGED
|
@@ -52,6 +52,12 @@ const reasons = [
|
|
|
52
52
|
const id = z.string().regex(ANALYSIS_ID_PATTERN);
|
|
53
53
|
const date = z.iso.datetime();
|
|
54
54
|
const digest = z.string().regex(SHA256_HEX_PATTERN);
|
|
55
|
+
const usd = z.number().min(0).max(1e9);
|
|
56
|
+
const refusedCostSchema = z.strictObject({
|
|
57
|
+
expectedCostUsd: usd,
|
|
58
|
+
worstCaseCostUsd: usd,
|
|
59
|
+
maxCostUsd: usd,
|
|
60
|
+
});
|
|
55
61
|
const jobSchema = z.strictObject({
|
|
56
62
|
schema: z.literal(AUTOMATIC_ANALYSIS_SCHEMA),
|
|
57
63
|
runId: id,
|
|
@@ -70,6 +76,7 @@ const jobSchema = z.strictObject({
|
|
|
70
76
|
receiptSha256: digest.nullable(),
|
|
71
77
|
startedAt: date.nullable(),
|
|
72
78
|
finishedAt: date.nullable(),
|
|
79
|
+
admission: refusedCostSchema.optional(),
|
|
73
80
|
});
|
|
74
81
|
const cancelSchema = z.strictObject({
|
|
75
82
|
schema: z.literal(CANCEL_SCHEMA),
|
|
@@ -111,11 +118,12 @@ export function projectAutomaticAnalysisView(value) {
|
|
|
111
118
|
if (value === undefined)
|
|
112
119
|
return undefined;
|
|
113
120
|
const parsed = jobSchema
|
|
114
|
-
.pick({ state: true, analysisId: true, reason: true, updatedAt: true })
|
|
121
|
+
.pick({ state: true, analysisId: true, reason: true, updatedAt: true, admission: true })
|
|
115
122
|
.safeParse(value);
|
|
116
123
|
if (!parsed.success || containsSensitive(JSON.stringify(parsed.data)))
|
|
117
124
|
return unknown();
|
|
118
|
-
|
|
125
|
+
const { admission, ...view } = parsed.data;
|
|
126
|
+
return admission === undefined ? view : { ...view, admission };
|
|
119
127
|
}
|
|
120
128
|
async function bindJob(prepared) {
|
|
121
129
|
await validatePreparedRunRootIdentity(prepared);
|
|
@@ -162,6 +170,7 @@ export async function readAutomaticAnalysisPrepared(prepared, now = Date.now())
|
|
|
162
170
|
analysisId: record.analysisId,
|
|
163
171
|
reason: record.reason,
|
|
164
172
|
updatedAt: record.updatedAt,
|
|
173
|
+
...(record.admission === undefined ? {} : { admission: record.admission }),
|
|
165
174
|
};
|
|
166
175
|
const updated = Date.parse(record.updatedAt);
|
|
167
176
|
if (updated > now || (pending(record.state) && now - updated > AUTOMATIC_ANALYSIS_STALE_MS)) {
|
package/dist/analysis/service.js
CHANGED
|
@@ -17,6 +17,8 @@ import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalys
|
|
|
17
17
|
import { beginAnalysisExecution, writeAnalysisExecutionReceipt } from "./store-executions.js";
|
|
18
18
|
import { keepRejectedAnalysisOutput } from "./diagnostics.js";
|
|
19
19
|
import { loadAnalysis } from "./load.js";
|
|
20
|
+
import { costRefusal, refusedCost } from "./admission.js";
|
|
21
|
+
import { cli } from "../cli/invocation.js";
|
|
20
22
|
import { hashAnalysisValue } from "./validation.js";
|
|
21
23
|
import { ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
22
24
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
@@ -188,17 +190,20 @@ function analyzeConfigRefusal(config) {
|
|
|
188
190
|
}
|
|
189
191
|
/** The result when admission refuses the attempt, or when a dry run stops after admission. */
|
|
190
192
|
function admissionOnlyResult(base, config) {
|
|
191
|
-
if (!base.admission.allowed)
|
|
193
|
+
if (!base.admission.allowed) {
|
|
194
|
+
const cost = refusedCost(base.admission);
|
|
195
|
+
const refusal = cost && costRefusal(cost, `--run ${base.run}`, cli);
|
|
192
196
|
return {
|
|
193
197
|
...base,
|
|
194
198
|
ok: false,
|
|
195
199
|
error: {
|
|
196
200
|
code: base.admission.error ?? "analysis_admission_denied",
|
|
197
|
-
message:
|
|
198
|
-
?
|
|
201
|
+
message: refusal
|
|
202
|
+
? `${refusal.text} ${refusal.command}`
|
|
199
203
|
: "The analysis input or configuration did not pass admission. No provider request was sent.",
|
|
200
204
|
},
|
|
201
205
|
};
|
|
206
|
+
}
|
|
202
207
|
return {
|
|
203
208
|
...base,
|
|
204
209
|
ok: true,
|
|
@@ -29,7 +29,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
29
29
|
.option("--run <id>", RUN_OPTION_DESCRIPTION, "latest")
|
|
30
30
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
31
31
|
.option("--provider <name>", "Analyst: openai (default) or restricted codex account. No provider fallback.")
|
|
32
|
-
.option("--max-cost <usd>", "Required for OpenAI, also with --dry-run: send nothing when the
|
|
32
|
+
.option("--max-cost <usd>", "Required for OpenAI, also with --dry-run: send nothing when the expected cost in USD, plus a 10% margin, is above this. It does not limit billing. Unsupported for Codex.")
|
|
33
33
|
.option("--model <id>", "Supported vision model. OpenAI uses high effort; qualified Codex account analysis uses low effort.", DEFAULT_ANALYSIS_MODEL)
|
|
34
34
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
35
35
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
@@ -125,7 +125,7 @@ async function handleAnalyze(io, options, command) {
|
|
|
125
125
|
if (value.ok && value.dryRun && selected.config?.provider === "codex")
|
|
126
126
|
return "Local evidence and configuration passed admission. Codex CLI, login and model access were not checked. Dollar cost and output-token ceiling are unknown. No provider request sent.\n";
|
|
127
127
|
if (value.ok && value.dryRun)
|
|
128
|
-
return
|
|
128
|
+
return admittedDryRunText(value.admission);
|
|
129
129
|
const lines = [];
|
|
130
130
|
if (value.artifactPath)
|
|
131
131
|
lines.push(`${value.reused ? "Reused" : "Saved"} ${value.status} analysis: ${value.artifactPath}`);
|
|
@@ -231,3 +231,8 @@ async function handleAnalyzeCorrect(io, options, command) {
|
|
|
231
231
|
io.setExitCode(2);
|
|
232
232
|
}
|
|
233
233
|
}
|
|
234
|
+
/** An admitted dry run's costs: the expected cost, the worst case and the cap it was admitted under. */
|
|
235
|
+
function admittedDryRunText(admission) {
|
|
236
|
+
const usd = (value) => value === null || value === undefined ? "unknown" : `$${value.toFixed(2)}`;
|
|
237
|
+
return `Expected cost: ${usd(admission?.estimatedCostUsd)}. Worst case: ${usd(admission?.worstCaseCostUsd)}, if the analyst writes its whole ${admission?.outputTokenAllowance ?? "unknown"}-token output allowance, reasoning included. Admitted under the $${admission?.maxCostUsd ?? "unknown"} cap. No request sent.\n`;
|
|
238
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { renderObserver } from "../../observer/render.js";
|
|
2
|
+
import { loadRunBundlePrepared, resolveRunPath } from "../../run/locate.js";
|
|
3
|
+
import { readRunNotes } from "../../run/note-files.js";
|
|
4
|
+
import { byRunTime } from "../../run/note-shape.js";
|
|
5
|
+
import { addRunNote, formatRunTime } from "../../run/notes.js";
|
|
6
|
+
import { runParticipantCaptions, streamParticipantIdOf } from "../../run/participant-records.js";
|
|
7
|
+
import { resolvePhysicalCwd, runIdOf } from "../../run/paths.js";
|
|
8
|
+
import { runNotFoundMessage } from "../../run/run-not-found.js";
|
|
9
|
+
import { cli } from "../invocation.js";
|
|
10
|
+
import { CWD_OPTION_DESCRIPTION, humanError, JSON_OPTION_DESCRIPTION, writeResult, } from "../io.js";
|
|
11
|
+
const NOTES_RESULT_SCHEMA = "humanish.notes-result.v1";
|
|
12
|
+
export function registerNotesCommand(parent, io) {
|
|
13
|
+
parent
|
|
14
|
+
.command("notes")
|
|
15
|
+
.description("List the reviewer notes on a run, or add one at a moment of its recording. A note's time counts from the run's first timed capture, as the Observer's study timeline shows it.")
|
|
16
|
+
.summary("List or add reviewer notes on a run.")
|
|
17
|
+
// The text is read from the words after the run, not declared, so the root command list
|
|
18
|
+
// stays as narrow as its other rows and an unquoted note still arrives whole.
|
|
19
|
+
.usage("[options] <run> [text...]")
|
|
20
|
+
.argument("<run>", "Run id, or latest.")
|
|
21
|
+
.allowExcessArguments(true)
|
|
22
|
+
.option("--add", "Add a note. Its text is the words after the run; --at gives the moment.")
|
|
23
|
+
.option("--at <mm:ss>", "With --add: the moment, in minutes and seconds, such as 02:31.")
|
|
24
|
+
.option("--participant <id>", "With --add: the participant's id from the study, or its stream id. Leave it out for a note on the whole run.")
|
|
25
|
+
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
26
|
+
.option("--json", JSON_OPTION_DESCRIPTION)
|
|
27
|
+
.addHelpText("after", [
|
|
28
|
+
"",
|
|
29
|
+
"Examples:",
|
|
30
|
+
" humanish notes latest",
|
|
31
|
+
' humanish notes latest --add --at 02:31 "The save button was hidden here."',
|
|
32
|
+
' humanish notes <runId> --add --at 00:45 --participant <id> "They gave up."',
|
|
33
|
+
"",
|
|
34
|
+
"Each note is saved in the run directory as notes/<id>.json. Text that looks like a secret",
|
|
35
|
+
"or a local path is replaced before it is saved, and verify scans notes like other run text.",
|
|
36
|
+
].join("\n"))
|
|
37
|
+
.action((run, options, command) => {
|
|
38
|
+
const words = command.args.slice(1);
|
|
39
|
+
return handleNotes(io, run, words.length > 0 ? words.join(" ") : undefined, options, command);
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
/** Minutes and seconds as the Observer shows them, `02:31`, in milliseconds; null otherwise. */
|
|
43
|
+
function parseRunTime(value) {
|
|
44
|
+
const match = /^(\d{1,4}):([0-5]\d)$/.exec(value.trim());
|
|
45
|
+
return match ? (Number(match[1]) * 60 + Number(match[2])) * 1000 : null;
|
|
46
|
+
}
|
|
47
|
+
async function handleNotes(io, run, text, options, command) {
|
|
48
|
+
const base = {
|
|
49
|
+
schema: NOTES_RESULT_SCHEMA,
|
|
50
|
+
cwd: options.cwd,
|
|
51
|
+
run,
|
|
52
|
+
notes: [],
|
|
53
|
+
participants: [],
|
|
54
|
+
warnings: [],
|
|
55
|
+
};
|
|
56
|
+
const fail = (code, message) => {
|
|
57
|
+
writeResult(command, io, { ...base, ok: false, error: { code, message } }, formatNotesHuman);
|
|
58
|
+
io.setExitCode(2);
|
|
59
|
+
};
|
|
60
|
+
const adding = options.add === true;
|
|
61
|
+
if (!adding && (text !== undefined || options.at !== undefined))
|
|
62
|
+
return fail("HUMANISH_NOTES_OPTION_CONFLICT", `Adding a note needs --add: ${cli(`notes ${run} --add --at 02:31 "text"`)}.`);
|
|
63
|
+
if (adding && (options.at === undefined || text === undefined))
|
|
64
|
+
return fail("HUMANISH_NOTES_OPTION_CONFLICT", `--add needs --at and the note's text: ${cli(`notes ${run} --add --at 02:31 "text"`)}.`);
|
|
65
|
+
const atMs = options.at === undefined ? null : parseRunTime(options.at);
|
|
66
|
+
if (adding && atMs === null)
|
|
67
|
+
return fail("HUMANISH_NOTE_INVALID_TIME", `--at ${options.at ?? ""} is not a time on the run clock. Give minutes and seconds from the run's first capture, as the Observer shows them, such as 02:31.`);
|
|
68
|
+
const cwd = await resolvePhysicalCwd(options.cwd);
|
|
69
|
+
const prepared = await resolveRunPath(cwd, run).catch(() => null);
|
|
70
|
+
if (!prepared)
|
|
71
|
+
return fail("HUMANISH_RUN_NOT_FOUND", await runNotFoundMessage(cwd, run));
|
|
72
|
+
const runId = runIdOf(prepared);
|
|
73
|
+
let added;
|
|
74
|
+
const warnings = [];
|
|
75
|
+
if (adding && atMs !== null && text !== undefined) {
|
|
76
|
+
const result = await addRunNote(prepared, {
|
|
77
|
+
atMs,
|
|
78
|
+
participant: options.participant ?? null,
|
|
79
|
+
text,
|
|
80
|
+
});
|
|
81
|
+
if (!result.ok)
|
|
82
|
+
return fail(result.error.code, result.error.message);
|
|
83
|
+
added = result.note;
|
|
84
|
+
if (result.scrubbed)
|
|
85
|
+
warnings.push("Part of the note looked like a secret or a local path, so it was replaced before the note was saved.");
|
|
86
|
+
// The saved Observer page shows notes read-only, so it is rendered again with the new one.
|
|
87
|
+
const rendered = await renderObserver(cwd, runId, { open: false, expectedRun: prepared })
|
|
88
|
+
.then((observer) => observer.ok)
|
|
89
|
+
.catch(() => false);
|
|
90
|
+
if (!rendered)
|
|
91
|
+
warnings.push(`The note is saved, but the run's saved Observer page could not be rendered again. ${cli(`observe --run ${runId}`)} shows it.`);
|
|
92
|
+
}
|
|
93
|
+
const notes = await readRunNotes(prepared);
|
|
94
|
+
// A note file that cannot be read is named and left as it is; the others are listed.
|
|
95
|
+
warnings.push(...notes.skipped);
|
|
96
|
+
const result = {
|
|
97
|
+
...base,
|
|
98
|
+
ok: true,
|
|
99
|
+
run: runId,
|
|
100
|
+
notes: byRunTime(notes.notes),
|
|
101
|
+
participants: await participantsOf(cwd, prepared),
|
|
102
|
+
...(added === undefined ? {} : { added }),
|
|
103
|
+
warnings,
|
|
104
|
+
};
|
|
105
|
+
writeResult(command, io, result, formatNotesHuman);
|
|
106
|
+
io.setExitCode(0);
|
|
107
|
+
}
|
|
108
|
+
/** The run's participants with their captions; none when its run.json cannot be read. */
|
|
109
|
+
async function participantsOf(cwd, prepared) {
|
|
110
|
+
const loaded = await loadRunBundlePrepared(cwd, prepared).catch(() => null);
|
|
111
|
+
if (!loaded)
|
|
112
|
+
return [];
|
|
113
|
+
const captions = runParticipantCaptions(loaded.bundle);
|
|
114
|
+
return loaded.bundle.streams.map((stream) => {
|
|
115
|
+
const participantId = streamParticipantIdOf(stream);
|
|
116
|
+
return {
|
|
117
|
+
id: stream.id,
|
|
118
|
+
...(participantId === undefined ? {} : { participantId }),
|
|
119
|
+
caption: captions.get(stream.id) ?? stream.id,
|
|
120
|
+
};
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
/** Who a note is about, as a person reads it. */
|
|
124
|
+
function participantName(result, participant) {
|
|
125
|
+
if (participant === null)
|
|
126
|
+
return "whole run";
|
|
127
|
+
return result.participants.find((entry) => entry.id === participant)?.caption ?? participant;
|
|
128
|
+
}
|
|
129
|
+
function noteLine(note, name, width) {
|
|
130
|
+
const head = ` ${formatRunTime(note.atMs)} ${name.padEnd(width)} `;
|
|
131
|
+
const [first = "", ...rest] = note.text.split("\n");
|
|
132
|
+
return [`${head}${note.author}: ${first}`, ...rest.map((line) => ` ${line}`)].join("\n");
|
|
133
|
+
}
|
|
134
|
+
function formatNotesHuman(result) {
|
|
135
|
+
if (!result.ok)
|
|
136
|
+
return humanError(result.error);
|
|
137
|
+
const warnings = result.warnings.map((warning) => `warning: ${warning}\n`).join("");
|
|
138
|
+
if (result.added)
|
|
139
|
+
return `Added a note at ${formatRunTime(result.added.atMs)} ${result.added.participant === null ? "on the whole run" : `for ${participantName(result, result.added.participant)}`} to run ${result.run}. The Observer marks it on the study timeline, and ${cli(`notes ${result.run}`)} lists every note.\n${warnings}`;
|
|
140
|
+
if (result.notes.length === 0)
|
|
141
|
+
return `Run ${result.run} has no reviewer notes. Add one: ${cli(`notes ${result.run} --add --at 00:30 "text"`)}\n${warnings}`;
|
|
142
|
+
const names = result.notes.map((note) => participantName(result, note.participant));
|
|
143
|
+
const width = Math.max(...names.map((name) => name.length));
|
|
144
|
+
return `${[`Reviewer notes on run ${result.run}: ${result.notes.length}`, ...result.notes.map((note, index) => noteLine(note, names[index], width))].join("\n")}\n${warnings}`;
|
|
145
|
+
}
|
|
@@ -152,7 +152,7 @@ async function observeRun(io, options, command) {
|
|
|
152
152
|
...(openResult.command ? { openCommand: openResult.command } : {}),
|
|
153
153
|
warnings: [
|
|
154
154
|
...rendered.warnings,
|
|
155
|
-
"Observer is served
|
|
155
|
+
"Observer is served over loopback http on 127.0.0.1; only this run's bundle directory is exposed, and the one change the page can make is adding a reviewer note.",
|
|
156
156
|
...(openResult.warning ? [openResult.warning] : []),
|
|
157
157
|
],
|
|
158
158
|
};
|
|
@@ -7,6 +7,7 @@ import { warnAndQueue } from "../deprecations.js";
|
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
|
9
9
|
import { routeOf } from "../../study/plan.js";
|
|
10
|
+
import { computerUseParticipantCount } from "../../study/routing.js";
|
|
10
11
|
import { computerUseRouteRun } from "./study-route-computer-use.js";
|
|
11
12
|
import { runRoute } from "./study-route-run.js";
|
|
12
13
|
import { scriptedRouteRun } from "./study-route-scripted.js";
|
|
@@ -130,7 +131,7 @@ export async function runStudyCommand(args) {
|
|
|
130
131
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
131
132
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
132
133
|
}
|
|
133
|
-
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route);
|
|
134
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
134
135
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
135
136
|
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
136
137
|
}
|
package/dist/cli/findings.js
CHANGED
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
// `humanish.analysis-findings.v1` projection and its text. The analysis artifact stays the record;
|
|
3
3
|
// this reads the version `analyze show` selects and never starts an analysis.
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import { costRefusal } from "../analysis/admission.js";
|
|
5
6
|
import { DEFAULT_ANALYSIS_MAX_COST_USD } from "../analysis/automatic-config.js";
|
|
7
|
+
import { bySeverity, findingLead } from "../analysis/finding-lead.js";
|
|
6
8
|
import { loadAnalysis } from "../analysis/load.js";
|
|
7
9
|
import { projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
8
10
|
import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
@@ -63,20 +65,13 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
63
65
|
...entry,
|
|
64
66
|
}));
|
|
65
67
|
}
|
|
66
|
-
const SEVERITY_RANK = {
|
|
67
|
-
major: 0,
|
|
68
|
-
moderate: 1,
|
|
69
|
-
minor: 2,
|
|
70
|
-
};
|
|
71
68
|
/** Most severe first; equal severity keeps the analysis order. */
|
|
72
69
|
function designFindingViews(analysis, runRoot) {
|
|
73
70
|
const designFindings = analysis.result?.designFindings;
|
|
74
71
|
if (designFindings === undefined)
|
|
75
72
|
return null;
|
|
76
73
|
const labels = new Map(analysis.participants.map((p) => [p.streamId, p.label]));
|
|
77
|
-
return
|
|
78
|
-
.sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity])
|
|
79
|
-
.map((finding) => ({
|
|
74
|
+
return bySeverity(designFindings).map((finding) => ({
|
|
80
75
|
id: finding.id,
|
|
81
76
|
headline: finding.headline,
|
|
82
77
|
screen: finding.screen,
|
|
@@ -199,9 +194,13 @@ export function analysisFindings(source) {
|
|
|
199
194
|
const outcome = analysisOutcomeText(job);
|
|
200
195
|
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? "codex" : undefined));
|
|
201
196
|
}
|
|
197
|
+
if (job?.state === "skipped" && job.admission) {
|
|
198
|
+
const refusal = costRefusal(job.admission, `--run ${source.runId}${source.cwdFlag}`, cli);
|
|
199
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${refusal.text}`, refusal.command);
|
|
200
|
+
}
|
|
202
201
|
if (job?.state === "skipped") {
|
|
203
202
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
204
|
-
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run
|
|
203
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run says why, and for a cost refusal names the --max-cost that admits it." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
205
204
|
? null
|
|
206
205
|
: `${analyzeCommand(source, undefined)}${refused ? " --dry-run" : ""}`);
|
|
207
206
|
}
|
|
@@ -339,18 +338,6 @@ function captureFiles(evidence, runRoot) {
|
|
|
339
338
|
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
340
339
|
];
|
|
341
340
|
}
|
|
342
|
-
/**
|
|
343
|
-
* What a reader sees first: the headline, or the reviewer's claim when a review amended the finding,
|
|
344
|
-
* since the headline and experience describe the claim it replaced. Null before headlines existed.
|
|
345
|
-
*/
|
|
346
|
-
function leadOf(finding) {
|
|
347
|
-
const amended = finding.correction?.status === "amended" ? finding.correction : null;
|
|
348
|
-
if (finding.headline === null)
|
|
349
|
-
return null;
|
|
350
|
-
return amended?.replacementClaim
|
|
351
|
-
? { headline: amended.replacementClaim, corrected: true }
|
|
352
|
-
: { headline: finding.headline, corrected: false };
|
|
353
|
-
}
|
|
354
341
|
function findingLines(finding, runRoot) {
|
|
355
342
|
const cited = [...new Set(finding.evidence.map((entry) => entry.streamId))];
|
|
356
343
|
const labels = new Map(finding.affected.map((p) => [p.streamId, p.label]));
|
|
@@ -358,19 +345,13 @@ function findingLines(finding, runRoot) {
|
|
|
358
345
|
const streams = [...affectedIds, ...cited.filter((id) => !labels.has(id))];
|
|
359
346
|
const captures = captureFiles(finding.evidence, runRoot);
|
|
360
347
|
const correction = finding.correction;
|
|
361
|
-
const lead =
|
|
348
|
+
const lead = findingLead(finding, correction);
|
|
362
349
|
return [
|
|
363
350
|
...(lead === null
|
|
364
351
|
? [`${finding.id} ${finding.title}`]
|
|
365
352
|
: [
|
|
366
353
|
`${finding.id} ${lead.headline}`,
|
|
367
|
-
...(lead.
|
|
368
|
-
? [
|
|
369
|
-
" Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
370
|
-
]
|
|
371
|
-
: finding.experience === null
|
|
372
|
-
? []
|
|
373
|
-
: [` ${finding.experience}`]),
|
|
354
|
+
...(lead.account === null ? [] : [` ${lead.account}`]),
|
|
374
355
|
` evidence: ${finding.title}`,
|
|
375
356
|
]),
|
|
376
357
|
` impact: ${IMPACT_TEXT[finding.impact]} · confidence: ${finding.confidence} · recovery: ${RECOVERY_TEXT[finding.recovery]}`,
|
|
@@ -448,7 +429,7 @@ export function formatFindingsSummary(view, fullCommand, limit = 3) {
|
|
|
448
429
|
`findings: ${view.findings.length === 0 ? "none" : view.findings.length}${view.state === "stale" ? " (stale)" : ""}`,
|
|
449
430
|
...shown.map((finding) => {
|
|
450
431
|
const qualities = `${IMPACT_TEXT[finding.impact]}, ${finding.confidence} confidence, ${RECOVERY_TEXT[finding.recovery]}`;
|
|
451
|
-
const lead =
|
|
432
|
+
const lead = findingLead(finding, finding.correction);
|
|
452
433
|
if (lead === null)
|
|
453
434
|
return `- ${finding.id} ${qualities}: ${finding.title}`;
|
|
454
435
|
return `- ${finding.id} ${lead.headline} (${lead.corrected ? "corrected in human review" : qualities})`;
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -138,4 +138,5 @@ export declare function analysisOutcomeText(analysis: {
|
|
|
138
138
|
state: string;
|
|
139
139
|
reason: string | null;
|
|
140
140
|
}): string;
|
|
141
|
+
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
141
142
|
export declare function automaticAnalysisEnvelope<T>(result: T): T;
|
package/dist/cli/io.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { automaticAnalysisSucceeded,
|
|
2
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js";
|
|
2
|
+
import { costRefusal, refusedCost } from "../analysis/admission.js";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
5
5
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
@@ -188,13 +188,15 @@ export function writeResult(command, io, result, formatHuman) {
|
|
|
188
188
|
writeHuman(command, io, formatHuman(output));
|
|
189
189
|
if (output !== null && typeof output === "object" && "automaticAnalysis" in output) {
|
|
190
190
|
const analysis = output.automaticAnalysis;
|
|
191
|
-
|
|
191
|
+
const cost = analysis?.result?.admission && refusedCost(analysis.result.admission);
|
|
192
|
+
const refusal = cost && costRefusal(cost, `--run ${analysis.result?.run ?? "latest"}`, cli);
|
|
193
|
+
if (refusal)
|
|
194
|
+
io.writeOut(`analysis: refused before it started. ${refusal.text} ${refusal.command}\n`);
|
|
195
|
+
else if (analysis)
|
|
192
196
|
io.writeOut(`analysis: ${analysisOutcomeText(analysis)}\n`);
|
|
193
197
|
const rejected = analysis?.result?.rejectedOutputPath;
|
|
194
198
|
if (rejected)
|
|
195
199
|
io.writeOut(`analysis: ${analysis.result?.error?.code ?? "rejected"}; the rejected output is kept locally at ${rejected}\n`);
|
|
196
|
-
if (defaultAnalysisOverBudget(output))
|
|
197
|
-
io.writeOut(overBudgetAnalysisHint(output));
|
|
198
200
|
}
|
|
199
201
|
}
|
|
200
202
|
markInvocationEnvelopeWritten(command);
|
|
@@ -286,7 +288,7 @@ const ANALYSIS_REASON_TEXT = {
|
|
|
286
288
|
AUTOMATIC_ANALYSIS_SOURCE_UNAVAILABLE: "skipped because the run's evidence could not be read",
|
|
287
289
|
AUTOMATIC_ANALYSIS_ALREADY_REQUESTED: "skipped because this run's analysis was already requested",
|
|
288
290
|
AUTOMATIC_ANALYSIS_BUSY: "skipped because another analysis is running",
|
|
289
|
-
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "
|
|
291
|
+
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "refused before it started: its configuration, question, evidence or cost did not pass admission",
|
|
290
292
|
AUTOMATIC_ANALYSIS_CLEANUP_UNCONFIRMED: "not started because the run's cleanup was not confirmed",
|
|
291
293
|
AUTOMATIC_ANALYSIS_REUSED: "complete, reusing an earlier analysis of the same evidence",
|
|
292
294
|
AUTOMATIC_ANALYSIS_LIMITATIONS: "partial, because it covered only part of the evidence",
|
|
@@ -306,12 +308,6 @@ export function analysisOutcomeText(analysis) {
|
|
|
306
308
|
return ANALYSIS_REASON_TEXT[analysis.reason] ?? `${analysis.state} (${analysis.reason})`;
|
|
307
309
|
}
|
|
308
310
|
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
309
|
-
function overBudgetAnalysisHint(result) {
|
|
310
|
-
const estimate = result.automaticAnalysis?.result?.admission?.estimatedCostUsd ?? null;
|
|
311
|
-
const suggested = Math.ceil(estimate ?? DEFAULT_ANALYSIS_MAX_COST_USD + 1);
|
|
312
|
-
const shown = estimate === null ? "" : ` ($${estimate.toFixed(2)})`;
|
|
313
|
-
return `analysis: its estimate${shown} is over the default $${DEFAULT_ANALYSIS_MAX_COST_USD} cap, so no request was sent. To analyze this run: ${cli(`analyze --run ${result.runId ?? "latest"} --max-cost ${suggested}`)}\n`;
|
|
314
|
-
}
|
|
315
311
|
export function automaticAnalysisEnvelope(result) {
|
|
316
312
|
if (result === null ||
|
|
317
313
|
typeof result !== "object" ||
|
package/dist/cli/program.js
CHANGED
|
@@ -13,6 +13,7 @@ import { registerCommsCommands } from "./commands/comms.js";
|
|
|
13
13
|
import { registerFeedbackCommands } from "./commands/feedback.js";
|
|
14
14
|
import { registerStudyCommands } from "./commands/study.js";
|
|
15
15
|
import { registerMigrateCommand } from "./commands/migrate.js";
|
|
16
|
+
import { registerNotesCommand } from "./commands/notes.js";
|
|
16
17
|
import { registerObserveCommand } from "./commands/observe.js";
|
|
17
18
|
import { registerCleanupCommand, registerExportCommand, registerReclaimCommand, registerReviewCommand, registerRunCommand, registerRunsCommand, registerRuntimeCommands, registerStatsCommand, registerVerifyCommand, } from "./commands/runs.js";
|
|
18
19
|
import { registerDoctorCommand, registerInitCommand, registerKeysCommand, registerTelemetryCommand, } from "./commands/setup.js";
|
|
@@ -392,6 +393,7 @@ export function createProgram(io = {}) {
|
|
|
392
393
|
registerVerifyCommand(program, cliIo);
|
|
393
394
|
registerReviewCommand(program, cliIo);
|
|
394
395
|
registerAnalyzeCommand(program, cliIo);
|
|
396
|
+
registerNotesCommand(program, cliIo);
|
|
395
397
|
registerFeedbackCommands(program, cliIo);
|
|
396
398
|
registerExportCommand(program, cliIo);
|
|
397
399
|
registerRunsCommand(program, cliIo);
|