humanish 0.113.0 → 0.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/CONTEXT.md +1 -0
- package/README.md +1 -1
- package/dist/actors/computer-use/openai-conversation.js +9 -7
- package/dist/actors/contract.d.ts +4 -2
- package/dist/analysis/admission.d.ts +37 -0
- package/dist/analysis/admission.js +60 -0
- package/dist/analysis/automatic-completion.d.ts +0 -6
- package/dist/analysis/automatic-completion.js +3 -3
- package/dist/analysis/automatic-config.d.ts +10 -2
- package/dist/analysis/automatic-config.js +29 -14
- package/dist/analysis/automatic.js +7 -0
- package/dist/analysis/execute.d.ts +19 -5
- package/dist/analysis/execute.js +58 -27
- package/dist/analysis/image-tokens.d.ts +2 -0
- package/dist/analysis/image-tokens.js +1 -1
- package/dist/analysis/job.d.ts +9 -1
- package/dist/analysis/job.js +11 -2
- package/dist/analysis/service.js +6 -3
- package/dist/cli/commands/analyze.js +7 -2
- package/dist/cli/commands/notes.d.ts +3 -0
- package/dist/cli/commands/notes.js +144 -0
- package/dist/cli/commands/observe.js +1 -1
- package/dist/cli/commands/study-run.js +2 -1
- package/dist/cli/findings.js +4 -1
- package/dist/cli/io.d.ts +1 -0
- package/dist/cli/io.js +11 -12
- package/dist/cli/program.js +2 -0
- package/dist/evidence/encoded-text.d.ts +15 -2
- package/dist/evidence/encoded-text.js +68 -4
- package/dist/evidence/secret-scrub.d.ts +1 -1
- package/dist/evidence/secret-scrub.js +23 -2
- package/dist/feedback/draft.d.ts +19 -0
- package/dist/feedback/draft.js +51 -1
- package/dist/feedback/export-bundle.js +2 -2
- package/dist/feedback/export.js +47 -3
- package/dist/feedback/feedback.js +42 -2
- package/dist/observer/artifact.d.ts +5 -0
- package/dist/observer/artifact.js +18 -2
- package/dist/observer/notes-route.d.ts +21 -0
- package/dist/observer/notes-route.js +140 -0
- package/dist/observer/pinned-files.d.ts +8 -1
- package/dist/observer/pinned-files.js +32 -3
- package/dist/observer/render.js +38 -14
- package/dist/observer/run-routes.d.ts +8 -2
- package/dist/observer/run-routes.js +47 -6
- package/dist/observer/serve.d.ts +3 -0
- package/dist/observer/serve.js +28 -3
- package/dist/observer-app.html +7 -6
- package/dist/run/contained-output.d.ts +7 -0
- package/dist/run/contained-output.js +35 -1
- package/dist/run/note-files.d.ts +73 -0
- package/dist/run/note-files.js +184 -0
- package/dist/run/notes.d.ts +37 -0
- package/dist/run/notes.js +198 -0
- package/dist/run/participant-caption.d.ts +9 -0
- package/dist/run/participant-caption.js +19 -0
- package/dist/study/doctor.js +1 -1
- package/dist/study/preflight-result.js +2 -1
- package/dist/study/summary.js +2 -2
- package/docs/contracts/errors.md +2 -2
- package/docs/contracts/run-bundle.md +57 -0
- package/docs/contracts/schemas.md +5 -2
- package/docs/contracts/study-analysis.md +26 -8
- package/docs/principles/invariants-and-defaults.md +2 -2
- package/docs/product/automatic-analysis.md +22 -16
- package/package.json +2 -2
- package/skills/humanish/SKILL.md +5 -0
package/dist/analysis/service.js
CHANGED
|
@@ -17,6 +17,7 @@ import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalys
|
|
|
17
17
|
import { beginAnalysisExecution, writeAnalysisExecutionReceipt } from "./store-executions.js";
|
|
18
18
|
import { keepRejectedAnalysisOutput } from "./diagnostics.js";
|
|
19
19
|
import { loadAnalysis } from "./load.js";
|
|
20
|
+
import { admittingMaxCost, costRefusalText, refusedCost } from "./admission.js";
|
|
20
21
|
import { hashAnalysisValue } from "./validation.js";
|
|
21
22
|
import { ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
22
23
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
@@ -188,17 +189,19 @@ function analyzeConfigRefusal(config) {
|
|
|
188
189
|
}
|
|
189
190
|
/** The result when admission refuses the attempt, or when a dry run stops after admission. */
|
|
190
191
|
function admissionOnlyResult(base, config) {
|
|
191
|
-
if (!base.admission.allowed)
|
|
192
|
+
if (!base.admission.allowed) {
|
|
193
|
+
const cost = refusedCost(base.admission);
|
|
192
194
|
return {
|
|
193
195
|
...base,
|
|
194
196
|
ok: false,
|
|
195
197
|
error: {
|
|
196
198
|
code: base.admission.error ?? "analysis_admission_denied",
|
|
197
|
-
message:
|
|
198
|
-
?
|
|
199
|
+
message: cost
|
|
200
|
+
? `${costRefusalText(cost)} To run it, raise the cap: humanish analyze --run ${base.run} --max-cost ${admittingMaxCost(cost)}`
|
|
199
201
|
: "The analysis input or configuration did not pass admission. No provider request was sent.",
|
|
200
202
|
},
|
|
201
203
|
};
|
|
204
|
+
}
|
|
202
205
|
return {
|
|
203
206
|
...base,
|
|
204
207
|
ok: true,
|
|
@@ -29,7 +29,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
29
29
|
.option("--run <id>", RUN_OPTION_DESCRIPTION, "latest")
|
|
30
30
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
31
31
|
.option("--provider <name>", "Analyst: openai (default) or restricted codex account. No provider fallback.")
|
|
32
|
-
.option("--max-cost <usd>", "Required for OpenAI, also with --dry-run: send nothing when the
|
|
32
|
+
.option("--max-cost <usd>", "Required for OpenAI, also with --dry-run: send nothing when the expected cost in USD, plus a 10% margin, is above this. It does not limit billing. Unsupported for Codex.")
|
|
33
33
|
.option("--model <id>", "Supported vision model. OpenAI uses high effort; qualified Codex account analysis uses low effort.", DEFAULT_ANALYSIS_MODEL)
|
|
34
34
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
35
35
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
@@ -125,7 +125,7 @@ async function handleAnalyze(io, options, command) {
|
|
|
125
125
|
if (value.ok && value.dryRun && selected.config?.provider === "codex")
|
|
126
126
|
return "Local evidence and configuration passed admission. Codex CLI, login and model access were not checked. Dollar cost and output-token ceiling are unknown. No provider request sent.\n";
|
|
127
127
|
if (value.ok && value.dryRun)
|
|
128
|
-
return
|
|
128
|
+
return admittedDryRunText(value.admission);
|
|
129
129
|
const lines = [];
|
|
130
130
|
if (value.artifactPath)
|
|
131
131
|
lines.push(`${value.reused ? "Reused" : "Saved"} ${value.status} analysis: ${value.artifactPath}`);
|
|
@@ -231,3 +231,8 @@ async function handleAnalyzeCorrect(io, options, command) {
|
|
|
231
231
|
io.setExitCode(2);
|
|
232
232
|
}
|
|
233
233
|
}
|
|
234
|
+
/** An admitted dry run's costs: the expected cost, the worst case and the cap it was admitted under. */
|
|
235
|
+
function admittedDryRunText(admission) {
|
|
236
|
+
const usd = (value) => value === null || value === undefined ? "unknown" : `$${value.toFixed(2)}`;
|
|
237
|
+
return `Expected cost: ${usd(admission?.estimatedCostUsd)}. Worst case: ${usd(admission?.worstCaseCostUsd)}, if the analyst writes its whole ${admission?.outputTokenAllowance ?? "unknown"}-token output allowance, reasoning included. Admitted under the $${admission?.maxCostUsd ?? "unknown"} cap. No request sent.\n`;
|
|
238
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { renderObserver } from "../../observer/render.js";
|
|
2
|
+
import { loadRunBundlePrepared, resolveRunPath } from "../../run/locate.js";
|
|
3
|
+
import { addRunNote, formatRunTime, readRunNotes, runParticipantCaptions, } from "../../run/notes.js";
|
|
4
|
+
import { streamParticipantIdOf } from "../../run/participant-records.js";
|
|
5
|
+
import { resolvePhysicalCwd, runIdOf } from "../../run/paths.js";
|
|
6
|
+
import { runNotFoundMessage } from "../../run/run-not-found.js";
|
|
7
|
+
import { cli } from "../invocation.js";
|
|
8
|
+
import { CWD_OPTION_DESCRIPTION, humanError, JSON_OPTION_DESCRIPTION, writeResult, } from "../io.js";
|
|
9
|
+
const NOTES_RESULT_SCHEMA = "humanish.notes-result.v1";
|
|
10
|
+
export function registerNotesCommand(parent, io) {
|
|
11
|
+
parent
|
|
12
|
+
.command("notes")
|
|
13
|
+
.description("List the reviewer notes on a run, or add one at a moment of its recording. A note's time counts from the run's first timed capture, as the Observer's study timeline shows it.")
|
|
14
|
+
.summary("List or add reviewer notes on a run.")
|
|
15
|
+
// The text is read from the words after the run, not declared, so the root command list
|
|
16
|
+
// stays as narrow as its other rows and an unquoted note still arrives whole.
|
|
17
|
+
.usage("[options] <run> [text...]")
|
|
18
|
+
.argument("<run>", "Run id, or latest.")
|
|
19
|
+
.allowExcessArguments(true)
|
|
20
|
+
.option("--add", "Add a note. Its text is the words after the run; --at gives the moment.")
|
|
21
|
+
.option("--at <mm:ss>", "With --add: the moment, in minutes and seconds, such as 02:31.")
|
|
22
|
+
.option("--participant <id>", "With --add: the participant's id from the study, or its stream id. Leave it out for a note on the whole run.")
|
|
23
|
+
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
24
|
+
.option("--json", JSON_OPTION_DESCRIPTION)
|
|
25
|
+
.addHelpText("after", [
|
|
26
|
+
"",
|
|
27
|
+
"Examples:",
|
|
28
|
+
" humanish notes latest",
|
|
29
|
+
' humanish notes latest --add --at 02:31 "The save button was hidden here."',
|
|
30
|
+
' humanish notes <runId> --add --at 00:45 --participant <id> "They gave up."',
|
|
31
|
+
"",
|
|
32
|
+
"Each note is saved in the run directory as notes/<id>.json. Text that looks like a secret",
|
|
33
|
+
"or a local path is replaced before it is saved, and verify scans notes like other run text.",
|
|
34
|
+
].join("\n"))
|
|
35
|
+
.action((run, options, command) => {
|
|
36
|
+
const words = command.args.slice(1);
|
|
37
|
+
return handleNotes(io, run, words.length > 0 ? words.join(" ") : undefined, options, command);
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
/** Minutes and seconds as the Observer shows them, `02:31`, in milliseconds; null otherwise. */
|
|
41
|
+
function parseRunTime(value) {
|
|
42
|
+
const match = /^(\d{1,4}):([0-5]\d)$/.exec(value.trim());
|
|
43
|
+
return match ? (Number(match[1]) * 60 + Number(match[2])) * 1000 : null;
|
|
44
|
+
}
|
|
45
|
+
const byRunTime = (notes) => [...notes].sort((left, right) => left.atMs - right.atMs || left.createdAt.localeCompare(right.createdAt));
|
|
46
|
+
async function handleNotes(io, run, text, options, command) {
|
|
47
|
+
const base = {
|
|
48
|
+
schema: NOTES_RESULT_SCHEMA,
|
|
49
|
+
cwd: options.cwd,
|
|
50
|
+
run,
|
|
51
|
+
notes: [],
|
|
52
|
+
participants: [],
|
|
53
|
+
warnings: [],
|
|
54
|
+
};
|
|
55
|
+
const fail = (code, message) => {
|
|
56
|
+
writeResult(command, io, { ...base, ok: false, error: { code, message } }, formatNotesHuman);
|
|
57
|
+
io.setExitCode(2);
|
|
58
|
+
};
|
|
59
|
+
const adding = options.add === true;
|
|
60
|
+
if (!adding && (text !== undefined || options.at !== undefined))
|
|
61
|
+
return fail("HUMANISH_NOTES_OPTION_CONFLICT", `Adding a note needs --add: ${cli(`notes ${run} --add --at 02:31 "text"`)}.`);
|
|
62
|
+
if (adding && (options.at === undefined || text === undefined))
|
|
63
|
+
return fail("HUMANISH_NOTES_OPTION_CONFLICT", `--add needs --at and the note's text: ${cli(`notes ${run} --add --at 02:31 "text"`)}.`);
|
|
64
|
+
const atMs = options.at === undefined ? null : parseRunTime(options.at);
|
|
65
|
+
if (adding && atMs === null)
|
|
66
|
+
return fail("HUMANISH_NOTE_INVALID_TIME", `--at ${options.at ?? ""} is not a time on the run clock. Give minutes and seconds from the run's first capture, as the Observer shows them, such as 02:31.`);
|
|
67
|
+
const cwd = await resolvePhysicalCwd(options.cwd);
|
|
68
|
+
const prepared = await resolveRunPath(cwd, run).catch(() => null);
|
|
69
|
+
if (!prepared)
|
|
70
|
+
return fail("HUMANISH_RUN_NOT_FOUND", await runNotFoundMessage(cwd, run));
|
|
71
|
+
const runId = runIdOf(prepared);
|
|
72
|
+
let added;
|
|
73
|
+
const warnings = [];
|
|
74
|
+
if (adding && atMs !== null && text !== undefined) {
|
|
75
|
+
const result = await addRunNote(prepared, {
|
|
76
|
+
atMs,
|
|
77
|
+
participant: options.participant ?? null,
|
|
78
|
+
text,
|
|
79
|
+
});
|
|
80
|
+
if (!result.ok)
|
|
81
|
+
return fail(result.error.code, result.error.message);
|
|
82
|
+
added = result.note;
|
|
83
|
+
if (result.scrubbed)
|
|
84
|
+
warnings.push("Part of the note looked like a secret or a local path, so it was replaced before the note was saved.");
|
|
85
|
+
// The saved Observer page shows notes read-only, so it is rendered again with the new one.
|
|
86
|
+
const rendered = await renderObserver(cwd, runId, { open: false, expectedRun: prepared })
|
|
87
|
+
.then((observer) => observer.ok)
|
|
88
|
+
.catch(() => false);
|
|
89
|
+
if (!rendered)
|
|
90
|
+
warnings.push(`The note is saved, but the run's saved Observer page could not be rendered again. ${cli(`observe --run ${runId}`)} shows it.`);
|
|
91
|
+
}
|
|
92
|
+
const notes = await readRunNotes(prepared);
|
|
93
|
+
// A note file that cannot be read is named and left as it is; the others are listed.
|
|
94
|
+
warnings.push(...notes.skipped);
|
|
95
|
+
const result = {
|
|
96
|
+
...base,
|
|
97
|
+
ok: true,
|
|
98
|
+
run: runId,
|
|
99
|
+
notes: byRunTime(notes.notes),
|
|
100
|
+
participants: await participantsOf(cwd, prepared),
|
|
101
|
+
...(added === undefined ? {} : { added }),
|
|
102
|
+
warnings,
|
|
103
|
+
};
|
|
104
|
+
writeResult(command, io, result, formatNotesHuman);
|
|
105
|
+
io.setExitCode(0);
|
|
106
|
+
}
|
|
107
|
+
/** The run's participants with their captions; none when its run.json cannot be read. */
|
|
108
|
+
async function participantsOf(cwd, prepared) {
|
|
109
|
+
const loaded = await loadRunBundlePrepared(cwd, prepared).catch(() => null);
|
|
110
|
+
if (!loaded)
|
|
111
|
+
return [];
|
|
112
|
+
const captions = runParticipantCaptions(loaded.bundle);
|
|
113
|
+
return loaded.bundle.streams.map((stream) => {
|
|
114
|
+
const participantId = streamParticipantIdOf(stream);
|
|
115
|
+
return {
|
|
116
|
+
id: stream.id,
|
|
117
|
+
...(participantId === undefined ? {} : { participantId }),
|
|
118
|
+
caption: captions.get(stream.id) ?? stream.id,
|
|
119
|
+
};
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
/** Who a note is about, as a person reads it. */
|
|
123
|
+
function participantName(result, participant) {
|
|
124
|
+
if (participant === null)
|
|
125
|
+
return "whole run";
|
|
126
|
+
return result.participants.find((entry) => entry.id === participant)?.caption ?? participant;
|
|
127
|
+
}
|
|
128
|
+
function noteLine(note, name, width) {
|
|
129
|
+
const head = ` ${formatRunTime(note.atMs)} ${name.padEnd(width)} `;
|
|
130
|
+
const [first = "", ...rest] = note.text.split("\n");
|
|
131
|
+
return [`${head}${note.author}: ${first}`, ...rest.map((line) => ` ${line}`)].join("\n");
|
|
132
|
+
}
|
|
133
|
+
function formatNotesHuman(result) {
|
|
134
|
+
if (!result.ok)
|
|
135
|
+
return humanError(result.error);
|
|
136
|
+
const warnings = result.warnings.map((warning) => `warning: ${warning}\n`).join("");
|
|
137
|
+
if (result.added)
|
|
138
|
+
return `Added a note at ${formatRunTime(result.added.atMs)} ${result.added.participant === null ? "on the whole run" : `for ${participantName(result, result.added.participant)}`} to run ${result.run}. The Observer marks it on the study timeline, and ${cli(`notes ${result.run}`)} lists every note.\n${warnings}`;
|
|
139
|
+
if (result.notes.length === 0)
|
|
140
|
+
return `Run ${result.run} has no reviewer notes. Add one: ${cli(`notes ${result.run} --add --at 00:30 "text"`)}\n${warnings}`;
|
|
141
|
+
const names = result.notes.map((note) => participantName(result, note.participant));
|
|
142
|
+
const width = Math.max(...names.map((name) => name.length));
|
|
143
|
+
return `${[`Reviewer notes on run ${result.run}: ${result.notes.length}`, ...result.notes.map((note, index) => noteLine(note, names[index], width))].join("\n")}\n${warnings}`;
|
|
144
|
+
}
|
|
@@ -152,7 +152,7 @@ async function observeRun(io, options, command) {
|
|
|
152
152
|
...(openResult.command ? { openCommand: openResult.command } : {}),
|
|
153
153
|
warnings: [
|
|
154
154
|
...rendered.warnings,
|
|
155
|
-
"Observer is served
|
|
155
|
+
"Observer is served over loopback http on 127.0.0.1; only this run's bundle directory is exposed, and the one change the page can make is adding a reviewer note.",
|
|
156
156
|
...(openResult.warning ? [openResult.warning] : []),
|
|
157
157
|
],
|
|
158
158
|
};
|
|
@@ -7,6 +7,7 @@ import { warnAndQueue } from "../deprecations.js";
|
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
|
9
9
|
import { routeOf } from "../../study/plan.js";
|
|
10
|
+
import { computerUseParticipantCount } from "../../study/routing.js";
|
|
10
11
|
import { computerUseRouteRun } from "./study-route-computer-use.js";
|
|
11
12
|
import { runRoute } from "./study-route-run.js";
|
|
12
13
|
import { scriptedRouteRun } from "./study-route-scripted.js";
|
|
@@ -130,7 +131,7 @@ export async function runStudyCommand(args) {
|
|
|
130
131
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
131
132
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
132
133
|
}
|
|
133
|
-
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route);
|
|
134
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
134
135
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
135
136
|
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
136
137
|
}
|
package/dist/cli/findings.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// `humanish.analysis-findings.v1` projection and its text. The analysis artifact stays the record;
|
|
3
3
|
// this reads the version `analyze show` selects and never starts an analysis.
|
|
4
4
|
import path from "node:path";
|
|
5
|
+
import { admittingMaxCost, costRefusalText } from "../analysis/admission.js";
|
|
5
6
|
import { DEFAULT_ANALYSIS_MAX_COST_USD } from "../analysis/automatic-config.js";
|
|
6
7
|
import { loadAnalysis } from "../analysis/load.js";
|
|
7
8
|
import { projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
@@ -199,9 +200,11 @@ export function analysisFindings(source) {
|
|
|
199
200
|
const outcome = analysisOutcomeText(job);
|
|
200
201
|
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? "codex" : undefined));
|
|
201
202
|
}
|
|
203
|
+
if (job?.state === "skipped" && job.admission)
|
|
204
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${costRefusalText(job.admission)}`, cli(`analyze --run ${source.runId}${source.cwdFlag} --max-cost ${admittingMaxCost(job.admission)}`));
|
|
202
205
|
if (job?.state === "skipped") {
|
|
203
206
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
204
|
-
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run
|
|
207
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run says why, and for a cost refusal names the --max-cost that admits it." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
205
208
|
? null
|
|
206
209
|
: `${analyzeCommand(source, undefined)}${refused ? " --dry-run" : ""}`);
|
|
207
210
|
}
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -138,4 +138,5 @@ export declare function analysisOutcomeText(analysis: {
|
|
|
138
138
|
state: string;
|
|
139
139
|
reason: string | null;
|
|
140
140
|
}): string;
|
|
141
|
+
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
141
142
|
export declare function automaticAnalysisEnvelope<T>(result: T): T;
|
package/dist/cli/io.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { automaticAnalysisSucceeded,
|
|
2
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js";
|
|
2
|
+
import { admittingMaxCost, costRefusalText, refusedCost, } from "../analysis/admission.js";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
5
5
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
@@ -188,13 +188,14 @@ export function writeResult(command, io, result, formatHuman) {
|
|
|
188
188
|
writeHuman(command, io, formatHuman(output));
|
|
189
189
|
if (output !== null && typeof output === "object" && "automaticAnalysis" in output) {
|
|
190
190
|
const analysis = output.automaticAnalysis;
|
|
191
|
-
|
|
191
|
+
const cost = analysis?.result?.admission && refusedCost(analysis.result.admission);
|
|
192
|
+
if (cost)
|
|
193
|
+
io.writeOut(`analysis: refused before it started. ${refusedAnalysisText(cost, analysis.result?.run ?? "latest")}\n`);
|
|
194
|
+
else if (analysis)
|
|
192
195
|
io.writeOut(`analysis: ${analysisOutcomeText(analysis)}\n`);
|
|
193
196
|
const rejected = analysis?.result?.rejectedOutputPath;
|
|
194
197
|
if (rejected)
|
|
195
198
|
io.writeOut(`analysis: ${analysis.result?.error?.code ?? "rejected"}; the rejected output is kept locally at ${rejected}\n`);
|
|
196
|
-
if (defaultAnalysisOverBudget(output))
|
|
197
|
-
io.writeOut(overBudgetAnalysisHint(output));
|
|
198
199
|
}
|
|
199
200
|
}
|
|
200
201
|
markInvocationEnvelopeWritten(command);
|
|
@@ -286,7 +287,7 @@ const ANALYSIS_REASON_TEXT = {
|
|
|
286
287
|
AUTOMATIC_ANALYSIS_SOURCE_UNAVAILABLE: "skipped because the run's evidence could not be read",
|
|
287
288
|
AUTOMATIC_ANALYSIS_ALREADY_REQUESTED: "skipped because this run's analysis was already requested",
|
|
288
289
|
AUTOMATIC_ANALYSIS_BUSY: "skipped because another analysis is running",
|
|
289
|
-
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "
|
|
290
|
+
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "refused before it started: its configuration, question, evidence or cost did not pass admission",
|
|
290
291
|
AUTOMATIC_ANALYSIS_CLEANUP_UNCONFIRMED: "not started because the run's cleanup was not confirmed",
|
|
291
292
|
AUTOMATIC_ANALYSIS_REUSED: "complete, reusing an earlier analysis of the same evidence",
|
|
292
293
|
AUTOMATIC_ANALYSIS_LIMITATIONS: "partial, because it covered only part of the evidence",
|
|
@@ -305,13 +306,11 @@ export function analysisOutcomeText(analysis) {
|
|
|
305
306
|
return analysis.state;
|
|
306
307
|
return ANALYSIS_REASON_TEXT[analysis.reason] ?? `${analysis.state} (${analysis.reason})`;
|
|
307
308
|
}
|
|
308
|
-
/**
|
|
309
|
-
function
|
|
310
|
-
|
|
311
|
-
const suggested = Math.ceil(estimate ?? DEFAULT_ANALYSIS_MAX_COST_USD + 1);
|
|
312
|
-
const shown = estimate === null ? "" : ` ($${estimate.toFixed(2)})`;
|
|
313
|
-
return `analysis: its estimate${shown} is over the default $${DEFAULT_ANALYSIS_MAX_COST_USD} cap, so no request was sent. To analyze this run: ${cli(`analyze --run ${result.runId ?? "latest"} --max-cost ${suggested}`)}\n`;
|
|
309
|
+
/** What a refused analysis would cost and the command that runs it with a cap that admits it. */
|
|
310
|
+
function refusedAnalysisText(cost, runId) {
|
|
311
|
+
return `${costRefusalText(cost)} To run it, raise the cap: ${cli(`analyze --run ${runId} --max-cost ${admittingMaxCost(cost)}`)}`;
|
|
314
312
|
}
|
|
313
|
+
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
315
314
|
export function automaticAnalysisEnvelope(result) {
|
|
316
315
|
if (result === null ||
|
|
317
316
|
typeof result !== "object" ||
|
package/dist/cli/program.js
CHANGED
|
@@ -13,6 +13,7 @@ import { registerCommsCommands } from "./commands/comms.js";
|
|
|
13
13
|
import { registerFeedbackCommands } from "./commands/feedback.js";
|
|
14
14
|
import { registerStudyCommands } from "./commands/study.js";
|
|
15
15
|
import { registerMigrateCommand } from "./commands/migrate.js";
|
|
16
|
+
import { registerNotesCommand } from "./commands/notes.js";
|
|
16
17
|
import { registerObserveCommand } from "./commands/observe.js";
|
|
17
18
|
import { registerCleanupCommand, registerExportCommand, registerReclaimCommand, registerReviewCommand, registerRunCommand, registerRunsCommand, registerRuntimeCommands, registerStatsCommand, registerVerifyCommand, } from "./commands/runs.js";
|
|
18
19
|
import { registerDoctorCommand, registerInitCommand, registerKeysCommand, registerTelemetryCommand, } from "./commands/setup.js";
|
|
@@ -392,6 +393,7 @@ export function createProgram(io = {}) {
|
|
|
392
393
|
registerVerifyCommand(program, cliIo);
|
|
393
394
|
registerReviewCommand(program, cliIo);
|
|
394
395
|
registerAnalyzeCommand(program, cliIo);
|
|
396
|
+
registerNotesCommand(program, cliIo);
|
|
395
397
|
registerFeedbackCommands(program, cliIo);
|
|
396
398
|
registerExportCommand(program, cliIo);
|
|
397
399
|
registerRunsCommand(program, cliIo);
|
|
@@ -1,5 +1,17 @@
|
|
|
1
|
-
/**
|
|
1
|
+
/**
|
|
2
|
+
* Undoes JSON and JS escapes, percent-encoding and HTML character references, one pass each. Each
|
|
3
|
+
* percent escape is one character, the byte's code, so `caf%C3%A9` reads as `café`;
|
|
4
|
+
* decodeEscapesUtf8 reads it as `café`. A scan checks both: either reading can split a value that
|
|
5
|
+
* the other keeps whole, as `%E2%80%80` (U+2000, a space, in UTF-8) does.
|
|
6
|
+
*/
|
|
2
7
|
export declare function decodeEscapes(text: string): string;
|
|
8
|
+
/** decodeEscapes with each run of percent escapes read as UTF-8. */
|
|
9
|
+
export declare function decodeEscapesUtf8(text: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* The UTF-8 reading of text's escapes when it differs from decodeEscapes, which it can only where
|
|
12
|
+
* a percent escape holds a byte of 0x80 or more.
|
|
13
|
+
*/
|
|
14
|
+
export declare function utf8ReadingOf(text: string, decoded: string): string | undefined;
|
|
3
15
|
/** How scanEncodedText reads text. */
|
|
4
16
|
export interface EncodedTextScanOptions {
|
|
5
17
|
/** Do not count an encoded binary run as opaque. */
|
|
@@ -14,7 +26,8 @@ export interface EncodedTextScan {
|
|
|
14
26
|
readonly opaque: boolean;
|
|
15
27
|
}
|
|
16
28
|
/**
|
|
17
|
-
* Scans text for secrets as written, after decodeEscapes
|
|
29
|
+
* Scans text for secrets as written, after decodeEscapes (also with percent escapes read as UTF-8)
|
|
30
|
+
* and transfer escapes, and inside each
|
|
18
31
|
* base64 (standard, URL-safe or line-wrapped) and hex run. A run that decodes to text is scanned in
|
|
19
32
|
* turn. A run that decodes to an archive, or to other binary past MIN_OPAQUE_BASE64_RUN characters,
|
|
20
33
|
* is opaque unless the caller allows opaque runs.
|
|
@@ -44,13 +44,74 @@ function codePoint(value, original) {
|
|
|
44
44
|
? String.fromCodePoint(value)
|
|
45
45
|
: original;
|
|
46
46
|
}
|
|
47
|
-
|
|
47
|
+
// The length of the UTF-8 sequence a lead byte starts, or 0 for a byte that starts none.
|
|
48
|
+
function sequenceLength(lead) {
|
|
49
|
+
if (lead < 0x80)
|
|
50
|
+
return 1;
|
|
51
|
+
if (lead >= 0xc2 && lead <= 0xdf)
|
|
52
|
+
return 2;
|
|
53
|
+
if (lead >= 0xe0 && lead <= 0xef)
|
|
54
|
+
return 3;
|
|
55
|
+
if (lead >= 0xf0 && lead <= 0xf4)
|
|
56
|
+
return 4;
|
|
57
|
+
return 0;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* A run of percent escapes read as UTF-8, as a browser reads a URL. A byte that starts no valid
|
|
61
|
+
* sequence stands for itself, the character with that code, as decodeEscapes reads every byte.
|
|
62
|
+
*/
|
|
63
|
+
function decodePercentRun(run) {
|
|
64
|
+
const bytes = Buffer.from(run.replace(/%/g, ""), "hex");
|
|
65
|
+
let text = "";
|
|
66
|
+
for (let at = 0; at < bytes.length;) {
|
|
67
|
+
const length = sequenceLength(bytes[at]);
|
|
68
|
+
const sequence = length > 0 ? bytes.subarray(at, at + length) : undefined;
|
|
69
|
+
const decoded = sequence?.length === length ? sequence.toString("utf8") : undefined;
|
|
70
|
+
// Node writes the replacement character for an overlong form, a surrogate half or a bad
|
|
71
|
+
// continuation byte, so a sequence counts only when its character encodes back to its bytes.
|
|
72
|
+
if (decoded !== undefined && Buffer.from(decoded, "utf8").equals(sequence)) {
|
|
73
|
+
text += decoded;
|
|
74
|
+
at += length;
|
|
75
|
+
}
|
|
76
|
+
else {
|
|
77
|
+
text += String.fromCharCode(bytes[at]);
|
|
78
|
+
at += 1;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return text;
|
|
82
|
+
}
|
|
83
|
+
function decodePercentBytes(run) {
|
|
84
|
+
return Buffer.from(run.replace(/%/g, ""), "hex").toString("latin1");
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Undoes JSON and JS escapes, percent-encoding and HTML character references, one pass each. Each
|
|
88
|
+
* percent escape is one character, the byte's code, so `caf%C3%A9` reads as `café`;
|
|
89
|
+
* decodeEscapesUtf8 reads it as `café`. A scan checks both: either reading can split a value that
|
|
90
|
+
* the other keeps whole, as `%E2%80%80` (U+2000, a space, in UTF-8) does.
|
|
91
|
+
*/
|
|
48
92
|
export function decodeEscapes(text) {
|
|
93
|
+
return decodeWith(text, decodePercentBytes);
|
|
94
|
+
}
|
|
95
|
+
/** decodeEscapes with each run of percent escapes read as UTF-8. */
|
|
96
|
+
export function decodeEscapesUtf8(text) {
|
|
97
|
+
return decodeWith(text, decodePercentRun);
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* The UTF-8 reading of text's escapes when it differs from decodeEscapes, which it can only where
|
|
101
|
+
* a percent escape holds a byte of 0x80 or more.
|
|
102
|
+
*/
|
|
103
|
+
export function utf8ReadingOf(text, decoded) {
|
|
104
|
+
if (!/%[89a-f][0-9a-f]/i.test(text))
|
|
105
|
+
return undefined;
|
|
106
|
+
const utf8 = decodeEscapesUtf8(text);
|
|
107
|
+
return utf8 === decoded ? undefined : utf8;
|
|
108
|
+
}
|
|
109
|
+
function decodeWith(text, percent) {
|
|
49
110
|
return text
|
|
50
111
|
.replace(/\\u([0-9a-f]{4})/gi, (match, hex) => codePoint(Number.parseInt(hex, 16), match))
|
|
51
112
|
.replace(/\\x([0-9a-f]{2})/gi, (match, hex) => codePoint(Number.parseInt(hex, 16), match))
|
|
52
113
|
.replace(/\\\//g, "/")
|
|
53
|
-
.replace(
|
|
114
|
+
.replace(/(?:%[0-9a-f]{2})+/gi, percent)
|
|
54
115
|
.replace(/&#x([0-9a-f]{1,6});?/gi, (match, hex) => codePoint(Number.parseInt(hex, 16), match))
|
|
55
116
|
.replace(/&#(\d{1,7});?/g, (match, digits) => codePoint(Number.parseInt(digits, 10), match))
|
|
56
117
|
.replace(/&([a-z]+);/gi, (match, name) => ASCII_ENTITIES[name.toLowerCase()] ?? match);
|
|
@@ -215,7 +276,8 @@ function inspectDecoded(bytes, run, options, depth) {
|
|
|
215
276
|
};
|
|
216
277
|
}
|
|
217
278
|
/**
|
|
218
|
-
* Scans text for secrets as written, after decodeEscapes
|
|
279
|
+
* Scans text for secrets as written, after decodeEscapes (also with percent escapes read as UTF-8)
|
|
280
|
+
* and transfer escapes, and inside each
|
|
219
281
|
* base64 (standard, URL-safe or line-wrapped) and hex run. A run that decodes to text is scanned in
|
|
220
282
|
* turn. A run that decodes to an archive, or to other binary past MIN_OPAQUE_BASE64_RUN characters,
|
|
221
283
|
* is opaque unless the caller allows opaque runs.
|
|
@@ -223,10 +285,12 @@ function inspectDecoded(bytes, run, options, depth) {
|
|
|
223
285
|
export function scanEncodedText(text, options = {}, depth = 0) {
|
|
224
286
|
const matches = matcherOf(options);
|
|
225
287
|
const decoded = decodeEscapes(text);
|
|
288
|
+
const utf8 = utf8ReadingOf(text, decoded);
|
|
226
289
|
const expanded = decodeTransferEscapes(decoded);
|
|
227
290
|
// A decoding that returns the same string would match the same way, so it is not matched again.
|
|
228
291
|
if (matches(text) ||
|
|
229
292
|
(decoded !== text && matches(decoded)) ||
|
|
293
|
+
(utf8 !== undefined && matches(utf8)) ||
|
|
230
294
|
(expanded !== decoded && matches(expanded)))
|
|
231
295
|
return SENSITIVE;
|
|
232
296
|
const runs = [];
|
|
@@ -277,7 +341,7 @@ export function scanEncodedText(text, options = {}, depth = 0) {
|
|
|
277
341
|
// Bump when scanEncodedText, decodeEscapes or the sensitive patterns change what they return. The
|
|
278
342
|
// cache lives in one process, so the version guards results across a hot reload or a test that
|
|
279
343
|
// swaps the scanner.
|
|
280
|
-
const ENCODED_SCAN_VERSION =
|
|
344
|
+
const ENCODED_SCAN_VERSION = 4;
|
|
281
345
|
// Distinct files one process verifies in a burst (a run's files, a serve library's runs).
|
|
282
346
|
const SCAN_CACHE_LIMIT = 256;
|
|
283
347
|
const scanCache = new Map();
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
3
|
-
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
3
|
+
* `\u` escapes) and twice, its UTF-8 bytes read one per character, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
4
4
|
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
5
5
|
* is not ordinary text.
|
|
6
6
|
*/
|
|
@@ -27,7 +27,7 @@ function base64Middles(bytes, encoding) {
|
|
|
27
27
|
}
|
|
28
28
|
/**
|
|
29
29
|
* A value as written, and percent-encoded, JSON-escaped once (also with non-ASCII characters as
|
|
30
|
-
* `\u` escapes) and twice, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
30
|
+
* `\u` escapes) and twice, its UTF-8 bytes read one per character, base64 at each byte offset, base64url and hex. Twice, because a JSON event can carry a command's JSON output as a string. An
|
|
31
31
|
* escaped form is searched for at any length, as the value is: it holds a backslash or a `%`, so it
|
|
32
32
|
* is not ordinary text.
|
|
33
33
|
*/
|
|
@@ -52,8 +52,29 @@ export function encodedForms(value) {
|
|
|
52
52
|
...base64Middles(bytes, "base64"),
|
|
53
53
|
...base64Middles(bytes, "base64url"),
|
|
54
54
|
];
|
|
55
|
+
// decodeEscapes reads each percent escape as one character, so a percent-encoded non-ASCII value
|
|
56
|
+
// reads as its UTF-8 bytes one per character, é as é.
|
|
57
|
+
const bytewise = bytes.toString("latin1");
|
|
58
|
+
if (bytewise !== value)
|
|
59
|
+
escaped.push(bytewise);
|
|
55
60
|
return [value, ...escaped, ...binary.filter((form) => form.length >= MIN_ENCODED_FORM)];
|
|
56
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Whether [at, end) overlaps a marker. The markers are one regex's matches, so they are sorted and
|
|
64
|
+
* disjoint: only the first marker that ends after `at` can also start before `end`.
|
|
65
|
+
*/
|
|
66
|
+
function overlapsMarker(markers, at, end) {
|
|
67
|
+
let low = 0;
|
|
68
|
+
let high = markers.length;
|
|
69
|
+
while (low < high) {
|
|
70
|
+
const middle = (low + high) >> 1;
|
|
71
|
+
if (markers[middle][1] > at)
|
|
72
|
+
high = middle;
|
|
73
|
+
else
|
|
74
|
+
low = middle + 1;
|
|
75
|
+
}
|
|
76
|
+
return low < markers.length && markers[low][0] < end;
|
|
77
|
+
}
|
|
57
78
|
/** Every occurrence of every form, outside the markers, merged where they overlap or touch. */
|
|
58
79
|
function secretSpans(text, forms) {
|
|
59
80
|
const markers = [];
|
|
@@ -65,7 +86,7 @@ function secretSpans(text, forms) {
|
|
|
65
86
|
for (const form of forms) {
|
|
66
87
|
for (let at = text.indexOf(form); at !== -1; at = text.indexOf(form, at + 1)) {
|
|
67
88
|
const end = at + form.length;
|
|
68
|
-
if (!markers
|
|
89
|
+
if (!overlapsMarker(markers, at, end))
|
|
69
90
|
spans.push([at, end]);
|
|
70
91
|
}
|
|
71
92
|
}
|
package/dist/feedback/draft.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { RunBundle, RunFeedbackCandidate } from "../run/bundle.js";
|
|
2
2
|
import type { PreparedRunArtifactPaths } from "../run/paths.js";
|
|
3
3
|
import { loadRunBundlePrepared } from "../run/locate.js";
|
|
4
|
+
import { type RunNotes } from "../run/notes.js";
|
|
4
5
|
export declare const FEEDBACK_SCHEMA = "humanish.feedback.v1";
|
|
5
6
|
export interface FeedbackDraft {
|
|
6
7
|
schema: typeof FEEDBACK_SCHEMA;
|
|
@@ -35,6 +36,19 @@ export interface FeedbackDraft {
|
|
|
35
36
|
idempotency_key: string;
|
|
36
37
|
proposed_next_state: RunFeedbackCandidate["proposed_next_state"];
|
|
37
38
|
acceptance_proof: string[];
|
|
39
|
+
/** Notes a person added while reviewing the recording, in run clock order. Absent with none. */
|
|
40
|
+
reviewer_notes?: Array<{
|
|
41
|
+
id: string;
|
|
42
|
+
/** The moment on the run clock, as the Observer shows it: `02:31`. */
|
|
43
|
+
at: string;
|
|
44
|
+
at_ms: number;
|
|
45
|
+
/** The stream id the note file records, or null for a note on the whole run. */
|
|
46
|
+
participant: string | null;
|
|
47
|
+
/** The participant's caption, as the Observer names it. */
|
|
48
|
+
participant_caption: string | null;
|
|
49
|
+
author: string;
|
|
50
|
+
text: string;
|
|
51
|
+
}>;
|
|
38
52
|
}
|
|
39
53
|
export interface FeedbackDraftOptions {
|
|
40
54
|
/** A candidate id from `feedback list`. Absent: the first usable candidate, as before. */
|
|
@@ -53,6 +67,11 @@ export interface FeedbackRunContext {
|
|
|
53
67
|
}
|
|
54
68
|
export declare function buildDraft(bundle: RunBundle, bundlePath: string, candidateId?: string): FeedbackDraft;
|
|
55
69
|
export declare function buildAnalysisDraft(context: FeedbackRunContext, options: FeedbackDraftOptions): Promise<FeedbackDraft | null>;
|
|
70
|
+
/**
|
|
71
|
+
* The draft with the run's reviewer notes and each note's file cited. The caller passes the set it
|
|
72
|
+
* read and checked for sharing (readNotesForSharing).
|
|
73
|
+
*/
|
|
74
|
+
export declare function withReviewerNotes(draft: FeedbackDraft, notes: RunNotes | null, bundle: RunBundle, runRoot: string): FeedbackDraft;
|
|
56
75
|
export declare function isUsableFeedbackCandidate(candidate: unknown): candidate is RunFeedbackCandidate;
|
|
57
76
|
export declare function renderMarkdown(draft: FeedbackDraft, repo: string): string;
|
|
58
77
|
export {};
|