humanish 0.139.0 → 0.140.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +5 -4
- package/dist/actors/codex/operator-instructions.d.ts +5 -0
- package/dist/actors/codex/operator-instructions.js +9 -0
- package/dist/actors/codex/restricted-launch.d.ts +10 -2
- package/dist/actors/codex/restricted-launch.js +65 -16
- package/dist/actors/codex/restricted-participant.d.ts +4 -0
- package/dist/actors/codex/restricted-participant.js +9 -0
- package/dist/actors/codex/restricted-policy.d.ts +5 -2
- package/dist/actors/codex/restricted-policy.js +90 -72
- package/dist/actors/codex/restricted-session.d.ts +4 -0
- package/dist/actors/codex/restricted-session.js +8 -0
- package/dist/analysis/automatic-completion.js +24 -0
- package/dist/analysis/automatic-config.d.ts +15 -6
- package/dist/analysis/automatic-config.js +25 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -3
- package/dist/cli/commands/study-route-scripted.js +3 -4
- package/dist/cli/commands/study-route-shared-world.js +2 -3
- package/dist/cli/commands/study-route-terminal.js +3 -4
- package/dist/cli/commands/study-run.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +3 -0
- package/dist/cli/findings.d.ts +17 -14
- package/dist/cli/findings.js +37 -31
- package/dist/cli/io.d.ts +5 -1
- package/dist/cli/io.js +11 -2
- package/dist/cli/run-findings.d.ts +19 -0
- package/dist/cli/run-findings.js +106 -0
- package/dist/guest/bootstrap.d.ts +4 -2
- package/dist/guest/bootstrap.js +15 -7
- package/dist/guest/display-config.d.ts +31 -0
- package/dist/guest/display-config.js +35 -0
- package/dist/guest/runtime-desktop.d.ts +9 -3
- package/dist/guest/runtime-desktop.js +59 -25
- package/dist/guest/runtime.d.ts +2 -1
- package/dist/guest/runtime.js +1 -1
- package/dist/guest-runtime-main.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +10 -17
- package/dist/routes/computer-use/local-vm.js +34 -1
- package/dist/routes/computer-use/plan.js +1 -1
- package/dist/routes/scripted/plan.js +1 -1
- package/dist/routes/shared-world/plan.js +1 -1
- package/dist/routes/terminal/plan.js +1 -1
- package/dist/run/route-shell.js +1 -1
- package/dist/study/doctor.d.ts +3 -0
- package/dist/study/doctor.js +22 -7
- package/dist/study/init-templates.js +9 -3
- package/dist/study/mobile-emulation.d.ts +27 -0
- package/dist/study/mobile-emulation.js +60 -0
- package/dist/study/plan.js +1 -1
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/summary.js +1 -1
- package/dist/study/warnings.js +5 -2
- package/dist/substrates/e2b/cdp-probe.d.ts +2 -6
- package/dist/substrates/e2b/desktop-cdp.d.ts +0 -2
- package/dist/substrates/e2b/desktop-cdp.js +0 -2
- package/dist/substrates/local/firecracker-desktop.d.ts +5 -0
- package/dist/substrates/local/firecracker-desktop.js +7 -1
- package/dist/substrates/local/runtime-config.d.ts +12 -0
- package/dist/substrates/local/runtime-config.js +55 -13
- package/dist/substrates/local/runtime-release.js +8 -8
- package/dist/substrates/local/runtime.js +1 -0
- package/docs/contracts/study-analysis.md +14 -8
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +46 -11
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +45 -26
|
@@ -27,14 +27,29 @@ const SENSITIVE_QUESTION = {
|
|
|
27
27
|
ok: false,
|
|
28
28
|
message: "review.analysis.question contains sensitive text and cannot be sent for analysis.",
|
|
29
29
|
};
|
|
30
|
-
/** The spend cap on an analysis the study did not declare. */
|
|
31
|
-
export const DEFAULT_ANALYSIS_MAX_COST_USD =
|
|
32
|
-
/**
|
|
33
|
-
|
|
30
|
+
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
31
|
+
export const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
32
|
+
/**
|
|
33
|
+
* The analysis a study gets when it declares none. A Codex local-agent participant signs in with a
|
|
34
|
+
* ChatGPT account and needs no API key, so its run is analyzed by the same account; every other
|
|
35
|
+
* participant gets the OpenAI analyst under the default cap. A Claude participant stays on OpenAI:
|
|
36
|
+
* moving it to the Codex account because OPENAI_API_KEY is missing would bill an account the study
|
|
37
|
+
* never named.
|
|
38
|
+
*/
|
|
39
|
+
function undeclaredAnalysis(participant) {
|
|
40
|
+
return participant?.type === "local-agent" && (participant.localAgent ?? "codex") === "codex"
|
|
41
|
+
? { provider: "codex" }
|
|
42
|
+
: { maxCostUsd: DEFAULT_ANALYSIS_MAX_COST_USD };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Called for parsed manifests and direct library configs, before participant execution. An
|
|
46
|
+
* undefined `raw` is the study's default, chosen by its participant (the study's `actor`).
|
|
47
|
+
*/
|
|
48
|
+
export function resolveAutomaticAnalysis(raw, participant) {
|
|
34
49
|
if (raw === false)
|
|
35
50
|
return { ok: true, config: undefined };
|
|
36
51
|
if (raw === undefined)
|
|
37
|
-
raw =
|
|
52
|
+
raw = undeclaredAnalysis(participant);
|
|
38
53
|
if (raw === null ||
|
|
39
54
|
typeof raw !== "object" ||
|
|
40
55
|
Array.isArray(raw) ||
|
|
@@ -117,12 +132,13 @@ export function resolveAutomaticAnalysis(raw) {
|
|
|
117
132
|
};
|
|
118
133
|
}
|
|
119
134
|
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
120
|
-
* participant count it also gives the expected cost range for that many participants
|
|
121
|
-
|
|
135
|
+
* participant count it also gives the expected cost range for that many participants, and with the
|
|
136
|
+
* study's participant it resolves an undeclared analysis the way the run will. */
|
|
137
|
+
export function automaticAnalysisBudget(raw, route, participants, participant) {
|
|
122
138
|
// The synthetic preview has no participants, so nothing is analyzed after it.
|
|
123
139
|
if (route === "preview")
|
|
124
140
|
return undefined;
|
|
125
|
-
const resolved = resolveAutomaticAnalysis(raw);
|
|
141
|
+
const resolved = resolveAutomaticAnalysis(raw, participant);
|
|
126
142
|
if (!resolved.ok || !resolved.config)
|
|
127
143
|
return undefined;
|
|
128
144
|
const { config } = resolved;
|
|
@@ -150,7 +166,7 @@ export function analysisRequestsText(requests) {
|
|
|
150
166
|
}
|
|
151
167
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
152
168
|
if (budget.provider === "codex")
|
|
153
|
-
return `After live runs: Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
169
|
+
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
154
170
|
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
155
171
|
// The participants, and the requests that analyse them when there is more than one.
|
|
156
172
|
const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
|
|
@@ -9,7 +9,7 @@ import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
|
9
9
|
import { onRunShutdown } from "./run-signals.js";
|
|
10
10
|
import { parseStudyCount, parseParticipantIds, parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
11
11
|
import { exposureRequestFromOptions, followDecision, followObserver, planObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
12
|
-
import {
|
|
12
|
+
import { writeRunResult } from "../run-findings.js";
|
|
13
13
|
import { formatCuaStudyHuman } from "./study-format.js";
|
|
14
14
|
import { resolveRouteShouldOpen } from "./study-route-open.js";
|
|
15
15
|
import { refusedResult } from "../../run/study-result.js";
|
|
@@ -236,9 +236,8 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
236
236
|
],
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
|
-
|
|
239
|
+
await writeRunResult(args.command, args.io, args.options.cwd, output, (value) => formatCuaStudyHuman(value, args.config.subject));
|
|
240
240
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
241
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
242
241
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
243
242
|
const followResult = output.observer?.ok
|
|
244
243
|
? output.observer
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
|
-
import { wantsJson
|
|
5
|
-
import {
|
|
4
|
+
import { wantsJson } from "../io.js";
|
|
5
|
+
import { writeRunResult } from "../run-findings.js";
|
|
6
6
|
import { showObserver } from "../observer-follow.js";
|
|
7
7
|
import { formatScriptedStudyHuman } from "./study-format.js";
|
|
8
8
|
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
@@ -34,9 +34,8 @@ export function scriptedRouteRun(args) {
|
|
|
34
34
|
throw new Error(`Expected the scripted route, got ${outcome.route}.`);
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
|
-
|
|
37
|
+
await writeRunResult(args.command, args.io, args.options.cwd, result, (value) => formatScriptedStudyHuman(value, args.config.subject));
|
|
38
38
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
39
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
41
40
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
42
41
|
await showObserver({
|
|
@@ -6,7 +6,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
6
6
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
7
7
|
import { parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
8
8
|
import { followDecision, followObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
9
|
-
import {
|
|
9
|
+
import { writeRunResult } from "../run-findings.js";
|
|
10
10
|
import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
|
|
11
11
|
import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
12
12
|
import { refusedResult } from "../../run/study-result.js";
|
|
@@ -120,9 +120,8 @@ export function sharedWorldRouteRun(args) {
|
|
|
120
120
|
],
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
|
-
|
|
123
|
+
await writeRunResult(args.command, args.io, args.options.cwd, output, formatConcurrentSharedWorldStudyHuman);
|
|
124
124
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
125
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
126
125
|
if (server && output.observer?.ok) {
|
|
127
126
|
await followObserver(args.io, output.observer, server);
|
|
128
127
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
|
-
import { wantsJson
|
|
5
|
-
import {
|
|
4
|
+
import { wantsJson } from "../io.js";
|
|
5
|
+
import { writeRunResult } from "../run-findings.js";
|
|
6
6
|
import { showObserver } from "../observer-follow.js";
|
|
7
7
|
import { formatTerminalStudyHuman } from "./study-format.js";
|
|
8
8
|
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
@@ -34,9 +34,8 @@ export function terminalRouteRun(args) {
|
|
|
34
34
|
throw new Error(`Expected the terminal route, got ${outcome.route}.`);
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
|
-
|
|
37
|
+
await writeRunResult(args.command, args.io, args.options.cwd, result, formatTerminalStudyHuman);
|
|
38
38
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
39
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
41
40
|
await showObserver({
|
|
42
41
|
command: args.command,
|
|
@@ -138,7 +138,7 @@ export async function runStudyCommand(args) {
|
|
|
138
138
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
139
139
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
140
140
|
}
|
|
141
|
-
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
141
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config), config.actor);
|
|
142
142
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
143
143
|
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
144
144
|
}
|
package/dist/cli/doctor.d.ts
CHANGED
|
@@ -53,6 +53,8 @@ export declare function doctor(cwdInput: string, options?: {
|
|
|
53
53
|
keyDeps?: KeyResolutionDeps;
|
|
54
54
|
/** The hosted Codex participant's operator handshake; tests replace it. */
|
|
55
55
|
codexParticipantReadiness?: StudySetupCheckArgs["codexParticipantReadiness"];
|
|
56
|
+
/** The Codex account analyst's setup check; tests replace it. */
|
|
57
|
+
codexAnalysisReadiness?: StudySetupCheckArgs["codexAnalysisReadiness"];
|
|
56
58
|
/** What `--dotenv` loaded, so a key row names the file that supplied it. */
|
|
57
59
|
dotenv?: DotenvLoad;
|
|
58
60
|
}): Promise<DoctorResult>;
|
package/dist/cli/doctor.js
CHANGED
|
@@ -89,6 +89,9 @@ export async function doctor(cwdInput, options = {}) {
|
|
|
89
89
|
...(options.codexParticipantReadiness === undefined
|
|
90
90
|
? {}
|
|
91
91
|
: { codexParticipantReadiness: options.codexParticipantReadiness }),
|
|
92
|
+
...(options.codexAnalysisReadiness === undefined
|
|
93
|
+
? {}
|
|
94
|
+
: { codexAnalysisReadiness: options.codexAnalysisReadiness }),
|
|
92
95
|
})
|
|
93
96
|
: undefined;
|
|
94
97
|
// Without --study, each key row names the studies that need it instead of failing.
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import type { Command } from "commander";
|
|
2
|
-
import type { AutomaticAnalysisOutcome } from "../analysis/job.js";
|
|
3
1
|
import type { AnalysisArtifact, LoadedAnalysis } from "../analysis/types.js";
|
|
4
|
-
import { type
|
|
2
|
+
import { type RunStudyProvenance } from "../run/study-provenance.js";
|
|
5
3
|
export declare const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
6
4
|
type Finding = NonNullable<AnalysisArtifact["result"]>["findings"][number];
|
|
7
5
|
type DesignFinding = NonNullable<NonNullable<AnalysisArtifact["result"]>["designFindings"]>[number];
|
|
@@ -116,6 +114,13 @@ interface FindingsSource {
|
|
|
116
114
|
analysisOffIn?: string;
|
|
117
115
|
/** For a dry run, the study a live run can start from, read when the view is built. */
|
|
118
116
|
liveStudy?: LiveStudy;
|
|
117
|
+
/** The run's study, from its bundle. */
|
|
118
|
+
study?: Pick<RunStudyProvenance, "id" | "path">;
|
|
119
|
+
/**
|
|
120
|
+
* Whether Codex is signed in with a ChatGPT account on this machine, read only for an analysis
|
|
121
|
+
* skipped for a missing OpenAI key, whose next step it picks.
|
|
122
|
+
*/
|
|
123
|
+
codexSignedIn?: boolean;
|
|
119
124
|
}
|
|
120
125
|
/** A study `run` can start live after a dry run. */
|
|
121
126
|
interface LiveStudy {
|
|
@@ -126,24 +131,22 @@ interface LiveStudy {
|
|
|
126
131
|
/** The study file to set `mode: live` in first, when the file still starts as a dry run. */
|
|
127
132
|
setLiveIn?: string;
|
|
128
133
|
}
|
|
134
|
+
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
135
|
+
export declare function cwdFlag(cwd: string): string;
|
|
129
136
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
130
137
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
|
131
138
|
/**
|
|
132
139
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
133
140
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
134
141
|
*/
|
|
135
|
-
export declare function readRunAnalysis(cwd: string, run: string, analysisId?: string): Promise<FindingsSource | null>;
|
|
142
|
+
export declare function readRunAnalysis(cwd: string, run: string, analysisId?: string, deps?: FindingsDeps): Promise<FindingsSource | null>;
|
|
143
|
+
/** What reading a run's findings needs from this machine; tests replace it. */
|
|
144
|
+
export interface FindingsDeps {
|
|
145
|
+
/** Whether Codex is signed in with a ChatGPT account. */
|
|
146
|
+
codexSignedIn?: () => Promise<boolean>;
|
|
147
|
+
}
|
|
136
148
|
/** The view of a run that could not be found. */
|
|
137
149
|
export declare function missingRunFindings(cwd: string, run: string): AnalysisFindings;
|
|
138
150
|
/** The findings of a run's selected analysis, or why it has none. */
|
|
139
|
-
export declare function readRunFindings(cwd: string, run: string, analysisId?: string): Promise<AnalysisFindings>;
|
|
140
|
-
/**
|
|
141
|
-
* The end of a live run's human output, after its `analysis:` line: the findings of the analysis
|
|
142
|
-
* the run just completed. Prints nothing for --json, or when the run's analysis did not complete,
|
|
143
|
-
* since that line already says why.
|
|
144
|
-
*/
|
|
145
|
-
export declare function writeRunFindings(command: Command, io: CliIo, cwd: string, result: {
|
|
146
|
-
runId?: string;
|
|
147
|
-
automaticAnalysis?: AutomaticAnalysisOutcome;
|
|
148
|
-
}): Promise<void>;
|
|
151
|
+
export declare function readRunFindings(cwd: string, run: string, analysisId?: string, deps?: FindingsDeps): Promise<AnalysisFindings>;
|
|
149
152
|
export {};
|
package/dist/cli/findings.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
// A run's analysis findings for `review`, `analyze show` and the end of a live `run
|
|
2
|
-
// `humanish.analysis-findings.v1` projection, read from the run.
|
|
1
|
+
// A run's analysis findings for `review`, `analyze show` and the end of a live `run`
|
|
2
|
+
// (run-findings.ts): the `humanish.analysis-findings.v1` projection, read from the run.
|
|
3
|
+
// findings-text.ts prints it. The
|
|
3
4
|
// analysis artifact stays the record; this reads the version `analyze show` selects and never
|
|
4
5
|
// starts an analysis.
|
|
5
6
|
import path from "node:path";
|
|
@@ -15,12 +16,11 @@ import { resolveStudyManifest } from "../study/discover.js";
|
|
|
15
16
|
import { routeOf } from "../study/routing.js";
|
|
16
17
|
import { plural } from "../run/text.js";
|
|
17
18
|
import { shellQuote } from "../substrates/shell.js";
|
|
18
|
-
import { analysisOutcomeText
|
|
19
|
-
import { formatFindingsSummary } from "./findings-text.js";
|
|
19
|
+
import { analysisOutcomeText } from "./io.js";
|
|
20
20
|
import { cli } from "./invocation.js";
|
|
21
21
|
export const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
22
22
|
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
23
|
-
function cwdFlag(cwd) {
|
|
23
|
+
export function cwdFlag(cwd) {
|
|
24
24
|
if (path.resolve(cwd) === process.cwd())
|
|
25
25
|
return "";
|
|
26
26
|
return ` --cwd ${/^[\w@%+=:,./-]+$/.test(cwd) ? cwd : shellQuote(cwd)}`;
|
|
@@ -152,6 +152,18 @@ function dryRunFindings(source) {
|
|
|
152
152
|
: `set mode: live in ${live.setLiveIn}, then run the next command.`;
|
|
153
153
|
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs: ${howLive}`, cli(`run ${live.run}${source.cwdFlag}`));
|
|
154
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* An analysis skipped because OPENAI_API_KEY is not set. With Codex signed in, the next step is the
|
|
157
|
+
* Codex account analyst, named as an opt-in: the study chose the OpenAI analyst, and switching
|
|
158
|
+
* the account that pays for it is the operator's choice.
|
|
159
|
+
*/
|
|
160
|
+
function missingKeyFindings(source, reason) {
|
|
161
|
+
const opening = "The automatic analysis was skipped because OPENAI_API_KEY is not set.";
|
|
162
|
+
if (source.codexSignedIn !== true)
|
|
163
|
+
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, undefined));
|
|
164
|
+
const file = source.study?.path === undefined ? "the study file" : source.study.path;
|
|
165
|
+
return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, "codex"));
|
|
166
|
+
}
|
|
155
167
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
156
168
|
export function analysisFindings(source) {
|
|
157
169
|
if (source.mode !== undefined && source.mode !== "live")
|
|
@@ -200,6 +212,8 @@ export function analysisFindings(source) {
|
|
|
200
212
|
const refusal = costRefusal(job.admission, `--run ${source.runId}${source.cwdFlag}`, cli);
|
|
201
213
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${refusal.text}`, refusal.command);
|
|
202
214
|
}
|
|
215
|
+
if (job?.state === "skipped" && job.reason === "AUTOMATIC_ANALYSIS_KEY_MISSING")
|
|
216
|
+
return missingKeyFindings(source, job.reason);
|
|
203
217
|
if (job?.state === "skipped") {
|
|
204
218
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
205
219
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run says why, and for a cost refusal names the --max-cost that admits it." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
@@ -253,7 +267,7 @@ async function liveStudyAfterDryRun(cwd, study) {
|
|
|
253
267
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
254
268
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
255
269
|
*/
|
|
256
|
-
export async function readRunAnalysis(cwd, run, analysisId) {
|
|
270
|
+
export async function readRunAnalysis(cwd, run, analysisId, deps = {}) {
|
|
257
271
|
const physical = await resolvePhysicalCwd(cwd);
|
|
258
272
|
const prepared = await resolveRunPath(physical, run).catch(() => null);
|
|
259
273
|
if (!prepared)
|
|
@@ -263,16 +277,30 @@ export async function readRunAnalysis(cwd, run, analysisId) {
|
|
|
263
277
|
const study = bundle === null ? undefined : studyProvenanceOf(bundle.bundle);
|
|
264
278
|
const analysisOffIn = mode === "live" ? await studyAnalysisOff(physical, study) : undefined;
|
|
265
279
|
const liveStudy = mode !== undefined && mode !== "live" ? await liveStudyAfterDryRun(physical, study) : undefined;
|
|
280
|
+
const loaded = await loadAnalysis(prepared, analysisId);
|
|
281
|
+
const codexSignedIn = loaded.automatic?.reason === "AUTOMATIC_ANALYSIS_KEY_MISSING"
|
|
282
|
+
? await (deps.codexSignedIn ?? codexSignedInHere)().catch(() => false)
|
|
283
|
+
: undefined;
|
|
266
284
|
return {
|
|
267
285
|
runId: runIdOf(prepared),
|
|
268
286
|
mode,
|
|
269
287
|
runRoot: prepared.relativeRunRoot,
|
|
270
|
-
loaded
|
|
288
|
+
loaded,
|
|
271
289
|
cwdFlag: cwdFlag(cwd),
|
|
272
290
|
...(analysisOffIn === undefined ? {} : { analysisOffIn }),
|
|
273
291
|
...(liveStudy === undefined ? {} : { liveStudy }),
|
|
292
|
+
...(study === undefined
|
|
293
|
+
? {}
|
|
294
|
+
: { study: { id: study.id, ...(study.path === undefined ? {} : { path: study.path }) } }),
|
|
295
|
+
...(codexSignedIn === undefined ? {} : { codexSignedIn }),
|
|
274
296
|
};
|
|
275
297
|
}
|
|
298
|
+
async function codexSignedInHere() {
|
|
299
|
+
const { detectLocalAgents } = await import("../actors/local-agent/cli.js");
|
|
300
|
+
const agents = await detectLocalAgents({ only: "codex" });
|
|
301
|
+
// The Codex analyst refuses an API-key login, so only a ChatGPT login counts.
|
|
302
|
+
return agents.some((agent) => agent.authStatus === "authenticated" && agent.billing === "account-unknown");
|
|
303
|
+
}
|
|
276
304
|
/** The view of a run that could not be found. */
|
|
277
305
|
export function missingRunFindings(cwd, run) {
|
|
278
306
|
const loaded = {
|
|
@@ -285,29 +313,7 @@ export function missingRunFindings(cwd, run) {
|
|
|
285
313
|
return withoutFindings({ runId: run, mode: undefined, runRoot: "", loaded, cwdFlag: flag }, "unavailable", "ANALYSIS_RUN_NOT_FOUND", `Run ${run} was not found or its storage is unsafe.`, cli(`runs${flag}`));
|
|
286
314
|
}
|
|
287
315
|
/** The findings of a run's selected analysis, or why it has none. */
|
|
288
|
-
export async function readRunFindings(cwd, run, analysisId) {
|
|
289
|
-
const source = await readRunAnalysis(cwd, run, analysisId);
|
|
316
|
+
export async function readRunFindings(cwd, run, analysisId, deps = {}) {
|
|
317
|
+
const source = await readRunAnalysis(cwd, run, analysisId, deps);
|
|
290
318
|
return source ? analysisFindings(source) : missingRunFindings(cwd, run);
|
|
291
319
|
}
|
|
292
|
-
/**
|
|
293
|
-
* The end of a live run's human output, after its `analysis:` line: the findings of the analysis
|
|
294
|
-
* the run just completed. Prints nothing for --json, or when the run's analysis did not complete,
|
|
295
|
-
* since that line already says why.
|
|
296
|
-
*/
|
|
297
|
-
export async function writeRunFindings(command, io, cwd, result) {
|
|
298
|
-
const outcome = result.automaticAnalysis;
|
|
299
|
-
const analysisId = outcome?.result?.analysisId;
|
|
300
|
-
if (wantsJson(command) ||
|
|
301
|
-
result.runId === undefined ||
|
|
302
|
-
analysisId === undefined ||
|
|
303
|
-
(outcome?.state !== "complete" && outcome?.state !== "partial"))
|
|
304
|
-
return;
|
|
305
|
-
try {
|
|
306
|
-
const view = await readRunFindings(cwd, result.runId, analysisId);
|
|
307
|
-
const full = cli(`review --run ${view.runId}${cwdFlag(cwd)}`);
|
|
308
|
-
io.writeOut(`${formatFindingsSummary(view, full).join("\n")}\n`);
|
|
309
|
-
}
|
|
310
|
-
catch {
|
|
311
|
-
// The run's own result is already printed; review reads the same record later.
|
|
312
|
-
}
|
|
313
|
-
}
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -113,7 +113,11 @@ export declare function messageOutput(value: {
|
|
|
113
113
|
export declare function humanError(error: CliError | undefined): HumanOutput;
|
|
114
114
|
/** Prints a human formatter's output: text to stdout, an error to stderr through formatCliError. */
|
|
115
115
|
export declare function writeHuman(command: Command, io: CliIo, human: HumanOutput): void;
|
|
116
|
-
|
|
116
|
+
/**
|
|
117
|
+
* Prints a result: JSON for --json, else the human formatter's text and, for a run, its analysis
|
|
118
|
+
* line. `analysisLines` replaces that line, for a caller that has read what the analysis left.
|
|
119
|
+
*/
|
|
120
|
+
export declare function writeResult<T>(command: Command, io: CliIo, result: T, formatHuman: (result: T) => HumanOutput, analysisLines?: readonly string[]): void;
|
|
117
121
|
export declare function noteRunFacts(command: Command, facts: TelemetryProperties): void;
|
|
118
122
|
/** Exported for tests: the facts writeResult read off a command's result document. */
|
|
119
123
|
export declare function runFactsFor(command: Command): TelemetryProperties;
|
package/dist/cli/io.js
CHANGED
|
@@ -178,7 +178,11 @@ export function writeHuman(command, io, human) {
|
|
|
178
178
|
if (human.error)
|
|
179
179
|
io.writeErr(formatCliError(commandLabel(command), human.error));
|
|
180
180
|
}
|
|
181
|
-
|
|
181
|
+
/**
|
|
182
|
+
* Prints a result: JSON for --json, else the human formatter's text and, for a run, its analysis
|
|
183
|
+
* line. `analysisLines` replaces that line, for a caller that has read what the analysis left.
|
|
184
|
+
*/
|
|
185
|
+
export function writeResult(command, io, result, formatHuman, analysisLines) {
|
|
182
186
|
const output = automaticAnalysisEnvelope(result);
|
|
183
187
|
if (wantsJson(command)) {
|
|
184
188
|
// Queued warnings are on stderr already; a JSON caller gets them in warnings[] as well.
|
|
@@ -186,9 +190,14 @@ export function writeResult(command, io, result, formatHuman) {
|
|
|
186
190
|
}
|
|
187
191
|
else {
|
|
188
192
|
writeHuman(command, io, formatHuman(output));
|
|
193
|
+
if (analysisLines !== undefined) {
|
|
194
|
+
for (const line of analysisLines)
|
|
195
|
+
io.writeOut(`${line}\n`);
|
|
196
|
+
}
|
|
197
|
+
else if (
|
|
189
198
|
// A run refused before it was created has no analysis to report, so its output ends with the
|
|
190
199
|
// refusal. Its JSON keeps the skip record.
|
|
191
|
-
|
|
200
|
+
output !== null &&
|
|
192
201
|
typeof output === "object" &&
|
|
193
202
|
"automaticAnalysis" in output &&
|
|
194
203
|
output.runId !== "not-created") {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
import type { AutomaticAnalysisOutcome } from "../analysis/job.js";
|
|
3
|
+
import { type FindingsDeps } from "./findings.js";
|
|
4
|
+
import { type CliIo, type HumanOutput } from "./io.js";
|
|
5
|
+
interface RunResultShape {
|
|
6
|
+
runId?: string;
|
|
7
|
+
automaticAnalysis?: AutomaticAnalysisOutcome;
|
|
8
|
+
observer?: {
|
|
9
|
+
observerPath?: string;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Writes a finished run's result. After the route's own output it says what the automatic
|
|
14
|
+
* analysis did and the command a person runs next; when the analysis left findings, the top ones.
|
|
15
|
+
* `--json` carries the same message and command on `automaticAnalysis` and the findings summary
|
|
16
|
+
* as `findings`. A run with no analysis record is written as writeResult writes it.
|
|
17
|
+
*/
|
|
18
|
+
export declare function writeRunResult<T extends RunResultShape>(command: Command, io: CliIo, cwd: string, result: T, formatHuman: (result: T) => HumanOutput, deps?: FindingsDeps): Promise<void>;
|
|
19
|
+
export {};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// The end of a live run's output: what its automatic analysis left, read from the run once and
|
|
2
|
+
// written both ways. The human output gets the analysis line, then the top findings or the command
|
|
3
|
+
// that gets them; `--json` gets the same message and command on `automaticAnalysis`, and a bounded
|
|
4
|
+
// `findings` summary when the analysis left findings.
|
|
5
|
+
import { cwdFlag, readRunFindings } from "./findings.js";
|
|
6
|
+
import { formatFindingsSummary } from "./findings-text.js";
|
|
7
|
+
import { cli } from "./invocation.js";
|
|
8
|
+
import { analysisOutcomeText, wantsJson, writeResult } from "./io.js";
|
|
9
|
+
/** At most this many findings go in `run --json`; `review --json` has every one. */
|
|
10
|
+
const TOP_FINDINGS = 5;
|
|
11
|
+
function findingsSummary(view, review, observerPath) {
|
|
12
|
+
return {
|
|
13
|
+
analysisId: view.analysisId,
|
|
14
|
+
status: view.status,
|
|
15
|
+
count: view.findings.length,
|
|
16
|
+
designFindingCount: view.designFindings === null ? null : view.designFindings.length,
|
|
17
|
+
top: view.findings.slice(0, TOP_FINDINGS).map((finding) => ({
|
|
18
|
+
id: finding.id,
|
|
19
|
+
title: finding.title,
|
|
20
|
+
headline: finding.headline,
|
|
21
|
+
impact: finding.impact,
|
|
22
|
+
confidence: finding.confidence,
|
|
23
|
+
participants: finding.affected.map((participant) => participant.streamId),
|
|
24
|
+
})),
|
|
25
|
+
review,
|
|
26
|
+
observer: observerPath === undefined ? null : `${observerPath}#/report`,
|
|
27
|
+
path: view.path,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** What the run's analysis left: the message, the command, the human lines and any findings. */
|
|
31
|
+
async function readRunReport(cwd, result, deps) {
|
|
32
|
+
const outcome = result.automaticAnalysis;
|
|
33
|
+
if (outcome === undefined || result.runId === undefined || result.runId === "not-created")
|
|
34
|
+
return undefined;
|
|
35
|
+
const outcomeLine = `analysis: ${analysisOutcomeText(outcome)}`;
|
|
36
|
+
const plain = {
|
|
37
|
+
message: `The automatic analysis was ${analysisOutcomeText(outcome)}.`,
|
|
38
|
+
nextCommand: null,
|
|
39
|
+
lines: [outcomeLine],
|
|
40
|
+
};
|
|
41
|
+
// A dry run left no evidence, and its own output names the live run.
|
|
42
|
+
if (outcome.reason === "AUTOMATIC_ANALYSIS_DRY_RUN")
|
|
43
|
+
return plain;
|
|
44
|
+
const analysisId = outcome.result?.analysisId;
|
|
45
|
+
const finished = outcome.state === "complete" || outcome.state === "partial";
|
|
46
|
+
let view;
|
|
47
|
+
try {
|
|
48
|
+
view = await readRunFindings(cwd, result.runId, finished ? analysisId : undefined, deps);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return plain;
|
|
52
|
+
}
|
|
53
|
+
const rejected = outcome.result?.rejectedOutputPath;
|
|
54
|
+
const rejectedLine = rejected
|
|
55
|
+
? [
|
|
56
|
+
`analysis: ${outcome.result?.error?.code ?? "rejected"}; the rejected output is kept locally at ${rejected}`,
|
|
57
|
+
]
|
|
58
|
+
: [];
|
|
59
|
+
const review = cli(`review --run ${view.runId}${cwdFlag(cwd)}`);
|
|
60
|
+
if (finished && (view.state === "ready" || view.state === "stale"))
|
|
61
|
+
return {
|
|
62
|
+
message: view.message,
|
|
63
|
+
nextCommand: review,
|
|
64
|
+
lines: [outcomeLine, ...rejectedLine, ...formatFindingsSummary(view, review)],
|
|
65
|
+
findings: findingsSummary(view, review, result.observer?.observerPath),
|
|
66
|
+
};
|
|
67
|
+
// The line reads "analysis: skipped because ...", as it does without the view.
|
|
68
|
+
const said = view.message.replace(/^The automatic analysis (?:was )?/, "");
|
|
69
|
+
return {
|
|
70
|
+
message: view.message,
|
|
71
|
+
nextCommand: view.next,
|
|
72
|
+
lines: [
|
|
73
|
+
`analysis: ${said}`,
|
|
74
|
+
...rejectedLine,
|
|
75
|
+
...(view.next === null ? [] : [`next: ${view.next}`]),
|
|
76
|
+
],
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Writes a finished run's result. After the route's own output it says what the automatic
|
|
81
|
+
* analysis did and the command a person runs next; when the analysis left findings, the top ones.
|
|
82
|
+
* `--json` carries the same message and command on `automaticAnalysis` and the findings summary
|
|
83
|
+
* as `findings`. A run with no analysis record is written as writeResult writes it.
|
|
84
|
+
*/
|
|
85
|
+
export async function writeRunResult(command, io, cwd, result, formatHuman, deps = {}) {
|
|
86
|
+
const report = await readRunReport(cwd, result, deps);
|
|
87
|
+
if (report === undefined || result.automaticAnalysis === undefined) {
|
|
88
|
+
writeResult(command, io, result, formatHuman);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (!wantsJson(command)) {
|
|
92
|
+
writeResult(command, io, result, formatHuman, report.lines);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
writeResult(command, io, {
|
|
96
|
+
...result,
|
|
97
|
+
automaticAnalysis: {
|
|
98
|
+
...result.automaticAnalysis,
|
|
99
|
+
message: report.message,
|
|
100
|
+
nextCommand: report.nextCommand,
|
|
101
|
+
},
|
|
102
|
+
...(report.findings === undefined ? {} : { findings: report.findings }),
|
|
103
|
+
},
|
|
104
|
+
// JSON output never calls the human formatter.
|
|
105
|
+
() => "");
|
|
106
|
+
}
|
|
@@ -3,6 +3,7 @@ import type { Duplex } from "node:stream";
|
|
|
3
3
|
import { type BrowserControlIdentity } from "../browser-control/protocol.js";
|
|
4
4
|
import { createBrowserControlClient } from "../browser-control/client.js";
|
|
5
5
|
import { type GuestMediaConfig } from "./media-config.js";
|
|
6
|
+
import { type GuestDisplayConfig } from "./display-config.js";
|
|
6
7
|
export declare const GUEST_BOOTSTRAP_VERSION = 1;
|
|
7
8
|
export declare const GUEST_BOOTSTRAP_LIMITS: Readonly<{
|
|
8
9
|
bytes: number;
|
|
@@ -18,7 +19,7 @@ export declare const GUEST_BOOTSTRAP_LIMITS: Readonly<{
|
|
|
18
19
|
/** Same entry authority as the local desktop's opaque app-port forward. */
|
|
19
20
|
export declare function validateGuestInitialUrl(value: string): string;
|
|
20
21
|
export declare function guestReadyTimeoutMs(initialUrl?: string): number;
|
|
21
|
-
export declare function encodeGuestBootstrap(identity: BrowserControlIdentity, ready?: boolean, initialUrl?: string, media?: GuestMediaConfig, recording?: DesktopRecordingConfig): Buffer;
|
|
22
|
+
export declare function encodeGuestBootstrap(identity: BrowserControlIdentity, ready?: boolean, initialUrl?: string, media?: GuestMediaConfig, recording?: DesktopRecordingConfig, display?: GuestDisplayConfig): Buffer;
|
|
22
23
|
export declare function parseGuestBootstrap(bytes: Buffer, revision: string, ready?: boolean): BrowserControlIdentity;
|
|
23
24
|
/** Own the first frame and continue refusing early bytes until synchronous handoff. */
|
|
24
25
|
export declare class GuestBootstrapReader {
|
|
@@ -31,6 +32,7 @@ export declare class GuestBootstrapReader {
|
|
|
31
32
|
initialUrl: string | undefined;
|
|
32
33
|
media: GuestMediaConfig | undefined;
|
|
33
34
|
recording: DesktopRecordingConfig | undefined;
|
|
35
|
+
display: GuestDisplayConfig | undefined;
|
|
34
36
|
private resolve;
|
|
35
37
|
private reject;
|
|
36
38
|
private readonly header;
|
|
@@ -52,4 +54,4 @@ export declare class GuestBootstrapReader {
|
|
|
52
54
|
close(): void;
|
|
53
55
|
}
|
|
54
56
|
/** Fixed preface on an already acquired Firecracker stream; no path discovery/retry. */
|
|
55
|
-
export declare function connectGuestBootstrap(stream: Duplex, identity: BrowserControlIdentity, signal: AbortSignal, initialUrl?: string, media?: GuestMediaConfig, recording?: DesktopRecordingConfig): Promise<ReturnType<typeof createBrowserControlClient>>;
|
|
57
|
+
export declare function connectGuestBootstrap(stream: Duplex, identity: BrowserControlIdentity, signal: AbortSignal, initialUrl?: string, media?: GuestMediaConfig, recording?: DesktopRecordingConfig, display?: GuestDisplayConfig): Promise<ReturnType<typeof createBrowserControlClient>>;
|