humanish 0.139.1 → 0.140.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +5 -4
- package/dist/analysis/automatic-completion.js +24 -0
- package/dist/analysis/automatic-config.d.ts +15 -6
- package/dist/analysis/automatic-config.js +25 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -3
- package/dist/cli/commands/study-route-scripted.js +3 -4
- package/dist/cli/commands/study-route-shared-world.js +2 -3
- package/dist/cli/commands/study-route-terminal.js +3 -4
- package/dist/cli/commands/study-run.js +1 -1
- package/dist/cli/doctor.d.ts +2 -0
- package/dist/cli/doctor.js +3 -0
- package/dist/cli/findings.d.ts +17 -14
- package/dist/cli/findings.js +37 -31
- package/dist/cli/io.d.ts +5 -1
- package/dist/cli/io.js +11 -2
- package/dist/cli/run-findings.d.ts +19 -0
- package/dist/cli/run-findings.js +106 -0
- package/dist/guest/bootstrap.d.ts +4 -2
- package/dist/guest/bootstrap.js +15 -7
- package/dist/guest/display-config.d.ts +31 -0
- package/dist/guest/display-config.js +35 -0
- package/dist/guest/runtime-desktop.d.ts +9 -3
- package/dist/guest/runtime-desktop.js +59 -25
- package/dist/guest/runtime.d.ts +2 -1
- package/dist/guest/runtime.js +1 -1
- package/dist/guest-runtime-main.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/fidelity.js +10 -17
- package/dist/routes/computer-use/local-vm.js +34 -1
- package/dist/routes/computer-use/plan.js +1 -1
- package/dist/routes/scripted/plan.js +1 -1
- package/dist/routes/shared-world/plan.js +1 -1
- package/dist/routes/terminal/plan.js +1 -1
- package/dist/run/route-shell.js +1 -1
- package/dist/study/doctor.js +11 -6
- package/dist/study/init-templates.js +9 -3
- package/dist/study/mobile-emulation.d.ts +27 -0
- package/dist/study/mobile-emulation.js +60 -0
- package/dist/study/plan.js +1 -1
- package/dist/study/preflight-result.js +1 -1
- package/dist/study/summary.js +1 -1
- package/dist/study/warnings.js +5 -2
- package/dist/substrates/e2b/cdp-probe.d.ts +2 -6
- package/dist/substrates/e2b/desktop-cdp.d.ts +0 -2
- package/dist/substrates/e2b/desktop-cdp.js +0 -2
- package/dist/substrates/local/firecracker-desktop.d.ts +5 -0
- package/dist/substrates/local/firecracker-desktop.js +7 -1
- package/dist/substrates/local/runtime-config.d.ts +12 -0
- package/dist/substrates/local/runtime-config.js +55 -13
- package/dist/substrates/local/runtime-release.js +8 -8
- package/dist/substrates/local/runtime.js +1 -0
- package/docs/contracts/study-analysis.md +14 -8
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +46 -11
- package/package.json +1 -1
- package/skills/humanish/SKILL.md +45 -26
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,17 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.140.0: One local study can test phone, tablet and desktop, and every run ends with findings (2026-10-21)
|
|
12
|
+
|
|
13
|
+
humanish 0.140.0 lets a local browser study give each participant its own device. One study with
|
|
14
|
+
`mobile`, `tablet` and `desktop` participants runs them side by side on your machine, with mobile
|
|
15
|
+
emulation for the phone, as hosted desktops already did; `humanish runtime setup` installs the new
|
|
16
|
+
local runtime. A study with a Codex participant is now analyzed by the same Codex account on every
|
|
17
|
+
target, so a run ends with findings without an OpenAI key; the default OpenAI analysis cap is $10;
|
|
18
|
+
a skipped analysis names the command that runs it; and `run --json` carries the findings.
|
|
19
|
+
|
|
20
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.140.0)
|
|
21
|
+
|
|
11
22
|
## 0.139.1: Hosted Codex participants work with an AGENTS.md in the Codex home (2026-10-21)
|
|
12
23
|
|
|
13
24
|
humanish 0.139.1 fixes hosted Codex participants for operators whose Codex home
|
package/README.md
CHANGED
|
@@ -235,18 +235,19 @@ a separately verified copy while preserving the readable original.
|
|
|
235
235
|
participant's report, task outcomes, costs, and how to turn a finding into an issue.
|
|
236
236
|
|
|
237
237
|
For a ranked review of a completed live study, run
|
|
238
|
-
`npx humanish analyze --run latest --max-cost
|
|
238
|
+
`npx humanish analyze --run latest --max-cost 10 --dry-run --json` to inspect
|
|
239
239
|
admission, then remove `--dry-run` to generate findings. This sends selected
|
|
240
240
|
retained evidence to OpenAI and requires `OPENAI_API_KEY`. Findings appear
|
|
241
241
|
beside Participants in Observer, with evidence links and separate participant
|
|
242
242
|
feedback. You can explicitly select a restricted Codex account analyst with
|
|
243
243
|
`npx humanish analyze --run latest --provider codex`; it uses remote inference,
|
|
244
244
|
requires the qualified CLI and account login, and reports dollar cost as unknown.
|
|
245
|
-
|
|
245
|
+
A study whose participant is a Codex `local-agent` uses that analyst by default. See the [analysis contract](docs/contracts/study-analysis.md) for
|
|
246
246
|
coverage limits, estimated cost controls, corrections and sharing behavior.
|
|
247
247
|
|
|
248
|
-
Supported live studies generate findings automatically after participants finish
|
|
249
|
-
|
|
248
|
+
Supported live studies generate findings automatically after participants finish.
|
|
249
|
+
A Codex participant's run is analyzed by the same Codex account; every other run uses
|
|
250
|
+
`gpt-6-astra` with high reasoning and a separate $10 cap on the expected cost.
|
|
250
251
|
This is additional to participant execution costs, and is not a hard provider
|
|
251
252
|
billing cap. To disable the extra request:
|
|
252
253
|
|
|
@@ -96,10 +96,34 @@ function defaultAnalysisOverBudget(result) {
|
|
|
96
96
|
value.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED" &&
|
|
97
97
|
value.result?.error?.code === "analysis_budget_exceeded");
|
|
98
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* Codex refusals that stop the analyst before any request: the account or CLI cannot start it here.
|
|
101
|
+
* A hosted Codex participant can run where the analyst cannot, for example with a keyring-only
|
|
102
|
+
* login, since the analyst reads auth.json from a private home.
|
|
103
|
+
*/
|
|
104
|
+
const CODEX_ANALYST_SETUP_CODES = [
|
|
105
|
+
"analysis_codex_unavailable",
|
|
106
|
+
"analysis_codex_unsupported_version",
|
|
107
|
+
"analysis_codex_incompatible_release",
|
|
108
|
+
"analysis_codex_unsupported_platform",
|
|
109
|
+
"analysis_codex_login_required",
|
|
110
|
+
"analysis_codex_unsupported_auth",
|
|
111
|
+
"analysis_codex_busy",
|
|
112
|
+
];
|
|
113
|
+
/** True when a default Codex analyst could not start, so the study's author never asked for it. */
|
|
114
|
+
function defaultCodexAnalystUnavailable(result) {
|
|
115
|
+
const value = result.automaticAnalysis;
|
|
116
|
+
return (result.automaticAnalysisTrigger === "default" &&
|
|
117
|
+
value?.state === "failed" &&
|
|
118
|
+
value.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" &&
|
|
119
|
+
CODEX_ANALYST_SETUP_CODES.includes(value.result?.error?.code ?? ""));
|
|
120
|
+
}
|
|
99
121
|
export function automaticAnalysisSucceeded(result) {
|
|
100
122
|
const value = result.automaticAnalysis;
|
|
101
123
|
if (value === undefined)
|
|
102
124
|
return true;
|
|
125
|
+
if (defaultCodexAnalystUnavailable(result))
|
|
126
|
+
return true;
|
|
103
127
|
if (value.state === "skipped")
|
|
104
128
|
return (value.reason === "AUTOMATIC_ANALYSIS_DRY_RUN" ||
|
|
105
129
|
(result.automaticAnalysisTrigger === "default" &&
|
|
@@ -17,10 +17,18 @@ export type StudyAnalysis = StudyAnalysisSettings & ({
|
|
|
17
17
|
maxCostUsd?: null;
|
|
18
18
|
maxOutputTokens?: null;
|
|
19
19
|
});
|
|
20
|
-
/** The spend cap on an analysis the study did not declare. */
|
|
21
|
-
export declare const DEFAULT_ANALYSIS_MAX_COST_USD =
|
|
22
|
-
/**
|
|
23
|
-
export
|
|
20
|
+
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
21
|
+
export declare const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
22
|
+
/** The study's participant, which picks the analyst when the study declares none. */
|
|
23
|
+
export interface AnalysisParticipant {
|
|
24
|
+
readonly type?: string;
|
|
25
|
+
readonly localAgent?: "codex" | "claude";
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Called for parsed manifests and direct library configs, before participant execution. An
|
|
29
|
+
* undefined `raw` is the study's default, chosen by its participant (the study's `actor`).
|
|
30
|
+
*/
|
|
31
|
+
export declare function resolveAutomaticAnalysis(raw: unknown, participant?: AnalysisParticipant): {
|
|
24
32
|
ok: true;
|
|
25
33
|
config: AnalysisConfig | undefined;
|
|
26
34
|
preferLargerOutput?: boolean;
|
|
@@ -48,8 +56,9 @@ export interface AutomaticAnalysisBudget {
|
|
|
48
56
|
refusedFromUsd?: number;
|
|
49
57
|
}
|
|
50
58
|
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
51
|
-
* participant count it also gives the expected cost range for that many participants
|
|
52
|
-
|
|
59
|
+
* participant count it also gives the expected cost range for that many participants, and with the
|
|
60
|
+
* study's participant it resolves an undeclared analysis the way the run will. */
|
|
61
|
+
export declare function automaticAnalysisBudget(raw: unknown, route: StudyRoute, participants?: number, participant?: AnalysisParticipant): AutomaticAnalysisBudget | undefined;
|
|
53
62
|
/**
|
|
54
63
|
* An analysis's requests in words, from their count: undefined for one request. The plan's
|
|
55
64
|
* analysis line and the dry run's worst case both say it.
|
|
@@ -27,14 +27,29 @@ const SENSITIVE_QUESTION = {
|
|
|
27
27
|
ok: false,
|
|
28
28
|
message: "review.analysis.question contains sensitive text and cannot be sent for analysis.",
|
|
29
29
|
};
|
|
30
|
-
/** The spend cap on an analysis the study did not declare. */
|
|
31
|
-
export const DEFAULT_ANALYSIS_MAX_COST_USD =
|
|
32
|
-
/**
|
|
33
|
-
|
|
30
|
+
/** The spend cap on an OpenAI analysis the study did not declare. */
|
|
31
|
+
export const DEFAULT_ANALYSIS_MAX_COST_USD = 10;
|
|
32
|
+
/**
|
|
33
|
+
* The analysis a study gets when it declares none. A Codex local-agent participant signs in with a
|
|
34
|
+
* ChatGPT account and needs no API key, so its run is analyzed by the same account; every other
|
|
35
|
+
* participant gets the OpenAI analyst under the default cap. A Claude participant stays on OpenAI:
|
|
36
|
+
* moving it to the Codex account because OPENAI_API_KEY is missing would bill an account the study
|
|
37
|
+
* never named.
|
|
38
|
+
*/
|
|
39
|
+
function undeclaredAnalysis(participant) {
|
|
40
|
+
return participant?.type === "local-agent" && (participant.localAgent ?? "codex") === "codex"
|
|
41
|
+
? { provider: "codex" }
|
|
42
|
+
: { maxCostUsd: DEFAULT_ANALYSIS_MAX_COST_USD };
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Called for parsed manifests and direct library configs, before participant execution. An
|
|
46
|
+
* undefined `raw` is the study's default, chosen by its participant (the study's `actor`).
|
|
47
|
+
*/
|
|
48
|
+
export function resolveAutomaticAnalysis(raw, participant) {
|
|
34
49
|
if (raw === false)
|
|
35
50
|
return { ok: true, config: undefined };
|
|
36
51
|
if (raw === undefined)
|
|
37
|
-
raw =
|
|
52
|
+
raw = undeclaredAnalysis(participant);
|
|
38
53
|
if (raw === null ||
|
|
39
54
|
typeof raw !== "object" ||
|
|
40
55
|
Array.isArray(raw) ||
|
|
@@ -117,12 +132,13 @@ export function resolveAutomaticAnalysis(raw) {
|
|
|
117
132
|
};
|
|
118
133
|
}
|
|
119
134
|
/** Metadata only: resolving the future live-run budget never reads keys or dispatches. With a
|
|
120
|
-
* participant count it also gives the expected cost range for that many participants
|
|
121
|
-
|
|
135
|
+
* participant count it also gives the expected cost range for that many participants, and with the
|
|
136
|
+
* study's participant it resolves an undeclared analysis the way the run will. */
|
|
137
|
+
export function automaticAnalysisBudget(raw, route, participants, participant) {
|
|
122
138
|
// The synthetic preview has no participants, so nothing is analyzed after it.
|
|
123
139
|
if (route === "preview")
|
|
124
140
|
return undefined;
|
|
125
|
-
const resolved = resolveAutomaticAnalysis(raw);
|
|
141
|
+
const resolved = resolveAutomaticAnalysis(raw, participant);
|
|
126
142
|
if (!resolved.ok || !resolved.config)
|
|
127
143
|
return undefined;
|
|
128
144
|
const { config } = resolved;
|
|
@@ -150,7 +166,7 @@ export function analysisRequestsText(requests) {
|
|
|
150
166
|
}
|
|
151
167
|
export function formatAutomaticAnalysisBudget(budget) {
|
|
152
168
|
if (budget.provider === "codex")
|
|
153
|
-
return `After live runs: Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
169
|
+
return `After live runs: ${budget.trigger === "default" ? "default " : ""}Codex account analysis · ${budget.model} · separate restricted analyst with remote inference. Account limits apply; dollar cost and output-token ceiling are unknown. Set review.analysis: false to disable.`;
|
|
154
170
|
const rule = admissionRule(`$${budget.maxCostUsd}`);
|
|
155
171
|
// The participants, and the requests that analyse them when there is more than one.
|
|
156
172
|
const requests = analysisRequestsText(analysisRequestCount(Math.min(budget.participants ?? 0, EVIDENCE_LIMITS.participants)));
|
|
@@ -9,7 +9,7 @@ import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
|
9
9
|
import { onRunShutdown } from "./run-signals.js";
|
|
10
10
|
import { parseStudyCount, parseParticipantIds, parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
11
11
|
import { exposureRequestFromOptions, followDecision, followObserver, planObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
12
|
-
import {
|
|
12
|
+
import { writeRunResult } from "../run-findings.js";
|
|
13
13
|
import { formatCuaStudyHuman } from "./study-format.js";
|
|
14
14
|
import { resolveRouteShouldOpen } from "./study-route-open.js";
|
|
15
15
|
import { refusedResult } from "../../run/study-result.js";
|
|
@@ -236,9 +236,8 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
236
236
|
],
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
|
-
|
|
239
|
+
await writeRunResult(args.command, args.io, args.options.cwd, output, (value) => formatCuaStudyHuman(value, args.config.subject));
|
|
240
240
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
241
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
242
241
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
243
242
|
const followResult = output.observer?.ok
|
|
244
243
|
? output.observer
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
|
-
import { wantsJson
|
|
5
|
-
import {
|
|
4
|
+
import { wantsJson } from "../io.js";
|
|
5
|
+
import { writeRunResult } from "../run-findings.js";
|
|
6
6
|
import { showObserver } from "../observer-follow.js";
|
|
7
7
|
import { formatScriptedStudyHuman } from "./study-format.js";
|
|
8
8
|
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
@@ -34,9 +34,8 @@ export function scriptedRouteRun(args) {
|
|
|
34
34
|
throw new Error(`Expected the scripted route, got ${outcome.route}.`);
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
|
-
|
|
37
|
+
await writeRunResult(args.command, args.io, args.options.cwd, result, (value) => formatScriptedStudyHuman(value, args.config.subject));
|
|
38
38
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
39
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
41
40
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
42
41
|
await showObserver({
|
|
@@ -6,7 +6,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
6
6
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
7
7
|
import { parseObserverPort, wantsJson, writeResult, } from "../io.js";
|
|
8
8
|
import { followDecision, followObserver, showObserver, staticObserverOpen, withObserverServer, } from "../observer-follow.js";
|
|
9
|
-
import {
|
|
9
|
+
import { writeRunResult } from "../run-findings.js";
|
|
10
10
|
import { formatConcurrentSharedWorldStudyHuman } from "./study-format.js";
|
|
11
11
|
import { resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
12
12
|
import { refusedResult } from "../../run/study-result.js";
|
|
@@ -120,9 +120,8 @@ export function sharedWorldRouteRun(args) {
|
|
|
120
120
|
],
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
|
-
|
|
123
|
+
await writeRunResult(args.command, args.io, args.options.cwd, output, formatConcurrentSharedWorldStudyHuman);
|
|
124
124
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
125
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
126
125
|
if (server && output.observer?.ok) {
|
|
127
126
|
await followObserver(args.io, output.observer, server);
|
|
128
127
|
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
|
-
import { wantsJson
|
|
5
|
-
import {
|
|
4
|
+
import { wantsJson } from "../io.js";
|
|
5
|
+
import { writeRunResult } from "../run-findings.js";
|
|
6
6
|
import { showObserver } from "../observer-follow.js";
|
|
7
7
|
import { formatTerminalStudyHuman } from "./study-format.js";
|
|
8
8
|
import { observerOpen, resolveRouteShouldOpen, watchFinishedPlan } from "./study-route-open.js";
|
|
@@ -34,9 +34,8 @@ export function terminalRouteRun(args) {
|
|
|
34
34
|
throw new Error(`Expected the terminal route, got ${outcome.route}.`);
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
|
-
|
|
37
|
+
await writeRunResult(args.command, args.io, args.options.cwd, result, formatTerminalStudyHuman);
|
|
38
38
|
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
|
-
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
39
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
41
40
|
await showObserver({
|
|
42
41
|
command: args.command,
|
|
@@ -138,7 +138,7 @@ export async function runStudyCommand(args) {
|
|
|
138
138
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
139
139
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
140
140
|
}
|
|
141
|
-
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
141
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config), config.actor);
|
|
142
142
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
143
143
|
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
144
144
|
}
|
package/dist/cli/doctor.d.ts
CHANGED
|
@@ -53,6 +53,8 @@ export declare function doctor(cwdInput: string, options?: {
|
|
|
53
53
|
keyDeps?: KeyResolutionDeps;
|
|
54
54
|
/** The hosted Codex participant's operator handshake; tests replace it. */
|
|
55
55
|
codexParticipantReadiness?: StudySetupCheckArgs["codexParticipantReadiness"];
|
|
56
|
+
/** The Codex account analyst's setup check; tests replace it. */
|
|
57
|
+
codexAnalysisReadiness?: StudySetupCheckArgs["codexAnalysisReadiness"];
|
|
56
58
|
/** What `--dotenv` loaded, so a key row names the file that supplied it. */
|
|
57
59
|
dotenv?: DotenvLoad;
|
|
58
60
|
}): Promise<DoctorResult>;
|
package/dist/cli/doctor.js
CHANGED
|
@@ -89,6 +89,9 @@ export async function doctor(cwdInput, options = {}) {
|
|
|
89
89
|
...(options.codexParticipantReadiness === undefined
|
|
90
90
|
? {}
|
|
91
91
|
: { codexParticipantReadiness: options.codexParticipantReadiness }),
|
|
92
|
+
...(options.codexAnalysisReadiness === undefined
|
|
93
|
+
? {}
|
|
94
|
+
: { codexAnalysisReadiness: options.codexAnalysisReadiness }),
|
|
92
95
|
})
|
|
93
96
|
: undefined;
|
|
94
97
|
// Without --study, each key row names the studies that need it instead of failing.
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import type { Command } from "commander";
|
|
2
|
-
import type { AutomaticAnalysisOutcome } from "../analysis/job.js";
|
|
3
1
|
import type { AnalysisArtifact, LoadedAnalysis } from "../analysis/types.js";
|
|
4
|
-
import { type
|
|
2
|
+
import { type RunStudyProvenance } from "../run/study-provenance.js";
|
|
5
3
|
export declare const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
6
4
|
type Finding = NonNullable<AnalysisArtifact["result"]>["findings"][number];
|
|
7
5
|
type DesignFinding = NonNullable<NonNullable<AnalysisArtifact["result"]>["designFindings"]>[number];
|
|
@@ -116,6 +114,13 @@ interface FindingsSource {
|
|
|
116
114
|
analysisOffIn?: string;
|
|
117
115
|
/** For a dry run, the study a live run can start from, read when the view is built. */
|
|
118
116
|
liveStudy?: LiveStudy;
|
|
117
|
+
/** The run's study, from its bundle. */
|
|
118
|
+
study?: Pick<RunStudyProvenance, "id" | "path">;
|
|
119
|
+
/**
|
|
120
|
+
* Whether Codex is signed in with a ChatGPT account on this machine, read only for an analysis
|
|
121
|
+
* skipped for a missing OpenAI key, whose next step it picks.
|
|
122
|
+
*/
|
|
123
|
+
codexSignedIn?: boolean;
|
|
119
124
|
}
|
|
120
125
|
/** A study `run` can start live after a dry run. */
|
|
121
126
|
interface LiveStudy {
|
|
@@ -126,24 +131,22 @@ interface LiveStudy {
|
|
|
126
131
|
/** The study file to set `mode: live` in first, when the file still starts as a dry run. */
|
|
127
132
|
setLiveIn?: string;
|
|
128
133
|
}
|
|
134
|
+
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
135
|
+
export declare function cwdFlag(cwd: string): string;
|
|
129
136
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
130
137
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
|
131
138
|
/**
|
|
132
139
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
133
140
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
134
141
|
*/
|
|
135
|
-
export declare function readRunAnalysis(cwd: string, run: string, analysisId?: string): Promise<FindingsSource | null>;
|
|
142
|
+
export declare function readRunAnalysis(cwd: string, run: string, analysisId?: string, deps?: FindingsDeps): Promise<FindingsSource | null>;
|
|
143
|
+
/** What reading a run's findings needs from this machine; tests replace it. */
|
|
144
|
+
export interface FindingsDeps {
|
|
145
|
+
/** Whether Codex is signed in with a ChatGPT account. */
|
|
146
|
+
codexSignedIn?: () => Promise<boolean>;
|
|
147
|
+
}
|
|
136
148
|
/** The view of a run that could not be found. */
|
|
137
149
|
export declare function missingRunFindings(cwd: string, run: string): AnalysisFindings;
|
|
138
150
|
/** The findings of a run's selected analysis, or why it has none. */
|
|
139
|
-
export declare function readRunFindings(cwd: string, run: string, analysisId?: string): Promise<AnalysisFindings>;
|
|
140
|
-
/**
|
|
141
|
-
* The end of a live run's human output, after its `analysis:` line: the findings of the analysis
|
|
142
|
-
* the run just completed. Prints nothing for --json, or when the run's analysis did not complete,
|
|
143
|
-
* since that line already says why.
|
|
144
|
-
*/
|
|
145
|
-
export declare function writeRunFindings(command: Command, io: CliIo, cwd: string, result: {
|
|
146
|
-
runId?: string;
|
|
147
|
-
automaticAnalysis?: AutomaticAnalysisOutcome;
|
|
148
|
-
}): Promise<void>;
|
|
151
|
+
export declare function readRunFindings(cwd: string, run: string, analysisId?: string, deps?: FindingsDeps): Promise<AnalysisFindings>;
|
|
149
152
|
export {};
|
package/dist/cli/findings.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
// A run's analysis findings for `review`, `analyze show` and the end of a live `run
|
|
2
|
-
// `humanish.analysis-findings.v1` projection, read from the run.
|
|
1
|
+
// A run's analysis findings for `review`, `analyze show` and the end of a live `run`
|
|
2
|
+
// (run-findings.ts): the `humanish.analysis-findings.v1` projection, read from the run.
|
|
3
|
+
// findings-text.ts prints it. The
|
|
3
4
|
// analysis artifact stays the record; this reads the version `analyze show` selects and never
|
|
4
5
|
// starts an analysis.
|
|
5
6
|
import path from "node:path";
|
|
@@ -15,12 +16,11 @@ import { resolveStudyManifest } from "../study/discover.js";
|
|
|
15
16
|
import { routeOf } from "../study/routing.js";
|
|
16
17
|
import { plural } from "../run/text.js";
|
|
17
18
|
import { shellQuote } from "../substrates/shell.js";
|
|
18
|
-
import { analysisOutcomeText
|
|
19
|
-
import { formatFindingsSummary } from "./findings-text.js";
|
|
19
|
+
import { analysisOutcomeText } from "./io.js";
|
|
20
20
|
import { cli } from "./invocation.js";
|
|
21
21
|
export const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
22
22
|
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
23
|
-
function cwdFlag(cwd) {
|
|
23
|
+
export function cwdFlag(cwd) {
|
|
24
24
|
if (path.resolve(cwd) === process.cwd())
|
|
25
25
|
return "";
|
|
26
26
|
return ` --cwd ${/^[\w@%+=:,./-]+$/.test(cwd) ? cwd : shellQuote(cwd)}`;
|
|
@@ -152,6 +152,18 @@ function dryRunFindings(source) {
|
|
|
152
152
|
: `set mode: live in ${live.setLiveIn}, then run the next command.`;
|
|
153
153
|
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs: ${howLive}`, cli(`run ${live.run}${source.cwdFlag}`));
|
|
154
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* An analysis skipped because OPENAI_API_KEY is not set. With Codex signed in, the next step is the
|
|
157
|
+
* Codex account analyst, named as an opt-in: the study chose the OpenAI analyst, and switching
|
|
158
|
+
* the account that pays for it is the operator's choice.
|
|
159
|
+
*/
|
|
160
|
+
function missingKeyFindings(source, reason) {
|
|
161
|
+
const opening = "The automatic analysis was skipped because OPENAI_API_KEY is not set.";
|
|
162
|
+
if (source.codexSignedIn !== true)
|
|
163
|
+
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, undefined));
|
|
164
|
+
const file = source.study?.path === undefined ? "the study file" : source.study.path;
|
|
165
|
+
return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, "codex"));
|
|
166
|
+
}
|
|
155
167
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
156
168
|
export function analysisFindings(source) {
|
|
157
169
|
if (source.mode !== undefined && source.mode !== "live")
|
|
@@ -200,6 +212,8 @@ export function analysisFindings(source) {
|
|
|
200
212
|
const refusal = costRefusal(job.admission, `--run ${source.runId}${source.cwdFlag}`, cli);
|
|
201
213
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${refusal.text}`, refusal.command);
|
|
202
214
|
}
|
|
215
|
+
if (job?.state === "skipped" && job.reason === "AUTOMATIC_ANALYSIS_KEY_MISSING")
|
|
216
|
+
return missingKeyFindings(source, job.reason);
|
|
203
217
|
if (job?.state === "skipped") {
|
|
204
218
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
205
219
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run says why, and for a cost refusal names the --max-cost that admits it." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
@@ -253,7 +267,7 @@ async function liveStudyAfterDryRun(cwd, study) {
|
|
|
253
267
|
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
254
268
|
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
255
269
|
*/
|
|
256
|
-
export async function readRunAnalysis(cwd, run, analysisId) {
|
|
270
|
+
export async function readRunAnalysis(cwd, run, analysisId, deps = {}) {
|
|
257
271
|
const physical = await resolvePhysicalCwd(cwd);
|
|
258
272
|
const prepared = await resolveRunPath(physical, run).catch(() => null);
|
|
259
273
|
if (!prepared)
|
|
@@ -263,16 +277,30 @@ export async function readRunAnalysis(cwd, run, analysisId) {
|
|
|
263
277
|
const study = bundle === null ? undefined : studyProvenanceOf(bundle.bundle);
|
|
264
278
|
const analysisOffIn = mode === "live" ? await studyAnalysisOff(physical, study) : undefined;
|
|
265
279
|
const liveStudy = mode !== undefined && mode !== "live" ? await liveStudyAfterDryRun(physical, study) : undefined;
|
|
280
|
+
const loaded = await loadAnalysis(prepared, analysisId);
|
|
281
|
+
const codexSignedIn = loaded.automatic?.reason === "AUTOMATIC_ANALYSIS_KEY_MISSING"
|
|
282
|
+
? await (deps.codexSignedIn ?? codexSignedInHere)().catch(() => false)
|
|
283
|
+
: undefined;
|
|
266
284
|
return {
|
|
267
285
|
runId: runIdOf(prepared),
|
|
268
286
|
mode,
|
|
269
287
|
runRoot: prepared.relativeRunRoot,
|
|
270
|
-
loaded
|
|
288
|
+
loaded,
|
|
271
289
|
cwdFlag: cwdFlag(cwd),
|
|
272
290
|
...(analysisOffIn === undefined ? {} : { analysisOffIn }),
|
|
273
291
|
...(liveStudy === undefined ? {} : { liveStudy }),
|
|
292
|
+
...(study === undefined
|
|
293
|
+
? {}
|
|
294
|
+
: { study: { id: study.id, ...(study.path === undefined ? {} : { path: study.path }) } }),
|
|
295
|
+
...(codexSignedIn === undefined ? {} : { codexSignedIn }),
|
|
274
296
|
};
|
|
275
297
|
}
|
|
298
|
+
async function codexSignedInHere() {
|
|
299
|
+
const { detectLocalAgents } = await import("../actors/local-agent/cli.js");
|
|
300
|
+
const agents = await detectLocalAgents({ only: "codex" });
|
|
301
|
+
// The Codex analyst refuses an API-key login, so only a ChatGPT login counts.
|
|
302
|
+
return agents.some((agent) => agent.authStatus === "authenticated" && agent.billing === "account-unknown");
|
|
303
|
+
}
|
|
276
304
|
/** The view of a run that could not be found. */
|
|
277
305
|
export function missingRunFindings(cwd, run) {
|
|
278
306
|
const loaded = {
|
|
@@ -285,29 +313,7 @@ export function missingRunFindings(cwd, run) {
|
|
|
285
313
|
return withoutFindings({ runId: run, mode: undefined, runRoot: "", loaded, cwdFlag: flag }, "unavailable", "ANALYSIS_RUN_NOT_FOUND", `Run ${run} was not found or its storage is unsafe.`, cli(`runs${flag}`));
|
|
286
314
|
}
|
|
287
315
|
/** The findings of a run's selected analysis, or why it has none. */
|
|
288
|
-
export async function readRunFindings(cwd, run, analysisId) {
|
|
289
|
-
const source = await readRunAnalysis(cwd, run, analysisId);
|
|
316
|
+
export async function readRunFindings(cwd, run, analysisId, deps = {}) {
|
|
317
|
+
const source = await readRunAnalysis(cwd, run, analysisId, deps);
|
|
290
318
|
return source ? analysisFindings(source) : missingRunFindings(cwd, run);
|
|
291
319
|
}
|
|
292
|
-
/**
|
|
293
|
-
* The end of a live run's human output, after its `analysis:` line: the findings of the analysis
|
|
294
|
-
* the run just completed. Prints nothing for --json, or when the run's analysis did not complete,
|
|
295
|
-
* since that line already says why.
|
|
296
|
-
*/
|
|
297
|
-
export async function writeRunFindings(command, io, cwd, result) {
|
|
298
|
-
const outcome = result.automaticAnalysis;
|
|
299
|
-
const analysisId = outcome?.result?.analysisId;
|
|
300
|
-
if (wantsJson(command) ||
|
|
301
|
-
result.runId === undefined ||
|
|
302
|
-
analysisId === undefined ||
|
|
303
|
-
(outcome?.state !== "complete" && outcome?.state !== "partial"))
|
|
304
|
-
return;
|
|
305
|
-
try {
|
|
306
|
-
const view = await readRunFindings(cwd, result.runId, analysisId);
|
|
307
|
-
const full = cli(`review --run ${view.runId}${cwdFlag(cwd)}`);
|
|
308
|
-
io.writeOut(`${formatFindingsSummary(view, full).join("\n")}\n`);
|
|
309
|
-
}
|
|
310
|
-
catch {
|
|
311
|
-
// The run's own result is already printed; review reads the same record later.
|
|
312
|
-
}
|
|
313
|
-
}
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -113,7 +113,11 @@ export declare function messageOutput(value: {
|
|
|
113
113
|
export declare function humanError(error: CliError | undefined): HumanOutput;
|
|
114
114
|
/** Prints a human formatter's output: text to stdout, an error to stderr through formatCliError. */
|
|
115
115
|
export declare function writeHuman(command: Command, io: CliIo, human: HumanOutput): void;
|
|
116
|
-
|
|
116
|
+
/**
|
|
117
|
+
* Prints a result: JSON for --json, else the human formatter's text and, for a run, its analysis
|
|
118
|
+
* line. `analysisLines` replaces that line, for a caller that has read what the analysis left.
|
|
119
|
+
*/
|
|
120
|
+
export declare function writeResult<T>(command: Command, io: CliIo, result: T, formatHuman: (result: T) => HumanOutput, analysisLines?: readonly string[]): void;
|
|
117
121
|
export declare function noteRunFacts(command: Command, facts: TelemetryProperties): void;
|
|
118
122
|
/** Exported for tests: the facts writeResult read off a command's result document. */
|
|
119
123
|
export declare function runFactsFor(command: Command): TelemetryProperties;
|
package/dist/cli/io.js
CHANGED
|
@@ -178,7 +178,11 @@ export function writeHuman(command, io, human) {
|
|
|
178
178
|
if (human.error)
|
|
179
179
|
io.writeErr(formatCliError(commandLabel(command), human.error));
|
|
180
180
|
}
|
|
181
|
-
|
|
181
|
+
/**
|
|
182
|
+
* Prints a result: JSON for --json, else the human formatter's text and, for a run, its analysis
|
|
183
|
+
* line. `analysisLines` replaces that line, for a caller that has read what the analysis left.
|
|
184
|
+
*/
|
|
185
|
+
export function writeResult(command, io, result, formatHuman, analysisLines) {
|
|
182
186
|
const output = automaticAnalysisEnvelope(result);
|
|
183
187
|
if (wantsJson(command)) {
|
|
184
188
|
// Queued warnings are on stderr already; a JSON caller gets them in warnings[] as well.
|
|
@@ -186,9 +190,14 @@ export function writeResult(command, io, result, formatHuman) {
|
|
|
186
190
|
}
|
|
187
191
|
else {
|
|
188
192
|
writeHuman(command, io, formatHuman(output));
|
|
193
|
+
if (analysisLines !== undefined) {
|
|
194
|
+
for (const line of analysisLines)
|
|
195
|
+
io.writeOut(`${line}\n`);
|
|
196
|
+
}
|
|
197
|
+
else if (
|
|
189
198
|
// A run refused before it was created has no analysis to report, so its output ends with the
|
|
190
199
|
// refusal. Its JSON keeps the skip record.
|
|
191
|
-
|
|
200
|
+
output !== null &&
|
|
192
201
|
typeof output === "object" &&
|
|
193
202
|
"automaticAnalysis" in output &&
|
|
194
203
|
output.runId !== "not-created") {
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { Command } from "commander";
|
|
2
|
+
import type { AutomaticAnalysisOutcome } from "../analysis/job.js";
|
|
3
|
+
import { type FindingsDeps } from "./findings.js";
|
|
4
|
+
import { type CliIo, type HumanOutput } from "./io.js";
|
|
5
|
+
interface RunResultShape {
|
|
6
|
+
runId?: string;
|
|
7
|
+
automaticAnalysis?: AutomaticAnalysisOutcome;
|
|
8
|
+
observer?: {
|
|
9
|
+
observerPath?: string;
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Writes a finished run's result. After the route's own output it says what the automatic
|
|
14
|
+
* analysis did and the command a person runs next; when the analysis left findings, the top ones.
|
|
15
|
+
* `--json` carries the same message and command on `automaticAnalysis` and the findings summary
|
|
16
|
+
* as `findings`. A run with no analysis record is written as writeResult writes it.
|
|
17
|
+
*/
|
|
18
|
+
export declare function writeRunResult<T extends RunResultShape>(command: Command, io: CliIo, cwd: string, result: T, formatHuman: (result: T) => HumanOutput, deps?: FindingsDeps): Promise<void>;
|
|
19
|
+
export {};
|