humanish 0.141.0 → 0.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +1 -0
- package/CHANGELOG.md +21 -0
- package/README.md +3 -1
- package/dist/analysis/admission.d.ts +8 -0
- package/dist/analysis/admission.js +15 -4
- package/dist/analysis/automatic-config.d.ts +17 -3
- package/dist/analysis/automatic-config.js +56 -12
- package/dist/analysis/automatic.js +13 -7
- package/dist/analysis/execute.d.ts +6 -5
- package/dist/analysis/execute.js +27 -26
- package/dist/analysis/gemini-wire.d.ts +9 -0
- package/dist/analysis/gemini-wire.js +166 -0
- package/dist/analysis/image-tokens.d.ts +4 -2
- package/dist/analysis/image-tokens.js +24 -3
- package/dist/analysis/job.d.ts +1 -0
- package/dist/analysis/job.js +1 -0
- package/dist/analysis/models.d.ts +12 -0
- package/dist/analysis/models.js +47 -0
- package/dist/analysis/openai-wire.d.ts +6 -0
- package/dist/analysis/openai-wire.js +102 -0
- package/dist/analysis/provider.d.ts +19 -7
- package/dist/analysis/provider.js +21 -101
- package/dist/analysis/responses.d.ts +3 -0
- package/dist/analysis/responses.js +19 -0
- package/dist/analysis/service.js +11 -3
- package/dist/analysis/types.d.ts +12 -2
- package/dist/analysis/validation.d.ts +1 -0
- package/dist/analysis/validation.js +17 -15
- package/dist/cli/commands/analyze.js +11 -7
- package/dist/cli/commands/observe.js +26 -1
- package/dist/cli/commands/run-command.d.ts +1 -1
- package/dist/cli/commands/run-command.js +12 -0
- package/dist/cli/commands/study-remote.d.ts +13 -0
- package/dist/cli/commands/study-remote.js +125 -0
- package/dist/cli/commands/study-route-computer-use.js +26 -2
- package/dist/cli/commands/study-route-scripted.js +1 -0
- package/dist/cli/commands/study-route-shared-world.js +1 -0
- package/dist/cli/commands/study-route-terminal.js +1 -0
- package/dist/cli/commands/study-run.js +34 -0
- package/dist/cli/commands/watch.js +5 -0
- package/dist/cli/doctor.js +7 -2
- package/dist/cli/findings.d.ts +9 -1
- package/dist/cli/findings.js +62 -17
- package/dist/cli/io.d.ts +8 -0
- package/dist/cli/io.js +2 -1
- package/dist/keys/key-resolution.d.ts +10 -0
- package/dist/keys/key-resolution.js +33 -5
- package/dist/observer/listen.d.ts +7 -2
- package/dist/observer/listen.js +10 -4
- package/dist/observer/render.d.ts +2 -0
- package/dist/observer/render.js +1 -1
- package/dist/observer-app.html +1 -1
- package/dist/remote/launch.d.ts +56 -0
- package/dist/remote/launch.js +365 -0
- package/dist/remote/observer-proxy.d.ts +11 -0
- package/dist/remote/observer-proxy.js +66 -0
- package/dist/remote/plan.d.ts +35 -0
- package/dist/remote/plan.js +114 -0
- package/dist/remote/reattach.d.ts +17 -0
- package/dist/remote/reattach.js +81 -0
- package/dist/remote/runner.d.ts +84 -0
- package/dist/remote/runner.js +241 -0
- package/dist/routes/computer-use/e2b-desktop/start.js +1 -1
- package/dist/routes/computer-use/types.d.ts +2 -0
- package/dist/routes/scripted/types.d.ts +2 -0
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/routes/terminal/types.d.ts +2 -0
- package/dist/run/costs.d.ts +2 -1
- package/dist/run/pricing.d.ts +13 -1
- package/dist/run/pricing.js +50 -4
- package/dist/run/results.d.ts +1 -1
- package/dist/run/run-cost.d.ts +2 -1
- package/dist/run/run-cost.js +6 -1
- package/dist/run/run.d.ts +13 -0
- package/dist/run/run.js +18 -7
- package/dist/run/sandbox-creates.d.ts +10 -3
- package/dist/run/sandbox-creates.js +48 -6
- package/dist/run-study.d.ts +5 -0
- package/dist/study/doctor.js +14 -1
- package/dist/study/requirements.d.ts +1 -1
- package/dist/study/requirements.js +8 -6
- package/dist/study/route-inputs.js +4 -0
- package/dist/study/run-study-events.d.ts +1 -1
- package/dist/study/run-study-events.js +9 -2
- package/dist/substrates/e2b/base-module.d.ts +3 -0
- package/dist/substrates/e2b/base-module.js +40 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +1 -1
- package/dist/substrates/e2b/desktop-browser.js +24 -6
- package/dist/substrates/e2b/sdk.d.ts +24 -0
- package/dist/substrates/e2b/sdk.js +5 -1
- package/dist/substrates/e2b/shell-only.d.ts +5 -0
- package/dist/substrates/e2b/shell-only.js +27 -0
- package/dist/substrates/modal/sandbox.js +27 -28
- package/dist/substrates/modal/sdk.d.ts +7 -0
- package/dist/tui-app.js +46 -46
- package/docs/contracts/study-analysis.md +85 -4
- package/docs/product/automatic-analysis.md +42 -8
- package/package.json +2 -2
- package/skills/humanish/SKILL.md +18 -1
package/dist/cli/doctor.js
CHANGED
|
@@ -2,7 +2,7 @@ import { readFile, readdir } from "node:fs/promises";
|
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { DOCTOR_KEYS, probeKeySources, } from "../keys/key-resolution.js";
|
|
5
|
+
import { DOCTOR_KEYS, analysisKeyNames, probeKeySources, } from "../keys/key-resolution.js";
|
|
6
6
|
import { desktopPeerAdvice, humanishInstall, } from "../substrates/e2b/peer-install.js";
|
|
7
7
|
import { concurrentSandboxes, describeConcurrentSandboxes, describeSandboxCeiling, sandboxCeiling, } from "../substrates/e2b/lifetime.js";
|
|
8
8
|
import { nodeSupportsTui, terminalSurfaceMessage, TUI_BUNDLE_URL } from "../tui/contract.js";
|
|
@@ -304,7 +304,8 @@ function versionCheck(env) {
|
|
|
304
304
|
},
|
|
305
305
|
];
|
|
306
306
|
}
|
|
307
|
-
/** Probes the live-run keys, plus the receiving-email key a real-comms study names
|
|
307
|
+
/** Probes the live-run keys, plus the receiving-email key a real-comms study names and the Gemini
|
|
308
|
+
* key a study analyzed on Gemini reads. */
|
|
308
309
|
async function probeDoctorKeys(cwd, env, study, keyDeps, dotenv) {
|
|
309
310
|
const keyNames = new Set(DOCTOR_KEYS);
|
|
310
311
|
let receivingKey = null;
|
|
@@ -315,6 +316,10 @@ async function probeDoctorKeys(cwd, env, study, keyDeps, dotenv) {
|
|
|
315
316
|
if (resolved.ok && resolved.config.execution?.target === "modal-terminal")
|
|
316
317
|
for (const name of ["MODAL_TOKEN_ID", "MODAL_TOKEN_SECRET"])
|
|
317
318
|
keyNames.add(name);
|
|
319
|
+
// A study analyzed on Gemini reads the Gemini key, which no other study needs.
|
|
320
|
+
const analysis = resolved.ok ? resolved.config.review?.analysis : undefined;
|
|
321
|
+
if (typeof analysis === "object" && analysis !== null && analysis.provider === "gemini")
|
|
322
|
+
keyNames.add(analysisKeyNames("gemini")[0]);
|
|
318
323
|
if (resolved.ok && resolved.config.comms?.email?.kind === "real") {
|
|
319
324
|
const { receivingRequiredKey } = await import("../comms/setup.js");
|
|
320
325
|
receivingKey = await receivingRequiredKey(cwd, resolved.config.comms.email.connection);
|
package/dist/cli/findings.d.ts
CHANGED
|
@@ -117,6 +117,8 @@ interface FindingsSource {
|
|
|
117
117
|
liveStudy?: LiveStudy;
|
|
118
118
|
/** The run's study, from its bundle. */
|
|
119
119
|
study?: Pick<RunStudyProvenance, "id" | "path">;
|
|
120
|
+
/** The analyst the run's study gets, read only for an analysis skipped for a missing key. */
|
|
121
|
+
analyst?: Analyst;
|
|
120
122
|
/**
|
|
121
123
|
* Whether Codex is signed in with a ChatGPT account on this machine, read only for an analysis
|
|
122
124
|
* skipped for a missing OpenAI key, whose next step it picks.
|
|
@@ -140,7 +142,7 @@ interface FindingsSource {
|
|
|
140
142
|
/** status.json's last heartbeat. */
|
|
141
143
|
updatedAt: string;
|
|
142
144
|
/** The analyst the run's study gets, read only for an analysis whose process stopped. */
|
|
143
|
-
|
|
145
|
+
analyst?: Analyst;
|
|
144
146
|
};
|
|
145
147
|
/** When the view was built, for how long an analysis has run. */
|
|
146
148
|
nowMs?: number;
|
|
@@ -156,6 +158,12 @@ interface LiveStudy {
|
|
|
156
158
|
}
|
|
157
159
|
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
158
160
|
export declare function cwdFlag(cwd: string): string;
|
|
161
|
+
/** An analysis provider, model and effort, as a study or an analysis record names them. */
|
|
162
|
+
interface Analyst {
|
|
163
|
+
provider?: string | undefined;
|
|
164
|
+
model?: string;
|
|
165
|
+
reasoningEffort?: string | undefined;
|
|
166
|
+
}
|
|
159
167
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
160
168
|
export declare function analysisFindings(source: FindingsSource): AnalysisFindings;
|
|
161
169
|
/**
|
package/dist/cli/findings.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
// starts an analysis.
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { costRefusal } from "../analysis/admission.js";
|
|
8
|
-
import { DEFAULT_ANALYSIS_MAX_COST_USD, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
|
|
8
|
+
import { DEFAULT_ANALYSIS_EFFORT, DEFAULT_ANALYSIS_MAX_COST_USD, DEFAULT_ANALYSIS_MODEL, resolveAutomaticAnalysis, } from "../analysis/automatic-config.js";
|
|
9
9
|
import { ANALYSIS_USUALLY_TAKES } from "../analysis/analysis-limits.js";
|
|
10
10
|
import { bySeverity } from "../analysis/finding-lead.js";
|
|
11
11
|
import { loadAnalysis } from "../analysis/load.js";
|
|
@@ -22,6 +22,7 @@ import { plural } from "../run/text.js";
|
|
|
22
22
|
import { shellQuote } from "../substrates/shell.js";
|
|
23
23
|
import { analysisOutcomeText } from "./io.js";
|
|
24
24
|
import { cli } from "./invocation.js";
|
|
25
|
+
import { analysisKeyNames } from "../keys/key-resolution.js";
|
|
25
26
|
export const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
26
27
|
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
27
28
|
export function cwdFlag(cwd) {
|
|
@@ -29,11 +30,30 @@ export function cwdFlag(cwd) {
|
|
|
29
30
|
return "";
|
|
30
31
|
return ` --cwd ${/^[\w@%+=:,./-]+$/.test(cwd) ? cwd : shellQuote(cwd)}`;
|
|
31
32
|
}
|
|
32
|
-
|
|
33
|
+
/** The skip reasons for an analysis whose API key was missing, one per API analyst. */
|
|
34
|
+
const KEY_MISSING_REASONS = new Set([
|
|
35
|
+
"AUTOMATIC_ANALYSIS_KEY_MISSING",
|
|
36
|
+
"AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING",
|
|
37
|
+
]);
|
|
38
|
+
/** The effort an analysis configuration declares; Codex configurations declare none. */
|
|
39
|
+
const declaredEffort = (config) => "reasoningEffort" in config ? config.reasoningEffort : undefined;
|
|
40
|
+
/** The `analyze` command that runs `analyst` on the run; undefined is the OpenAI default. */
|
|
41
|
+
function analyzeCommand(source, analyst) {
|
|
33
42
|
const run = cli(`analyze --run ${source.runId}${source.cwdFlag}`);
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
43
|
+
if (analyst?.provider === "codex")
|
|
44
|
+
return `${run} --provider codex`;
|
|
45
|
+
if (analyst?.provider === "gemini") {
|
|
46
|
+
const model = analyst.model === undefined ? "" : ` --model ${analyst.model}`;
|
|
47
|
+
return `${run} --provider gemini${model} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
48
|
+
}
|
|
49
|
+
// A named model or effort stays in the command, so the rerun is the same analyst. The default
|
|
50
|
+
// analyst needs neither: `analyze` without flags runs it.
|
|
51
|
+
const isDefault = (analyst?.model ?? DEFAULT_ANALYSIS_MODEL) === DEFAULT_ANALYSIS_MODEL &&
|
|
52
|
+
(analyst?.model === undefined || analyst.reasoningEffort === DEFAULT_ANALYSIS_EFFORT);
|
|
53
|
+
const flags = isDefault
|
|
54
|
+
? ""
|
|
55
|
+
: ` --model ${analyst.model}${analyst.reasoningEffort === undefined ? "" : ` --reasoning-effort ${analyst.reasoningEffort}`}`;
|
|
56
|
+
return `${run}${flags} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
37
57
|
}
|
|
38
58
|
/** The cited items the analysis evidence resolves, each once, in citation order. */
|
|
39
59
|
function citedEvidence(ids, analysis, runRoot) {
|
|
@@ -157,16 +177,22 @@ function dryRunFindings(source) {
|
|
|
157
177
|
return withoutFindings(source, "dry_run", null, `${opening} Findings come from live runs: ${howLive}`, cli(`run ${live.run}${source.cwdFlag}`));
|
|
158
178
|
}
|
|
159
179
|
/**
|
|
160
|
-
* An analysis skipped because
|
|
180
|
+
* An analysis skipped because its API key is not set. For OpenAI with Codex signed in, the next step is the
|
|
161
181
|
* Codex account analyst, named as an opt-in: the study chose the OpenAI analyst, and switching
|
|
162
182
|
* the account that pays for it is the operator's choice.
|
|
163
183
|
*/
|
|
164
184
|
function missingKeyFindings(source, reason) {
|
|
185
|
+
// The skip reason records which analyst's key was missing; the study file only adds its model.
|
|
186
|
+
if (reason === "AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING")
|
|
187
|
+
return withoutFindings(source, "skipped", reason, `The automatic analysis was skipped because ${analysisKeyNames("gemini").join(" and ")} are not set. Store the key with ${cli("keys set gemini")}, then run the next command.`, analyzeCommand(source, {
|
|
188
|
+
provider: "gemini",
|
|
189
|
+
...(source.analyst?.provider === "gemini" ? { model: source.analyst.model } : {}),
|
|
190
|
+
}));
|
|
165
191
|
const opening = "The automatic analysis was skipped because OPENAI_API_KEY is not set.";
|
|
166
192
|
if (source.codexSignedIn !== true)
|
|
167
|
-
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, undefined));
|
|
193
|
+
return withoutFindings(source, "skipped", reason, `${opening} Set it with ${cli("keys set openai")}, then run the next command.`, analyzeCommand(source, source.analyst?.provider === "gemini" ? undefined : source.analyst));
|
|
168
194
|
const file = source.study?.path === undefined ? "the study file" : source.study.path;
|
|
169
|
-
return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, "codex"));
|
|
195
|
+
return withoutFindings(source, "skipped", reason, `${opening} Codex is signed in on this machine, so its ChatGPT account can analyze the run instead, sending the evidence to OpenAI under that account. The next command does it once; review.analysis.provider: codex in ${file} makes it this study's analyst.`, analyzeCommand(source, { provider: "codex" }));
|
|
170
196
|
}
|
|
171
197
|
/**
|
|
172
198
|
* The view of a run whose findings are not written yet: participants still running, an analysis
|
|
@@ -194,7 +220,7 @@ function pendingFindings(source, job) {
|
|
|
194
220
|
if (!stopped)
|
|
195
221
|
return undefined;
|
|
196
222
|
const at = job?.updatedAt ?? (progress?.stage === "analysis" ? progress.updatedAt : undefined);
|
|
197
|
-
return withoutFindings(source, "failed", "AUTOMATIC_ANALYSIS_STOPPED", `The automatic analysis stopped before it wrote findings: the process running it stopped updating its record${at === undefined ? "" : ` at ${at}`}, most likely because it ended. Analyze the run again with the next command.`, analyzeCommand(source, progress?.stage === "analysis" ? progress.
|
|
223
|
+
return withoutFindings(source, "failed", "AUTOMATIC_ANALYSIS_STOPPED", `The automatic analysis stopped before it wrote findings: the process running it stopped updating its record${at === undefined ? "" : ` at ${at}`}, most likely because it ended. Analyze the run again with the next command.`, analyzeCommand(source, progress?.stage === "analysis" ? progress.analyst : undefined));
|
|
198
224
|
}
|
|
199
225
|
/**
|
|
200
226
|
* A run that ended, by a signal, a process that left without publishing, or a heartbeat that
|
|
@@ -217,7 +243,13 @@ async function studyAnalyst(cwd, study) {
|
|
|
217
243
|
if (!resolved?.ok || resolved.config.id !== study.id)
|
|
218
244
|
return undefined;
|
|
219
245
|
const analysis = resolveAutomaticAnalysis(resolved.config.review?.analysis, resolved.config.actor);
|
|
220
|
-
return analysis.ok
|
|
246
|
+
return analysis.ok && analysis.config !== undefined
|
|
247
|
+
? {
|
|
248
|
+
provider: analysis.config.provider,
|
|
249
|
+
model: analysis.config.model,
|
|
250
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
251
|
+
}
|
|
252
|
+
: undefined;
|
|
221
253
|
}
|
|
222
254
|
/** The findings view of a run's selected analysis, or why it has none. */
|
|
223
255
|
export function analysisFindings(source) {
|
|
@@ -236,7 +268,13 @@ export function analysisFindings(source) {
|
|
|
236
268
|
message: stale
|
|
237
269
|
? `The run's evidence changed after analysis ${analysis.id}, so its findings may not match the run.`
|
|
238
270
|
: `Analysis ${analysis.id} is ${analysis.status}, with ${plural(result.findings.length, "finding")}.`,
|
|
239
|
-
next: stale
|
|
271
|
+
next: stale
|
|
272
|
+
? analyzeCommand(source, {
|
|
273
|
+
provider: analysis.provider,
|
|
274
|
+
model: analysis.config.model,
|
|
275
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
276
|
+
})
|
|
277
|
+
: null,
|
|
240
278
|
analysisId: analysis.id,
|
|
241
279
|
status: analysis.status,
|
|
242
280
|
provider: analysis.provider,
|
|
@@ -259,16 +297,20 @@ export function analysisFindings(source) {
|
|
|
259
297
|
if (pending !== undefined)
|
|
260
298
|
return pending;
|
|
261
299
|
if (analysis && analysis.result === null)
|
|
262
|
-
return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source,
|
|
300
|
+
return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, {
|
|
301
|
+
provider: analysis.provider,
|
|
302
|
+
model: analysis.config.model,
|
|
303
|
+
reasoningEffort: declaredEffort(analysis.config),
|
|
304
|
+
}));
|
|
263
305
|
if (job?.state === "failed" || job?.state === "cancelled") {
|
|
264
306
|
const outcome = analysisOutcomeText(job);
|
|
265
|
-
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? "codex" : undefined));
|
|
307
|
+
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? { provider: "codex" } : undefined));
|
|
266
308
|
}
|
|
267
309
|
if (job?.state === "skipped" && job.admission) {
|
|
268
310
|
const refusal = costRefusal(job.admission, `--run ${source.runId}${source.cwdFlag}`, cli);
|
|
269
311
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${refusal.text}`, refusal.command);
|
|
270
312
|
}
|
|
271
|
-
if (job?.state === "skipped" && job.reason
|
|
313
|
+
if (job?.state === "skipped" && job.reason !== null && KEY_MISSING_REASONS.has(job.reason))
|
|
272
314
|
return missingKeyFindings(source, job.reason);
|
|
273
315
|
if (job?.state === "skipped") {
|
|
274
316
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
@@ -339,20 +381,22 @@ export async function readRunAnalysis(cwd, run, analysisId, deps = {}) {
|
|
|
339
381
|
const record = isRunStatusRecord(status) && status.runId === runIdOf(prepared) ? status : null;
|
|
340
382
|
const phase = record === null ? undefined : runAnalysisPhase(record, nowMs);
|
|
341
383
|
const stopped = phase?.state === "stopped" || loaded.automatic?.reason === "AUTOMATIC_ANALYSIS_STOPPED";
|
|
342
|
-
const
|
|
384
|
+
const missingKeyReason = loaded.automatic?.reason ?? null;
|
|
385
|
+
const keyMissing = missingKeyReason !== null && KEY_MISSING_REASONS.has(missingKeyReason);
|
|
386
|
+
const analyst = stopped || keyMissing ? await studyAnalyst(physical, study) : undefined;
|
|
343
387
|
const progress = phase !== undefined && record !== null
|
|
344
388
|
? {
|
|
345
389
|
stage: "analysis",
|
|
346
390
|
phase,
|
|
347
391
|
updatedAt: record.updatedAt,
|
|
348
|
-
...(
|
|
392
|
+
...(analyst === undefined ? {} : { analyst }),
|
|
349
393
|
}
|
|
350
394
|
: bundle?.bundle.outcome === undefined &&
|
|
351
395
|
record !== null &&
|
|
352
396
|
classifyRunStatus(record, nowMs) === "running"
|
|
353
397
|
? { stage: "participants" }
|
|
354
398
|
: cutOff(bundle?.bundle, record, nowMs);
|
|
355
|
-
const codexSignedIn =
|
|
399
|
+
const codexSignedIn = missingKeyReason === "AUTOMATIC_ANALYSIS_KEY_MISSING"
|
|
356
400
|
? await (deps.codexSignedIn ?? codexSignedInHere)().catch(() => false)
|
|
357
401
|
: undefined;
|
|
358
402
|
return {
|
|
@@ -367,6 +411,7 @@ export async function readRunAnalysis(cwd, run, analysisId, deps = {}) {
|
|
|
367
411
|
? {}
|
|
368
412
|
: { study: { id: study.id, ...(study.path === undefined ? {} : { path: study.path }) } }),
|
|
369
413
|
...(codexSignedIn === undefined ? {} : { codexSignedIn }),
|
|
414
|
+
...(keyMissing && analyst !== undefined ? { analyst } : {}),
|
|
370
415
|
...(progress === undefined ? {} : { progress }),
|
|
371
416
|
nowMs,
|
|
372
417
|
};
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -45,6 +45,14 @@ export interface StudyCommandOptions {
|
|
|
45
45
|
port?: string | undefined;
|
|
46
46
|
rerunFailedFrom?: string | undefined;
|
|
47
47
|
runId?: string | undefined;
|
|
48
|
+
/** Run the study in a runner sandbox on its participants' provider. */
|
|
49
|
+
remote?: boolean | undefined;
|
|
50
|
+
/** Hidden: the run key the run's sandboxes carry, chosen by a remote run's launcher. */
|
|
51
|
+
ownerKey?: string | undefined;
|
|
52
|
+
/** Hidden, watch only: bind the served Observer on every interface (a remote run's runner). */
|
|
53
|
+
allInterfaces?: boolean | undefined;
|
|
54
|
+
/** Hidden, watch only: also write the run's result and exit code to this file as JSON. */
|
|
55
|
+
resultFile?: string | undefined;
|
|
48
56
|
/** watch only: serve and follow the Observer even without an interactive terminal. */
|
|
49
57
|
serve?: boolean | undefined;
|
|
50
58
|
/** Repo-relative path to an adopter scorer module; overrides review.scorer.ref when set. */
|
package/dist/cli/io.js
CHANGED
|
@@ -2,7 +2,7 @@ import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js
|
|
|
2
2
|
import { costRefusal, refusedCost } from "../analysis/admission.js";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
5
|
-
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
5
|
+
import { analysisKeyNames, discoverProviderKeys } from "../keys/key-resolution.js";
|
|
6
6
|
import { deriveRunFacts } from "./telemetry.js";
|
|
7
7
|
import { warnAndQueue, withQueuedWarnings } from "./deprecations.js";
|
|
8
8
|
import { forTerminal } from "../routes/terminal/encoding.js";
|
|
@@ -297,6 +297,7 @@ export function wantsJson(command) {
|
|
|
297
297
|
const ANALYSIS_REASON_TEXT = {
|
|
298
298
|
AUTOMATIC_ANALYSIS_DRY_RUN: "skipped for dry runs",
|
|
299
299
|
AUTOMATIC_ANALYSIS_KEY_MISSING: "skipped because OPENAI_API_KEY is not set",
|
|
300
|
+
AUTOMATIC_ANALYSIS_GEMINI_KEY_MISSING: `skipped because ${analysisKeyNames("gemini").join(" and ")} are not set`,
|
|
300
301
|
AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE: "skipped because no participant left evidence",
|
|
301
302
|
AUTOMATIC_ANALYSIS_ACTOR_CANCELLED: "skipped because the run stopped before its participants finished",
|
|
302
303
|
AUTOMATIC_ANALYSIS_SOURCE_UNAVAILABLE: "skipped because the run's evidence could not be read",
|
|
@@ -1,3 +1,12 @@
|
|
|
1
|
+
/** The key each API analyst reads, under its first spelling. */
|
|
2
|
+
declare const ANALYSIS_KEYS: {
|
|
3
|
+
readonly openai: "OPENAI_API_KEY";
|
|
4
|
+
readonly gemini: "GEMINI_API_KEY";
|
|
5
|
+
};
|
|
6
|
+
/** The env names an API analyst's key may have, in the order they are read. */
|
|
7
|
+
export declare const analysisKeyNames: (vendor: keyof typeof ANALYSIS_KEYS) => readonly string[];
|
|
8
|
+
/** An API analyst's key from `env`: the first of its names with a non-blank value, else "". */
|
|
9
|
+
export declare function analysisApiKey(vendor: keyof typeof ANALYSIS_KEYS, env: Readonly<Record<string, string | undefined>>): string;
|
|
1
10
|
/** The vendor aliases `humanish keys set` accepts. A raw ENV_NAME is also accepted. */
|
|
2
11
|
export declare const KEY_VENDORS: string[];
|
|
3
12
|
/** The keys `humanish keys` lists, in its order, each with what humanish uses it for. */
|
|
@@ -77,3 +86,4 @@ export declare function setUserKey(name: string, value: string, env: NodeJS.Proc
|
|
|
77
86
|
export declare function unsetUserKey(name: string, env: NodeJS.ProcessEnv, deps?: KeyResolutionDeps): boolean;
|
|
78
87
|
/** The names (never values) currently in the user store. */
|
|
79
88
|
export declare function listUserKeys(env: NodeJS.ProcessEnv, deps?: KeyResolutionDeps): string[];
|
|
89
|
+
export {};
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
// GH_TOKEN <- `gh auth token` (the gh CLI's credential chain);
|
|
12
12
|
// 4. the humanish user-level store `$XDG_CONFIG_HOME/humanish/keys.env`, written only by
|
|
13
13
|
// `humanish keys set` (0600, prompted or --stdin), the fallback for vendors that ship
|
|
14
|
-
// no machine chain of their own (OpenAI, Anthropic).
|
|
14
|
+
// no machine chain of their own (OpenAI, Anthropic, Gemini).
|
|
15
15
|
//
|
|
16
16
|
// Every fill is announced by key name + source, never by value. The subject app's own
|
|
17
17
|
// `.env`/`.env.local` are never read: those hold product credentials, a different class
|
|
@@ -34,7 +34,8 @@ import { OPENAI_EGRESS_PLACEHOLDER } from "../routes/terminal/runtime-auth.js";
|
|
|
34
34
|
* full-file load). ANTHROPIC_API_KEY is not listed, and doctor does not report it: a Claude Code
|
|
35
35
|
* participant runs on its own login and never receives it. CODEX_API_KEY is an alternative spelling
|
|
36
36
|
* of the OpenAI key for terminal studies. Doctor reports AGENTMAIL_API_KEY only under `--study`, for
|
|
37
|
-
* a study whose email connection names it, since a connection may name another variable
|
|
37
|
+
* a study whose email connection names it, since a connection may name another variable, and
|
|
38
|
+
* GEMINI_API_KEY only for a study whose analysis runs on Gemini.
|
|
38
39
|
*/
|
|
39
40
|
const PROVIDER_KEYS = [
|
|
40
41
|
{
|
|
@@ -69,9 +70,33 @@ const PROVIDER_KEYS = [
|
|
|
69
70
|
{ name: "GITHUB_TOKEN", spellingOf: "GH_TOKEN" },
|
|
70
71
|
{ name: "CODEX_API_KEY", doctor: true },
|
|
71
72
|
{ name: "AGENTMAIL_API_KEY", vendor: "agentmail", listed: { order: 4, use: "email in studies" } },
|
|
73
|
+
{ name: "GEMINI_API_KEY", vendor: "gemini", listed: { order: 5, use: "analysis on Gemini" } },
|
|
74
|
+
// Google's client libraries also read GOOGLE_API_KEY, and prefer it when both are set. humanish
|
|
75
|
+
// reads GEMINI_API_KEY first: a GOOGLE_API_KEY exported for Maps or Firebase must not take the
|
|
76
|
+
// place of the key `humanish keys set gemini` stored.
|
|
77
|
+
{ name: "GOOGLE_API_KEY", spellingOf: "GEMINI_API_KEY" },
|
|
72
78
|
];
|
|
73
79
|
const PROVIDER_KEY_SET = new Set(PROVIDER_KEYS.map((key) => key.name));
|
|
74
80
|
const providerKey = (name) => PROVIDER_KEYS.find((key) => key.name === name);
|
|
81
|
+
/**
|
|
82
|
+
* Every env name of the key `name` is a spelling of, in the order humanish reads them: the key's
|
|
83
|
+
* own name, then its second spellings. Discovery treats them as one key.
|
|
84
|
+
*/
|
|
85
|
+
function keyNames(name) {
|
|
86
|
+
const own = providerKey(name)?.spellingOf ?? name;
|
|
87
|
+
return [own, ...PROVIDER_KEYS.filter((key) => key.spellingOf === own).map((key) => key.name)];
|
|
88
|
+
}
|
|
89
|
+
/** The key each API analyst reads, under its first spelling. */
|
|
90
|
+
const ANALYSIS_KEYS = { openai: "OPENAI_API_KEY", gemini: "GEMINI_API_KEY" };
|
|
91
|
+
/** The env names an API analyst's key may have, in the order they are read. */
|
|
92
|
+
export const analysisKeyNames = (vendor) => keyNames(ANALYSIS_KEYS[vendor]);
|
|
93
|
+
/** An API analyst's key from `env`: the first of its names with a non-blank value, else "". */
|
|
94
|
+
export function analysisApiKey(vendor, env) {
|
|
95
|
+
for (const name of analysisKeyNames(vendor))
|
|
96
|
+
if (env[name]?.trim())
|
|
97
|
+
return env[name];
|
|
98
|
+
return "";
|
|
99
|
+
}
|
|
75
100
|
/** The vendor aliases `humanish keys set` accepts. A raw ENV_NAME is also accepted. */
|
|
76
101
|
export const KEY_VENDORS = PROVIDER_KEYS.flatMap((key) => key.vendor === undefined ? [] : [key.vendor]);
|
|
77
102
|
/** The keys `humanish keys` lists, in its order, each with what humanish uses it for. */
|
|
@@ -212,8 +237,11 @@ export async function discoverProviderKeys(args) {
|
|
|
212
237
|
const ignored = [];
|
|
213
238
|
// Fill-only means presence wins, including an explicitly-set empty value: an operator who
|
|
214
239
|
// exported OPENAI_API_KEY="" said "off", and a rung that "fixes" that has overridden them.
|
|
215
|
-
// Only a truly-unset name is fillable
|
|
216
|
-
|
|
240
|
+
// Only a truly-unset name is fillable, and only while no spelling of its key was set before
|
|
241
|
+
// discovery: a repo overlay's GOOGLE_API_KEY must not stand beside, and outrank, an exported
|
|
242
|
+
// GEMINI_API_KEY. Spellings that discovery itself fills may stand together.
|
|
243
|
+
const explicit = new Set(Object.keys(env).filter((name) => env[name] !== undefined));
|
|
244
|
+
const fillable = (name) => env[name] === undefined && keyNames(name).every((spelling) => !explicit.has(spelling));
|
|
217
245
|
const fill = (name, value, source) => {
|
|
218
246
|
env[name] = value;
|
|
219
247
|
fills.push({ name, source });
|
|
@@ -259,7 +287,7 @@ export async function discoverProviderKeys(args) {
|
|
|
259
287
|
// Rung 3c: the gh CLI's credential chain. Only consulted when neither GitHub env name is
|
|
260
288
|
// present; fills GH_TOKEN (the name humanish reads first). gh runs without any provider key in
|
|
261
289
|
// its environment, so the keys the rungs above just filled stay in this process.
|
|
262
|
-
if (
|
|
290
|
+
if (keyNames("GH_TOKEN").every((name) => env[name] === undefined)) {
|
|
263
291
|
const exec = deps.execText ?? defaultExecText;
|
|
264
292
|
const token = await exec("gh", ["auth", "token"], 3_000, withoutProviderKeys(env));
|
|
265
293
|
if (token !== null)
|
|
@@ -12,5 +12,10 @@ export declare class PortInUseError extends Error {
|
|
|
12
12
|
* that cannot decide says "other", which is the answer that suggests the safer action.
|
|
13
13
|
*/
|
|
14
14
|
export declare function probePortHolder(port: number, fetchFn?: typeof fetch): Promise<PortHolder>;
|
|
15
|
-
/**
|
|
16
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Bind on loopback; resolve the port that was actually bound; name a taken port. `allInterfaces`
|
|
17
|
+
* binds every interface instead, for the Observer of a remote run's runner sandbox: Modal's proxy
|
|
18
|
+
* cannot reach a loopback-only server, and only the provider's authenticated proxy reaches the
|
|
19
|
+
* sandbox's ports.
|
|
20
|
+
*/
|
|
21
|
+
export declare function listenOnLoopback(server: Server, port: number, probe?: (port: number) => Promise<PortHolder>, allInterfaces?: boolean): Promise<number>;
|
package/dist/observer/listen.js
CHANGED
|
@@ -61,8 +61,13 @@ async function probeRoutes(port, fetchFn) {
|
|
|
61
61
|
}
|
|
62
62
|
return "other";
|
|
63
63
|
}
|
|
64
|
-
/**
|
|
65
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Bind on loopback; resolve the port that was actually bound; name a taken port. `allInterfaces`
|
|
66
|
+
* binds every interface instead, for the Observer of a remote run's runner sandbox: Modal's proxy
|
|
67
|
+
* cannot reach a loopback-only server, and only the provider's authenticated proxy reaches the
|
|
68
|
+
* sandbox's ports.
|
|
69
|
+
*/
|
|
70
|
+
export function listenOnLoopback(server, port, probe = probePortHolder, allInterfaces = false) {
|
|
66
71
|
return new Promise((resolve, reject) => {
|
|
67
72
|
const onError = (error) => {
|
|
68
73
|
if (error.code === "EADDRINUSE") {
|
|
@@ -72,8 +77,9 @@ export function listenOnLoopback(server, port, probe = probePortHolder) {
|
|
|
72
77
|
reject(error);
|
|
73
78
|
};
|
|
74
79
|
server.once("error", onError);
|
|
75
|
-
//
|
|
76
|
-
|
|
80
|
+
// Otherwise never beyond loopback; exposure only ever happens through a tunnel or proxy
|
|
81
|
+
// forwarding here.
|
|
82
|
+
server.listen(port, allInterfaces ? "0.0.0.0" : "127.0.0.1", () => {
|
|
77
83
|
server.off("error", onError);
|
|
78
84
|
const address = server.address();
|
|
79
85
|
if (!address || typeof address === "string") {
|
|
@@ -31,6 +31,8 @@ export interface ObserverServeOptions {
|
|
|
31
31
|
/** Restrict history and evidence routes to the selected run. Exposed viewers always do this. */
|
|
32
32
|
scope?: "run" | "library";
|
|
33
33
|
exposed?: boolean;
|
|
34
|
+
/** Bind every interface: only a remote run's runner sandbox, behind its provider's proxy. */
|
|
35
|
+
allInterfaces?: boolean;
|
|
34
36
|
}
|
|
35
37
|
export interface ObserverServer {
|
|
36
38
|
url: string;
|
package/dist/observer/render.js
CHANGED
|
@@ -293,7 +293,7 @@ export async function serveObserver(result, options = {}) {
|
|
|
293
293
|
const server = createServer((request, response) => {
|
|
294
294
|
void handleRequest(request, response);
|
|
295
295
|
});
|
|
296
|
-
port = await listenOnLoopback(server, options.port ?? 0);
|
|
296
|
+
port = await listenOnLoopback(server, options.port ?? 0, undefined, options.allInterfaces === true);
|
|
297
297
|
if (exposed) {
|
|
298
298
|
hostAllowlist.add(`127.0.0.1:${port}`);
|
|
299
299
|
hostAllowlist.add(`localhost:${port}`);
|