humanish 0.109.1 → 0.110.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +14 -11
- package/CHANGELOG.md +24 -0
- package/CONTEXT.md +8 -6
- package/README.md +97 -48
- package/dist/actors/computer-use/loop/ending.js +4 -4
- package/dist/actors/local-agent/claude-participant.d.ts +74 -0
- package/dist/actors/local-agent/claude-participant.js +243 -0
- package/dist/actors/local-agent/claude-session.d.ts +16 -3
- package/dist/actors/local-agent/claude-session.js +64 -23
- package/dist/actors/local-agent/cli.d.ts +11 -0
- package/dist/actors/local-agent/cli.js +103 -24
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +18 -11
- package/dist/cli/commands/analyze.js +21 -5
- package/dist/cli/commands/comms.js +25 -7
- package/dist/cli/commands/observe.js +4 -2
- package/dist/cli/commands/run-command.js +4 -3
- package/dist/cli/commands/run-signals.d.ts +2 -0
- package/dist/cli/commands/run-signals.js +96 -44
- package/dist/cli/commands/runs.d.ts +5 -0
- package/dist/cli/commands/runs.js +89 -67
- package/dist/cli/commands/setup.js +19 -5
- package/dist/cli/commands/study-format.js +29 -2
- package/dist/cli/commands/study-route-computer-use.js +2 -0
- package/dist/cli/commands/study-route-scripted.js +2 -0
- package/dist/cli/commands/study-route-shared-world.js +2 -0
- package/dist/cli/commands/study-route-terminal.js +2 -0
- package/dist/cli/commands/study.js +4 -3
- package/dist/cli/commands/tui.js +4 -3
- package/dist/cli/commands/watch.js +2 -2
- package/dist/cli/doctor.d.ts +7 -2
- package/dist/cli/doctor.js +47 -8
- package/dist/cli/findings.d.ts +103 -0
- package/dist/cli/findings.js +316 -0
- package/dist/cli/first-run-path.d.ts +5 -4
- package/dist/cli/first-run-path.js +14 -7
- package/dist/cli/io.d.ts +17 -3
- package/dist/cli/io.js +22 -4
- package/dist/cli/product-sentence.d.ts +3 -3
- package/dist/cli/product-sentence.js +3 -3
- package/dist/comms/receiving-runtime.js +1 -1
- package/dist/comms/setup.js +1 -1
- package/dist/evidence/redaction.d.ts +7 -3
- package/dist/evidence/redaction.js +68 -24
- package/dist/feedback/export-bundle.js +6 -1
- package/dist/feedback/export.js +3 -0
- package/dist/keys/env-file.d.ts +4 -0
- package/dist/keys/env-file.js +17 -0
- package/dist/keys/key-resolution.d.ts +7 -0
- package/dist/keys/key-resolution.js +10 -8
- package/dist/observer/data.d.ts +14 -1
- package/dist/observer/data.js +19 -9
- package/dist/observer/library.d.ts +4 -0
- package/dist/observer/library.js +18 -15
- package/dist/observer/render.js +8 -2
- package/dist/observer/run-routes.d.ts +2 -0
- package/dist/observer/run-routes.js +43 -34
- package/dist/observer-app.html +3 -3
- package/dist/routes/computer-use/bundle.js +3 -1
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/live-phase.d.ts +1 -0
- package/dist/routes/computer-use/participant-execution.js +3 -1
- package/dist/routes/computer-use/participant-model.js +5 -2
- package/dist/routes/computer-use/plan.js +4 -4
- package/dist/routes/computer-use/preflight.js +2 -2
- package/dist/routes/computer-use/result.js +17 -28
- package/dist/routes/computer-use/single-bundle.js +7 -1
- package/dist/routes/computer-use/types.d.ts +6 -0
- package/dist/routes/scripted/bundle.js +4 -2
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/scripted/result.d.ts +3 -5
- package/dist/routes/scripted/result.js +16 -22
- package/dist/routes/scripted/scenario.js +4 -4
- package/dist/routes/scripted/types.d.ts +1 -0
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.js +2 -0
- package/dist/routes/shared-world/plan.js +1 -1
- package/dist/routes/shared-world/result.js +22 -22
- package/dist/routes/shared-world/route.js +1 -1
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/routes/terminal/bundle.js +4 -2
- package/dist/routes/terminal/credentials.js +1 -1
- package/dist/routes/terminal/dry-run.js +8 -12
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +6 -6
- package/dist/routes/terminal/live-finish.js +23 -17
- package/dist/routes/terminal/plan.js +4 -4
- package/dist/routes/terminal/result.d.ts +6 -7
- package/dist/routes/terminal/result.js +6 -22
- package/dist/routes/terminal/session.js +1 -1
- package/dist/routes/terminal/types.d.ts +3 -0
- package/dist/run/bundle-shape.d.ts +3 -1
- package/dist/run/bundle-shape.js +26 -1
- package/dist/run/bundle.d.ts +30 -0
- package/dist/run/display.d.ts +59 -0
- package/dist/run/display.js +106 -0
- package/dist/run/dry-run.js +11 -7
- package/dist/run/judge.d.ts +3 -2
- package/dist/run/judge.js +10 -0
- package/dist/run/pricing.js +1 -1
- package/dist/run/projection.d.ts +0 -2
- package/dist/run/projection.js +0 -11
- package/dist/run/reclaim-outcomes.d.ts +89 -0
- package/dist/run/reclaim-outcomes.js +213 -0
- package/dist/run/reclaim.d.ts +18 -14
- package/dist/run/reclaim.js +179 -105
- package/dist/run/results.d.ts +11 -18
- package/dist/run/results.js +2 -2
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +11 -0
- package/dist/run/run.d.ts +34 -18
- package/dist/run/run.js +177 -72
- package/dist/run/sandbox-creates.d.ts +59 -0
- package/dist/run/sandbox-creates.js +138 -0
- package/dist/run/sandbox-ids.d.ts +53 -0
- package/dist/run/sandbox-ids.js +198 -0
- package/dist/run/sandbox-receipts.d.ts +16 -0
- package/dist/run/sandbox-receipts.js +70 -0
- package/dist/run/stats.d.ts +5 -0
- package/dist/run/stats.js +8 -4
- package/dist/run/status.d.ts +6 -9
- package/dist/run/status.js +7 -20
- package/dist/run/stored-runs.d.ts +5 -14
- package/dist/run/stored-runs.js +31 -137
- package/dist/run/streams.d.ts +3 -2
- package/dist/run-study.js +6 -1
- package/dist/study/doctor.d.ts +1 -1
- package/dist/study/doctor.js +7 -4
- package/dist/study/init-templates.js +2 -2
- package/dist/study/init.js +14 -3
- package/dist/study/keys.d.ts +12 -0
- package/dist/study/keys.js +32 -0
- package/dist/study/persona.d.ts +4 -0
- package/dist/study/persona.js +4 -2
- package/dist/study/types.d.ts +1 -1
- package/dist/substrates/e2b/peer-install.d.ts +51 -0
- package/dist/substrates/e2b/peer-install.js +161 -0
- package/dist/substrates/e2b/sandbox.d.ts +34 -6
- package/dist/substrates/e2b/sandbox.js +128 -35
- package/dist/substrates/e2b/sdk.d.ts +33 -0
- package/dist/substrates/e2b/sdk.js +21 -21
- package/dist/substrates/shell.d.ts +2 -0
- package/dist/substrates/shell.js +4 -0
- package/dist/tui-app.js +133 -133
- package/dist/verify/artifacts.d.ts +4 -0
- package/dist/verify/artifacts.js +8 -0
- package/dist/verify/liveness.d.ts +19 -3
- package/dist/verify/liveness.js +53 -28
- package/dist/verify/verify.d.ts +4 -1
- package/dist/verify/verify.js +27 -3
- package/docs/architecture/actor-contract.md +2 -1
- package/docs/architecture/observer.md +22 -7
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/task-protocol-support.md +13 -13
- package/docs/architecture/terminal-product-route.md +1 -1
- package/docs/contracts/run-bundle.md +90 -7
- package/docs/contracts/schemas.md +39 -22
- package/docs/contracts/study-analysis.md +45 -0
- package/docs/contracts/study-costs.md +8 -0
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
- package/docs/principles/invariants-and-defaults.md +11 -8
- package/package.json +5 -3
- package/skills/humanish/SKILL.md +18 -4
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
// A run's analysis findings for `review`, `analyze show` and the end of a live `run`: the
|
|
2
|
+
// `humanish.analysis-findings.v1` projection and its text. The analysis artifact stays the record;
|
|
3
|
+
// this reads the version `analyze show` selects and never starts an analysis.
|
|
4
|
+
import path from "node:path";
|
|
5
|
+
import { DEFAULT_ANALYSIS_MAX_COST_USD } from "../analysis/automatic-config.js";
|
|
6
|
+
import { loadAnalysis } from "../analysis/load.js";
|
|
7
|
+
import { projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
8
|
+
import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
9
|
+
import { resolvePhysicalCwd, runIdOf } from "../run/paths.js";
|
|
10
|
+
import { plural } from "../run/text.js";
|
|
11
|
+
import { shellQuote } from "../substrates/shell.js";
|
|
12
|
+
import { analysisOutcomeText, wantsJson } from "./io.js";
|
|
13
|
+
export const ANALYSIS_FINDINGS_SCHEMA = "humanish.analysis-findings.v1";
|
|
14
|
+
/** ` --cwd <dir>`, or "" when `cwd` is the current directory. */
|
|
15
|
+
function cwdFlag(cwd) {
|
|
16
|
+
if (path.resolve(cwd) === process.cwd())
|
|
17
|
+
return "";
|
|
18
|
+
return ` --cwd ${/^[\w@%+=:,./-]+$/.test(cwd) ? cwd : shellQuote(cwd)}`;
|
|
19
|
+
}
|
|
20
|
+
function analyzeCommand(source, provider) {
|
|
21
|
+
const run = `humanish analyze --run ${source.runId}${source.cwdFlag}`;
|
|
22
|
+
return provider === "codex"
|
|
23
|
+
? `${run} --provider codex`
|
|
24
|
+
: `${run} --max-cost ${DEFAULT_ANALYSIS_MAX_COST_USD}`;
|
|
25
|
+
}
|
|
26
|
+
function evidenceOf(finding, analysis, runRoot) {
|
|
27
|
+
const bases = new Map();
|
|
28
|
+
for (const observation of finding.observations)
|
|
29
|
+
for (const id of observation.evidenceIds) {
|
|
30
|
+
const set = bases.get(id) ?? new Set();
|
|
31
|
+
set.add(observation.basis);
|
|
32
|
+
bases.set(id, set);
|
|
33
|
+
}
|
|
34
|
+
const byId = new Map(analysis.evidence.map((entry) => [entry.id, entry]));
|
|
35
|
+
return [...bases].flatMap(([id, used]) => {
|
|
36
|
+
const entry = byId.get(id);
|
|
37
|
+
if (!entry)
|
|
38
|
+
return [];
|
|
39
|
+
return [
|
|
40
|
+
{
|
|
41
|
+
id,
|
|
42
|
+
streamId: entry.streamId,
|
|
43
|
+
kind: entry.kind,
|
|
44
|
+
bases: [...used],
|
|
45
|
+
frame: entry.frame,
|
|
46
|
+
elapsedMs: entry.elapsedMs,
|
|
47
|
+
at: entry.at,
|
|
48
|
+
capture: entry.capture ? path.join(runRoot, entry.capture.path) : null,
|
|
49
|
+
},
|
|
50
|
+
];
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
function findingView(finding, loaded, runRoot) {
|
|
54
|
+
const labels = new Map(loaded.analysis.participants.map((p) => [p.streamId, p.label]));
|
|
55
|
+
const correction = loaded.corrections.filter((entry) => entry.findingId === finding.id).at(-1);
|
|
56
|
+
return {
|
|
57
|
+
id: finding.id,
|
|
58
|
+
title: finding.title,
|
|
59
|
+
summary: finding.summary,
|
|
60
|
+
impact: finding.impact,
|
|
61
|
+
confidence: finding.confidence,
|
|
62
|
+
recovery: finding.recovery,
|
|
63
|
+
affected: finding.affectedStreamIds.map((streamId) => ({
|
|
64
|
+
streamId,
|
|
65
|
+
label: labels.get(streamId) ?? streamId,
|
|
66
|
+
})),
|
|
67
|
+
exposedCount: finding.exposedStreamIds.length,
|
|
68
|
+
evidence: evidenceOf(finding, loaded.analysis, runRoot),
|
|
69
|
+
nextStep: finding.nextStep,
|
|
70
|
+
correction: correction
|
|
71
|
+
? {
|
|
72
|
+
status: correction.status,
|
|
73
|
+
reason: correction.reason,
|
|
74
|
+
replacementClaim: correction.replacementClaim,
|
|
75
|
+
}
|
|
76
|
+
: null,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** A view with no findings: what happened and the command that gets them. */
|
|
80
|
+
function withoutFindings(source, state, reason, message, next) {
|
|
81
|
+
return {
|
|
82
|
+
schema: ANALYSIS_FINDINGS_SCHEMA,
|
|
83
|
+
runId: source.runId,
|
|
84
|
+
state,
|
|
85
|
+
reason,
|
|
86
|
+
message,
|
|
87
|
+
next,
|
|
88
|
+
analysisId: null,
|
|
89
|
+
status: null,
|
|
90
|
+
provider: null,
|
|
91
|
+
model: null,
|
|
92
|
+
completedAt: null,
|
|
93
|
+
estimatedCostUsd: null,
|
|
94
|
+
runPath: source.runRoot || null,
|
|
95
|
+
path: null,
|
|
96
|
+
summary: null,
|
|
97
|
+
findings: [],
|
|
98
|
+
limitations: [],
|
|
99
|
+
warnings: source.loaded.warnings,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/** The findings view of a run's selected analysis, or why it has none. */
|
|
103
|
+
export function analysisFindings(source) {
|
|
104
|
+
if (source.mode !== undefined && source.mode !== "live")
|
|
105
|
+
return withoutFindings(source, "dry_run", null, "This is a dry run: no participant used the product, so there is nothing to analyze. Findings come from live runs.", null);
|
|
106
|
+
const loaded = projectShareCheckedAnalysis(source.loaded);
|
|
107
|
+
const analysis = loaded.analysis;
|
|
108
|
+
if (analysis?.result && (loaded.state === "ready" || loaded.state === "stale")) {
|
|
109
|
+
const result = analysis.result;
|
|
110
|
+
const stale = loaded.state === "stale";
|
|
111
|
+
return {
|
|
112
|
+
schema: ANALYSIS_FINDINGS_SCHEMA,
|
|
113
|
+
runId: source.runId,
|
|
114
|
+
state: loaded.state,
|
|
115
|
+
reason: stale ? (loaded.warnings[0] ?? null) : null,
|
|
116
|
+
message: stale
|
|
117
|
+
? `The run's evidence changed after analysis ${analysis.id}, so its findings may not match the run.`
|
|
118
|
+
: `Analysis ${analysis.id} is ${analysis.status}, with ${plural(result.findings.length, "finding")}.`,
|
|
119
|
+
next: stale ? analyzeCommand(source, analysis.provider) : null,
|
|
120
|
+
analysisId: analysis.id,
|
|
121
|
+
status: analysis.status,
|
|
122
|
+
provider: analysis.provider,
|
|
123
|
+
model: analysis.config.model,
|
|
124
|
+
completedAt: analysis.completedAt,
|
|
125
|
+
estimatedCostUsd: analysis.usage.estimatedCostUsd,
|
|
126
|
+
runPath: source.runRoot,
|
|
127
|
+
path: path.join(source.runRoot, "analysis", analysis.id, "analysis.json"),
|
|
128
|
+
summary: result.summary,
|
|
129
|
+
findings: result.findings.map((finding) => findingView(finding, { ...loaded, analysis }, source.runRoot)),
|
|
130
|
+
limitations: result.limitations,
|
|
131
|
+
warnings: loaded.warnings,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
if (loaded.warnings.includes("ANALYSIS_SENSITIVE_TEXT_QUARANTINED"))
|
|
135
|
+
return withoutFindings(source, "unavailable", "ANALYSIS_SENSITIVE_TEXT_QUARANTINED", `The analysis text matched a sensitive-text pattern, so it is not printed. humanish analyze show --run ${source.runId}${source.cwdFlag} --json prints the raw record.`, null);
|
|
136
|
+
const job = loaded.automatic;
|
|
137
|
+
if (job?.state === "queued" || job?.state === "running")
|
|
138
|
+
return withoutFindings(source, "running", null, "The automatic analysis is still running. Check again when it finishes.", `humanish review --run ${source.runId}${source.cwdFlag}`);
|
|
139
|
+
if (analysis && analysis.result === null)
|
|
140
|
+
return withoutFindings(source, "failed", analysis.error, `Analysis attempt ${analysis.id} ${analysis.status === "cancelled" ? "was cancelled" : "failed"}${analysis.error ? ` (${analysis.error})` : ""} and left no findings.`, analyzeCommand(source, analysis.provider));
|
|
141
|
+
if (job?.state === "failed" || job?.state === "cancelled") {
|
|
142
|
+
const outcome = analysisOutcomeText(job);
|
|
143
|
+
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? "codex" : undefined));
|
|
144
|
+
}
|
|
145
|
+
if (job?.state === "skipped") {
|
|
146
|
+
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
147
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run prints its cost estimate; run it again with a --max-cost above the estimate." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
148
|
+
? null
|
|
149
|
+
: `${analyzeCommand(source, undefined)}${refused ? " --dry-run" : ""}`);
|
|
150
|
+
}
|
|
151
|
+
if (job !== undefined || loaded.state === "invalid")
|
|
152
|
+
return withoutFindings(source, "unavailable", job?.reason ?? loaded.warnings[0] ?? null, "This run's analysis records could not be read or checked, so no findings are shown.", `humanish analyze list --run ${source.runId}${source.cwdFlag}`);
|
|
153
|
+
return withoutFindings(source, "none", null, "No analysis has run for this run.", analyzeCommand(source, undefined));
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* The run's bundle mode and its selected analysis: `analysisId`, or the version `analyze show`
|
|
157
|
+
* defaults to. Null when the run is missing or its storage is unsafe.
|
|
158
|
+
*/
|
|
159
|
+
export async function readRunAnalysis(cwd, run, analysisId) {
|
|
160
|
+
const physical = await resolvePhysicalCwd(cwd);
|
|
161
|
+
const prepared = await resolveRunPath(physical, run).catch(() => null);
|
|
162
|
+
if (!prepared)
|
|
163
|
+
return null;
|
|
164
|
+
const bundle = await loadRunBundlePrepared(physical, prepared).catch(() => null);
|
|
165
|
+
return {
|
|
166
|
+
runId: runIdOf(prepared),
|
|
167
|
+
mode: bundle?.bundle.mode,
|
|
168
|
+
runRoot: prepared.relativeRunRoot,
|
|
169
|
+
loaded: await loadAnalysis(prepared, analysisId),
|
|
170
|
+
cwdFlag: cwdFlag(cwd),
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
/** The view of a run that could not be found. */
|
|
174
|
+
export function missingRunFindings(cwd, run) {
|
|
175
|
+
const loaded = {
|
|
176
|
+
state: "invalid",
|
|
177
|
+
analysis: null,
|
|
178
|
+
corrections: [],
|
|
179
|
+
warnings: ["ANALYSIS_RUN_NOT_FOUND"],
|
|
180
|
+
};
|
|
181
|
+
const flag = cwdFlag(cwd);
|
|
182
|
+
return withoutFindings({ runId: run, mode: undefined, runRoot: "", loaded, cwdFlag: flag }, "unavailable", "ANALYSIS_RUN_NOT_FOUND", `Run ${run} was not found or its storage is unsafe.`, `humanish runs${flag}`);
|
|
183
|
+
}
|
|
184
|
+
/** The findings of a run's selected analysis, or why it has none. */
|
|
185
|
+
export async function readRunFindings(cwd, run, analysisId) {
|
|
186
|
+
const source = await readRunAnalysis(cwd, run, analysisId);
|
|
187
|
+
return source ? analysisFindings(source) : missingRunFindings(cwd, run);
|
|
188
|
+
}
|
|
189
|
+
const IMPACT_TEXT = {
|
|
190
|
+
blocked_task: "blocked task",
|
|
191
|
+
friction: "friction",
|
|
192
|
+
recovery: "recovery",
|
|
193
|
+
uncertain: "uncertain",
|
|
194
|
+
};
|
|
195
|
+
const RECOVERY_TEXT = {
|
|
196
|
+
recovered: "recovered",
|
|
197
|
+
not_observed: "no recovery observed",
|
|
198
|
+
unknown: "recovery unknown",
|
|
199
|
+
};
|
|
200
|
+
/** `+1:05` from milliseconds since the first retained capture. */
|
|
201
|
+
function elapsedText(ms) {
|
|
202
|
+
const seconds = Math.floor(ms / 1000);
|
|
203
|
+
const minutes = Math.floor(seconds / 60);
|
|
204
|
+
const hours = Math.floor(minutes / 60);
|
|
205
|
+
const ss = String(seconds % 60).padStart(2, "0");
|
|
206
|
+
return hours > 0
|
|
207
|
+
? `+${hours}:${String(minutes % 60).padStart(2, "0")}:${ss}`
|
|
208
|
+
: `+${minutes}:${ss}`;
|
|
209
|
+
}
|
|
210
|
+
/** One participant's cited moments: each frame once, at its earliest cited time. */
|
|
211
|
+
function momentsText(evidence) {
|
|
212
|
+
const frames = new Map();
|
|
213
|
+
let unframed = 0;
|
|
214
|
+
for (const entry of evidence) {
|
|
215
|
+
if (entry.frame === null) {
|
|
216
|
+
unframed++;
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
const earlier = frames.get(entry.frame);
|
|
220
|
+
if (earlier === undefined ||
|
|
221
|
+
(entry.elapsedMs !== null && (earlier ?? Infinity) > entry.elapsedMs))
|
|
222
|
+
frames.set(entry.frame, entry.elapsedMs);
|
|
223
|
+
}
|
|
224
|
+
const parts = [...frames]
|
|
225
|
+
.sort(([a], [b]) => a - b)
|
|
226
|
+
.map(([frame, ms]) => `frame ${frame}${ms === null ? "" : ` at ${elapsedText(ms)}`}`);
|
|
227
|
+
if (unframed > 0)
|
|
228
|
+
parts.push(`${plural(unframed, "item")} with no frame`);
|
|
229
|
+
return parts.join(", ");
|
|
230
|
+
}
|
|
231
|
+
function findingLines(finding, runRoot) {
|
|
232
|
+
const cited = [...new Set(finding.evidence.map((entry) => entry.streamId))];
|
|
233
|
+
const labels = new Map(finding.affected.map((p) => [p.streamId, p.label]));
|
|
234
|
+
const affectedIds = finding.affected.map((p) => p.streamId);
|
|
235
|
+
const streams = [...affectedIds, ...cited.filter((id) => !labels.has(id))];
|
|
236
|
+
const captures = [
|
|
237
|
+
...new Set(finding.evidence
|
|
238
|
+
.filter((entry) => entry.capture !== null)
|
|
239
|
+
.sort((a, b) => (a.frame ?? 0) - (b.frame ?? 0))
|
|
240
|
+
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
241
|
+
];
|
|
242
|
+
const correction = finding.correction;
|
|
243
|
+
return [
|
|
244
|
+
`${finding.id} ${finding.title}`,
|
|
245
|
+
` impact: ${IMPACT_TEXT[finding.impact]} · confidence: ${finding.confidence} · recovery: ${RECOVERY_TEXT[finding.recovery]}`,
|
|
246
|
+
` ${finding.summary}`,
|
|
247
|
+
` affected: ${finding.affected.length} of ${plural(finding.exposedCount, "exposed participant")}`,
|
|
248
|
+
...streams.map((streamId) => {
|
|
249
|
+
const moments = momentsText(finding.evidence.filter((entry) => entry.streamId === streamId));
|
|
250
|
+
const label = `${labels.get(streamId) ?? streamId}${labels.has(streamId) ? "" : " (cited, not affected)"}`;
|
|
251
|
+
return ` - ${label}${moments ? `: ${moments}` : ""}`;
|
|
252
|
+
}),
|
|
253
|
+
...(captures.length === 0 ? [] : [` captures: ${captures.join(", ")}`]),
|
|
254
|
+
` next step: ${finding.nextStep}`,
|
|
255
|
+
...(correction === null
|
|
256
|
+
? []
|
|
257
|
+
: [
|
|
258
|
+
` human review: ${correction.status}, ${correction.reason}${correction.replacementClaim ? ` Amended claim: ${correction.replacementClaim}` : ""}`,
|
|
259
|
+
]),
|
|
260
|
+
];
|
|
261
|
+
}
|
|
262
|
+
/** The full findings block: the analysis, every finding with its evidence, and its limitations. */
|
|
263
|
+
export function formatFindings(view) {
|
|
264
|
+
if (view.state !== "ready" && view.state !== "stale")
|
|
265
|
+
return [`findings: none. ${view.message}`, ...(view.next ? [`next: ${view.next}`] : [])];
|
|
266
|
+
const model = [view.provider, view.model].filter(Boolean).join(" ");
|
|
267
|
+
const cost = view.estimatedCostUsd === null ? "" : `, estimated $${view.estimatedCostUsd.toFixed(2)}`;
|
|
268
|
+
return [
|
|
269
|
+
`findings: ${view.findings.length} from analysis ${view.analysisId} (${view.status}, ${model}${cost})`,
|
|
270
|
+
...(view.state === "stale" ? [`warning: ${view.message}`, `next: ${view.next}`] : []),
|
|
271
|
+
...(view.summary ? [view.summary] : []),
|
|
272
|
+
...view.findings.flatMap((finding) => ["", ...findingLines(finding, view.runPath ?? "")]),
|
|
273
|
+
...(view.limitations.length === 0
|
|
274
|
+
? []
|
|
275
|
+
: ["", "limitations:", ...view.limitations.map((limitation) => `- ${limitation}`)]),
|
|
276
|
+
...view.warnings.map((warning) => `warning: ${warning}`),
|
|
277
|
+
"",
|
|
278
|
+
`analysis: ${view.path}`,
|
|
279
|
+
];
|
|
280
|
+
}
|
|
281
|
+
/** At most `limit` findings, one line each, and the command that prints all of them. */
|
|
282
|
+
export function formatFindingsSummary(view, fullCommand, limit = 3) {
|
|
283
|
+
if (view.state !== "ready" && view.state !== "stale")
|
|
284
|
+
return [`findings: ${view.message}`, ...(view.next ? [`next: ${view.next}`] : [])];
|
|
285
|
+
const shown = view.findings.slice(0, limit);
|
|
286
|
+
return [
|
|
287
|
+
`findings: ${view.findings.length === 0 ? "none" : view.findings.length}${view.state === "stale" ? " (stale)" : ""}`,
|
|
288
|
+
...shown.map((finding) => `- ${finding.id} ${IMPACT_TEXT[finding.impact]}, ${finding.confidence} confidence, ${RECOVERY_TEXT[finding.recovery]}: ${finding.title}`),
|
|
289
|
+
...(view.findings.length > limit
|
|
290
|
+
? [`- and ${plural(view.findings.length - limit, "more finding")}`]
|
|
291
|
+
: []),
|
|
292
|
+
`all findings: ${fullCommand}`,
|
|
293
|
+
];
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* The end of a live run's human output, after its `analysis:` line: the findings of the analysis
|
|
297
|
+
* the run just completed. Prints nothing for --json, or when the run's analysis did not complete,
|
|
298
|
+
* since that line already says why.
|
|
299
|
+
*/
|
|
300
|
+
export async function writeRunFindings(command, io, cwd, result) {
|
|
301
|
+
const outcome = result.automaticAnalysis;
|
|
302
|
+
const analysisId = outcome?.result?.analysisId;
|
|
303
|
+
if (wantsJson(command) ||
|
|
304
|
+
result.runId === undefined ||
|
|
305
|
+
analysisId === undefined ||
|
|
306
|
+
(outcome?.state !== "complete" && outcome?.state !== "partial"))
|
|
307
|
+
return;
|
|
308
|
+
try {
|
|
309
|
+
const view = await readRunFindings(cwd, result.runId, analysisId);
|
|
310
|
+
const full = `humanish review --run ${view.runId}${cwdFlag(cwd)}`;
|
|
311
|
+
io.writeOut(`${formatFindingsSummary(view, full).join("\n")}\n`);
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
// The run's own result is already printed; review reads the same record later.
|
|
315
|
+
}
|
|
316
|
+
}
|
|
@@ -15,11 +15,12 @@ export interface FirstRunEnvironment {
|
|
|
15
15
|
*/
|
|
16
16
|
hasDesktopSdk: boolean;
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
18
|
+
* The command that installs `@e2b/desktop` where this humanish resolves it, ahead of
|
|
19
|
+
* `npx humanish`: beside humanish in its project, or humanish and the peer together in this one
|
|
20
|
+
* when humanish runs from an npx cache, a global install or another directory. Undefined where
|
|
21
|
+
* `npm i` would prune a node_modules that no package.json declares.
|
|
21
22
|
*/
|
|
22
|
-
|
|
23
|
+
desktopPeerCommand: string | undefined;
|
|
23
24
|
/** A provider API key for the model. */
|
|
24
25
|
hasProviderKey: boolean;
|
|
25
26
|
/** The coding agents already signed in locally: Codex, Claude Code, or both. */
|
|
@@ -79,13 +79,16 @@ export function firstRunSteps(env) {
|
|
|
79
79
|
: `${preferredAgent(env)?.label} (already signed in, so no API key is needed)`;
|
|
80
80
|
// Everything the step needs, in one line. Splitting it across two commands means the second
|
|
81
81
|
// one fails, which is the same dead end this guidance exists to remove.
|
|
82
|
+
if (!env.hasDesktopSdk && env.desktopPeerCommand === undefined) {
|
|
83
|
+
steps.push({
|
|
84
|
+
command: `${HUMANISH} doctor`,
|
|
85
|
+
why: "the hosted starter needs the desktop SDK, and installing it from here would remove what this directory's node_modules holds; doctor names the command to run from your project's directory",
|
|
86
|
+
});
|
|
87
|
+
return steps;
|
|
88
|
+
}
|
|
82
89
|
const command = env.hasDesktopSdk
|
|
83
90
|
? `${HUMANISH} run try-live`
|
|
84
|
-
: env.
|
|
85
|
-
? `npm i -D @e2b/desktop && ${HUMANISH} run try-live`
|
|
86
|
-
: // Running from an npx cache: installing only the peer here would not be found, because
|
|
87
|
-
// Node resolves it relative to humanish. Both, or neither.
|
|
88
|
-
`npm i -D humanish @e2b/desktop && ${HUMANISH} run try-live`;
|
|
91
|
+
: `${env.desktopPeerCommand} && ${HUMANISH} run try-live`;
|
|
89
92
|
steps.push({
|
|
90
93
|
command,
|
|
91
94
|
why: `a live study: one participant drives a real app in a hosted desktop, using ${brain}` +
|
|
@@ -99,7 +102,7 @@ export function firstRunSteps(env) {
|
|
|
99
102
|
}
|
|
100
103
|
steps.push({
|
|
101
104
|
command: `${HUMANISH} keys set openai`,
|
|
102
|
-
why: "openai-computer-use needs an API key; to use a Codex or Claude Code login instead, sign in and change the study to
|
|
105
|
+
why: "openai-computer-use needs an API key; to use a Codex or Claude Code login instead, sign in and change the study to actor.type: local-agent" +
|
|
103
106
|
localUnavailable,
|
|
104
107
|
});
|
|
105
108
|
return steps;
|
|
@@ -134,8 +137,12 @@ export function agentsSection() {
|
|
|
134
137
|
"humanish run local-browser # your loopback app; no E2B or model API key",
|
|
135
138
|
"humanish run try-live # demo app study: E2B plus the selected participant's authentication",
|
|
136
139
|
"humanish verify --run latest --json # is the evidence share-safe",
|
|
140
|
+
"humanish review --run <id> # the run's outcome and its analysis findings",
|
|
137
141
|
"```",
|
|
138
142
|
"",
|
|
143
|
+
"- After a live run, run `humanish review --run <id>` (or `--json`) and report its findings to the",
|
|
144
|
+
" person you are working for: each one's title, impact, confidence, recovery and cited captures.",
|
|
145
|
+
" When it shows no findings, report its message and the command it names.",
|
|
139
146
|
"- Studies are declared in `humanish/studies/*.yaml`. Edit `try-live.yaml`'s `subject` to point",
|
|
140
147
|
" at this project's own app once you have seen a run work.",
|
|
141
148
|
"- Configure the local study without editing YAML: `humanish init --yes --local-browser",
|
|
@@ -144,7 +151,7 @@ export function agentsSection() {
|
|
|
144
151
|
" bundles into an issue; `humanish feedback issue` writes a redacted, share-safe draft.",
|
|
145
152
|
"- `humanish tui` is for people and refuses to run in an agent session. Use the `--json`",
|
|
146
153
|
" commands above instead, and tell the person you are working for that `humanish tui` exists.",
|
|
147
|
-
"- A live run spends money. `
|
|
154
|
+
"- A live run spends money. `caps.maxUsd` in each study caps estimated model spend: the run",
|
|
148
155
|
" stops before its next request once the estimate passes it, so the last request can go over, and",
|
|
149
156
|
" hosted desktop time is billed on top. Do not raise it without asking the person you are working for.",
|
|
150
157
|
"",
|
package/dist/cli/io.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Command, Option } from "commander";
|
|
2
|
-
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
2
|
+
import { discoverProviderKeys, type DotenvLoad } from "../keys/key-resolution.js";
|
|
3
3
|
import { type TelemetryProperties } from "./telemetry.js";
|
|
4
4
|
import type { RunResult } from "../run/results.js";
|
|
5
5
|
export declare const CLI_RESPONSE_SCHEMA = "humanish.cli-response.v1";
|
|
@@ -13,7 +13,18 @@ export interface CliIo {
|
|
|
13
13
|
export declare const JSON_OPTION_DESCRIPTION = "Print a machine-readable JSON response.";
|
|
14
14
|
export declare const CWD_OPTION_DESCRIPTION = "Project directory.";
|
|
15
15
|
export declare const RUN_OPTION_DESCRIPTION = "Run id, or latest.";
|
|
16
|
-
export declare const
|
|
16
|
+
export declare const DOTENV_OPTION_DESCRIPTION = "Load unset variables from this dotenv file. Values are never printed or saved.";
|
|
17
|
+
/**
|
|
18
|
+
* The hidden `--env-file <path>` that `--dotenv` replaced. Node scans the whole argv for
|
|
19
|
+
* `--env-file` before a script runs, and stops only at `--` (src/node_dotenv.cc), so a missing file
|
|
20
|
+
* exits 9 through Node and never reaches humanish. `--dotenv` does not collide. Removed in 0.111.0.
|
|
21
|
+
*/
|
|
22
|
+
export declare function envFileAliasOption(): Option;
|
|
23
|
+
/** The dotenv path a command was given: `--dotenv`, or the `--env-file` alias with its warning. */
|
|
24
|
+
export declare function dotenvPathOf(options: {
|
|
25
|
+
dotenv?: string | undefined;
|
|
26
|
+
envFile?: string | undefined;
|
|
27
|
+
}, command: Command, io: CliIo): string | undefined;
|
|
17
28
|
export declare const PORT_OPTION_DESCRIPTION = "Port to listen on at 127.0.0.1.";
|
|
18
29
|
/** `--port` for a loopback server whose default, 0, lets the OS pick a free port. */
|
|
19
30
|
export declare function freePortOption(): Option;
|
|
@@ -22,6 +33,7 @@ export interface StudyCommandOptions {
|
|
|
22
33
|
cwd: string;
|
|
23
34
|
detach?: boolean | undefined;
|
|
24
35
|
dryRun?: boolean | undefined;
|
|
36
|
+
dotenv?: string | undefined;
|
|
25
37
|
envFile?: string | undefined;
|
|
26
38
|
json?: boolean | undefined;
|
|
27
39
|
open?: boolean | undefined;
|
|
@@ -50,13 +62,15 @@ export declare function applyEnvFileOption(args: {
|
|
|
50
62
|
io: CliIo;
|
|
51
63
|
env?: NodeJS.ProcessEnv;
|
|
52
64
|
onDiscovered?: (names: string[]) => void;
|
|
65
|
+
/** Called with the dotenv path and the names it set, which were unset before. */
|
|
66
|
+
onLoaded?: (load: DotenvLoad) => void;
|
|
53
67
|
/** False for the study-running commands, which discover only once the study resolves to live. */
|
|
54
68
|
discoverKeys?: boolean;
|
|
55
69
|
}): Promise<boolean>;
|
|
56
70
|
/**
|
|
57
71
|
* Provider-key discovery: fill still-missing keys from the documented project overlay, the
|
|
58
72
|
* owning vendors' native stores, and the humanish user store. It is fill-only (an explicit
|
|
59
|
-
* --
|
|
73
|
+
* --dotenv or process env always wins), and each fill is announced by name and source on
|
|
60
74
|
* stderr, never by value. HUMANISH_STRICT_KEYS=1 restores env-only behavior.
|
|
61
75
|
*/
|
|
62
76
|
export declare function discoverCliKeys(args: {
|
package/dist/cli/io.js
CHANGED
|
@@ -4,10 +4,10 @@ import { readFileSync } from "node:fs";
|
|
|
4
4
|
import { dirname, resolve } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { Command, Option } from "commander";
|
|
7
|
-
import { loadEnvFile } from "../keys/env-file.js";
|
|
7
|
+
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
8
8
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
9
9
|
import { deriveRunFacts } from "./telemetry.js";
|
|
10
|
-
import { withQueuedWarnings } from "./deprecations.js";
|
|
10
|
+
import { warnAndQueue, withQueuedWarnings } from "./deprecations.js";
|
|
11
11
|
import { forTerminal } from "../routes/terminal/encoding.js";
|
|
12
12
|
export const CLI_RESPONSE_SCHEMA = "humanish.cli-response.v1";
|
|
13
13
|
function readCliVersion() {
|
|
@@ -23,7 +23,23 @@ export const CLI_VERSION = readCliVersion();
|
|
|
23
23
|
export const JSON_OPTION_DESCRIPTION = "Print a machine-readable JSON response.";
|
|
24
24
|
export const CWD_OPTION_DESCRIPTION = "Project directory.";
|
|
25
25
|
export const RUN_OPTION_DESCRIPTION = "Run id, or latest.";
|
|
26
|
-
export const
|
|
26
|
+
export const DOTENV_OPTION_DESCRIPTION = "Load unset variables from this dotenv file. Values are never printed or saved.";
|
|
27
|
+
const ENV_FILE_RENAMED = "`--env-file` is now `--dotenv`; `--env-file` is removed in 0.111.0.";
|
|
28
|
+
/**
|
|
29
|
+
* The hidden `--env-file <path>` that `--dotenv` replaced. Node scans the whole argv for
|
|
30
|
+
* `--env-file` before a script runs, and stops only at `--` (src/node_dotenv.cc), so a missing file
|
|
31
|
+
* exits 9 through Node and never reaches humanish. `--dotenv` does not collide. Removed in 0.111.0.
|
|
32
|
+
*/
|
|
33
|
+
export function envFileAliasOption() {
|
|
34
|
+
return new Option("--env-file <path>").hideHelp().conflicts("dotenv");
|
|
35
|
+
}
|
|
36
|
+
/** The dotenv path a command was given: `--dotenv`, or the `--env-file` alias with its warning. */
|
|
37
|
+
export function dotenvPathOf(options, command, io) {
|
|
38
|
+
if (options.envFile === undefined)
|
|
39
|
+
return options.dotenv;
|
|
40
|
+
warnAndQueue(command, io, ENV_FILE_RENAMED);
|
|
41
|
+
return options.envFile;
|
|
42
|
+
}
|
|
27
43
|
export const PORT_OPTION_DESCRIPTION = "Port to listen on at 127.0.0.1.";
|
|
28
44
|
/** `--port` for a loopback server whose default, 0, lets the OS pick a free port. */
|
|
29
45
|
export function freePortOption() {
|
|
@@ -74,6 +90,8 @@ export async function applyEnvFileOption(args) {
|
|
|
74
90
|
}
|
|
75
91
|
for (const name of result.loaded)
|
|
76
92
|
env[name] = stagedEnv[name];
|
|
93
|
+
recordDotenvNames(env, result.loaded);
|
|
94
|
+
args.onLoaded?.({ path: args.envFile, names: result.loaded });
|
|
77
95
|
}
|
|
78
96
|
if (args.discoverKeys !== false) {
|
|
79
97
|
await discoverCliKeys({
|
|
@@ -88,7 +106,7 @@ export async function applyEnvFileOption(args) {
|
|
|
88
106
|
/**
|
|
89
107
|
* Provider-key discovery: fill still-missing keys from the documented project overlay, the
|
|
90
108
|
* owning vendors' native stores, and the humanish user store. It is fill-only (an explicit
|
|
91
|
-
* --
|
|
109
|
+
* --dotenv or process env always wins), and each fill is announced by name and source on
|
|
92
110
|
* stderr, never by value. HUMANISH_STRICT_KEYS=1 restores env-only behavior.
|
|
93
111
|
*/
|
|
94
112
|
export async function discoverCliKeys(args) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The product in one sentence.
|
|
3
|
-
*
|
|
4
|
-
* equal.
|
|
2
|
+
* The product in one sentence. package.json's description, `humanish --help` and bare `humanish`
|
|
3
|
+
* use it, and line 3 of `README.md` and the humanish skill's description open with it;
|
|
4
|
+
* tests/cli/product-sentence.test.ts holds them equal.
|
|
5
5
|
*/
|
|
6
6
|
export declare const PRODUCT_SENTENCE = "Synthetic user research for apps, CLIs, and agent-facing product flows.";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The product in one sentence.
|
|
3
|
-
*
|
|
4
|
-
* equal.
|
|
2
|
+
* The product in one sentence. package.json's description, `humanish --help` and bare `humanish`
|
|
3
|
+
* use it, and line 3 of `README.md` and the humanish skill's description open with it;
|
|
4
|
+
* tests/cli/product-sentence.test.ts holds them equal.
|
|
5
5
|
*/
|
|
6
6
|
export const PRODUCT_SENTENCE = "Synthetic user research for apps, CLIs, and agent-facing product flows.";
|
|
@@ -29,7 +29,7 @@ export async function resolveReceivingConnection(cwd, name, env, keyDeps) {
|
|
|
29
29
|
});
|
|
30
30
|
const apiKey = resolved[connection.apiKeyEnv]?.trim();
|
|
31
31
|
if (!apiKey)
|
|
32
|
-
throw new Error("Email connection credential is missing. Add it in Connections, or provide the configured key with --
|
|
32
|
+
throw new Error("Email connection credential is missing. Add it in Connections, or provide the configured key with --dotenv.");
|
|
33
33
|
return { connection, apiKey, adapter: createReceivingAdapter(connection, apiKey) };
|
|
34
34
|
}
|
|
35
35
|
export async function prepareReceivingRun(args) {
|
package/dist/comms/setup.js
CHANGED
|
@@ -53,7 +53,7 @@ export async function checkCommsConnection(args) {
|
|
|
53
53
|
...base,
|
|
54
54
|
ready: false,
|
|
55
55
|
code: "credential_missing",
|
|
56
|
-
message: "The configured credential is missing. Add it in Connections or provide it with --
|
|
56
|
+
message: "The configured credential is missing. Add it in Connections or provide it with --dotenv.",
|
|
57
57
|
};
|
|
58
58
|
base.credentialPresent = true;
|
|
59
59
|
if (!args.online)
|
|
@@ -7,10 +7,14 @@ export declare function containsSensitive(text: string): boolean;
|
|
|
7
7
|
* passes only this value at a sandbox-id key.
|
|
8
8
|
*/
|
|
9
9
|
export declare const REDACTED_SANDBOX_ID = "[redacted-sandbox-id]";
|
|
10
|
+
/** A sandbox id's public stand-in: its digest matches a receipt without naming the sandbox. */
|
|
11
|
+
export declare function sandboxIdDigest(id: string): string;
|
|
12
|
+
/** The raw ids at the keys redactSandboxIds replaces, anywhere in `value`. */
|
|
13
|
+
export declare function collectSandboxIds(value: unknown, into?: Set<string>): Set<string>;
|
|
10
14
|
/**
|
|
11
|
-
* `value` with every sandbox id replaced by REDACTED_SANDBOX_ID: strings at
|
|
12
|
-
* `subjectSandboxId`, and the `id` of each `providerResources` entry.
|
|
13
|
-
*
|
|
15
|
+
* `value` with every sandbox id replaced by REDACTED_SANDBOX_ID plus its digest: strings at
|
|
16
|
+
* `sandboxId` and `subjectSandboxId`, and the `id` of each `providerResources` or `resources` entry. Raw ids live
|
|
17
|
+
* only in a run's sandbox-receipts.ndjson; every record and output names a sandbox by digest.
|
|
14
18
|
*/
|
|
15
19
|
export declare function redactSandboxIds(value: unknown): unknown;
|
|
16
20
|
/** Redact secrets to [REDACTED_SECRET] and local paths to their path labels. */
|