humanish 0.114.0 → 0.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/CONTEXT.md +1 -1
- package/dist/actors/closing-report-limits.d.ts +11 -0
- package/dist/actors/closing-report-limits.js +13 -0
- package/dist/actors/closing-report.d.ts +68 -0
- package/dist/actors/closing-report.js +55 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +4 -47
- package/dist/actors/codex/restricted-participant-policy.js +12 -40
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -10
- package/dist/actors/computer-use/loop/debrief.js +10 -46
- package/dist/actors/computer-use/loop/impressions.d.ts +2 -2
- package/dist/actors/computer-use/loop/impressions.js +6 -1
- package/dist/actors/computer-use/loop.d.ts +0 -1
- package/dist/actors/computer-use/loop.js +3 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +1 -1
- package/dist/actors/computer-use/openai-conversation.js +3 -29
- package/dist/actors/computer-use/openai-provider.js +11 -5
- package/dist/actors/contract.d.ts +2 -20
- package/dist/actors/contract.js +0 -13
- package/dist/analysis/admission.d.ts +14 -4
- package/dist/analysis/admission.js +14 -9
- package/dist/analysis/execute.d.ts +5 -0
- package/dist/analysis/execute.js +3 -0
- package/dist/analysis/finding-lead.d.ts +27 -0
- package/dist/analysis/finding-lead.js +23 -0
- package/dist/analysis/service.js +5 -3
- package/dist/cli/commands/notes.js +4 -3
- package/dist/cli/findings.js +10 -32
- package/dist/cli/io.js +4 -7
- package/dist/cli/telemetry.js +10 -22
- package/dist/cli/update-check.js +6 -18
- package/dist/cli/user-config.d.ts +11 -0
- package/dist/cli/user-config.js +25 -0
- package/dist/comms/email-catch.js +7 -31
- package/dist/evidence/encoded-text.d.ts +12 -11
- package/dist/evidence/encoded-text.js +81 -70
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +21 -10
- package/dist/evidence/secret-scrub.d.ts +14 -1
- package/dist/evidence/secret-scrub.js +108 -32
- package/dist/feedback/draft.d.ts +1 -1
- package/dist/feedback/draft.js +13 -16
- package/dist/feedback/export-bundle.js +3 -3
- package/dist/feedback/export.js +32 -46
- package/dist/feedback/feedback.js +6 -19
- package/dist/keys/key-resolution.js +12 -17
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/http.d.ts +7 -0
- package/dist/observer/http.js +19 -1
- package/dist/observer/notes-route.d.ts +0 -1
- package/dist/observer/notes-route.js +24 -44
- package/dist/observer/pinned-files.js +4 -19
- package/dist/observer/render.js +3 -2
- package/dist/observer/run-routes.js +13 -28
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +3 -3
- package/dist/run/contained-output.d.ts +6 -5
- package/dist/run/contained-output.js +19 -26
- package/dist/run/evidence-files.d.ts +7 -0
- package/dist/run/evidence-files.js +24 -14
- package/dist/run/note-files.d.ts +26 -45
- package/dist/run/note-files.js +86 -58
- package/dist/run/note-shape.d.ts +35 -0
- package/dist/run/note-shape.js +20 -0
- package/dist/run/notes.d.ts +3 -6
- package/dist/run/notes.js +18 -24
- package/dist/run/participant-records.d.ts +2 -0
- package/dist/run/participant-records.js +17 -2
- package/dist/run/transient-comms-secrets.d.ts +6 -5
- package/dist/run/transient-comms-secrets.js +19 -24
- package/dist/study/summary.d.ts +4 -5
- package/dist/study/summary.js +4 -2
- package/dist/study/warnings.js +2 -1
- package/dist/tui-app.js +1 -1
- package/docs/architecture/actor-contract.md +3 -3
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/state-driven-executor.md +1 -1
- package/docs/contracts/run-bundle.md +7 -3
- package/docs/contracts/schemas.md +12 -12
- package/docs/contracts/study-analysis.md +12 -4
- package/package.json +7 -6
- package/skills/humanish/SKILL.md +1 -2
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { AffordanceUse } from "./affordance.js";
|
|
2
2
|
import type { ActorEstimatedCost } from "../run/pricing.js";
|
|
3
3
|
import type { TaskFunnel } from "../study/tasks.js";
|
|
4
|
+
import type { ParticipantClosingReport, ParticipantImpression } from "./closing-report.js";
|
|
4
5
|
import { type CuaProviderFailurePhase } from "./computer-use/provider-error.js";
|
|
5
6
|
export declare const ACTOR_TRACE_SCHEMA = "humanish.actor-trace.v1";
|
|
7
|
+
export type { ParticipantClosingReport, ParticipantImpression } from "./closing-report.js";
|
|
6
8
|
/**
|
|
7
9
|
* How a session ended, from the point of view of the study rather than the harness.
|
|
8
10
|
*
|
|
@@ -182,25 +184,6 @@ export interface ActorProviderRequest extends ProviderRequestReceipt {
|
|
|
182
184
|
export declare function validActorExecutionProfile(value: unknown): value is ActorExecutionProfile;
|
|
183
185
|
/** Closed per-attempt evidence; dollar amounts are never part of account usage. */
|
|
184
186
|
export declare function validActorProviderRequests(value: unknown): value is ActorProviderRequest[];
|
|
185
|
-
/** The participant's account, not an independently confirmed product diagnosis. */
|
|
186
|
-
export interface ParticipantClosingReport {
|
|
187
|
-
summary: string;
|
|
188
|
-
frictionReports: string[];
|
|
189
|
-
/** Absent when the provider did not ask for impressions; empty when the participant had none. */
|
|
190
|
-
impressions?: ParticipantImpression[];
|
|
191
|
-
}
|
|
192
|
-
/**
|
|
193
|
-
* What a participant can say about the product at the end of a session, beyond friction.
|
|
194
|
-
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
195
|
-
* life.
|
|
196
|
-
*/
|
|
197
|
-
export declare const PARTICIPANT_IMPRESSION_KINDS: readonly ["unclear", "unfinished", "untrustworthy", "liked", "missing", "unlike_my_work"];
|
|
198
|
-
type ParticipantImpressionKind = (typeof PARTICIPANT_IMPRESSION_KINDS)[number];
|
|
199
|
-
/** One first-person impression about something the participant saw in the session. */
|
|
200
|
-
export interface ParticipantImpression {
|
|
201
|
-
kind: ParticipantImpressionKind;
|
|
202
|
-
text: string;
|
|
203
|
-
}
|
|
204
187
|
/** An impression as the trace keeps it, with the message that quotes it. */
|
|
205
188
|
interface RecordedImpression extends ParticipantImpression {
|
|
206
189
|
/** The trace message carrying the impression, which the analysis quotes and cites. */
|
|
@@ -399,4 +382,3 @@ export interface ActorSessionResult {
|
|
|
399
382
|
export declare const CODEX_APP_SERVER_CAPABILITIES: ActorCapabilities;
|
|
400
383
|
export declare const SCRIPTED_BROWSER_CAPABILITIES: ActorCapabilities;
|
|
401
384
|
export declare const TERMINAL_AGENT_CAPABILITIES: ActorCapabilities;
|
|
402
|
-
export {};
|
package/dist/actors/contract.js
CHANGED
|
@@ -168,19 +168,6 @@ export function validActorProviderRequests(value) {
|
|
|
168
168
|
usage.input));
|
|
169
169
|
});
|
|
170
170
|
}
|
|
171
|
-
/**
|
|
172
|
-
* What a participant can say about the product at the end of a session, beyond friction.
|
|
173
|
-
* `unlike_my_work` compares the screen with how the persona does the same task in its own work or
|
|
174
|
-
* life.
|
|
175
|
-
*/
|
|
176
|
-
export const PARTICIPANT_IMPRESSION_KINDS = [
|
|
177
|
-
"unclear",
|
|
178
|
-
"unfinished",
|
|
179
|
-
"untrustworthy",
|
|
180
|
-
"liked",
|
|
181
|
-
"missing",
|
|
182
|
-
"unlike_my_work",
|
|
183
|
-
];
|
|
184
171
|
export const CODEX_APP_SERVER_CAPABILITIES = {
|
|
185
172
|
headless: true,
|
|
186
173
|
structuredTrace: true,
|
|
@@ -24,10 +24,20 @@ export interface RefusedAnalysisCost {
|
|
|
24
24
|
worstCaseCostUsd: number;
|
|
25
25
|
maxCostUsd: number;
|
|
26
26
|
}
|
|
27
|
-
/**
|
|
28
|
-
export
|
|
29
|
-
/**
|
|
30
|
-
|
|
27
|
+
/** A cost refusal in plain words, and the command that runs the analysis under a cap that admits it. */
|
|
28
|
+
export interface CostRefusal {
|
|
29
|
+
/** The costs, the cap and that no request was sent, ending in the lead-in to `command`. */
|
|
30
|
+
text: string;
|
|
31
|
+
/** `analyze` with `--max-cost` at the worst case rounded up to a whole dollar, at least 1. */
|
|
32
|
+
command: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
36
|
+
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
37
|
+
* src/cli/invocation.ts, and the Observer, which cannot read the install, a bare `humanish`.
|
|
38
|
+
* This module keeps only type imports, so the Observer can bundle it.
|
|
39
|
+
*/
|
|
40
|
+
export declare function costRefusal(cost: RefusedAnalysisCost, runFlags: string, cli: (rest: string) => string): CostRefusal;
|
|
31
41
|
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
32
42
|
export declare function refusedCost(admission: {
|
|
33
43
|
error: string | null;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// The admission cost model for one study analysis request: the expected cost, the worst case,
|
|
2
|
-
//
|
|
1
|
+
// The admission cost model for one study analysis request: the expected cost, the worst case, the
|
|
2
|
+
// figure admission compares with the cap, and the words and command a cost refusal gives.
|
|
3
3
|
/**
|
|
4
4
|
* UTF-8 bytes per input token for the instructions, evidence packet and schema. On 148 billed
|
|
5
5
|
* gpt-6-astra analyses the evidence packet ran 3.1 to 3.3 bytes per token and the instructions
|
|
@@ -40,13 +40,18 @@ export function estimateAnalysisCost(rate, size) {
|
|
|
40
40
|
admittedCostUsd: Math.min(worstCaseCostUsd, roundUp(expectedCostUsd * ADMISSION_MARGIN)),
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
43
|
+
/**
|
|
44
|
+
* Words a cost refusal. `runFlags` are the `analyze` flags that select the run, such as
|
|
45
|
+
* `--run <id>`. `cli` prefixes a humanish command: the CLI passes `cli` from
|
|
46
|
+
* src/cli/invocation.ts, and the Observer, which cannot read the install, a bare `humanish`.
|
|
47
|
+
* This module keeps only type imports, so the Observer can bundle it.
|
|
48
|
+
*/
|
|
49
|
+
export function costRefusal(cost, runFlags, cli) {
|
|
50
|
+
const margin = Math.round((ADMISSION_MARGIN - 1) * 100);
|
|
51
|
+
return {
|
|
52
|
+
text: `The expected cost is $${cost.expectedCostUsd.toFixed(2)} and the worst case is $${cost.worstCaseCostUsd.toFixed(2)}. With a ${margin}% margin the expected cost is over the $${cost.maxCostUsd} cap, so no request was sent. To run it, raise the cap:`,
|
|
53
|
+
command: cli(`analyze ${runFlags} --max-cost ${Math.max(1, Math.ceil(cost.worstCaseCostUsd))}`),
|
|
54
|
+
};
|
|
50
55
|
}
|
|
51
56
|
/** The admission's refused cost, when it refused the analysis for its cost. */
|
|
52
57
|
export function refusedCost(admission) {
|
|
@@ -16,6 +16,11 @@ export interface AnalysisAdmission {
|
|
|
16
16
|
estimatedCostUsd: number | null;
|
|
17
17
|
/** The cost if the analyst spends its whole output allowance. */
|
|
18
18
|
worstCaseCostUsd: number | null;
|
|
19
|
+
/**
|
|
20
|
+
* The expected cost plus the margin, or the worst case when that is lower: the figure admission
|
|
21
|
+
* compares with maxCostUsd, so the smallest cap that admits this analysis.
|
|
22
|
+
*/
|
|
23
|
+
admittedCostUsd: number | null;
|
|
19
24
|
maxCostUsd: number | null;
|
|
20
25
|
ratesAsOf: string | null;
|
|
21
26
|
}
|
package/dist/analysis/execute.js
CHANGED
|
@@ -145,6 +145,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
145
145
|
outputTokenAllowance: 0,
|
|
146
146
|
estimatedCostUsd: null,
|
|
147
147
|
worstCaseCostUsd: null,
|
|
148
|
+
admittedCostUsd: null,
|
|
148
149
|
maxCostUsd: null,
|
|
149
150
|
ratesAsOf: null,
|
|
150
151
|
});
|
|
@@ -178,6 +179,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
178
179
|
outputTokenAllowance: null,
|
|
179
180
|
estimatedCostUsd: null,
|
|
180
181
|
worstCaseCostUsd: null,
|
|
182
|
+
admittedCostUsd: null,
|
|
181
183
|
maxCostUsd: null,
|
|
182
184
|
ratesAsOf: null,
|
|
183
185
|
};
|
|
@@ -207,6 +209,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
207
209
|
outputTokenAllowance: config.maxOutputTokens,
|
|
208
210
|
estimatedCostUsd: cost.expectedCostUsd,
|
|
209
211
|
worstCaseCostUsd: cost.worstCaseCostUsd,
|
|
212
|
+
admittedCostUsd: cost.admittedCostUsd,
|
|
210
213
|
maxCostUsd: config.maxCostUsd,
|
|
211
214
|
ratesAsOf: rate.asOf,
|
|
212
215
|
};
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { AnalysisCorrection, AnalysisResult } from "./types.js";
|
|
2
|
+
type Severity = NonNullable<AnalysisResult["designFindings"]>[number]["severity"];
|
|
3
|
+
export interface FindingLead {
|
|
4
|
+
/** The reviewer's amended claim, or the plain headline. */
|
|
5
|
+
headline: string;
|
|
6
|
+
/**
|
|
7
|
+
* The line under the headline: the plain experience, or after an amendment a sentence saying the
|
|
8
|
+
* reviewer's claim replaced the headline and account. Null when the analysis wrote no experience.
|
|
9
|
+
*/
|
|
10
|
+
account: string | null;
|
|
11
|
+
/** The headline is a reviewer's amended claim. */
|
|
12
|
+
corrected: boolean;
|
|
13
|
+
}
|
|
14
|
+
/**
|
|
15
|
+
* The headline and account a finding leads with, given its latest human review note. An amendment
|
|
16
|
+
* replaces both, since they describe the claim the reviewer replaced. Null for an analysis written
|
|
17
|
+
* before headlines, which leads with the finding's title.
|
|
18
|
+
*/
|
|
19
|
+
export declare function findingLead(finding: {
|
|
20
|
+
headline?: string | null;
|
|
21
|
+
experience?: string | null;
|
|
22
|
+
}, correction: Pick<AnalysisCorrection, "status" | "replacementClaim"> | null | undefined): FindingLead | null;
|
|
23
|
+
/** Design findings most severe first, in a new array; equal severity keeps the analysis order. */
|
|
24
|
+
export declare function bySeverity<T extends {
|
|
25
|
+
severity: Severity;
|
|
26
|
+
}>(findings: readonly T[]): T[];
|
|
27
|
+
export {};
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// How a finding opens for a reader, the same in `review`, a feedback draft and the Observer. The
|
|
2
|
+
// Observer bundles this module, so it imports types only (observer/tests/contract-lock.test.ts).
|
|
3
|
+
/**
|
|
4
|
+
* The headline and account a finding leads with, given its latest human review note. An amendment
|
|
5
|
+
* replaces both, since they describe the claim the reviewer replaced. Null for an analysis written
|
|
6
|
+
* before headlines, which leads with the finding's title.
|
|
7
|
+
*/
|
|
8
|
+
export function findingLead(finding, correction) {
|
|
9
|
+
if (finding.headline === undefined || finding.headline === null)
|
|
10
|
+
return null;
|
|
11
|
+
if (correction?.status === "amended" && correction.replacementClaim)
|
|
12
|
+
return {
|
|
13
|
+
headline: correction.replacementClaim,
|
|
14
|
+
account: "Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
15
|
+
corrected: true,
|
|
16
|
+
};
|
|
17
|
+
return { headline: finding.headline, account: finding.experience ?? null, corrected: false };
|
|
18
|
+
}
|
|
19
|
+
const SEVERITY_ORDER = ["major", "moderate", "minor"];
|
|
20
|
+
/** Design findings most severe first, in a new array; equal severity keeps the analysis order. */
|
|
21
|
+
export function bySeverity(findings) {
|
|
22
|
+
return findings.toSorted((a, b) => SEVERITY_ORDER.indexOf(a.severity) - SEVERITY_ORDER.indexOf(b.severity));
|
|
23
|
+
}
|
package/dist/analysis/service.js
CHANGED
|
@@ -17,7 +17,8 @@ import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalys
|
|
|
17
17
|
import { beginAnalysisExecution, writeAnalysisExecutionReceipt } from "./store-executions.js";
|
|
18
18
|
import { keepRejectedAnalysisOutput } from "./diagnostics.js";
|
|
19
19
|
import { loadAnalysis } from "./load.js";
|
|
20
|
-
import {
|
|
20
|
+
import { costRefusal, refusedCost } from "./admission.js";
|
|
21
|
+
import { cli } from "../cli/invocation.js";
|
|
21
22
|
import { hashAnalysisValue } from "./validation.js";
|
|
22
23
|
import { ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
23
24
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
@@ -191,13 +192,14 @@ function analyzeConfigRefusal(config) {
|
|
|
191
192
|
function admissionOnlyResult(base, config) {
|
|
192
193
|
if (!base.admission.allowed) {
|
|
193
194
|
const cost = refusedCost(base.admission);
|
|
195
|
+
const refusal = cost && costRefusal(cost, `--run ${base.run}`, cli);
|
|
194
196
|
return {
|
|
195
197
|
...base,
|
|
196
198
|
ok: false,
|
|
197
199
|
error: {
|
|
198
200
|
code: base.admission.error ?? "analysis_admission_denied",
|
|
199
|
-
message:
|
|
200
|
-
? `${
|
|
201
|
+
message: refusal
|
|
202
|
+
? `${refusal.text} ${refusal.command}`
|
|
201
203
|
: "The analysis input or configuration did not pass admission. No provider request was sent.",
|
|
202
204
|
},
|
|
203
205
|
};
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { renderObserver } from "../../observer/render.js";
|
|
2
2
|
import { loadRunBundlePrepared, resolveRunPath } from "../../run/locate.js";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { readRunNotes } from "../../run/note-files.js";
|
|
4
|
+
import { byRunTime } from "../../run/note-shape.js";
|
|
5
|
+
import { addRunNote, formatRunTime } from "../../run/notes.js";
|
|
6
|
+
import { runParticipantCaptions, streamParticipantIdOf } from "../../run/participant-records.js";
|
|
5
7
|
import { resolvePhysicalCwd, runIdOf } from "../../run/paths.js";
|
|
6
8
|
import { runNotFoundMessage } from "../../run/run-not-found.js";
|
|
7
9
|
import { cli } from "../invocation.js";
|
|
@@ -42,7 +44,6 @@ function parseRunTime(value) {
|
|
|
42
44
|
const match = /^(\d{1,4}):([0-5]\d)$/.exec(value.trim());
|
|
43
45
|
return match ? (Number(match[1]) * 60 + Number(match[2])) * 1000 : null;
|
|
44
46
|
}
|
|
45
|
-
const byRunTime = (notes) => [...notes].sort((left, right) => left.atMs - right.atMs || left.createdAt.localeCompare(right.createdAt));
|
|
46
47
|
async function handleNotes(io, run, text, options, command) {
|
|
47
48
|
const base = {
|
|
48
49
|
schema: NOTES_RESULT_SCHEMA,
|
package/dist/cli/findings.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// `humanish.analysis-findings.v1` projection and its text. The analysis artifact stays the record;
|
|
3
3
|
// this reads the version `analyze show` selects and never starts an analysis.
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import {
|
|
5
|
+
import { costRefusal } from "../analysis/admission.js";
|
|
6
6
|
import { DEFAULT_ANALYSIS_MAX_COST_USD } from "../analysis/automatic-config.js";
|
|
7
|
+
import { bySeverity, findingLead } from "../analysis/finding-lead.js";
|
|
7
8
|
import { loadAnalysis } from "../analysis/load.js";
|
|
8
9
|
import { projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
9
10
|
import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
@@ -64,20 +65,13 @@ function evidenceOf(finding, analysis, runRoot) {
|
|
|
64
65
|
...entry,
|
|
65
66
|
}));
|
|
66
67
|
}
|
|
67
|
-
const SEVERITY_RANK = {
|
|
68
|
-
major: 0,
|
|
69
|
-
moderate: 1,
|
|
70
|
-
minor: 2,
|
|
71
|
-
};
|
|
72
68
|
/** Most severe first; equal severity keeps the analysis order. */
|
|
73
69
|
function designFindingViews(analysis, runRoot) {
|
|
74
70
|
const designFindings = analysis.result?.designFindings;
|
|
75
71
|
if (designFindings === undefined)
|
|
76
72
|
return null;
|
|
77
73
|
const labels = new Map(analysis.participants.map((p) => [p.streamId, p.label]));
|
|
78
|
-
return
|
|
79
|
-
.sort((a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity])
|
|
80
|
-
.map((finding) => ({
|
|
74
|
+
return bySeverity(designFindings).map((finding) => ({
|
|
81
75
|
id: finding.id,
|
|
82
76
|
headline: finding.headline,
|
|
83
77
|
screen: finding.screen,
|
|
@@ -200,8 +194,10 @@ export function analysisFindings(source) {
|
|
|
200
194
|
const outcome = analysisOutcomeText(job);
|
|
201
195
|
return withoutFindings(source, "failed", job.reason, `The automatic analysis ${outcome.startsWith("failed") ? "" : "was "}${outcome}.`, analyzeCommand(source, job.reason === "AUTOMATIC_ANALYSIS_CODEX_UNAVAILABLE" ? "codex" : undefined));
|
|
202
196
|
}
|
|
203
|
-
if (job?.state === "skipped" && job.admission)
|
|
204
|
-
|
|
197
|
+
if (job?.state === "skipped" && job.admission) {
|
|
198
|
+
const refusal = costRefusal(job.admission, `--run ${source.runId}${source.cwdFlag}`, cli);
|
|
199
|
+
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was refused before it started. ${refusal.text}`, refusal.command);
|
|
200
|
+
}
|
|
205
201
|
if (job?.state === "skipped") {
|
|
206
202
|
const refused = job.reason === "AUTOMATIC_ANALYSIS_ADMISSION_REFUSED";
|
|
207
203
|
return withoutFindings(source, "skipped", job.reason, `The automatic analysis was ${analysisOutcomeText(job)}.${refused ? " The dry run says why, and for a cost refusal names the --max-cost that admits it." : ""}`, job.reason === "AUTOMATIC_ANALYSIS_NO_PARTICIPANT_EVIDENCE"
|
|
@@ -342,18 +338,6 @@ function captureFiles(evidence, runRoot) {
|
|
|
342
338
|
.map((entry) => (runRoot ? path.relative(runRoot, entry.capture) : entry.capture))),
|
|
343
339
|
];
|
|
344
340
|
}
|
|
345
|
-
/**
|
|
346
|
-
* What a reader sees first: the headline, or the reviewer's claim when a review amended the finding,
|
|
347
|
-
* since the headline and experience describe the claim it replaced. Null before headlines existed.
|
|
348
|
-
*/
|
|
349
|
-
function leadOf(finding) {
|
|
350
|
-
const amended = finding.correction?.status === "amended" ? finding.correction : null;
|
|
351
|
-
if (finding.headline === null)
|
|
352
|
-
return null;
|
|
353
|
-
return amended?.replacementClaim
|
|
354
|
-
? { headline: amended.replacementClaim, corrected: true }
|
|
355
|
-
: { headline: finding.headline, corrected: false };
|
|
356
|
-
}
|
|
357
341
|
function findingLines(finding, runRoot) {
|
|
358
342
|
const cited = [...new Set(finding.evidence.map((entry) => entry.streamId))];
|
|
359
343
|
const labels = new Map(finding.affected.map((p) => [p.streamId, p.label]));
|
|
@@ -361,19 +345,13 @@ function findingLines(finding, runRoot) {
|
|
|
361
345
|
const streams = [...affectedIds, ...cited.filter((id) => !labels.has(id))];
|
|
362
346
|
const captures = captureFiles(finding.evidence, runRoot);
|
|
363
347
|
const correction = finding.correction;
|
|
364
|
-
const lead =
|
|
348
|
+
const lead = findingLead(finding, correction);
|
|
365
349
|
return [
|
|
366
350
|
...(lead === null
|
|
367
351
|
? [`${finding.id} ${finding.title}`]
|
|
368
352
|
: [
|
|
369
353
|
`${finding.id} ${lead.headline}`,
|
|
370
|
-
...(lead.
|
|
371
|
-
? [
|
|
372
|
-
" Corrected in human review. The reviewer's claim replaces the original headline and account.",
|
|
373
|
-
]
|
|
374
|
-
: finding.experience === null
|
|
375
|
-
? []
|
|
376
|
-
: [` ${finding.experience}`]),
|
|
354
|
+
...(lead.account === null ? [] : [` ${lead.account}`]),
|
|
377
355
|
` evidence: ${finding.title}`,
|
|
378
356
|
]),
|
|
379
357
|
` impact: ${IMPACT_TEXT[finding.impact]} · confidence: ${finding.confidence} · recovery: ${RECOVERY_TEXT[finding.recovery]}`,
|
|
@@ -451,7 +429,7 @@ export function formatFindingsSummary(view, fullCommand, limit = 3) {
|
|
|
451
429
|
`findings: ${view.findings.length === 0 ? "none" : view.findings.length}${view.state === "stale" ? " (stale)" : ""}`,
|
|
452
430
|
...shown.map((finding) => {
|
|
453
431
|
const qualities = `${IMPACT_TEXT[finding.impact]}, ${finding.confidence} confidence, ${RECOVERY_TEXT[finding.recovery]}`;
|
|
454
|
-
const lead =
|
|
432
|
+
const lead = findingLead(finding, finding.correction);
|
|
455
433
|
if (lead === null)
|
|
456
434
|
return `- ${finding.id} ${qualities}: ${finding.title}`;
|
|
457
435
|
return `- ${finding.id} ${lead.headline} (${lead.corrected ? "corrected in human review" : qualities})`;
|
package/dist/cli/io.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js";
|
|
2
|
-
import {
|
|
2
|
+
import { costRefusal, refusedCost } from "../analysis/admission.js";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
5
5
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
@@ -189,8 +189,9 @@ export function writeResult(command, io, result, formatHuman) {
|
|
|
189
189
|
if (output !== null && typeof output === "object" && "automaticAnalysis" in output) {
|
|
190
190
|
const analysis = output.automaticAnalysis;
|
|
191
191
|
const cost = analysis?.result?.admission && refusedCost(analysis.result.admission);
|
|
192
|
-
|
|
193
|
-
|
|
192
|
+
const refusal = cost && costRefusal(cost, `--run ${analysis.result?.run ?? "latest"}`, cli);
|
|
193
|
+
if (refusal)
|
|
194
|
+
io.writeOut(`analysis: refused before it started. ${refusal.text} ${refusal.command}\n`);
|
|
194
195
|
else if (analysis)
|
|
195
196
|
io.writeOut(`analysis: ${analysisOutcomeText(analysis)}\n`);
|
|
196
197
|
const rejected = analysis?.result?.rejectedOutputPath;
|
|
@@ -306,10 +307,6 @@ export function analysisOutcomeText(analysis) {
|
|
|
306
307
|
return analysis.state;
|
|
307
308
|
return ANALYSIS_REASON_TEXT[analysis.reason] ?? `${analysis.state} (${analysis.reason})`;
|
|
308
309
|
}
|
|
309
|
-
/** What a refused analysis would cost and the command that runs it with a cap that admits it. */
|
|
310
|
-
function refusedAnalysisText(cost, runId) {
|
|
311
|
-
return `${costRefusalText(cost)} To run it, raise the cap: ${cli(`analyze --run ${runId} --max-cost ${admittingMaxCost(cost)}`)}`;
|
|
312
|
-
}
|
|
313
310
|
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
314
311
|
export function automaticAnalysisEnvelope(result) {
|
|
315
312
|
if (result === null ||
|
package/dist/cli/telemetry.js
CHANGED
|
@@ -12,9 +12,9 @@
|
|
|
12
12
|
import { isCuaDiagnosticCategory, isCuaDiagnosticStopCause, } from "../routes/computer-use/diagnostics.js";
|
|
13
13
|
import { randomUUID } from "node:crypto";
|
|
14
14
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
15
|
-
import { homedir } from "node:os";
|
|
16
15
|
import path from "node:path";
|
|
17
16
|
import { STUDY_RESULT_SCHEMA } from "../run/study-result.js";
|
|
17
|
+
import { envFlag, humanishConfigFile } from "./user-config.js";
|
|
18
18
|
/** Write-only PostHog project key. Public by design: it can ingest, and can read nothing. */
|
|
19
19
|
const INGEST_KEY = "phc_oeMeBqxDZhZ9tCHMSnuDFimLqHpU5Myc847WD33hAh4C";
|
|
20
20
|
const INGEST_HOST = "https://us.i.posthog.com";
|
|
@@ -26,26 +26,16 @@ const STARTER_STUDIES = new Set([
|
|
|
26
26
|
"local-browser",
|
|
27
27
|
"lobby-trivia-3player",
|
|
28
28
|
]);
|
|
29
|
-
export function telemetryStatePath(env = process.env, home
|
|
30
|
-
|
|
31
|
-
// Same XDG rule the key store uses: a relative value must be ignored, or state becomes
|
|
32
|
-
// cwd-relative and follows people between projects.
|
|
33
|
-
const configHome = declared !== undefined && declared !== "" && path.isAbsolute(declared)
|
|
34
|
-
? declared
|
|
35
|
-
: path.join(home, ".config");
|
|
36
|
-
return path.join(configHome, "humanish", "telemetry.json");
|
|
29
|
+
export function telemetryStatePath(env = process.env, home) {
|
|
30
|
+
return humanishConfigFile(env, "telemetry.json", home);
|
|
37
31
|
}
|
|
38
32
|
/**
|
|
39
33
|
* Off means off, from any of the three switches people already expect: the cross-tool
|
|
40
34
|
* `DO_NOT_TRACK` standard, our own env var, and the persisted opt-out.
|
|
41
35
|
*/
|
|
42
36
|
export function disabledByEnvironment(env) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
value.trim() !== "0" &&
|
|
46
|
-
value.trim().toLowerCase() !== "false";
|
|
47
|
-
return (truthy(env.DO_NOT_TRACK) ||
|
|
48
|
-
truthy(env.HUMANISH_TELEMETRY_DISABLED) ||
|
|
37
|
+
return (envFlag(env.DO_NOT_TRACK) ||
|
|
38
|
+
envFlag(env.HUMANISH_TELEMETRY_DISABLED) ||
|
|
49
39
|
// Our own development and test runs must never reach the adoption dataset. In the first two
|
|
50
40
|
// days after telemetry shipped, 82% of events (4,042 of 4,932, from 49 of 59 anonymous ids)
|
|
51
41
|
// came from humanish's own CI and suite: ~50 ids each running nearly every subcommand about
|
|
@@ -55,7 +45,7 @@ export function disabledByEnvironment(env) {
|
|
|
55
45
|
// Deliberately not keyed on CI. An adopter running humanish in their pipeline is real usage
|
|
56
46
|
// and stays countable; the `ci` property already separates it, which is what Next.js does.
|
|
57
47
|
// This keys on being inside the humanish source tree, which only we ever are.
|
|
58
|
-
|
|
48
|
+
envFlag(env.HUMANISH_DEV));
|
|
59
49
|
}
|
|
60
50
|
/**
|
|
61
51
|
* True when this process is running from a humanish source checkout rather than an install.
|
|
@@ -101,7 +91,7 @@ export function inHumanishCheckout(startDir, readFileSyncFn) {
|
|
|
101
91
|
export function isOwnCheckoutRun(cwd, cliDir, readFileSyncFn) {
|
|
102
92
|
return inHumanishCheckout(cwd, readFileSyncFn) || inHumanishCheckout(cliDir, readFileSyncFn);
|
|
103
93
|
}
|
|
104
|
-
export async function readTelemetryState(env = process.env, home
|
|
94
|
+
export async function readTelemetryState(env = process.env, home) {
|
|
105
95
|
try {
|
|
106
96
|
const raw = await readFile(telemetryStatePath(env, home), "utf8");
|
|
107
97
|
const parsed = JSON.parse(raw);
|
|
@@ -115,7 +105,7 @@ export async function readTelemetryState(env = process.env, home = homedir()) {
|
|
|
115
105
|
return { enabled: true, anonymousId: randomUUID(), noticed: false };
|
|
116
106
|
}
|
|
117
107
|
}
|
|
118
|
-
export async function writeTelemetryState(state, env = process.env, home
|
|
108
|
+
export async function writeTelemetryState(state, env = process.env, home) {
|
|
119
109
|
const file = telemetryStatePath(env, home);
|
|
120
110
|
await mkdir(path.dirname(file), { recursive: true });
|
|
121
111
|
await writeFile(file, `${JSON.stringify(state, null, 2)}\n`, "utf8");
|
|
@@ -162,15 +152,13 @@ export function buildPayload(args) {
|
|
|
162
152
|
version: args.version,
|
|
163
153
|
os: args.platform ?? process.platform,
|
|
164
154
|
node: (args.nodeVersion ?? process.version).split(".")[0].replace("v", ""),
|
|
165
|
-
ci: env.CI
|
|
155
|
+
ci: envFlag(env.CI),
|
|
166
156
|
// A humanish study participant is a real cold install on a machine we do not own, so it emits
|
|
167
157
|
// like any other new adopter and is indistinguishable from one. It is also the exact
|
|
168
158
|
// population we are trying to count, so a busy self-study day reads as an adoption spike.
|
|
169
159
|
// Stamped rather than suppressed, the same shape as `ci`: participant runs stay visible and
|
|
170
160
|
// stay separable, and we keep a genuine measurement of what a cold install does.
|
|
171
|
-
studyParticipant: env.HUMANISH_STUDY_PARTICIPANT
|
|
172
|
-
env.HUMANISH_STUDY_PARTICIPANT !== "" &&
|
|
173
|
-
env.HUMANISH_STUDY_PARTICIPANT !== "0",
|
|
161
|
+
studyParticipant: envFlag(env.HUMANISH_STUDY_PARTICIPANT),
|
|
174
162
|
};
|
|
175
163
|
const given = args.properties ?? {};
|
|
176
164
|
if (given.command !== undefined)
|
package/dist/cli/update-check.js
CHANGED
|
@@ -4,20 +4,12 @@
|
|
|
4
4
|
// the command that started it.
|
|
5
5
|
import { spawn } from "node:child_process";
|
|
6
6
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
7
|
-
import { homedir } from "node:os";
|
|
8
7
|
import path from "node:path";
|
|
9
8
|
import { fileURLToPath } from "node:url";
|
|
10
9
|
import { disabledByEnvironment } from "./telemetry.js";
|
|
10
|
+
import { envFlag, humanishConfigFile } from "./user-config.js";
|
|
11
11
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
12
12
|
const REGISTRY_DIST_TAGS = "https://registry.npmjs.org/-/package/humanish/dist-tags";
|
|
13
|
-
/** Next to telemetry.json, under the same XDG rule the key store uses. */
|
|
14
|
-
function updateCachePath(env, home = homedir()) {
|
|
15
|
-
const declared = env.XDG_CONFIG_HOME?.trim();
|
|
16
|
-
const configHome = declared !== undefined && declared !== "" && path.isAbsolute(declared)
|
|
17
|
-
? declared
|
|
18
|
-
: path.join(home, ".config");
|
|
19
|
-
return path.join(configHome, "humanish", "update-check.json");
|
|
20
|
-
}
|
|
21
13
|
function readUpdateCache(cachePath) {
|
|
22
14
|
try {
|
|
23
15
|
const parsed = JSON.parse(readFileSync(cachePath, "utf8"));
|
|
@@ -65,10 +57,6 @@ function dayPassed(timestamp, now) {
|
|
|
65
57
|
const then = timestamp === undefined ? Number.NaN : Date.parse(timestamp);
|
|
66
58
|
return Number.isNaN(then) || now - then >= DAY_MS;
|
|
67
59
|
}
|
|
68
|
-
const truthy = (value) => value !== undefined &&
|
|
69
|
-
value.trim() !== "" &&
|
|
70
|
-
value.trim() !== "0" &&
|
|
71
|
-
value.trim().toLowerCase() !== "false";
|
|
72
60
|
/**
|
|
73
61
|
* Only a person at a terminal is told, and only when nothing has switched the check off. A pipe,
|
|
74
62
|
* CI and a JSON document have a program reading them. A study participant runs the version the
|
|
@@ -80,9 +68,9 @@ function checkAllowed(context) {
|
|
|
80
68
|
return (context.terminal &&
|
|
81
69
|
!context.json &&
|
|
82
70
|
!context.ownCheckout &&
|
|
83
|
-
!
|
|
84
|
-
!
|
|
85
|
-
!
|
|
71
|
+
!envFlag(env.CI) &&
|
|
72
|
+
!envFlag(env.HUMANISH_NO_UPDATE_CHECK) &&
|
|
73
|
+
!envFlag(env.HUMANISH_STUDY_PARTICIPANT) &&
|
|
86
74
|
!disabledByEnvironment(env));
|
|
87
75
|
}
|
|
88
76
|
/**
|
|
@@ -94,7 +82,7 @@ export function checkForUpdate(context) {
|
|
|
94
82
|
try {
|
|
95
83
|
if (!checkAllowed(context))
|
|
96
84
|
return undefined;
|
|
97
|
-
const cachePath =
|
|
85
|
+
const cachePath = humanishConfigFile(context.env, "update-check.json", context.home);
|
|
98
86
|
const cache = readUpdateCache(cachePath);
|
|
99
87
|
const stamp = new Date(context.now).toISOString();
|
|
100
88
|
const next = { ...cache };
|
|
@@ -126,7 +114,7 @@ export function checkForUpdate(context) {
|
|
|
126
114
|
* cache only; undefined until a check has succeeded on this machine.
|
|
127
115
|
*/
|
|
128
116
|
export function recordedVersion(installed, env, home) {
|
|
129
|
-
const cache = readUpdateCache(
|
|
117
|
+
const cache = readUpdateCache(humanishConfigFile(env, "update-check.json", home));
|
|
130
118
|
if (cache.latest === undefined)
|
|
131
119
|
return undefined;
|
|
132
120
|
const checked = cache.checkedAt === undefined ? "" : ` (checked ${cache.checkedAt.slice(0, 10)})`;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `$XDG_CONFIG_HOME/humanish/<fileName>`, or `<home>/.config/humanish/<fileName>`. The XDG spec
|
|
3
|
+
* says to ignore a relative XDG_CONFIG_HOME. Honoring one would put the file in whichever project
|
|
4
|
+
* is open, and for the key store that writes a secret into the repo.
|
|
5
|
+
*/
|
|
6
|
+
export declare function humanishConfigFile(env: NodeJS.ProcessEnv, fileName: string, home?: string): string;
|
|
7
|
+
/**
|
|
8
|
+
* Whether a flag variable such as `CI` or `DO_NOT_TRACK` is on. Unset, blank, `0` and `false` (any
|
|
9
|
+
* case, surrounding spaces ignored) are off, as is-in-ci reads `CI`; any other value is on.
|
|
10
|
+
*/
|
|
11
|
+
export declare function envFlag(value: string | undefined): boolean;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// Where humanish keeps per-user files (the key store, telemetry state, the update-check cache) and
|
|
2
|
+
// how it reads an on/off environment variable. The key store, telemetry and the update check each
|
|
3
|
+
// had their own copy of both.
|
|
4
|
+
import { homedir } from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
/**
|
|
7
|
+
* `$XDG_CONFIG_HOME/humanish/<fileName>`, or `<home>/.config/humanish/<fileName>`. The XDG spec
|
|
8
|
+
* says to ignore a relative XDG_CONFIG_HOME. Honoring one would put the file in whichever project
|
|
9
|
+
* is open, and for the key store that writes a secret into the repo.
|
|
10
|
+
*/
|
|
11
|
+
export function humanishConfigFile(env, fileName, home = homedir()) {
|
|
12
|
+
const declared = env.XDG_CONFIG_HOME?.trim();
|
|
13
|
+
const configHome = declared !== undefined && declared !== "" && path.isAbsolute(declared)
|
|
14
|
+
? declared
|
|
15
|
+
: path.join(home, ".config");
|
|
16
|
+
return path.join(configHome, "humanish", fileName);
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Whether a flag variable such as `CI` or `DO_NOT_TRACK` is on. Unset, blank, `0` and `false` (any
|
|
20
|
+
* case, surrounding spaces ignored) are off, as is-in-ci reads `CI`; any other value is on.
|
|
21
|
+
*/
|
|
22
|
+
export function envFlag(value) {
|
|
23
|
+
const trimmed = value?.trim() ?? "";
|
|
24
|
+
return trimmed !== "" && trimmed !== "0" && trimmed.toLowerCase() !== "false";
|
|
25
|
+
}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
//
|
|
10
10
|
// 127.0.0.1 only; request bodies are never logged; request size is capped.
|
|
11
11
|
import { createServer } from "node:http";
|
|
12
|
+
import { readBodyAtMost } from "../observer/http.js";
|
|
12
13
|
import { capturedInlineImages } from "./images.js";
|
|
13
14
|
// ---------------------------------------------------------------- shared parse helpers
|
|
14
15
|
function str(value) {
|
|
@@ -116,26 +117,6 @@ export const DEFAULT_EMAIL_PROFILES = [
|
|
|
116
117
|
sendgridEmailProfile,
|
|
117
118
|
];
|
|
118
119
|
// ---------------------------------------------------------------- server
|
|
119
|
-
class BodyTooLargeError extends Error {
|
|
120
|
-
}
|
|
121
|
-
function readBody(req, limit) {
|
|
122
|
-
return new Promise((resolve, reject) => {
|
|
123
|
-
let size = 0;
|
|
124
|
-
const chunks = [];
|
|
125
|
-
req.on("data", (chunk) => {
|
|
126
|
-
size += chunk.length;
|
|
127
|
-
if (size > limit) {
|
|
128
|
-
// Reject without destroying the socket here: the handler responds 413 cleanly first (a
|
|
129
|
-
// write-after-destroy would otherwise reset the connection), then tears the request down.
|
|
130
|
-
reject(new BodyTooLargeError("request body too large"));
|
|
131
|
-
return;
|
|
132
|
-
}
|
|
133
|
-
chunks.push(chunk);
|
|
134
|
-
});
|
|
135
|
-
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf8")));
|
|
136
|
-
req.on("error", reject);
|
|
137
|
-
});
|
|
138
|
-
}
|
|
139
120
|
const MAX_BODY_BYTES = 5 * 1024 * 1024;
|
|
140
121
|
/**
|
|
141
122
|
* Start the vendor-neutral email catch, routing accepted sends into `channel`. Returns the loopback
|
|
@@ -170,21 +151,16 @@ export async function startEmailCatchServer(channel, options = {}) {
|
|
|
170
151
|
respondJson(res, 404, { error: "not found" });
|
|
171
152
|
return;
|
|
172
153
|
}
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if (!res.headersSent)
|
|
179
|
-
respondJson(res, error instanceof BodyTooLargeError ? 413 : 400, {
|
|
180
|
-
error: "request body could not be read",
|
|
181
|
-
});
|
|
182
|
-
req.destroy();
|
|
154
|
+
const body = await readBodyAtMost(req, MAX_BODY_BYTES).catch(() => undefined);
|
|
155
|
+
if (body === null || body === undefined) {
|
|
156
|
+
// The rest of the body is never read, so the connection closes after this answer.
|
|
157
|
+
res.setHeader("connection", "close");
|
|
158
|
+
respondJson(res, body === null ? 413 : 400, { error: "request body could not be read" });
|
|
183
159
|
return;
|
|
184
160
|
}
|
|
185
161
|
let parsed;
|
|
186
162
|
try {
|
|
187
|
-
parsed = JSON.parse(
|
|
163
|
+
parsed = JSON.parse(body.length > 0 ? body.toString("utf8") : "{}");
|
|
188
164
|
}
|
|
189
165
|
catch {
|
|
190
166
|
respondJson(res, 422, { error: "invalid JSON body" });
|