humanish 0.118.0 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/analysis/admission.d.ts +16 -2
- package/dist/analysis/admission.js +35 -6
- package/dist/analysis/analysis-limits.d.ts +32 -0
- package/dist/analysis/analysis-limits.js +41 -3
- package/dist/analysis/automatic-config.d.ts +5 -0
- package/dist/analysis/automatic-config.js +15 -1
- package/dist/analysis/cohorts.d.ts +103 -0
- package/dist/analysis/cohorts.js +80 -0
- package/dist/analysis/evidence.d.ts +5 -10
- package/dist/analysis/evidence.js +37 -34
- package/dist/analysis/execute.d.ts +7 -2
- package/dist/analysis/execute.js +242 -297
- package/dist/analysis/responses.d.ts +19 -0
- package/dist/analysis/responses.js +201 -0
- package/dist/analysis/store.js +2 -1
- package/dist/analysis/validation.js +13 -7
- package/dist/cli/commands/analyze.js +8 -3
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/io.js +2 -3
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +4 -0
- package/dist/routes/computer-use/participant-execution.d.ts +6 -4
- package/dist/routes/computer-use/participant-execution.js +42 -14
- package/dist/routes/computer-use/participant-model.js +18 -8
- package/dist/routes/computer-use/participant-runs.js +12 -0
- package/dist/routes/computer-use/plan.js +23 -15
- package/dist/routes/computer-use/rerun-selection.js +14 -3
- package/dist/routes/computer-use/types.d.ts +13 -1
- package/dist/routes/shared-world/bundle-records.js +14 -0
- package/dist/routes/shared-world/bundle.js +2 -1
- package/dist/routes/shared-world/external-public.js +15 -3
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/plan.d.ts +7 -0
- package/dist/routes/shared-world/plan.js +79 -34
- package/dist/routes/shared-world/provisioned.js +15 -8
- package/dist/routes/shared-world/route.js +5 -2
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle-shape.js +10 -1
- package/dist/run/bundle.d.ts +15 -0
- package/dist/run/participant-records.js +1 -0
- package/dist/run/route-shell.js +5 -10
- package/dist/run/run.js +4 -2
- package/dist/study/arrivals.d.ts +114 -0
- package/dist/study/arrivals.js +204 -0
- package/dist/study/doctor.js +2 -15
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +3 -2
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +9 -3
- package/dist/study/parse/front.js +29 -1
- package/dist/study/plan-base.d.ts +1 -3
- package/dist/study/plan-base.js +0 -3
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +1 -0
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.js +12 -0
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +5 -3
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.js +5 -0
- package/dist/study/warnings.js +11 -3
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +19 -1
- package/docs/contracts/study-analysis.md +59 -6
- package/docs/principles/invariants-and-defaults.md +14 -14
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +1 -1
- package/dist/study/automatic-analysis-plan.d.ts +0 -28
- package/dist/study/automatic-analysis-plan.js +0 -57
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
2
|
+
import type { AnalysisProviderResult } from "./provider.js";
|
|
3
|
+
import type { AnalysisArtifact, AnalysisConfig, AnalysisInput, AnalysisResult } from "./types.js";
|
|
4
|
+
export type CheckedProviderAnalysis = {
|
|
5
|
+
ok: true;
|
|
6
|
+
result: AnalysisResult;
|
|
7
|
+
} | {
|
|
8
|
+
ok: false;
|
|
9
|
+
error: string;
|
|
10
|
+
rejected?: RejectedAnalysisOutput;
|
|
11
|
+
};
|
|
12
|
+
export declare function checkProviderAnalysis(input: AnalysisInput, value: unknown): CheckedProviderAnalysis;
|
|
13
|
+
/**
|
|
14
|
+
* Copy the reported tokens of every response onto the artifact, summed. OpenAI usage is priced with
|
|
15
|
+
* each response as its own request, since the long-context tier re-prices one request at a time.
|
|
16
|
+
*/
|
|
17
|
+
export declare function recordProviderUsage(artifact: AnalysisArtifact, config: AnalysisConfig, responses: readonly AnalysisProviderResult[]): void;
|
|
18
|
+
/** The warnings a Codex response carries about its CLI release and protocol. */
|
|
19
|
+
export declare function codexWarnings(config: AnalysisConfig, response: AnalysisProviderResult): string[];
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// What the provider responses of one attempt give it: the usage they report, the warnings a Codex
|
|
2
|
+
// response carries, and each response checked before it becomes an analysis result (the response
|
|
3
|
+
// schema, the scrub of known transient values from generated prose, and the evidence rules in
|
|
4
|
+
// validation.ts). Only an allowlisted code leaves a check; the rejected output goes to the caller
|
|
5
|
+
// for local diagnosis.
|
|
6
|
+
import { truncatedFrameWarning, unknownNotificationsWarning, } from "../actors/codex/restricted-notifications.js";
|
|
7
|
+
import { protocolAdditionsWarning, protocolIncompatibilityMessage, } from "../actors/codex/protocol-compat.js";
|
|
8
|
+
import { estimateActorCost } from "../run/pricing.js";
|
|
9
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
10
|
+
import { analysisResponseSchema, checkAnalysisResult } from "./validation.js";
|
|
11
|
+
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
|
|
12
|
+
* known value is found as written and in its encoded forms. */
|
|
13
|
+
function scrubGeneratedNarrative(result) {
|
|
14
|
+
const scrub = transientCommsKnownValueScrub();
|
|
15
|
+
const observation = (value) => ({
|
|
16
|
+
...value,
|
|
17
|
+
claim: scrub(value.claim),
|
|
18
|
+
limitation: scrub(value.limitation),
|
|
19
|
+
});
|
|
20
|
+
// A model can also echo a key as a syntactically valid finding ID. Refuse it without rewriting
|
|
21
|
+
// IDs, references or enums (including accidental collisions); never repair citation structure.
|
|
22
|
+
const structural = [
|
|
23
|
+
...result.participants.flatMap((value) => [
|
|
24
|
+
value.streamId,
|
|
25
|
+
value.outcome,
|
|
26
|
+
...value.evidenceIds,
|
|
27
|
+
...value.feedback.map((quote) => quote.evidenceId),
|
|
28
|
+
]),
|
|
29
|
+
...result.findings.flatMap((value) => [
|
|
30
|
+
value.id,
|
|
31
|
+
value.impact,
|
|
32
|
+
value.recovery,
|
|
33
|
+
value.confidence,
|
|
34
|
+
...value.affectedStreamIds,
|
|
35
|
+
...value.exposedStreamIds,
|
|
36
|
+
...value.observations.flatMap((item) => [item.basis, ...item.evidenceIds]),
|
|
37
|
+
]),
|
|
38
|
+
...(result.designFindings ?? []).flatMap((value) => [
|
|
39
|
+
value.id,
|
|
40
|
+
value.severity,
|
|
41
|
+
value.confidence,
|
|
42
|
+
...value.seenByStreamIds,
|
|
43
|
+
...value.evidenceIds,
|
|
44
|
+
]),
|
|
45
|
+
...(result.concernReviews ?? []).flatMap((value) => [
|
|
46
|
+
value.basis,
|
|
47
|
+
value.disposition,
|
|
48
|
+
...(value.findingId === null ? [] : [value.findingId]),
|
|
49
|
+
...value.evidenceIds,
|
|
50
|
+
]),
|
|
51
|
+
];
|
|
52
|
+
if (structural.some((value) => scrub(value) !== value))
|
|
53
|
+
throw new Error("ANALYSIS_TRANSIENT_SECRET_IN_STRUCTURE");
|
|
54
|
+
return {
|
|
55
|
+
...result,
|
|
56
|
+
summary: scrub(result.summary),
|
|
57
|
+
limitations: result.limitations.map(scrub),
|
|
58
|
+
participants: result.participants.map((value) => ({
|
|
59
|
+
...value,
|
|
60
|
+
summary: scrub(value.summary),
|
|
61
|
+
intent: scrub(value.intent),
|
|
62
|
+
outcomeReason: scrub(value.outcomeReason),
|
|
63
|
+
limitations: value.limitations.map(scrub),
|
|
64
|
+
feedback: value.feedback.map((quote) => ({ ...quote, text: scrub(quote.text) })),
|
|
65
|
+
})),
|
|
66
|
+
findings: result.findings.map((value) => ({
|
|
67
|
+
...value,
|
|
68
|
+
title: scrub(value.title),
|
|
69
|
+
...(value.headline === undefined ? {} : { headline: scrub(value.headline) }),
|
|
70
|
+
...(value.experience === undefined ? {} : { experience: scrub(value.experience) }),
|
|
71
|
+
summary: scrub(value.summary),
|
|
72
|
+
exposureReason: scrub(value.exposureReason),
|
|
73
|
+
nextStep: scrub(value.nextStep),
|
|
74
|
+
priorityReason: scrub(value.priorityReason),
|
|
75
|
+
observations: value.observations.map(observation),
|
|
76
|
+
})),
|
|
77
|
+
...(result.designFindings === undefined
|
|
78
|
+
? {}
|
|
79
|
+
: {
|
|
80
|
+
designFindings: result.designFindings.map((value) => ({
|
|
81
|
+
...value,
|
|
82
|
+
headline: scrub(value.headline),
|
|
83
|
+
screen: scrub(value.screen),
|
|
84
|
+
notice: scrub(value.notice),
|
|
85
|
+
whyItMatters: scrub(value.whyItMatters),
|
|
86
|
+
suggestion: scrub(value.suggestion),
|
|
87
|
+
})),
|
|
88
|
+
}),
|
|
89
|
+
...(result.concernReviews === undefined
|
|
90
|
+
? {}
|
|
91
|
+
: {
|
|
92
|
+
concernReviews: result.concernReviews.map((value) => ({
|
|
93
|
+
...observation(value),
|
|
94
|
+
reason: scrub(value.reason),
|
|
95
|
+
})),
|
|
96
|
+
}),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
const VALIDATION_FAILURES = Object.freeze({
|
|
100
|
+
ANALYSIS_RESULT_SCHEMA_INVALID: "analysis_validation_failed_schema_invalid",
|
|
101
|
+
ANALYSIS_DESIGN_FINDING_ID_DUPLICATE: "analysis_validation_failed_design_finding_id_duplicate",
|
|
102
|
+
ANALYSIS_DESIGN_REFERENCE_INVALID: "analysis_validation_failed_design_reference_invalid",
|
|
103
|
+
ANALYSIS_DESIGN_WITHOUT_CAPTURE: "analysis_validation_failed_design_without_capture",
|
|
104
|
+
ANALYSIS_DESIGN_MEMBERSHIP_INVALID: "analysis_validation_failed_design_membership_invalid",
|
|
105
|
+
ANALYSIS_INPUT_DUPLICATES: "analysis_validation_failed_input_duplicates",
|
|
106
|
+
ANALYSIS_PARTICIPANT_COVERAGE_INVALID: "analysis_validation_failed_participant_coverage_invalid",
|
|
107
|
+
ANALYSIS_PARTICIPANT_REFERENCE_INVALID: "analysis_validation_failed_participant_reference_invalid",
|
|
108
|
+
ANALYSIS_OUTCOME_WITHOUT_EVIDENCE: "analysis_validation_failed_outcome_without_evidence",
|
|
109
|
+
ANALYSIS_QUOTE_INVALID: "analysis_validation_failed_quote_invalid",
|
|
110
|
+
ANALYSIS_FINDING_ID_DUPLICATE: "analysis_validation_failed_finding_id_duplicate",
|
|
111
|
+
ANALYSIS_OBSERVATION_REFERENCE_INVALID: "analysis_validation_failed_observation_reference_invalid",
|
|
112
|
+
ANALYSIS_VISUAL_WITHOUT_CAPTURE: "analysis_validation_failed_visual_without_capture",
|
|
113
|
+
ANALYSIS_ACTION_SOURCE_INVALID: "analysis_validation_failed_action_source_invalid",
|
|
114
|
+
ANALYSIS_STATEMENT_SOURCE_INVALID: "analysis_validation_failed_statement_source_invalid",
|
|
115
|
+
ANALYSIS_FINDING_MEMBERSHIP_INVALID: "analysis_validation_failed_finding_membership_invalid",
|
|
116
|
+
ANALYSIS_AFFECTED_WITHOUT_EVIDENCE: "analysis_validation_failed_affected_without_evidence",
|
|
117
|
+
ANALYSIS_CONCERN_FINDING_INVALID: "analysis_validation_failed_concern_finding_invalid",
|
|
118
|
+
});
|
|
119
|
+
export function checkProviderAnalysis(input, value) {
|
|
120
|
+
try {
|
|
121
|
+
const parsed = analysisResponseSchema.safeParse(value);
|
|
122
|
+
if (!parsed.success) {
|
|
123
|
+
const error = VALIDATION_FAILURES.ANALYSIS_RESULT_SCHEMA_INVALID;
|
|
124
|
+
return { ok: false, error, rejected: { error, errors: [], output: value } };
|
|
125
|
+
}
|
|
126
|
+
let scrubbed;
|
|
127
|
+
try {
|
|
128
|
+
scrubbed = scrubGeneratedNarrative(parsed.data);
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
if (!(error instanceof Error && error.message === "ANALYSIS_TRANSIENT_SECRET_IN_STRUCTURE"))
|
|
132
|
+
return { ok: false, error: "analysis_validation_failed_unexpected" };
|
|
133
|
+
const rejected = "analysis_validation_failed_scrub_rejected";
|
|
134
|
+
return {
|
|
135
|
+
ok: false,
|
|
136
|
+
error: rejected,
|
|
137
|
+
rejected: { error: rejected, errors: [], output: value },
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
const checked = checkAnalysisResult(input, scrubbed);
|
|
141
|
+
if (!checked.ok) {
|
|
142
|
+
const error = VALIDATION_FAILURES[checked.errors[0] ?? ""] ?? "analysis_validation_failed_unexpected";
|
|
143
|
+
return {
|
|
144
|
+
ok: false,
|
|
145
|
+
error,
|
|
146
|
+
rejected: { error, errors: [...checked.errors], output: scrubbed },
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
return checked;
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return { ok: false, error: "analysis_validation_failed_unexpected" };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Copy the reported tokens of every response onto the artifact, summed. OpenAI usage is priced with
|
|
157
|
+
* each response as its own request, since the long-context tier re-prices one request at a time.
|
|
158
|
+
*/
|
|
159
|
+
export function recordProviderUsage(artifact, config, responses) {
|
|
160
|
+
const { usage } = artifact;
|
|
161
|
+
usage.dispatched = responses.some((response) => response.dispatched);
|
|
162
|
+
const turns = responses.flatMap((response) => (response.usage ? [response.usage] : []));
|
|
163
|
+
if (turns.length === 0)
|
|
164
|
+
return;
|
|
165
|
+
const total = (key) => turns.some((turn) => turn[key] !== undefined)
|
|
166
|
+
? turns.reduce((sum, turn) => sum + (turn[key] ?? 0), 0)
|
|
167
|
+
: undefined;
|
|
168
|
+
const tokens = {
|
|
169
|
+
input: total("input"),
|
|
170
|
+
output: total("output"),
|
|
171
|
+
...(total("cachedInput") === undefined ? {} : { cachedInput: total("cachedInput") }),
|
|
172
|
+
...(total("cacheWriteInput") === undefined
|
|
173
|
+
? {}
|
|
174
|
+
: { cacheWriteInput: total("cacheWriteInput") }),
|
|
175
|
+
};
|
|
176
|
+
const priced = config.provider === "codex"
|
|
177
|
+
? { estimatedCostUsd: null, ratesAsOf: null }
|
|
178
|
+
: estimateActorCost({ ...tokens, turns }, config.model);
|
|
179
|
+
usage.inputTokens = tokens.input;
|
|
180
|
+
usage.outputTokens = tokens.output;
|
|
181
|
+
usage.cachedInputTokens = tokens.cachedInput ?? null;
|
|
182
|
+
usage.cacheWriteInputTokens = tokens.cacheWriteInput ?? null;
|
|
183
|
+
usage.usageComplete = responses.every((response) => !response.dispatched ||
|
|
184
|
+
(response.usage !== null && (response.usageComplete ?? config.provider !== "codex")));
|
|
185
|
+
usage.estimatedCostUsd = priced.estimatedCostUsd;
|
|
186
|
+
usage.ratesAsOf = priced.ratesAsOf;
|
|
187
|
+
}
|
|
188
|
+
/** The warnings a Codex response carries about its CLI release and protocol. */
|
|
189
|
+
export function codexWarnings(config, response) {
|
|
190
|
+
if (config.provider !== "codex")
|
|
191
|
+
return [];
|
|
192
|
+
const { cliVersion } = config.identity;
|
|
193
|
+
return [
|
|
194
|
+
response.protocolIncompatibilities === undefined
|
|
195
|
+
? undefined
|
|
196
|
+
: protocolIncompatibilityMessage(cliVersion, response.protocolIncompatibilities),
|
|
197
|
+
protocolAdditionsWarning(cliVersion, response.protocolAdditions),
|
|
198
|
+
unknownNotificationsWarning(response.unknownNotifications, cliVersion),
|
|
199
|
+
truncatedFrameWarning(response.truncatedFrameBytes),
|
|
200
|
+
].filter((warning) => warning !== undefined);
|
|
201
|
+
}
|
package/dist/analysis/store.js
CHANGED
|
@@ -10,7 +10,8 @@ import { ANALYSIS_ID_PATTERN, } from "./types.js";
|
|
|
10
10
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
11
11
|
export const ANALYSIS_DIRECTORY = "analysis";
|
|
12
12
|
export const ANALYSIS_EXECUTION_DIRECTORY = "analysis-attempts";
|
|
13
|
-
|
|
13
|
+
// 4 MiB held one packet; a run of 128 participants stores eight cohorts' packets.
|
|
14
|
+
export const ANALYSIS_MAX_BYTES = 16 * 1024 * 1024;
|
|
14
15
|
export const MAX_VERSIONS = 256;
|
|
15
16
|
const MAX_CORRECTIONS = 256;
|
|
16
17
|
const MAX_CORRECTION_BYTES = 32 * 1024;
|
|
@@ -2,7 +2,7 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
-
import { ANALYSIS_LIMITS, hasControlCharacter, hasRevisionFields } from "./analysis-limits.js";
|
|
5
|
+
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
6
6
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
7
7
|
const text = (max) => z
|
|
8
8
|
.string()
|
|
@@ -17,7 +17,7 @@ const id = z
|
|
|
17
17
|
const sourceId = text(256).min(1);
|
|
18
18
|
const digest = z.string().regex(SHA256_HEX_PATTERN);
|
|
19
19
|
const timestamp = z.string().datetime({ offset: true });
|
|
20
|
-
const sourceIds = z.array(sourceId).max(
|
|
20
|
+
const sourceIds = z.array(sourceId).max(EVIDENCE_LIMITS.participants);
|
|
21
21
|
const refs = z.array(id).min(1).max(ANALYSIS_LIMITS.evidenceRefs);
|
|
22
22
|
const limitations = z.array(text(2000).min(1)).max(100);
|
|
23
23
|
const observationSchema = z
|
|
@@ -60,7 +60,7 @@ const analysisResultSchema = z
|
|
|
60
60
|
limitations,
|
|
61
61
|
})
|
|
62
62
|
.strict())
|
|
63
|
-
.max(
|
|
63
|
+
.max(EVIDENCE_LIMITS.participants),
|
|
64
64
|
findings: z
|
|
65
65
|
.array(z
|
|
66
66
|
.object({
|
|
@@ -116,12 +116,18 @@ const normalizedResult = ({ concernReviews, designFindings, findings, ...result
|
|
|
116
116
|
...(designFindings === undefined ? {} : { designFindings }),
|
|
117
117
|
...(concernReviews === undefined ? {} : { concernReviews }),
|
|
118
118
|
});
|
|
119
|
+
// A run's packet holds every cohort's packet.
|
|
120
|
+
const MAX_PACKET_EVIDENCE = EVIDENCE_LIMITS.evidence * MAX_ANALYSIS_COHORTS;
|
|
119
121
|
const analysisCoverageSchema = z
|
|
120
122
|
.object({
|
|
121
123
|
includedStreamIds: sourceIds,
|
|
122
124
|
omittedStreamIds: sourceIds,
|
|
123
|
-
evidenceCount: z.number().int().min(0).max(
|
|
124
|
-
captureCount: z
|
|
125
|
+
evidenceCount: z.number().int().min(0).max(MAX_PACKET_EVIDENCE),
|
|
126
|
+
captureCount: z
|
|
127
|
+
.number()
|
|
128
|
+
.int()
|
|
129
|
+
.min(0)
|
|
130
|
+
.max(EVIDENCE_LIMITS.captures * MAX_ANALYSIS_COHORTS),
|
|
125
131
|
complete: z.boolean(),
|
|
126
132
|
omissions: limitations,
|
|
127
133
|
})
|
|
@@ -264,9 +270,9 @@ const analysisArtifactSchema = z
|
|
|
264
270
|
provenance: analysisParticipantProvenanceSchema,
|
|
265
271
|
})
|
|
266
272
|
.strict())
|
|
267
|
-
.max(
|
|
273
|
+
.max(EVIDENCE_LIMITS.participants),
|
|
268
274
|
coverage: analysisCoverageSchema,
|
|
269
|
-
evidence: z.array(analysisEvidenceSchema).max(
|
|
275
|
+
evidence: z.array(analysisEvidenceSchema).max(MAX_PACKET_EVIDENCE),
|
|
270
276
|
result: analysisResultSchema.nullable(),
|
|
271
277
|
error: z
|
|
272
278
|
.string()
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, } from "../../analysis/automatic-config.js";
|
|
1
|
+
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, } from "../../analysis/automatic-config.js";
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
@@ -233,8 +233,13 @@ async function handleAnalyzeCorrect(io, options, command) {
|
|
|
233
233
|
io.setExitCode(2);
|
|
234
234
|
}
|
|
235
235
|
}
|
|
236
|
-
/**
|
|
236
|
+
/**
|
|
237
|
+
* An admitted dry run's costs: the expected cost, the worst case and the cap it was admitted
|
|
238
|
+
* under. The worst case of an analysis in cohorts is every request writing its whole allowance.
|
|
239
|
+
*/
|
|
237
240
|
function admittedDryRunText(admission) {
|
|
238
241
|
const usd = (value) => value === null || value === undefined ? "unknown" : `$${value.toFixed(2)}`;
|
|
239
|
-
|
|
242
|
+
const requests = analysisRequestsText(admission?.requests ?? 1);
|
|
243
|
+
const writer = requests === undefined ? "the analyst writes its" : `every request (${requests}) writes its`;
|
|
244
|
+
return `Expected cost: ${usd(admission?.estimatedCostUsd)}. Worst case: ${usd(admission?.worstCaseCostUsd)}, if ${writer} whole ${admission?.outputTokenAllowance ?? "unknown"}-token output allowance, reasoning included. Admitted under the $${admission?.maxCostUsd ?? "unknown"} cap. No request sent.\n`;
|
|
240
245
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import {} from "../../run-study.js";
|
|
4
4
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
@@ -237,7 +237,7 @@ async function reportCuaRun(args, prepared, result, live) {
|
|
|
237
237
|
};
|
|
238
238
|
}
|
|
239
239
|
writeResult(args.command, args.io, output, (value) => formatCuaStudyHuman(value, args.config.subject));
|
|
240
|
-
args.io.setExitCode(result.ok &&
|
|
240
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
241
241
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
242
242
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
243
243
|
const followResult = output.observer?.ok
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function scriptedRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, (value) => formatScriptedStudyHuman(value, args.config.subject));
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
// Watch mode serves the Observer the route rendered through its finished run.
|
|
41
41
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { resolveStudyDryRun } from "../../study/plan.js";
|
|
4
4
|
import { serveObserver } from "../../observer/render.js";
|
|
@@ -121,7 +121,7 @@ export function sharedWorldRouteRun(args) {
|
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
123
|
writeResult(args.command, args.io, output, formatConcurrentSharedWorldStudyHuman);
|
|
124
|
-
args.io.setExitCode(result.ok &&
|
|
124
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
125
125
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
126
126
|
if (server && output.observer?.ok) {
|
|
127
127
|
await followObserver(args.io, output.observer, server);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { automaticAnalysisSucceeded } from "../../analysis/automatic-completion.js";
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { cliAnalysisOptions } from "./analysis-signals.js";
|
|
4
4
|
import { wantsJson, writeResult } from "../io.js";
|
|
@@ -35,7 +35,7 @@ export function terminalRouteRun(args) {
|
|
|
35
35
|
}
|
|
36
36
|
const result = outcome.result;
|
|
37
37
|
writeResult(args.command, args.io, result, formatTerminalStudyHuman);
|
|
38
|
-
args.io.setExitCode(result.ok &&
|
|
38
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
39
39
|
await writeRunFindings(args.command, args.io, args.options.cwd, result);
|
|
40
40
|
if (finishedPlan !== undefined && result.ok && result.observer !== undefined) {
|
|
41
41
|
await showObserver({
|
|
@@ -2,7 +2,7 @@ import { resolve } from "node:path";
|
|
|
2
2
|
import { Command } from "commander";
|
|
3
3
|
import { deriveRunFacts } from "../telemetry.js";
|
|
4
4
|
import { resolveStudyManifest } from "../../study/discover.js";
|
|
5
|
-
import {
|
|
5
|
+
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../analysis/automatic-config.js";
|
|
6
6
|
import { warnAndQueue } from "../deprecations.js";
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
|
@@ -131,9 +131,9 @@ export async function runStudyCommand(args) {
|
|
|
131
131
|
// Surface it so the invoker (who may not be the manifest author) knows executable code ran.
|
|
132
132
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is code that humanish loaded and ran in this process. Review it as you would any code you run.\n`);
|
|
133
133
|
}
|
|
134
|
-
const analysisBudget =
|
|
134
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, route, computerUseParticipantCount(config));
|
|
135
135
|
if (analysisBudget && resolveStudyDryRun(config, args.options.dryRun, true) === false) {
|
|
136
|
-
args.io.writeErr(`${
|
|
136
|
+
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
137
137
|
}
|
|
138
138
|
return scorer === undefined ? {} : { scorer };
|
|
139
139
|
},
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { formatAutomaticAnalysisBudget } from "../../analysis/automatic-config.js";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { Command, Option } from "commander";
|
|
4
4
|
import { inspectStudyManifest, listStudyManifests } from "../../study/discover.js";
|
|
@@ -136,7 +136,7 @@ function formatStudyCheckHuman(result) {
|
|
|
136
136
|
`targets: ${checkedTargets.length ? `${reachableTargets.length}/${checkedTargets.length} reachable` : `${result.targets.length} declared, not checked`}`,
|
|
137
137
|
...(blockedTargets.length ? [`blocked-targets: ${blockedTargets.length}`] : []),
|
|
138
138
|
`spend: ${result.spend.e2bDesktop ? "one e2b desktop, no model calls" : "none"}`,
|
|
139
|
-
...(result.analysis ? [
|
|
139
|
+
...(result.analysis ? [formatAutomaticAnalysisBudget(result.analysis)] : []),
|
|
140
140
|
...(result.sandbox.created
|
|
141
141
|
? [`sandbox: created=yes killed=${result.sandbox.killed === true ? "yes" : "no"}`]
|
|
142
142
|
: []),
|
package/dist/cli/io.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
+
import { automaticAnalysisSucceeded, } from "../analysis/automatic-completion.js";
|
|
1
2
|
import { costRefusal, refusedCost } from "../analysis/admission.js";
|
|
2
3
|
import { Command, Option } from "commander";
|
|
3
4
|
import { loadEnvFile, recordDotenvNames } from "../keys/env-file.js";
|
|
4
5
|
import { discoverProviderKeys } from "../keys/key-resolution.js";
|
|
5
6
|
import { deriveRunFacts } from "./telemetry.js";
|
|
6
7
|
import { warnAndQueue, withQueuedWarnings } from "./deprecations.js";
|
|
7
|
-
import { PARTICIPANT_LIMIT_SKIP, PARTICIPANT_LIMIT_SKIP_TEXT, studyAnalysisSucceeded, } from "../study/automatic-analysis-plan.js";
|
|
8
8
|
import { forTerminal } from "../routes/terminal/encoding.js";
|
|
9
9
|
import { cli } from "./invocation.js";
|
|
10
10
|
export const CLI_RESPONSE_SCHEMA = "humanish.cli-response.v1";
|
|
@@ -295,7 +295,6 @@ const ANALYSIS_REASON_TEXT = {
|
|
|
295
295
|
AUTOMATIC_ANALYSIS_BUSY: "skipped because another analysis is running",
|
|
296
296
|
AUTOMATIC_ANALYSIS_ADMISSION_REFUSED: "refused before it started: its configuration, question, evidence or cost did not pass admission",
|
|
297
297
|
AUTOMATIC_ANALYSIS_CLEANUP_UNCONFIRMED: "not started because the run's cleanup was not confirmed",
|
|
298
|
-
[PARTICIPANT_LIMIT_SKIP]: PARTICIPANT_LIMIT_SKIP_TEXT,
|
|
299
298
|
AUTOMATIC_ANALYSIS_REUSED: "complete, reusing an earlier analysis of the same evidence",
|
|
300
299
|
AUTOMATIC_ANALYSIS_LIMITATIONS: "partial, because it covered only part of the evidence",
|
|
301
300
|
AUTOMATIC_ANALYSIS_ADMISSION_EXCEEDED: "partial, because it went over its estimated cost or output",
|
|
@@ -321,5 +320,5 @@ export function automaticAnalysisEnvelope(result) {
|
|
|
321
320
|
!("ok" in result))
|
|
322
321
|
return result;
|
|
323
322
|
const run = result;
|
|
324
|
-
return { ...run, runOk: run.ok, ok: run.ok &&
|
|
323
|
+
return { ...run, runOk: run.ok, ok: run.ok && automaticAnalysisSucceeded(run) };
|
|
325
324
|
}
|