humanish 0.117.0 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/analysis/admission.d.ts +23 -4
- package/dist/analysis/admission.js +47 -9
- package/dist/analysis/analysis-limits.d.ts +32 -0
- package/dist/analysis/analysis-limits.js +41 -3
- package/dist/analysis/automatic-config.d.ts +11 -1
- package/dist/analysis/automatic-config.js +26 -11
- package/dist/analysis/cohorts.d.ts +103 -0
- package/dist/analysis/cohorts.js +80 -0
- package/dist/analysis/evidence.d.ts +5 -10
- package/dist/analysis/evidence.js +37 -34
- package/dist/analysis/execute.d.ts +24 -7
- package/dist/analysis/execute.js +290 -307
- package/dist/analysis/responses.d.ts +19 -0
- package/dist/analysis/responses.js +201 -0
- package/dist/analysis/store.js +2 -1
- package/dist/analysis/validation.js +13 -7
- package/dist/cli/commands/analyze.js +10 -4
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-shared-world.js +3 -1
- package/dist/cli/commands/study-run.js +1 -1
- package/dist/cli/doctor.js +11 -1
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/fanout-records.js +6 -1
- package/dist/routes/computer-use/participant-execution.d.ts +6 -4
- package/dist/routes/computer-use/participant-execution.js +42 -14
- package/dist/routes/computer-use/participant-model.js +18 -8
- package/dist/routes/computer-use/participant-runs.js +15 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +38 -12
- package/dist/routes/computer-use/rerun-selection.js +14 -3
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/computer-use/types.d.ts +13 -1
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/bundle-records.js +14 -0
- package/dist/routes/shared-world/bundle.js +2 -1
- package/dist/routes/shared-world/external-public.js +15 -3
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +10 -1
- package/dist/routes/shared-world/plan.js +86 -10
- package/dist/routes/shared-world/provisioned.js +15 -8
- package/dist/routes/shared-world/route.js +7 -4
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle-shape.js +12 -2
- package/dist/run/bundle.d.ts +20 -3
- package/dist/run/participant-records.js +1 -0
- package/dist/run/projection.js +7 -2
- package/dist/run/run.js +4 -2
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/arrivals.d.ts +114 -0
- package/dist/study/arrivals.js +204 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +6 -20
- package/dist/study/doctor.js +2 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +3 -2
- package/dist/study/parse/actors.d.ts +2 -2
- package/dist/study/parse/actors.js +10 -4
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +36 -2
- package/dist/study/plan-base.d.ts +1 -1
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +3 -5
- package/dist/study/plan-types.d.ts +13 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +24 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.d.ts +2 -2
- package/dist/study/validation.js +27 -14
- package/dist/study/warnings.js +11 -3
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +112 -112
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/contracts/run-bundle.md +8 -7
- package/docs/contracts/schemas.md +33 -9
- package/docs/contracts/study-analysis.md +61 -8
- package/docs/principles/invariants-and-defaults.md +14 -14
- package/docs/product/automatic-analysis.md +10 -3
- package/package.json +1 -1
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { RejectedAnalysisOutput } from "./diagnostics.js";
|
|
2
|
+
import type { AnalysisProviderResult } from "./provider.js";
|
|
3
|
+
import type { AnalysisArtifact, AnalysisConfig, AnalysisInput, AnalysisResult } from "./types.js";
|
|
4
|
+
export type CheckedProviderAnalysis = {
|
|
5
|
+
ok: true;
|
|
6
|
+
result: AnalysisResult;
|
|
7
|
+
} | {
|
|
8
|
+
ok: false;
|
|
9
|
+
error: string;
|
|
10
|
+
rejected?: RejectedAnalysisOutput;
|
|
11
|
+
};
|
|
12
|
+
export declare function checkProviderAnalysis(input: AnalysisInput, value: unknown): CheckedProviderAnalysis;
|
|
13
|
+
/**
|
|
14
|
+
* Copy the reported tokens of every response onto the artifact, summed. OpenAI usage is priced with
|
|
15
|
+
* each response as its own request, since the long-context tier re-prices one request at a time.
|
|
16
|
+
*/
|
|
17
|
+
export declare function recordProviderUsage(artifact: AnalysisArtifact, config: AnalysisConfig, responses: readonly AnalysisProviderResult[]): void;
|
|
18
|
+
/** The warnings a Codex response carries about its CLI release and protocol. */
|
|
19
|
+
export declare function codexWarnings(config: AnalysisConfig, response: AnalysisProviderResult): string[];
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// What the provider responses of one attempt give it: the usage they report, the warnings a Codex
|
|
2
|
+
// response carries, and each response checked before it becomes an analysis result (the response
|
|
3
|
+
// schema, the scrub of known transient values from generated prose, and the evidence rules in
|
|
4
|
+
// validation.ts). Only an allowlisted code leaves a check; the rejected output goes to the caller
|
|
5
|
+
// for local diagnosis.
|
|
6
|
+
import { truncatedFrameWarning, unknownNotificationsWarning, } from "../actors/codex/restricted-notifications.js";
|
|
7
|
+
import { protocolAdditionsWarning, protocolIncompatibilityMessage, } from "../actors/codex/protocol-compat.js";
|
|
8
|
+
import { estimateActorCost } from "../run/pricing.js";
|
|
9
|
+
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
10
|
+
import { analysisResponseSchema, checkAnalysisResult } from "./validation.js";
|
|
11
|
+
/** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
|
|
12
|
+
* known value is found as written and in its encoded forms. */
|
|
13
|
+
function scrubGeneratedNarrative(result) {
|
|
14
|
+
const scrub = transientCommsKnownValueScrub();
|
|
15
|
+
const observation = (value) => ({
|
|
16
|
+
...value,
|
|
17
|
+
claim: scrub(value.claim),
|
|
18
|
+
limitation: scrub(value.limitation),
|
|
19
|
+
});
|
|
20
|
+
// A model can also echo a key as a syntactically valid finding ID. Refuse it without rewriting
|
|
21
|
+
// IDs, references or enums (including accidental collisions); never repair citation structure.
|
|
22
|
+
const structural = [
|
|
23
|
+
...result.participants.flatMap((value) => [
|
|
24
|
+
value.streamId,
|
|
25
|
+
value.outcome,
|
|
26
|
+
...value.evidenceIds,
|
|
27
|
+
...value.feedback.map((quote) => quote.evidenceId),
|
|
28
|
+
]),
|
|
29
|
+
...result.findings.flatMap((value) => [
|
|
30
|
+
value.id,
|
|
31
|
+
value.impact,
|
|
32
|
+
value.recovery,
|
|
33
|
+
value.confidence,
|
|
34
|
+
...value.affectedStreamIds,
|
|
35
|
+
...value.exposedStreamIds,
|
|
36
|
+
...value.observations.flatMap((item) => [item.basis, ...item.evidenceIds]),
|
|
37
|
+
]),
|
|
38
|
+
...(result.designFindings ?? []).flatMap((value) => [
|
|
39
|
+
value.id,
|
|
40
|
+
value.severity,
|
|
41
|
+
value.confidence,
|
|
42
|
+
...value.seenByStreamIds,
|
|
43
|
+
...value.evidenceIds,
|
|
44
|
+
]),
|
|
45
|
+
...(result.concernReviews ?? []).flatMap((value) => [
|
|
46
|
+
value.basis,
|
|
47
|
+
value.disposition,
|
|
48
|
+
...(value.findingId === null ? [] : [value.findingId]),
|
|
49
|
+
...value.evidenceIds,
|
|
50
|
+
]),
|
|
51
|
+
];
|
|
52
|
+
if (structural.some((value) => scrub(value) !== value))
|
|
53
|
+
throw new Error("ANALYSIS_TRANSIENT_SECRET_IN_STRUCTURE");
|
|
54
|
+
return {
|
|
55
|
+
...result,
|
|
56
|
+
summary: scrub(result.summary),
|
|
57
|
+
limitations: result.limitations.map(scrub),
|
|
58
|
+
participants: result.participants.map((value) => ({
|
|
59
|
+
...value,
|
|
60
|
+
summary: scrub(value.summary),
|
|
61
|
+
intent: scrub(value.intent),
|
|
62
|
+
outcomeReason: scrub(value.outcomeReason),
|
|
63
|
+
limitations: value.limitations.map(scrub),
|
|
64
|
+
feedback: value.feedback.map((quote) => ({ ...quote, text: scrub(quote.text) })),
|
|
65
|
+
})),
|
|
66
|
+
findings: result.findings.map((value) => ({
|
|
67
|
+
...value,
|
|
68
|
+
title: scrub(value.title),
|
|
69
|
+
...(value.headline === undefined ? {} : { headline: scrub(value.headline) }),
|
|
70
|
+
...(value.experience === undefined ? {} : { experience: scrub(value.experience) }),
|
|
71
|
+
summary: scrub(value.summary),
|
|
72
|
+
exposureReason: scrub(value.exposureReason),
|
|
73
|
+
nextStep: scrub(value.nextStep),
|
|
74
|
+
priorityReason: scrub(value.priorityReason),
|
|
75
|
+
observations: value.observations.map(observation),
|
|
76
|
+
})),
|
|
77
|
+
...(result.designFindings === undefined
|
|
78
|
+
? {}
|
|
79
|
+
: {
|
|
80
|
+
designFindings: result.designFindings.map((value) => ({
|
|
81
|
+
...value,
|
|
82
|
+
headline: scrub(value.headline),
|
|
83
|
+
screen: scrub(value.screen),
|
|
84
|
+
notice: scrub(value.notice),
|
|
85
|
+
whyItMatters: scrub(value.whyItMatters),
|
|
86
|
+
suggestion: scrub(value.suggestion),
|
|
87
|
+
})),
|
|
88
|
+
}),
|
|
89
|
+
...(result.concernReviews === undefined
|
|
90
|
+
? {}
|
|
91
|
+
: {
|
|
92
|
+
concernReviews: result.concernReviews.map((value) => ({
|
|
93
|
+
...observation(value),
|
|
94
|
+
reason: scrub(value.reason),
|
|
95
|
+
})),
|
|
96
|
+
}),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
const VALIDATION_FAILURES = Object.freeze({
|
|
100
|
+
ANALYSIS_RESULT_SCHEMA_INVALID: "analysis_validation_failed_schema_invalid",
|
|
101
|
+
ANALYSIS_DESIGN_FINDING_ID_DUPLICATE: "analysis_validation_failed_design_finding_id_duplicate",
|
|
102
|
+
ANALYSIS_DESIGN_REFERENCE_INVALID: "analysis_validation_failed_design_reference_invalid",
|
|
103
|
+
ANALYSIS_DESIGN_WITHOUT_CAPTURE: "analysis_validation_failed_design_without_capture",
|
|
104
|
+
ANALYSIS_DESIGN_MEMBERSHIP_INVALID: "analysis_validation_failed_design_membership_invalid",
|
|
105
|
+
ANALYSIS_INPUT_DUPLICATES: "analysis_validation_failed_input_duplicates",
|
|
106
|
+
ANALYSIS_PARTICIPANT_COVERAGE_INVALID: "analysis_validation_failed_participant_coverage_invalid",
|
|
107
|
+
ANALYSIS_PARTICIPANT_REFERENCE_INVALID: "analysis_validation_failed_participant_reference_invalid",
|
|
108
|
+
ANALYSIS_OUTCOME_WITHOUT_EVIDENCE: "analysis_validation_failed_outcome_without_evidence",
|
|
109
|
+
ANALYSIS_QUOTE_INVALID: "analysis_validation_failed_quote_invalid",
|
|
110
|
+
ANALYSIS_FINDING_ID_DUPLICATE: "analysis_validation_failed_finding_id_duplicate",
|
|
111
|
+
ANALYSIS_OBSERVATION_REFERENCE_INVALID: "analysis_validation_failed_observation_reference_invalid",
|
|
112
|
+
ANALYSIS_VISUAL_WITHOUT_CAPTURE: "analysis_validation_failed_visual_without_capture",
|
|
113
|
+
ANALYSIS_ACTION_SOURCE_INVALID: "analysis_validation_failed_action_source_invalid",
|
|
114
|
+
ANALYSIS_STATEMENT_SOURCE_INVALID: "analysis_validation_failed_statement_source_invalid",
|
|
115
|
+
ANALYSIS_FINDING_MEMBERSHIP_INVALID: "analysis_validation_failed_finding_membership_invalid",
|
|
116
|
+
ANALYSIS_AFFECTED_WITHOUT_EVIDENCE: "analysis_validation_failed_affected_without_evidence",
|
|
117
|
+
ANALYSIS_CONCERN_FINDING_INVALID: "analysis_validation_failed_concern_finding_invalid",
|
|
118
|
+
});
|
|
119
|
+
export function checkProviderAnalysis(input, value) {
|
|
120
|
+
try {
|
|
121
|
+
const parsed = analysisResponseSchema.safeParse(value);
|
|
122
|
+
if (!parsed.success) {
|
|
123
|
+
const error = VALIDATION_FAILURES.ANALYSIS_RESULT_SCHEMA_INVALID;
|
|
124
|
+
return { ok: false, error, rejected: { error, errors: [], output: value } };
|
|
125
|
+
}
|
|
126
|
+
let scrubbed;
|
|
127
|
+
try {
|
|
128
|
+
scrubbed = scrubGeneratedNarrative(parsed.data);
|
|
129
|
+
}
|
|
130
|
+
catch (error) {
|
|
131
|
+
if (!(error instanceof Error && error.message === "ANALYSIS_TRANSIENT_SECRET_IN_STRUCTURE"))
|
|
132
|
+
return { ok: false, error: "analysis_validation_failed_unexpected" };
|
|
133
|
+
const rejected = "analysis_validation_failed_scrub_rejected";
|
|
134
|
+
return {
|
|
135
|
+
ok: false,
|
|
136
|
+
error: rejected,
|
|
137
|
+
rejected: { error: rejected, errors: [], output: value },
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
const checked = checkAnalysisResult(input, scrubbed);
|
|
141
|
+
if (!checked.ok) {
|
|
142
|
+
const error = VALIDATION_FAILURES[checked.errors[0] ?? ""] ?? "analysis_validation_failed_unexpected";
|
|
143
|
+
return {
|
|
144
|
+
ok: false,
|
|
145
|
+
error,
|
|
146
|
+
rejected: { error, errors: [...checked.errors], output: scrubbed },
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
return checked;
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
return { ok: false, error: "analysis_validation_failed_unexpected" };
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Copy the reported tokens of every response onto the artifact, summed. OpenAI usage is priced with
|
|
157
|
+
* each response as its own request, since the long-context tier re-prices one request at a time.
|
|
158
|
+
*/
|
|
159
|
+
export function recordProviderUsage(artifact, config, responses) {
|
|
160
|
+
const { usage } = artifact;
|
|
161
|
+
usage.dispatched = responses.some((response) => response.dispatched);
|
|
162
|
+
const turns = responses.flatMap((response) => (response.usage ? [response.usage] : []));
|
|
163
|
+
if (turns.length === 0)
|
|
164
|
+
return;
|
|
165
|
+
const total = (key) => turns.some((turn) => turn[key] !== undefined)
|
|
166
|
+
? turns.reduce((sum, turn) => sum + (turn[key] ?? 0), 0)
|
|
167
|
+
: undefined;
|
|
168
|
+
const tokens = {
|
|
169
|
+
input: total("input"),
|
|
170
|
+
output: total("output"),
|
|
171
|
+
...(total("cachedInput") === undefined ? {} : { cachedInput: total("cachedInput") }),
|
|
172
|
+
...(total("cacheWriteInput") === undefined
|
|
173
|
+
? {}
|
|
174
|
+
: { cacheWriteInput: total("cacheWriteInput") }),
|
|
175
|
+
};
|
|
176
|
+
const priced = config.provider === "codex"
|
|
177
|
+
? { estimatedCostUsd: null, ratesAsOf: null }
|
|
178
|
+
: estimateActorCost({ ...tokens, turns }, config.model);
|
|
179
|
+
usage.inputTokens = tokens.input;
|
|
180
|
+
usage.outputTokens = tokens.output;
|
|
181
|
+
usage.cachedInputTokens = tokens.cachedInput ?? null;
|
|
182
|
+
usage.cacheWriteInputTokens = tokens.cacheWriteInput ?? null;
|
|
183
|
+
usage.usageComplete = responses.every((response) => !response.dispatched ||
|
|
184
|
+
(response.usage !== null && (response.usageComplete ?? config.provider !== "codex")));
|
|
185
|
+
usage.estimatedCostUsd = priced.estimatedCostUsd;
|
|
186
|
+
usage.ratesAsOf = priced.ratesAsOf;
|
|
187
|
+
}
|
|
188
|
+
/** The warnings a Codex response carries about its CLI release and protocol. */
|
|
189
|
+
export function codexWarnings(config, response) {
|
|
190
|
+
if (config.provider !== "codex")
|
|
191
|
+
return [];
|
|
192
|
+
const { cliVersion } = config.identity;
|
|
193
|
+
return [
|
|
194
|
+
response.protocolIncompatibilities === undefined
|
|
195
|
+
? undefined
|
|
196
|
+
: protocolIncompatibilityMessage(cliVersion, response.protocolIncompatibilities),
|
|
197
|
+
protocolAdditionsWarning(cliVersion, response.protocolAdditions),
|
|
198
|
+
unknownNotificationsWarning(response.unknownNotifications, cliVersion),
|
|
199
|
+
truncatedFrameWarning(response.truncatedFrameBytes),
|
|
200
|
+
].filter((warning) => warning !== undefined);
|
|
201
|
+
}
|
package/dist/analysis/store.js
CHANGED
|
@@ -10,7 +10,8 @@ import { ANALYSIS_ID_PATTERN, } from "./types.js";
|
|
|
10
10
|
import { RUN_BUNDLE_FILE } from "../run/bundle.js";
|
|
11
11
|
export const ANALYSIS_DIRECTORY = "analysis";
|
|
12
12
|
export const ANALYSIS_EXECUTION_DIRECTORY = "analysis-attempts";
|
|
13
|
-
|
|
13
|
+
// 4 MiB held one packet; a run of 128 participants stores eight cohorts' packets.
|
|
14
|
+
export const ANALYSIS_MAX_BYTES = 16 * 1024 * 1024;
|
|
14
15
|
export const MAX_VERSIONS = 256;
|
|
15
16
|
const MAX_CORRECTIONS = 256;
|
|
16
17
|
const MAX_CORRECTION_BYTES = 32 * 1024;
|
|
@@ -2,7 +2,7 @@ import { validStoredCodexAnalysisConfig } from "./codex-config.js";
|
|
|
2
2
|
import { createHash } from "node:crypto";
|
|
3
3
|
import { z } from "zod";
|
|
4
4
|
import { ACTOR_STATUSES, ACTOR_STOP_CAUSES } from "../actors/contract.js";
|
|
5
|
-
import { ANALYSIS_LIMITS, hasControlCharacter, hasRevisionFields } from "./analysis-limits.js";
|
|
5
|
+
import { ANALYSIS_LIMITS, EVIDENCE_LIMITS, MAX_ANALYSIS_COHORTS, hasControlCharacter, hasRevisionFields, } from "./analysis-limits.js";
|
|
6
6
|
import { SHA256_HEX_PATTERN, ACTION_CAPTURE_VERSION, ANALYSIS_SCHEMA, ANALYSIS_CORRECTION_SCHEMA, } from "./types.js";
|
|
7
7
|
const text = (max) => z
|
|
8
8
|
.string()
|
|
@@ -17,7 +17,7 @@ const id = z
|
|
|
17
17
|
const sourceId = text(256).min(1);
|
|
18
18
|
const digest = z.string().regex(SHA256_HEX_PATTERN);
|
|
19
19
|
const timestamp = z.string().datetime({ offset: true });
|
|
20
|
-
const sourceIds = z.array(sourceId).max(
|
|
20
|
+
const sourceIds = z.array(sourceId).max(EVIDENCE_LIMITS.participants);
|
|
21
21
|
const refs = z.array(id).min(1).max(ANALYSIS_LIMITS.evidenceRefs);
|
|
22
22
|
const limitations = z.array(text(2000).min(1)).max(100);
|
|
23
23
|
const observationSchema = z
|
|
@@ -60,7 +60,7 @@ const analysisResultSchema = z
|
|
|
60
60
|
limitations,
|
|
61
61
|
})
|
|
62
62
|
.strict())
|
|
63
|
-
.max(
|
|
63
|
+
.max(EVIDENCE_LIMITS.participants),
|
|
64
64
|
findings: z
|
|
65
65
|
.array(z
|
|
66
66
|
.object({
|
|
@@ -116,12 +116,18 @@ const normalizedResult = ({ concernReviews, designFindings, findings, ...result
|
|
|
116
116
|
...(designFindings === undefined ? {} : { designFindings }),
|
|
117
117
|
...(concernReviews === undefined ? {} : { concernReviews }),
|
|
118
118
|
});
|
|
119
|
+
// A run's packet holds every cohort's packet.
|
|
120
|
+
const MAX_PACKET_EVIDENCE = EVIDENCE_LIMITS.evidence * MAX_ANALYSIS_COHORTS;
|
|
119
121
|
const analysisCoverageSchema = z
|
|
120
122
|
.object({
|
|
121
123
|
includedStreamIds: sourceIds,
|
|
122
124
|
omittedStreamIds: sourceIds,
|
|
123
|
-
evidenceCount: z.number().int().min(0).max(
|
|
124
|
-
captureCount: z
|
|
125
|
+
evidenceCount: z.number().int().min(0).max(MAX_PACKET_EVIDENCE),
|
|
126
|
+
captureCount: z
|
|
127
|
+
.number()
|
|
128
|
+
.int()
|
|
129
|
+
.min(0)
|
|
130
|
+
.max(EVIDENCE_LIMITS.captures * MAX_ANALYSIS_COHORTS),
|
|
125
131
|
complete: z.boolean(),
|
|
126
132
|
omissions: limitations,
|
|
127
133
|
})
|
|
@@ -264,9 +270,9 @@ const analysisArtifactSchema = z
|
|
|
264
270
|
provenance: analysisParticipantProvenanceSchema,
|
|
265
271
|
})
|
|
266
272
|
.strict())
|
|
267
|
-
.max(
|
|
273
|
+
.max(EVIDENCE_LIMITS.participants),
|
|
268
274
|
coverage: analysisCoverageSchema,
|
|
269
|
-
evidence: z.array(analysisEvidenceSchema).max(
|
|
275
|
+
evidence: z.array(analysisEvidenceSchema).max(MAX_PACKET_EVIDENCE),
|
|
270
276
|
result: analysisResultSchema.nullable(),
|
|
271
277
|
error: z
|
|
272
278
|
.string()
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, } from "../../analysis/automatic-config.js";
|
|
1
|
+
import { analysisRequestsText, resolveAutomaticAnalysis, DEFAULT_ANALYSIS_MODEL, DEFAULT_ANALYSIS_TIMEOUT_MS, } from "../../analysis/automatic-config.js";
|
|
2
2
|
import { Command, Option } from "commander";
|
|
3
3
|
import { forTerminal } from "../../routes/terminal/encoding.js";
|
|
4
4
|
import { analyzeRun, correctAnalysis, dryRunBundleRefusal, showAnalysis, } from "../../analysis/service.js";
|
|
5
|
+
import { admissionRule } from "../../analysis/admission.js";
|
|
5
6
|
import { listAnalyses } from "../../analysis/store.js";
|
|
6
7
|
import { listAnalysisExecutions } from "../../analysis/store-executions.js";
|
|
7
8
|
import { resolveRunPath } from "../../run/locate.js";
|
|
@@ -30,7 +31,7 @@ export function registerAnalyzeCommand(parent, io) {
|
|
|
30
31
|
.option("--run <id>", RUN_OPTION_DESCRIPTION, "latest")
|
|
31
32
|
.option("--cwd <path>", CWD_OPTION_DESCRIPTION, ".")
|
|
32
33
|
.option("--provider <name>", "Analyst: openai (default) or restricted codex account. No provider fallback.")
|
|
33
|
-
.option("--max-cost <usd>",
|
|
34
|
+
.option("--max-cost <usd>", `Required for OpenAI, also with --dry-run: an analysis is refused before it sends anything if ${admissionRule("this")}. It does not limit billing. Unsupported for Codex.`)
|
|
34
35
|
.option("--model <id>", "Supported vision model. OpenAI uses high effort; qualified Codex account analysis uses low effort.", DEFAULT_ANALYSIS_MODEL)
|
|
35
36
|
.option("--question <text>", "Additional reviewer question; does not change participant instructions.")
|
|
36
37
|
.option("--timeout-ms <ms>", "Request timeout, at most 600000 ms.", String(DEFAULT_ANALYSIS_TIMEOUT_MS))
|
|
@@ -232,8 +233,13 @@ async function handleAnalyzeCorrect(io, options, command) {
|
|
|
232
233
|
io.setExitCode(2);
|
|
233
234
|
}
|
|
234
235
|
}
|
|
235
|
-
/**
|
|
236
|
+
/**
|
|
237
|
+
* An admitted dry run's costs: the expected cost, the worst case and the cap it was admitted
|
|
238
|
+
* under. The worst case of an analysis in cohorts is every request writing its whole allowance.
|
|
239
|
+
*/
|
|
236
240
|
function admittedDryRunText(admission) {
|
|
237
241
|
const usd = (value) => value === null || value === undefined ? "unknown" : `$${value.toFixed(2)}`;
|
|
238
|
-
|
|
242
|
+
const requests = analysisRequestsText(admission?.requests ?? 1);
|
|
243
|
+
const writer = requests === undefined ? "the analyst writes its" : `every request (${requests}) writes its`;
|
|
244
|
+
return `Expected cost: ${usd(admission?.estimatedCostUsd)}. Worst case: ${usd(admission?.worstCaseCostUsd)}, if ${writer} whole ${admission?.outputTokenAllowance ?? "unknown"}-token output allowance, reasoning included. Admitted under the $${admission?.maxCostUsd ?? "unknown"} cap. No request sent.\n`;
|
|
239
245
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
3
4
|
/**
|
|
4
5
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
5
6
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
@@ -38,16 +39,12 @@ function sandboxIdsLine(result) {
|
|
|
38
39
|
: [];
|
|
39
40
|
}
|
|
40
41
|
/**
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
* made before the route resolved a URL has none either, so it names the URL the study declares.
|
|
42
|
+
* The `subject:` line's name for the study's declared subject. A refusal made before the route
|
|
43
|
+
* resolved a URL has none, so it names the URL the study declares.
|
|
44
44
|
*/
|
|
45
|
-
function
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
if (subject.product !== undefined)
|
|
49
|
-
return `${subject.product.name} (${subject.source})`;
|
|
50
|
-
return subject.appUrl ?? subject.serve?.url ?? subject.source;
|
|
45
|
+
function subjectLine(appUrl, subject) {
|
|
46
|
+
const named = subjectName(appUrl, { source: subject.source, product: subject.product?.name });
|
|
47
|
+
return named !== "" ? named : (subject.appUrl ?? subject.serve?.url ?? subject.source);
|
|
51
48
|
}
|
|
52
49
|
/** A participant's status in words: a dry run's placeholder status says that nothing ran live. */
|
|
53
50
|
function participantStatus(status) {
|
|
@@ -107,7 +104,7 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
107
104
|
...runHeader(result, "scripted"),
|
|
108
105
|
`run: ${result.runId}`,
|
|
109
106
|
`actor: ${result.actor}`,
|
|
110
|
-
`subject: ${
|
|
107
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
111
108
|
...(result.scenario
|
|
112
109
|
? [
|
|
113
110
|
`scenario: ${result.scenario.id} @ ${result.scenario.sourceDigest.slice(0, 12)} (${result.scenario.source}, ${result.scenario.steps} step${result.scenario.steps === 1 ? "" : "s"})`,
|
|
@@ -133,7 +130,7 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
133
130
|
...runHeader(result, "computer-use"),
|
|
134
131
|
`run: ${result.runId}`,
|
|
135
132
|
`actor: ${result.actor}`,
|
|
136
|
-
`subject: ${
|
|
133
|
+
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
137
134
|
...(result.subject?.source === "clone"
|
|
138
135
|
? [
|
|
139
136
|
`repo: ${result.subject.repo}${result.subject.commit ? `@${result.subject.commit.slice(0, 12)}` : ""}${result.subject.envNames && result.subject.envNames.length > 0 ? ` env=[${result.subject.envNames.join(", ")}]` : ""}`,
|
|
@@ -39,7 +39,9 @@ export function sharedWorldRouteRun(args) {
|
|
|
39
39
|
topology: "shared-world",
|
|
40
40
|
topologyMode: "concurrent",
|
|
41
41
|
roleCount: participantList(args.config)?.length ?? 0,
|
|
42
|
-
|
|
42
|
+
// Refused before it planned: the declared cap, else every participant.
|
|
43
|
+
concurrency: args.config.execution?.concurrency ??
|
|
44
|
+
Math.max(1, participantList(args.config)?.length ?? 0),
|
|
43
45
|
dryRun,
|
|
44
46
|
runId: runId ?? args.options.runId ?? "not-created",
|
|
45
47
|
roles: [],
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../analysis/automatic-config.js";
|
|
2
1
|
import { resolve } from "node:path";
|
|
3
2
|
import { Command } from "commander";
|
|
4
3
|
import { deriveRunFacts } from "../telemetry.js";
|
|
5
4
|
import { resolveStudyManifest } from "../../study/discover.js";
|
|
5
|
+
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget, } from "../../analysis/automatic-config.js";
|
|
6
6
|
import { warnAndQueue } from "../deprecations.js";
|
|
7
7
|
import { planStudy, resolveStudyDryRun } from "../../study/plan.js";
|
|
8
8
|
import { keyNamesOf } from "../../study/requirements.js";
|
package/dist/cli/doctor.js
CHANGED
|
@@ -4,7 +4,7 @@ import { existsSync } from "node:fs";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { DOCTOR_KEYS, probeKeySources, } from "../keys/key-resolution.js";
|
|
6
6
|
import { desktopPeerAdvice, humanishInstall, } from "../substrates/e2b/peer-install.js";
|
|
7
|
-
import { describeSandboxCeiling, sandboxCeiling } from "../substrates/e2b/lifetime.js";
|
|
7
|
+
import { concurrentSandboxes, describeConcurrentSandboxes, describeSandboxCeiling, sandboxCeiling, } from "../substrates/e2b/lifetime.js";
|
|
8
8
|
import { nodeSupportsTui, terminalSurfaceMessage, TUI_BUNDLE_URL } from "../tui/contract.js";
|
|
9
9
|
import { detectLocalAgents, localAgentDoctorMessage, localAgentReady, NO_LOCAL_AGENT_MESSAGE, } from "../actors/local-agent/cli.js";
|
|
10
10
|
import { leftoverClaudeTranscripts } from "../actors/local-agent/claude-participant.js";
|
|
@@ -97,6 +97,7 @@ export async function doctor(cwdInput, options = {}) {
|
|
|
97
97
|
...(await projectChecks(projectRoot)),
|
|
98
98
|
await desktopSdkCheck(setup),
|
|
99
99
|
sandboxCeilingCheck(env),
|
|
100
|
+
concurrentSandboxesCheck(env),
|
|
100
101
|
terminalSurfaceCheck(),
|
|
101
102
|
...versionCheck(env),
|
|
102
103
|
...localAgentChecks(agents),
|
|
@@ -198,6 +199,15 @@ function sandboxCeilingCheck(env) {
|
|
|
198
199
|
const ceiling = sandboxCeiling(env);
|
|
199
200
|
return { name: "e2b sandbox ceiling", ok: ceiling.ok, message: describeSandboxCeiling(ceiling) };
|
|
200
201
|
}
|
|
202
|
+
/** How many sandboxes the E2B planners run at once, and where that number came from. */
|
|
203
|
+
function concurrentSandboxesCheck(env) {
|
|
204
|
+
const limit = concurrentSandboxes(env);
|
|
205
|
+
return {
|
|
206
|
+
name: "e2b concurrent sandboxes",
|
|
207
|
+
ok: limit.ok,
|
|
208
|
+
message: describeConcurrentSandboxes(limit),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
201
211
|
/**
|
|
202
212
|
* The optional peer dep every live browser and terminal route needs. `npx -y humanish` does not
|
|
203
213
|
* pull optional peers, so without this check an adopter's first live run fails on it, safely and at
|
|
@@ -6,6 +6,7 @@ import { homedir, hostname } from "node:os";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { isRecord } from "../run/type-guards.js";
|
|
8
8
|
import { isReceivingProviderId } from "./providers.js";
|
|
9
|
+
import { MAX_RECEIVING_INBOXES, } from "./receiving-types.js";
|
|
9
10
|
const SCHEMA = "humanish.comms-lease-journal.v1";
|
|
10
11
|
const MAX_BYTES = 1_048_576;
|
|
11
12
|
const UUID = /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/;
|
|
@@ -117,7 +118,7 @@ function parseJournal(text) {
|
|
|
117
118
|
!safeString(value.updatedAt) ||
|
|
118
119
|
!Array.isArray(value.leases) ||
|
|
119
120
|
value.leases.length < 1 ||
|
|
120
|
-
value.leases.length >
|
|
121
|
+
value.leases.length > MAX_RECEIVING_INBOXES)
|
|
121
122
|
fail();
|
|
122
123
|
const participants = new Set();
|
|
123
124
|
const leaseIds = new Set();
|
|
@@ -470,7 +471,7 @@ export class CommsLeaseStore {
|
|
|
470
471
|
!KEY_ENV.test(options.apiKeyEnv) ||
|
|
471
472
|
!validReceivingIdentity(options.identity) ||
|
|
472
473
|
options.participants.length < 1 ||
|
|
473
|
-
options.participants.length >
|
|
474
|
+
options.participants.length > MAX_RECEIVING_INBOXES ||
|
|
474
475
|
new Set(options.participants).size !== options.participants.length ||
|
|
475
476
|
options.participants.some((id) => !LOCAL_ID.test(id)))
|
|
476
477
|
fail();
|
|
@@ -3,6 +3,8 @@ import type { ReceivingProviderId } from "./providers.js";
|
|
|
3
3
|
import type { CommsInlineImage } from "./types.js";
|
|
4
4
|
/** authenticate() rejects with this code when the credential cannot acquire fresh inboxes. */
|
|
5
5
|
export declare const RECEIVING_SCOPE_UNSUPPORTED = "comms_scope_unsupported";
|
|
6
|
+
/** The most inboxes one run leases, one per participant; the lease journal holds this many. */
|
|
7
|
+
export declare const MAX_RECEIVING_INBOXES = 64;
|
|
6
8
|
export interface ReceivingContext {
|
|
7
9
|
signal?: AbortSignal;
|
|
8
10
|
timeoutMs?: number;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
1
|
/** authenticate() rejects with this code when the credential cannot acquire fresh inboxes. */
|
|
2
2
|
export const RECEIVING_SCOPE_UNSUPPORTED = "comms_scope_unsupported";
|
|
3
|
+
/** The most inboxes one run leases, one per participant; the lease journal holds this many. */
|
|
4
|
+
export const MAX_RECEIVING_INBOXES = 64;
|
|
3
5
|
export const COMMS_RECEIVING_SCHEMA = "humanish.comms-receiving.v2";
|