crysk 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +3 -2
- package/skills/crysk-ci-setup/assets/workflow.yml +3 -0
- package/src/asset-manifest.ts +11 -0
- package/src/assets.ts +1 -1
- package/src/ci/credential-refresh.ts +524 -0
- package/src/ci/crysk-secrets.ts +20 -6
- package/src/ci/force-comment.ts +14 -1
- package/src/ci/gates.ts +64 -21
- package/src/ci/outcome.ts +165 -15
- package/src/ci/platform-estimate.ts +111 -0
- package/src/ci/platform-trace.ts +30 -0
- package/src/ci/platform.ts +183 -5
- package/src/ci/reporter.ts +14 -4
- package/src/ci/setup.ts +96 -6
- package/src/ci/trace-bundle.ts +231 -27
- package/src/ci/trace-transcripts.ts +65 -0
- package/src/ci/trigger.ts +20 -4
- package/src/commands/ci-setup.ts +12 -3
- package/src/commands/doctor.ts +25 -23
- package/src/diversity/identity.ts +11 -1
- package/src/diversity/pipeline-integration.ts +19 -6
- package/src/doctor.ts +133 -4
- package/src/execution/contracts.ts +15 -3
- package/src/harness/materiality-judge.ts +153 -0
- package/src/harness/pipeline-plan.ts +133 -0
- package/src/harness/replay-rows.ts +172 -0
- package/src/harness/route-runner.ts +288 -0
- package/src/harness/single-step.ts +153 -0
- package/src/index.ts +1 -1
- package/src/model/catalog.ts +10 -7
- package/src/model/model-name.ts +14 -0
- package/src/model/profiles/claude.ts +34 -0
- package/src/model/profiles/openai.ts +44 -0
- package/src/model/profiles.ts +213 -0
- package/src/model/routing.ts +155 -50
- package/src/pr/ci-admission-gate.ts +2 -0
- package/src/pr/discovery.ts +18 -0
- package/src/pr/plan.ts +41 -2
- package/src/pr/review-pr.ts +43 -6
- package/src/pr/target.ts +67 -15
- package/src/production-runtime.ts +100 -13
- package/src/review/pipeline.ts +193 -58
- package/src/review/preflight.ts +91 -9
- package/src/review/prompt-set.ts +68 -0
- package/src/review/report.ts +4 -0
- package/src/review/review.ts +21 -11
- package/src/review/route-preflight.ts +117 -67
- package/src/review/run.ts +50 -4
- package/src/security/credential-broker.ts +21 -9
- package/src/security/oauth-providers.ts +101 -0
- package/src/transport-registry.ts +33 -0
- package/src/transports/opencode-sdk.ts +33 -1
- package/src/ui/config-edit.ts +9 -0
- package/src/ui/config.ts +1 -0
- package/src/ui/menu.ts +9 -2
- package/src/ui/plan.ts +8 -3
- package/src/updater.ts +21 -13
- package/src/wizard.ts +23 -0
package/src/doctor.ts
CHANGED
|
@@ -1,17 +1,25 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import os from "node:os";
|
|
3
3
|
import path from "node:path";
|
|
4
|
-
import {
|
|
4
|
+
import { loadEffectiveConfig } from "#config/config";
|
|
5
|
+
import { type ModelsConfig, routingNeedsOpenCode } from "#model/model-name";
|
|
6
|
+
import {
|
|
7
|
+
FALLBACK_PROFILE,
|
|
8
|
+
type ModelProfile,
|
|
9
|
+
resolveProfile,
|
|
10
|
+
} from "#model/profiles";
|
|
5
11
|
import {
|
|
6
12
|
exactBindingCapabilityIssues,
|
|
7
13
|
type ProviderCapabilityReport,
|
|
8
14
|
} from "#model/provider-capabilities";
|
|
15
|
+
import { buildResolvedRoutePlan, planRoutes } from "#model/routing";
|
|
9
16
|
import {
|
|
10
17
|
agentFilePath,
|
|
11
18
|
GOTCHAS_PLACEHOLDER_MARKER,
|
|
12
19
|
gotchasUnusableReason,
|
|
13
20
|
localReviewSpec,
|
|
14
21
|
resolveAgentsDirSetting,
|
|
22
|
+
resolveMaterialityGate,
|
|
15
23
|
} from "#review/preflight";
|
|
16
24
|
import {
|
|
17
25
|
REVIEW_MD_FILE_NAME,
|
|
@@ -30,6 +38,7 @@ import {
|
|
|
30
38
|
resolveEngineAssets,
|
|
31
39
|
selfInvocation,
|
|
32
40
|
} from "./assets";
|
|
41
|
+
import { readBrandEnv } from "./brand";
|
|
33
42
|
import { REPO_DIR_NAME, resolveHomeDir, resolveRepoDir } from "./brand-paths";
|
|
34
43
|
import {
|
|
35
44
|
gitIsTracked,
|
|
@@ -93,6 +102,108 @@ export interface RunDoctorOptions {
|
|
|
93
102
|
// Legacy injectable ProviderCapabilityReport. Ignored when
|
|
94
103
|
// probeExactBindings is provided.
|
|
95
104
|
produceCapabilityReport?: () => Promise<ProviderCapabilityReport>;
|
|
105
|
+
// Whether any step the configuration resolves runs on the Claude CLI
|
|
106
|
+
// (reviewNeedsClaude). Absent: required, as every caller that does not
|
|
107
|
+
// resolve the configuration (the updater, the menu) has always had it.
|
|
108
|
+
claudeRequired?: boolean;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
// Whether a review under this configuration sends any step to the Claude CLI
|
|
112
|
+
// (model-profiles S6): the route plan a review would build, every hunter of
|
|
113
|
+
// the bundled spec, the refuter, consolidate, scout, summarizer and, with the
|
|
114
|
+
// gate on, the materiality judge, resolved through `models` and the active
|
|
115
|
+
// profile. True whenever the answer is not proven: neither `models` nor a
|
|
116
|
+
// `profile`, a plan that fails to build, or a bad CRYSK_EXTRA_HUNTERS. So zero-config and
|
|
117
|
+
// every configuration naming a Claude model keep the CLI required.
|
|
118
|
+
export function reviewNeedsClaude(input: {
|
|
119
|
+
models: ModelsConfig | undefined;
|
|
120
|
+
// The config's `profile` (openai-profile S6). Absent: the fallback profile.
|
|
121
|
+
profile?: string;
|
|
122
|
+
// Whether a custom agents_dir applies (PR #521 R003). Its frontmatter
|
|
123
|
+
// outranks the profile's seat and is not read here, so with no
|
|
124
|
+
// `models.default` the answer stays "Claude required".
|
|
125
|
+
customAgents?: boolean;
|
|
126
|
+
materialityGate?: boolean;
|
|
127
|
+
env?: Record<string, string | undefined>;
|
|
128
|
+
}): boolean {
|
|
129
|
+
if (input.models === undefined && input.profile === undefined) return true;
|
|
130
|
+
if (input.customAgents === true && input.models?.default === undefined) {
|
|
131
|
+
return true;
|
|
132
|
+
}
|
|
133
|
+
try {
|
|
134
|
+
const plan = buildResolvedRoutePlan({
|
|
135
|
+
agents: localReviewSpec(input.env ?? process.env).agents,
|
|
136
|
+
models: input.models,
|
|
137
|
+
profile: resolveProfile(input.profile),
|
|
138
|
+
bundledAgents: true,
|
|
139
|
+
summarizer: { bundled: true },
|
|
140
|
+
scout: { bundled: true },
|
|
141
|
+
consolidate: { bundled: true },
|
|
142
|
+
// Unknown counts as on: the judge then joins only when it shares a
|
|
143
|
+
// backend with another step or `models.judge` names it.
|
|
144
|
+
...(input.materialityGate === false
|
|
145
|
+
? {}
|
|
146
|
+
: { materiality: true as const }),
|
|
147
|
+
});
|
|
148
|
+
// Fallbacks included: a step that falls back to Claude can start the CLI.
|
|
149
|
+
return planRoutes(plan).some(
|
|
150
|
+
(step) => step.route.backend === "claude-code",
|
|
151
|
+
);
|
|
152
|
+
} catch {
|
|
153
|
+
return true;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// What a doctor run reads from the configuration: whether any step needs the
|
|
158
|
+
// Claude CLI, and the models and profile its binding probe resolves
|
|
159
|
+
// (openai-profile S6). One reading for `crysk doctor`, `crysk update`, the
|
|
160
|
+
// menu and the wizard, so none of them asks a person who chose a profile with
|
|
161
|
+
// no Claude step for the Claude CLI. A config that fails to load keeps Claude
|
|
162
|
+
// required on the fallback profile, and `error` says why.
|
|
163
|
+
export interface DoctorRouting {
|
|
164
|
+
readonly claudeRequired: boolean;
|
|
165
|
+
readonly models?: ModelsConfig;
|
|
166
|
+
readonly profile: ModelProfile;
|
|
167
|
+
readonly warnings: readonly string[];
|
|
168
|
+
readonly error?: string;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export async function loadDoctorRouting(input: {
|
|
172
|
+
home: string;
|
|
173
|
+
repoRoot?: string | undefined;
|
|
174
|
+
configFlag?: string | undefined;
|
|
175
|
+
// The environment the AGENTS_DIR override is read from. Absent: the process's.
|
|
176
|
+
env?: Record<string, string | undefined>;
|
|
177
|
+
}): Promise<DoctorRouting> {
|
|
178
|
+
try {
|
|
179
|
+
const loaded = await loadEffectiveConfig({
|
|
180
|
+
root: input.repoRoot,
|
|
181
|
+
home: input.home,
|
|
182
|
+
configFlag: input.configFlag,
|
|
183
|
+
});
|
|
184
|
+
const { models, profile } = loaded.effective;
|
|
185
|
+
return {
|
|
186
|
+
claudeRequired: reviewNeedsClaude({
|
|
187
|
+
models,
|
|
188
|
+
...(profile === undefined ? {} : { profile }),
|
|
189
|
+
customAgents:
|
|
190
|
+
loaded.effective.agents_dir !== undefined ||
|
|
191
|
+
(readBrandEnv(input.env ?? process.env, "AGENTS_DIR")?.value ??
|
|
192
|
+
"") !== "",
|
|
193
|
+
materialityGate: resolveMaterialityGate(loaded.effective),
|
|
194
|
+
}),
|
|
195
|
+
...(models === undefined ? {} : { models }),
|
|
196
|
+
profile: resolveProfile(profile),
|
|
197
|
+
warnings: loaded.warnings,
|
|
198
|
+
};
|
|
199
|
+
} catch (error) {
|
|
200
|
+
return {
|
|
201
|
+
claudeRequired: true,
|
|
202
|
+
profile: FALLBACK_PROFILE,
|
|
203
|
+
warnings: [],
|
|
204
|
+
error: error instanceof Error ? error.message : String(error),
|
|
205
|
+
};
|
|
206
|
+
}
|
|
96
207
|
}
|
|
97
208
|
|
|
98
209
|
// Remediation hints for the report's known non-blocking codes; blocking
|
|
@@ -162,9 +273,13 @@ function pushProviderIssues(
|
|
|
162
273
|
|
|
163
274
|
export function openCodeRoutingSdkCheck(input: {
|
|
164
275
|
models: ModelsConfig | undefined;
|
|
276
|
+
// The configuration's profile (openai-profile S6). Absent: the fallback.
|
|
277
|
+
profile?: ModelProfile;
|
|
165
278
|
sdkVersion: string | undefined;
|
|
166
279
|
}): DoctorCheckItem | undefined {
|
|
167
|
-
if (!
|
|
280
|
+
if (!routingNeedsOpenCode(input.models, input.profile ?? FALLBACK_PROFILE)) {
|
|
281
|
+
return undefined;
|
|
282
|
+
}
|
|
168
283
|
const version =
|
|
169
284
|
typeof input.sdkVersion === "string" ? input.sdkVersion.trim() : "";
|
|
170
285
|
if (version === SUPPORTED_OPENCODE_SDK_VERSION) return undefined;
|
|
@@ -254,8 +369,22 @@ export async function runDoctor(
|
|
|
254
369
|
});
|
|
255
370
|
}
|
|
256
371
|
|
|
257
|
-
// Claude (required + auth)
|
|
258
|
-
|
|
372
|
+
// Claude (required + auth). Required unless the caller resolved the
|
|
373
|
+
// configuration and found no step on the Claude CLI (model-profiles S6,
|
|
374
|
+
// reviewNeedsClaude): then a missing or signed-out CLI is reported, but it
|
|
375
|
+
// blocks nothing, because no review would start it.
|
|
376
|
+
const claudeRequired = options.claudeRequired !== false;
|
|
377
|
+
if (!claudeRequired && !(tools.claude.installed && tools.claude.authOk)) {
|
|
378
|
+
checks.push({
|
|
379
|
+
name: "claude",
|
|
380
|
+
severity: "healthy",
|
|
381
|
+
message: `${
|
|
382
|
+
tools.claude.installed
|
|
383
|
+
? `Claude Code CLI is installed (${tools.claude.version ?? "unknown"}) but not authenticated`
|
|
384
|
+
: "Claude Code CLI is not installed"
|
|
385
|
+
}; not needed, since no configured step runs on it`,
|
|
386
|
+
});
|
|
387
|
+
} else if (!tools.claude.installed) {
|
|
259
388
|
checks.push({
|
|
260
389
|
name: "claude",
|
|
261
390
|
severity: "blocking",
|
|
@@ -237,12 +237,24 @@ export interface TransportOutcome {
|
|
|
237
237
|
readonly exitCode?: number;
|
|
238
238
|
}
|
|
239
239
|
|
|
240
|
-
// The
|
|
241
|
-
//
|
|
242
|
-
|
|
240
|
+
// The secret a failed attempt ran on (see TransportOutcome.failureCredential).
|
|
241
|
+
//
|
|
242
|
+
// A Claude route names the Claude CLI's own variable; in the action the CI texts map it
|
|
243
|
+
// to the CRYSK_CLAUDE_* secret that filled it. The CI record mirrors this list in
|
|
244
|
+
// src/ci/outcome.ts, tied by `satisfies`.
|
|
245
|
+
export type ClaudeFailureCredential =
|
|
243
246
|
| "ANTHROPIC_API_KEY"
|
|
244
247
|
| "ANTHROPIC_AUTH_TOKEN"
|
|
245
248
|
| "CLAUDE_CODE_OAUTH_TOKEN";
|
|
249
|
+
// An OpenCode route names its CRYSK_OPENCODE_* secret directly (S7): the name is per
|
|
250
|
+
// provider, built by opencodeSecretName (src/ci/crysk-secrets.ts) from the route's
|
|
251
|
+
// provider and credential kind. The CI record checks it against that grammar.
|
|
252
|
+
export type OpenCodeFailureCredential =
|
|
253
|
+
| `CRYSK_OPENCODE_${string}_API_KEY`
|
|
254
|
+
| `CRYSK_OPENCODE_${string}_OAUTH`;
|
|
255
|
+
export type FailureCredential =
|
|
256
|
+
| ClaudeFailureCredential
|
|
257
|
+
| OpenCodeFailureCredential;
|
|
246
258
|
|
|
247
259
|
export type TransportFailureCause =
|
|
248
260
|
| "auth_invalid"
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// The materiality judge's one spawn for the probes (scripts/materiality-spawn.ts,
|
|
2
|
+
// shared by the exam, the noise baseline and the lab's judgeClaim), on the
|
|
3
|
+
// route seam (odd/tasks/route-plan-harness.md S4). The judge's route comes
|
|
4
|
+
// from a plan through the production path, and every outcome records it.
|
|
5
|
+
//
|
|
6
|
+
// Isolation: tools [], empty MCP config, a fresh tmp dir per spawn, which is
|
|
7
|
+
// also the workspace admission names. Opus by default (S6): the hunters run
|
|
8
|
+
// sonnet, so the judge never grades its own model's output, and the lab's
|
|
9
|
+
// opus-x3 experiment assumes it. Never throws: a refused plan, a failed
|
|
10
|
+
// admission, spawn or parse comes back as `label: null` with the error, so one
|
|
11
|
+
// bad claim never discards judgments already paid for.
|
|
12
|
+
import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import path from "node:path";
|
|
15
|
+
import {
|
|
16
|
+
JUDGE_SYSTEM,
|
|
17
|
+
type MaterialityLabel,
|
|
18
|
+
parseJudgeVerdict,
|
|
19
|
+
renderJudgePrompt,
|
|
20
|
+
} from "#compare/materiality";
|
|
21
|
+
import { defaultEffortForRole } from "#model/catalog";
|
|
22
|
+
import { MATERIALITY_STEP_KEY, type ResolvedRoutePlan } from "#model/routing";
|
|
23
|
+
import type { StepRunner } from "#review/step-runner";
|
|
24
|
+
import type { ResolvedModelRoute } from "../execution/contracts";
|
|
25
|
+
import { resolveRunClaudeCredential } from "../security/claude-credential";
|
|
26
|
+
import {
|
|
27
|
+
buildHarnessRoutePlan,
|
|
28
|
+
type HarnessIdentity,
|
|
29
|
+
harnessIdentity,
|
|
30
|
+
prepareHarnessRunner,
|
|
31
|
+
routeStep,
|
|
32
|
+
} from "./route-runner";
|
|
33
|
+
|
|
34
|
+
// A `models.judge` entry, so it must be on the profile's measured list.
|
|
35
|
+
export const MATERIALITY_JUDGE_CHOICE = "claude:opus-latest";
|
|
36
|
+
|
|
37
|
+
export function materialityJudgePlan(
|
|
38
|
+
judge: string = MATERIALITY_JUDGE_CHOICE,
|
|
39
|
+
): ResolvedRoutePlan {
|
|
40
|
+
return buildHarnessRoutePlan({
|
|
41
|
+
materiality: true,
|
|
42
|
+
models: { judge: [judge] },
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// The judge's identity, for a results file's header. Pure: no spawn.
|
|
47
|
+
export function materialityJudgeIdentity(
|
|
48
|
+
judge: string = MATERIALITY_JUDGE_CHOICE,
|
|
49
|
+
): HarnessIdentity {
|
|
50
|
+
return harnessIdentity(materialityJudgePlan(judge));
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface JudgeOutcome {
|
|
54
|
+
label: MaterialityLabel | null;
|
|
55
|
+
reason?: string;
|
|
56
|
+
error?: string;
|
|
57
|
+
cost: number;
|
|
58
|
+
// The route the judge was planned on, absent only when no plan was built.
|
|
59
|
+
route?: ResolvedModelRoute;
|
|
60
|
+
routeFingerprint?: string;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface PreparedJudge {
|
|
64
|
+
runner: StepRunner;
|
|
65
|
+
dispose(): Promise<void>;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface MaterialityJudgeOptions {
|
|
69
|
+
judge?: string;
|
|
70
|
+
// Test seam: prepares the runner for the plan in the spawn's workspace.
|
|
71
|
+
prepare?: (
|
|
72
|
+
workspaceRoot: string,
|
|
73
|
+
plan: ResolvedRoutePlan,
|
|
74
|
+
) => Promise<PreparedJudge>;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const prepareLive = (workspaceRoot: string, plan: ResolvedRoutePlan) =>
|
|
78
|
+
prepareHarnessRunner({
|
|
79
|
+
workspaceRoot,
|
|
80
|
+
buildRoutePlan: () => plan,
|
|
81
|
+
claudeRun: resolveRunClaudeCredential({}),
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
export async function runMaterialityJudge(
|
|
85
|
+
input: { name: string; prTitle: string; claim: string },
|
|
86
|
+
options: MaterialityJudgeOptions = {},
|
|
87
|
+
): Promise<JudgeOutcome> {
|
|
88
|
+
let identity: HarnessIdentity | undefined;
|
|
89
|
+
const tmp = mkdtempSync(path.join(tmpdir(), "materiality-judge-"));
|
|
90
|
+
let prepared: PreparedJudge | undefined;
|
|
91
|
+
try {
|
|
92
|
+
const plan = materialityJudgePlan(options.judge);
|
|
93
|
+
identity = harnessIdentity(plan);
|
|
94
|
+
const recorded = {
|
|
95
|
+
route: identity.route,
|
|
96
|
+
routeFingerprint: identity.routeFingerprint,
|
|
97
|
+
};
|
|
98
|
+
const systemPromptPath = path.join(tmp, "system.md");
|
|
99
|
+
const mcpConfigPath = path.join(tmp, "mcp.json");
|
|
100
|
+
writeFileSync(systemPromptPath, JUDGE_SYSTEM);
|
|
101
|
+
writeFileSync(mcpConfigPath, JSON.stringify({ mcpServers: {} }));
|
|
102
|
+
chmodSync(systemPromptPath, 0o600);
|
|
103
|
+
prepared = await (options.prepare ?? prepareLive)(tmp, plan);
|
|
104
|
+
const result = await prepared.runner.run(
|
|
105
|
+
routeStep(plan, MATERIALITY_STEP_KEY, {
|
|
106
|
+
name: input.name,
|
|
107
|
+
systemPromptPath,
|
|
108
|
+
prompt: renderJudgePrompt(input.prTitle, input.claim),
|
|
109
|
+
tools: [],
|
|
110
|
+
mcpConfigPath,
|
|
111
|
+
model: identity.route.modelSnapshot,
|
|
112
|
+
effort: defaultEffortForRole("other"),
|
|
113
|
+
cwd: tmp,
|
|
114
|
+
outPath: path.join(tmp, "out.json"),
|
|
115
|
+
timeoutMs: 3 * 60 * 1000,
|
|
116
|
+
maxAttempts: 2,
|
|
117
|
+
parse: parseJudgeVerdict,
|
|
118
|
+
}),
|
|
119
|
+
);
|
|
120
|
+
const cost = result.usage.cost_usd_est;
|
|
121
|
+
if (result.status !== "ok" || result.output === undefined) {
|
|
122
|
+
return {
|
|
123
|
+
label: null,
|
|
124
|
+
error: result.stderrTail.slice(-300),
|
|
125
|
+
cost,
|
|
126
|
+
...recorded,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
const verdict = result.output as ReturnType<typeof parseJudgeVerdict>;
|
|
130
|
+
return {
|
|
131
|
+
label: verdict.label,
|
|
132
|
+
...(verdict.reason === undefined ? {} : { reason: verdict.reason }),
|
|
133
|
+
cost,
|
|
134
|
+
...recorded,
|
|
135
|
+
};
|
|
136
|
+
} catch (error) {
|
|
137
|
+
return {
|
|
138
|
+
label: null,
|
|
139
|
+
error: error instanceof Error ? error.message : String(error),
|
|
140
|
+
cost: 0,
|
|
141
|
+
...(identity === undefined
|
|
142
|
+
? {}
|
|
143
|
+
: {
|
|
144
|
+
route: identity.route,
|
|
145
|
+
routeFingerprint: identity.routeFingerprint,
|
|
146
|
+
}),
|
|
147
|
+
};
|
|
148
|
+
} finally {
|
|
149
|
+
// The runtime first: it may still hold the workspace it was admitted on.
|
|
150
|
+
await prepared?.dispose().catch(() => undefined);
|
|
151
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// The route plan a whole-pipeline harness (refuter-probe, fixture-eval)
|
|
2
|
+
// passes runPipeline (odd/tasks/route-plan-harness.md S4). Today such a
|
|
3
|
+
// harness passes no plan, and runPipeline builds one for itself but runs the
|
|
4
|
+
// runner it was given, so a ClaudeCodeRunner beside an OpenCode route runs
|
|
5
|
+
// silently on Claude. With this plan the harness prepares the runner through
|
|
6
|
+
// the seam (prepareHarnessRunner) and runPipeline binds every step to its
|
|
7
|
+
// route.
|
|
8
|
+
//
|
|
9
|
+
// The plan is built from the SAME input runPipeline receives, with the same
|
|
10
|
+
// inputs runPipeline's own no-plan build reads (pipeline.ts, "Step 2.5"), so
|
|
11
|
+
// what a harness measured before is what it measures now (S6): the plan's
|
|
12
|
+
// fingerprint equals the one a no-plan run records. buildCliRoutePlan is the
|
|
13
|
+
// CLI's plan and differs on purpose (bundled root prompts, consolidation on),
|
|
14
|
+
// so a harness that mirrors a lab-style run cannot use it.
|
|
15
|
+
import { readFile } from "node:fs/promises";
|
|
16
|
+
import path from "node:path";
|
|
17
|
+
import { buildResolvedRoutePlan, type ResolvedRoutePlan } from "#model/routing";
|
|
18
|
+
import type { PipelineInput } from "#review/pipeline";
|
|
19
|
+
import { parseAgentFile } from "#review/prompt-set";
|
|
20
|
+
import { defaultReviewSpec, validateReviewSpec } from "#review/spec";
|
|
21
|
+
import type { ResolvedModelRoute } from "../execution/contracts";
|
|
22
|
+
import { HarnessRouteError, harnessModelChoice } from "./route-runner";
|
|
23
|
+
|
|
24
|
+
// Every PipelineInput field runPipeline reads to build its own plan.
|
|
25
|
+
export type PipelinePlanInput = Pick<
|
|
26
|
+
PipelineInput,
|
|
27
|
+
| "spec"
|
|
28
|
+
| "agentsDir"
|
|
29
|
+
| "bundledAgents"
|
|
30
|
+
| "modelProfile"
|
|
31
|
+
| "model"
|
|
32
|
+
| "models"
|
|
33
|
+
| "config"
|
|
34
|
+
| "summarizer"
|
|
35
|
+
| "scout"
|
|
36
|
+
| "consolidate"
|
|
37
|
+
| "materiality"
|
|
38
|
+
>;
|
|
39
|
+
|
|
40
|
+
async function rootFrontmatter(
|
|
41
|
+
stage: { promptPath: string; bundled?: boolean } | undefined,
|
|
42
|
+
): Promise<string | undefined> {
|
|
43
|
+
if (stage === undefined || stage.bundled === true) return undefined;
|
|
44
|
+
return (await parseAgentFile(stage.promptPath)).model;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Unlike runPipeline, an unreadable prompt fails here: the harness learns of
|
|
48
|
+
// it before admission rather than from a plan that silently lacks a model.
|
|
49
|
+
export async function buildPipelineHarnessPlan(
|
|
50
|
+
input: PipelinePlanInput,
|
|
51
|
+
): Promise<ResolvedRoutePlan> {
|
|
52
|
+
const spec = validateReviewSpec(input.spec ?? defaultReviewSpec());
|
|
53
|
+
// A diversity run routes its expanded legs (diversityCtx.routeAgents), which
|
|
54
|
+
// this plan does not build; runPipeline would refuse it as a missing route.
|
|
55
|
+
if (spec.multiModelDiversity?.enabled === true) {
|
|
56
|
+
throw new HarnessRouteError(
|
|
57
|
+
"a multi-model diversity spec routes its expanded legs; a pipeline harness plan does not build them",
|
|
58
|
+
);
|
|
59
|
+
}
|
|
60
|
+
const frontmatter = new Map<string, string | undefined>();
|
|
61
|
+
for (const agent of spec.agents) {
|
|
62
|
+
const parsed = await parseAgentFile(path.join(input.agentsDir, agent.file));
|
|
63
|
+
frontmatter.set(agent.key, parsed.model);
|
|
64
|
+
}
|
|
65
|
+
const models = input.models ?? input.config?.effective?.models;
|
|
66
|
+
const stage = async (
|
|
67
|
+
value:
|
|
68
|
+
| { promptPath: string; model?: string; bundled?: boolean }
|
|
69
|
+
| undefined,
|
|
70
|
+
) =>
|
|
71
|
+
value === undefined
|
|
72
|
+
? undefined
|
|
73
|
+
: {
|
|
74
|
+
model: value.model,
|
|
75
|
+
frontmatterModel: await rootFrontmatter(value),
|
|
76
|
+
bundled: value.bundled === true,
|
|
77
|
+
};
|
|
78
|
+
const summarizer = await stage(input.summarizer);
|
|
79
|
+
const scout = await stage(input.scout);
|
|
80
|
+
const consolidate = await stage(input.consolidate);
|
|
81
|
+
return buildResolvedRoutePlan({
|
|
82
|
+
agents: spec.agents,
|
|
83
|
+
cliModel: harnessModelChoice(input.model),
|
|
84
|
+
models,
|
|
85
|
+
...(input.modelProfile === undefined
|
|
86
|
+
? {}
|
|
87
|
+
: { profile: input.modelProfile }),
|
|
88
|
+
...(input.bundledAgents === true ? { bundledAgents: true } : {}),
|
|
89
|
+
frontmatterModel: (agentKey) => frontmatter.get(agentKey),
|
|
90
|
+
...(summarizer === undefined ? {} : { summarizer }),
|
|
91
|
+
...(scout === undefined ? {} : { scout }),
|
|
92
|
+
...(consolidate === undefined ? {} : { consolidate }),
|
|
93
|
+
...(input.materiality === undefined ? {} : { materiality: true as const }),
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface PipelineRouteIdentity {
|
|
98
|
+
readonly route: ResolvedModelRoute;
|
|
99
|
+
readonly routeFingerprint: string;
|
|
100
|
+
readonly stepRouteFingerprint: string;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
interface RecordedPlan {
|
|
104
|
+
route_fingerprint?: string;
|
|
105
|
+
route_plan?: {
|
|
106
|
+
steps?: Array<{
|
|
107
|
+
stepKey: string;
|
|
108
|
+
route: ResolvedModelRoute;
|
|
109
|
+
routeFingerprint: string;
|
|
110
|
+
}>;
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// The identity of the route a pipeline run executed for one step, read back
|
|
115
|
+
// from the run's own pipeline.json, never from the harness's intent.
|
|
116
|
+
export async function readPipelineRouteIdentity(
|
|
117
|
+
runDir: string,
|
|
118
|
+
stepKey: string,
|
|
119
|
+
): Promise<PipelineRouteIdentity> {
|
|
120
|
+
const file = path.join(runDir, "pipeline.json");
|
|
121
|
+
const recorded = JSON.parse(await readFile(file, "utf8")) as RecordedPlan;
|
|
122
|
+
const step = recorded.route_plan?.steps?.find((s) => s.stepKey === stepKey);
|
|
123
|
+
if (step === undefined || recorded.route_fingerprint === undefined) {
|
|
124
|
+
throw new HarnessRouteError(
|
|
125
|
+
`${file} records no route for step "${stepKey}"`,
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
route: step.route,
|
|
130
|
+
routeFingerprint: recorded.route_fingerprint,
|
|
131
|
+
stepRouteFingerprint: step.routeFingerprint,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// The rows and resume rules of refuter-replay and consolidate-replay
|
|
2
|
+
// (odd/tasks/route-plan-harness.md S3, S6), here so the gates and offline
|
|
3
|
+
// tests cover them; the scripts keep only their CLI wiring. Every row that ran
|
|
4
|
+
// a step records the model string it was given and the route identity of the
|
|
5
|
+
// step it ran. Resume accepts rows written before the seam, keyed by the model
|
|
6
|
+
// string alone, so no paid row is orphaned.
|
|
7
|
+
import type { PlanEntry } from "#review/consolidate";
|
|
8
|
+
import type { RefuterResult } from "#review/drafts";
|
|
9
|
+
import type { Finding } from "#review/findings";
|
|
10
|
+
import type { StepResult } from "#review/step-runner";
|
|
11
|
+
import {
|
|
12
|
+
type RouteIdentityFields,
|
|
13
|
+
ranOnRoute,
|
|
14
|
+
routeIdentityFields,
|
|
15
|
+
type SingleStepRoute,
|
|
16
|
+
} from "./single-step";
|
|
17
|
+
|
|
18
|
+
export interface RefuterReplayCase {
|
|
19
|
+
readonly run: string;
|
|
20
|
+
readonly finding: string;
|
|
21
|
+
readonly expect: "keep" | "drop";
|
|
22
|
+
readonly label: string;
|
|
23
|
+
// "refuted": the finding lives in findings.json debug.refuted, not in the
|
|
24
|
+
// posted list (the run deleted it). Its id may collide with a posted one.
|
|
25
|
+
readonly source?: "posted" | "refuted";
|
|
26
|
+
readonly note?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// The source is part of the key: a refuted finding's id may collide with a
|
|
30
|
+
// posted one, and the two must not resume each other. The model is not: which
|
|
31
|
+
// route a row ran on is ranOnRoute's question.
|
|
32
|
+
export function refuterReplayCaseKey(c: {
|
|
33
|
+
readonly run: string;
|
|
34
|
+
readonly finding: string;
|
|
35
|
+
readonly source?: string;
|
|
36
|
+
}): string {
|
|
37
|
+
return `${c.run}|${c.finding}|${c.source ?? "posted"}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
// The case keys already answered on this route in an existing --out. A failed
|
|
41
|
+
// step (quota, auth, timeout) is not an answer: it reruns.
|
|
42
|
+
export function refuterReplayDone(
|
|
43
|
+
jsonl: string,
|
|
44
|
+
route: SingleStepRoute,
|
|
45
|
+
): Set<string> {
|
|
46
|
+
const done = new Set<string>();
|
|
47
|
+
for (const line of jsonl.split("\n")) {
|
|
48
|
+
if (!line.trim()) continue;
|
|
49
|
+
const row = JSON.parse(line) as {
|
|
50
|
+
run: string;
|
|
51
|
+
finding: string;
|
|
52
|
+
source?: string;
|
|
53
|
+
model?: string;
|
|
54
|
+
step_route_fingerprint?: string;
|
|
55
|
+
status: string;
|
|
56
|
+
};
|
|
57
|
+
if (row.status === "ok" && ranOnRoute(row, route)) {
|
|
58
|
+
done.add(refuterReplayCaseKey(row));
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return done;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type RefuterPromptCheck = "match" | "unrecorded" | "candidate";
|
|
65
|
+
|
|
66
|
+
export function refuterReplayRow(input: {
|
|
67
|
+
readonly replayCase: RefuterReplayCase;
|
|
68
|
+
readonly finding: Finding;
|
|
69
|
+
readonly route: SingleStepRoute;
|
|
70
|
+
readonly result: StepResult;
|
|
71
|
+
readonly promptCheck: RefuterPromptCheck;
|
|
72
|
+
}) {
|
|
73
|
+
const { replayCase: c, finding: f, route, result } = input;
|
|
74
|
+
// A failed step reads as `inconclusive`, exactly as runRefuter treats it.
|
|
75
|
+
const entry =
|
|
76
|
+
result.status === "ok"
|
|
77
|
+
? (result.output as RefuterResult).results[0]
|
|
78
|
+
: undefined;
|
|
79
|
+
return {
|
|
80
|
+
run: c.run,
|
|
81
|
+
finding: f.id,
|
|
82
|
+
source: c.source ?? "posted",
|
|
83
|
+
model: route.model,
|
|
84
|
+
...routeIdentityFields(route.identity),
|
|
85
|
+
severity: f.severity,
|
|
86
|
+
tier: f.tier,
|
|
87
|
+
original_verdict: c.source === "refuted" ? "refuted" : f.refuter_verdict,
|
|
88
|
+
verdict: entry?.outcome ?? "inconclusive",
|
|
89
|
+
refuter_proof_refs: entry?.proof_refs ?? [],
|
|
90
|
+
status: result.status,
|
|
91
|
+
denial: result.denialCode ?? null,
|
|
92
|
+
stderr_tail:
|
|
93
|
+
result.status === "ok" ? null : (result.stderrTail ?? "").slice(-400),
|
|
94
|
+
expect: c.expect,
|
|
95
|
+
label: c.label,
|
|
96
|
+
prompt_check: input.promptCheck,
|
|
97
|
+
notional_usd: result.usageV2?.notionalCostUsd ?? null,
|
|
98
|
+
wall_ms: result.usageV2?.wallMs ?? null,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// consolidate-replay pins its experiment in config.json, because its audit is
|
|
103
|
+
// keyed by PR label only. The pin keeps the shape it had before the seam, so
|
|
104
|
+
// an existing --out keeps resuming; the model matches on its route, so another
|
|
105
|
+
// spelling of the same model is the same experiment.
|
|
106
|
+
export interface ConsolidateReplayConfig {
|
|
107
|
+
readonly arm: string;
|
|
108
|
+
readonly tool: string;
|
|
109
|
+
readonly model: string;
|
|
110
|
+
readonly variant: string;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export function consolidateConfigMatches(
|
|
114
|
+
seen: Partial<Record<keyof ConsolidateReplayConfig, unknown>>,
|
|
115
|
+
current: ConsolidateReplayConfig,
|
|
116
|
+
route: SingleStepRoute,
|
|
117
|
+
): boolean {
|
|
118
|
+
return (
|
|
119
|
+
seen.arm === current.arm &&
|
|
120
|
+
seen.tool === current.tool &&
|
|
121
|
+
seen.variant === current.variant &&
|
|
122
|
+
ranOnRoute({ model: seen.model }, route)
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export type ConsolidateStepRow = {
|
|
127
|
+
readonly label: string;
|
|
128
|
+
readonly status: "ok" | "fallback";
|
|
129
|
+
readonly reason?: string;
|
|
130
|
+
readonly plan?: PlanEntry[];
|
|
131
|
+
readonly in: number;
|
|
132
|
+
readonly out: number;
|
|
133
|
+
readonly notional_usd: number | null;
|
|
134
|
+
readonly model: string;
|
|
135
|
+
} & RouteIdentityFields;
|
|
136
|
+
|
|
137
|
+
// The audit row of a PR whose consolidation step ran. A failed step is a
|
|
138
|
+
// fallback that keeps every finding, and is retried on resume.
|
|
139
|
+
export function consolidateStepRow(input: {
|
|
140
|
+
readonly label: string;
|
|
141
|
+
readonly ids: readonly string[];
|
|
142
|
+
readonly route: SingleStepRoute;
|
|
143
|
+
readonly result: StepResult;
|
|
144
|
+
}): ConsolidateStepRow {
|
|
145
|
+
const { label, ids, route, result } = input;
|
|
146
|
+
const ran = {
|
|
147
|
+
model: route.model,
|
|
148
|
+
...routeIdentityFields(route.identity),
|
|
149
|
+
};
|
|
150
|
+
const notional_usd = result.usageV2?.notionalCostUsd ?? null;
|
|
151
|
+
if (result.status !== "ok") {
|
|
152
|
+
return {
|
|
153
|
+
label,
|
|
154
|
+
status: "fallback",
|
|
155
|
+
reason: (result.stderrTail ?? "").slice(-300),
|
|
156
|
+
in: ids.length,
|
|
157
|
+
out: ids.length,
|
|
158
|
+
notional_usd,
|
|
159
|
+
...ran,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
const plan = result.output as PlanEntry[];
|
|
163
|
+
return {
|
|
164
|
+
label,
|
|
165
|
+
status: "ok",
|
|
166
|
+
plan,
|
|
167
|
+
in: ids.length,
|
|
168
|
+
out: plan.length,
|
|
169
|
+
notional_usd,
|
|
170
|
+
...ran,
|
|
171
|
+
};
|
|
172
|
+
}
|