crysk 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/package.json +3 -2
- package/skills/crysk-ci-setup/assets/workflow.yml +3 -0
- package/src/asset-manifest.ts +11 -0
- package/src/assets.ts +1 -1
- package/src/ci/credential-refresh.ts +524 -0
- package/src/ci/crysk-secrets.ts +20 -6
- package/src/ci/outcome.ts +165 -15
- package/src/ci/platform.ts +155 -7
- package/src/ci/setup.ts +64 -6
- package/src/ci/trigger.ts +20 -4
- package/src/commands/ci-setup.ts +12 -3
- package/src/commands/doctor.ts +25 -23
- package/src/diversity/identity.ts +11 -1
- package/src/diversity/pipeline-integration.ts +19 -6
- package/src/doctor.ts +133 -4
- package/src/execution/contracts.ts +15 -3
- package/src/harness/materiality-judge.ts +153 -0
- package/src/harness/pipeline-plan.ts +133 -0
- package/src/harness/replay-rows.ts +172 -0
- package/src/harness/route-runner.ts +288 -0
- package/src/harness/single-step.ts +153 -0
- package/src/index.ts +1 -1
- package/src/model/catalog.ts +10 -7
- package/src/model/model-name.ts +14 -0
- package/src/model/profiles/claude.ts +34 -0
- package/src/model/profiles/openai.ts +44 -0
- package/src/model/profiles.ts +213 -0
- package/src/model/routing.ts +155 -50
- package/src/pr/plan.ts +6 -1
- package/src/pr/review-pr.ts +32 -6
- package/src/pr/target.ts +16 -7
- package/src/production-runtime.ts +100 -13
- package/src/review/pipeline.ts +193 -58
- package/src/review/preflight.ts +91 -9
- package/src/review/prompt-set.ts +68 -0
- package/src/review/review.ts +21 -11
- package/src/review/route-preflight.ts +117 -67
- package/src/review/run.ts +34 -4
- package/src/security/credential-broker.ts +21 -9
- package/src/security/oauth-providers.ts +101 -0
- package/src/transport-registry.ts +33 -0
- package/src/transports/opencode-sdk.ts +33 -1
- package/src/ui/config-edit.ts +9 -0
- package/src/ui/config.ts +1 -0
- package/src/ui/menu.ts +9 -2
- package/src/updater.ts +21 -13
- package/src/wizard.ts +23 -0
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
// The one seam probes, replays and benchmarks run steps through
|
|
2
|
+
// (odd/tasks/route-plan-harness.md S1). It replaces `new ClaudeCodeRunner`
|
|
3
|
+
// in harness code: a harness builds a route plan for the steps it runs,
|
|
4
|
+
// prepares the runner exactly as a review does (production admission, then
|
|
5
|
+
// prepareRunnerForRoute), gives each hand-built step its route, and records
|
|
6
|
+
// the identity of the route it ran. Measuring another profile or provider is
|
|
7
|
+
// then a different plan, never a harness change.
|
|
8
|
+
//
|
|
9
|
+
// Nothing here routes, admits or binds on its own: every decision is the
|
|
10
|
+
// production helper's. What the seam adds is refusal. Production has three
|
|
11
|
+
// silent ways onto the Claude CLI that a harness must never take: a plan that
|
|
12
|
+
// fails to build falls back to the no-plan path, prepareRunnerForRoute builds
|
|
13
|
+
// a ClaudeCodeRunner without a plan, and MultiProviderRunner sends a spec with
|
|
14
|
+
// no route to its Claude binding. Each is a refusal here, so a harness
|
|
15
|
+
// measuring an OpenCode model can never record a Claude run under its name.
|
|
16
|
+
import type { ModelsConfig } from "#model/model-name";
|
|
17
|
+
import { FALLBACK_PROFILE, profileRoute } from "#model/profiles";
|
|
18
|
+
import {
|
|
19
|
+
type BuildRoutePlanInput,
|
|
20
|
+
buildResolvedRoutePlan,
|
|
21
|
+
MATERIALITY_STEP_KEY,
|
|
22
|
+
type ResolvedRoutePlan,
|
|
23
|
+
} from "#model/routing";
|
|
24
|
+
import {
|
|
25
|
+
type PipelineDeps,
|
|
26
|
+
routeForStepKey,
|
|
27
|
+
stepEffortFields,
|
|
28
|
+
stepRouteFields,
|
|
29
|
+
} from "#review/pipeline";
|
|
30
|
+
import {
|
|
31
|
+
type ProductionRoutePlanResult,
|
|
32
|
+
resolveProductionRoutePlanAtConfirm,
|
|
33
|
+
} from "#review/route-preflight";
|
|
34
|
+
import { prepareRunnerForRoute } from "#review/run";
|
|
35
|
+
import type { StepRunner, StepSpec } from "#review/step-runner";
|
|
36
|
+
import type { ResolvedModelRoute } from "../execution/contracts";
|
|
37
|
+
import type { ProductionRuntime } from "../production-runtime";
|
|
38
|
+
|
|
39
|
+
export class HarnessRouteError extends Error {
|
|
40
|
+
override name = "HarnessRouteError";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// buildResolvedRoutePlan's input with every step optional and the profile
|
|
44
|
+
// defaulting to `claude`, as a review's does. A single-step harness names its
|
|
45
|
+
// one step: `agents: [{ key: "refuter", role: "refuter" }]`, `scout: {...}`,
|
|
46
|
+
// `materiality: true`. `bundledAgents` (and `bundled` on a root prompt) gives
|
|
47
|
+
// the step the profile seat; a lab prompt set passes `frontmatterModel`
|
|
48
|
+
// instead, and its frontmatter wins as in production.
|
|
49
|
+
export type HarnessRoutePlanInput = Partial<BuildRoutePlanInput>;
|
|
50
|
+
|
|
51
|
+
// A full config's `hunter.<key>` roles for hunters this plan does not run. A
|
|
52
|
+
// review refuses them (they name no hunter in that review), but a harness
|
|
53
|
+
// running one step from a production config runs a subset by design.
|
|
54
|
+
function modelsForAgents(
|
|
55
|
+
models: ModelsConfig | undefined,
|
|
56
|
+
agents: BuildRoutePlanInput["agents"],
|
|
57
|
+
): ModelsConfig | undefined {
|
|
58
|
+
if (models === undefined) return undefined;
|
|
59
|
+
const hunters = new Set(
|
|
60
|
+
agents
|
|
61
|
+
.filter((agent) => agent.role === "hunter")
|
|
62
|
+
.map((agent) => agent.specialty ?? agent.key),
|
|
63
|
+
);
|
|
64
|
+
return Object.fromEntries(
|
|
65
|
+
Object.entries(models).filter(
|
|
66
|
+
([role]) =>
|
|
67
|
+
!role.startsWith("hunter.") ||
|
|
68
|
+
hunters.has(role.slice("hunter.".length)),
|
|
69
|
+
),
|
|
70
|
+
);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// The route plan for the steps a harness runs. A whole-pipeline harness uses
|
|
74
|
+
// buildCliRoutePlan (#review/route-preflight) instead, the review's own plan.
|
|
75
|
+
// A bare pinned Claude id (`claude-sonnet-4-5-20250929`) reached the Claude
|
|
76
|
+
// CLI as given before harnesses ran on route plans, and the lab's recipes still
|
|
77
|
+
// pass one. Production's `--model` refuses that spelling, so a harness names it
|
|
78
|
+
// `claude:<id>` for the user: the same model, now through a route.
|
|
79
|
+
const BARE_CLAUDE_ID_RE = /^claude-[a-z0-9.-]+[a-z0-9]$/;
|
|
80
|
+
|
|
81
|
+
export function harnessModelChoice(
|
|
82
|
+
model: string | undefined,
|
|
83
|
+
): string | undefined {
|
|
84
|
+
return model !== undefined && BARE_CLAUDE_ID_RE.test(model)
|
|
85
|
+
? `claude:${model}`
|
|
86
|
+
: model;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function buildHarnessRoutePlan(
|
|
90
|
+
input: HarnessRoutePlanInput,
|
|
91
|
+
): ResolvedRoutePlan {
|
|
92
|
+
const agents = input.agents ?? [];
|
|
93
|
+
const profile = input.profile ?? FALLBACK_PROFILE;
|
|
94
|
+
let models = modelsForAgents(input.models, agents);
|
|
95
|
+
// Production adds the judge only beside a step that shares its credential,
|
|
96
|
+
// so a judge alone would silently vanish. A harness that asks for the judge
|
|
97
|
+
// runs it: with no choice, on the profile's judge seat.
|
|
98
|
+
if (input.materiality === true && models?.judge === undefined) {
|
|
99
|
+
models = { ...models, judge: [profileRoute(profile, "judge").model] };
|
|
100
|
+
}
|
|
101
|
+
const cliModel = harnessModelChoice(input.cliModel);
|
|
102
|
+
const plan = buildResolvedRoutePlan({
|
|
103
|
+
...input,
|
|
104
|
+
...(cliModel === undefined ? {} : { cliModel }),
|
|
105
|
+
agents,
|
|
106
|
+
profile,
|
|
107
|
+
...(models === undefined ? {} : { models }),
|
|
108
|
+
});
|
|
109
|
+
if (
|
|
110
|
+
input.materiality === true &&
|
|
111
|
+
!plan.steps.some((step) => step.stepKey === MATERIALITY_STEP_KEY)
|
|
112
|
+
) {
|
|
113
|
+
throw new HarnessRouteError(
|
|
114
|
+
`models.judge names no model the materiality judge was measured on (${(models?.judge ?? []).join(", ")}); a harness never runs the judge on another`,
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
if (plan.steps.length === 0) {
|
|
118
|
+
throw new HarnessRouteError(
|
|
119
|
+
"the harness route plan has no step: name the agents or stages it runs",
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
return plan;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function planStep(plan: ResolvedRoutePlan, stepKey: string) {
|
|
126
|
+
const step = plan.steps.find((s) => s.stepKey === stepKey);
|
|
127
|
+
if (step === undefined) {
|
|
128
|
+
throw new HarnessRouteError(
|
|
129
|
+
`no route for step "${stepKey}": the plan routes ${plan.steps.map((s) => s.stepKey).join(", ")}`,
|
|
130
|
+
);
|
|
131
|
+
}
|
|
132
|
+
return step;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// A hand-built step with its route, fallbacks and efforts from the plan,
|
|
136
|
+
// through the pipeline's own stepRouteFields and stepEffortFields, so it binds
|
|
137
|
+
// and runs exactly as a review's step. The spec's effort is the level a
|
|
138
|
+
// review takes from the prompt's frontmatter or the role default: a Claude
|
|
139
|
+
// `models` entry's effort outranks it, as in production.
|
|
140
|
+
export function routeStep(
|
|
141
|
+
plan: ResolvedRoutePlan,
|
|
142
|
+
stepKey: string,
|
|
143
|
+
spec: StepSpec,
|
|
144
|
+
): StepSpec {
|
|
145
|
+
if (routeForStepKey(plan, stepKey) === undefined) planStep(plan, stepKey);
|
|
146
|
+
return {
|
|
147
|
+
...spec,
|
|
148
|
+
...stepRouteFields(plan, stepKey),
|
|
149
|
+
...stepEffortFields(plan, stepKey, spec.effort, spec.effort),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export interface HarnessIdentity {
|
|
154
|
+
// The step's primary route in the plan, the one its binding was frozen on.
|
|
155
|
+
readonly route: ResolvedModelRoute;
|
|
156
|
+
// The plan's fingerprint: what pipeline.json records as route_fingerprint,
|
|
157
|
+
// so a single-step row and a pipeline run compare on the same field.
|
|
158
|
+
readonly routeFingerprint: string;
|
|
159
|
+
// The step's own route fingerprint (route only, no effort).
|
|
160
|
+
readonly stepRouteFingerprint: string;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// The identity a harness row records. A single-step plan needs no step key.
|
|
164
|
+
// It is the plan's primary model: a row whose step fell back to a later
|
|
165
|
+
// model must say so from the step's own result.
|
|
166
|
+
export function harnessIdentity(
|
|
167
|
+
plan: ResolvedRoutePlan,
|
|
168
|
+
stepKey?: string,
|
|
169
|
+
): HarnessIdentity {
|
|
170
|
+
if (stepKey === undefined && plan.steps.length !== 1) {
|
|
171
|
+
throw new HarnessRouteError(
|
|
172
|
+
`the plan routes ${plan.steps.length} steps (${plan.steps.map((s) => s.stepKey).join(", ")}): name the step whose identity to record`,
|
|
173
|
+
);
|
|
174
|
+
}
|
|
175
|
+
const step = stepKey === undefined ? plan.steps[0] : planStep(plan, stepKey);
|
|
176
|
+
if (step === undefined) throw new HarnessRouteError("the plan has no step");
|
|
177
|
+
if (plan.routeFingerprint === undefined) {
|
|
178
|
+
throw new HarnessRouteError(
|
|
179
|
+
"the route plan has no fingerprint: build it with createResolvedRoutePlan",
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
route: step.route,
|
|
184
|
+
routeFingerprint: plan.routeFingerprint,
|
|
185
|
+
stepRouteFingerprint: step.routeFingerprint,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
// A runner that refuses a spec with no route instead of letting the
|
|
190
|
+
// production runner send it to its Claude binding.
|
|
191
|
+
export function requireRoutedRunner(inner: StepRunner): StepRunner {
|
|
192
|
+
return {
|
|
193
|
+
run: async (spec) => {
|
|
194
|
+
if (spec.route === undefined) {
|
|
195
|
+
throw new HarnessRouteError(
|
|
196
|
+
`step "${spec.name}" has no route: give it one with routeStep before running it`,
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
return inner.run(spec);
|
|
200
|
+
},
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
type AdmissionInput = Parameters<typeof resolveProductionRoutePlanAtConfirm>[0];
|
|
205
|
+
|
|
206
|
+
export interface PrepareHarnessRunnerInput {
|
|
207
|
+
readonly workspaceRoot: string;
|
|
208
|
+
// buildHarnessRoutePlan for a single-step harness, buildCliRoutePlan for a
|
|
209
|
+
// whole pipeline. Called once, before admission, so its error surfaces.
|
|
210
|
+
readonly buildRoutePlan: () => ResolvedRoutePlan | Promise<ResolvedRoutePlan>;
|
|
211
|
+
// Passed through to production admission as given. A live harness passes
|
|
212
|
+
// the run's Claude credential (resolveRunClaudeCredential) as a review
|
|
213
|
+
// does; `checkCredentials: false` is a dry run's, never a default here.
|
|
214
|
+
readonly claudeRun?: AdmissionInput["claudeRun"];
|
|
215
|
+
readonly env?: AdmissionInput["env"];
|
|
216
|
+
readonly checkCredentials?: boolean;
|
|
217
|
+
// The production helpers' own test seams, passed through unchanged.
|
|
218
|
+
readonly authorityDeps?: AdmissionInput["authorityDeps"];
|
|
219
|
+
readonly loadSdk?: AdmissionInput["loadSdk"];
|
|
220
|
+
readonly ceilingController?: AbortController;
|
|
221
|
+
readonly onProgress?: PipelineDeps["onProgress"];
|
|
222
|
+
readonly maxConcurrentSteps?: number;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export interface HarnessRunner {
|
|
226
|
+
readonly plan: ResolvedRoutePlan;
|
|
227
|
+
// The production runner behind requireRoutedRunner.
|
|
228
|
+
readonly runner: StepRunner;
|
|
229
|
+
// runPipeline's second argument, on the same guarded runner.
|
|
230
|
+
readonly deps: PipelineDeps;
|
|
231
|
+
readonly runtime: ProductionRuntime;
|
|
232
|
+
readonly productionAdmission: ProductionRoutePlanResult["productionAdmission"];
|
|
233
|
+
// The caller's `finally` runs it, as a review's does with the runtime.
|
|
234
|
+
dispose(): Promise<void>;
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export async function prepareHarnessRunner(
|
|
238
|
+
input: PrepareHarnessRunnerInput,
|
|
239
|
+
): Promise<HarnessRunner> {
|
|
240
|
+
const plan = await input.buildRoutePlan();
|
|
241
|
+
const admitted = await resolveProductionRoutePlanAtConfirm({
|
|
242
|
+
// A built plan cannot fail to build, so production's legacy fallback is
|
|
243
|
+
// unreachable; `true` keeps it so if this ever changes.
|
|
244
|
+
modelsConfigured: true,
|
|
245
|
+
workspaceRoot: input.workspaceRoot,
|
|
246
|
+
buildRoutePlan: async () => plan,
|
|
247
|
+
...(input.claudeRun === undefined ? {} : { claudeRun: input.claudeRun }),
|
|
248
|
+
...(input.env === undefined ? {} : { env: input.env }),
|
|
249
|
+
...(input.checkCredentials === undefined
|
|
250
|
+
? {}
|
|
251
|
+
: { checkCredentials: input.checkCredentials }),
|
|
252
|
+
...(input.authorityDeps === undefined
|
|
253
|
+
? {}
|
|
254
|
+
: { authorityDeps: input.authorityDeps }),
|
|
255
|
+
...(input.loadSdk === undefined ? {} : { loadSdk: input.loadSdk }),
|
|
256
|
+
});
|
|
257
|
+
if (admitted === undefined) {
|
|
258
|
+
throw new HarnessRouteError("production admission returned no route plan");
|
|
259
|
+
}
|
|
260
|
+
const ceilingController = input.ceilingController ?? new AbortController();
|
|
261
|
+
const prepared = await prepareRunnerForRoute({
|
|
262
|
+
routePlan: admitted.routePlan,
|
|
263
|
+
productionAdmission: admitted.productionAdmission,
|
|
264
|
+
workspaceRoot: input.workspaceRoot,
|
|
265
|
+
// Read only on the legacy path, which a harness never takes (below).
|
|
266
|
+
runnerAuthority: { error: "a harness never runs the legacy Claude runner" },
|
|
267
|
+
ceilingController,
|
|
268
|
+
onProgress: input.onProgress,
|
|
269
|
+
...(input.maxConcurrentSteps === undefined
|
|
270
|
+
? {}
|
|
271
|
+
: { maxConcurrentSteps: input.maxConcurrentSteps }),
|
|
272
|
+
});
|
|
273
|
+
const runtime = prepared.productionRuntime;
|
|
274
|
+
if (runtime === undefined) {
|
|
275
|
+
throw new HarnessRouteError(
|
|
276
|
+
"no production runtime was prepared; a harness never falls back to the legacy Claude runner",
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
const runner = requireRoutedRunner(runtime.runner);
|
|
280
|
+
return {
|
|
281
|
+
plan: admitted.routePlan,
|
|
282
|
+
runner,
|
|
283
|
+
deps: { ...prepared.deps, runner },
|
|
284
|
+
runtime,
|
|
285
|
+
productionAdmission: admitted.productionAdmission,
|
|
286
|
+
dispose: () => runtime.dispose(),
|
|
287
|
+
};
|
|
288
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// Single-step harnesses on the route seam (odd/tasks/route-plan-harness.md S3,
|
|
2
|
+
// S6): refuter-replay, scout-probe, consolidate-replay, live-micro-eval and
|
|
3
|
+
// martian-judge each run one step. Each builds a one-step plan from the model
|
|
4
|
+
// it measures, prepares the runner through route-runner.ts, routes its spec,
|
|
5
|
+
// and records the route identity in its rows.
|
|
6
|
+
//
|
|
7
|
+
// The model is always the harness's own explicit choice, passed as the step's
|
|
8
|
+
// `--model` (cliModel): a bare alias (`sonnet`) is `claude:sonnet-latest`, the
|
|
9
|
+
// route the harness ran before the seam, and `opencode:<provider>/<model>`
|
|
10
|
+
// measures an OpenCode model through the same flag. Passing it explicitly
|
|
11
|
+
// means a profile or `models` change can never move what a harness measures.
|
|
12
|
+
import type { StepResult, StepSpec } from "#review/step-runner";
|
|
13
|
+
import type { ResolvedModelRoute } from "../execution/contracts";
|
|
14
|
+
import {
|
|
15
|
+
buildHarnessRoutePlan,
|
|
16
|
+
type HarnessIdentity,
|
|
17
|
+
type HarnessRoutePlanInput,
|
|
18
|
+
harnessIdentity,
|
|
19
|
+
harnessModelChoice,
|
|
20
|
+
type PrepareHarnessRunnerInput,
|
|
21
|
+
prepareHarnessRunner,
|
|
22
|
+
routeStep,
|
|
23
|
+
} from "./route-runner";
|
|
24
|
+
|
|
25
|
+
// A production step a harness replays, or a harness-only step with no
|
|
26
|
+
// production role (the Martian judge, the trivial micro-eval) under its own
|
|
27
|
+
// key. A harness-only step has no profile seat or `models` role: only the
|
|
28
|
+
// harness's model routes it.
|
|
29
|
+
export type SingleStep =
|
|
30
|
+
| "refuter"
|
|
31
|
+
| "scout"
|
|
32
|
+
| "consolidate"
|
|
33
|
+
| { readonly key: string };
|
|
34
|
+
|
|
35
|
+
export function singleStepPlanInput(
|
|
36
|
+
step: SingleStep,
|
|
37
|
+
model: string,
|
|
38
|
+
): HarnessRoutePlanInput {
|
|
39
|
+
switch (step) {
|
|
40
|
+
case "refuter":
|
|
41
|
+
return {
|
|
42
|
+
agents: [{ key: "refuter", role: "refuter" }],
|
|
43
|
+
bundledAgents: true,
|
|
44
|
+
cliModel: model,
|
|
45
|
+
};
|
|
46
|
+
// These take the model as the stage's own, not cliModel, so the bare
|
|
47
|
+
// pinned id is spelled out here as buildHarnessRoutePlan does for cliModel.
|
|
48
|
+
case "scout":
|
|
49
|
+
return { scout: { model: harnessModelChoice(model) ?? model } };
|
|
50
|
+
case "consolidate":
|
|
51
|
+
return { consolidate: { model: harnessModelChoice(model) ?? model } };
|
|
52
|
+
default:
|
|
53
|
+
return { agents: [{ key: step.key, role: "other" }], cliModel: model };
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface SingleStepRoute {
|
|
58
|
+
readonly step: SingleStep;
|
|
59
|
+
// The model string the harness was given, as its rows record it.
|
|
60
|
+
readonly model: string;
|
|
61
|
+
readonly plan: ReturnType<typeof buildHarnessRoutePlan>;
|
|
62
|
+
readonly stepKey: string;
|
|
63
|
+
readonly identity: HarnessIdentity;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Pure: an invalid model fails here, before anything is admitted or spent.
|
|
67
|
+
export function singleStepRoute(
|
|
68
|
+
step: SingleStep,
|
|
69
|
+
model: string,
|
|
70
|
+
): SingleStepRoute {
|
|
71
|
+
const plan = buildHarnessRoutePlan(singleStepPlanInput(step, model));
|
|
72
|
+
const identity = harnessIdentity(plan);
|
|
73
|
+
const stepKey = plan.steps[0]?.stepKey as string;
|
|
74
|
+
return { step, model, plan, stepKey, identity };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// The identity a row records, under the field names pipeline.json uses.
|
|
78
|
+
export interface RouteIdentityFields {
|
|
79
|
+
readonly route: ResolvedModelRoute;
|
|
80
|
+
readonly route_fingerprint: string;
|
|
81
|
+
readonly step_route_fingerprint: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function routeIdentityFields(
|
|
85
|
+
identity: HarnessIdentity,
|
|
86
|
+
): RouteIdentityFields {
|
|
87
|
+
return {
|
|
88
|
+
route: identity.route,
|
|
89
|
+
route_fingerprint: identity.routeFingerprint,
|
|
90
|
+
step_route_fingerprint: identity.stepRouteFingerprint,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Whether a recorded row ran on the route this run measures, so a resume skips
|
|
95
|
+
// it. A row with a step route fingerprint matches on it alone. A row written
|
|
96
|
+
// before the seam carries only the model string it was keyed by: it matches
|
|
97
|
+
// that string, or any spelling of the same route (`sonnet` and
|
|
98
|
+
// `claude:sonnet-latest`), so no paid row is orphaned.
|
|
99
|
+
export function ranOnRoute(
|
|
100
|
+
row: { readonly model?: unknown; readonly step_route_fingerprint?: unknown },
|
|
101
|
+
current: SingleStepRoute,
|
|
102
|
+
): boolean {
|
|
103
|
+
const fingerprint = current.identity.stepRouteFingerprint;
|
|
104
|
+
if (typeof row.step_route_fingerprint === "string") {
|
|
105
|
+
return row.step_route_fingerprint === fingerprint;
|
|
106
|
+
}
|
|
107
|
+
if (typeof row.model !== "string") return false;
|
|
108
|
+
if (row.model === current.model) return true;
|
|
109
|
+
try {
|
|
110
|
+
return (
|
|
111
|
+
singleStepRoute(current.step, row.model).identity.stepRouteFingerprint ===
|
|
112
|
+
fingerprint
|
|
113
|
+
);
|
|
114
|
+
} catch {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export interface SingleStepRunner {
|
|
120
|
+
readonly route: SingleStepRoute;
|
|
121
|
+
// Routes the hand-built spec on the plan, then runs it on the guarded
|
|
122
|
+
// production runner.
|
|
123
|
+
run(spec: StepSpec): Promise<StepResult>;
|
|
124
|
+
// The caller's `finally` runs it.
|
|
125
|
+
dispose(): Promise<void>;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export type PrepareSingleStepRunnerInput = Omit<
|
|
129
|
+
PrepareHarnessRunnerInput,
|
|
130
|
+
"buildRoutePlan"
|
|
131
|
+
> & {
|
|
132
|
+
readonly route: SingleStepRoute;
|
|
133
|
+
// Test seam: prepareHarnessRunner unless given.
|
|
134
|
+
readonly prepare?: typeof prepareHarnessRunner;
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
// A live harness passes `claudeRun: resolveRunClaudeCredential({})`, the
|
|
138
|
+
// credential a review chooses at run start; the rest passes through.
|
|
139
|
+
export async function prepareSingleStepRunner(
|
|
140
|
+
input: PrepareSingleStepRunnerInput,
|
|
141
|
+
): Promise<SingleStepRunner> {
|
|
142
|
+
const { route, prepare = prepareHarnessRunner, ...rest } = input;
|
|
143
|
+
const prepared = await prepare({
|
|
144
|
+
...rest,
|
|
145
|
+
buildRoutePlan: () => route.plan,
|
|
146
|
+
});
|
|
147
|
+
return {
|
|
148
|
+
route,
|
|
149
|
+
run: (spec) =>
|
|
150
|
+
prepared.runner.run(routeStep(prepared.plan, route.stepKey, spec)),
|
|
151
|
+
dispose: () => prepared.dispose(),
|
|
152
|
+
};
|
|
153
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Public API surface; the deep-review lab consumes exactly these exports.
|
|
3
3
|
|
|
4
4
|
export const ENGINE_NAME = "pr-hero";
|
|
5
|
-
export const ENGINE_VERSION = "0.
|
|
5
|
+
export const ENGINE_VERSION = "0.6.0";
|
|
6
6
|
|
|
7
7
|
// errors and git/refs used to reach this surface only through re-exports in
|
|
8
8
|
// review/preflight.ts; they are exported directly now that those re-exports
|
package/src/model/catalog.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import anthropicCatalog from "../../config/models/anthropic.json";
|
|
2
2
|
import type { RunnerBackend } from "../execution/contracts";
|
|
3
|
+
import { FALLBACK_PROFILE, profileRoleEffort } from "./profiles";
|
|
3
4
|
|
|
4
5
|
// #175, 2026-09-02. This catalogue holds ALIAS NAMES and nothing else.
|
|
5
6
|
//
|
|
@@ -182,9 +183,11 @@ export type StepRole =
|
|
|
182
183
|
| "verifier"
|
|
183
184
|
| "other";
|
|
184
185
|
|
|
185
|
-
//
|
|
186
|
-
//
|
|
187
|
-
//
|
|
186
|
+
// The per-role effort table: the fallback when a prompt's frontmatter omits
|
|
187
|
+
// `effort:`, so a custom or lab prompt set still runs at an EXPLICIT level.
|
|
188
|
+
// The levels a profile assigns are read from the `claude` profile
|
|
189
|
+
// (src/model/profiles.ts), the one place they are chosen; `verifier` and
|
|
190
|
+
// `other` have no profile role and stay here.
|
|
188
191
|
//
|
|
189
192
|
// WHY the engine owns this at all (#299, 2026-10-02): pr-hero never passed
|
|
190
193
|
// `--effort`, so every step ran at the CLI's implicit default, which is `high`
|
|
@@ -208,10 +211,10 @@ export type StepRole =
|
|
|
208
211
|
// - everything else high: unmeasured, so it keeps what every model ran at
|
|
209
212
|
// before the 5.5 family, i.e. the behavior those roles were validated under.
|
|
210
213
|
const ROLE_EFFORT: Readonly<Record<StepRole, Effort>> = Object.freeze({
|
|
211
|
-
hunter: "
|
|
212
|
-
refuter: "
|
|
213
|
-
summarizer: "
|
|
214
|
-
scout: "
|
|
214
|
+
hunter: profileRoleEffort(FALLBACK_PROFILE, "hunter"),
|
|
215
|
+
refuter: profileRoleEffort(FALLBACK_PROFILE, "refuter"),
|
|
216
|
+
summarizer: profileRoleEffort(FALLBACK_PROFILE, "summary"),
|
|
217
|
+
scout: profileRoleEffort(FALLBACK_PROFILE, "scout"),
|
|
215
218
|
verifier: "high",
|
|
216
219
|
other: "high",
|
|
217
220
|
});
|
package/src/model/model-name.ts
CHANGED
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
isModelAlias,
|
|
30
30
|
type ModelAlias,
|
|
31
31
|
} from "./catalog";
|
|
32
|
+
import { type ModelProfile, profileModels } from "./profiles";
|
|
32
33
|
|
|
33
34
|
export class ModelNameError extends Error {}
|
|
34
35
|
|
|
@@ -402,3 +403,16 @@ export function modelsNeedOpenCode(models: ModelsConfig | undefined): boolean {
|
|
|
402
403
|
list.some((entry) => modelEntry(entry).name.startsWith(OPENCODE_PREFIX)),
|
|
403
404
|
);
|
|
404
405
|
}
|
|
406
|
+
|
|
407
|
+
// The same question for a whole configuration (openai-profile S6): `models`,
|
|
408
|
+
// or for a role `models` does not set (no `default`), the profile's routes.
|
|
409
|
+
export function routingNeedsOpenCode(
|
|
410
|
+
models: ModelsConfig | undefined,
|
|
411
|
+
profile: ModelProfile,
|
|
412
|
+
): boolean {
|
|
413
|
+
if (modelsNeedOpenCode(models)) return true;
|
|
414
|
+
if (models?.default !== undefined) return false;
|
|
415
|
+
return profileModels(profile).some((model) =>
|
|
416
|
+
model.startsWith(OPENCODE_PREFIX),
|
|
417
|
+
);
|
|
418
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { ModelProfile } from "../profiles";
|
|
2
|
+
|
|
3
|
+
// The `claude` profile: every step on Anthropic models through the Claude Code
|
|
4
|
+
// CLI. It is one profile among others; the registry (src/model/profiles.ts)
|
|
5
|
+
// also makes it the fallback, which is the only thing that sets it apart.
|
|
6
|
+
//
|
|
7
|
+
// Exactly what 4898e1b4 runs with no configuration, read off the bundled
|
|
8
|
+
// prompts' frontmatter (prompts/default/*.md, consolidate.md, summarizer.md),
|
|
9
|
+
// the old DEFAULT_SCOUT_MODEL / DEFAULT_CONSOLIDATE_MODEL /
|
|
10
|
+
// DEFAULT_CONSOLIDATE_EFFORT / DEFAULT_SUMMARY_MODEL / MATERIALITY_JUDGE_MODEL
|
|
11
|
+
// constants and ROLE_EFFORT (catalog.ts, where the #299 evidence for the
|
|
12
|
+
// hunter and refuter levels lives). Pinned by
|
|
13
|
+
// test/fixtures/model-profiles/baseline and preflight-bundled-prompts.test.ts.
|
|
14
|
+
export const CLAUDE_PROFILE: ModelProfile = Object.freeze({
|
|
15
|
+
id: "claude",
|
|
16
|
+
roles: Object.freeze({
|
|
17
|
+
hunter: Object.freeze({ model: "claude:sonnet-latest", effort: "xhigh" }),
|
|
18
|
+
refuter: Object.freeze({ model: "claude:opus-latest", effort: "high" }),
|
|
19
|
+
consolidate: Object.freeze({
|
|
20
|
+
model: "claude:sonnet-latest",
|
|
21
|
+
effort: "high",
|
|
22
|
+
}),
|
|
23
|
+
scout: Object.freeze({ model: "claude:sonnet-latest", effort: "high" }),
|
|
24
|
+
summary: Object.freeze({ model: "claude:haiku-latest", effort: "high" }),
|
|
25
|
+
judge: Object.freeze({ model: "claude:sonnet-latest", effort: "high" }),
|
|
26
|
+
}),
|
|
27
|
+
// The judge was measured on Claude sonnet and opus through the Claude Code
|
|
28
|
+
// CLI (same bundled prompt, same bar; both passed, lab bench/METRICS.md
|
|
29
|
+
// 2026-11-01); sonnet was chosen because it costs half for a difference
|
|
30
|
+
// within one run's noise.
|
|
31
|
+
measured: Object.freeze({
|
|
32
|
+
judgeModels: Object.freeze(["claude:sonnet-latest", "claude:opus-latest"]),
|
|
33
|
+
}),
|
|
34
|
+
} satisfies ModelProfile);
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import type { ModelProfile } from "../profiles";
|
|
2
|
+
|
|
3
|
+
// The `openai` profile: every step on OpenAI models through OpenCode, the same
|
|
4
|
+
// parallel hunters `claude` runs. Juanma's approved table (odd/tasks/
|
|
5
|
+
// openai-profile.md S2, "Asi si"): Sol for the refuter, Terra for the hunters,
|
|
6
|
+
// consolidate, scout and judge, Luna for the summary.
|
|
7
|
+
//
|
|
8
|
+
// An effort becomes the OpenCode variant (OpenAI's Responses variants are
|
|
9
|
+
// none..xhigh). A role with no effort sends no variant, so it runs at the
|
|
10
|
+
// provider's default level.
|
|
11
|
+
//
|
|
12
|
+
// Nothing was measured on these models, and no measurement is planned (S10),
|
|
13
|
+
// so no `promptSet` (the default prompts) and no `measured`. The materiality
|
|
14
|
+
// judge runs on Terra by Juanma's choice, not a measurement (S9, L8: "yo
|
|
15
|
+
// dejaria encendido el juez pero es opcional"). The gate stays opt-in
|
|
16
|
+
// (`materiality_gate`, off by default).
|
|
17
|
+
export const OPENAI_PROFILE: ModelProfile = Object.freeze({
|
|
18
|
+
id: "openai",
|
|
19
|
+
roles: Object.freeze({
|
|
20
|
+
hunter: Object.freeze({
|
|
21
|
+
model: "opencode:openai/gpt-6.1-terra",
|
|
22
|
+
effort: "xhigh",
|
|
23
|
+
}),
|
|
24
|
+
refuter: Object.freeze({
|
|
25
|
+
model: "opencode:openai/gpt-6.1-sol",
|
|
26
|
+
effort: "high",
|
|
27
|
+
}),
|
|
28
|
+
consolidate: Object.freeze({
|
|
29
|
+
model: "opencode:openai/gpt-6.1-terra",
|
|
30
|
+
effort: "high",
|
|
31
|
+
}),
|
|
32
|
+
scout: Object.freeze({ model: "opencode:openai/gpt-6.1-terra" }),
|
|
33
|
+
summary: Object.freeze({ model: "opencode:openai/gpt-6-luna" }),
|
|
34
|
+
judge: Object.freeze({ model: "opencode:openai/gpt-6.1-terra" }),
|
|
35
|
+
}),
|
|
36
|
+
// Terra first, the judge a plan with no `models.judge` runs. Then the two
|
|
37
|
+
// models the judge was measured on through Claude Code: this profile runs
|
|
38
|
+
// the same default prompts, so `models.judge` may still pick them.
|
|
39
|
+
judgeModels: Object.freeze([
|
|
40
|
+
"opencode:openai/gpt-6.1-terra",
|
|
41
|
+
"claude:sonnet-latest",
|
|
42
|
+
"claude:opus-latest",
|
|
43
|
+
]),
|
|
44
|
+
} satisfies ModelProfile);
|