crysk 0.5.1 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +2 -1
  2. package/package.json +3 -2
  3. package/skills/crysk-ci-setup/assets/workflow.yml +3 -0
  4. package/src/asset-manifest.ts +11 -0
  5. package/src/assets.ts +1 -1
  6. package/src/ci/credential-refresh.ts +524 -0
  7. package/src/ci/crysk-secrets.ts +20 -6
  8. package/src/ci/outcome.ts +165 -15
  9. package/src/ci/platform.ts +155 -7
  10. package/src/ci/setup.ts +64 -6
  11. package/src/ci/trigger.ts +20 -4
  12. package/src/commands/ci-setup.ts +12 -3
  13. package/src/commands/doctor.ts +25 -23
  14. package/src/diversity/identity.ts +11 -1
  15. package/src/diversity/pipeline-integration.ts +19 -6
  16. package/src/doctor.ts +133 -4
  17. package/src/execution/contracts.ts +15 -3
  18. package/src/harness/materiality-judge.ts +153 -0
  19. package/src/harness/pipeline-plan.ts +133 -0
  20. package/src/harness/replay-rows.ts +172 -0
  21. package/src/harness/route-runner.ts +288 -0
  22. package/src/harness/single-step.ts +153 -0
  23. package/src/index.ts +1 -1
  24. package/src/model/catalog.ts +10 -7
  25. package/src/model/model-name.ts +14 -0
  26. package/src/model/profiles/claude.ts +34 -0
  27. package/src/model/profiles/openai.ts +44 -0
  28. package/src/model/profiles.ts +213 -0
  29. package/src/model/routing.ts +155 -50
  30. package/src/pr/plan.ts +6 -1
  31. package/src/pr/review-pr.ts +32 -6
  32. package/src/pr/target.ts +16 -7
  33. package/src/production-runtime.ts +100 -13
  34. package/src/review/pipeline.ts +193 -58
  35. package/src/review/preflight.ts +91 -9
  36. package/src/review/prompt-set.ts +68 -0
  37. package/src/review/review.ts +21 -11
  38. package/src/review/route-preflight.ts +117 -67
  39. package/src/review/run.ts +34 -4
  40. package/src/security/credential-broker.ts +21 -9
  41. package/src/security/oauth-providers.ts +101 -0
  42. package/src/transport-registry.ts +33 -0
  43. package/src/transports/opencode-sdk.ts +33 -1
  44. package/src/ui/config-edit.ts +9 -0
  45. package/src/ui/config.ts +1 -0
  46. package/src/ui/menu.ts +9 -2
  47. package/src/updater.ts +21 -13
  48. package/src/wizard.ts +23 -0
@@ -0,0 +1,153 @@
1
+ // The materiality judge's one spawn for the probes (scripts/materiality-spawn.ts,
2
+ // shared by the exam, the noise baseline and the lab's judgeClaim), on the
3
+ // route seam (odd/tasks/route-plan-harness.md S4). The judge's route comes
4
+ // from a plan through the production path, and every outcome records it.
5
+ //
6
+ // Isolation: tools [], empty MCP config, a fresh tmp dir per spawn, which is
7
+ // also the workspace admission names. Opus by default (S6): the hunters run
8
+ // sonnet, so the judge never grades its own model's output, and the lab's
9
+ // opus-x3 experiment assumes it. Never throws: a refused plan, a failed
10
+ // admission, spawn or parse comes back as `label: null` with the error, so one
11
+ // bad claim never discards judgments already paid for.
12
+ import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
13
+ import { tmpdir } from "node:os";
14
+ import path from "node:path";
15
+ import {
16
+ JUDGE_SYSTEM,
17
+ type MaterialityLabel,
18
+ parseJudgeVerdict,
19
+ renderJudgePrompt,
20
+ } from "#compare/materiality";
21
+ import { defaultEffortForRole } from "#model/catalog";
22
+ import { MATERIALITY_STEP_KEY, type ResolvedRoutePlan } from "#model/routing";
23
+ import type { StepRunner } from "#review/step-runner";
24
+ import type { ResolvedModelRoute } from "../execution/contracts";
25
+ import { resolveRunClaudeCredential } from "../security/claude-credential";
26
+ import {
27
+ buildHarnessRoutePlan,
28
+ type HarnessIdentity,
29
+ harnessIdentity,
30
+ prepareHarnessRunner,
31
+ routeStep,
32
+ } from "./route-runner";
33
+
34
+ // A `models.judge` entry, so it must be on the profile's measured list.
35
+ export const MATERIALITY_JUDGE_CHOICE = "claude:opus-latest";
36
+
37
+ export function materialityJudgePlan(
38
+ judge: string = MATERIALITY_JUDGE_CHOICE,
39
+ ): ResolvedRoutePlan {
40
+ return buildHarnessRoutePlan({
41
+ materiality: true,
42
+ models: { judge: [judge] },
43
+ });
44
+ }
45
+
46
+ // The judge's identity, for a results file's header. Pure: no spawn.
47
+ export function materialityJudgeIdentity(
48
+ judge: string = MATERIALITY_JUDGE_CHOICE,
49
+ ): HarnessIdentity {
50
+ return harnessIdentity(materialityJudgePlan(judge));
51
+ }
52
+
53
+ export interface JudgeOutcome {
54
+ label: MaterialityLabel | null;
55
+ reason?: string;
56
+ error?: string;
57
+ cost: number;
58
+ // The route the judge was planned on, absent only when no plan was built.
59
+ route?: ResolvedModelRoute;
60
+ routeFingerprint?: string;
61
+ }
62
+
63
+ interface PreparedJudge {
64
+ runner: StepRunner;
65
+ dispose(): Promise<void>;
66
+ }
67
+
68
+ export interface MaterialityJudgeOptions {
69
+ judge?: string;
70
+ // Test seam: prepares the runner for the plan in the spawn's workspace.
71
+ prepare?: (
72
+ workspaceRoot: string,
73
+ plan: ResolvedRoutePlan,
74
+ ) => Promise<PreparedJudge>;
75
+ }
76
+
77
+ const prepareLive = (workspaceRoot: string, plan: ResolvedRoutePlan) =>
78
+ prepareHarnessRunner({
79
+ workspaceRoot,
80
+ buildRoutePlan: () => plan,
81
+ claudeRun: resolveRunClaudeCredential({}),
82
+ });
83
+
84
+ export async function runMaterialityJudge(
85
+ input: { name: string; prTitle: string; claim: string },
86
+ options: MaterialityJudgeOptions = {},
87
+ ): Promise<JudgeOutcome> {
88
+ let identity: HarnessIdentity | undefined;
89
+ const tmp = mkdtempSync(path.join(tmpdir(), "materiality-judge-"));
90
+ let prepared: PreparedJudge | undefined;
91
+ try {
92
+ const plan = materialityJudgePlan(options.judge);
93
+ identity = harnessIdentity(plan);
94
+ const recorded = {
95
+ route: identity.route,
96
+ routeFingerprint: identity.routeFingerprint,
97
+ };
98
+ const systemPromptPath = path.join(tmp, "system.md");
99
+ const mcpConfigPath = path.join(tmp, "mcp.json");
100
+ writeFileSync(systemPromptPath, JUDGE_SYSTEM);
101
+ writeFileSync(mcpConfigPath, JSON.stringify({ mcpServers: {} }));
102
+ chmodSync(systemPromptPath, 0o600);
103
+ prepared = await (options.prepare ?? prepareLive)(tmp, plan);
104
+ const result = await prepared.runner.run(
105
+ routeStep(plan, MATERIALITY_STEP_KEY, {
106
+ name: input.name,
107
+ systemPromptPath,
108
+ prompt: renderJudgePrompt(input.prTitle, input.claim),
109
+ tools: [],
110
+ mcpConfigPath,
111
+ model: identity.route.modelSnapshot,
112
+ effort: defaultEffortForRole("other"),
113
+ cwd: tmp,
114
+ outPath: path.join(tmp, "out.json"),
115
+ timeoutMs: 3 * 60 * 1000,
116
+ maxAttempts: 2,
117
+ parse: parseJudgeVerdict,
118
+ }),
119
+ );
120
+ const cost = result.usage.cost_usd_est;
121
+ if (result.status !== "ok" || result.output === undefined) {
122
+ return {
123
+ label: null,
124
+ error: result.stderrTail.slice(-300),
125
+ cost,
126
+ ...recorded,
127
+ };
128
+ }
129
+ const verdict = result.output as ReturnType<typeof parseJudgeVerdict>;
130
+ return {
131
+ label: verdict.label,
132
+ ...(verdict.reason === undefined ? {} : { reason: verdict.reason }),
133
+ cost,
134
+ ...recorded,
135
+ };
136
+ } catch (error) {
137
+ return {
138
+ label: null,
139
+ error: error instanceof Error ? error.message : String(error),
140
+ cost: 0,
141
+ ...(identity === undefined
142
+ ? {}
143
+ : {
144
+ route: identity.route,
145
+ routeFingerprint: identity.routeFingerprint,
146
+ }),
147
+ };
148
+ } finally {
149
+ // The runtime first: it may still hold the workspace it was admitted on.
150
+ await prepared?.dispose().catch(() => undefined);
151
+ rmSync(tmp, { recursive: true, force: true });
152
+ }
153
+ }
@@ -0,0 +1,133 @@
1
+ // The route plan a whole-pipeline harness (refuter-probe, fixture-eval)
2
+ // passes runPipeline (odd/tasks/route-plan-harness.md S4). Today such a
3
+ // harness passes no plan, and runPipeline builds one for itself but runs the
4
+ // runner it was given, so a ClaudeCodeRunner beside an OpenCode route runs
5
+ // silently on Claude. With this plan the harness prepares the runner through
6
+ // the seam (prepareHarnessRunner) and runPipeline binds every step to its
7
+ // route.
8
+ //
9
+ // The plan is built from the SAME input runPipeline receives, with the same
10
+ // inputs runPipeline's own no-plan build reads (pipeline.ts, "Step 2.5"), so
11
+ // what a harness measured before is what it measures now (S6): the plan's
12
+ // fingerprint equals the one a no-plan run records. buildCliRoutePlan is the
13
+ // CLI's plan and differs on purpose (bundled root prompts, consolidation on),
14
+ // so a harness that mirrors a lab-style run cannot use it.
15
+ import { readFile } from "node:fs/promises";
16
+ import path from "node:path";
17
+ import { buildResolvedRoutePlan, type ResolvedRoutePlan } from "#model/routing";
18
+ import type { PipelineInput } from "#review/pipeline";
19
+ import { parseAgentFile } from "#review/prompt-set";
20
+ import { defaultReviewSpec, validateReviewSpec } from "#review/spec";
21
+ import type { ResolvedModelRoute } from "../execution/contracts";
22
+ import { HarnessRouteError, harnessModelChoice } from "./route-runner";
23
+
24
+ // Every PipelineInput field runPipeline reads to build its own plan.
25
+ export type PipelinePlanInput = Pick<
26
+ PipelineInput,
27
+ | "spec"
28
+ | "agentsDir"
29
+ | "bundledAgents"
30
+ | "modelProfile"
31
+ | "model"
32
+ | "models"
33
+ | "config"
34
+ | "summarizer"
35
+ | "scout"
36
+ | "consolidate"
37
+ | "materiality"
38
+ >;
39
+
40
+ async function rootFrontmatter(
41
+ stage: { promptPath: string; bundled?: boolean } | undefined,
42
+ ): Promise<string | undefined> {
43
+ if (stage === undefined || stage.bundled === true) return undefined;
44
+ return (await parseAgentFile(stage.promptPath)).model;
45
+ }
46
+
47
+ // Unlike runPipeline, an unreadable prompt fails here: the harness learns of
48
+ // it before admission rather than from a plan that silently lacks a model.
49
+ export async function buildPipelineHarnessPlan(
50
+ input: PipelinePlanInput,
51
+ ): Promise<ResolvedRoutePlan> {
52
+ const spec = validateReviewSpec(input.spec ?? defaultReviewSpec());
53
+ // A diversity run routes its expanded legs (diversityCtx.routeAgents), which
54
+ // this plan does not build; runPipeline would refuse it as a missing route.
55
+ if (spec.multiModelDiversity?.enabled === true) {
56
+ throw new HarnessRouteError(
57
+ "a multi-model diversity spec routes its expanded legs; a pipeline harness plan does not build them",
58
+ );
59
+ }
60
+ const frontmatter = new Map<string, string | undefined>();
61
+ for (const agent of spec.agents) {
62
+ const parsed = await parseAgentFile(path.join(input.agentsDir, agent.file));
63
+ frontmatter.set(agent.key, parsed.model);
64
+ }
65
+ const models = input.models ?? input.config?.effective?.models;
66
+ const stage = async (
67
+ value:
68
+ | { promptPath: string; model?: string; bundled?: boolean }
69
+ | undefined,
70
+ ) =>
71
+ value === undefined
72
+ ? undefined
73
+ : {
74
+ model: value.model,
75
+ frontmatterModel: await rootFrontmatter(value),
76
+ bundled: value.bundled === true,
77
+ };
78
+ const summarizer = await stage(input.summarizer);
79
+ const scout = await stage(input.scout);
80
+ const consolidate = await stage(input.consolidate);
81
+ return buildResolvedRoutePlan({
82
+ agents: spec.agents,
83
+ cliModel: harnessModelChoice(input.model),
84
+ models,
85
+ ...(input.modelProfile === undefined
86
+ ? {}
87
+ : { profile: input.modelProfile }),
88
+ ...(input.bundledAgents === true ? { bundledAgents: true } : {}),
89
+ frontmatterModel: (agentKey) => frontmatter.get(agentKey),
90
+ ...(summarizer === undefined ? {} : { summarizer }),
91
+ ...(scout === undefined ? {} : { scout }),
92
+ ...(consolidate === undefined ? {} : { consolidate }),
93
+ ...(input.materiality === undefined ? {} : { materiality: true as const }),
94
+ });
95
+ }
96
+
97
+ export interface PipelineRouteIdentity {
98
+ readonly route: ResolvedModelRoute;
99
+ readonly routeFingerprint: string;
100
+ readonly stepRouteFingerprint: string;
101
+ }
102
+
103
+ interface RecordedPlan {
104
+ route_fingerprint?: string;
105
+ route_plan?: {
106
+ steps?: Array<{
107
+ stepKey: string;
108
+ route: ResolvedModelRoute;
109
+ routeFingerprint: string;
110
+ }>;
111
+ };
112
+ }
113
+
114
+ // The identity of the route a pipeline run executed for one step, read back
115
+ // from the run's own pipeline.json, never from the harness's intent.
116
+ export async function readPipelineRouteIdentity(
117
+ runDir: string,
118
+ stepKey: string,
119
+ ): Promise<PipelineRouteIdentity> {
120
+ const file = path.join(runDir, "pipeline.json");
121
+ const recorded = JSON.parse(await readFile(file, "utf8")) as RecordedPlan;
122
+ const step = recorded.route_plan?.steps?.find((s) => s.stepKey === stepKey);
123
+ if (step === undefined || recorded.route_fingerprint === undefined) {
124
+ throw new HarnessRouteError(
125
+ `${file} records no route for step "${stepKey}"`,
126
+ );
127
+ }
128
+ return {
129
+ route: step.route,
130
+ routeFingerprint: recorded.route_fingerprint,
131
+ stepRouteFingerprint: step.routeFingerprint,
132
+ };
133
+ }
@@ -0,0 +1,172 @@
1
+ // The rows and resume rules of refuter-replay and consolidate-replay
2
+ // (odd/tasks/route-plan-harness.md S3, S6), here so the gates and offline
3
+ // tests cover them; the scripts keep only their CLI wiring. Every row that ran
4
+ // a step records the model string it was given and the route identity of the
5
+ // step it ran. Resume accepts rows written before the seam, keyed by the model
6
+ // string alone, so no paid row is orphaned.
7
+ import type { PlanEntry } from "#review/consolidate";
8
+ import type { RefuterResult } from "#review/drafts";
9
+ import type { Finding } from "#review/findings";
10
+ import type { StepResult } from "#review/step-runner";
11
+ import {
12
+ type RouteIdentityFields,
13
+ ranOnRoute,
14
+ routeIdentityFields,
15
+ type SingleStepRoute,
16
+ } from "./single-step";
17
+
18
+ export interface RefuterReplayCase {
19
+ readonly run: string;
20
+ readonly finding: string;
21
+ readonly expect: "keep" | "drop";
22
+ readonly label: string;
23
+ // "refuted": the finding lives in findings.json debug.refuted, not in the
24
+ // posted list (the run deleted it). Its id may collide with a posted one.
25
+ readonly source?: "posted" | "refuted";
26
+ readonly note?: string;
27
+ }
28
+
29
+ // The source is part of the key: a refuted finding's id may collide with a
30
+ // posted one, and the two must not resume each other. The model is not: which
31
+ // route a row ran on is ranOnRoute's question.
32
+ export function refuterReplayCaseKey(c: {
33
+ readonly run: string;
34
+ readonly finding: string;
35
+ readonly source?: string;
36
+ }): string {
37
+ return `${c.run}|${c.finding}|${c.source ?? "posted"}`;
38
+ }
39
+
40
+ // The case keys already answered on this route in an existing --out. A failed
41
+ // step (quota, auth, timeout) is not an answer: it reruns.
42
+ export function refuterReplayDone(
43
+ jsonl: string,
44
+ route: SingleStepRoute,
45
+ ): Set<string> {
46
+ const done = new Set<string>();
47
+ for (const line of jsonl.split("\n")) {
48
+ if (!line.trim()) continue;
49
+ const row = JSON.parse(line) as {
50
+ run: string;
51
+ finding: string;
52
+ source?: string;
53
+ model?: string;
54
+ step_route_fingerprint?: string;
55
+ status: string;
56
+ };
57
+ if (row.status === "ok" && ranOnRoute(row, route)) {
58
+ done.add(refuterReplayCaseKey(row));
59
+ }
60
+ }
61
+ return done;
62
+ }
63
+
64
+ export type RefuterPromptCheck = "match" | "unrecorded" | "candidate";
65
+
66
+ export function refuterReplayRow(input: {
67
+ readonly replayCase: RefuterReplayCase;
68
+ readonly finding: Finding;
69
+ readonly route: SingleStepRoute;
70
+ readonly result: StepResult;
71
+ readonly promptCheck: RefuterPromptCheck;
72
+ }) {
73
+ const { replayCase: c, finding: f, route, result } = input;
74
+ // A failed step reads as `inconclusive`, exactly as runRefuter treats it.
75
+ const entry =
76
+ result.status === "ok"
77
+ ? (result.output as RefuterResult).results[0]
78
+ : undefined;
79
+ return {
80
+ run: c.run,
81
+ finding: f.id,
82
+ source: c.source ?? "posted",
83
+ model: route.model,
84
+ ...routeIdentityFields(route.identity),
85
+ severity: f.severity,
86
+ tier: f.tier,
87
+ original_verdict: c.source === "refuted" ? "refuted" : f.refuter_verdict,
88
+ verdict: entry?.outcome ?? "inconclusive",
89
+ refuter_proof_refs: entry?.proof_refs ?? [],
90
+ status: result.status,
91
+ denial: result.denialCode ?? null,
92
+ stderr_tail:
93
+ result.status === "ok" ? null : (result.stderrTail ?? "").slice(-400),
94
+ expect: c.expect,
95
+ label: c.label,
96
+ prompt_check: input.promptCheck,
97
+ notional_usd: result.usageV2?.notionalCostUsd ?? null,
98
+ wall_ms: result.usageV2?.wallMs ?? null,
99
+ };
100
+ }
101
+
102
+ // consolidate-replay pins its experiment in config.json, because its audit is
103
+ // keyed by PR label only. The pin keeps the shape it had before the seam, so
104
+ // an existing --out keeps resuming; the model matches on its route, so another
105
+ // spelling of the same model is the same experiment.
106
+ export interface ConsolidateReplayConfig {
107
+ readonly arm: string;
108
+ readonly tool: string;
109
+ readonly model: string;
110
+ readonly variant: string;
111
+ }
112
+
113
+ export function consolidateConfigMatches(
114
+ seen: Partial<Record<keyof ConsolidateReplayConfig, unknown>>,
115
+ current: ConsolidateReplayConfig,
116
+ route: SingleStepRoute,
117
+ ): boolean {
118
+ return (
119
+ seen.arm === current.arm &&
120
+ seen.tool === current.tool &&
121
+ seen.variant === current.variant &&
122
+ ranOnRoute({ model: seen.model }, route)
123
+ );
124
+ }
125
+
126
+ export type ConsolidateStepRow = {
127
+ readonly label: string;
128
+ readonly status: "ok" | "fallback";
129
+ readonly reason?: string;
130
+ readonly plan?: PlanEntry[];
131
+ readonly in: number;
132
+ readonly out: number;
133
+ readonly notional_usd: number | null;
134
+ readonly model: string;
135
+ } & RouteIdentityFields;
136
+
137
+ // The audit row of a PR whose consolidation step ran. A failed step is a
138
+ // fallback that keeps every finding, and is retried on resume.
139
+ export function consolidateStepRow(input: {
140
+ readonly label: string;
141
+ readonly ids: readonly string[];
142
+ readonly route: SingleStepRoute;
143
+ readonly result: StepResult;
144
+ }): ConsolidateStepRow {
145
+ const { label, ids, route, result } = input;
146
+ const ran = {
147
+ model: route.model,
148
+ ...routeIdentityFields(route.identity),
149
+ };
150
+ const notional_usd = result.usageV2?.notionalCostUsd ?? null;
151
+ if (result.status !== "ok") {
152
+ return {
153
+ label,
154
+ status: "fallback",
155
+ reason: (result.stderrTail ?? "").slice(-300),
156
+ in: ids.length,
157
+ out: ids.length,
158
+ notional_usd,
159
+ ...ran,
160
+ };
161
+ }
162
+ const plan = result.output as PlanEntry[];
163
+ return {
164
+ label,
165
+ status: "ok",
166
+ plan,
167
+ in: ids.length,
168
+ out: plan.length,
169
+ notional_usd,
170
+ ...ran,
171
+ };
172
+ }