@crewhaus/spec-patch 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,106 @@
1
+ /**
2
+ * Pillar 2 — `spec-patch`. The structured mutation primitive that the
3
+ * active eval optimizer uses to translate "the prompt that scored higher"
4
+ * back into a YAML edit the user can review, commit, and re-run.
5
+ *
6
+ * Why patch the SPEC (not the IR)?
7
+ *
8
+ * The compiler does destructive normalisation in `lower()`: sub-agent
9
+ * maps become arrays, role names get alphabetically sorted, secrets get
10
+ * rewritten to env-var refs via `lowerSecret`, permission rules get
11
+ * de-duped and re-ordered. The IR is intentionally lossy and ordering-
12
+ * canonical because its job is to feed codegen, not round-trip to YAML.
13
+ *
14
+ * A patch at the IR layer therefore can't write back to YAML — there's
15
+ * no path from a frozen sorted array back to the source-author's map
16
+ * with their comments and field ordering. Patching at the SPEC layer
17
+ * (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
18
+ * --write-back` rewrites the user's file with their comments preserved
19
+ * and only the touched values changed.
20
+ *
21
+ * The `yaml` package's `parseDocument` gives us the CST we need. The
22
+ * Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
23
+ * that operate on the live AST; `toString()` renders back with comment
24
+ * and key-order fidelity.
25
+ *
26
+ * IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
27
+ * time optimisations that run AFTER lowering. They don't get conflated
28
+ * with eval-driven mutation. The two systems share nothing; that's
29
+ * intentional.
30
+ *
31
+ * Catalog layer: F2 (compiler periphery). Brief: 278.
32
+ */
33
+ import { CrewhausError } from "@crewhaus/errors";
34
+ import { type Spec } from "@crewhaus/spec";
35
+ export declare class SpecPatchError extends CrewhausError {
36
+ readonly name = "SpecPatchError";
37
+ constructor(message: string, cause?: unknown);
38
+ }
39
+ export type SpecPatchOp = "replace" | "add" | "remove";
40
+ /**
41
+ * A structured edit to a spec. `path` is a property-key chain (no array
42
+ * indices today — the spec's mutation surface is all object fields). The
43
+ * `target` discriminator must match the spec being patched; the validator
44
+ * refuses cross-target patches so an optimizer can't pass an `IrCli`
45
+ * patch to a `pipeline` spec.
46
+ */
47
+ export type SpecPatch = {
48
+ readonly target: Spec["target"];
49
+ readonly path: ReadonlyArray<string>;
50
+ readonly op: SpecPatchOp;
51
+ /** Required for `"replace"` and `"add"`; ignored for `"remove"`. */
52
+ readonly value?: unknown;
53
+ /**
54
+ * Optional rationale string for audit / write-back commit messages.
55
+ * The optimizer fills this with the mutation kind + observed delta.
56
+ */
57
+ readonly rationale?: string;
58
+ };
59
+ export type ApplySpecPatchResult = {
60
+ /** The mutated YAML text, with comments and key order preserved. */
61
+ readonly yaml: string;
62
+ /** The re-parsed Spec after the patch. */
63
+ readonly spec: Spec;
64
+ };
65
+ /**
66
+ * Apply a structured patch to a YAML spec source. Uses the `yaml`
67
+ * package's CST so comments and key order survive the round-trip; the
68
+ * only bytes that change are the ones the patch targets.
69
+ *
70
+ * Validates the resulting document against the Spec schema; throws
71
+ * `SpecPatchError` if the mutation produces invalid spec.
72
+ */
73
+ export declare function applySpecPatch(yamlText: string, patch: SpecPatch): ApplySpecPatchResult;
74
+ /**
75
+ * Type-check a patch against a parsed Spec WITHOUT applying it. Used
76
+ * by the optimizer to refuse cross-target patches and patches that
77
+ * reference paths outside the optimisation surface.
78
+ *
79
+ * The path-validity check is the "soft" version: it verifies the path
80
+ * is structurally reachable (each segment names a defined field). It
81
+ * does NOT instantiate the patched value — `applySpecPatch` does that
82
+ * via `parseSpec`.
83
+ */
84
+ export declare function validatePatch(spec: Spec, patch: SpecPatch): void;
85
+ /**
86
+ * Per-target whitelist of mutation paths the active optimizer is
87
+ * allowed to touch. Adding a new field here is the explicit signal that
88
+ * it's safe to autotune. Skipping this list means the optimizer can
89
+ * only mutate prompts (the default), preserving the "spec safety floor"
90
+ * that an optimizer can't accidentally rewrite security-critical fields
91
+ * like `permissions.mode` or `model_router` rules.
92
+ */
93
+ export declare const OPTIMIZABLE_PATHS: Readonly<Record<Spec["target"], ReadonlyArray<ReadonlyArray<string>>>>;
94
+ /**
95
+ * Format a YAML header comment to prepend to a written-back file. The
96
+ * orchestrator's `--write-back` writes this above the original spec so
97
+ * a human reviewer can see when and by what an optimisation pass ran.
98
+ */
99
+ export declare function formatWriteBackHeader(opts: {
100
+ readonly runId: string;
101
+ readonly mutator: string;
102
+ readonly scoreBefore: number;
103
+ readonly scoreAfter: number;
104
+ readonly iterations: number;
105
+ readonly timestamp?: string;
106
+ }): string;
package/dist/index.js ADDED
@@ -0,0 +1,294 @@
1
+ /**
2
+ * Pillar 2 — `spec-patch`. The structured mutation primitive that the
3
+ * active eval optimizer uses to translate "the prompt that scored higher"
4
+ * back into a YAML edit the user can review, commit, and re-run.
5
+ *
6
+ * Why patch the SPEC (not the IR)?
7
+ *
8
+ * The compiler does destructive normalisation in `lower()`: sub-agent
9
+ * maps become arrays, role names get alphabetically sorted, secrets get
10
+ * rewritten to env-var refs via `lowerSecret`, permission rules get
11
+ * de-duped and re-ordered. The IR is intentionally lossy and ordering-
12
+ * canonical because its job is to feed codegen, not round-trip to YAML.
13
+ *
14
+ * A patch at the IR layer therefore can't write back to YAML — there's
15
+ * no path from a frozen sorted array back to the source-author's map
16
+ * with their comments and field ordering. Patching at the SPEC layer
17
+ * (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
18
+ * --write-back` rewrites the user's file with their comments preserved
19
+ * and only the touched values changed.
20
+ *
21
+ * The `yaml` package's `parseDocument` gives us the CST we need. The
22
+ * Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
23
+ * that operate on the live AST; `toString()` renders back with comment
24
+ * and key-order fidelity.
25
+ *
26
+ * IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
27
+ * time optimisations that run AFTER lowering. They don't get conflated
28
+ * with eval-driven mutation. The two systems share nothing; that's
29
+ * intentional.
30
+ *
31
+ * Catalog layer: F2 (compiler periphery). Brief: 278.
32
+ */
33
+ import { CrewhausError } from "@crewhaus/errors";
34
+ import { parseSpec } from "@crewhaus/spec";
35
+ import { parseDocument } from "yaml";
36
+ import { z } from "zod";
37
+ export class SpecPatchError extends CrewhausError {
38
+ name = "SpecPatchError";
39
+ constructor(message, cause) {
40
+ super("compiler", message, cause);
41
+ }
42
+ }
43
+ const specPatchSchema = z.object({
44
+ target: z.string().min(1),
45
+ path: z.array(z.string().min(1)).min(1),
46
+ op: z.enum(["replace", "add", "remove"]),
47
+ value: z.unknown().optional(),
48
+ rationale: z.string().optional(),
49
+ });
50
+ /**
51
+ * Apply a structured patch to a YAML spec source. Uses the `yaml`
52
+ * package's CST so comments and key order survive the round-trip; the
53
+ * only bytes that change are the ones the patch targets.
54
+ *
55
+ * Validates the resulting document against the Spec schema; throws
56
+ * `SpecPatchError` if the mutation produces invalid spec.
57
+ */
58
+ export function applySpecPatch(yamlText, patch) {
59
+ const parsed = specPatchSchema.safeParse(patch);
60
+ if (!parsed.success) {
61
+ throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
62
+ }
63
+ let doc;
64
+ try {
65
+ doc = parseDocument(yamlText);
66
+ }
67
+ catch (err) {
68
+ throw new SpecPatchError("input YAML is not parseable", err);
69
+ }
70
+ // Verify the patch matches the spec's target. Cheap pre-check so we
71
+ // fail with a useful message before the Zod validation later.
72
+ const docTarget = doc.getIn(["target"]);
73
+ if (typeof docTarget === "string" && docTarget !== patch.target) {
74
+ throw new SpecPatchError(`patch target "${patch.target}" does not match spec target "${docTarget}"`);
75
+ }
76
+ const path = [...patch.path];
77
+ switch (patch.op) {
78
+ case "replace": {
79
+ if (!doc.hasIn(path)) {
80
+ throw new SpecPatchError(`cannot replace ${formatPath(path)}: path does not exist`);
81
+ }
82
+ doc.setIn(path, patch.value);
83
+ break;
84
+ }
85
+ case "add": {
86
+ if (doc.hasIn(path)) {
87
+ throw new SpecPatchError(`cannot add ${formatPath(path)}: path already exists (use "replace")`);
88
+ }
89
+ doc.setIn(path, patch.value);
90
+ break;
91
+ }
92
+ case "remove": {
93
+ if (!doc.hasIn(path)) {
94
+ throw new SpecPatchError(`cannot remove ${formatPath(path)}: path does not exist`);
95
+ }
96
+ doc.deleteIn(path);
97
+ break;
98
+ }
99
+ }
100
+ const newYaml = doc.toString();
101
+ let spec;
102
+ try {
103
+ spec = parseSpec(newYaml);
104
+ }
105
+ catch (err) {
106
+ throw new SpecPatchError(`patched YAML failed spec validation: ${err.message}`, err);
107
+ }
108
+ return { yaml: newYaml, spec };
109
+ }
110
+ /**
111
+ * Type-check a patch against a parsed Spec WITHOUT applying it. Used
112
+ * by the optimizer to refuse cross-target patches and patches that
113
+ * reference paths outside the optimisation surface.
114
+ *
115
+ * The path-validity check is the "soft" version: it verifies the path
116
+ * is structurally reachable (each segment names a defined field). It
117
+ * does NOT instantiate the patched value — `applySpecPatch` does that
118
+ * via `parseSpec`.
119
+ */
120
+ export function validatePatch(spec, patch) {
121
+ const parsed = specPatchSchema.safeParse(patch);
122
+ if (!parsed.success) {
123
+ throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
124
+ }
125
+ if (patch.target !== spec.target) {
126
+ throw new SpecPatchError(`patch target "${patch.target}" does not match spec target "${spec.target}"`);
127
+ }
128
+ if (!isOptimizable(spec.target, patch.path)) {
129
+ throw new SpecPatchError(`path ${formatPath(patch.path)} is not listed in OPTIMIZABLE_PATHS for target "${spec.target}"; add it to packages/spec-patch/src/index.ts if it's intended to be tunable`);
130
+ }
131
+ }
132
+ function formatPath(path) {
133
+ return path.join(".");
134
+ }
135
+ /**
136
+ * Per-target whitelist of mutation paths the active optimizer is
137
+ * allowed to touch. Adding a new field here is the explicit signal that
138
+ * it's safe to autotune. Skipping this list means the optimizer can
139
+ * only mutate prompts (the default), preserving the "spec safety floor"
140
+ * that an optimizer can't accidentally rewrite security-critical fields
141
+ * like `permissions.mode` or `model_router` rules.
142
+ */
143
+ export const OPTIMIZABLE_PATHS = Object.freeze({
144
+ cli: Object.freeze([
145
+ Object.freeze(["agent", "instructions"]),
146
+ Object.freeze(["failure_taxonomy"]),
147
+ Object.freeze(["compaction", "threshold"]),
148
+ // Pillar 2 active context curation — eval-optimizer can flip the
149
+ // semantic-dedupe + relevance-reorder pass on/off and tune its
150
+ // similarity threshold, which dominates input-token cost on long runs.
151
+ Object.freeze(["compaction", "curate"]),
152
+ Object.freeze(["compaction", "dedupeThreshold"]),
153
+ Object.freeze(["compaction", "relevanceTopK"]),
154
+ // Pillar 3 sink-side fabric — egress policy + intent-gate thresholds
155
+ // are tunable so the eval-optimizer can find the sweet spot between
156
+ // false-positive denials and false-negative exfil bypasses.
157
+ Object.freeze(["security", "egressPolicy"]),
158
+ Object.freeze(["security", "justification"]),
159
+ // §47 blockchain subsystem (slice 0). Whole-block replacement so the
160
+ // optimizer can tune `chains[*].finality.count`, `chains[*].rpcPolicy`,
161
+ // `transaction_policy.maxValueWei`, and `transaction_policy.simulationRequired`
162
+ // by patching their parent block.
163
+ Object.freeze(["chains"]),
164
+ Object.freeze(["transaction_policy"]),
165
+ ]),
166
+ workflow: Object.freeze([
167
+ Object.freeze(["steps"]),
168
+ Object.freeze(["failure_taxonomy"]),
169
+ Object.freeze(["chains"]),
170
+ Object.freeze(["transaction_policy"]),
171
+ ]) /* whole-step replacement allowed */,
172
+ channel: Object.freeze([
173
+ Object.freeze(["agent", "instructions"]),
174
+ Object.freeze(["failure_taxonomy"]),
175
+ Object.freeze(["chains"]),
176
+ Object.freeze(["transaction_policy"]),
177
+ ]),
178
+ graph: Object.freeze([
179
+ Object.freeze(["nodes"]),
180
+ Object.freeze(["failure_taxonomy"]),
181
+ Object.freeze(["chains"]),
182
+ Object.freeze(["transaction_policy"]),
183
+ ]),
184
+ managed: Object.freeze([
185
+ Object.freeze(["agent", "instructions"]),
186
+ Object.freeze(["failure_taxonomy"]),
187
+ ]),
188
+ pipeline: Object.freeze([
189
+ Object.freeze(["agent", "instructions"]),
190
+ Object.freeze(["failure_taxonomy"]),
191
+ Object.freeze(["indexing", "chunkSize"]),
192
+ Object.freeze(["indexing", "chunkOverlap"]),
193
+ Object.freeze(["retrieve", "defaultK"]),
194
+ ]),
195
+ crew: Object.freeze([
196
+ Object.freeze(["roles"]),
197
+ Object.freeze(["failure_taxonomy"]),
198
+ Object.freeze(["chains"]),
199
+ Object.freeze(["transaction_policy"]),
200
+ ]) /* whole-role replacement */,
201
+ research: Object.freeze([
202
+ Object.freeze(["agent", "instructions"]),
203
+ Object.freeze(["failure_taxonomy"]),
204
+ Object.freeze(["retrieve", "maxDepth"]),
205
+ Object.freeze(["chains"]),
206
+ Object.freeze(["transaction_policy"]),
207
+ ]),
208
+ batch: Object.freeze([
209
+ Object.freeze(["agent", "instructions"]),
210
+ Object.freeze(["failure_taxonomy"]),
211
+ Object.freeze(["chains"]),
212
+ Object.freeze(["transaction_policy"]),
213
+ ]),
214
+ voice: Object.freeze([
215
+ Object.freeze(["agent", "instructions"]),
216
+ Object.freeze(["failure_taxonomy"]),
217
+ ]),
218
+ browser: Object.freeze([
219
+ Object.freeze(["agent", "instructions"]),
220
+ Object.freeze(["failure_taxonomy"]),
221
+ ]),
222
+ eval: Object.freeze([
223
+ Object.freeze(["agent", "instructions"]),
224
+ Object.freeze(["failure_taxonomy"]),
225
+ ]),
226
+ // §47 onchain daemon: full cross-cutting blocks are optimizable.
227
+ onchain: Object.freeze([
228
+ Object.freeze(["agent", "instructions"]),
229
+ Object.freeze(["failure_taxonomy"]),
230
+ Object.freeze(["chains"]),
231
+ Object.freeze(["triggers"]),
232
+ Object.freeze(["transaction_policy"]),
233
+ Object.freeze(["idempotencyWindowMs"]),
234
+ ]),
235
+ // §47 onchain-game: instructions, game.objective, and the policy are
236
+ // the productive knobs; move-timeout-ms is the realtime quality knob.
237
+ "onchain-game": Object.freeze([
238
+ Object.freeze(["agent", "instructions"]),
239
+ Object.freeze(["failure_taxonomy"]),
240
+ Object.freeze(["game"]),
241
+ Object.freeze(["transaction_policy"]),
242
+ ]),
243
+ });
244
+ function isOptimizable(target, path) {
245
+ const allowed = OPTIMIZABLE_PATHS[target];
246
+ for (const ok of allowed) {
247
+ if (ok.length !== path.length)
248
+ continue;
249
+ let match = true;
250
+ for (let i = 0; i < ok.length; i++) {
251
+ if (ok[i] !== path[i]) {
252
+ match = false;
253
+ break;
254
+ }
255
+ }
256
+ if (match)
257
+ return true;
258
+ }
259
+ // Allow patches with a prefix that matches an optimizable path
260
+ // (e.g. `["nodes", "0", "instructions"]` if `["nodes"]` is whitelisted)
261
+ // so the optimizer can do fine-grained updates without listing every
262
+ // sub-path.
263
+ for (const ok of allowed) {
264
+ if (path.length < ok.length)
265
+ continue;
266
+ let match = true;
267
+ for (let i = 0; i < ok.length; i++) {
268
+ if (ok[i] !== path[i]) {
269
+ match = false;
270
+ break;
271
+ }
272
+ }
273
+ if (match)
274
+ return true;
275
+ }
276
+ return false;
277
+ }
278
+ /**
279
+ * Format a YAML header comment to prepend to a written-back file. The
280
+ * orchestrator's `--write-back` writes this above the original spec so
281
+ * a human reviewer can see when and by what an optimisation pass ran.
282
+ */
283
+ export function formatWriteBackHeader(opts) {
284
+ const ts = opts.timestamp ?? new Date().toISOString();
285
+ const delta = (opts.scoreAfter - opts.scoreBefore).toFixed(3);
286
+ return [
287
+ `# crewhaus optimize: runId ${opts.runId}`,
288
+ `# - mutator: ${opts.mutator}`,
289
+ `# - iterations: ${opts.iterations}`,
290
+ `# - score: ${opts.scoreBefore.toFixed(3)} → ${opts.scoreAfter.toFixed(3)} (Δ ${delta})`,
291
+ `# - generated: ${ts}`,
292
+ "",
293
+ ].join("\n");
294
+ }
package/package.json CHANGED
@@ -1,19 +1,22 @@
1
1
  {
2
2
  "name": "@crewhaus/spec-patch",
3
- "version": "0.1.3",
3
+ "version": "0.1.5",
4
4
  "type": "module",
5
5
  "description": "Pillar-2 patch infrastructure — apply a SpecPatch to a YAML source preserving comments and key order via the yaml CST. Drives the active eval optimizer's spec-level mutation loop.",
6
- "main": "src/index.ts",
7
- "types": "src/index.ts",
6
+ "main": "dist/index.js",
7
+ "types": "dist/index.d.ts",
8
8
  "exports": {
9
- ".": "./src/index.ts"
9
+ ".": {
10
+ "types": "./dist/index.d.ts",
11
+ "import": "./dist/index.js"
12
+ }
10
13
  },
11
14
  "scripts": {
12
15
  "test": "bun test src"
13
16
  },
14
17
  "dependencies": {
15
- "@crewhaus/errors": "0.1.3",
16
- "@crewhaus/spec": "0.1.3",
18
+ "@crewhaus/errors": "0.1.5",
19
+ "@crewhaus/spec": "0.1.5",
17
20
  "yaml": "^2.6.0",
18
21
  "zod": "^3.23.8"
19
22
  },
@@ -35,5 +38,5 @@
35
38
  "publishConfig": {
36
39
  "access": "public"
37
40
  },
38
- "files": ["src", "README.md", "LICENSE", "NOTICE"]
41
+ "files": ["dist", "README.md", "LICENSE", "NOTICE"]
39
42
  }
package/src/index.test.ts DELETED
@@ -1,236 +0,0 @@
1
- import { describe, expect, test } from "bun:test";
2
- import { OPTIMIZABLE_PATHS, applySpecPatch, formatWriteBackHeader, validatePatch } from "./index";
3
-
4
- const CLI_YAML = `# A simple CLI agent
5
- target: cli
6
- name: hello-cli
7
- agent:
8
- model: claude-sonnet-4-5
9
- # The system prompt the model sees on every turn:
10
- instructions: You are a helpful assistant.
11
- tools:
12
- - Read
13
- - Write
14
- `;
15
-
16
- const PIPELINE_YAML = `target: pipeline
17
- name: hello-rag
18
- agent:
19
- model: claude-sonnet-4-5
20
- instructions: Answer using only the retrieved docs.
21
- retrieve:
22
- embedderModel: text-embedding-3-small
23
- defaultK: 3
24
- indexing:
25
- # Default chunking — deliberately short for testing
26
- chunkSize: 200
27
- chunkOverlap: 20
28
- documents:
29
- - id: doc1
30
- text: The capital of France is Paris.
31
- `;
32
-
33
- describe("applySpecPatch — round-trip", () => {
34
- test("replaces agent.instructions and re-parses", () => {
35
- const patch = {
36
- target: "cli" as const,
37
- path: ["agent", "instructions"] as const,
38
- op: "replace" as const,
39
- value: "Think step by step before answering.",
40
- };
41
- const { yaml, spec } = applySpecPatch(CLI_YAML, patch);
42
- if (spec.target !== "cli") throw new Error("expected cli spec");
43
- expect(spec.agent.instructions).toBe("Think step by step before answering.");
44
- expect(yaml).toContain("instructions: Think step by step before answering.");
45
- });
46
-
47
- test("preserves leading comments and structural comments", () => {
48
- const patch = {
49
- target: "cli" as const,
50
- path: ["agent", "instructions"] as const,
51
- op: "replace" as const,
52
- value: "Be concise.",
53
- };
54
- const { yaml } = applySpecPatch(CLI_YAML, patch);
55
- expect(yaml).toContain("# A simple CLI agent");
56
- expect(yaml).toContain("# The system prompt the model sees on every turn");
57
- });
58
-
59
- test("replaces a numeric field on a pipeline spec", () => {
60
- const patch = {
61
- target: "pipeline" as const,
62
- path: ["indexing", "chunkOverlap"] as const,
63
- op: "replace" as const,
64
- value: 50,
65
- };
66
- const { yaml, spec } = applySpecPatch(PIPELINE_YAML, patch);
67
- if (spec.target !== "pipeline") throw new Error("expected pipeline spec");
68
- expect(spec.indexing.chunkOverlap).toBe(50);
69
- expect(yaml).toContain("chunkOverlap: 50");
70
- // Comment on indexing block survives
71
- expect(yaml).toContain("# Default chunking — deliberately short for testing");
72
- });
73
- });
74
-
75
- describe("applySpecPatch — error cases", () => {
76
- test("cross-target patch is rejected with a clear error", () => {
77
- expect(() =>
78
- applySpecPatch(CLI_YAML, {
79
- target: "pipeline",
80
- path: ["agent", "instructions"],
81
- op: "replace",
82
- value: "x",
83
- }),
84
- ).toThrow(/does not match spec target/);
85
- });
86
-
87
- test("non-existent replace path is rejected", () => {
88
- expect(() =>
89
- applySpecPatch(CLI_YAML, {
90
- target: "cli",
91
- path: ["agent", "nonexistent_field"],
92
- op: "replace",
93
- value: "x",
94
- }),
95
- ).toThrow(/path does not exist/);
96
- });
97
-
98
- test("add to an existing path is rejected", () => {
99
- expect(() =>
100
- applySpecPatch(CLI_YAML, {
101
- target: "cli",
102
- path: ["agent", "instructions"],
103
- op: "add",
104
- value: "x",
105
- }),
106
- ).toThrow(/path already exists/);
107
- });
108
-
109
- test("remove a non-existent path is rejected", () => {
110
- expect(() =>
111
- applySpecPatch(CLI_YAML, {
112
- target: "cli",
113
- path: ["agent", "nope"],
114
- op: "remove",
115
- }),
116
- ).toThrow(/path does not exist/);
117
- });
118
-
119
- test("invalid patch shape (empty path array) is rejected", () => {
120
- expect(() =>
121
- applySpecPatch(CLI_YAML, {
122
- target: "cli",
123
- // biome-ignore lint/suspicious/noExplicitAny: testing runtime validation
124
- path: [] as any,
125
- op: "replace",
126
- value: "x",
127
- }),
128
- ).toThrow(/patch shape is invalid/);
129
- });
130
-
131
- test("patched YAML that breaks spec schema is rejected", () => {
132
- // Replacing model with a non-string number should be rejected.
133
- expect(() =>
134
- applySpecPatch(CLI_YAML, {
135
- target: "cli",
136
- path: ["agent", "model"],
137
- op: "replace",
138
- value: 42,
139
- }),
140
- ).toThrow(/spec validation failed/);
141
- });
142
- });
143
-
144
- describe("validatePatch", () => {
145
- test("accepts a whitelisted optimizable path", () => {
146
- const { spec } = applySpecPatch(PIPELINE_YAML, {
147
- target: "pipeline",
148
- path: ["indexing", "chunkOverlap"],
149
- op: "replace",
150
- value: 30,
151
- });
152
- expect(() =>
153
- validatePatch(spec, {
154
- target: "pipeline",
155
- path: ["indexing", "chunkOverlap"],
156
- op: "replace",
157
- value: 40,
158
- }),
159
- ).not.toThrow();
160
- });
161
-
162
- test("rejects a path that's not in OPTIMIZABLE_PATHS", () => {
163
- const { spec } = applySpecPatch(CLI_YAML, {
164
- target: "cli",
165
- path: ["agent", "instructions"],
166
- op: "replace",
167
- value: "x",
168
- });
169
- expect(() =>
170
- validatePatch(spec, {
171
- target: "cli",
172
- path: ["agent", "model"], // intentionally not in cli's OPTIMIZABLE_PATHS
173
- op: "replace",
174
- value: "opus-4",
175
- }),
176
- ).toThrow(/not listed in OPTIMIZABLE_PATHS/);
177
- });
178
-
179
- test("cross-target patch is rejected before path-check", () => {
180
- const { spec } = applySpecPatch(CLI_YAML, {
181
- target: "cli",
182
- path: ["agent", "instructions"],
183
- op: "replace",
184
- value: "x",
185
- });
186
- expect(() =>
187
- validatePatch(spec, {
188
- target: "pipeline",
189
- path: ["agent", "instructions"],
190
- op: "replace",
191
- value: "y",
192
- }),
193
- ).toThrow(/does not match spec target/);
194
- });
195
- });
196
-
197
- describe("OPTIMIZABLE_PATHS", () => {
198
- test("every shipped target shape has at least one whitelisted path", () => {
199
- const targets = [
200
- "cli",
201
- "workflow",
202
- "channel",
203
- "graph",
204
- "managed",
205
- "pipeline",
206
- "crew",
207
- "research",
208
- "batch",
209
- "voice",
210
- "browser",
211
- "eval",
212
- ] as const;
213
- for (const t of targets) {
214
- expect(OPTIMIZABLE_PATHS[t]).toBeDefined();
215
- expect(OPTIMIZABLE_PATHS[t].length).toBeGreaterThanOrEqual(1);
216
- }
217
- });
218
- });
219
-
220
- describe("formatWriteBackHeader", () => {
221
- test("produces a deterministic header given a fixed timestamp", () => {
222
- const header = formatWriteBackHeader({
223
- runId: "opt_abc123",
224
- mutator: "rule-based",
225
- scoreBefore: 0.45,
226
- scoreAfter: 0.78,
227
- iterations: 12,
228
- timestamp: "2026-05-10T12:00:00Z",
229
- });
230
- expect(header).toContain("runId opt_abc123");
231
- expect(header).toContain("mutator: rule-based");
232
- expect(header).toContain("iterations: 12");
233
- expect(header).toContain("score: 0.450 → 0.780 (Δ 0.330)");
234
- expect(header).toContain("2026-05-10T12:00:00Z");
235
- });
236
- });
package/src/index.ts DELETED
@@ -1,344 +0,0 @@
1
- /**
2
- * Pillar 2 — `spec-patch`. The structured mutation primitive that the
3
- * active eval optimizer uses to translate "the prompt that scored higher"
4
- * back into a YAML edit the user can review, commit, and re-run.
5
- *
6
- * Why patch the SPEC (not the IR)?
7
- *
8
- * The compiler does destructive normalisation in `lower()`: sub-agent
9
- * maps become arrays, role names get alphabetically sorted, secrets get
10
- * rewritten to env-var refs via `lowerSecret`, permission rules get
11
- * de-duped and re-ordered. The IR is intentionally lossy and ordering-
12
- * canonical because its job is to feed codegen, not round-trip to YAML.
13
- *
14
- * A patch at the IR layer therefore can't write back to YAML — there's
15
- * no path from a frozen sorted array back to the source-author's map
16
- * with their comments and field ordering. Patching at the SPEC layer
17
- * (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
18
- * --write-back` rewrites the user's file with their comments preserved
19
- * and only the touched values changed.
20
- *
21
- * The `yaml` package's `parseDocument` gives us the CST we need. The
22
- * Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
23
- * that operate on the live AST; `toString()` renders back with comment
24
- * and key-order fidelity.
25
- *
26
- * IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
27
- * time optimisations that run AFTER lowering. They don't get conflated
28
- * with eval-driven mutation. The two systems share nothing; that's
29
- * intentional.
30
- *
31
- * Catalog layer: F2 (compiler periphery). Brief: 278.
32
- */
33
- import { CrewhausError } from "@crewhaus/errors";
34
- import { type Spec, parseSpec } from "@crewhaus/spec";
35
- import { parseDocument } from "yaml";
36
- import { z } from "zod";
37
-
38
- export class SpecPatchError extends CrewhausError {
39
- override readonly name = "SpecPatchError";
40
- constructor(message: string, cause?: unknown) {
41
- super("compiler", message, cause);
42
- }
43
- }
44
-
45
- export type SpecPatchOp = "replace" | "add" | "remove";
46
-
47
- /**
48
- * A structured edit to a spec. `path` is a property-key chain (no array
49
- * indices today — the spec's mutation surface is all object fields). The
50
- * `target` discriminator must match the spec being patched; the validator
51
- * refuses cross-target patches so an optimizer can't pass an `IrCli`
52
- * patch to a `pipeline` spec.
53
- */
54
- export type SpecPatch = {
55
- readonly target: Spec["target"];
56
- readonly path: ReadonlyArray<string>;
57
- readonly op: SpecPatchOp;
58
- /** Required for `"replace"` and `"add"`; ignored for `"remove"`. */
59
- readonly value?: unknown;
60
- /**
61
- * Optional rationale string for audit / write-back commit messages.
62
- * The optimizer fills this with the mutation kind + observed delta.
63
- */
64
- readonly rationale?: string;
65
- };
66
-
67
- const specPatchSchema = z.object({
68
- target: z.string().min(1),
69
- path: z.array(z.string().min(1)).min(1),
70
- op: z.enum(["replace", "add", "remove"]),
71
- value: z.unknown().optional(),
72
- rationale: z.string().optional(),
73
- });
74
-
75
- export type ApplySpecPatchResult = {
76
- /** The mutated YAML text, with comments and key order preserved. */
77
- readonly yaml: string;
78
- /** The re-parsed Spec after the patch. */
79
- readonly spec: Spec;
80
- };
81
-
82
- /**
83
- * Apply a structured patch to a YAML spec source. Uses the `yaml`
84
- * package's CST so comments and key order survive the round-trip; the
85
- * only bytes that change are the ones the patch targets.
86
- *
87
- * Validates the resulting document against the Spec schema; throws
88
- * `SpecPatchError` if the mutation produces invalid spec.
89
- */
90
- export function applySpecPatch(yamlText: string, patch: SpecPatch): ApplySpecPatchResult {
91
- const parsed = specPatchSchema.safeParse(patch);
92
- if (!parsed.success) {
93
- throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
94
- }
95
- let doc: ReturnType<typeof parseDocument>;
96
- try {
97
- doc = parseDocument(yamlText);
98
- } catch (err) {
99
- throw new SpecPatchError("input YAML is not parseable", err);
100
- }
101
- // Verify the patch matches the spec's target. Cheap pre-check so we
102
- // fail with a useful message before the Zod validation later.
103
- const docTarget = doc.getIn(["target"]);
104
- if (typeof docTarget === "string" && docTarget !== patch.target) {
105
- throw new SpecPatchError(
106
- `patch target "${patch.target}" does not match spec target "${docTarget}"`,
107
- );
108
- }
109
-
110
- const path = [...patch.path];
111
- switch (patch.op) {
112
- case "replace": {
113
- if (!doc.hasIn(path)) {
114
- throw new SpecPatchError(`cannot replace ${formatPath(path)}: path does not exist`);
115
- }
116
- doc.setIn(path, patch.value);
117
- break;
118
- }
119
- case "add": {
120
- if (doc.hasIn(path)) {
121
- throw new SpecPatchError(
122
- `cannot add ${formatPath(path)}: path already exists (use "replace")`,
123
- );
124
- }
125
- doc.setIn(path, patch.value);
126
- break;
127
- }
128
- case "remove": {
129
- if (!doc.hasIn(path)) {
130
- throw new SpecPatchError(`cannot remove ${formatPath(path)}: path does not exist`);
131
- }
132
- doc.deleteIn(path);
133
- break;
134
- }
135
- }
136
-
137
- const newYaml = doc.toString();
138
- let spec: Spec;
139
- try {
140
- spec = parseSpec(newYaml);
141
- } catch (err) {
142
- throw new SpecPatchError(`patched YAML failed spec validation: ${(err as Error).message}`, err);
143
- }
144
- return { yaml: newYaml, spec };
145
- }
146
-
147
- /**
148
- * Type-check a patch against a parsed Spec WITHOUT applying it. Used
149
- * by the optimizer to refuse cross-target patches and patches that
150
- * reference paths outside the optimisation surface.
151
- *
152
- * The path-validity check is the "soft" version: it verifies the path
153
- * is structurally reachable (each segment names a defined field). It
154
- * does NOT instantiate the patched value — `applySpecPatch` does that
155
- * via `parseSpec`.
156
- */
157
- export function validatePatch(spec: Spec, patch: SpecPatch): void {
158
- const parsed = specPatchSchema.safeParse(patch);
159
- if (!parsed.success) {
160
- throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
161
- }
162
- if (patch.target !== spec.target) {
163
- throw new SpecPatchError(
164
- `patch target "${patch.target}" does not match spec target "${spec.target}"`,
165
- );
166
- }
167
- if (!isOptimizable(spec.target, patch.path)) {
168
- throw new SpecPatchError(
169
- `path ${formatPath(patch.path)} is not listed in OPTIMIZABLE_PATHS for target "${spec.target}"; add it to packages/spec-patch/src/index.ts if it's intended to be tunable`,
170
- );
171
- }
172
- }
173
-
174
- function formatPath(path: ReadonlyArray<string>): string {
175
- return path.join(".");
176
- }
177
-
178
- /**
179
- * Per-target whitelist of mutation paths the active optimizer is
180
- * allowed to touch. Adding a new field here is the explicit signal that
181
- * it's safe to autotune. Skipping this list means the optimizer can
182
- * only mutate prompts (the default), preserving the "spec safety floor"
183
- * that an optimizer can't accidentally rewrite security-critical fields
184
- * like `permissions.mode` or `model_router` rules.
185
- */
186
- export const OPTIMIZABLE_PATHS: Readonly<
187
- Record<Spec["target"], ReadonlyArray<ReadonlyArray<string>>>
188
- > = Object.freeze({
189
- cli: Object.freeze([
190
- Object.freeze(["agent", "instructions"]),
191
- Object.freeze(["failure_taxonomy"]),
192
- Object.freeze(["compaction", "threshold"]),
193
- // Pillar 2 active context curation — eval-optimizer can flip the
194
- // semantic-dedupe + relevance-reorder pass on/off and tune its
195
- // similarity threshold, which dominates input-token cost on long runs.
196
- Object.freeze(["compaction", "curate"]),
197
- Object.freeze(["compaction", "dedupeThreshold"]),
198
- Object.freeze(["compaction", "relevanceTopK"]),
199
- // Pillar 3 sink-side fabric — egress policy + intent-gate thresholds
200
- // are tunable so the eval-optimizer can find the sweet spot between
201
- // false-positive denials and false-negative exfil bypasses.
202
- Object.freeze(["security", "egressPolicy"]),
203
- Object.freeze(["security", "justification"]),
204
- // §47 blockchain subsystem (slice 0). Whole-block replacement so the
205
- // optimizer can tune `chains[*].finality.count`, `chains[*].rpcPolicy`,
206
- // `transaction_policy.maxValueUsd`, and `transaction_policy.simulationRequired`
207
- // by patching their parent block.
208
- Object.freeze(["chains"]),
209
- Object.freeze(["transaction_policy"]),
210
- ]),
211
- workflow: Object.freeze([
212
- Object.freeze(["steps"]),
213
- Object.freeze(["failure_taxonomy"]),
214
- Object.freeze(["chains"]),
215
- Object.freeze(["transaction_policy"]),
216
- ]) /* whole-step replacement allowed */,
217
- channel: Object.freeze([
218
- Object.freeze(["agent", "instructions"]),
219
- Object.freeze(["failure_taxonomy"]),
220
- Object.freeze(["chains"]),
221
- Object.freeze(["transaction_policy"]),
222
- ]),
223
- graph: Object.freeze([
224
- Object.freeze(["nodes"]),
225
- Object.freeze(["failure_taxonomy"]),
226
- Object.freeze(["chains"]),
227
- Object.freeze(["transaction_policy"]),
228
- ]),
229
- managed: Object.freeze([
230
- Object.freeze(["agent", "instructions"]),
231
- Object.freeze(["failure_taxonomy"]),
232
- ]),
233
- pipeline: Object.freeze([
234
- Object.freeze(["agent", "instructions"]),
235
- Object.freeze(["failure_taxonomy"]),
236
- Object.freeze(["indexing", "chunkSize"]),
237
- Object.freeze(["indexing", "chunkOverlap"]),
238
- Object.freeze(["retrieve", "defaultK"]),
239
- ]),
240
- crew: Object.freeze([
241
- Object.freeze(["roles"]),
242
- Object.freeze(["failure_taxonomy"]),
243
- Object.freeze(["chains"]),
244
- Object.freeze(["transaction_policy"]),
245
- ]) /* whole-role replacement */,
246
- research: Object.freeze([
247
- Object.freeze(["agent", "instructions"]),
248
- Object.freeze(["failure_taxonomy"]),
249
- Object.freeze(["retrieve", "maxDepth"]),
250
- Object.freeze(["chains"]),
251
- Object.freeze(["transaction_policy"]),
252
- ]),
253
- batch: Object.freeze([
254
- Object.freeze(["agent", "instructions"]),
255
- Object.freeze(["failure_taxonomy"]),
256
- Object.freeze(["chains"]),
257
- Object.freeze(["transaction_policy"]),
258
- ]),
259
- voice: Object.freeze([
260
- Object.freeze(["agent", "instructions"]),
261
- Object.freeze(["failure_taxonomy"]),
262
- ]),
263
- browser: Object.freeze([
264
- Object.freeze(["agent", "instructions"]),
265
- Object.freeze(["failure_taxonomy"]),
266
- ]),
267
- eval: Object.freeze([
268
- Object.freeze(["agent", "instructions"]),
269
- Object.freeze(["failure_taxonomy"]),
270
- ]),
271
- // §47 onchain daemon: full cross-cutting blocks are optimizable.
272
- onchain: Object.freeze([
273
- Object.freeze(["agent", "instructions"]),
274
- Object.freeze(["failure_taxonomy"]),
275
- Object.freeze(["chains"]),
276
- Object.freeze(["triggers"]),
277
- Object.freeze(["transaction_policy"]),
278
- Object.freeze(["idempotencyWindowMs"]),
279
- ]),
280
- // §47 onchain-game: instructions, game.objective, and the policy are
281
- // the productive knobs; move-timeout-ms is the realtime quality knob.
282
- "onchain-game": Object.freeze([
283
- Object.freeze(["agent", "instructions"]),
284
- Object.freeze(["failure_taxonomy"]),
285
- Object.freeze(["game"]),
286
- Object.freeze(["transaction_policy"]),
287
- ]),
288
- });
289
-
290
- function isOptimizable(target: Spec["target"], path: ReadonlyArray<string>): boolean {
291
- const allowed = OPTIMIZABLE_PATHS[target];
292
- for (const ok of allowed) {
293
- if (ok.length !== path.length) continue;
294
- let match = true;
295
- for (let i = 0; i < ok.length; i++) {
296
- if (ok[i] !== path[i]) {
297
- match = false;
298
- break;
299
- }
300
- }
301
- if (match) return true;
302
- }
303
- // Allow patches with a prefix that matches an optimizable path
304
- // (e.g. `["nodes", "0", "instructions"]` if `["nodes"]` is whitelisted)
305
- // so the optimizer can do fine-grained updates without listing every
306
- // sub-path.
307
- for (const ok of allowed) {
308
- if (path.length < ok.length) continue;
309
- let match = true;
310
- for (let i = 0; i < ok.length; i++) {
311
- if (ok[i] !== path[i]) {
312
- match = false;
313
- break;
314
- }
315
- }
316
- if (match) return true;
317
- }
318
- return false;
319
- }
320
-
321
- /**
322
- * Format a YAML header comment to prepend to a written-back file. The
323
- * orchestrator's `--write-back` writes this above the original spec so
324
- * a human reviewer can see when and by what an optimisation pass ran.
325
- */
326
- export function formatWriteBackHeader(opts: {
327
- readonly runId: string;
328
- readonly mutator: string;
329
- readonly scoreBefore: number;
330
- readonly scoreAfter: number;
331
- readonly iterations: number;
332
- readonly timestamp?: string;
333
- }): string {
334
- const ts = opts.timestamp ?? new Date().toISOString();
335
- const delta = (opts.scoreAfter - opts.scoreBefore).toFixed(3);
336
- return [
337
- `# crewhaus optimize: runId ${opts.runId}`,
338
- `# - mutator: ${opts.mutator}`,
339
- `# - iterations: ${opts.iterations}`,
340
- `# - score: ${opts.scoreBefore.toFixed(3)} → ${opts.scoreAfter.toFixed(3)} (Δ ${delta})`,
341
- `# - generated: ${ts}`,
342
- "",
343
- ].join("\n");
344
- }