@crewhaus/spec-patch 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +106 -0
- package/dist/index.js +294 -0
- package/package.json +10 -7
- package/src/index.test.ts +0 -236
- package/src/index.ts +0 -344
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pillar 2 — `spec-patch`. The structured mutation primitive that the
|
|
3
|
+
* active eval optimizer uses to translate "the prompt that scored higher"
|
|
4
|
+
* back into a YAML edit the user can review, commit, and re-run.
|
|
5
|
+
*
|
|
6
|
+
* Why patch the SPEC (not the IR)?
|
|
7
|
+
*
|
|
8
|
+
* The compiler does destructive normalisation in `lower()`: sub-agent
|
|
9
|
+
* maps become arrays, role names get alphabetically sorted, secrets get
|
|
10
|
+
* rewritten to env-var refs via `lowerSecret`, permission rules get
|
|
11
|
+
* de-duped and re-ordered. The IR is intentionally lossy and ordering-
|
|
12
|
+
* canonical because its job is to feed codegen, not round-trip to YAML.
|
|
13
|
+
*
|
|
14
|
+
* A patch at the IR layer therefore can't write back to YAML — there's
|
|
15
|
+
* no path from a frozen sorted array back to the source-author's map
|
|
16
|
+
* with their comments and field ordering. Patching at the SPEC layer
|
|
17
|
+
* (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
|
|
18
|
+
* --write-back` rewrites the user's file with their comments preserved
|
|
19
|
+
* and only the touched values changed.
|
|
20
|
+
*
|
|
21
|
+
* The `yaml` package's `parseDocument` gives us the CST we need. The
|
|
22
|
+
* Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
|
|
23
|
+
* that operate on the live AST; `toString()` renders back with comment
|
|
24
|
+
* and key-order fidelity.
|
|
25
|
+
*
|
|
26
|
+
* IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
|
|
27
|
+
* time optimisations that run AFTER lowering. They don't get conflated
|
|
28
|
+
* with eval-driven mutation. The two systems share nothing; that's
|
|
29
|
+
* intentional.
|
|
30
|
+
*
|
|
31
|
+
* Catalog layer: F2 (compiler periphery). Brief: 278.
|
|
32
|
+
*/
|
|
33
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
34
|
+
import { type Spec } from "@crewhaus/spec";
|
|
35
|
+
export declare class SpecPatchError extends CrewhausError {
|
|
36
|
+
readonly name = "SpecPatchError";
|
|
37
|
+
constructor(message: string, cause?: unknown);
|
|
38
|
+
}
|
|
39
|
+
export type SpecPatchOp = "replace" | "add" | "remove";
|
|
40
|
+
/**
|
|
41
|
+
* A structured edit to a spec. `path` is a property-key chain (no array
|
|
42
|
+
* indices today — the spec's mutation surface is all object fields). The
|
|
43
|
+
* `target` discriminator must match the spec being patched; the validator
|
|
44
|
+
* refuses cross-target patches so an optimizer can't pass an `IrCli`
|
|
45
|
+
* patch to a `pipeline` spec.
|
|
46
|
+
*/
|
|
47
|
+
export type SpecPatch = {
|
|
48
|
+
readonly target: Spec["target"];
|
|
49
|
+
readonly path: ReadonlyArray<string>;
|
|
50
|
+
readonly op: SpecPatchOp;
|
|
51
|
+
/** Required for `"replace"` and `"add"`; ignored for `"remove"`. */
|
|
52
|
+
readonly value?: unknown;
|
|
53
|
+
/**
|
|
54
|
+
* Optional rationale string for audit / write-back commit messages.
|
|
55
|
+
* The optimizer fills this with the mutation kind + observed delta.
|
|
56
|
+
*/
|
|
57
|
+
readonly rationale?: string;
|
|
58
|
+
};
|
|
59
|
+
export type ApplySpecPatchResult = {
|
|
60
|
+
/** The mutated YAML text, with comments and key order preserved. */
|
|
61
|
+
readonly yaml: string;
|
|
62
|
+
/** The re-parsed Spec after the patch. */
|
|
63
|
+
readonly spec: Spec;
|
|
64
|
+
};
|
|
65
|
+
/**
|
|
66
|
+
* Apply a structured patch to a YAML spec source. Uses the `yaml`
|
|
67
|
+
* package's CST so comments and key order survive the round-trip; the
|
|
68
|
+
* only bytes that change are the ones the patch targets.
|
|
69
|
+
*
|
|
70
|
+
* Validates the resulting document against the Spec schema; throws
|
|
71
|
+
* `SpecPatchError` if the mutation produces invalid spec.
|
|
72
|
+
*/
|
|
73
|
+
export declare function applySpecPatch(yamlText: string, patch: SpecPatch): ApplySpecPatchResult;
|
|
74
|
+
/**
|
|
75
|
+
* Type-check a patch against a parsed Spec WITHOUT applying it. Used
|
|
76
|
+
* by the optimizer to refuse cross-target patches and patches that
|
|
77
|
+
* reference paths outside the optimisation surface.
|
|
78
|
+
*
|
|
79
|
+
* The path-validity check is the "soft" version: it verifies the path
|
|
80
|
+
* is structurally reachable (each segment names a defined field). It
|
|
81
|
+
* does NOT instantiate the patched value — `applySpecPatch` does that
|
|
82
|
+
* via `parseSpec`.
|
|
83
|
+
*/
|
|
84
|
+
export declare function validatePatch(spec: Spec, patch: SpecPatch): void;
|
|
85
|
+
/**
|
|
86
|
+
* Per-target whitelist of mutation paths the active optimizer is
|
|
87
|
+
* allowed to touch. Adding a new field here is the explicit signal that
|
|
88
|
+
* it's safe to autotune. Skipping this list means the optimizer can
|
|
89
|
+
* only mutate prompts (the default), preserving the "spec safety floor"
|
|
90
|
+
* that an optimizer can't accidentally rewrite security-critical fields
|
|
91
|
+
* like `permissions.mode` or `model_router` rules.
|
|
92
|
+
*/
|
|
93
|
+
export declare const OPTIMIZABLE_PATHS: Readonly<Record<Spec["target"], ReadonlyArray<ReadonlyArray<string>>>>;
|
|
94
|
+
/**
|
|
95
|
+
* Format a YAML header comment to prepend to a written-back file. The
|
|
96
|
+
* orchestrator's `--write-back` writes this above the original spec so
|
|
97
|
+
* a human reviewer can see when and by what an optimisation pass ran.
|
|
98
|
+
*/
|
|
99
|
+
export declare function formatWriteBackHeader(opts: {
|
|
100
|
+
readonly runId: string;
|
|
101
|
+
readonly mutator: string;
|
|
102
|
+
readonly scoreBefore: number;
|
|
103
|
+
readonly scoreAfter: number;
|
|
104
|
+
readonly iterations: number;
|
|
105
|
+
readonly timestamp?: string;
|
|
106
|
+
}): string;
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pillar 2 — `spec-patch`. The structured mutation primitive that the
|
|
3
|
+
* active eval optimizer uses to translate "the prompt that scored higher"
|
|
4
|
+
* back into a YAML edit the user can review, commit, and re-run.
|
|
5
|
+
*
|
|
6
|
+
* Why patch the SPEC (not the IR)?
|
|
7
|
+
*
|
|
8
|
+
* The compiler does destructive normalisation in `lower()`: sub-agent
|
|
9
|
+
* maps become arrays, role names get alphabetically sorted, secrets get
|
|
10
|
+
* rewritten to env-var refs via `lowerSecret`, permission rules get
|
|
11
|
+
* de-duped and re-ordered. The IR is intentionally lossy and ordering-
|
|
12
|
+
* canonical because its job is to feed codegen, not round-trip to YAML.
|
|
13
|
+
*
|
|
14
|
+
* A patch at the IR layer therefore can't write back to YAML — there's
|
|
15
|
+
* no path from a frozen sorted array back to the source-author's map
|
|
16
|
+
* with their comments and field ordering. Patching at the SPEC layer
|
|
17
|
+
* (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
|
|
18
|
+
* --write-back` rewrites the user's file with their comments preserved
|
|
19
|
+
* and only the touched values changed.
|
|
20
|
+
*
|
|
21
|
+
* The `yaml` package's `parseDocument` gives us the CST we need. The
|
|
22
|
+
* Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
|
|
23
|
+
* that operate on the live AST; `toString()` renders back with comment
|
|
24
|
+
* and key-order fidelity.
|
|
25
|
+
*
|
|
26
|
+
* IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
|
|
27
|
+
* time optimisations that run AFTER lowering. They don't get conflated
|
|
28
|
+
* with eval-driven mutation. The two systems share nothing; that's
|
|
29
|
+
* intentional.
|
|
30
|
+
*
|
|
31
|
+
* Catalog layer: F2 (compiler periphery). Brief: 278.
|
|
32
|
+
*/
|
|
33
|
+
import { CrewhausError } from "@crewhaus/errors";
|
|
34
|
+
import { parseSpec } from "@crewhaus/spec";
|
|
35
|
+
import { parseDocument } from "yaml";
|
|
36
|
+
import { z } from "zod";
|
|
37
|
+
export class SpecPatchError extends CrewhausError {
|
|
38
|
+
name = "SpecPatchError";
|
|
39
|
+
constructor(message, cause) {
|
|
40
|
+
super("compiler", message, cause);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const specPatchSchema = z.object({
|
|
44
|
+
target: z.string().min(1),
|
|
45
|
+
path: z.array(z.string().min(1)).min(1),
|
|
46
|
+
op: z.enum(["replace", "add", "remove"]),
|
|
47
|
+
value: z.unknown().optional(),
|
|
48
|
+
rationale: z.string().optional(),
|
|
49
|
+
});
|
|
50
|
+
/**
|
|
51
|
+
* Apply a structured patch to a YAML spec source. Uses the `yaml`
|
|
52
|
+
* package's CST so comments and key order survive the round-trip; the
|
|
53
|
+
* only bytes that change are the ones the patch targets.
|
|
54
|
+
*
|
|
55
|
+
* Validates the resulting document against the Spec schema; throws
|
|
56
|
+
* `SpecPatchError` if the mutation produces invalid spec.
|
|
57
|
+
*/
|
|
58
|
+
export function applySpecPatch(yamlText, patch) {
|
|
59
|
+
const parsed = specPatchSchema.safeParse(patch);
|
|
60
|
+
if (!parsed.success) {
|
|
61
|
+
throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
|
|
62
|
+
}
|
|
63
|
+
let doc;
|
|
64
|
+
try {
|
|
65
|
+
doc = parseDocument(yamlText);
|
|
66
|
+
}
|
|
67
|
+
catch (err) {
|
|
68
|
+
throw new SpecPatchError("input YAML is not parseable", err);
|
|
69
|
+
}
|
|
70
|
+
// Verify the patch matches the spec's target. Cheap pre-check so we
|
|
71
|
+
// fail with a useful message before the Zod validation later.
|
|
72
|
+
const docTarget = doc.getIn(["target"]);
|
|
73
|
+
if (typeof docTarget === "string" && docTarget !== patch.target) {
|
|
74
|
+
throw new SpecPatchError(`patch target "${patch.target}" does not match spec target "${docTarget}"`);
|
|
75
|
+
}
|
|
76
|
+
const path = [...patch.path];
|
|
77
|
+
switch (patch.op) {
|
|
78
|
+
case "replace": {
|
|
79
|
+
if (!doc.hasIn(path)) {
|
|
80
|
+
throw new SpecPatchError(`cannot replace ${formatPath(path)}: path does not exist`);
|
|
81
|
+
}
|
|
82
|
+
doc.setIn(path, patch.value);
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
case "add": {
|
|
86
|
+
if (doc.hasIn(path)) {
|
|
87
|
+
throw new SpecPatchError(`cannot add ${formatPath(path)}: path already exists (use "replace")`);
|
|
88
|
+
}
|
|
89
|
+
doc.setIn(path, patch.value);
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
case "remove": {
|
|
93
|
+
if (!doc.hasIn(path)) {
|
|
94
|
+
throw new SpecPatchError(`cannot remove ${formatPath(path)}: path does not exist`);
|
|
95
|
+
}
|
|
96
|
+
doc.deleteIn(path);
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
const newYaml = doc.toString();
|
|
101
|
+
let spec;
|
|
102
|
+
try {
|
|
103
|
+
spec = parseSpec(newYaml);
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
throw new SpecPatchError(`patched YAML failed spec validation: ${err.message}`, err);
|
|
107
|
+
}
|
|
108
|
+
return { yaml: newYaml, spec };
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Type-check a patch against a parsed Spec WITHOUT applying it. Used
|
|
112
|
+
* by the optimizer to refuse cross-target patches and patches that
|
|
113
|
+
* reference paths outside the optimisation surface.
|
|
114
|
+
*
|
|
115
|
+
* The path-validity check is the "soft" version: it verifies the path
|
|
116
|
+
* is structurally reachable (each segment names a defined field). It
|
|
117
|
+
* does NOT instantiate the patched value — `applySpecPatch` does that
|
|
118
|
+
* via `parseSpec`.
|
|
119
|
+
*/
|
|
120
|
+
export function validatePatch(spec, patch) {
|
|
121
|
+
const parsed = specPatchSchema.safeParse(patch);
|
|
122
|
+
if (!parsed.success) {
|
|
123
|
+
throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
|
|
124
|
+
}
|
|
125
|
+
if (patch.target !== spec.target) {
|
|
126
|
+
throw new SpecPatchError(`patch target "${patch.target}" does not match spec target "${spec.target}"`);
|
|
127
|
+
}
|
|
128
|
+
if (!isOptimizable(spec.target, patch.path)) {
|
|
129
|
+
throw new SpecPatchError(`path ${formatPath(patch.path)} is not listed in OPTIMIZABLE_PATHS for target "${spec.target}"; add it to packages/spec-patch/src/index.ts if it's intended to be tunable`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function formatPath(path) {
|
|
133
|
+
return path.join(".");
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Per-target whitelist of mutation paths the active optimizer is
|
|
137
|
+
* allowed to touch. Adding a new field here is the explicit signal that
|
|
138
|
+
* it's safe to autotune. Skipping this list means the optimizer can
|
|
139
|
+
* only mutate prompts (the default), preserving the "spec safety floor"
|
|
140
|
+
* that an optimizer can't accidentally rewrite security-critical fields
|
|
141
|
+
* like `permissions.mode` or `model_router` rules.
|
|
142
|
+
*/
|
|
143
|
+
export const OPTIMIZABLE_PATHS = Object.freeze({
|
|
144
|
+
cli: Object.freeze([
|
|
145
|
+
Object.freeze(["agent", "instructions"]),
|
|
146
|
+
Object.freeze(["failure_taxonomy"]),
|
|
147
|
+
Object.freeze(["compaction", "threshold"]),
|
|
148
|
+
// Pillar 2 active context curation — eval-optimizer can flip the
|
|
149
|
+
// semantic-dedupe + relevance-reorder pass on/off and tune its
|
|
150
|
+
// similarity threshold, which dominates input-token cost on long runs.
|
|
151
|
+
Object.freeze(["compaction", "curate"]),
|
|
152
|
+
Object.freeze(["compaction", "dedupeThreshold"]),
|
|
153
|
+
Object.freeze(["compaction", "relevanceTopK"]),
|
|
154
|
+
// Pillar 3 sink-side fabric — egress policy + intent-gate thresholds
|
|
155
|
+
// are tunable so the eval-optimizer can find the sweet spot between
|
|
156
|
+
// false-positive denials and false-negative exfil bypasses.
|
|
157
|
+
Object.freeze(["security", "egressPolicy"]),
|
|
158
|
+
Object.freeze(["security", "justification"]),
|
|
159
|
+
// §47 blockchain subsystem (slice 0). Whole-block replacement so the
|
|
160
|
+
// optimizer can tune `chains[*].finality.count`, `chains[*].rpcPolicy`,
|
|
161
|
+
// `transaction_policy.maxValueWei`, and `transaction_policy.simulationRequired`
|
|
162
|
+
// by patching their parent block.
|
|
163
|
+
Object.freeze(["chains"]),
|
|
164
|
+
Object.freeze(["transaction_policy"]),
|
|
165
|
+
]),
|
|
166
|
+
workflow: Object.freeze([
|
|
167
|
+
Object.freeze(["steps"]),
|
|
168
|
+
Object.freeze(["failure_taxonomy"]),
|
|
169
|
+
Object.freeze(["chains"]),
|
|
170
|
+
Object.freeze(["transaction_policy"]),
|
|
171
|
+
]) /* whole-step replacement allowed */,
|
|
172
|
+
channel: Object.freeze([
|
|
173
|
+
Object.freeze(["agent", "instructions"]),
|
|
174
|
+
Object.freeze(["failure_taxonomy"]),
|
|
175
|
+
Object.freeze(["chains"]),
|
|
176
|
+
Object.freeze(["transaction_policy"]),
|
|
177
|
+
]),
|
|
178
|
+
graph: Object.freeze([
|
|
179
|
+
Object.freeze(["nodes"]),
|
|
180
|
+
Object.freeze(["failure_taxonomy"]),
|
|
181
|
+
Object.freeze(["chains"]),
|
|
182
|
+
Object.freeze(["transaction_policy"]),
|
|
183
|
+
]),
|
|
184
|
+
managed: Object.freeze([
|
|
185
|
+
Object.freeze(["agent", "instructions"]),
|
|
186
|
+
Object.freeze(["failure_taxonomy"]),
|
|
187
|
+
]),
|
|
188
|
+
pipeline: Object.freeze([
|
|
189
|
+
Object.freeze(["agent", "instructions"]),
|
|
190
|
+
Object.freeze(["failure_taxonomy"]),
|
|
191
|
+
Object.freeze(["indexing", "chunkSize"]),
|
|
192
|
+
Object.freeze(["indexing", "chunkOverlap"]),
|
|
193
|
+
Object.freeze(["retrieve", "defaultK"]),
|
|
194
|
+
]),
|
|
195
|
+
crew: Object.freeze([
|
|
196
|
+
Object.freeze(["roles"]),
|
|
197
|
+
Object.freeze(["failure_taxonomy"]),
|
|
198
|
+
Object.freeze(["chains"]),
|
|
199
|
+
Object.freeze(["transaction_policy"]),
|
|
200
|
+
]) /* whole-role replacement */,
|
|
201
|
+
research: Object.freeze([
|
|
202
|
+
Object.freeze(["agent", "instructions"]),
|
|
203
|
+
Object.freeze(["failure_taxonomy"]),
|
|
204
|
+
Object.freeze(["retrieve", "maxDepth"]),
|
|
205
|
+
Object.freeze(["chains"]),
|
|
206
|
+
Object.freeze(["transaction_policy"]),
|
|
207
|
+
]),
|
|
208
|
+
batch: Object.freeze([
|
|
209
|
+
Object.freeze(["agent", "instructions"]),
|
|
210
|
+
Object.freeze(["failure_taxonomy"]),
|
|
211
|
+
Object.freeze(["chains"]),
|
|
212
|
+
Object.freeze(["transaction_policy"]),
|
|
213
|
+
]),
|
|
214
|
+
voice: Object.freeze([
|
|
215
|
+
Object.freeze(["agent", "instructions"]),
|
|
216
|
+
Object.freeze(["failure_taxonomy"]),
|
|
217
|
+
]),
|
|
218
|
+
browser: Object.freeze([
|
|
219
|
+
Object.freeze(["agent", "instructions"]),
|
|
220
|
+
Object.freeze(["failure_taxonomy"]),
|
|
221
|
+
]),
|
|
222
|
+
eval: Object.freeze([
|
|
223
|
+
Object.freeze(["agent", "instructions"]),
|
|
224
|
+
Object.freeze(["failure_taxonomy"]),
|
|
225
|
+
]),
|
|
226
|
+
// §47 onchain daemon: full cross-cutting blocks are optimizable.
|
|
227
|
+
onchain: Object.freeze([
|
|
228
|
+
Object.freeze(["agent", "instructions"]),
|
|
229
|
+
Object.freeze(["failure_taxonomy"]),
|
|
230
|
+
Object.freeze(["chains"]),
|
|
231
|
+
Object.freeze(["triggers"]),
|
|
232
|
+
Object.freeze(["transaction_policy"]),
|
|
233
|
+
Object.freeze(["idempotencyWindowMs"]),
|
|
234
|
+
]),
|
|
235
|
+
// §47 onchain-game: instructions, game.objective, and the policy are
|
|
236
|
+
// the productive knobs; move-timeout-ms is the realtime quality knob.
|
|
237
|
+
"onchain-game": Object.freeze([
|
|
238
|
+
Object.freeze(["agent", "instructions"]),
|
|
239
|
+
Object.freeze(["failure_taxonomy"]),
|
|
240
|
+
Object.freeze(["game"]),
|
|
241
|
+
Object.freeze(["transaction_policy"]),
|
|
242
|
+
]),
|
|
243
|
+
});
|
|
244
|
+
function isOptimizable(target, path) {
|
|
245
|
+
const allowed = OPTIMIZABLE_PATHS[target];
|
|
246
|
+
for (const ok of allowed) {
|
|
247
|
+
if (ok.length !== path.length)
|
|
248
|
+
continue;
|
|
249
|
+
let match = true;
|
|
250
|
+
for (let i = 0; i < ok.length; i++) {
|
|
251
|
+
if (ok[i] !== path[i]) {
|
|
252
|
+
match = false;
|
|
253
|
+
break;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
if (match)
|
|
257
|
+
return true;
|
|
258
|
+
}
|
|
259
|
+
// Allow patches with a prefix that matches an optimizable path
|
|
260
|
+
// (e.g. `["nodes", "0", "instructions"]` if `["nodes"]` is whitelisted)
|
|
261
|
+
// so the optimizer can do fine-grained updates without listing every
|
|
262
|
+
// sub-path.
|
|
263
|
+
for (const ok of allowed) {
|
|
264
|
+
if (path.length < ok.length)
|
|
265
|
+
continue;
|
|
266
|
+
let match = true;
|
|
267
|
+
for (let i = 0; i < ok.length; i++) {
|
|
268
|
+
if (ok[i] !== path[i]) {
|
|
269
|
+
match = false;
|
|
270
|
+
break;
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
if (match)
|
|
274
|
+
return true;
|
|
275
|
+
}
|
|
276
|
+
return false;
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Format a YAML header comment to prepend to a written-back file. The
|
|
280
|
+
* orchestrator's `--write-back` writes this above the original spec so
|
|
281
|
+
* a human reviewer can see when and by what an optimisation pass ran.
|
|
282
|
+
*/
|
|
283
|
+
export function formatWriteBackHeader(opts) {
|
|
284
|
+
const ts = opts.timestamp ?? new Date().toISOString();
|
|
285
|
+
const delta = (opts.scoreAfter - opts.scoreBefore).toFixed(3);
|
|
286
|
+
return [
|
|
287
|
+
`# crewhaus optimize: runId ${opts.runId}`,
|
|
288
|
+
`# - mutator: ${opts.mutator}`,
|
|
289
|
+
`# - iterations: ${opts.iterations}`,
|
|
290
|
+
`# - score: ${opts.scoreBefore.toFixed(3)} → ${opts.scoreAfter.toFixed(3)} (Δ ${delta})`,
|
|
291
|
+
`# - generated: ${ts}`,
|
|
292
|
+
"",
|
|
293
|
+
].join("\n");
|
|
294
|
+
}
|
package/package.json
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crewhaus/spec-patch",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Pillar-2 patch infrastructure — apply a SpecPatch to a YAML source preserving comments and key order via the yaml CST. Drives the active eval optimizer's spec-level mutation loop.",
|
|
6
|
-
"main": "
|
|
7
|
-
"types": "
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
8
|
"exports": {
|
|
9
|
-
".":
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./dist/index.d.ts",
|
|
11
|
+
"import": "./dist/index.js"
|
|
12
|
+
}
|
|
10
13
|
},
|
|
11
14
|
"scripts": {
|
|
12
15
|
"test": "bun test src"
|
|
13
16
|
},
|
|
14
17
|
"dependencies": {
|
|
15
|
-
"@crewhaus/errors": "0.1.
|
|
16
|
-
"@crewhaus/spec": "0.1.
|
|
18
|
+
"@crewhaus/errors": "0.1.5",
|
|
19
|
+
"@crewhaus/spec": "0.1.5",
|
|
17
20
|
"yaml": "^2.6.0",
|
|
18
21
|
"zod": "^3.23.8"
|
|
19
22
|
},
|
|
@@ -35,5 +38,5 @@
|
|
|
35
38
|
"publishConfig": {
|
|
36
39
|
"access": "public"
|
|
37
40
|
},
|
|
38
|
-
"files": ["
|
|
41
|
+
"files": ["dist", "README.md", "LICENSE", "NOTICE"]
|
|
39
42
|
}
|
package/src/index.test.ts
DELETED
|
@@ -1,236 +0,0 @@
|
|
|
1
|
-
import { describe, expect, test } from "bun:test";
|
|
2
|
-
import { OPTIMIZABLE_PATHS, applySpecPatch, formatWriteBackHeader, validatePatch } from "./index";
|
|
3
|
-
|
|
4
|
-
const CLI_YAML = `# A simple CLI agent
|
|
5
|
-
target: cli
|
|
6
|
-
name: hello-cli
|
|
7
|
-
agent:
|
|
8
|
-
model: claude-sonnet-4-5
|
|
9
|
-
# The system prompt the model sees on every turn:
|
|
10
|
-
instructions: You are a helpful assistant.
|
|
11
|
-
tools:
|
|
12
|
-
- Read
|
|
13
|
-
- Write
|
|
14
|
-
`;
|
|
15
|
-
|
|
16
|
-
const PIPELINE_YAML = `target: pipeline
|
|
17
|
-
name: hello-rag
|
|
18
|
-
agent:
|
|
19
|
-
model: claude-sonnet-4-5
|
|
20
|
-
instructions: Answer using only the retrieved docs.
|
|
21
|
-
retrieve:
|
|
22
|
-
embedderModel: text-embedding-3-small
|
|
23
|
-
defaultK: 3
|
|
24
|
-
indexing:
|
|
25
|
-
# Default chunking — deliberately short for testing
|
|
26
|
-
chunkSize: 200
|
|
27
|
-
chunkOverlap: 20
|
|
28
|
-
documents:
|
|
29
|
-
- id: doc1
|
|
30
|
-
text: The capital of France is Paris.
|
|
31
|
-
`;
|
|
32
|
-
|
|
33
|
-
describe("applySpecPatch — round-trip", () => {
|
|
34
|
-
test("replaces agent.instructions and re-parses", () => {
|
|
35
|
-
const patch = {
|
|
36
|
-
target: "cli" as const,
|
|
37
|
-
path: ["agent", "instructions"] as const,
|
|
38
|
-
op: "replace" as const,
|
|
39
|
-
value: "Think step by step before answering.",
|
|
40
|
-
};
|
|
41
|
-
const { yaml, spec } = applySpecPatch(CLI_YAML, patch);
|
|
42
|
-
if (spec.target !== "cli") throw new Error("expected cli spec");
|
|
43
|
-
expect(spec.agent.instructions).toBe("Think step by step before answering.");
|
|
44
|
-
expect(yaml).toContain("instructions: Think step by step before answering.");
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
test("preserves leading comments and structural comments", () => {
|
|
48
|
-
const patch = {
|
|
49
|
-
target: "cli" as const,
|
|
50
|
-
path: ["agent", "instructions"] as const,
|
|
51
|
-
op: "replace" as const,
|
|
52
|
-
value: "Be concise.",
|
|
53
|
-
};
|
|
54
|
-
const { yaml } = applySpecPatch(CLI_YAML, patch);
|
|
55
|
-
expect(yaml).toContain("# A simple CLI agent");
|
|
56
|
-
expect(yaml).toContain("# The system prompt the model sees on every turn");
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
test("replaces a numeric field on a pipeline spec", () => {
|
|
60
|
-
const patch = {
|
|
61
|
-
target: "pipeline" as const,
|
|
62
|
-
path: ["indexing", "chunkOverlap"] as const,
|
|
63
|
-
op: "replace" as const,
|
|
64
|
-
value: 50,
|
|
65
|
-
};
|
|
66
|
-
const { yaml, spec } = applySpecPatch(PIPELINE_YAML, patch);
|
|
67
|
-
if (spec.target !== "pipeline") throw new Error("expected pipeline spec");
|
|
68
|
-
expect(spec.indexing.chunkOverlap).toBe(50);
|
|
69
|
-
expect(yaml).toContain("chunkOverlap: 50");
|
|
70
|
-
// Comment on indexing block survives
|
|
71
|
-
expect(yaml).toContain("# Default chunking — deliberately short for testing");
|
|
72
|
-
});
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
describe("applySpecPatch — error cases", () => {
|
|
76
|
-
test("cross-target patch is rejected with a clear error", () => {
|
|
77
|
-
expect(() =>
|
|
78
|
-
applySpecPatch(CLI_YAML, {
|
|
79
|
-
target: "pipeline",
|
|
80
|
-
path: ["agent", "instructions"],
|
|
81
|
-
op: "replace",
|
|
82
|
-
value: "x",
|
|
83
|
-
}),
|
|
84
|
-
).toThrow(/does not match spec target/);
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
test("non-existent replace path is rejected", () => {
|
|
88
|
-
expect(() =>
|
|
89
|
-
applySpecPatch(CLI_YAML, {
|
|
90
|
-
target: "cli",
|
|
91
|
-
path: ["agent", "nonexistent_field"],
|
|
92
|
-
op: "replace",
|
|
93
|
-
value: "x",
|
|
94
|
-
}),
|
|
95
|
-
).toThrow(/path does not exist/);
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
test("add to an existing path is rejected", () => {
|
|
99
|
-
expect(() =>
|
|
100
|
-
applySpecPatch(CLI_YAML, {
|
|
101
|
-
target: "cli",
|
|
102
|
-
path: ["agent", "instructions"],
|
|
103
|
-
op: "add",
|
|
104
|
-
value: "x",
|
|
105
|
-
}),
|
|
106
|
-
).toThrow(/path already exists/);
|
|
107
|
-
});
|
|
108
|
-
|
|
109
|
-
test("remove a non-existent path is rejected", () => {
|
|
110
|
-
expect(() =>
|
|
111
|
-
applySpecPatch(CLI_YAML, {
|
|
112
|
-
target: "cli",
|
|
113
|
-
path: ["agent", "nope"],
|
|
114
|
-
op: "remove",
|
|
115
|
-
}),
|
|
116
|
-
).toThrow(/path does not exist/);
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
test("invalid patch shape (empty path array) is rejected", () => {
|
|
120
|
-
expect(() =>
|
|
121
|
-
applySpecPatch(CLI_YAML, {
|
|
122
|
-
target: "cli",
|
|
123
|
-
// biome-ignore lint/suspicious/noExplicitAny: testing runtime validation
|
|
124
|
-
path: [] as any,
|
|
125
|
-
op: "replace",
|
|
126
|
-
value: "x",
|
|
127
|
-
}),
|
|
128
|
-
).toThrow(/patch shape is invalid/);
|
|
129
|
-
});
|
|
130
|
-
|
|
131
|
-
test("patched YAML that breaks spec schema is rejected", () => {
|
|
132
|
-
// Replacing model with a non-string number should be rejected.
|
|
133
|
-
expect(() =>
|
|
134
|
-
applySpecPatch(CLI_YAML, {
|
|
135
|
-
target: "cli",
|
|
136
|
-
path: ["agent", "model"],
|
|
137
|
-
op: "replace",
|
|
138
|
-
value: 42,
|
|
139
|
-
}),
|
|
140
|
-
).toThrow(/spec validation failed/);
|
|
141
|
-
});
|
|
142
|
-
});
|
|
143
|
-
|
|
144
|
-
describe("validatePatch", () => {
|
|
145
|
-
test("accepts a whitelisted optimizable path", () => {
|
|
146
|
-
const { spec } = applySpecPatch(PIPELINE_YAML, {
|
|
147
|
-
target: "pipeline",
|
|
148
|
-
path: ["indexing", "chunkOverlap"],
|
|
149
|
-
op: "replace",
|
|
150
|
-
value: 30,
|
|
151
|
-
});
|
|
152
|
-
expect(() =>
|
|
153
|
-
validatePatch(spec, {
|
|
154
|
-
target: "pipeline",
|
|
155
|
-
path: ["indexing", "chunkOverlap"],
|
|
156
|
-
op: "replace",
|
|
157
|
-
value: 40,
|
|
158
|
-
}),
|
|
159
|
-
).not.toThrow();
|
|
160
|
-
});
|
|
161
|
-
|
|
162
|
-
test("rejects a path that's not in OPTIMIZABLE_PATHS", () => {
|
|
163
|
-
const { spec } = applySpecPatch(CLI_YAML, {
|
|
164
|
-
target: "cli",
|
|
165
|
-
path: ["agent", "instructions"],
|
|
166
|
-
op: "replace",
|
|
167
|
-
value: "x",
|
|
168
|
-
});
|
|
169
|
-
expect(() =>
|
|
170
|
-
validatePatch(spec, {
|
|
171
|
-
target: "cli",
|
|
172
|
-
path: ["agent", "model"], // intentionally not in cli's OPTIMIZABLE_PATHS
|
|
173
|
-
op: "replace",
|
|
174
|
-
value: "opus-4",
|
|
175
|
-
}),
|
|
176
|
-
).toThrow(/not listed in OPTIMIZABLE_PATHS/);
|
|
177
|
-
});
|
|
178
|
-
|
|
179
|
-
test("cross-target patch is rejected before path-check", () => {
|
|
180
|
-
const { spec } = applySpecPatch(CLI_YAML, {
|
|
181
|
-
target: "cli",
|
|
182
|
-
path: ["agent", "instructions"],
|
|
183
|
-
op: "replace",
|
|
184
|
-
value: "x",
|
|
185
|
-
});
|
|
186
|
-
expect(() =>
|
|
187
|
-
validatePatch(spec, {
|
|
188
|
-
target: "pipeline",
|
|
189
|
-
path: ["agent", "instructions"],
|
|
190
|
-
op: "replace",
|
|
191
|
-
value: "y",
|
|
192
|
-
}),
|
|
193
|
-
).toThrow(/does not match spec target/);
|
|
194
|
-
});
|
|
195
|
-
});
|
|
196
|
-
|
|
197
|
-
describe("OPTIMIZABLE_PATHS", () => {
|
|
198
|
-
test("every shipped target shape has at least one whitelisted path", () => {
|
|
199
|
-
const targets = [
|
|
200
|
-
"cli",
|
|
201
|
-
"workflow",
|
|
202
|
-
"channel",
|
|
203
|
-
"graph",
|
|
204
|
-
"managed",
|
|
205
|
-
"pipeline",
|
|
206
|
-
"crew",
|
|
207
|
-
"research",
|
|
208
|
-
"batch",
|
|
209
|
-
"voice",
|
|
210
|
-
"browser",
|
|
211
|
-
"eval",
|
|
212
|
-
] as const;
|
|
213
|
-
for (const t of targets) {
|
|
214
|
-
expect(OPTIMIZABLE_PATHS[t]).toBeDefined();
|
|
215
|
-
expect(OPTIMIZABLE_PATHS[t].length).toBeGreaterThanOrEqual(1);
|
|
216
|
-
}
|
|
217
|
-
});
|
|
218
|
-
});
|
|
219
|
-
|
|
220
|
-
describe("formatWriteBackHeader", () => {
|
|
221
|
-
test("produces a deterministic header given a fixed timestamp", () => {
|
|
222
|
-
const header = formatWriteBackHeader({
|
|
223
|
-
runId: "opt_abc123",
|
|
224
|
-
mutator: "rule-based",
|
|
225
|
-
scoreBefore: 0.45,
|
|
226
|
-
scoreAfter: 0.78,
|
|
227
|
-
iterations: 12,
|
|
228
|
-
timestamp: "2026-05-10T12:00:00Z",
|
|
229
|
-
});
|
|
230
|
-
expect(header).toContain("runId opt_abc123");
|
|
231
|
-
expect(header).toContain("mutator: rule-based");
|
|
232
|
-
expect(header).toContain("iterations: 12");
|
|
233
|
-
expect(header).toContain("score: 0.450 → 0.780 (Δ 0.330)");
|
|
234
|
-
expect(header).toContain("2026-05-10T12:00:00Z");
|
|
235
|
-
});
|
|
236
|
-
});
|
package/src/index.ts
DELETED
|
@@ -1,344 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pillar 2 — `spec-patch`. The structured mutation primitive that the
|
|
3
|
-
* active eval optimizer uses to translate "the prompt that scored higher"
|
|
4
|
-
* back into a YAML edit the user can review, commit, and re-run.
|
|
5
|
-
*
|
|
6
|
-
* Why patch the SPEC (not the IR)?
|
|
7
|
-
*
|
|
8
|
-
* The compiler does destructive normalisation in `lower()`: sub-agent
|
|
9
|
-
* maps become arrays, role names get alphabetically sorted, secrets get
|
|
10
|
-
* rewritten to env-var refs via `lowerSecret`, permission rules get
|
|
11
|
-
* de-duped and re-ordered. The IR is intentionally lossy and ordering-
|
|
12
|
-
* canonical because its job is to feed codegen, not round-trip to YAML.
|
|
13
|
-
*
|
|
14
|
-
* A patch at the IR layer therefore can't write back to YAML — there's
|
|
15
|
-
* no path from a frozen sorted array back to the source-author's map
|
|
16
|
-
* with their comments and field ordering. Patching at the SPEC layer
|
|
17
|
-
* (i.e. the YAML AST) keeps the round-trip honest: a `crewhaus optimize
|
|
18
|
-
* --write-back` rewrites the user's file with their comments preserved
|
|
19
|
-
* and only the touched values changed.
|
|
20
|
-
*
|
|
21
|
-
* The `yaml` package's `parseDocument` gives us the CST we need. The
|
|
22
|
-
* Document API has `setIn(path, value)`, `getIn(path)`, `deleteIn(path)`
|
|
23
|
-
* that operate on the live AST; `toString()` renders back with comment
|
|
24
|
-
* and key-order fidelity.
|
|
25
|
-
*
|
|
26
|
-
* IR-passes (in `@crewhaus/ir-passes`) stay what they are — codegen-
|
|
27
|
-
* time optimisations that run AFTER lowering. They don't get conflated
|
|
28
|
-
* with eval-driven mutation. The two systems share nothing; that's
|
|
29
|
-
* intentional.
|
|
30
|
-
*
|
|
31
|
-
* Catalog layer: F2 (compiler periphery). Brief: 278.
|
|
32
|
-
*/
|
|
33
|
-
import { CrewhausError } from "@crewhaus/errors";
|
|
34
|
-
import { type Spec, parseSpec } from "@crewhaus/spec";
|
|
35
|
-
import { parseDocument } from "yaml";
|
|
36
|
-
import { z } from "zod";
|
|
37
|
-
|
|
38
|
-
export class SpecPatchError extends CrewhausError {
|
|
39
|
-
override readonly name = "SpecPatchError";
|
|
40
|
-
constructor(message: string, cause?: unknown) {
|
|
41
|
-
super("compiler", message, cause);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
export type SpecPatchOp = "replace" | "add" | "remove";
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* A structured edit to a spec. `path` is a property-key chain (no array
|
|
49
|
-
* indices today — the spec's mutation surface is all object fields). The
|
|
50
|
-
* `target` discriminator must match the spec being patched; the validator
|
|
51
|
-
* refuses cross-target patches so an optimizer can't pass an `IrCli`
|
|
52
|
-
* patch to a `pipeline` spec.
|
|
53
|
-
*/
|
|
54
|
-
export type SpecPatch = {
|
|
55
|
-
readonly target: Spec["target"];
|
|
56
|
-
readonly path: ReadonlyArray<string>;
|
|
57
|
-
readonly op: SpecPatchOp;
|
|
58
|
-
/** Required for `"replace"` and `"add"`; ignored for `"remove"`. */
|
|
59
|
-
readonly value?: unknown;
|
|
60
|
-
/**
|
|
61
|
-
* Optional rationale string for audit / write-back commit messages.
|
|
62
|
-
* The optimizer fills this with the mutation kind + observed delta.
|
|
63
|
-
*/
|
|
64
|
-
readonly rationale?: string;
|
|
65
|
-
};
|
|
66
|
-
|
|
67
|
-
const specPatchSchema = z.object({
|
|
68
|
-
target: z.string().min(1),
|
|
69
|
-
path: z.array(z.string().min(1)).min(1),
|
|
70
|
-
op: z.enum(["replace", "add", "remove"]),
|
|
71
|
-
value: z.unknown().optional(),
|
|
72
|
-
rationale: z.string().optional(),
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
export type ApplySpecPatchResult = {
|
|
76
|
-
/** The mutated YAML text, with comments and key order preserved. */
|
|
77
|
-
readonly yaml: string;
|
|
78
|
-
/** The re-parsed Spec after the patch. */
|
|
79
|
-
readonly spec: Spec;
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
/**
|
|
83
|
-
* Apply a structured patch to a YAML spec source. Uses the `yaml`
|
|
84
|
-
* package's CST so comments and key order survive the round-trip; the
|
|
85
|
-
* only bytes that change are the ones the patch targets.
|
|
86
|
-
*
|
|
87
|
-
* Validates the resulting document against the Spec schema; throws
|
|
88
|
-
* `SpecPatchError` if the mutation produces invalid spec.
|
|
89
|
-
*/
|
|
90
|
-
export function applySpecPatch(yamlText: string, patch: SpecPatch): ApplySpecPatchResult {
|
|
91
|
-
const parsed = specPatchSchema.safeParse(patch);
|
|
92
|
-
if (!parsed.success) {
|
|
93
|
-
throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
|
|
94
|
-
}
|
|
95
|
-
let doc: ReturnType<typeof parseDocument>;
|
|
96
|
-
try {
|
|
97
|
-
doc = parseDocument(yamlText);
|
|
98
|
-
} catch (err) {
|
|
99
|
-
throw new SpecPatchError("input YAML is not parseable", err);
|
|
100
|
-
}
|
|
101
|
-
// Verify the patch matches the spec's target. Cheap pre-check so we
|
|
102
|
-
// fail with a useful message before the Zod validation later.
|
|
103
|
-
const docTarget = doc.getIn(["target"]);
|
|
104
|
-
if (typeof docTarget === "string" && docTarget !== patch.target) {
|
|
105
|
-
throw new SpecPatchError(
|
|
106
|
-
`patch target "${patch.target}" does not match spec target "${docTarget}"`,
|
|
107
|
-
);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
const path = [...patch.path];
|
|
111
|
-
switch (patch.op) {
|
|
112
|
-
case "replace": {
|
|
113
|
-
if (!doc.hasIn(path)) {
|
|
114
|
-
throw new SpecPatchError(`cannot replace ${formatPath(path)}: path does not exist`);
|
|
115
|
-
}
|
|
116
|
-
doc.setIn(path, patch.value);
|
|
117
|
-
break;
|
|
118
|
-
}
|
|
119
|
-
case "add": {
|
|
120
|
-
if (doc.hasIn(path)) {
|
|
121
|
-
throw new SpecPatchError(
|
|
122
|
-
`cannot add ${formatPath(path)}: path already exists (use "replace")`,
|
|
123
|
-
);
|
|
124
|
-
}
|
|
125
|
-
doc.setIn(path, patch.value);
|
|
126
|
-
break;
|
|
127
|
-
}
|
|
128
|
-
case "remove": {
|
|
129
|
-
if (!doc.hasIn(path)) {
|
|
130
|
-
throw new SpecPatchError(`cannot remove ${formatPath(path)}: path does not exist`);
|
|
131
|
-
}
|
|
132
|
-
doc.deleteIn(path);
|
|
133
|
-
break;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
const newYaml = doc.toString();
|
|
138
|
-
let spec: Spec;
|
|
139
|
-
try {
|
|
140
|
-
spec = parseSpec(newYaml);
|
|
141
|
-
} catch (err) {
|
|
142
|
-
throw new SpecPatchError(`patched YAML failed spec validation: ${(err as Error).message}`, err);
|
|
143
|
-
}
|
|
144
|
-
return { yaml: newYaml, spec };
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
/**
|
|
148
|
-
* Type-check a patch against a parsed Spec WITHOUT applying it. Used
|
|
149
|
-
* by the optimizer to refuse cross-target patches and patches that
|
|
150
|
-
* reference paths outside the optimisation surface.
|
|
151
|
-
*
|
|
152
|
-
* The path-validity check is the "soft" version: it verifies the path
|
|
153
|
-
* is structurally reachable (each segment names a defined field). It
|
|
154
|
-
* does NOT instantiate the patched value — `applySpecPatch` does that
|
|
155
|
-
* via `parseSpec`.
|
|
156
|
-
*/
|
|
157
|
-
export function validatePatch(spec: Spec, patch: SpecPatch): void {
|
|
158
|
-
const parsed = specPatchSchema.safeParse(patch);
|
|
159
|
-
if (!parsed.success) {
|
|
160
|
-
throw new SpecPatchError(`patch shape is invalid: ${parsed.error.message}`);
|
|
161
|
-
}
|
|
162
|
-
if (patch.target !== spec.target) {
|
|
163
|
-
throw new SpecPatchError(
|
|
164
|
-
`patch target "${patch.target}" does not match spec target "${spec.target}"`,
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
if (!isOptimizable(spec.target, patch.path)) {
|
|
168
|
-
throw new SpecPatchError(
|
|
169
|
-
`path ${formatPath(patch.path)} is not listed in OPTIMIZABLE_PATHS for target "${spec.target}"; add it to packages/spec-patch/src/index.ts if it's intended to be tunable`,
|
|
170
|
-
);
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
function formatPath(path: ReadonlyArray<string>): string {
|
|
175
|
-
return path.join(".");
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/**
|
|
179
|
-
* Per-target whitelist of mutation paths the active optimizer is
|
|
180
|
-
* allowed to touch. Adding a new field here is the explicit signal that
|
|
181
|
-
* it's safe to autotune. Skipping this list means the optimizer can
|
|
182
|
-
* only mutate prompts (the default), preserving the "spec safety floor"
|
|
183
|
-
* that an optimizer can't accidentally rewrite security-critical fields
|
|
184
|
-
* like `permissions.mode` or `model_router` rules.
|
|
185
|
-
*/
|
|
186
|
-
export const OPTIMIZABLE_PATHS: Readonly<
|
|
187
|
-
Record<Spec["target"], ReadonlyArray<ReadonlyArray<string>>>
|
|
188
|
-
> = Object.freeze({
|
|
189
|
-
cli: Object.freeze([
|
|
190
|
-
Object.freeze(["agent", "instructions"]),
|
|
191
|
-
Object.freeze(["failure_taxonomy"]),
|
|
192
|
-
Object.freeze(["compaction", "threshold"]),
|
|
193
|
-
// Pillar 2 active context curation — eval-optimizer can flip the
|
|
194
|
-
// semantic-dedupe + relevance-reorder pass on/off and tune its
|
|
195
|
-
// similarity threshold, which dominates input-token cost on long runs.
|
|
196
|
-
Object.freeze(["compaction", "curate"]),
|
|
197
|
-
Object.freeze(["compaction", "dedupeThreshold"]),
|
|
198
|
-
Object.freeze(["compaction", "relevanceTopK"]),
|
|
199
|
-
// Pillar 3 sink-side fabric — egress policy + intent-gate thresholds
|
|
200
|
-
// are tunable so the eval-optimizer can find the sweet spot between
|
|
201
|
-
// false-positive denials and false-negative exfil bypasses.
|
|
202
|
-
Object.freeze(["security", "egressPolicy"]),
|
|
203
|
-
Object.freeze(["security", "justification"]),
|
|
204
|
-
// §47 blockchain subsystem (slice 0). Whole-block replacement so the
|
|
205
|
-
// optimizer can tune `chains[*].finality.count`, `chains[*].rpcPolicy`,
|
|
206
|
-
// `transaction_policy.maxValueUsd`, and `transaction_policy.simulationRequired`
|
|
207
|
-
// by patching their parent block.
|
|
208
|
-
Object.freeze(["chains"]),
|
|
209
|
-
Object.freeze(["transaction_policy"]),
|
|
210
|
-
]),
|
|
211
|
-
workflow: Object.freeze([
|
|
212
|
-
Object.freeze(["steps"]),
|
|
213
|
-
Object.freeze(["failure_taxonomy"]),
|
|
214
|
-
Object.freeze(["chains"]),
|
|
215
|
-
Object.freeze(["transaction_policy"]),
|
|
216
|
-
]) /* whole-step replacement allowed */,
|
|
217
|
-
channel: Object.freeze([
|
|
218
|
-
Object.freeze(["agent", "instructions"]),
|
|
219
|
-
Object.freeze(["failure_taxonomy"]),
|
|
220
|
-
Object.freeze(["chains"]),
|
|
221
|
-
Object.freeze(["transaction_policy"]),
|
|
222
|
-
]),
|
|
223
|
-
graph: Object.freeze([
|
|
224
|
-
Object.freeze(["nodes"]),
|
|
225
|
-
Object.freeze(["failure_taxonomy"]),
|
|
226
|
-
Object.freeze(["chains"]),
|
|
227
|
-
Object.freeze(["transaction_policy"]),
|
|
228
|
-
]),
|
|
229
|
-
managed: Object.freeze([
|
|
230
|
-
Object.freeze(["agent", "instructions"]),
|
|
231
|
-
Object.freeze(["failure_taxonomy"]),
|
|
232
|
-
]),
|
|
233
|
-
pipeline: Object.freeze([
|
|
234
|
-
Object.freeze(["agent", "instructions"]),
|
|
235
|
-
Object.freeze(["failure_taxonomy"]),
|
|
236
|
-
Object.freeze(["indexing", "chunkSize"]),
|
|
237
|
-
Object.freeze(["indexing", "chunkOverlap"]),
|
|
238
|
-
Object.freeze(["retrieve", "defaultK"]),
|
|
239
|
-
]),
|
|
240
|
-
crew: Object.freeze([
|
|
241
|
-
Object.freeze(["roles"]),
|
|
242
|
-
Object.freeze(["failure_taxonomy"]),
|
|
243
|
-
Object.freeze(["chains"]),
|
|
244
|
-
Object.freeze(["transaction_policy"]),
|
|
245
|
-
]) /* whole-role replacement */,
|
|
246
|
-
research: Object.freeze([
|
|
247
|
-
Object.freeze(["agent", "instructions"]),
|
|
248
|
-
Object.freeze(["failure_taxonomy"]),
|
|
249
|
-
Object.freeze(["retrieve", "maxDepth"]),
|
|
250
|
-
Object.freeze(["chains"]),
|
|
251
|
-
Object.freeze(["transaction_policy"]),
|
|
252
|
-
]),
|
|
253
|
-
batch: Object.freeze([
|
|
254
|
-
Object.freeze(["agent", "instructions"]),
|
|
255
|
-
Object.freeze(["failure_taxonomy"]),
|
|
256
|
-
Object.freeze(["chains"]),
|
|
257
|
-
Object.freeze(["transaction_policy"]),
|
|
258
|
-
]),
|
|
259
|
-
voice: Object.freeze([
|
|
260
|
-
Object.freeze(["agent", "instructions"]),
|
|
261
|
-
Object.freeze(["failure_taxonomy"]),
|
|
262
|
-
]),
|
|
263
|
-
browser: Object.freeze([
|
|
264
|
-
Object.freeze(["agent", "instructions"]),
|
|
265
|
-
Object.freeze(["failure_taxonomy"]),
|
|
266
|
-
]),
|
|
267
|
-
eval: Object.freeze([
|
|
268
|
-
Object.freeze(["agent", "instructions"]),
|
|
269
|
-
Object.freeze(["failure_taxonomy"]),
|
|
270
|
-
]),
|
|
271
|
-
// §47 onchain daemon: full cross-cutting blocks are optimizable.
|
|
272
|
-
onchain: Object.freeze([
|
|
273
|
-
Object.freeze(["agent", "instructions"]),
|
|
274
|
-
Object.freeze(["failure_taxonomy"]),
|
|
275
|
-
Object.freeze(["chains"]),
|
|
276
|
-
Object.freeze(["triggers"]),
|
|
277
|
-
Object.freeze(["transaction_policy"]),
|
|
278
|
-
Object.freeze(["idempotencyWindowMs"]),
|
|
279
|
-
]),
|
|
280
|
-
// §47 onchain-game: instructions, game.objective, and the policy are
|
|
281
|
-
// the productive knobs; move-timeout-ms is the realtime quality knob.
|
|
282
|
-
"onchain-game": Object.freeze([
|
|
283
|
-
Object.freeze(["agent", "instructions"]),
|
|
284
|
-
Object.freeze(["failure_taxonomy"]),
|
|
285
|
-
Object.freeze(["game"]),
|
|
286
|
-
Object.freeze(["transaction_policy"]),
|
|
287
|
-
]),
|
|
288
|
-
});
|
|
289
|
-
|
|
290
|
-
function isOptimizable(target: Spec["target"], path: ReadonlyArray<string>): boolean {
|
|
291
|
-
const allowed = OPTIMIZABLE_PATHS[target];
|
|
292
|
-
for (const ok of allowed) {
|
|
293
|
-
if (ok.length !== path.length) continue;
|
|
294
|
-
let match = true;
|
|
295
|
-
for (let i = 0; i < ok.length; i++) {
|
|
296
|
-
if (ok[i] !== path[i]) {
|
|
297
|
-
match = false;
|
|
298
|
-
break;
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
if (match) return true;
|
|
302
|
-
}
|
|
303
|
-
// Allow patches with a prefix that matches an optimizable path
|
|
304
|
-
// (e.g. `["nodes", "0", "instructions"]` if `["nodes"]` is whitelisted)
|
|
305
|
-
// so the optimizer can do fine-grained updates without listing every
|
|
306
|
-
// sub-path.
|
|
307
|
-
for (const ok of allowed) {
|
|
308
|
-
if (path.length < ok.length) continue;
|
|
309
|
-
let match = true;
|
|
310
|
-
for (let i = 0; i < ok.length; i++) {
|
|
311
|
-
if (ok[i] !== path[i]) {
|
|
312
|
-
match = false;
|
|
313
|
-
break;
|
|
314
|
-
}
|
|
315
|
-
}
|
|
316
|
-
if (match) return true;
|
|
317
|
-
}
|
|
318
|
-
return false;
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
/**
|
|
322
|
-
* Format a YAML header comment to prepend to a written-back file. The
|
|
323
|
-
* orchestrator's `--write-back` writes this above the original spec so
|
|
324
|
-
* a human reviewer can see when and by what an optimisation pass ran.
|
|
325
|
-
*/
|
|
326
|
-
export function formatWriteBackHeader(opts: {
|
|
327
|
-
readonly runId: string;
|
|
328
|
-
readonly mutator: string;
|
|
329
|
-
readonly scoreBefore: number;
|
|
330
|
-
readonly scoreAfter: number;
|
|
331
|
-
readonly iterations: number;
|
|
332
|
-
readonly timestamp?: string;
|
|
333
|
-
}): string {
|
|
334
|
-
const ts = opts.timestamp ?? new Date().toISOString();
|
|
335
|
-
const delta = (opts.scoreAfter - opts.scoreBefore).toFixed(3);
|
|
336
|
-
return [
|
|
337
|
-
`# crewhaus optimize: runId ${opts.runId}`,
|
|
338
|
-
`# - mutator: ${opts.mutator}`,
|
|
339
|
-
`# - iterations: ${opts.iterations}`,
|
|
340
|
-
`# - score: ${opts.scoreBefore.toFixed(3)} → ${opts.scoreAfter.toFixed(3)} (Δ ${delta})`,
|
|
341
|
-
`# - generated: ${ts}`,
|
|
342
|
-
"",
|
|
343
|
-
].join("\n");
|
|
344
|
-
}
|