@cairnkeep/cli 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -1
- package/README.md +23 -0
- package/bin/cairn +6 -0
- package/docs/design/meta-agent.md +39 -35
- package/docs/learning/CURRICULUM-MAP.md +9 -4
- package/docs/learning/FEATURE-GUIDE.md +6 -4
- package/docs/learning/README.md +4 -2
- package/docs/learning/lessons/L00-why-cairnkeep.md +1 -1
- package/docs/learning/lessons/L01-safe-trial.md +3 -3
- package/docs/learning/lessons/L02-installation.md +2 -2
- package/docs/learning/lessons/L03-first-project.md +1 -1
- package/docs/learning/lessons/L17-agent-boundary.md +12 -8
- package/docs/learning/lessons/L18-local-code-graph.md +97 -0
- package/docs/learning/lessons/L19-validated-skill-improvement.md +87 -0
- package/docs/learning/tracks/evidence-and-evaluation.md +5 -3
- package/docs/learning/tracks/practitioner.md +5 -4
- package/docs/operating.md +23 -0
- package/docs/privacy-and-data-flow.md +42 -1
- package/docs/skill-improvement.md +124 -0
- package/docs/storage.md +25 -0
- package/examples/eval/bundled-fake.json +1 -1
- package/mcp-memory-server/dist/eval-plan.js +22 -5
- package/mcp-memory-server/dist/eval-process.js +31 -0
- package/mcp-memory-server/dist/eval-runner.js +10 -4
- package/mcp-memory-server/dist/eval-schema.js +1 -1
- package/mcp-memory-server/dist/eval-workspace.js +50 -1
- package/mcp-memory-server/dist/skill-cli.js +236 -0
- package/mcp-memory-server/dist/skill-evaluation.js +244 -0
- package/mcp-memory-server/dist/skill-schema.js +164 -0
- package/mcp-memory-server/dist/skill-store.js +583 -0
- package/mcp-memory-server/package.json +2 -1
- package/package.json +1 -1
- package/schemas/eval-report.schema.json +1 -1
- package/schemas/skill-adapter.schema.json +39 -0
- package/schemas/skill-proposal-protocol.schema.json +97 -0
- package/scripts/completion.sh +41 -3
- package/scripts/test-completion.sh +11 -0
- package/scripts/test-learning-docs.sh +11 -2
- package/scripts/test-package-install.sh +18 -0
- package/scripts/test-skill-cli.sh +30 -0
- package/templates/graph-policy.md.template +3 -1
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { canonicalDigest } from "./eval-schema.js";
|
|
4
|
+
import { evaluateSkillProposal } from "./skill-evaluation.js";
|
|
5
|
+
import { applySkillProposal, approveSkillCandidate, harvestSkillCandidates, listSkillApplications, listSkillCandidates, listSkillEvaluations, listSkillProposals, proposeSkill, readSkillApplication, readSkillCandidate, readSkillEvaluation, readSkillProposal, rollbackSkillApplication, } from "./skill-store.js";
|
|
6
|
+
process.stdout.on("error", (error) => {
|
|
7
|
+
if (error.code === "EPIPE")
|
|
8
|
+
process.exit(0);
|
|
9
|
+
throw error;
|
|
10
|
+
});
|
|
11
|
+
const ID_PATTERN = /^[a-z0-9][a-z0-9._-]{0,127}$/;
|
|
12
|
+
const DIGEST_PATTERN = /^[a-f0-9]{64}$/;
|
|
13
|
+
const KINDS = ["candidate", "proposal", "evaluation", "application"];
|
|
14
|
+
function usage() {
|
|
15
|
+
return `cairn skill: reviewed, evidence-gated skill improvement
|
|
16
|
+
|
|
17
|
+
Usage:
|
|
18
|
+
cairn skill harvest [--project PATH] [--minimum-occurrences N] [--json]
|
|
19
|
+
cairn skill list [--project PATH] [--kind KIND] [--json]
|
|
20
|
+
cairn skill show --kind KIND --id ID [--project PATH] [--json]
|
|
21
|
+
cairn skill review --candidate ID --approve [--project PATH] [--json]
|
|
22
|
+
cairn skill propose --candidate ID --target RELATIVE_PATH --adapter PATH [--edit-budget N] [--project PATH] [--json]
|
|
23
|
+
cairn skill evaluate --proposal ID --exploration-task-set PATH --confirmation-task-set PATH --adapter PATH
|
|
24
|
+
[--output ROOT] [--repetitions N] [--seed VALUE] [--minimum-improvement N]
|
|
25
|
+
[--project PATH] --yes [--json]
|
|
26
|
+
cairn skill apply --proposal ID --evaluation ID --confirm PROPOSAL_DIGEST [--project PATH] [--json]
|
|
27
|
+
cairn skill rollback --application ID --confirm [--project PATH] [--json]
|
|
28
|
+
|
|
29
|
+
KIND is candidate, proposal, evaluation, or application. Proposal generation
|
|
30
|
+
receives evidence only after review. Evaluation is opt-in with CAIRN_EVAL=1,
|
|
31
|
+
uses disjoint committed exploration and confirmation task sets, and invokes the
|
|
32
|
+
operator-owned adapter. Apply is allowed only for an eligible exact evaluation.
|
|
33
|
+
`;
|
|
34
|
+
}
|
|
35
|
+
const valueFlags = new Set([
|
|
36
|
+
"--project", "--minimum-occurrences", "--kind", "--id", "--candidate", "--target", "--adapter",
|
|
37
|
+
"--edit-budget", "--proposal", "--exploration-task-set", "--confirmation-task-set", "--output",
|
|
38
|
+
"--repetitions", "--seed", "--minimum-improvement", "--evaluation", "--application", "--confirm",
|
|
39
|
+
]);
|
|
40
|
+
const booleanFlags = new Set(["--json", "--approve", "--yes"]);
|
|
41
|
+
function parse(args, allowed) {
|
|
42
|
+
const permitted = new Set(allowed);
|
|
43
|
+
const result = new Map();
|
|
44
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
45
|
+
const flag = args[index];
|
|
46
|
+
if (!flag.startsWith("--") || !permitted.has(flag))
|
|
47
|
+
throw new Error(`Unknown option: ${flag}`);
|
|
48
|
+
if (result.has(flag))
|
|
49
|
+
throw new Error(`Option may be supplied only once: ${flag}`);
|
|
50
|
+
if (flag === "--confirm" && (!args[index + 1] || args[index + 1].startsWith("--"))) {
|
|
51
|
+
result.set(flag, true);
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (booleanFlags.has(flag)) {
|
|
55
|
+
result.set(flag, true);
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
if (!valueFlags.has(flag))
|
|
59
|
+
throw new Error(`Unsupported option: ${flag}`);
|
|
60
|
+
const value = args[index + 1];
|
|
61
|
+
if (!value || value.startsWith("--"))
|
|
62
|
+
throw new Error(`${flag} requires a value.`);
|
|
63
|
+
result.set(flag, value);
|
|
64
|
+
index += 1;
|
|
65
|
+
}
|
|
66
|
+
return result;
|
|
67
|
+
}
|
|
68
|
+
function stringValue(flags, name, required = false) {
|
|
69
|
+
const value = flags.get(name);
|
|
70
|
+
if (required && typeof value !== "string")
|
|
71
|
+
throw new Error(`${name} is required.`);
|
|
72
|
+
return typeof value === "string" ? value : undefined;
|
|
73
|
+
}
|
|
74
|
+
function positiveInteger(flags, name, fallback) {
|
|
75
|
+
const raw = stringValue(flags, name);
|
|
76
|
+
if (raw === undefined)
|
|
77
|
+
return fallback;
|
|
78
|
+
if (!/^[1-9][0-9]*$/.test(raw))
|
|
79
|
+
throw new Error(`${name} must be a positive integer.`);
|
|
80
|
+
const value = Number(raw);
|
|
81
|
+
if (!Number.isSafeInteger(value))
|
|
82
|
+
throw new Error(`${name} is outside the supported range.`);
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
function identifier(flags, name) {
|
|
86
|
+
const value = stringValue(flags, name, true);
|
|
87
|
+
if (!ID_PATTERN.test(value))
|
|
88
|
+
throw new Error(`${name} must be a canonical artifact ID.`);
|
|
89
|
+
return value;
|
|
90
|
+
}
|
|
91
|
+
function projectRoot(flags) {
|
|
92
|
+
return resolve(stringValue(flags, "--project") ?? process.cwd());
|
|
93
|
+
}
|
|
94
|
+
function kind(flags, fallback) {
|
|
95
|
+
const value = stringValue(flags, "--kind") ?? fallback;
|
|
96
|
+
if (!value || !KINDS.includes(value))
|
|
97
|
+
throw new Error(`--kind must be one of: ${KINDS.join(", ")}.`);
|
|
98
|
+
return value;
|
|
99
|
+
}
|
|
100
|
+
function emit(value, json) {
|
|
101
|
+
process.stdout.write(`${JSON.stringify(value, null, json ? 0 : 2)}\n`);
|
|
102
|
+
}
|
|
103
|
+
function list(root, selected) {
|
|
104
|
+
const values = {
|
|
105
|
+
candidates: listSkillCandidates(root),
|
|
106
|
+
proposals: listSkillProposals(root),
|
|
107
|
+
evaluations: listSkillEvaluations(root),
|
|
108
|
+
applications: listSkillApplications(root),
|
|
109
|
+
};
|
|
110
|
+
if (!selected)
|
|
111
|
+
return values;
|
|
112
|
+
return values[`${selected}s`];
|
|
113
|
+
}
|
|
114
|
+
function show(root, selected, id) {
|
|
115
|
+
if (selected === "candidate")
|
|
116
|
+
return readSkillCandidate(root, id);
|
|
117
|
+
if (selected === "proposal")
|
|
118
|
+
return readSkillProposal(root, id);
|
|
119
|
+
if (selected === "evaluation")
|
|
120
|
+
return readSkillEvaluation(root, id);
|
|
121
|
+
return readSkillApplication(root, id);
|
|
122
|
+
}
|
|
123
|
+
async function main() {
|
|
124
|
+
const [command = "help", ...args] = process.argv.slice(2);
|
|
125
|
+
if (command === "help" || command === "--help" || command === "-h") {
|
|
126
|
+
process.stdout.write(usage());
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
if (command === "harvest") {
|
|
130
|
+
const flags = parse(args, ["--project", "--minimum-occurrences", "--json"]);
|
|
131
|
+
emit(harvestSkillCandidates({
|
|
132
|
+
projectRoot: projectRoot(flags),
|
|
133
|
+
minimumOccurrences: positiveInteger(flags, "--minimum-occurrences"),
|
|
134
|
+
}), flags.has("--json"));
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
if (command === "list") {
|
|
138
|
+
const flags = parse(args, ["--project", "--kind", "--json"]);
|
|
139
|
+
const selected = stringValue(flags, "--kind") ? kind(flags) : undefined;
|
|
140
|
+
emit(list(projectRoot(flags), selected), flags.has("--json"));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (command === "show") {
|
|
144
|
+
const flags = parse(args, ["--project", "--kind", "--id", "--json"]);
|
|
145
|
+
emit(show(projectRoot(flags), kind(flags), identifier(flags, "--id")), flags.has("--json"));
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
if (command === "review") {
|
|
149
|
+
const flags = parse(args, ["--project", "--candidate", "--approve", "--json"]);
|
|
150
|
+
if (!flags.has("--approve"))
|
|
151
|
+
throw new Error("Review currently requires --approve.");
|
|
152
|
+
emit(approveSkillCandidate(projectRoot(flags), identifier(flags, "--candidate")), flags.has("--json"));
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
if (command === "propose") {
|
|
156
|
+
const flags = parse(args, ["--project", "--candidate", "--target", "--adapter", "--edit-budget", "--json"]);
|
|
157
|
+
const controller = new AbortController();
|
|
158
|
+
const cancel = () => controller.abort();
|
|
159
|
+
process.once("SIGINT", cancel);
|
|
160
|
+
process.once("SIGTERM", cancel);
|
|
161
|
+
try {
|
|
162
|
+
const proposal = await proposeSkill({
|
|
163
|
+
projectRoot: projectRoot(flags),
|
|
164
|
+
candidateId: identifier(flags, "--candidate"),
|
|
165
|
+
targetPath: stringValue(flags, "--target", true),
|
|
166
|
+
adapterPath: stringValue(flags, "--adapter", true),
|
|
167
|
+
editBudget: positiveInteger(flags, "--edit-budget"),
|
|
168
|
+
signal: controller.signal,
|
|
169
|
+
});
|
|
170
|
+
emit({ ...proposal, proposal_digest: canonicalDigest(proposal) }, flags.has("--json"));
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
process.removeListener("SIGINT", cancel);
|
|
174
|
+
process.removeListener("SIGTERM", cancel);
|
|
175
|
+
}
|
|
176
|
+
return;
|
|
177
|
+
}
|
|
178
|
+
if (command === "evaluate") {
|
|
179
|
+
const flags = parse(args, [
|
|
180
|
+
"--project", "--proposal", "--exploration-task-set", "--confirmation-task-set", "--adapter",
|
|
181
|
+
"--output", "--repetitions", "--seed", "--minimum-improvement", "--yes", "--json",
|
|
182
|
+
]);
|
|
183
|
+
const controller = new AbortController();
|
|
184
|
+
const cancel = () => controller.abort();
|
|
185
|
+
process.once("SIGINT", cancel);
|
|
186
|
+
process.once("SIGTERM", cancel);
|
|
187
|
+
try {
|
|
188
|
+
emit(await evaluateSkillProposal({
|
|
189
|
+
projectRoot: projectRoot(flags),
|
|
190
|
+
proposalId: identifier(flags, "--proposal"),
|
|
191
|
+
explorationTaskSetPath: stringValue(flags, "--exploration-task-set", true),
|
|
192
|
+
confirmationTaskSetPath: stringValue(flags, "--confirmation-task-set", true),
|
|
193
|
+
adapterPath: stringValue(flags, "--adapter", true),
|
|
194
|
+
outputRoot: stringValue(flags, "--output"),
|
|
195
|
+
repetitions: positiveInteger(flags, "--repetitions"),
|
|
196
|
+
seed: stringValue(flags, "--seed"),
|
|
197
|
+
minimumImprovement: positiveInteger(flags, "--minimum-improvement"),
|
|
198
|
+
confirm: flags.has("--yes"),
|
|
199
|
+
signal: controller.signal,
|
|
200
|
+
}), flags.has("--json"));
|
|
201
|
+
}
|
|
202
|
+
finally {
|
|
203
|
+
process.removeListener("SIGINT", cancel);
|
|
204
|
+
process.removeListener("SIGTERM", cancel);
|
|
205
|
+
}
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
208
|
+
if (command === "apply") {
|
|
209
|
+
const flags = parse(args, ["--project", "--proposal", "--evaluation", "--confirm", "--json"]);
|
|
210
|
+
const confirm = flags.get("--confirm");
|
|
211
|
+
if (typeof confirm !== "string" || !DIGEST_PATTERN.test(confirm)) {
|
|
212
|
+
throw new Error("--confirm requires the exact 64-character proposal digest.");
|
|
213
|
+
}
|
|
214
|
+
emit(applySkillProposal({
|
|
215
|
+
projectRoot: projectRoot(flags),
|
|
216
|
+
proposalId: identifier(flags, "--proposal"),
|
|
217
|
+
evaluationId: identifier(flags, "--evaluation"),
|
|
218
|
+
confirmDigest: confirm,
|
|
219
|
+
}), flags.has("--json"));
|
|
220
|
+
return;
|
|
221
|
+
}
|
|
222
|
+
if (command === "rollback") {
|
|
223
|
+
const flags = parse(args, ["--project", "--application", "--confirm", "--json"]);
|
|
224
|
+
emit(rollbackSkillApplication({
|
|
225
|
+
projectRoot: projectRoot(flags),
|
|
226
|
+
applicationId: identifier(flags, "--application"),
|
|
227
|
+
confirm: flags.has("--confirm"),
|
|
228
|
+
}), flags.has("--json"));
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
throw new Error(`Unknown skill command: ${command}`);
|
|
232
|
+
}
|
|
233
|
+
main().catch((error) => {
|
|
234
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
|
235
|
+
process.exitCode = 1;
|
|
236
|
+
});
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
import { canonicalDigest } from "./eval-schema.js";
|
|
2
|
+
import { isEvalEnabled, loadEvalPlan } from "./eval-plan.js";
|
|
3
|
+
import { runTwoPassExperiment } from "./eval-runner.js";
|
|
4
|
+
import { getSkillStore, readSkillProposal, readSkillTarget, sha256File, writeSkillEvaluation, } from "./skill-store.js";
|
|
5
|
+
import { SKILL_SCHEMA_VERSION, skillEvaluationSchema } from "./skill-schema.js";
|
|
6
|
+
function requireRootWorkspaces(plan) {
|
|
7
|
+
if (plan.task_set.tasks.some((task) => task.workspace.path !== ".")) {
|
|
8
|
+
throw new Error("Skill evaluation tasks must use the repository root as their workspace.");
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
function requireDisjointPlans(exploration, confirmation) {
|
|
12
|
+
if (exploration.task_set.tasks.length < 2 || confirmation.task_set.tasks.length < 2) {
|
|
13
|
+
throw new Error("Skill evaluation requires at least two exploration and two confirmation tasks.");
|
|
14
|
+
}
|
|
15
|
+
if (exploration.task_set_digest === confirmation.task_set_digest) {
|
|
16
|
+
throw new Error("Exploration and confirmation task sets must be different committed documents.");
|
|
17
|
+
}
|
|
18
|
+
if (exploration.source.kind !== confirmation.source.kind) {
|
|
19
|
+
throw new Error("Exploration and confirmation task sets must bind the same source kind.");
|
|
20
|
+
}
|
|
21
|
+
const explorationRevision = exploration.source.kind === "git"
|
|
22
|
+
? exploration.source.revision
|
|
23
|
+
: exploration.source.package_version;
|
|
24
|
+
const confirmationRevision = confirmation.source.kind === "git"
|
|
25
|
+
? confirmation.source.revision
|
|
26
|
+
: confirmation.source.package_version;
|
|
27
|
+
if (explorationRevision !== confirmationRevision) {
|
|
28
|
+
throw new Error("Exploration and confirmation task sets must bind the same immutable source revision.");
|
|
29
|
+
}
|
|
30
|
+
const explorationIds = new Set(exploration.task_set.tasks.map(({ id }) => id));
|
|
31
|
+
if (confirmation.task_set.tasks.some(({ id }) => explorationIds.has(id))) {
|
|
32
|
+
throw new Error("Exploration and confirmation task IDs must not overlap.");
|
|
33
|
+
}
|
|
34
|
+
const taskDefinitionDigest = ({ id: _id, ...task }) => canonicalDigest(task);
|
|
35
|
+
const explorationDefinitions = new Set(exploration.task_set.tasks.map(taskDefinitionDigest));
|
|
36
|
+
if (confirmation.task_set.tasks.some((task) => explorationDefinitions.has(taskDefinitionDigest(task)))) {
|
|
37
|
+
throw new Error("Exploration and confirmation task definitions must not overlap.");
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function observationKey(observation) {
|
|
41
|
+
return `${observation.task_id}\0${observation.repetition}\0${observation.pass}`;
|
|
42
|
+
}
|
|
43
|
+
export function summarizeSkillReport(report) {
|
|
44
|
+
if (report.status !== "final" || report.experiment_kind !== "skill_candidate") {
|
|
45
|
+
throw new Error("Skill evaluation requires a final skill-candidate report.");
|
|
46
|
+
}
|
|
47
|
+
const baseline = new Map();
|
|
48
|
+
const treatment = new Map();
|
|
49
|
+
for (const observation of report.observations) {
|
|
50
|
+
(observation.arm === "baseline" ? baseline : treatment).set(observationKey(observation), observation);
|
|
51
|
+
}
|
|
52
|
+
let baselinePassed = 0;
|
|
53
|
+
let candidatePassed = 0;
|
|
54
|
+
let eligiblePairs = 0;
|
|
55
|
+
const improvedTasks = new Set();
|
|
56
|
+
const regressedTasks = new Set();
|
|
57
|
+
let unknown = 0;
|
|
58
|
+
const scheduledPairs = report.schedule.length / 2;
|
|
59
|
+
for (const [key, baselineObservation] of baseline) {
|
|
60
|
+
const candidateObservation = treatment.get(key);
|
|
61
|
+
const eligible = candidateObservation !== undefined
|
|
62
|
+
&& baselineObservation.state === "terminal"
|
|
63
|
+
&& candidateObservation.state === "terminal"
|
|
64
|
+
&& baselineObservation.capability_status === "valid"
|
|
65
|
+
&& candidateObservation.capability_status === "valid"
|
|
66
|
+
&& baselineObservation.pass_state !== "unknown"
|
|
67
|
+
&& candidateObservation.pass_state !== "unknown";
|
|
68
|
+
if (!eligible || !candidateObservation)
|
|
69
|
+
continue;
|
|
70
|
+
eligiblePairs += 1;
|
|
71
|
+
const baselinePass = baselineObservation.pass_state === "passed";
|
|
72
|
+
const candidatePass = candidateObservation.pass_state === "passed";
|
|
73
|
+
if (baselinePass)
|
|
74
|
+
baselinePassed += 1;
|
|
75
|
+
if (candidatePass)
|
|
76
|
+
candidatePassed += 1;
|
|
77
|
+
if (!baselinePass && candidatePass)
|
|
78
|
+
improvedTasks.add(baselineObservation.task_id);
|
|
79
|
+
if (baselinePass && !candidatePass)
|
|
80
|
+
regressedTasks.add(baselineObservation.task_id);
|
|
81
|
+
}
|
|
82
|
+
unknown = Math.max(0, scheduledPairs - eligiblePairs);
|
|
83
|
+
return {
|
|
84
|
+
task_set_digest: report.task_set_digest,
|
|
85
|
+
report_digest: canonicalDigest(report),
|
|
86
|
+
experiment_id: report.experiment_id,
|
|
87
|
+
baseline_passed: baselinePassed,
|
|
88
|
+
candidate_passed: candidatePassed,
|
|
89
|
+
eligible_pairs: eligiblePairs,
|
|
90
|
+
improvements: improvedTasks.size,
|
|
91
|
+
regressions: regressedTasks.size,
|
|
92
|
+
unknown,
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
function gate(summary, minimumImprovement, prefix) {
|
|
96
|
+
const reasons = [];
|
|
97
|
+
if (summary.eligible_pairs === 0 || summary.unknown > 0)
|
|
98
|
+
reasons.push(`${prefix}_incomplete`);
|
|
99
|
+
if (summary.regressions > 0)
|
|
100
|
+
reasons.push(`${prefix}_regression`);
|
|
101
|
+
if (summary.improvements < minimumImprovement)
|
|
102
|
+
reasons.push(`${prefix}_no_improvement`);
|
|
103
|
+
return reasons;
|
|
104
|
+
}
|
|
105
|
+
export async function evaluateSkillProposal(options) {
|
|
106
|
+
if (!options.confirm)
|
|
107
|
+
throw new Error("Skill evaluation requires --yes because it invokes the configured evaluation adapter.");
|
|
108
|
+
if (!isEvalEnabled())
|
|
109
|
+
throw new Error("Skill evaluation is disabled. Set CAIRN_EVAL=1 to opt in.");
|
|
110
|
+
const repetitions = options.repetitions ?? 2;
|
|
111
|
+
const minimumImprovement = options.minimumImprovement ?? 1;
|
|
112
|
+
if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 1_000) {
|
|
113
|
+
throw new Error("Evaluation repetitions must be an integer from 1 through 1000.");
|
|
114
|
+
}
|
|
115
|
+
if (!Number.isSafeInteger(minimumImprovement) || minimumImprovement < 1 || minimumImprovement > 10_000) {
|
|
116
|
+
throw new Error("Minimum improvement must be an integer from 1 through 10000.");
|
|
117
|
+
}
|
|
118
|
+
const store = getSkillStore(options.projectRoot);
|
|
119
|
+
const proposal = readSkillProposal(store.project_root, options.proposalId);
|
|
120
|
+
const target = readSkillTarget(store.project_root, proposal.target_path);
|
|
121
|
+
if (target.digest !== proposal.baseline_digest)
|
|
122
|
+
throw new Error("Skill target changed after proposal generation.");
|
|
123
|
+
const arms = [
|
|
124
|
+
{ id: "baseline", disabled_capability: null },
|
|
125
|
+
{ id: "treatment", disabled_capability: null },
|
|
126
|
+
];
|
|
127
|
+
const outputRoot = options.outputRoot ?? ".agentfs/eval/experiments";
|
|
128
|
+
const seed = options.seed ?? "cairn-skill-eval-v1";
|
|
129
|
+
const common = {
|
|
130
|
+
adapterPath: options.adapterPath,
|
|
131
|
+
outputRoot,
|
|
132
|
+
repetitions,
|
|
133
|
+
arms,
|
|
134
|
+
experimentKind: "skill_candidate",
|
|
135
|
+
cwd: store.project_root,
|
|
136
|
+
};
|
|
137
|
+
const explorationPlan = loadEvalPlan({
|
|
138
|
+
...common,
|
|
139
|
+
taskSetPath: options.explorationTaskSetPath,
|
|
140
|
+
seed: `${seed}:exploration`,
|
|
141
|
+
});
|
|
142
|
+
const confirmationPlan = loadEvalPlan({
|
|
143
|
+
...common,
|
|
144
|
+
taskSetPath: options.confirmationTaskSetPath,
|
|
145
|
+
seed: `${seed}:confirmation`,
|
|
146
|
+
});
|
|
147
|
+
requireRootWorkspaces(explorationPlan);
|
|
148
|
+
requireRootWorkspaces(confirmationPlan);
|
|
149
|
+
requireDisjointPlans(explorationPlan, confirmationPlan);
|
|
150
|
+
if (minimumImprovement > explorationPlan.task_set.tasks.length
|
|
151
|
+
|| minimumImprovement > confirmationPlan.task_set.tasks.length) {
|
|
152
|
+
throw new Error("Minimum improvement cannot exceed the task count in either evaluation set.");
|
|
153
|
+
}
|
|
154
|
+
if (explorationPlan.adapter_config_digest !== confirmationPlan.adapter_config_digest
|
|
155
|
+
|| explorationPlan.resolved_programs.adapter !== confirmationPlan.resolved_programs.adapter) {
|
|
156
|
+
throw new Error("Exploration and confirmation must use the same evaluation adapter.");
|
|
157
|
+
}
|
|
158
|
+
const evaluationAdapterProgramDigest = sha256File(explorationPlan.resolved_programs.adapter);
|
|
159
|
+
const proposalDigest = canonicalDigest(proposal);
|
|
160
|
+
const binding = {
|
|
161
|
+
schema_version: SKILL_SCHEMA_VERSION,
|
|
162
|
+
proposal_id: proposal.id,
|
|
163
|
+
proposal_digest: proposalDigest,
|
|
164
|
+
baseline_digest: proposal.baseline_digest,
|
|
165
|
+
candidate_digest: proposal.candidate_content_digest,
|
|
166
|
+
exploration_plan_digest: explorationPlan.plan_digest,
|
|
167
|
+
confirmation_plan_digest: confirmationPlan.plan_digest,
|
|
168
|
+
evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
|
|
169
|
+
repetitions,
|
|
170
|
+
minimum_improvement: minimumImprovement,
|
|
171
|
+
};
|
|
172
|
+
const bindingDigest = canonicalDigest(binding);
|
|
173
|
+
const overlay = {
|
|
174
|
+
baseline: {
|
|
175
|
+
relative_path: proposal.target_path,
|
|
176
|
+
content: target.content,
|
|
177
|
+
digest: proposal.baseline_digest,
|
|
178
|
+
},
|
|
179
|
+
treatment: {
|
|
180
|
+
relative_path: proposal.target_path,
|
|
181
|
+
content: proposal.candidate_content,
|
|
182
|
+
digest: proposal.candidate_content_digest,
|
|
183
|
+
},
|
|
184
|
+
};
|
|
185
|
+
const explorationRun = await runTwoPassExperiment({
|
|
186
|
+
plan: explorationPlan,
|
|
187
|
+
experiment_id: `skill-explore-${bindingDigest.slice(0, 20)}`,
|
|
188
|
+
workspace_overlays: overlay,
|
|
189
|
+
signal: options.signal,
|
|
190
|
+
});
|
|
191
|
+
const exploration = summarizeSkillReport(explorationRun.report);
|
|
192
|
+
if (sha256File(explorationPlan.resolved_programs.adapter) !== evaluationAdapterProgramDigest) {
|
|
193
|
+
throw new Error("Evaluation adapter executable changed during exploration.");
|
|
194
|
+
}
|
|
195
|
+
const explorationReasons = gate(exploration, minimumImprovement, "exploration");
|
|
196
|
+
if (explorationReasons.length > 0) {
|
|
197
|
+
const evaluation = skillEvaluationSchema.parse({
|
|
198
|
+
schema_version: SKILL_SCHEMA_VERSION,
|
|
199
|
+
id: `evaluation-${bindingDigest.slice(0, 24)}`,
|
|
200
|
+
proposal_id: proposal.id,
|
|
201
|
+
proposal_digest: proposalDigest,
|
|
202
|
+
binding_digest: bindingDigest,
|
|
203
|
+
evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
|
|
204
|
+
status: exploration.unknown > 0 ? "inconclusive" : "rejected",
|
|
205
|
+
minimum_improvement: minimumImprovement,
|
|
206
|
+
exploration,
|
|
207
|
+
confirmation: null,
|
|
208
|
+
reasons: explorationReasons,
|
|
209
|
+
created_at: new Date().toISOString(),
|
|
210
|
+
});
|
|
211
|
+
writeSkillEvaluation(store.project_root, evaluation);
|
|
212
|
+
return evaluation;
|
|
213
|
+
}
|
|
214
|
+
const confirmationRun = await runTwoPassExperiment({
|
|
215
|
+
plan: confirmationPlan,
|
|
216
|
+
experiment_id: `skill-confirm-${bindingDigest.slice(0, 20)}`,
|
|
217
|
+
workspace_overlays: overlay,
|
|
218
|
+
signal: options.signal,
|
|
219
|
+
});
|
|
220
|
+
if (sha256File(confirmationPlan.resolved_programs.adapter) !== evaluationAdapterProgramDigest) {
|
|
221
|
+
throw new Error("Evaluation adapter executable changed during confirmation.");
|
|
222
|
+
}
|
|
223
|
+
const confirmation = summarizeSkillReport(confirmationRun.report);
|
|
224
|
+
const confirmationReasons = gate(confirmation, minimumImprovement, "confirmation");
|
|
225
|
+
const status = confirmationReasons.length === 0
|
|
226
|
+
? "eligible"
|
|
227
|
+
: confirmation.unknown > 0 ? "inconclusive" : "rejected";
|
|
228
|
+
const evaluation = skillEvaluationSchema.parse({
|
|
229
|
+
schema_version: SKILL_SCHEMA_VERSION,
|
|
230
|
+
id: `evaluation-${bindingDigest.slice(0, 24)}`,
|
|
231
|
+
proposal_id: proposal.id,
|
|
232
|
+
proposal_digest: proposalDigest,
|
|
233
|
+
binding_digest: bindingDigest,
|
|
234
|
+
evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
|
|
235
|
+
status,
|
|
236
|
+
minimum_improvement: minimumImprovement,
|
|
237
|
+
exploration,
|
|
238
|
+
confirmation,
|
|
239
|
+
reasons: confirmationReasons,
|
|
240
|
+
created_at: new Date().toISOString(),
|
|
241
|
+
});
|
|
242
|
+
writeSkillEvaluation(store.project_root, evaluation);
|
|
243
|
+
return evaluation;
|
|
244
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const SKILL_SCHEMA_VERSION = 1;
|
|
3
|
+
export const MAX_SKILL_BYTES = 256 * 1024;
|
|
4
|
+
export const MAX_SKILL_EDITS = 16;
|
|
5
|
+
const identifierSchema = z.string().min(1).max(128).regex(/^[a-z0-9][a-z0-9._-]*$/);
|
|
6
|
+
const digestSchema = z.string().regex(/^[a-f0-9]{64}$/);
|
|
7
|
+
const timestampSchema = z.iso.datetime();
|
|
8
|
+
const boundedTextSchema = z.string().max(4096);
|
|
9
|
+
const canonicalRelativePathSchema = z.string().min(1).max(1024).superRefine((value, context) => {
|
|
10
|
+
if (value.startsWith("/") || value.endsWith("/") || value.includes("\\")
|
|
11
|
+
|| /[\u0000-\u001f\u007f]/.test(value)) {
|
|
12
|
+
context.addIssue({ code: "custom", message: "Skill paths must be relative canonical paths." });
|
|
13
|
+
return;
|
|
14
|
+
}
|
|
15
|
+
const segments = value.split("/");
|
|
16
|
+
if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
|
|
17
|
+
context.addIssue({ code: "custom", message: "Skill paths contain an invalid segment." });
|
|
18
|
+
}
|
|
19
|
+
});
|
|
20
|
+
export const skillRelativePathSchema = canonicalRelativePathSchema.refine((value) => value.split("/").at(-1) === "SKILL.md", "Skill targets must be existing files named SKILL.md.");
|
|
21
|
+
export const skillEvidenceOccurrenceSchema = z.strictObject({
|
|
22
|
+
session_id: z.string().min(1).max(256),
|
|
23
|
+
session_digest: digestSchema,
|
|
24
|
+
ended_at: timestampSchema,
|
|
25
|
+
outcome: z.enum(["failure", "resolution", "abandonment"]),
|
|
26
|
+
tool_name: z.string().min(1).max(256).optional(),
|
|
27
|
+
evidence: boundedTextSchema,
|
|
28
|
+
});
|
|
29
|
+
export const skillCandidateSchema = z.strictObject({
|
|
30
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
31
|
+
id: identifierSchema,
|
|
32
|
+
project_id: identifierSchema,
|
|
33
|
+
status: z.enum(["pending_review", "approved"]),
|
|
34
|
+
note_id: identifierSchema,
|
|
35
|
+
source_digest: digestSchema,
|
|
36
|
+
title: z.string().min(1).max(512),
|
|
37
|
+
description: boundedTextSchema,
|
|
38
|
+
failure_family: identifierSchema,
|
|
39
|
+
failure_count: z.number().int().min(2).max(1024),
|
|
40
|
+
resolution_count: z.number().int().min(1).max(1024),
|
|
41
|
+
occurrences: z.array(skillEvidenceOccurrenceSchema).min(2).max(64),
|
|
42
|
+
lessons: z.array(z.string().min(1).max(1024)).max(16),
|
|
43
|
+
caveats: z.array(z.string().min(1).max(1024)).max(16),
|
|
44
|
+
created_at: timestampSchema,
|
|
45
|
+
reviewed_at: timestampSchema.nullable(),
|
|
46
|
+
});
|
|
47
|
+
export const skillEditSchema = z.strictObject({
|
|
48
|
+
operation: z.enum(["add", "replace", "delete"]),
|
|
49
|
+
anchor: z.string().min(1).max(16 * 1024).nullable(),
|
|
50
|
+
content: z.string().max(64 * 1024),
|
|
51
|
+
rationale: z.string().min(1).max(2048),
|
|
52
|
+
}).superRefine((edit, context) => {
|
|
53
|
+
if (edit.operation !== "add" && edit.anchor === null) {
|
|
54
|
+
context.addIssue({ code: "custom", path: ["anchor"], message: "Replace and delete edits require an anchor." });
|
|
55
|
+
}
|
|
56
|
+
if (edit.operation === "delete" && edit.content !== "") {
|
|
57
|
+
context.addIssue({ code: "custom", path: ["content"], message: "Delete edits require empty content." });
|
|
58
|
+
}
|
|
59
|
+
if (edit.operation !== "delete" && edit.content.length === 0) {
|
|
60
|
+
context.addIssue({ code: "custom", path: ["content"], message: "Add and replace edits require content." });
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
export const skillAdapterConfigSchema = z.strictObject({
|
|
64
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
65
|
+
id: identifierSchema,
|
|
66
|
+
command: z.strictObject({
|
|
67
|
+
program: z.string().min(1).max(4096),
|
|
68
|
+
args: z.array(z.string().max(16 * 1024)).max(128),
|
|
69
|
+
}),
|
|
70
|
+
environment_allowlist: z.array(z.string().regex(/^[A-Z][A-Z0-9_]{0,127}$/)).max(64).default([]),
|
|
71
|
+
limits: z.strictObject({
|
|
72
|
+
elapsed_ms: z.number().int().positive().max(3_600_000),
|
|
73
|
+
stdout_bytes: z.number().int().positive().max(4 * 1024 * 1024),
|
|
74
|
+
}),
|
|
75
|
+
});
|
|
76
|
+
export const skillProposalRequestSchema = z.strictObject({
|
|
77
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
78
|
+
operation: z.literal("propose"),
|
|
79
|
+
candidate: skillCandidateSchema.refine((candidate) => candidate.status === "approved", "Candidate must be approved."),
|
|
80
|
+
candidate_digest: digestSchema,
|
|
81
|
+
target: z.strictObject({
|
|
82
|
+
path: skillRelativePathSchema,
|
|
83
|
+
baseline_digest: digestSchema,
|
|
84
|
+
content: z.string().max(MAX_SKILL_BYTES),
|
|
85
|
+
}),
|
|
86
|
+
edit_budget: z.number().int().positive().max(MAX_SKILL_EDITS),
|
|
87
|
+
});
|
|
88
|
+
export const skillProposalResponseSchema = z.strictObject({
|
|
89
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
90
|
+
status: z.enum(["completed", "adapter_error"]),
|
|
91
|
+
error_code: identifierSchema.optional(),
|
|
92
|
+
edits: z.array(skillEditSchema).max(MAX_SKILL_EDITS).default([]),
|
|
93
|
+
}).superRefine((response, context) => {
|
|
94
|
+
if (response.status === "completed" && response.edits.length === 0) {
|
|
95
|
+
context.addIssue({ code: "custom", path: ["edits"], message: "A completed proposal requires at least one edit." });
|
|
96
|
+
}
|
|
97
|
+
if (response.status === "adapter_error" && !response.error_code) {
|
|
98
|
+
context.addIssue({ code: "custom", path: ["error_code"], message: "Adapter errors require an error code." });
|
|
99
|
+
}
|
|
100
|
+
});
|
|
101
|
+
export const skillProposalSchema = z.strictObject({
|
|
102
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
103
|
+
id: identifierSchema,
|
|
104
|
+
candidate_id: identifierSchema,
|
|
105
|
+
candidate_digest: digestSchema,
|
|
106
|
+
target_path: skillRelativePathSchema,
|
|
107
|
+
baseline_digest: digestSchema,
|
|
108
|
+
candidate_content_digest: digestSchema,
|
|
109
|
+
candidate_content: z.string().max(MAX_SKILL_BYTES),
|
|
110
|
+
target_mode: z.number().int().min(0).max(0o777),
|
|
111
|
+
adapter_id: identifierSchema,
|
|
112
|
+
adapter_config_digest: digestSchema,
|
|
113
|
+
adapter_program_digest: digestSchema,
|
|
114
|
+
edits: z.array(skillEditSchema).min(1).max(MAX_SKILL_EDITS),
|
|
115
|
+
created_at: timestampSchema,
|
|
116
|
+
});
|
|
117
|
+
const skillEvaluationSummarySchema = z.strictObject({
|
|
118
|
+
task_set_digest: digestSchema,
|
|
119
|
+
report_digest: digestSchema,
|
|
120
|
+
experiment_id: z.string().min(1).max(128),
|
|
121
|
+
baseline_passed: z.number().int().nonnegative(),
|
|
122
|
+
candidate_passed: z.number().int().nonnegative(),
|
|
123
|
+
eligible_pairs: z.number().int().nonnegative(),
|
|
124
|
+
improvements: z.number().int().nonnegative(),
|
|
125
|
+
regressions: z.number().int().nonnegative(),
|
|
126
|
+
unknown: z.number().int().nonnegative(),
|
|
127
|
+
});
|
|
128
|
+
export const skillEvaluationSchema = z.strictObject({
|
|
129
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
130
|
+
id: identifierSchema,
|
|
131
|
+
proposal_id: identifierSchema,
|
|
132
|
+
proposal_digest: digestSchema,
|
|
133
|
+
binding_digest: digestSchema,
|
|
134
|
+
evaluation_adapter_program_digest: digestSchema,
|
|
135
|
+
status: z.enum(["eligible", "rejected", "inconclusive"]),
|
|
136
|
+
minimum_improvement: z.number().int().positive().max(10_000),
|
|
137
|
+
exploration: skillEvaluationSummarySchema,
|
|
138
|
+
confirmation: skillEvaluationSummarySchema.nullable(),
|
|
139
|
+
reasons: z.array(identifierSchema).max(64),
|
|
140
|
+
created_at: timestampSchema,
|
|
141
|
+
}).superRefine((evaluation, context) => {
|
|
142
|
+
if (evaluation.status === "eligible" && (evaluation.confirmation === null || evaluation.reasons.length > 0)) {
|
|
143
|
+
context.addIssue({ code: "custom", message: "Eligible evaluations require confirmation and no rejection reasons." });
|
|
144
|
+
}
|
|
145
|
+
if (evaluation.status !== "eligible" && evaluation.reasons.length === 0) {
|
|
146
|
+
context.addIssue({ code: "custom", path: ["reasons"], message: "Non-eligible evaluations require at least one reason." });
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
export const skillApplicationSchema = z.strictObject({
|
|
150
|
+
schema_version: z.literal(SKILL_SCHEMA_VERSION),
|
|
151
|
+
id: identifierSchema,
|
|
152
|
+
proposal_id: identifierSchema,
|
|
153
|
+
proposal_digest: digestSchema,
|
|
154
|
+
evaluation_id: identifierSchema,
|
|
155
|
+
evaluation_digest: digestSchema,
|
|
156
|
+
target_path: skillRelativePathSchema,
|
|
157
|
+
before_digest: digestSchema,
|
|
158
|
+
applied_digest: digestSchema,
|
|
159
|
+
backup_path: canonicalRelativePathSchema,
|
|
160
|
+
target_mode: z.number().int().min(0).max(0o777),
|
|
161
|
+
state: z.enum(["applied", "rolled_back"]),
|
|
162
|
+
applied_at: timestampSchema,
|
|
163
|
+
rolled_back_at: timestampSchema.nullable(),
|
|
164
|
+
});
|