@cairnkeep/cli 2.6.0 → 2.8.0

package/mcp-memory-server/dist/skill-evaluation.js

@@ -0,0 +1,244 @@
+import { canonicalDigest } from "./eval-schema.js";
+import { isEvalEnabled, loadEvalPlan } from "./eval-plan.js";
+import { runTwoPassExperiment } from "./eval-runner.js";
+import { getSkillStore, readSkillProposal, readSkillTarget, sha256File, writeSkillEvaluation, } from "./skill-store.js";
+import { SKILL_SCHEMA_VERSION, skillEvaluationSchema } from "./skill-schema.js";
+function requireRootWorkspaces(plan) {
+    if (plan.task_set.tasks.some((task) => task.workspace.path !== ".")) {
+        throw new Error("Skill evaluation tasks must use the repository root as their workspace.");
+    }
+}
+function requireDisjointPlans(exploration, confirmation) {
+    if (exploration.task_set.tasks.length < 2 || confirmation.task_set.tasks.length < 2) {
+        throw new Error("Skill evaluation requires at least two exploration and two confirmation tasks.");
+    }
+    if (exploration.task_set_digest === confirmation.task_set_digest) {
+        throw new Error("Exploration and confirmation task sets must be different committed documents.");
+    }
+    if (exploration.source.kind !== confirmation.source.kind) {
+        throw new Error("Exploration and confirmation task sets must bind the same source kind.");
+    }
+    const explorationRevision = exploration.source.kind === "git"
+        ? exploration.source.revision
+        : exploration.source.package_version;
+    const confirmationRevision = confirmation.source.kind === "git"
+        ? confirmation.source.revision
+        : confirmation.source.package_version;
+    if (explorationRevision !== confirmationRevision) {
+        throw new Error("Exploration and confirmation task sets must bind the same immutable source revision.");
+    }
+    const explorationIds = new Set(exploration.task_set.tasks.map(({ id }) => id));
+    if (confirmation.task_set.tasks.some(({ id }) => explorationIds.has(id))) {
+        throw new Error("Exploration and confirmation task IDs must not overlap.");
+    }
+    const taskDefinitionDigest = ({ id: _id, ...task }) => canonicalDigest(task);
+    const explorationDefinitions = new Set(exploration.task_set.tasks.map(taskDefinitionDigest));
+    if (confirmation.task_set.tasks.some((task) => explorationDefinitions.has(taskDefinitionDigest(task)))) {
+        throw new Error("Exploration and confirmation task definitions must not overlap.");
+    }
+}
+function observationKey(observation) {
+    return `${observation.task_id}\0${observation.repetition}\0${observation.pass}`;
+}
+export function summarizeSkillReport(report) {
+    if (report.status !== "final" || report.experiment_kind !== "skill_candidate") {
+        throw new Error("Skill evaluation requires a final skill-candidate report.");
+    }
+    const baseline = new Map();
+    const treatment = new Map();
+    for (const observation of report.observations) {
+        (observation.arm === "baseline" ? baseline : treatment).set(observationKey(observation), observation);
+    }
+    let baselinePassed = 0;
+    let candidatePassed = 0;
+    let eligiblePairs = 0;
+    const improvedTasks = new Set();
+    const regressedTasks = new Set();
+    let unknown = 0;
+    const scheduledPairs = report.schedule.length / 2;
+    for (const [key, baselineObservation] of baseline) {
+        const candidateObservation = treatment.get(key);
+        const eligible = candidateObservation !== undefined
+            && baselineObservation.state === "terminal"
+            && candidateObservation.state === "terminal"
+            && baselineObservation.capability_status === "valid"
+            && candidateObservation.capability_status === "valid"
+            && baselineObservation.pass_state !== "unknown"
+            && candidateObservation.pass_state !== "unknown";
+        if (!eligible || !candidateObservation)
+            continue;
+        eligiblePairs += 1;
+        const baselinePass = baselineObservation.pass_state === "passed";
+        const candidatePass = candidateObservation.pass_state === "passed";
+        if (baselinePass)
+            baselinePassed += 1;
+        if (candidatePass)
+            candidatePassed += 1;
+        if (!baselinePass && candidatePass)
+            improvedTasks.add(baselineObservation.task_id);
+        if (baselinePass && !candidatePass)
+            regressedTasks.add(baselineObservation.task_id);
+    }
+    unknown = Math.max(0, scheduledPairs - eligiblePairs);
+    return {
+        task_set_digest: report.task_set_digest,
+        report_digest: canonicalDigest(report),
+        experiment_id: report.experiment_id,
+        baseline_passed: baselinePassed,
+        candidate_passed: candidatePassed,
+        eligible_pairs: eligiblePairs,
+        improvements: improvedTasks.size,
+        regressions: regressedTasks.size,
+        unknown,
+    };
+}
+function gate(summary, minimumImprovement, prefix) {
+    const reasons = [];
+    if (summary.eligible_pairs === 0 || summary.unknown > 0)
+        reasons.push(`${prefix}_incomplete`);
+    if (summary.regressions > 0)
+        reasons.push(`${prefix}_regression`);
+    if (summary.improvements < minimumImprovement)
+        reasons.push(`${prefix}_no_improvement`);
+    return reasons;
+}
+export async function evaluateSkillProposal(options) {
+    if (!options.confirm)
+        throw new Error("Skill evaluation requires --yes because it invokes the configured evaluation adapter.");
+    if (!isEvalEnabled())
+        throw new Error("Skill evaluation is disabled. Set CAIRN_EVAL=1 to opt in.");
+    const repetitions = options.repetitions ?? 2;
+    const minimumImprovement = options.minimumImprovement ?? 1;
+    if (!Number.isSafeInteger(repetitions) || repetitions < 1 || repetitions > 1_000) {
+        throw new Error("Evaluation repetitions must be an integer from 1 through 1000.");
+    }
+    if (!Number.isSafeInteger(minimumImprovement) || minimumImprovement < 1 || minimumImprovement > 10_000) {
+        throw new Error("Minimum improvement must be an integer from 1 through 10000.");
+    }
+    const store = getSkillStore(options.projectRoot);
+    const proposal = readSkillProposal(store.project_root, options.proposalId);
+    const target = readSkillTarget(store.project_root, proposal.target_path);
+    if (target.digest !== proposal.baseline_digest)
+        throw new Error("Skill target changed after proposal generation.");
+    const arms = [
+        { id: "baseline", disabled_capability: null },
+        { id: "treatment", disabled_capability: null },
+    ];
+    const outputRoot = options.outputRoot ?? ".agentfs/eval/experiments";
+    const seed = options.seed ?? "cairn-skill-eval-v1";
+    const common = {
+        adapterPath: options.adapterPath,
+        outputRoot,
+        repetitions,
+        arms,
+        experimentKind: "skill_candidate",
+        cwd: store.project_root,
+    };
+    const explorationPlan = loadEvalPlan({
+        ...common,
+        taskSetPath: options.explorationTaskSetPath,
+        seed: `${seed}:exploration`,
+    });
+    const confirmationPlan = loadEvalPlan({
+        ...common,
+        taskSetPath: options.confirmationTaskSetPath,
+        seed: `${seed}:confirmation`,
+    });
+    requireRootWorkspaces(explorationPlan);
+    requireRootWorkspaces(confirmationPlan);
+    requireDisjointPlans(explorationPlan, confirmationPlan);
+    if (minimumImprovement > explorationPlan.task_set.tasks.length
+        || minimumImprovement > confirmationPlan.task_set.tasks.length) {
+        throw new Error("Minimum improvement cannot exceed the task count in either evaluation set.");
+    }
+    if (explorationPlan.adapter_config_digest !== confirmationPlan.adapter_config_digest
+        || explorationPlan.resolved_programs.adapter !== confirmationPlan.resolved_programs.adapter) {
+        throw new Error("Exploration and confirmation must use the same evaluation adapter.");
+    }
+    const evaluationAdapterProgramDigest = sha256File(explorationPlan.resolved_programs.adapter);
+    const proposalDigest = canonicalDigest(proposal);
+    const binding = {
+        schema_version: SKILL_SCHEMA_VERSION,
+        proposal_id: proposal.id,
+        proposal_digest: proposalDigest,
+        baseline_digest: proposal.baseline_digest,
+        candidate_digest: proposal.candidate_content_digest,
+        exploration_plan_digest: explorationPlan.plan_digest,
+        confirmation_plan_digest: confirmationPlan.plan_digest,
+        evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
+        repetitions,
+        minimum_improvement: minimumImprovement,
+    };
+    const bindingDigest = canonicalDigest(binding);
+    const overlay = {
+        baseline: {
+            relative_path: proposal.target_path,
+            content: target.content,
+            digest: proposal.baseline_digest,
+        },
+        treatment: {
+            relative_path: proposal.target_path,
+            content: proposal.candidate_content,
+            digest: proposal.candidate_content_digest,
+        },
+    };
+    const explorationRun = await runTwoPassExperiment({
+        plan: explorationPlan,
+        experiment_id: `skill-explore-${bindingDigest.slice(0, 20)}`,
+        workspace_overlays: overlay,
+        signal: options.signal,
+    });
+    const exploration = summarizeSkillReport(explorationRun.report);
+    if (sha256File(explorationPlan.resolved_programs.adapter) !== evaluationAdapterProgramDigest) {
+        throw new Error("Evaluation adapter executable changed during exploration.");
+    }
+    const explorationReasons = gate(exploration, minimumImprovement, "exploration");
+    if (explorationReasons.length > 0) {
+        const evaluation = skillEvaluationSchema.parse({
+            schema_version: SKILL_SCHEMA_VERSION,
+            id: `evaluation-${bindingDigest.slice(0, 24)}`,
+            proposal_id: proposal.id,
+            proposal_digest: proposalDigest,
+            binding_digest: bindingDigest,
+            evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
+            status: exploration.unknown > 0 ? "inconclusive" : "rejected",
+            minimum_improvement: minimumImprovement,
+            exploration,
+            confirmation: null,
+            reasons: explorationReasons,
+            created_at: new Date().toISOString(),
+        });
+        writeSkillEvaluation(store.project_root, evaluation);
+        return evaluation;
+    }
+    const confirmationRun = await runTwoPassExperiment({
+        plan: confirmationPlan,
+        experiment_id: `skill-confirm-${bindingDigest.slice(0, 20)}`,
+        workspace_overlays: overlay,
+        signal: options.signal,
+    });
+    if (sha256File(confirmationPlan.resolved_programs.adapter) !== evaluationAdapterProgramDigest) {
+        throw new Error("Evaluation adapter executable changed during confirmation.");
+    }
+    const confirmation = summarizeSkillReport(confirmationRun.report);
+    const confirmationReasons = gate(confirmation, minimumImprovement, "confirmation");
+    const status = confirmationReasons.length === 0
+        ? "eligible"
+        : confirmation.unknown > 0 ? "inconclusive" : "rejected";
+    const evaluation = skillEvaluationSchema.parse({
+        schema_version: SKILL_SCHEMA_VERSION,
+        id: `evaluation-${bindingDigest.slice(0, 24)}`,
+        proposal_id: proposal.id,
+        proposal_digest: proposalDigest,
+        binding_digest: bindingDigest,
+        evaluation_adapter_program_digest: evaluationAdapterProgramDigest,
+        status,
+        minimum_improvement: minimumImprovement,
+        exploration,
+        confirmation,
+        reasons: confirmationReasons,
+        created_at: new Date().toISOString(),
+    });
+    writeSkillEvaluation(store.project_root, evaluation);
+    return evaluation;
+}

package/mcp-memory-server/dist/skill-schema.js

@@ -0,0 +1,164 @@
+import { z } from "zod";
+export const SKILL_SCHEMA_VERSION = 1;
+export const MAX_SKILL_BYTES = 256 * 1024;
+export const MAX_SKILL_EDITS = 16;
+const identifierSchema = z.string().min(1).max(128).regex(/^[a-z0-9][a-z0-9._-]*$/);
+const digestSchema = z.string().regex(/^[a-f0-9]{64}$/);
+const timestampSchema = z.iso.datetime();
+const boundedTextSchema = z.string().max(4096);
+const canonicalRelativePathSchema = z.string().min(1).max(1024).superRefine((value, context) => {
+    if (value.startsWith("/") || value.endsWith("/") || value.includes("\\")
+        || /[\u0000-\u001f\u007f]/.test(value)) {
+        context.addIssue({ code: "custom", message: "Skill paths must be relative canonical paths." });
+        return;
+    }
+    const segments = value.split("/");
+    if (segments.some((segment) => segment === "" || segment === "." || segment === "..")) {
+        context.addIssue({ code: "custom", message: "Skill paths contain an invalid segment." });
+    }
+});
+export const skillRelativePathSchema = canonicalRelativePathSchema.refine((value) => value.split("/").at(-1) === "SKILL.md", "Skill targets must be existing files named SKILL.md.");
+export const skillEvidenceOccurrenceSchema = z.strictObject({
+    session_id: z.string().min(1).max(256),
+    session_digest: digestSchema,
+    ended_at: timestampSchema,
+    outcome: z.enum(["failure", "resolution", "abandonment"]),
+    tool_name: z.string().min(1).max(256).optional(),
+    evidence: boundedTextSchema,
+});
+export const skillCandidateSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    id: identifierSchema,
+    project_id: identifierSchema,
+    status: z.enum(["pending_review", "approved"]),
+    note_id: identifierSchema,
+    source_digest: digestSchema,
+    title: z.string().min(1).max(512),
+    description: boundedTextSchema,
+    failure_family: identifierSchema,
+    failure_count: z.number().int().min(2).max(1024),
+    resolution_count: z.number().int().min(1).max(1024),
+    occurrences: z.array(skillEvidenceOccurrenceSchema).min(2).max(64),
+    lessons: z.array(z.string().min(1).max(1024)).max(16),
+    caveats: z.array(z.string().min(1).max(1024)).max(16),
+    created_at: timestampSchema,
+    reviewed_at: timestampSchema.nullable(),
+});
+export const skillEditSchema = z.strictObject({
+    operation: z.enum(["add", "replace", "delete"]),
+    anchor: z.string().min(1).max(16 * 1024).nullable(),
+    content: z.string().max(64 * 1024),
+    rationale: z.string().min(1).max(2048),
+}).superRefine((edit, context) => {
+    if (edit.operation !== "add" && edit.anchor === null) {
+        context.addIssue({ code: "custom", path: ["anchor"], message: "Replace and delete edits require an anchor." });
+    }
+    if (edit.operation === "delete" && edit.content !== "") {
+        context.addIssue({ code: "custom", path: ["content"], message: "Delete edits require empty content." });
+    }
+    if (edit.operation !== "delete" && edit.content.length === 0) {
+        context.addIssue({ code: "custom", path: ["content"], message: "Add and replace edits require content." });
+    }
+});
+export const skillAdapterConfigSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    id: identifierSchema,
+    command: z.strictObject({
+        program: z.string().min(1).max(4096),
+        args: z.array(z.string().max(16 * 1024)).max(128),
+    }),
+    environment_allowlist: z.array(z.string().regex(/^[A-Z][A-Z0-9_]{0,127}$/)).max(64).default([]),
+    limits: z.strictObject({
+        elapsed_ms: z.number().int().positive().max(3_600_000),
+        stdout_bytes: z.number().int().positive().max(4 * 1024 * 1024),
+    }),
+});
+export const skillProposalRequestSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    operation: z.literal("propose"),
+    candidate: skillCandidateSchema.refine((candidate) => candidate.status === "approved", "Candidate must be approved."),
+    candidate_digest: digestSchema,
+    target: z.strictObject({
+        path: skillRelativePathSchema,
+        baseline_digest: digestSchema,
+        content: z.string().max(MAX_SKILL_BYTES),
+    }),
+    edit_budget: z.number().int().positive().max(MAX_SKILL_EDITS),
+});
+export const skillProposalResponseSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    status: z.enum(["completed", "adapter_error"]),
+    error_code: identifierSchema.optional(),
+    edits: z.array(skillEditSchema).max(MAX_SKILL_EDITS).default([]),
+}).superRefine((response, context) => {
+    if (response.status === "completed" && response.edits.length === 0) {
+        context.addIssue({ code: "custom", path: ["edits"], message: "A completed proposal requires at least one edit." });
+    }
+    if (response.status === "adapter_error" && !response.error_code) {
+        context.addIssue({ code: "custom", path: ["error_code"], message: "Adapter errors require an error code." });
+    }
+});
+export const skillProposalSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    id: identifierSchema,
+    candidate_id: identifierSchema,
+    candidate_digest: digestSchema,
+    target_path: skillRelativePathSchema,
+    baseline_digest: digestSchema,
+    candidate_content_digest: digestSchema,
+    candidate_content: z.string().max(MAX_SKILL_BYTES),
+    target_mode: z.number().int().min(0).max(0o777),
+    adapter_id: identifierSchema,
+    adapter_config_digest: digestSchema,
+    adapter_program_digest: digestSchema,
+    edits: z.array(skillEditSchema).min(1).max(MAX_SKILL_EDITS),
+    created_at: timestampSchema,
+});
+const skillEvaluationSummarySchema = z.strictObject({
+    task_set_digest: digestSchema,
+    report_digest: digestSchema,
+    experiment_id: z.string().min(1).max(128),
+    baseline_passed: z.number().int().nonnegative(),
+    candidate_passed: z.number().int().nonnegative(),
+    eligible_pairs: z.number().int().nonnegative(),
+    improvements: z.number().int().nonnegative(),
+    regressions: z.number().int().nonnegative(),
+    unknown: z.number().int().nonnegative(),
+});
+export const skillEvaluationSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    id: identifierSchema,
+    proposal_id: identifierSchema,
+    proposal_digest: digestSchema,
+    binding_digest: digestSchema,
+    evaluation_adapter_program_digest: digestSchema,
+    status: z.enum(["eligible", "rejected", "inconclusive"]),
+    minimum_improvement: z.number().int().positive().max(10_000),
+    exploration: skillEvaluationSummarySchema,
+    confirmation: skillEvaluationSummarySchema.nullable(),
+    reasons: z.array(identifierSchema).max(64),
+    created_at: timestampSchema,
+}).superRefine((evaluation, context) => {
+    if (evaluation.status === "eligible" && (evaluation.confirmation === null || evaluation.reasons.length > 0)) {
+        context.addIssue({ code: "custom", message: "Eligible evaluations require confirmation and no rejection reasons." });
+    }
+    if (evaluation.status !== "eligible" && evaluation.reasons.length === 0) {
+        context.addIssue({ code: "custom", path: ["reasons"], message: "Non-eligible evaluations require at least one reason." });
+    }
+});
+export const skillApplicationSchema = z.strictObject({
+    schema_version: z.literal(SKILL_SCHEMA_VERSION),
+    id: identifierSchema,
+    proposal_id: identifierSchema,
+    proposal_digest: digestSchema,
+    evaluation_id: identifierSchema,
+    evaluation_digest: digestSchema,
+    target_path: skillRelativePathSchema,
+    before_digest: digestSchema,
+    applied_digest: digestSchema,
+    backup_path: canonicalRelativePathSchema,
+    target_mode: z.number().int().min(0).max(0o777),
+    state: z.enum(["applied", "rolled_back"]),
+    applied_at: timestampSchema,
+    rolled_back_at: timestampSchema.nullable(),
+});