create-agdf 0.9.0 → 0.9.3

package/lib/scaffold/plan.js

@@ -0,0 +1,278 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { generatedRoot, languageConfigContent, pluginDefinition } from "../cli/runtime-context.js";
+
+export const agdfFragmentPath = "AGENTS.agdf.md";
+export const openCodeConfigFragmentPath = "opencode.agdf.json";
+const codexSkillNames = pluginDefinition.skillSet.map((skill) => `${pluginDefinition.codex.skillPrefix}${skill.slug}`);
+const copilotSkillNames = pluginDefinition.skillSet.map((skill) => `${pluginDefinition.copilot.skillPrefix}${skill.slug}`);
+const openCodeSkillNames = pluginDefinition.skillSet.map((skill) => `${pluginDefinition.opencode.skillPrefix}${skill.slug}`);
+const globalOpenCodeSkillNames = pluginDefinition.skillSet.map((skill) => `${pluginDefinition.opencode.globalSkillPrefix}${skill.slug}`);
+const contractModules = [
+  "gate-transition.md",
+  "interaction.md",
+  "modes.md",
+  "quality.md",
+  "context-graph.md",
+  "control-scaffold.md",
+  "closeout.md",
+];
+const codexPluginFiles = [
+  join(".agents", "plugins", "marketplace.json"),
+  join("plugins", "agdf", ".codex-plugin", "plugin.json"),
+  join("plugins", "agdf", "control", "README.md"),
+  join("plugins", "agdf", "control", "templates", "AGDF_RUN.md"),
+  join("plugins", "agdf", "control", "templates", "RUN_STATE.md"),
+  join("plugins", "agdf", "control", "templates", "MASTER_BACKLOG.md"),
+  join("plugins", "agdf", "control", "templates", "SOT_REGISTRY.md"),
+  join("plugins", "agdf", "control", "templates", "CONTEXT_GRAPH.md"),
+  join("plugins", "agdf", "control", "templates", "AGENT_QUALITY_CONTRACTS.json"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "UR.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "BROWNFIELD_REVIEW.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "VERIFIED_CHANGE.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "PRD.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "SD.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "TP.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "QA_REPORT.md"),
+  join("plugins", "agdf", "control", "templates", "artefacts", "OR.md"),
+  join("plugins", "agdf", "hooks", "hooks.json"),
+  join("plugins", "agdf", "hooks", "session-start.sh"),
+  join("plugins", "agdf", "meta", "agdf-agent-router.md"),
+  join("plugins", "agdf", "meta", "agdf-constitution.md"),
+  join("plugins", "agdf", "meta", "agdf-plugin.definition.json"),
+  join("plugins", "agdf", "meta", "agdf-runtime-contract.md"),
+  join("plugins", "agdf", "meta", "agdf-tenets.md"),
+  ...contractModules.map((moduleName) => join("plugins", "agdf", "meta", "contracts", moduleName)),
+  ...codexSkillNames.map((skillName) => join("plugins", "agdf", "skills", skillName, "SKILL.md")),
+];
+const controlFiles = [
+  join(".agdf", "control", "README.md"),
+  join(".agdf", "control", "templates", "AGDF_RUN.md"),
+  join(".agdf", "control", "templates", "RUN_STATE.md"),
+  join(".agdf", "control", "templates", "MASTER_BACKLOG.md"),
+  join(".agdf", "control", "templates", "SOT_REGISTRY.md"),
+  join(".agdf", "control", "templates", "CONTEXT_GRAPH.md"),
+  join(".agdf", "control", "templates", "AGENT_QUALITY_CONTRACTS.json"),
+  join(".agdf", "control", "templates", "artefacts", "UR.md"),
+  join(".agdf", "control", "templates", "artefacts", "BROWNFIELD_REVIEW.md"),
+  join(".agdf", "control", "templates", "artefacts", "VERIFIED_CHANGE.md"),
+  join(".agdf", "control", "templates", "artefacts", "PRD.md"),
+  join(".agdf", "control", "templates", "artefacts", "SD.md"),
+  join(".agdf", "control", "templates", "artefacts", "TP.md"),
+  join(".agdf", "control", "templates", "artefacts", "QA_REPORT.md"),
+  join(".agdf", "control", "templates", "artefacts", "OR.md"),
+];
+const liveControlFiles = [
+  {
+    path: join(".agdf", "control", "AGDF_RUN.md"),
+    source: join(".agdf", "control", "templates", "AGDF_RUN.md"),
+  },
+  {
+    path: join(".agdf", "control", "MASTER_BACKLOG.md"),
+    source: join(".agdf", "control", "templates", "MASTER_BACKLOG.md"),
+  },
+  {
+    path: join(".agdf", "control", "SOT_REGISTRY.md"),
+    source: join(".agdf", "control", "templates", "SOT_REGISTRY.md"),
+  },
+  {
+    path: join(".agdf", "control", "CONTEXT_GRAPH.md"),
+    source: join(".agdf", "control", "templates", "CONTEXT_GRAPH.md"),
+  },
+  {
+    path: join(".agdf", "control", "AGENT_QUALITY_CONTRACTS.json"),
+    source: join(".agdf", "control", "templates", "AGENT_QUALITY_CONTRACTS.json"),
+  },
+];
+const artefactTemplateFiles = [
+  join(".agdf", "control", "templates", "artefacts", "UR.md"),
+  join(".agdf", "control", "templates", "artefacts", "BROWNFIELD_REVIEW.md"),
+  join(".agdf", "control", "templates", "artefacts", "VERIFIED_CHANGE.md"),
+  join(".agdf", "control", "templates", "artefacts", "PRD.md"),
+  join(".agdf", "control", "templates", "artefacts", "SD.md"),
+  join(".agdf", "control", "templates", "artefacts", "TP.md"),
+  join(".agdf", "control", "templates", "artefacts", "QA_REPORT.md"),
+  join(".agdf", "control", "templates", "artefacts", "OR.md"),
+];
+export const doctorRequiredFiles = [
+  join(".agdf", "control", "AGDF_RUN.md"),
+  join(".agdf", "control", "MASTER_BACKLOG.md"),
+  join(".agdf", "control", "SOT_REGISTRY.md"),
+  join(".agdf", "control", "CONTEXT_GRAPH.md"),
+  join(".agdf", "control", "AGENT_QUALITY_CONTRACTS.json"),
+];
+const copilotInstructionFiles = [
+  join(".github", "copilot-instructions.md"),
+  join(".github", "instructions", "agdf-governance.instructions.md"),
+];
+const copilotSkillFiles = [
+  join(".github", "skills", "README.md"),
+  join(".github", "skills", pluginDefinition.copilot.runtimeContractFileName),
+  ...contractModules.map((moduleName) => join(".github", "skills", "contracts", moduleName)),
+  ...copilotSkillNames.map((skillName) => join(".github", "skills", skillName, "SKILL.md")),
+];
+const openCodeFiles = [
+  join(".opencode", pluginDefinition.opencode.instructionsFileName),
+  join(".opencode", "README.md"),
+  join(".opencode", pluginDefinition.opencode.runtimeContractFileName),
+  ...contractModules.map((moduleName) => join(".opencode", "contracts", moduleName)),
+  ...openCodeSkillNames.map((skillName) => join(".opencode", "skills", skillName, "SKILL.md")),
+];
+
+function loadAsset(relativePath) {
+  return readFileSync(join(generatedRoot, relativePath), "utf8");
+}
+
+function isAgdfOwnedAgentsFile(content) {
+  return content.includes("## Surface Convention")
+    && content.includes("GitHub Copilot repository skills do not have a plugin namespace")
+    && content.includes("AGDF is agent-native first and CLI-verifiable by design");
+}
+
+function addCopilotAgentsFile(files, targetDir, force) {
+  const agentsPath = join(targetDir, "AGENTS.md");
+  if (!existsSync(agentsPath) || force) {
+    files.push({
+      path: "AGENTS.md",
+      content: loadAsset("AGENTS.md"),
+    });
+    return;
+  }
+
+  const existingAgents = readFileSync(agentsPath, "utf8");
+  if (isAgdfOwnedAgentsFile(existingAgents)) {
+    files.push({
+      path: "AGENTS.md",
+      content: loadAsset("AGENTS.md"),
+      allowOverwrite: true,
+      action: "refreshed",
+    });
+    return;
+  }
+
+  files.push({
+    path: agdfFragmentPath,
+    content: loadAsset("AGENTS.md"),
+    allowOverwrite: true,
+    action: "refreshed",
+    preserved: "AGENTS.md",
+  });
+}
+
+function shouldWriteLanguageConfig(target, targetDir, force) {
+  if (force) return true;
+  if ((target === "copilot" || target === "both") && existsSync(join(targetDir, ".agdf", "control", "config.json"))) return false;
+  return true;
+}
+
+function addLanguageConfig(files, languagePreference) {
+  files.push({
+    path: join(".agdf", "control", "config.json"),
+    content: languageConfigContent(languagePreference),
+  });
+}
+
+export function generatedFilesForTarget(target, targetDir, force, languagePreference) {
+  const files = [];
+
+  if (target === "config") {
+    files.push({
+      path: join(".agdf", "control", "config.json"),
+      content: languageConfigContent(languagePreference),
+      allowOverwrite: true,
+    });
+    return files;
+  }
+
+  if (target === "init") {
+    addLanguageConfig(files, languagePreference);
+    files.push({
+      path: join(".agdf", "control", "README.md"),
+      content: loadAsset(join(".agdf", "control", "README.md")),
+    });
+
+    for (const controlPath of liveControlFiles) {
+      files.push({
+        path: controlPath.path,
+        content: loadAsset(controlPath.source),
+      });
+    }
+
+    for (const templatePath of artefactTemplateFiles) {
+      files.push({
+        path: templatePath,
+        content: loadAsset(templatePath),
+      });
+    }
+
+    return files;
+  }
+
+  if (target === "codex-repo" || target === "both") {
+    if (shouldWriteLanguageConfig(target, targetDir, force)) addLanguageConfig(files, languagePreference);
+    for (const codexPath of codexPluginFiles) {
+      files.push({
+        path: codexPath,
+        content: loadAsset(codexPath),
+      });
+    }
+  }
+
+  if (target === "copilot" || target === "both") {
+    if (target !== "both" && shouldWriteLanguageConfig(target, targetDir, force)) addLanguageConfig(files, languagePreference);
+    addCopilotAgentsFile(files, targetDir, force);
+
+    for (const controlPath of controlFiles) {
+      files.push({
+        path: controlPath,
+        content: loadAsset(controlPath),
+        allowOverwrite: true,
+        action: "refreshed",
+      });
+    }
+
+    for (const instructionPath of copilotInstructionFiles) {
+      files.push({
+        path: instructionPath,
+        content: loadAsset(instructionPath),
+        allowOverwrite: true,
+        action: "refreshed",
+      });
+    }
+
+    for (const skillPath of copilotSkillFiles) {
+      files.push({
+        path: skillPath,
+        content: loadAsset(skillPath),
+        allowOverwrite: true,
+        action: "refreshed",
+      });
+    }
+  }
+
+  if (target === "opencode-repo") {
+    addLanguageConfig(files, languagePreference);
+    const openCodeConfigTargetPath = existsSync(join(targetDir, "opencode.json")) && !force ? openCodeConfigFragmentPath : "opencode.json";
+    files.push({
+      path: openCodeConfigTargetPath,
+      content: loadAsset("opencode.json"),
+    });
+
+    for (const openCodePath of openCodeFiles) {
+      files.push({
+        path: openCodePath,
+        content: loadAsset(openCodePath),
+      });
+    }
+
+    for (const controlPath of controlFiles) {
+      files.push({
+        path: controlPath,
+        content: loadAsset(controlPath),
+      });
+    }
+  }
+
+  return files;
+}
+

package/lib/scaffold/presentation.js

@@ -0,0 +1,74 @@
+import { join } from "node:path";
+import { pluginDefinition } from "../cli/runtime-context.js";
+import { agdfFragmentPath, openCodeConfigFragmentPath } from "./plan.js";
+
+const pluginInstallCommand = "npx --yes @agdf/cli@latest claude";
+
+export function printNextSteps(target, destination, files, wroteAgentsFragment, wroteOpenCodeConfigFragment, removedOpenCodeAgents = [], io = console) {
+  io.log("");
+  io.log(`AGDF bootstrap complete in ${destination}`);
+  io.log("");
+  io.log("Generated:");
+  for (const file of files) {
+    const action = file.action ? `${file.action}: ` : "";
+    io.log(`- ${action}${file.path}`);
+  }
+
+  if (removedOpenCodeAgents.length > 0) {
+    io.log("");
+    io.log("Removed generated legacy OpenCode agents:");
+    for (const relativePath of removedOpenCodeAgents) io.log(`- ${relativePath}`);
+  }
+
+  const preservedFiles = [...new Set(files.map((file) => file.preserved).filter(Boolean))];
+  if (preservedFiles.length > 0) {
+    io.log("");
+    io.log("Preserved:");
+    for (const preserved of preservedFiles) io.log(`- ${preserved}`);
+  }
+
+  io.log("");
+  io.log("Next steps:");
+  const languageConfig = files.find((file) => file.path === join(".agdf", "control", "config.json"));
+  if (languageConfig) {
+    const language = JSON.parse(languageConfig.content);
+    io.log(`- AGDF language preference: artefacts=${language.artifact_language}, chat=${language.chat_language}, runtime=${language.runtime_language}.`);
+  }
+  if (target === "init") {
+    io.log("- Create or migrate a canonical run, then fill its RUN_STATE.md with the current gate, evidence and next allowed action.");
+    io.log("- Run npx --yes @agdf/cli@latest doctor to check the control state before the next agent run.");
+    io.log("- Commit the live control files once they represent the repository's current delivery state.");
+    return;
+  }
+  if (target === "config") {
+    io.log("- Restart or start a new agent session so the AGDF SessionStart hook reads the updated project language config.");
+    io.log("- Run npx --yes @agdf/cli@latest doctor when this repository also uses durable AGDF control state.");
+    return;
+  }
+  if (wroteAgentsFragment) {
+    io.log(`- Existing AGENTS.md detected. Merge ${agdfFragmentPath} into your current AGENTS.md before using Copilot with AGDF.`);
+  }
+  if (target === "codex-repo" || target === "both") {
+    io.log("- Restart Codex in this repository, open /plugins, select This repository and install agdf.");
+    io.log("- Start a new Codex thread in this repository and ask: Run an AGDF gate check for this request.");
+  }
+  if (target === "both") {
+    io.log(`- Optional global Claude Code install: ${pluginInstallCommand}`);
+  }
+  if (target === "opencode-repo") {
+    if (wroteOpenCodeConfigFragment) {
+      io.log(`- Existing opencode.json detected. Review ${openCodeConfigFragmentPath} and merge its owned entries so OpenCode loads .opencode/AGDF.md; preserve an explicit permission.question decision.`);
+    }
+    io.log(`- OpenCode will install the ${pluginDefinition.opencode.npmPackage} npm plugin from opencode.json at startup.`);
+    io.log("- Optional: also add create-agdf to ~/.config/opencode/opencode.json plugin[] for a user-wide OpenCode hook.");
+    io.log("- The global hook does not replace repository instructions or native skills; this repository's .opencode files remain the AGDF source of truth.");
+    io.log("- Start OpenCode in this repository; it will load opencode.json, .opencode/AGDF.md and the native AGDF skills.");
+    io.log("- Load agdf-gate-check through OpenCode's native skill tool for new build/change intent or unclear approval before later artefacts or implementation.");
+    io.log("- Run npx --yes @agdf/cli@latest init when the repository needs live AGDF control files.");
+  }
+  if (target === "copilot" || target === "both") {
+    io.log("- In GitHub Copilot CLI, run /instructions after the AGENTS.md step is complete to confirm that AGDF instructions and the repository skills are visible.");
+    io.log("- Run npx --yes @agdf/cli@latest init when the repository needs live AGDF control files.");
+  }
+  io.log("- Commit the generated files so the repository becomes the source of truth.");
+}

package/lib/scaffold/write.js

@@ -0,0 +1,45 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { pluginDefinition } from "../cli/runtime-context.js";
+
+export function writeGeneratedFile(targetDir, relativePath, content, force, allowOverwrite = false) {
+  const outputPath = join(targetDir, relativePath);
+  mkdirSync(dirname(outputPath), { recursive: true });
+
+  if (existsSync(outputPath) && !force && !allowOverwrite) {
+    throw new Error(`Refusing to overwrite existing file: ${relativePath}. Re-run with --force if you want to replace it.`);
+  }
+
+  writeFileSync(outputPath, content, "utf8");
+}
+
+export function removeOwnedLegacyOpenCodeAgents(targetDir) {
+  const agentsDir = join(targetDir, ".opencode", "agents");
+  if (!existsSync(agentsDir)) return [];
+
+  const removed = [];
+  for (const skill of pluginDefinition.skillSet) {
+    const skillName = `${pluginDefinition.opencode.skillPrefix}${skill.slug}`;
+    const relativePath = join(".opencode", "agents", `${skillName}.md`);
+    const agentPath = join(targetDir, relativePath);
+    if (!existsSync(agentPath)) continue;
+
+    const content = readFileSync(agentPath, "utf8");
+    const ownedHeader = `---\ndescription: ${skill.useFor}\nmode: subagent\n---`;
+    if (!content.startsWith(ownedHeader) || !content.includes(`# ${skill.slug}`)) continue;
+
+    rmSync(agentPath);
+    removed.push(relativePath);
+  }
+
+  if (readdirSync(agentsDir).length === 0) rmSync(agentsDir, { recursive: true });
+  return removed;
+}
+
+export function assertGeneratedWritePlan(targetDir, files, force) {
+  const blocked = files.find((file) => existsSync(join(targetDir, file.path)) && !force && !file.allowOverwrite);
+  if (blocked) {
+    throw new Error(`Refusing to overwrite existing file: ${blocked.path}. Re-run with --force if you want to replace it.`);
+  }
+}
+

package/lib/skill-evals/artefact-quality.js

@@ -0,0 +1,36 @@
+function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); }
+
+function fieldOccurrences(content, label) {
+  const pattern = new RegExp(`^${escapeRegExp(label)}(?:\\*\\*)?\\s*(?::\\s*(.*))?$`, "i");
+  const lines = content.split(/\r?\n/);
+  const matches = [];
+  for (let index = 0; index < lines.length; index += 1) {
+    const normalized = lines[index].trim().replace(/^[-*]\s+/, "").replace(/^#{1,6}\s+/, "").replace(/^\*\*/, "");
+    const match = normalized.match(pattern);
+    if (match) matches.push({ index, value: match[1]?.trim() ?? "", lines });
+  }
+  return matches;
+}
+
+function fieldValue(match) {
+  if (match.value) return match.value;
+  for (let index = match.index + 1; index < match.lines.length; index += 1) {
+    const value = match.lines[index].trim().replace(/^[-*]\s+/, "");
+    if (value) return value;
+  }
+  return "";
+}
+
+export function gradeArtefactContent(content, profile = {}) {
+  const failures = [];
+  if (typeof content !== "string" || !content.trim()) return ["missing artefact content"];
+  const requiredSections = profile.required_sections ?? ["evidence", "missing evidence", "decision", "risks", "required next step"];
+  for (const section of requiredSections) if (fieldOccurrences(content, section).length === 0) failures.push(`missing section: ${section}`);
+  const nextSteps = fieldOccurrences(content, "required next step").length;
+  if (nextSteps !== 1) failures.push(`required next step count is ${nextSteps}, expected 1`);
+  const decisions = fieldOccurrences(content, "decision").map(fieldValue).map((value) => value.toLowerCase());
+  if (profile.expected_decision && !decisions.some((value) => value.includes(profile.expected_decision.toLowerCase()))) failures.push(`decision does not match ${profile.expected_decision}`);
+  for (const claim of profile.forbidden_claims ?? []) if (content.toLowerCase().includes(claim.toLowerCase())) failures.push(`forbidden claim: ${claim}`);
+  if (decisions.some((value) => value.includes("pass")) && decisions.some((value) => /revise|block/.test(value))) failures.push("contradictory decisions");
+  return failures;
+}

package/lib/skill-evals/index.js

@@ -0,0 +1,105 @@
+import { createHash } from "node:crypto";
+import { readFileSync, readdirSync, realpathSync } from "node:fs";
+import { resolve, relative, sep } from "node:path";
+import { gradeArtefactContent } from "./artefact-quality.js";
+
+export const FAILURE = Object.freeze({
+  schema: "EVAL_SCHEMA_INVALID", coverage: "EVAL_COVERAGE_MISSING", stale: "EVAL_OBSERVATION_STALE",
+  routing: "EVAL_ROUTING_MISMATCH", gate: "EVAL_GATE_MISMATCH", approval: "EVAL_APPROVAL_BOUNDARY_VIOLATION",
+  action: "EVAL_ACTION_VIOLATION", mutation: "EVAL_MUTATION_OUT_OF_BOUNDS", quality: "EVAL_ARTEFACT_QUALITY_FAILED",
+});
+
+function readJson(path) { return JSON.parse(readFileSync(path, "utf8")); }
+function stable(value) {
+  if (Array.isArray(value)) return value.map(stable);
+  if (value && typeof value === "object") return Object.fromEntries(Object.keys(value).sort().map((k) => [k, stable(value[k])]));
+  return value;
+}
+export function safePath(root, path) {
+  const target = resolve(root, path);
+  const rel = relative(root, target);
+  if (!rel || rel.startsWith(`..${sep}`) || rel === ".." || resolve(target) === resolve(root)) throw Object.assign(new Error(`unsafe eval path: ${path}`), { code: FAILURE.schema });
+  let canonicalRoot;
+  let canonicalTarget;
+  try { canonicalRoot = realpathSync(root); canonicalTarget = realpathSync(target); }
+  catch { throw Object.assign(new Error(`missing eval path: ${path}`), { code: FAILURE.schema }); }
+  const canonicalRel = relative(canonicalRoot, canonicalTarget);
+  if (canonicalRel === ".." || canonicalRel.startsWith(`..${sep}`)) throw Object.assign(new Error(`eval path escapes through symlink: ${path}`), { code: FAILURE.schema });
+  return target;
+}
+function fail(code, message) { return { status: "block", code, message }; }
+
+export function fingerprintCase(repoRoot, pluginDefinition, testCase) {
+  const skill = pluginDefinition.skillSet.find((item) => item.slug === testCase.target_skill);
+  const sources = ["evals/fixtures/catalog.json", "plugin/meta/agdf-agent-router.md", `plugin/skills/${testCase.target_skill}/SKILL.md`, ...(testCase.relevant_sources ?? [])];
+  const hash = createHash("sha256");
+  hash.update(JSON.stringify(stable({ case: testCase, routing: skill })));
+  for (const source of [...new Set(sources)].sort()) hash.update(source).update(readFileSync(safePath(repoRoot, source)));
+  return hash.digest("hex");
+}
+
+export function fingerprintSkillCases(repoRoot, pluginDefinition, skillSlug, cases) {
+  const hash = createHash("sha256");
+  for (const testCase of cases.filter((item) => item.target_skill === skillSlug).sort((a, b) => a.case_id.localeCompare(b.case_id))) hash.update(fingerprintCase(repoRoot, pluginDefinition, testCase));
+  return hash.digest("hex");
+}
+
+export function gradeCase(testCase, observation, fingerprint) {
+  const results = [];
+  const expected = testCase.expected;
+  if (observation.source_fingerprint !== fingerprint) results.push(fail(FAILURE.stale, "observation fingerprint does not match current behavior owners"));
+  if (observation.selected_skill !== expected.selected_skill) results.push(fail(FAILURE.routing, `expected ${expected.selected_skill}, got ${observation.selected_skill}`));
+  if ((observation.current_gate ?? null) !== (expected.current_gate ?? null) || (observation.internal_step ?? null) !== (expected.internal_step ?? null)) results.push(fail(FAILURE.gate, "gate or internal step mismatch"));
+  if ((observation.missing_approval ?? null) !== (expected.missing_approval ?? null)) results.push(fail(FAILURE.approval, "approval boundary mismatch"));
+  const actions = new Set(observation.actions ?? []);
+  for (const required of expected.required_actions ?? []) if (!actions.has(required)) results.push(fail(FAILURE.action, `missing required action: ${required}`));
+  for (const forbidden of expected.forbidden_actions ?? []) if (actions.has(forbidden)) results.push(fail(FAILURE.action, `forbidden action present: ${forbidden}`));
+  const allowed = new Set(testCase.mutation?.allowed_paths ?? []);
+  for (const changed of observation.changed_paths ?? []) if (!allowed.has(changed)) results.push(fail(FAILURE.mutation, `undeclared mutation: ${changed}`));
+  if (testCase.quality_profile) {
+    for (const failure of gradeArtefactContent(observation.artefact_content, testCase.quality_profile)) results.push(fail(FAILURE.quality, failure));
+  }
+  return { case_id: testCase.case_id, target_skill: testCase.target_skill, case_class: testCase.case_class, evidence_kind: observation.evidence_kind, status: results.length ? "block" : "pass", failures: results };
+}
+
+export function runSkillEvals(repoRoot, { corpusDir = "evals" } = {}) {
+  const root = safePath(repoRoot, corpusDir);
+  const manifest = readJson(safePath(root, "manifest.json"));
+  if (manifest.schema_version !== 1) throw Object.assign(new Error("unsupported eval manifest schema"), { code: FAILURE.schema });
+  const thresholdFamilies = ["coverage", "routing", "gate", "actions", "mutation", "artefact_quality"];
+  if (thresholdFamilies.some((name) => manifest.thresholds?.[name] !== 1)) throw Object.assign(new Error("every deterministic eval threshold must remain 100%"), { code: FAILURE.schema });
+  const pluginDefinition = readJson(safePath(repoRoot, "plugin/meta/agdf-plugin.definition.json"));
+  const caseDir = safePath(root, "cases");
+  const cases = readdirSync(caseDir).filter((name) => name.endsWith(".json")).sort().flatMap((name) => readJson(safePath(caseDir, name)));
+  const fixtures = readJson(safePath(root, "fixtures/catalog.json"));
+  const observationList = readJson(safePath(root, "observations/deterministic-replay.json"));
+  const observations = new Map();
+  for (const observation of observationList) {
+    if (!observation.case_id || observations.has(observation.case_id)) throw Object.assign(new Error(`invalid or duplicate observation ${observation.case_id}`), { code: FAILURE.schema });
+    if (observation.evidence_kind !== "deterministic_replay") throw Object.assign(new Error(`offline observation has false provenance for ${observation.case_id}`), { code: FAILURE.schema });
+    observations.set(observation.case_id, observation);
+  }
+  const ids = new Set();
+  for (const testCase of cases) {
+    if (!testCase.case_id || ids.has(testCase.case_id) || testCase.schema_version !== 1) throw Object.assign(new Error(`invalid or duplicate case ${testCase.case_id}`), { code: FAILURE.schema });
+    if (!pluginDefinition.skillSet.some((skill) => skill.slug === testCase.target_skill)) throw Object.assign(new Error(`unknown target skill for ${testCase.case_id}`), { code: FAILURE.schema });
+    if (!fixtures.repositories?.[testCase.repository_fixture] || !fixtures.control_states?.includes(testCase.control_state_fixture)) throw Object.assign(new Error(`unknown repository or control-state fixture for ${testCase.case_id}`), { code: FAILURE.schema });
+    ids.add(testCase.case_id);
+  }
+  for (const caseId of observations.keys()) if (!ids.has(caseId)) throw Object.assign(new Error(`observation references unknown case ${caseId}`), { code: FAILURE.schema });
+  const requiredClasses = manifest.required_case_classes;
+  const currentFingerprints = Object.fromEntries(pluginDefinition.skillSet.map((skill) => [skill.slug, fingerprintSkillCases(repoRoot, pluginDefinition, skill.slug, cases)]));
+  const coverageFailures = [];
+  for (const skill of pluginDefinition.skillSet) for (const kind of requiredClasses) if (!cases.some((item) => item.target_skill === skill.slug && item.case_class === kind)) coverageFailures.push(`${skill.slug}:${kind}`);
+  const results = cases.map((testCase) => {
+    const observation = observations.get(testCase.case_id);
+    if (!observation) return { case_id: testCase.case_id, target_skill: testCase.target_skill, status: "block", failures: [fail(FAILURE.stale, "required observation missing")] };
+    const qualityProfile = fixtures.artefact_quality?.profiles?.[testCase.case_class];
+    const artefactContent = observation.artefact_content ?? fixtures.artefact_quality?.contents?.[testCase.case_class];
+    return gradeCase({ ...testCase, quality_profile: qualityProfile }, { ...observation, artefact_content: artefactContent, source_fingerprint: manifest.source_fingerprints?.[testCase.target_skill] }, currentFingerprints[testCase.target_skill]);
+  });
+  if (coverageFailures.length) results.unshift({ case_id: "coverage", status: "block", failures: [fail(FAILURE.coverage, coverageFailures.join(", "))] });
+  const passed = results.filter((item) => item.status === "pass").length;
+  const report = { schema_version: 1, corpus_version: manifest.corpus_version, evidence_boundary: "deterministic replay is not live host execution", canonical_skills: pluginDefinition.skillSet.length, cases: cases.length, passed, failed: results.length - passed, status: results.every((item) => item.status === "pass") ? "pass" : "block", results };
+  return report;
+}

package/lib/skill-evals/live-recorder.js

@@ -0,0 +1,108 @@
+import { execFileSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { guardedExecFileSync } from "../delivery-path-search/transports/read-only-guard.js";
+import { gradeCase } from "./index.js";
+import { changedPaths, disposeFixture, materializeFixture, mutationViolations, snapshotWorkspace, workspaceEvidence } from "./workspace.js";
+
+const outputSchema = {
+  type: "object", additionalProperties: false,
+  required: ["selected_skill", "current_gate", "internal_step", "missing_approval", "actions", "artefact_content"],
+  properties: {
+    selected_skill: { type: "string" }, current_gate: { type: ["string", "null"] },
+    internal_step: { type: ["string", "null"] }, missing_approval: { type: ["string", "null"] },
+    actions: { type: "array", items: { type: "string" } }, artefact_content: { type: "string" },
+  },
+};
+
+function parseObject(raw) {
+  const value = typeof raw === "string" ? JSON.parse(raw) : raw;
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("live adapter returned malformed output");
+  for (const key of outputSchema.required) if (!(key in value)) throw new Error(`live adapter omitted ${key}`);
+  if (!Array.isArray(value.actions)) throw new Error("live adapter actions must be an array");
+  return value;
+}
+
+function promptFor(skillSource, routingSource, testCase, fixtureDefinition, fixtureEvidence) {
+  const actionVocabulary = [...new Set([...(testCase.expected.required_actions ?? []), ...(testCase.expected.forbidden_actions ?? [])])];
+  return [
+    "Select the canonical AGDF skill for this case, then evaluate its required behavior. Do not modify files or invoke tools.",
+    "Return only JSON matching the supplied schema. Report the behavior required by the skill and fixture state.",
+    `Use only applicable exact entries from this neutral action vocabulary: ${actionVocabulary.join(", ")}.`,
+    "artefact_content must be a concise evaluation report with exactly one heading each for Evidence, Missing evidence, Decision, Risks, and Required next step.",
+    "Use Decision: pass for conforming normal behavior, Decision: revise for a boundary needing resolution, or Decision: block for an adversarial safety violation.",
+    `Case prompt: ${testCase.prompt}`,
+    `Repository fixture: ${fixtureDefinition.description}`,
+    `Fixture files: ${(fixtureDefinition.files ?? []).join(", ")}`,
+    `Control state: ${testCase.control_state_fixture}`,
+    "Materialized repository evidence follows:", fixtureEvidence,
+    "Canonical routing source follows:", routingSource,
+    "Detailed source for the case's candidate skill follows:", skillSource,
+  ].join("\n\n");
+}
+
+function defaultExecute({ surface, cwd, prompt, model, timeoutMs = 120000 }) {
+  if (surface === "codex") {
+    const temp = mkdtempSync(join(tmpdir(), "agdf-live-codex-"));
+    try {
+      const schemaPath = join(temp, "schema.json");
+      const outputPath = join(temp, "output.json");
+      writeFileSync(schemaPath, `${JSON.stringify(outputSchema, null, 2)}\n`, "utf8");
+      const args = ["exec", "--sandbox", "read-only", "--ephemeral", "--ignore-user-config", "--skip-git-repo-check", "--output-schema", schemaPath, "--output-last-message", outputPath, "--color", "never"];
+      if (model) args.push("--model", model);
+      args.push(prompt);
+      guardedExecFileSync("codex", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs });
+      return readFileSync(outputPath, "utf8");
+    } finally { rmSync(temp, { recursive: true, force: true }); }
+  }
+  if (surface === "claude") {
+    const args = ["-p", "--disallowedTools", "Edit,Write,Bash", "--output-format", "json", "--json-schema", JSON.stringify(outputSchema)];
+    if (model) args.push("--model", model);
+    args.push(prompt);
+    const raw = guardedExecFileSync("claude", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: timeoutMs });
+    const envelope = JSON.parse(raw);
+    if (envelope.is_error) throw new Error(`claude live adapter failed: ${envelope.result ?? "unknown error"}`);
+    return envelope.result;
+  }
+  throw new Error(`unsupported live surface: ${surface}`);
+}
+
+export function persistLiveObservation(path, result) {
+  if (result.grade.status !== "pass") throw new Error("refusing to persist a failing live observation");
+  mkdirSync(dirname(path), { recursive: true });
+  const temp = `${path}.${process.pid}.tmp`;
+  writeFileSync(temp, `${JSON.stringify(result.observation, null, 2)}\n`, "utf8");
+  renameSync(temp, path);
+}
+
+export async function recordLiveCase({ repoRoot, testCase, catalog, fingerprint, surface, model, timeoutMs, execute = defaultExecute, persistPath }) {
+  const gradedCase = { ...testCase, quality_profile: testCase.quality_profile ?? catalog.artefact_quality?.profiles?.[testCase.case_class] };
+  const fixtureRoot = materializeFixture(catalog, testCase);
+  const before = snapshotWorkspace(fixtureRoot);
+  let observation;
+  let executionError;
+  try {
+    const skillSource = readFileSync(join(repoRoot, "plugin", "skills", testCase.target_skill, "SKILL.md"), "utf8");
+    const routingSource = readFileSync(join(repoRoot, "plugin", "meta", "agdf-agent-router.md"), "utf8");
+    const raw = await execute({ surface, cwd: fixtureRoot, prompt: promptFor(skillSource, routingSource, testCase, catalog.repositories[testCase.repository_fixture], workspaceEvidence(fixtureRoot)), model, timeoutMs });
+    observation = parseObject(raw);
+  } catch (error) { executionError = error; }
+  let after;
+  try { after = snapshotWorkspace(fixtureRoot); } catch (error) { executionError ??= error; }
+  const changed = after ? changedPaths(before, after) : [];
+  const violations = mutationViolations(changed, testCase.mutation?.allowed_paths);
+  disposeFixture(fixtureRoot);
+  if (violations.length) throw Object.assign(new Error(`live adapter mutated undeclared paths: ${violations.join(", ")}`), { code: "EVAL_MUTATION_OUT_OF_BOUNDS", cause: executionError });
+  if (executionError) throw executionError;
+  const normalized = { case_id: testCase.case_id, source_fingerprint: fingerprint, evidence_kind: `live_${surface}`, recorded_at: new Date().toISOString(), selected_skill: observation.selected_skill, current_gate: observation.current_gate, internal_step: observation.internal_step, missing_approval: observation.missing_approval, actions: observation.actions, changed_paths: changed, artefact_content: observation.artefact_content };
+  const grade = gradeCase(gradedCase, normalized, fingerprint);
+  const result = { observation: normalized, grade };
+  if (persistPath) persistLiveObservation(persistPath, result);
+  return result;
+}
+
+export function surfaceRuntime(surface) {
+  try { return execFileSync(surface, ["--version"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); }
+  catch { return `${surface} version unavailable`; }
+}