codeharness 0.8.0 → 0.9.0

package/dist/index.js

@@ -919,6 +919,14 @@ function storyVerificationPatch() {
 - [ ] All acceptance criteria verified with real-world evidence
 - [ ] Test coverage meets target (100%)
 
+### Verification Tags
+
+For each AC, append a verification tag to indicate how it can be verified:
+- \`<!-- verification: cli-verifiable -->\` \u2014 AC can be verified by running CLI commands in a subprocess
+- \`<!-- verification: integration-required -->\` \u2014 AC requires integration testing, multi-system interaction, or manual verification
+
+ACs referencing workflows, sprint planning, user sessions, or external system interactions should be tagged as \`integration-required\`. If no tag is present, a heuristic classifier will attempt to determine verifiability at runtime.
+
 ## Documentation Requirements
 
 - [ ] Relevant AGENTS.md files updated (list modules touched)
@@ -1340,7 +1348,7 @@ function importStoriesToBeads(stories, opts, beadsFns) {
 }
 
 // src/commands/init.ts
-var HARNESS_VERSION = true ? "0.8.0" : "0.0.0-dev";
+var HARNESS_VERSION = true ? "0.9.0" : "0.0.0-dev";
 function getStackLabel(stack) {
   if (stack === "nodejs") return "Node.js (package.json)";
   if (stack === "python") return "Python";
@@ -2573,6 +2581,29 @@ var DB_KEYWORDS = [
   "sql",
   "table"
 ];
+var INTEGRATION_KEYWORDS = [
+  "sprint planning",
+  "workflow",
+  "run /command",
+  "user session",
+  "multi-step",
+  "external system",
+  "real infrastructure",
+  "integration test",
+  "manual verification"
+];
+function classifyVerifiability(description) {
+  const lower = description.toLowerCase();
+  for (const kw of INTEGRATION_KEYWORDS) {
+    if (lower.includes(kw)) return "integration-required";
+  }
+  return "cli-verifiable";
+}
+var VERIFICATION_TAG_PATTERN = /<!--\s*verification:\s*(cli-verifiable|integration-required)\s*-->/;
+function parseVerificationTag(text) {
+  const match = VERIFICATION_TAG_PATTERN.exec(text);
+  return match ? match[1] : null;
+}
 function classifyAC(description) {
   const lower = description.toLowerCase();
   for (const kw of UI_KEYWORDS) {
@@ -2622,10 +2653,13 @@ function parseStoryACs(storyFilePath) {
     if (currentId !== null && currentDesc.length > 0) {
       const description = currentDesc.join(" ").trim();
       if (description) {
+        const tag = parseVerificationTag(description);
+        const verifiability = tag ?? classifyVerifiability(description);
         acs.push({
           id: currentId,
           description,
-          type: classifyAC(description)
+          type: classifyAC(description),
+          verifiability
         });
       } else {
         warn(`Skipping malformed AC #${currentId}: empty description`);
@@ -2764,6 +2798,62 @@ function getNewestSourceMtime(dir) {
   walk(dir);
   return newest;
 }
+function getSourceFilesInModule(modulePath) {
+  const files = [];
+  function walk(current) {
+    let entries;
+    try {
+      entries = readdirSync2(current);
+    } catch {
+      return;
+    }
+    const dirName = current.split("/").pop() ?? "";
+    if (dirName === "node_modules" || dirName === ".git" || dirName === "__tests__" || dirName === "dist" || dirName === "coverage" || dirName.startsWith(".") && current !== modulePath) return;
+    for (const entry of entries) {
+      const fullPath = join9(current, entry);
+      let stat;
+      try {
+        stat = statSync(fullPath);
+      } catch {
+        continue;
+      }
+      if (stat.isDirectory()) {
+        walk(fullPath);
+      } else if (stat.isFile()) {
+        const ext = getExtension(entry);
+        if (SOURCE_EXTENSIONS.has(ext) && !isTestFile(entry)) {
+          files.push(entry);
+        }
+      }
+    }
+  }
+  walk(modulePath);
+  return files;
+}
+function getMentionedFilesInAgentsMd(agentsPath) {
+  if (!existsSync11(agentsPath)) return [];
+  const content = readFileSync9(agentsPath, "utf-8");
+  const mentioned = /* @__PURE__ */ new Set();
+  const filenamePattern = /[\w./-]*[\w-]+\.(?:ts|js|py)\b/g;
+  let match;
+  while ((match = filenamePattern.exec(content)) !== null) {
+    const fullMatch = match[0];
+    const basename3 = fullMatch.split("/").pop();
+    if (!isTestFile(basename3)) {
+      mentioned.add(basename3);
+    }
+  }
+  return Array.from(mentioned);
+}
+function checkAgentsMdCompleteness(agentsPath, modulePath) {
+  const sourceFiles = getSourceFilesInModule(modulePath);
+  const mentionedFiles = new Set(getMentionedFilesInAgentsMd(agentsPath));
+  const missing = sourceFiles.filter((f) => !mentionedFiles.has(f));
+  return {
+    complete: missing.length === 0,
+    missing
+  };
+}
 function checkAgentsMdForModule(modulePath, dir) {
   const root = dir ?? process.cwd();
   const fullModulePath = join9(root, modulePath);
@@ -2782,13 +2872,15 @@ function checkAgentsMdForModule(modulePath, dir) {
   }
   const docMtime = statSync(agentsPath).mtime;
   const codeMtime = getNewestSourceMtime(fullModulePath);
-  if (codeMtime !== null && codeMtime.getTime() > docMtime.getTime()) {
+  const { complete, missing } = checkAgentsMdCompleteness(agentsPath, fullModulePath);
+  if (!complete) {
+    const missingList = missing.join(", ");
     return {
       path: relative(root, agentsPath),
       grade: "stale",
       lastModified: docMtime,
       codeLastModified: codeMtime,
-      reason: `AGENTS.md stale for module: ${modulePath}`
+      reason: `AGENTS.md stale for module: ${modulePath} \u2014 missing: ${missingList}`
     };
   }
   return {
@@ -2818,22 +2910,30 @@ function scanDocHealth(dir) {
   if (existsSync11(rootAgentsPath)) {
     if (modules.length > 0) {
       const docMtime = statSync(rootAgentsPath).mtime;
-      let newestCode = null;
+      let allMissing = [];
       let staleModule = "";
+      let newestCode = null;
       for (const mod of modules) {
-        const modMtime = getNewestSourceMtime(join9(root, mod));
+        const fullModPath = join9(root, mod);
+        const modAgentsPath = join9(fullModPath, "AGENTS.md");
+        if (existsSync11(modAgentsPath)) continue;
+        const { missing } = checkAgentsMdCompleteness(rootAgentsPath, fullModPath);
+        if (missing.length > 0 && staleModule === "") {
+          staleModule = mod;
+          allMissing = missing;
+        }
+        const modMtime = getNewestSourceMtime(fullModPath);
         if (modMtime !== null && (newestCode === null || modMtime.getTime() > newestCode.getTime())) {
           newestCode = modMtime;
-          staleModule = mod;
         }
       }
-      if (newestCode !== null && newestCode.getTime() > docMtime.getTime()) {
+      if (allMissing.length > 0) {
         documents.push({
           path: "AGENTS.md",
           grade: "stale",
           lastModified: docMtime,
           codeLastModified: newestCode,
-          reason: `AGENTS.md stale for module: ${staleModule}`
+          reason: `AGENTS.md stale for module: ${staleModule} \u2014 missing: ${allMissing.join(", ")}`
         });
       } else {
         documents.push({
@@ -3189,10 +3289,44 @@ function runShowboatVerify(proofPath) {
     return { passed: false, output: stdout || stderr || message };
   }
 }
-function proofHasContent(proofPath) {
-  if (!existsSync12(proofPath)) return false;
+function validateProofQuality(proofPath) {
+  if (!existsSync12(proofPath)) {
+    return { verified: 0, pending: 0, escalated: 0, total: 0, passed: false };
+  }
   const content = readFileSync10(proofPath, "utf-8");
-  return content.includes("<!-- /showboat exec -->") || content.includes("<!-- showboat image:");
+  const acHeaderPattern = /^## AC \d+:/gm;
+  const matches = [...content.matchAll(acHeaderPattern)];
+  if (matches.length === 0) {
+    return { verified: 0, pending: 0, escalated: 0, total: 0, passed: false };
+  }
+  let verified = 0;
+  let pending = 0;
+  let escalated = 0;
+  for (let i = 0; i < matches.length; i++) {
+    const start = matches[i].index;
+    const end = i + 1 < matches.length ? matches[i + 1].index : content.length;
+    const section = content.slice(start, end);
+    if (section.includes("[ESCALATE]")) {
+      escalated++;
+      continue;
+    }
+    const hasEvidence = section.includes("<!-- /showboat exec -->") || section.includes("<!-- showboat image:") || /```(?:bash|shell)\n[\s\S]*?```\n+```output\n/m.test(section);
+    if (hasEvidence) {
+      verified++;
+    } else {
+      pending++;
+    }
+  }
+  const total = verified + pending + escalated;
+  return {
+    verified,
+    pending,
+    escalated,
+    total,
+    // Proof passes when no pending ACs remain and at least one is verified.
+    // Escalated ACs are allowed — they are explicitly acknowledged as unverifiable.
+    passed: pending === 0 && verified > 0
+  };
 }
 function updateVerificationState(storyId, result, dir) {
   const { state, body } = readStateWithBody(dir);
@@ -3333,36 +3467,52 @@ function verifyStory(storyId, isJson, root) {
     return;
   }
   const storyTitle = extractStoryTitle(storyFilePath);
-  const proofPath = createProofDocument(storyId, storyTitle, acs, root);
-  let showboatStatus = "skipped";
-  if (proofHasContent(proofPath)) {
-    const showboatResult = runShowboatVerify(proofPath);
-    if (showboatResult.output === "showboat not available") {
-      showboatStatus = "skipped";
-      warn("Showboat not installed \u2014 skipping re-verification");
+  const expectedProofPath = join11(root, "verification", `${storyId}-proof.md`);
+  const proofPath = existsSync13(expectedProofPath) ? expectedProofPath : createProofDocument(storyId, storyTitle, acs, root);
+  const proofQuality = validateProofQuality(proofPath);
+  if (!proofQuality.passed) {
+    if (isJson) {
+      jsonOutput({
+        status: "fail",
+        message: `Proof quality check failed: ${proofQuality.verified}/${proofQuality.total} ACs verified`,
+        proofQuality: { verified: proofQuality.verified, pending: proofQuality.pending, escalated: proofQuality.escalated, total: proofQuality.total }
+      });
     } else {
-      showboatStatus = showboatResult.passed ? "pass" : "fail";
-      if (!showboatResult.passed) {
-        fail(`Showboat verify failed: ${showboatResult.output}`, { json: isJson });
-        process.exitCode = 1;
-        return;
-      }
+      fail(`Proof quality check failed: ${proofQuality.verified}/${proofQuality.total} ACs verified`);
+    }
+    process.exitCode = 1;
+    return;
+  }
+  if (proofQuality.escalated > 0) {
+    warn(`Story ${storyId} has ${proofQuality.escalated} ACs requiring integration verification`);
+    info("Run these ACs manually or in a dedicated verification session");
+  }
+  let showboatStatus = "skipped";
+  const showboatResult = runShowboatVerify(proofPath);
+  if (showboatResult.output === "showboat not available") {
+    showboatStatus = "skipped";
+    warn("Showboat not installed \u2014 skipping re-verification");
+  } else {
+    showboatStatus = showboatResult.passed ? "pass" : "fail";
+    if (!showboatResult.passed) {
+      fail(`Showboat verify failed: ${showboatResult.output}`, { json: isJson });
+      process.exitCode = 1;
+      return;
     }
   }
-  const acsVerified = showboatStatus === "pass";
-  const verifiedCount = acsVerified ? acs.length : 0;
   const result = {
     storyId,
     success: true,
-    totalACs: acs.length,
-    verifiedCount,
-    failedCount: acs.length - verifiedCount,
+    totalACs: proofQuality.total,
+    verifiedCount: proofQuality.verified,
+    failedCount: proofQuality.pending,
+    escalatedCount: proofQuality.escalated,
     proofPath: `verification/${storyId}-proof.md`,
     showboatVerifyStatus: showboatStatus,
     perAC: acs.map((ac) => ({
       id: ac.id,
       description: ac.description,
-      verified: acsVerified,
+      verified: true,
       evidencePaths: []
     }))
   };
@@ -3394,7 +3544,10 @@ function verifyStory(storyId, isJson, root) {
     warn(`Failed to complete exec-plan: ${message}`);
   }
   if (isJson) {
-    jsonOutput(result);
+    jsonOutput({
+      ...result,
+      proofQuality: { verified: proofQuality.verified, pending: proofQuality.pending, escalated: proofQuality.escalated, total: proofQuality.total }
+    });
   } else {
     ok(`Story ${storyId}: verified \u2014 proof at verification/${storyId}-proof.md`);
   }
@@ -6630,7 +6783,7 @@ function registerGithubImportCommand(program) {
 }
 
 // src/index.ts
-var VERSION = true ? "0.8.0" : "0.0.0-dev";
+var VERSION = true ? "0.9.0" : "0.0.0-dev";
 function createProgram() {
   const program = new Command();
   program.name("codeharness").description("Makes autonomous coding agents produce software that actually works").version(VERSION).option("--json", "Output in machine-readable JSON format");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "codeharness",
-  "version": "0.8.0",
+  "version": "0.9.0",
   "type": "module",
   "description": "CLI for codeharness — makes autonomous coding agents produce software that actually works",
   "bin": {