@agile-vibe-coding/avc 0.2.3 → 0.3.2

package/cli/micro-check-runner.js

@@ -0,0 +1,449 @@
+/**
+ * micro-check-runner.js
+ *
+ * Runs individual micro-checks (Tier 1 and Tier 2) against an LLM provider
+ * to evaluate work item quality.
+ */
+
+const SYSTEM_INSTRUCTIONS =
+  "You are a work item analyzer. Answer the applicability question about whether this check is relevant to the given work item. Return JSON with 'applicable' (boolean) and 'reason' (string).";
+
+const QUALITY_CHECK_INSTRUCTIONS =
+  "You are a work item quality checker. Answer YES (passed: true) or NO (passed: false) to the quality question. Provide brief evidence from the work item text supporting your answer. Return JSON with 'passed' (boolean) and 'evidence' (string).\n\n" +
+  "IMPORTANT calibration rules:\n" +
+  "- Accept SEMANTIC EQUIVALENCE: if the concept is addressed through a described mechanism, even without using the exact terminology the question uses, answer YES. Example: 'SameSite=Strict cookies' addresses CSRF even without the word 'CSRF'.\n" +
+  "- Match strictness to the WORK ITEM LEVEL: Epics are high-level planning artifacts — they describe WHAT and WHY, not HOW. Do not fail an epic for lacking implementation details (specific HTTP methods, error code enumerations, layer-by-layer validation). Stories should be more specific.\n" +
+  "- When in doubt, answer YES if the spirit of the requirement is met, even if the exact wording differs.";
+
+// ---------------------------------------------------------------------------
+// JSON parse helpers with regex fallback
+// ---------------------------------------------------------------------------
+
+/**
+ * Parse an applicability response from the LLM.
+ * Tries JSON.parse first, then regex fallback.
+ * Defaults to applicable = true on total failure (conservative).
+ * @param {string|Object} raw - Raw LLM response
+ * @returns {{ applicable: boolean, reason: string }}
+ */
+function parseApplicabilityResponse(raw) {
+  if (raw && typeof raw === 'object') {
+    return {
+      applicable: Boolean(raw.applicable),
+      reason: raw.reason || '',
+    };
+  }
+
+  const text = String(raw);
+
+  try {
+    const parsed = JSON.parse(text);
+    return {
+      applicable: Boolean(parsed.applicable),
+      reason: parsed.reason || '',
+    };
+  } catch {
+    // Regex fallback
+    const applicableMatch = text.match(/"applicable"\s*:\s*(true|false)/i);
+    if (applicableMatch) {
+      return {
+        applicable: applicableMatch[1].toLowerCase() === 'true',
+        reason: '',
+      };
+    }
+
+    // Total failure — default to not-applicable (skip) to avoid phantom failures
+    // from unparseable LLM responses inflating failure counts
+    return { applicable: false, reason: 'Failed to parse LLM response, defaulting to skip' };
+  }
+}
+
+/**
+ * Parse a quality check response from the LLM.
+ * Tries JSON.parse first, then regex fallback.
+ * Defaults to passed = false on total failure (conservative).
+ * @param {string|Object} raw - Raw LLM response
+ * @returns {{ passed: boolean, evidence: string }}
+ */
+function parseCheckResponse(raw) {
+  if (raw && typeof raw === 'object') {
+    return {
+      passed: Boolean(raw.passed),
+      evidence: raw.evidence || '',
+    };
+  }
+
+  const text = String(raw);
+
+  try {
+    const parsed = JSON.parse(text);
+    return {
+      passed: Boolean(parsed.passed),
+      evidence: parsed.evidence || '',
+    };
+  } catch {
+    // Regex fallback
+    const passedMatch = text.match(/"passed"\s*:\s*(true|false)/i);
+    const evidenceMatch = text.match(/"evidence"\s*:\s*"([^"]*)"/);
+
+    if (passedMatch) {
+      return {
+        passed: passedMatch[1].toLowerCase() === 'true',
+        evidence: evidenceMatch ? evidenceMatch[1] : '',
+      };
+    }
+
+    // Total failure — mark as indeterminate (null) so scorer can exclude it
+    return { passed: null, evidence: 'Failed to parse LLM response' };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Prompt builders
+// ---------------------------------------------------------------------------
+
+function buildApplicabilityPrompt(workItemText, applicabilityQuestion) {
+  return (
+    `Given the following work item:\n\n${workItemText}\n\n` +
+    `Question: ${applicabilityQuestion}\n\n` +
+    `Respond with JSON: {"applicable": true/false, "reason": "brief reason"}`
+  );
+}
+
+function buildQualityCheckPrompt(workItemText, question, workItemType) {
+  const levelHint = workItemType === 'epic'
+    ? 'This is an EPIC (high-level planning document). Evaluate at epic-level granularity — do not require implementation details.\n\n'
+    : 'This is a STORY (implementation-level specification). Evaluate at story-level granularity — expect specific details.\n\n';
+  return (
+    levelHint +
+    `Given the following work item:\n\n${workItemText}\n\n` +
+    `Question: ${question}\n\n` +
+    `Respond with JSON: {"passed": true/false, "evidence": "brief quote or explanation supporting your answer"}`
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Template variable resolution (Tier 2)
+// ---------------------------------------------------------------------------
+
+/**
+ * Resolve template variables like {{checkId.evidence}} in a question string.
+ * @param {string} question - The question template
+ * @param {Map} tier1Results - Map of checkId -> tier1Result
+ * @returns {string} The resolved question
+ */
+function resolveTemplateVariables(question, tier1Results) {
+  return question.replace(/\{\{([\w-]+)\.evidence\}\}/g, (_match, checkId) => {
+    const result = tier1Results.get(checkId);
+    if (result && result.evidence) {
+      return result.evidence;
+    }
+    return '(not available)';
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Exported check runners
+// ---------------------------------------------------------------------------
+
+/**
+ * Run a single Tier 1 check against a work item.
+ * @param {Object} check - Check definition from catalog JSON
+ * @param {string} workItemText - The full work item text (epic/story context markdown)
+ * @param {Object} llmProvider - LLM provider instance with generateJSON() method
+ * @param {string} [workItemType] - "epic" or "story" (falls back to check ID heuristic)
+ * @returns {Object} { id, tier, severity, category, perspective, universal, applicable, passed, evidence }
+ */
+export async function runTier1Check(check, workItemText, llmProvider, workItemType) {
+  const baseResult = {
+    id: check.id,
+    tier: 1,
+    severity: check.severity,
+    category: check.category,
+    perspective: check.perspective,
+    universal: check.universal,
+  };
+
+  // Step 1: Applicability gate
+  if (check.universal !== true) {
+    const applicabilityPrompt = buildApplicabilityPrompt(workItemText, check.applicabilityQuestion);
+    const applicabilityRaw = await llmProvider.generateJSON(applicabilityPrompt, SYSTEM_INSTRUCTIONS);
+    const applicability = parseApplicabilityResponse(applicabilityRaw);
+
+    if (!applicability.applicable) {
+      return {
+        ...baseResult,
+        applicable: false,
+        passed: null,
+        evidence: null,
+      };
+    }
+  }
+
+  // Step 2: Quality check
+  const itemType = workItemType || (check.id.includes('-epic-') ? 'epic' : 'story');
+  const qualityPrompt = buildQualityCheckPrompt(workItemText, check.question, itemType);
+  const qualityRaw = await llmProvider.generateJSON(qualityPrompt, QUALITY_CHECK_INSTRUCTIONS);
+  const qualityResult = parseCheckResponse(qualityRaw);
+
+  return {
+    ...baseResult,
+    applicable: true,
+    passed: qualityResult.passed,
+    evidence: qualityResult.evidence,
+    failDescription: check.failDescription,
+    failSuggestion: check.failSuggestion,
+  };
+}
+
+/**
+ * Run a single Tier 2 cross-reference check.
+ * @param {Object} check - Tier 2 check definition with dependsOn and template variables
+ * @param {string} workItemText - The full work item text
+ * @param {Map} tier1Results - Map of checkId -> tier1Result (for template resolution)
+ * @param {Object} llmProvider - LLM provider instance
+ * @returns {Object} { id, tier, severity, category, perspectives, applicable: true, passed, evidence }
+ */
+// ---------------------------------------------------------------------------
+// Batch operations — run N checks in a single LLM call
+// ---------------------------------------------------------------------------
+
+const BATCH_SYSTEM_INSTRUCTIONS =
+  "You are a work item quality checker evaluating MULTIPLE checks in a single pass.\n\n" +
+  "IMPORTANT calibration rules:\n" +
+  "- Accept SEMANTIC EQUIVALENCE: if the concept is addressed through a described mechanism, even without using the exact terminology the question uses, answer YES. Example: 'SameSite=Strict cookies' addresses CSRF even without the word 'CSRF'.\n" +
+  "- Match strictness to the WORK ITEM LEVEL: Epics are high-level planning artifacts — they describe WHAT and WHY, not HOW. Do not fail an epic for lacking implementation details. Stories should be more specific.\n" +
+  "- When in doubt, answer YES if the spirit of the requirement is met, even if the exact wording differs.\n\n" +
+  "For each check, determine:\n" +
+  "1. If it has an applicability question: is this check relevant to the work item? If not, set applicable=false.\n" +
+  "2. If applicable (or universal): does the work item pass the quality question?\n\n" +
+  "Return a JSON object keyed by check ID. Each value must have: {\"applicable\": boolean, \"passed\": boolean|null, \"evidence\": string}.\n" +
+  "If not applicable, set passed=null and evidence=\"\".";
+
+/**
+ * Build a single prompt that evaluates multiple checks against one work item.
+ * @param {Object[]} checks - Array of check definitions
+ * @param {string} workItemText - The full work item text
+ * @param {string} workItemType - "epic" or "story"
+ * @returns {string}
+ */
+function buildBatchPrompt(checks, workItemText, workItemType) {
+  const levelHint = workItemType === 'epic'
+    ? 'This is an EPIC (high-level planning document). Evaluate at epic-level granularity — do not require implementation details.\n\n'
+    : 'This is a STORY (implementation-level specification). Evaluate at story-level granularity — expect specific details.\n\n';
+
+  let prompt = levelHint;
+  prompt += `Given the following work item:\n\n${workItemText}\n\n`;
+  prompt += `Evaluate the following ${checks.length} checks:\n\n`;
+
+  for (const check of checks) {
+    prompt += `--- CHECK "${check.id}" ---\n`;
+    if (check.universal !== true && check.applicabilityQuestion) {
+      prompt += `Type: CONDITIONAL (first determine applicability)\n`;
+      prompt += `Applicability question: ${check.applicabilityQuestion}\n`;
+    } else {
+      prompt += `Type: UNIVERSAL (always applicable)\n`;
+    }
+    prompt += `Quality question: ${check.question}\n\n`;
+  }
+
+  prompt += `Respond with a JSON object. Keys are check IDs, values are {\"applicable\": boolean, \"passed\": boolean|null, \"evidence\": \"...\"}.\n`;
+  prompt += `Example: {"check-id-1": {"applicable": true, "passed": true, "evidence": "Found in section X"}, "check-id-2": {"applicable": false, "passed": null, "evidence": ""}}`;
+
+  return prompt;
+}
+
+/**
+ * Parse a batch LLM response into a Map of checkId → result or null.
+ * Tries JSON.parse first, then regex fallback per check ID.
+ * @param {string|Object} raw - Raw LLM response
+ * @param {Object[]} checks - The checks that were batched (for ID list)
+ * @returns {Map<string, Object|null>}
+ */
+function parseBatchResponse(raw, checks) {
+  const results = new Map();
+  const checkIds = checks.map(c => c.id);
+
+  // Try full JSON parse
+  let parsed = null;
+  if (raw && typeof raw === 'object') {
+    parsed = raw;
+  } else {
+    const text = String(raw);
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      // Fall through to regex
+    }
+  }
+
+  if (parsed && typeof parsed === 'object') {
+    for (const id of checkIds) {
+      if (parsed[id] && typeof parsed[id] === 'object') {
+        results.set(id, {
+          applicable: parsed[id].applicable !== false,
+          passed: parsed[id].applicable === false ? null : (parsed[id].passed === null ? null : Boolean(parsed[id].passed)),
+          evidence: parsed[id].applicable === false ? '' : (parsed[id].evidence || ''),
+        });
+      } else {
+        results.set(id, null); // Not found in response
+      }
+    }
+    return results;
+  }
+
+  // Regex fallback — try to extract per-check JSON blocks
+  const text = String(raw);
+  for (const id of checkIds) {
+    // Look for "check-id": { ... }
+    const escapedId = id.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+    const pattern = new RegExp(`"${escapedId}"\\s*:\\s*\\{([^}]+)\\}`, 'i');
+    const match = text.match(pattern);
+    if (match) {
+      try {
+        const block = JSON.parse(`{${match[1]}}`);
+        results.set(id, {
+          applicable: block.applicable !== false,
+          passed: block.applicable === false ? null : (block.passed === null ? null : Boolean(block.passed)),
+          evidence: block.applicable === false ? '' : (block.evidence || ''),
+        });
+      } catch {
+        results.set(id, null);
+      }
+    } else {
+      results.set(id, null);
+    }
+  }
+
+  return results;
+}
+
+/**
+ * Run a batch of Tier 1 checks in a single LLM call.
+ * Falls back to individual runTier1Check() for any unparseable results.
+ * @param {Object[]} checks - Array of Tier 1 check definitions
+ * @param {string} workItemText - The full work item text
+ * @param {Object} llmProvider - LLM provider instance
+ * @param {string} workItemType - "epic" or "story"
+ * @returns {Promise<Object[]>} Array of results with same shape as runTier1Check()
+ */
+export async function runTier1Batch(checks, workItemText, llmProvider, workItemType) {
+  const batchPrompt = buildBatchPrompt(checks, workItemText, workItemType);
+  const raw = await llmProvider.generateJSON(batchPrompt, BATCH_SYSTEM_INSTRUCTIONS);
+  const parsed = parseBatchResponse(raw, checks);
+
+  const results = [];
+  const fallbacks = [];
+
+  for (const check of checks) {
+    const entry = parsed.get(check.id);
+    if (entry) {
+      results.push({
+        id: check.id,
+        tier: 1,
+        severity: check.severity,
+        category: check.category,
+        perspective: check.perspective,
+        universal: check.universal,
+        applicable: entry.applicable,
+        passed: entry.passed,
+        evidence: entry.evidence,
+        failDescription: check.failDescription,
+        failSuggestion: check.failSuggestion,
+      });
+    } else {
+      fallbacks.push(check);
+    }
+  }
+
+  // Individual fallback for unparseable entries
+  if (fallbacks.length > 0) {
+    const fallbackResults = await Promise.all(
+      fallbacks.map(check => runTier1Check(check, workItemText, llmProvider, workItemType))
+    );
+    results.push(...fallbackResults);
+  }
+
+  return results;
+}
+
+/**
+ * Run a batch of Tier 2 checks in a single LLM call.
+ * Resolves template variables before building the batch prompt.
+ * Falls back to individual runTier2Check() for unparseable results.
+ * @param {Object[]} checks - Array of Tier 2 check definitions
+ * @param {string} workItemText - The full work item text
+ * @param {Map} tier1Results - Map of checkId → tier1Result
+ * @param {Object} llmProvider - LLM provider instance
+ * @param {string} workItemType - "epic" or "story"
+ * @returns {Promise<Object[]>} Array of results with same shape as runTier2Check()
+ */
+export async function runTier2Batch(checks, workItemText, tier1Results, llmProvider, workItemType) {
+  // Resolve template variables in each check's question before batching
+  const resolvedChecks = checks.map(check => ({
+    ...check,
+    question: resolveTemplateVariables(check.question, tier1Results),
+    universal: true, // Tier 2 checks skip applicability gate
+  }));
+
+  const batchPrompt = buildBatchPrompt(resolvedChecks, workItemText, workItemType);
+  const raw = await llmProvider.generateJSON(batchPrompt, BATCH_SYSTEM_INSTRUCTIONS);
+  const parsed = parseBatchResponse(raw, resolvedChecks);
+
+  const results = [];
+  const fallbacks = [];
+
+  for (const check of checks) {
+    const entry = parsed.get(check.id);
+    if (entry) {
+      results.push({
+        id: check.id,
+        tier: 2,
+        severity: check.severity,
+        category: check.category,
+        perspectives: check.perspectives,
+        applicable: true,
+        passed: entry.passed,
+        evidence: entry.evidence,
+        failDescription: check.failDescription,
+        failSuggestion: check.failSuggestion,
+      });
+    } else {
+      fallbacks.push(check);
+    }
+  }
+
+  // Individual fallback for unparseable entries
+  if (fallbacks.length > 0) {
+    const fallbackResults = await Promise.all(
+      fallbacks.map(check => runTier2Check(check, workItemText, tier1Results, llmProvider))
+    );
+    results.push(...fallbackResults);
+  }
+
+  return results;
+}
+
+export async function runTier2Check(check, workItemText, tier1Results, llmProvider) {
+  // Step 1: Resolve template variables in the question
+  const resolvedQuestion = resolveTemplateVariables(check.question, tier1Results);
+
+  // Step 2: Run quality check with resolved question
+  const itemType = check.id.includes('-epic') ? 'epic' : 'story';
+  const qualityPrompt = buildQualityCheckPrompt(workItemText, resolvedQuestion, itemType);
+  const qualityRaw = await llmProvider.generateJSON(qualityPrompt, QUALITY_CHECK_INSTRUCTIONS);
+  const qualityResult = parseCheckResponse(qualityRaw);
+
+  return {
+    id: check.id,
+    tier: 2,
+    severity: check.severity,
+    category: check.category,
+    perspectives: check.perspectives,
+    applicable: true,
+    passed: qualityResult.passed,
+    evidence: qualityResult.evidence,
+    failDescription: check.failDescription,
+    failSuggestion: check.failSuggestion,
+  };
+}

package/cli/micro-check-scorer.js

@@ -0,0 +1,148 @@
+/**
+ * Score micro-check results programmatically.
+ * Pure JS module — no LLM involvement, no external dependencies.
+ *
+ * @param {Object[]} checkResults - Array of completed check results from runner.
+ *   Each: { id, tier, severity, category, perspective, universal, applicable,
+ *           passed, evidence?, failDescription?, failSuggestion? }
+ * @returns {Object} Scoring result with failure counts, status, and pattern detection.
+ */
+export function scoreChecks(checkResults) {
+  // 1. Partition into applicable vs skipped vs errored
+  const applicable = [];
+  let skippedCount = 0;
+  let erroredCount = 0;
+
+  for (const check of checkResults) {
+    if (check.applicable === false) {
+      skippedCount++;
+    } else if (check.passed === null) {
+      // LLM parse error or runtime error — exclude from scoring
+      erroredCount++;
+    } else {
+      applicable.push(check);
+    }
+  }
+
+  const applicableCount = applicable.length;
+
+  // 2. Count failures by severity (only actual failures, not errors)
+  const failures = applicable.filter((c) => c.passed === false);
+
+  let criticalFails = 0;
+  let majorFails = 0;
+  let minorFails = 0;
+
+  for (const f of failures) {
+    switch (f.severity) {
+      case 'critical':
+        criticalFails++;
+        break;
+      case 'major':
+        majorFails++;
+        break;
+      case 'minor':
+        minorFails++;
+        break;
+    }
+  }
+
+  // 3. Scoring formula (matches existing validators)
+  let score;
+
+  if (criticalFails > 0) {
+    score = Math.max(0, Math.min(69, 60 - (criticalFails - 1) * 10));
+  } else if (majorFails > 0) {
+    score = Math.max(70, Math.min(89, 88 - (majorFails - 1) * 5));
+  } else {
+    score = Math.max(95, Math.min(100, 98 - minorFails));
+  }
+
+  // 4. Derive status
+  let status;
+
+  if (score >= 90) {
+    status = 'excellent';
+  } else if (score >= 70) {
+    status = 'acceptable';
+  } else {
+    status = 'needs-improvement';
+  }
+
+  // 5. Collect detailed failure info
+  const failedChecks = failures.map((f) => ({
+    id: f.id,
+    tier: f.tier,
+    severity: f.severity,
+    category: f.category,
+    perspective: f.perspective,
+    failDescription: f.failDescription,
+    failSuggestion: f.failSuggestion,
+    evidence: f.evidence,
+  }));
+
+  // 6. Tier 3 pattern detection
+
+  const result = {
+    score,
+    status,
+    applicableCount,
+    skippedCount,
+    erroredCount,
+    criticalFails,
+    majorFails,
+    minorFails,
+    failedChecks,
+  };
+
+  // 6a. Split recommendation — look for concentrated major/critical failures
+  //     in acceptance-criteria or implementation-clarity categories
+  const splitTargetCategories = new Set([
+    'acceptance-criteria',
+    'implementation-clarity',
+  ]);
+  const severeCategoryBuckets = {};
+
+  for (const f of failures) {
+    if (
+      (f.severity === 'major' || f.severity === 'critical') &&
+      splitTargetCategories.has(f.category)
+    ) {
+      severeCategoryBuckets[f.category] =
+        (severeCategoryBuckets[f.category] || 0) + 1;
+    }
+  }
+
+  for (const [category, count] of Object.entries(severeCategoryBuckets)) {
+    if (count >= 3) {
+      result.splitRecommendation = true;
+      result.splitReason =
+        `${count} major/critical issues in ${category} suggest this work item combines too many concerns`;
+      break; // report the first qualifying category
+    }
+  }
+
+  // 6b. Issue clustering — perspectives with 3+ failures
+  const perspectiveBuckets = {};
+
+  for (const f of failures) {
+    perspectiveBuckets[f.perspective] =
+      (perspectiveBuckets[f.perspective] || 0) + 1;
+  }
+
+  const clusterWarnings = [];
+
+  for (const [perspective, count] of Object.entries(perspectiveBuckets)) {
+    if (count >= 3) {
+      clusterWarnings.push(
+        `${count} issues from ${perspective} perspective — may indicate fundamental gap in this domain`
+      );
+    }
+  }
+
+  if (clusterWarnings.length > 0) {
+    result.clusterWarnings = clusterWarnings;
+  }
+
+  return result;
+}