@jaimevalasek/aioson 1.30.2 → 1.36.0

package/src/verification/runtime-telemetry.js

@@ -0,0 +1,144 @@
+'use strict';
+
+/**
+ * Best-effort runtime telemetry for `verify:implementation`.
+ *
+ * The verifier already stores durable artifacts under
+ * `.aioson/context/features/{slug}/verification-runs/`. Runtime telemetry is a
+ * query/index layer only: it must never store raw auditor output, stderr,
+ * prompt text, or finding evidence.
+ */
+
+const { openRuntimeDb, startTask, startRun } = require('../runtime-store');
+
+const SOURCE = 'verify_implementation';
+const EVENT_TYPE = 'implementation_verification_completed';
+const PHASE = 'implementation_verification';
+
+function bool(value) {
+  return Boolean(value);
+}
+
+function safeString(value) {
+  if (value === null || value === undefined) return null;
+  const text = String(value).trim();
+  return text || null;
+}
+
+function safeNumber(value) {
+  if (value === null || value === undefined) return null;
+  const number = Number(value);
+  return Number.isFinite(number) ? number : null;
+}
+
+function pathOrNull(value) {
+  const text = safeString(value);
+  return text ? text.replace(/\\/g, '/') : null;
+}
+
+function summarizeRunner(runner) {
+  if (!runner || typeof runner !== 'object') return null;
+  return {
+    status: safeString(runner.status),
+    tool_detected: bool(runner.detected),
+    permission_mode: safeString(runner.permission_mode),
+    destructive_commands_allowed: bool(runner.destructive_commands_allowed),
+    timeout_ms: safeNumber(runner.timeout_ms),
+    max_output_bytes: safeNumber(runner.max_output_bytes),
+    duration_ms: safeNumber(runner.duration_ms),
+    exit_code: safeNumber(runner.exit_code),
+    signal: safeString(runner.signal),
+    output_bytes: safeNumber(runner.output_bytes),
+    output_truncated: bool(runner.output_truncated)
+  };
+}
+
+function buildVerificationTelemetryPayload(result, { durationMs = null } = {}) {
+  const runner = summarizeRunner(result.runner);
+  const payload = {
+    feature_slug: safeString(result.feature_slug),
+    mode: safeString(result.mode),
+    policy: safeString(result.policy),
+    ok: bool(result.ok),
+    verdict: safeString(result.verdict),
+    recommended_route: safeString(result.recommended_route),
+    blocking_findings_count: safeNumber(result.blocking_findings_count),
+    reason: safeString(result.reason),
+    tool: safeString(result.tool),
+    model: safeString(result.model),
+    ledger_path: pathOrNull(result.ledger_path),
+    prompt_path: pathOrNull(result.prompt_path),
+    report_path: pathOrNull(result.report_path),
+    run_report_path: pathOrNull(result.run_report_path),
+    report_json_path: pathOrNull(result.report_json_path),
+    raw_output_stored: bool(result.raw_report_path),
+    stderr_stored: bool(result.stderr_path),
+    duration_ms: safeNumber(durationMs)
+  };
+
+  if (runner) payload.runner = runner;
+  return payload;
+}
+
+function runtimeStatus(result) {
+  if (!result || result.mode === 'verify-implementation') return 'failed';
+  return 'completed';
+}
+
+async function recordVerificationTelemetry(rootDir, result, { startedAt = null } = {}) {
+  if (!rootDir || !result || typeof result !== 'object') {
+    return { emitted: false, reason: 'invalid_input' };
+  }
+
+  try {
+    const durationMs = startedAt ? Math.max(0, Date.now() - Number(startedAt)) : null;
+    const payload = buildVerificationTelemetryPayload(result, { durationMs });
+    const { db } = await openRuntimeDb(rootDir);
+    try {
+      const status = runtimeStatus(result);
+      const title = `Implementation verification: ${payload.feature_slug || 'unknown'}`;
+      const taskKey = startTask(db, {
+        title,
+        taskKind: 'implementation_verification',
+        status,
+        createdBy: 'verify:implementation',
+        metaJson: {
+          feature_slug: payload.feature_slug,
+          mode: payload.mode,
+          policy: payload.policy,
+          source: SOURCE
+        }
+      });
+      const runKey = startRun(db, {
+        taskKey,
+        agentName: 'verify:implementation',
+        agentKind: 'system',
+        source: SOURCE,
+        title: `verify:implementation ${payload.mode || 'unknown'} ${payload.feature_slug || 'unknown'}`,
+        status,
+        summary: `${payload.verdict || (result.ok ? 'PASS' : 'INCONCLUSIVE')} via ${payload.mode || 'unknown'}`,
+        outputPath: payload.report_path || payload.prompt_path || payload.ledger_path || null,
+        eventType: EVENT_TYPE,
+        phase: PHASE,
+        toolName: payload.tool,
+        verdict: payload.verdict,
+        message: `Implementation verification completed: ${payload.verdict || (result.ok ? 'ok' : 'blocked')}`,
+        payload
+      });
+      return { emitted: true, task_key: taskKey, run_key: runKey, event_type: EVENT_TYPE };
+    } finally {
+      db.close();
+    }
+  } catch (err) {
+    return { emitted: false, reason: 'telemetry_error', message: err && err.message ? err.message : String(err) };
+  }
+}
+
+module.exports = {
+  SOURCE,
+  EVENT_TYPE,
+  PHASE,
+  buildVerificationTelemetryPayload,
+  recordVerificationTelemetry
+};
+

package/src/verification/schema.js

@@ -0,0 +1,276 @@
+'use strict';
+
+const { resolveInsideRoot } = require('./path-policy');
+
+const LEDGER_SCHEMA_VERSION = 'implementation-ledger/v1';
+const REPORT_SCHEMA_VERSION = 'verification-report/v1';
+
+const POLICIES = new Set(['advisory', 'standard', 'strict']);
+
+const CLAIM_STATUSES = new Set([
+  'planned',
+  'implemented',
+  'partial',
+  'blocked',
+  'not_applicable'
+]);
+
+const CLAIM_KINDS = new Set([
+  'required_behavior',
+  'acceptance_criterion',
+  'test_coverage',
+  'scope_constraint',
+  'security_constraint',
+  'prototype_contract',
+  'migration_or_data'
+]);
+
+const OWNERS = new Set([
+  'dev',
+  'deyvin',
+  'product',
+  'sheldon',
+  'architect',
+  'ux-ui',
+  'qa',
+  'tester',
+  'pentester',
+  'scope-check'
+]);
+
+const VERDICTS = new Set([
+  'PASS',
+  'NEEDS_DEV_FIX',
+  'NEEDS_SCOPE_DECISION',
+  'NEEDS_QA_RECHECK',
+  'NEEDS_SECURITY_REVIEW',
+  'INCONCLUSIVE'
+]);
+
+const FINDING_STATUSES = new Set([
+  'CONFIRMS',
+  'DOES_NOT_CONFIRM',
+  'PARTIAL',
+  'NOT_VERIFIED',
+  'NOT_APPLICABLE'
+]);
+
+const SEVERITIES = new Set([
+  'info',
+  'warning',
+  'blocking'
+]);
+
+function normalizePolicy(policy) {
+  const value = String(policy || 'standard').toLowerCase();
+  return POLICIES.has(value) ? value : null;
+}
+
+function validatePathInsideRoot(rootDir, relPath, field, errors) {
+  if (!relPath) {
+    errors.push({ field, reason: 'missing' });
+    return;
+  }
+  const safe = resolveInsideRoot(rootDir, relPath);
+  if (!safe.ok) errors.push({ field, reason: safe.reason });
+}
+
+function validateSourceArtifacts(rootDir, artifacts, errors) {
+  if (!Array.isArray(artifacts)) {
+    errors.push({ field: 'source_artifacts', reason: 'must_be_array' });
+    return;
+  }
+  for (const [index, artifact] of artifacts.entries()) {
+    if (!artifact || typeof artifact !== 'object') {
+      errors.push({ field: `source_artifacts[${index}]`, reason: 'must_be_object' });
+      continue;
+    }
+    if (!artifact.type) errors.push({ field: `source_artifacts[${index}].type`, reason: 'missing' });
+    validatePathInsideRoot(rootDir, artifact.path, `source_artifacts[${index}].path`, errors);
+  }
+}
+
+function validateClaims(rootDir, claims, errors) {
+  if (!Array.isArray(claims)) {
+    errors.push({ field: 'claims', reason: 'must_be_array' });
+    return;
+  }
+
+  for (const [index, claim] of claims.entries()) {
+    if (!claim || typeof claim !== 'object') {
+      errors.push({ field: `claims[${index}]`, reason: 'must_be_object' });
+      continue;
+    }
+    if (!claim.id) errors.push({ field: `claims[${index}].id`, reason: 'missing' });
+    if (!claim.summary) errors.push({ field: `claims[${index}].summary`, reason: 'missing' });
+    if (!CLAIM_KINDS.has(claim.kind)) errors.push({ field: `claims[${index}].kind`, reason: 'invalid' });
+    if (!OWNERS.has(claim.owner)) errors.push({ field: `claims[${index}].owner`, reason: 'invalid' });
+    if (!CLAIM_STATUSES.has(claim.status)) errors.push({ field: `claims[${index}].status`, reason: 'invalid' });
+    if (!Array.isArray(claim.evidence)) {
+      errors.push({ field: `claims[${index}].evidence`, reason: 'must_be_array' });
+      continue;
+    }
+    for (const [evidenceIndex, evidence] of claim.evidence.entries()) {
+      if (evidence && evidence.path) {
+        validatePathInsideRoot(
+          rootDir,
+          evidence.path,
+          `claims[${index}].evidence[${evidenceIndex}].path`,
+          errors
+        );
+      }
+    }
+  }
+}
+
+function validateLedgerVerificationCommands(commands, errors) {
+  if (!Array.isArray(commands)) {
+    errors.push({ field: 'verification_commands', reason: 'must_be_array' });
+    return;
+  }
+  for (const [index, command] of commands.entries()) {
+    if (!command || typeof command !== 'object') {
+      errors.push({ field: `verification_commands[${index}]`, reason: 'must_be_object' });
+      continue;
+    }
+    if (!command.command) errors.push({ field: `verification_commands[${index}].command`, reason: 'missing' });
+  }
+}
+
+function validateImplementationLedger(ledger, { rootDir, slug }) {
+  const errors = [];
+  if (!ledger || typeof ledger !== 'object') {
+    return [{ field: 'machine_ledger', reason: 'must_be_object' }];
+  }
+  if (ledger.schema_version !== LEDGER_SCHEMA_VERSION) {
+    errors.push({ field: 'schema_version', reason: 'invalid' });
+  }
+  if (ledger.feature_slug !== slug) {
+    errors.push({ field: 'feature_slug', reason: 'mismatch' });
+  }
+  validateSourceArtifacts(rootDir, ledger.source_artifacts, errors);
+  validateClaims(rootDir, ledger.claims, errors);
+  validateLedgerVerificationCommands(ledger.verification_commands, errors);
+  if (!Array.isArray(ledger.known_gaps)) {
+    errors.push({ field: 'known_gaps', reason: 'must_be_array' });
+  }
+  return errors;
+}
+
+function missingLedgerEvidence(ledger) {
+  const claims = Array.isArray(ledger && ledger.claims) ? ledger.claims : [];
+  return claims
+    .filter((claim) => claim && ['implemented', 'partial'].includes(claim.status))
+    .filter((claim) => !Array.isArray(claim.evidence) || claim.evidence.length === 0)
+    .map((claim) => claim.id || '(missing id)');
+}
+
+function validateReportCommands(commands, errors) {
+  if (!Array.isArray(commands)) {
+    errors.push({ field: 'commands_run', reason: 'must_be_array' });
+    return;
+  }
+  for (const [index, command] of commands.entries()) {
+    if (!command || typeof command !== 'object') {
+      errors.push({ field: `commands_run[${index}]`, reason: 'must_be_object' });
+      continue;
+    }
+    if (!command.command) errors.push({ field: `commands_run[${index}].command`, reason: 'missing' });
+  }
+}
+
+function validateFindings(findings, errors) {
+  if (!Array.isArray(findings)) {
+    errors.push({ field: 'findings', reason: 'must_be_array' });
+    return;
+  }
+  for (const [index, finding] of findings.entries()) {
+    if (!finding || typeof finding !== 'object') {
+      errors.push({ field: `findings[${index}]`, reason: 'must_be_object' });
+      continue;
+    }
+    if (!finding.id) errors.push({ field: `findings[${index}].id`, reason: 'missing' });
+    if (!FINDING_STATUSES.has(finding.status)) errors.push({ field: `findings[${index}].status`, reason: 'invalid' });
+    if (!SEVERITIES.has(finding.severity)) errors.push({ field: `findings[${index}].severity`, reason: 'invalid' });
+    if (!OWNERS.has(finding.owner)) errors.push({ field: `findings[${index}].owner`, reason: 'invalid' });
+    if (finding.kind && !CLAIM_KINDS.has(finding.kind)) errors.push({ field: `findings[${index}].kind`, reason: 'invalid' });
+    if (finding.recommended_route && !OWNERS.has(finding.recommended_route)) {
+      errors.push({ field: `findings[${index}].recommended_route`, reason: 'invalid' });
+    }
+  }
+}
+
+function validateVerificationReport(report, { slug, requestedPolicy }) {
+  const errors = [];
+  if (!report || typeof report !== 'object') {
+    return [{ field: 'machine_report', reason: 'must_be_object' }];
+  }
+  if (report.schema_version !== REPORT_SCHEMA_VERSION) {
+    errors.push({ field: 'schema_version', reason: 'invalid' });
+  }
+  if (report.feature_slug !== slug) {
+    errors.push({ field: 'feature_slug', reason: 'mismatch' });
+  }
+  if (!normalizePolicy(requestedPolicy || report.policy)) {
+    errors.push({ field: 'policy', reason: 'invalid' });
+  }
+  if (!VERDICTS.has(report.verdict)) {
+    errors.push({ field: 'verdict', reason: 'invalid' });
+  }
+  if (report.recommended_route && !OWNERS.has(report.recommended_route)) {
+    errors.push({ field: 'recommended_route', reason: 'invalid' });
+  }
+  validateReportCommands(report.commands_run, errors);
+  validateFindings(report.findings, errors);
+  return errors;
+}
+
+function machineReportSchemaExample(slug, policy) {
+  return {
+    schema_version: REPORT_SCHEMA_VERSION,
+    feature_slug: slug,
+    policy,
+    verdict: 'PASS|NEEDS_DEV_FIX|NEEDS_SCOPE_DECISION|NEEDS_QA_RECHECK|NEEDS_SECURITY_REVIEW|INCONCLUSIVE',
+    summary: 'Short evidence-grounded summary.',
+    commands_run: [
+      {
+        command: 'npm test',
+        status: 'passed|failed|not_run',
+        evidence: 'Short result summary'
+      }
+    ],
+    findings: [
+      {
+        id: 'FIND-001',
+        claim_id: 'CLAIM-001',
+        kind: 'required_behavior',
+        status: 'CONFIRMS|DOES_NOT_CONFIRM|PARTIAL|NOT_VERIFIED|NOT_APPLICABLE',
+        severity: 'info|warning|blocking',
+        owner: 'dev',
+        file: 'src/example.js',
+        line: 1,
+        evidence: 'Exact reason with file:line when available.',
+        recommended_route: 'dev'
+      }
+    ],
+    recommended_route: 'qa',
+    blocking_findings_count: 0
+  };
+}
+
+module.exports = {
+  LEDGER_SCHEMA_VERSION,
+  REPORT_SCHEMA_VERSION,
+  POLICIES,
+  CLAIM_STATUSES,
+  CLAIM_KINDS,
+  OWNERS,
+  VERDICTS,
+  FINDING_STATUSES,
+  SEVERITIES,
+  normalizePolicy,
+  validateImplementationLedger,
+  missingLedgerEvidence,
+  validateVerificationReport,
+  machineReportSchemaExample
+};

package/src/verification/source-discovery.js

@@ -0,0 +1,153 @@
+'use strict';
+
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+const { resolveInsideRoot, relativeFromRoot, toPosixPath } = require('./path-policy');
+
+async function fileExists(filePath) {
+  try {
+    const stat = await fs.stat(filePath);
+    return stat.isFile() || stat.isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+async function listDirSafe(dir) {
+  try {
+    return await fs.readdir(dir, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+}
+
+async function walkFiles(rootDir, relativeDir, predicate, limit = 100) {
+  const start = path.join(rootDir, relativeDir);
+  const found = [];
+
+  async function walk(absDir) {
+    if (found.length >= limit) return;
+    const entries = await listDirSafe(absDir);
+    for (const entry of entries) {
+      if (found.length >= limit) break;
+      const full = path.join(absDir, entry.name);
+      if (entry.isDirectory()) {
+        await walk(full);
+      } else if (entry.isFile() && predicate(entry.name, full)) {
+        found.push(relativeFromRoot(rootDir, full));
+      }
+    }
+  }
+
+  if (await fileExists(start)) {
+    await walk(start);
+  }
+  return found;
+}
+
+function addArtifact(artifacts, seen, artifact) {
+  if (!artifact || !artifact.path) return;
+  const key = `${artifact.type}:${toPosixPath(artifact.path)}`;
+  if (seen.has(key)) return;
+  seen.add(key);
+  artifacts.push({
+    type: artifact.type,
+    path: toPosixPath(artifact.path),
+    role: artifact.role || 'source'
+  });
+}
+
+async function addIfExists(rootDir, artifacts, seen, type, relPath, role = 'source') {
+  const safe = resolveInsideRoot(rootDir, relPath);
+  if (!safe.ok) return;
+  if (await fileExists(safe.path)) {
+    addArtifact(artifacts, seen, { type, path: safe.relative_path, role });
+  }
+}
+
+function ledgerSourceArtifacts(ledger) {
+  if (!ledger || !Array.isArray(ledger.source_artifacts)) return [];
+  return ledger.source_artifacts
+    .filter((artifact) => artifact && artifact.path)
+    .map((artifact) => ({
+      type: artifact.type || 'ledger_source',
+      path: artifact.path,
+      role: artifact.role || 'ledger_declared'
+    }));
+}
+
+function ledgerEvidenceFiles(ledger) {
+  if (!ledger || !Array.isArray(ledger.claims)) return [];
+  const files = [];
+  for (const claim of ledger.claims) {
+    if (!claim || !Array.isArray(claim.evidence)) continue;
+    for (const evidence of claim.evidence) {
+      if (evidence && evidence.path) {
+        files.push({
+          type: 'implementation_evidence',
+          path: evidence.path,
+          role: claim.id ? `claim:${claim.id}` : 'claim'
+        });
+      }
+    }
+  }
+  return files;
+}
+
+async function discoverSourceArtifacts(rootDir, slug, ledger = null) {
+  const artifacts = [];
+  const seen = new Set();
+
+  await addIfExists(rootDir, artifacts, seen, 'prd', `.aioson/context/prd-${slug}.md`, 'product_authority');
+  await addIfExists(rootDir, artifacts, seen, 'requirements', `.aioson/context/requirements-${slug}.md`, 'acceptance_criteria');
+  await addIfExists(rootDir, artifacts, seen, 'spec', `.aioson/context/spec-${slug}.md`, 'living_memory');
+  await addIfExists(rootDir, artifacts, seen, 'sheldon_enrichment', `.aioson/context/sheldon-enrichment-${slug}.md`, 'product_review');
+  await addIfExists(rootDir, artifacts, seen, 'sheldon_validation', `.aioson/context/sheldon-validation-${slug}.md`, 'product_review');
+  await addIfExists(rootDir, artifacts, seen, 'design_doc', `.aioson/context/design-doc-${slug}.md`, 'design_authority');
+  await addIfExists(rootDir, artifacts, seen, 'readiness', `.aioson/context/readiness-${slug}.md`, 'implementation_readiness');
+  await addIfExists(rootDir, artifacts, seen, 'ui_spec', `.aioson/context/ui-spec-${slug}.md`, 'ui_contract');
+  await addIfExists(rootDir, artifacts, seen, 'implementation_plan', `.aioson/context/implementation-plan-${slug}.md`, 'execution_plan');
+  await addIfExists(rootDir, artifacts, seen, 'simple_plan', `.aioson/context/simple-plans/${slug}.md`, 'simple_plan');
+  await addIfExists(rootDir, artifacts, seen, 'scope_check', `.aioson/context/scope-check-${slug}.md`, 'scope_alignment');
+  await addIfExists(rootDir, artifacts, seen, 'qa_report', `.aioson/context/qa-report-${slug}.md`, 'qa_findings');
+  await addIfExists(rootDir, artifacts, seen, 'security_findings', `.aioson/context/security-findings-${slug}.json`, 'security_findings');
+  await addIfExists(rootDir, artifacts, seen, 'dossier', `.aioson/context/features/${slug}/dossier.md`, 'feature_dossier');
+  await addIfExists(rootDir, artifacts, seen, 'sheldon_plan', `.aioson/plans/${slug}/manifest.md`, 'phased_plan');
+  await addIfExists(rootDir, artifacts, seen, 'harness', `.aioson/plans/${slug}/harness-contract.json`, 'binary_criteria');
+  await addIfExists(rootDir, artifacts, seen, 'harness_progress', `.aioson/plans/${slug}/progress.json`, 'binary_criteria_state');
+  await addIfExists(rootDir, artifacts, seen, 'harness_output', `.aioson/plans/${slug}/last-check-output.json`, 'binary_criteria_output');
+  await addIfExists(rootDir, artifacts, seen, 'prototype', `.aioson/briefings/${slug}/prototype.html`, 'prototype_contract');
+  await addIfExists(rootDir, artifacts, seen, 'prototype_manifest', `.aioson/briefings/${slug}/prototype-manifest.md`, 'prototype_contract');
+  await addIfExists(rootDir, artifacts, seen, 'prototype_report', `.aioson/briefings/${slug}/prototype-report.md`, 'prototype_contract');
+
+  const prdFiles = await walkFiles(rootDir, 'prds', (name) => name.includes(slug) && name.endsWith('.md'), 25);
+  for (const rel of prdFiles) addArtifact(artifacts, seen, { type: 'source_prd', path: rel, role: 'draft_source' });
+
+  const rootPlanFiles = await walkFiles(rootDir, 'plans', (name, full) => {
+    const rel = relativeFromRoot(rootDir, full);
+    return rel.includes(slug);
+  }, 50);
+  for (const rel of rootPlanFiles) addArtifact(artifacts, seen, { type: 'source_plan', path: rel, role: 'preproduction_source' });
+
+  const testFiles = await walkFiles(rootDir, 'tests', (name) => name.includes(slug), 50);
+  for (const rel of testFiles) addArtifact(artifacts, seen, { type: 'test', path: rel, role: 'verification' });
+
+  const contextReviewFiles = await walkFiles(rootDir, '.aioson/context', (name) => (
+    /^qa-report-/.test(name) && name.includes(slug) && name.endsWith('.md')
+  ), 25);
+  for (const rel of contextReviewFiles) addArtifact(artifacts, seen, { type: 'qa_report', path: rel, role: 'qa_findings' });
+
+  for (const artifact of [...ledgerSourceArtifacts(ledger), ...ledgerEvidenceFiles(ledger)]) {
+    const safe = resolveInsideRoot(rootDir, artifact.path);
+    if (safe.ok && await fileExists(safe.path)) {
+      addArtifact(artifacts, seen, { ...artifact, path: safe.relative_path });
+    }
+  }
+
+  return artifacts;
+}
+
+module.exports = {
+  discoverSourceArtifacts
+};