@intentset/verification 0.1.0

package/src/adapters/node-test.ts

@@ -0,0 +1,139 @@
+/**
+ * `node --test --test-reporter=tap`: TAP version 13 as Node's test runner
+ * writes it. Each test is announced by `# Subtest: <title>` and reported by an
+ * `ok` or `not ok` line at the same indentation, four spaces per level of
+ * nesting, followed by a YAML diagnostic block. A test's full title is its
+ * ancestors' titles and its own, joined with " > ".
+ *
+ * Results: `ok` is pass; `# SKIP` and `# TODO` are skip; `not ok` is fail when
+ * an assertion failed and error when anything else went wrong (a thrown
+ * non-assertion, a hook failure, a timeout, a cancellation). A test with
+ * subtests is not counted on its own unless it failed for a reason of its
+ * own: its subtests are what carry the result.
+ */
+import type { RunResult } from "../records.ts";
+import type { ParsedRun, ParsedTest } from "./common.ts";
+
+const SUBTEST = /^# Subtest: (.*)$/;
+const TEST_POINT = /^(not ok|ok)\b(?:\s+\d+)?(?:\s+-)?\s*(.*)$/;
+const ASSERTION_CODE = "ERR_ASSERTION";
+
+export function parseNodeTap(text: string): ParsedRun {
+  const lines = text.split(/\r?\n/);
+  const titles: string[] = [];
+  const hasChild: boolean[] = [];
+  const tests: ParsedTest[] = [];
+  const problems: string[] = [];
+
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i];
+    const indent = line.length - line.trimStart().length;
+    const body = line.slice(indent);
+    const depth = Math.floor(indent / 4);
+
+    const subtest = SUBTEST.exec(body);
+    if (subtest !== null) {
+      titles[depth] = unescapeTap(subtest[1]);
+      titles.length = depth + 1;
+      hasChild[depth] = false;
+      hasChild.length = depth + 1;
+      i++;
+      continue;
+    }
+
+    const point = TEST_POINT.exec(body);
+    if (point === null) {
+      if (body.startsWith("Bail out!")) problems.push(body);
+      i++;
+      continue;
+    }
+
+    i++;
+    const yaml = new Map<string, string>();
+    if (i < lines.length && lines[i].trim() === "---") {
+      const blockIndent = lines[i].length - lines[i].trimStart().length;
+      i++;
+      while (i < lines.length) {
+        const current = lines[i];
+        const currentIndent = current.length - current.trimStart().length;
+        i++;
+        if (current.trim() === "..." && currentIndent === blockIndent) break;
+        if (currentIndent !== blockIndent) continue;
+        const pair = /^([A-Za-z_][A-Za-z0-9_]*):(?:\s+(.*))?$/.exec(current.slice(blockIndent));
+        if (pair !== null) yaml.set(pair[1], unquote(pair[2] ?? ""));
+      }
+    }
+
+    const { title, directive } = splitDirective(point[2]);
+    const parent = hasChild[depth] === true;
+    hasChild[depth] = false;
+    if (depth > 0) hasChild[depth - 1] = true;
+
+    const failureType = yaml.get("failureType");
+    const result = resultOf(point[1] === "ok", directive, failureType, yaml);
+    if (parent && (result === "pass" || result === "skip" || failureType === "subtestsFailed")) continue;
+
+    const ancestors = titles.slice(0, depth).filter((t) => t !== undefined);
+    const duration = Number(yaml.get("duration_ms"));
+    tests.push({
+      fullName: [...ancestors, title].join(" > "),
+      result,
+      durationMs: yaml.has("duration_ms") && Number.isFinite(duration) ? duration : null,
+    });
+  }
+  return { tests, problems };
+}
+
+function resultOf(
+  ok: boolean,
+  directive: "skip" | "todo" | null,
+  failureType: string | undefined,
+  yaml: Map<string, string>,
+): RunResult {
+  if (directive !== null) return "skip";
+  if (ok) return "pass";
+  if (failureType === "subtestsFailed") return "fail";
+  const assertion = yaml.get("code") === ASSERTION_CODE || yaml.get("name") === "AssertionError";
+  if (assertion) return "fail";
+  if (failureType === undefined) return "fail";
+  return "error";
+}
+
+/** Split a test point's description at its first unescaped `#` into the title and the directive. */
+function splitDirective(description: string): { title: string; directive: "skip" | "todo" | null } {
+  for (let i = 0; i < description.length; i++) {
+    const c = description[i];
+    if (c === "\\") {
+      i++;
+      continue;
+    }
+    if (c !== "#") continue;
+    const rest = description.slice(i + 1).trim();
+    const directive = /^skip\b/i.test(rest) ? "skip" : /^todo\b/i.test(rest) ? "todo" : null;
+    return { title: unescapeTap(description.slice(0, i).trimEnd()), directive };
+  }
+  return { title: unescapeTap(description.trimEnd()), directive: null };
+}
+
+const ESCAPES: Record<string, string> = { b: "\b", f: "\f", n: "\n", r: "\r", t: "\t", v: "\v" };
+
+/** Undo Node's TAP escaping: `\\`, `\#` and the control-character escapes. */
+function unescapeTap(text: string): string {
+  return text.replace(/\\(.)/g, (_, c: string) => ESCAPES[c] ?? c);
+}
+
+function unquote(value: string): string {
+  const trimmed = value.trim();
+  if (trimmed.length >= 2 && trimmed.startsWith("'") && trimmed.endsWith("'")) {
+    return trimmed.slice(1, -1).replace(/''/g, "'");
+  }
+  if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
+    try {
+      return JSON.parse(trimmed) as string;
+    } catch {
+      return trimmed.slice(1, -1);
+    }
+  }
+  return trimmed;
+}

package/src/adapters/vitest.ts

@@ -0,0 +1,82 @@
+/**
+ * `vitest run --reporter=json`: `testResults[]`, one per file, each with
+ * `assertionResults[]` carrying `ancestorTitles`, `title`, `fullName`,
+ * `status` and `failureMessages`. Results: passed is pass; skipped, pending,
+ * todo and disabled are skip; failed is fail when the failure is an
+ * assertion and error when the test threw anything else. A file that failed
+ * without reporting any test (it did not load) is listed in `problems`,
+ * since no test title can carry it to a selector.
+ */
+import type { RunResult } from "../records.ts";
+import type { ParsedRun, ParsedTest } from "./common.ts";
+
+const STATUSES: Record<string, RunResult | "failed"> = {
+  passed: "pass",
+  failed: "failed",
+  skipped: "skip",
+  pending: "skip",
+  todo: "skip",
+  disabled: "skip",
+};
+
+export function parseVitestJson(json: unknown): ParsedRun {
+  if (!isRecord(json) || !Array.isArray(json.testResults)) {
+    throw new TypeError("This is not Vitest JSON reporter output: expected an object with a testResults array.");
+  }
+  const tests: ParsedTest[] = [];
+  const problems: string[] = [];
+  json.testResults.forEach((file, fileIndex) => {
+    if (!isRecord(file)) throw new TypeError(`testResults[${fileIndex}] is not an object.`);
+    const name = typeof file.name === "string" ? file.name : `testResults[${fileIndex}]`;
+    const assertions = Array.isArray(file.assertionResults) ? file.assertionResults : [];
+    if (assertions.length === 0 && file.status === "failed") {
+      problems.push(`${name} failed without running a test: ${firstLine(file.message) || "no message"}`);
+    }
+    assertions.forEach((assertion, index) => {
+      if (!isRecord(assertion))
+        throw new TypeError(`testResults[${fileIndex}].assertionResults[${index}] is not an object.`);
+      const title = typeof assertion.title === "string" ? assertion.title : "";
+      const ancestors = Array.isArray(assertion.ancestorTitles)
+        ? assertion.ancestorTitles.filter((t): t is string => typeof t === "string")
+        : [];
+      const fullName =
+        typeof assertion.fullName === "string" && assertion.fullName !== ""
+          ? assertion.fullName
+          : [...ancestors, title].join(" > ");
+      const status = typeof assertion.status === "string" ? STATUSES[assertion.status] : undefined;
+      let result: RunResult;
+      if (status === undefined) {
+        problems.push(`${fullName} has the unknown status ${JSON.stringify(assertion.status)}, read as error`);
+        result = "error";
+      } else if (status === "failed") {
+        const messages = Array.isArray(assertion.failureMessages) ? assertion.failureMessages : [];
+        result = failureKind(typeof messages[0] === "string" ? messages[0] : "");
+      } else {
+        result = status;
+      }
+      const duration = assertion.duration;
+      tests.push({
+        fullName,
+        result,
+        durationMs: typeof duration === "number" && Number.isFinite(duration) ? duration : null,
+      });
+    });
+  });
+  return { tests, problems };
+}
+
+/** An assertion failure is fail; any other thrown error class is error; a bare message is a failed expectation. */
+function failureKind(message: string): RunResult {
+  const head = message.trimStart();
+  if (/^AssertionError\b/.test(head)) return "fail";
+  if (/^[A-Za-z]*Error\b/.test(head)) return "error";
+  return "fail";
+}
+
+function firstLine(value: unknown): string {
+  return typeof value === "string" ? (value.trim().split("\n")[0] ?? "") : "";
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/check.ts

@@ -0,0 +1,137 @@
+/**
+ * The evidence check (Core §8, §11): classification, coverage, and the
+ * diagnostics that follow from them. CORE007 joins at L3, where current
+ * passing evidence is a conformance requirement; below L3 missing evidence is
+ * not a diagnostic, but the coverage report still shows every gap.
+ */
+import { type Diagnostic, type Graph, type Level, LEVELS, sortDiagnostics } from "@intentset/core";
+import { type Classification, classifyEvidence, type EvidenceSnapshot, type EvidenceStatus } from "./classify.ts";
+import { type CoverageReport, compareUnresolved, coverage } from "./coverage.ts";
+import { compareStrings, evidenceDiagnostic } from "./diagnostic.ts";
+import type { RunRecord } from "./records.ts";
+
+export interface CheckEvidenceOptions {
+  level?: Level;
+  /**
+   * Every repository-relative path in the tree. When given, each
+   * verification's locator must name one of them (EVID003); when absent the
+   * locators are not checked and each is listed as unresolved.
+   */
+  files?: ReadonlySet<string>;
+  /** The file the records were read from, for diagnostics about a record. */
+  evidencePath?: string | null;
+}
+
+export interface EvidenceCheck {
+  /**
+   * EVID001 (graph-aware), EVID002, EVID003 and, at L3 and above, CORE007 (an
+   * error for a required claim, a warning for a draft claim with no
+   * verification definition); sorted, origin "evidence".
+   */
+  diagnostics: Diagnostic[];
+  classification: Classification;
+  coverage: CoverageReport;
+}
+
+const PHRASES: Record<EvidenceStatus, string> = {
+  "current-pass": "passed at this snapshot",
+  "current-fail": "failed at this snapshot",
+  stale: "is stale, with runs only at another commit or graph hash",
+  skip: "was skipped at this snapshot",
+  error: "errored at this snapshot",
+  missing: "has no run record",
+  unresolved: "could not be classified",
+};
+
+export function checkEvidence(
+  graph: Graph,
+  records: readonly RunRecord[],
+  snapshot: EvidenceSnapshot,
+  options: CheckEvidenceOptions = {},
+): EvidenceCheck {
+  const level = options.level ?? "L1";
+  const classification = classifyEvidence(graph, records, snapshot, { evidencePath: options.evidencePath ?? null });
+  const report = coverage(graph, classification, { level });
+  const diagnostics: Diagnostic[] = [...classification.diagnostics];
+
+  for (const id of Object.keys(classification.verifications)) {
+    const artifact = graph.artifacts.get(id);
+    const locator = artifact?.meta.verification?.locator;
+    if (artifact === undefined || locator === undefined || artifact.meta.status === "retired") continue;
+    if (options.files === undefined) {
+      report.unresolved.push({ subject: id, reason: `locator ${locator} not checked: no file tree was supplied` });
+    } else if (!options.files.has(locator)) {
+      diagnostics.push(
+        evidenceDiagnostic({
+          code: "EVID003",
+          severity: "warning",
+          artifact: id,
+          path: artifact.path,
+          field: "/intentset/verification/locator",
+          message: `Verification ${id} names the locator ${locator}, which is not a file in the repository tree.`,
+          remediation:
+            "Correct the locator to the repository-relative path of the test or review procedure, or add the file.",
+        }),
+      );
+    }
+  }
+  report.unresolved.sort(compareUnresolved);
+
+  if (LEVELS.indexOf(level) >= LEVELS.indexOf("L3")) {
+    for (const claim of report.perClaim) {
+      // A draft claim is not required yet, but one with no verification at all
+      // is a gap CI output should show (decided 2026-10-03, after the Streamlane
+      // pilot reported nothing at L3 while two rules had no test): a warning,
+      // never an error, and only for a missing definition, not a missing run.
+      if (!claim.required && claim.lifecycle === "draft" && claim.verifications.length === 0) {
+        diagnostics.push(
+          evidenceDiagnostic({
+            code: "CORE007",
+            severity: "warning",
+            artifact: claim.id,
+            path: claim.path,
+            message: `Draft ${claim.type} ${claim.id} has no verification definition naming it in verifies; it is not required while draft, and will fail L3 once approved.`,
+            remediation: `Add a verification whose links.verifies names ${claim.id} before the claim leaves draft (Core §8).`,
+          }),
+        );
+        continue;
+      }
+      if (!claim.required || claim.verified) continue;
+      const subject = `${capitalize(claim.lifecycle)} ${claim.type} ${claim.id}`;
+      if (claim.verifications.length === 0) {
+        diagnostics.push(
+          evidenceDiagnostic({
+            code: "CORE007",
+            artifact: claim.id,
+            path: claim.path,
+            message: `${subject} has no verification definition naming it in verifies, so it cannot be verified in this snapshot.`,
+            remediation: `Add a verification whose links.verifies names ${claim.id}, then record a passing run at the snapshot (Core §8).`,
+          }),
+        );
+        continue;
+      }
+      const failing = claim.verifications.filter((v) => v.status !== "current-pass");
+      const where = `commit ${snapshot.commit ?? "(unknown)"} and graph hash ${snapshot.graphHash.slice(0, 12)}`;
+      diagnostics.push(
+        evidenceDiagnostic({
+          code: "CORE007",
+          artifact: claim.id,
+          path: claim.path,
+          message: `${subject} is not verified in this snapshot: ${failing.map((v) => `${v.id} ${PHRASES[v.status]}`).join("; ")}.`,
+          remediation: `Record a passing run of ${failing
+            .map((v) => v.id)
+            .sort(compareStrings)
+            .join(
+              ", ",
+            )} at ${where}, fixing whatever failed first; stale, skipped and errored runs never count (Core §8).`,
+        }),
+      );
+    }
+  }
+
+  return { diagnostics: sortDiagnostics(diagnostics), classification, coverage: report };
+}
+
+function capitalize(text: string): string {
+  return text.charAt(0).toUpperCase() + text.slice(1);
+}

package/src/classify.ts

@@ -0,0 +1,222 @@
+/**
+ * Evidence freshness (Core §8, invariant 3). Only a `pass` at the assessed
+ * commit and graph hash is current; everything else is something else, and
+ * says which. A failing run at the snapshot stays visible however many older
+ * passes there are, because the status is decided by the latest run at the
+ * snapshot and every run is kept in the history.
+ */
+import { type Diagnostic, type Graph, sortDiagnostics } from "@intentset/core";
+import { compareStrings, evidenceDiagnostic } from "./diagnostic.ts";
+import { CURRENT, compareRecords, type RunRecord, type RunResult, type RunScope } from "./records.ts";
+
+export const EVIDENCE_STATUSES = [
+  "current-pass",
+  "current-fail",
+  "stale",
+  "skip",
+  "error",
+  "missing",
+  "unresolved",
+] as const;
+export type EvidenceStatus = (typeof EVIDENCE_STATUSES)[number];
+
+/** What evidence is assessed against: a commit and graph hash, and the product and release it is for. */
+export interface EvidenceSnapshot {
+  /** The assessed commit, or null when none could be read: matching then uses the graph hash alone, and says so. */
+  commit: string | null;
+  /** The graph hash of the snapshot (ADR 0005). */
+  graphHash: string;
+  /** The exact product and release under assessment. Absent or null considers records of every scope, and reports that it did. */
+  scope?: RunScope | null;
+}
+
+export type ResultCounts = Record<RunResult, number>;
+
+export interface VerificationEvidence {
+  verificationId: string;
+  method: "automated" | "manual";
+  status: EvidenceStatus;
+  /** Why the status is qualified or what was set aside, when anything was. */
+  note?: string;
+  /** The record that decided the status: the latest at the snapshot, else the latest in scope; null when there is none. */
+  latest: RunRecord | null;
+  /** Every in-scope record for this verification, oldest first (finishedAt, then evidenceId). */
+  history: RunRecord[];
+  /** Results of the records at the snapshot, so a failure followed by a pass is still countable. */
+  atSnapshot: ResultCounts;
+  /** Records for this verification that named another product or release and were not considered. */
+  outOfScope: number;
+}
+
+export interface RejectedRecord {
+  record: RunRecord;
+  reason: string;
+}
+
+export interface Classification {
+  snapshot: { commit: string | null; graphHash: string; scope: RunScope | null };
+  /** One entry per verification artifact in the graph, keyed and ordered by ID. */
+  verifications: Record<string, VerificationEvidence>;
+  /** Records whose scope named another product or release, across every verification. */
+  ignoredOutOfScope: number;
+  /** Records naming an ID that is not a verification in the graph (EVID002), by evidence ID. */
+  unknown: RunRecord[];
+  /** Records that cannot be evidence for the verification they name (EVID001), by evidence ID. */
+  rejected: RejectedRecord[];
+  /** EVID001 and EVID002, sorted. */
+  diagnostics: Diagnostic[];
+}
+
+export interface ClassifyOptions {
+  /** The file the records were read from, for diagnostics; null when they did not come from one. */
+  evidencePath?: string | null;
+}
+
+const NOTE_COMMIT_UNKNOWN = "commit unknown; matched on graph hash";
+
+/** Core §8: classify every verification in the graph against the snapshot. Deterministic for identical inputs. */
+export function classifyEvidence(
+  graph: Graph,
+  records: readonly RunRecord[],
+  snapshot: EvidenceSnapshot,
+  options: ClassifyOptions = {},
+): Classification {
+  const path = options.evidencePath ?? null;
+  const scope = snapshot.scope ?? null;
+  const diagnostics: Diagnostic[] = [];
+  const unknown: RunRecord[] = [];
+  const rejected: RejectedRecord[] = [];
+  const inScope = new Map<string, RunRecord[]>();
+  const outOfScope = new Map<string, number>();
+  let ignoredOutOfScope = 0;
+
+  const ids = [...graph.artifacts.keys()]
+    .filter((id) => graph.artifacts.get(id)?.meta.type === "verification")
+    .sort(compareStrings);
+  for (const id of ids) inScope.set(id, []);
+
+  const ordered = [...records].sort((a, b) => compareStrings(a.evidenceId, b.evidenceId) || compareRecords(a, b));
+  for (const record of ordered) {
+    const target = graph.artifacts.get(record.verificationId);
+    if (target === undefined || target.meta.type !== "verification") {
+      unknown.push(record);
+      diagnostics.push(
+        evidenceDiagnostic({
+          code: "EVID002",
+          severity: "warning",
+          artifact: null,
+          path,
+          message:
+            target === undefined
+              ? `Run record ${record.evidenceId} names ${record.verificationId}, which is not in the graph, so it is evidence for nothing.`
+              : `Run record ${record.evidenceId} names ${record.verificationId}, which is a ${target.meta.type}, not a verification.`,
+          remediation:
+            "Point the record at the verification it ran, or create that verification; runs are evidence for verifications only (Core §8).",
+        }),
+      );
+      continue;
+    }
+    if (target.meta.verification?.method === "manual" && record.reviewer === null) {
+      const reason = `${record.verificationId} is a manual review, and record ${record.evidenceId} names no reviewer.`;
+      rejected.push({ record, reason });
+      diagnostics.push(
+        evidenceDiagnostic({
+          code: "EVID001",
+          artifact: record.verificationId,
+          path,
+          message: `Run record ${record.evidenceId} names no reviewer, but ${record.verificationId} is a manual review.`,
+          remediation:
+            "Record who performed the review and the rationale; a manual record must identify both (Core §8).",
+        }),
+      );
+      continue;
+    }
+    if (scope !== null && (record.scope.product !== scope.product || record.scope.release !== scope.release)) {
+      ignoredOutOfScope++;
+      outOfScope.set(record.verificationId, (outOfScope.get(record.verificationId) ?? 0) + 1);
+      continue;
+    }
+    inScope.get(record.verificationId)?.push(record);
+  }
+
+  const verifications: Record<string, VerificationEvidence> = {};
+  for (const id of ids) {
+    const artifact = graph.artifacts.get(id);
+    const history = (inScope.get(id) ?? []).sort(compareRecords);
+    verifications[id] = classifyOne(
+      id,
+      artifact?.meta.verification?.method ?? "automated",
+      history,
+      outOfScope.get(id) ?? 0,
+      snapshot,
+    );
+  }
+
+  return {
+    snapshot: { commit: snapshot.commit, graphHash: snapshot.graphHash, scope },
+    verifications,
+    ignoredOutOfScope,
+    unknown,
+    rejected,
+    diagnostics: sortDiagnostics(diagnostics),
+  };
+}
+
+/** True when a record was made at the snapshot: same graph hash, and same commit unless the snapshot's is unknown. */
+export function atSnapshot(record: RunRecord, snapshot: { commit: string | null; graphHash: string }): boolean {
+  return record.graphHash === snapshot.graphHash && (snapshot.commit === null || record.commit === snapshot.commit);
+}
+
+function classifyOne(
+  verificationId: string,
+  method: "automated" | "manual",
+  history: RunRecord[],
+  outOfScope: number,
+  snapshot: EvidenceSnapshot,
+): VerificationEvidence {
+  const counts: ResultCounts = { pass: 0, fail: 0, skip: 0, error: 0 };
+  const base = { verificationId, method, history, atSnapshot: counts, outOfScope };
+  const ignored =
+    outOfScope > 0 ? `${outOfScope} record${outOfScope === 1 ? "" : "s"} for another product or release ignored` : null;
+
+  if (history.length === 0) {
+    return withNote({ ...base, status: "missing", latest: null }, [ignored]);
+  }
+
+  const unbound = history.filter(
+    (record) => record.graphHash === CURRENT || (snapshot.commit !== null && record.commit === CURRENT),
+  );
+  if (unbound.length > 0) {
+    const names = unbound.map((record) => record.evidenceId).join(", ");
+    return withNote({ ...base, status: "unresolved", latest: history[history.length - 1] }, [
+      `unbound @current placeholder in ${names}; bind it to the snapshot before classifying`,
+      ignored,
+    ]);
+  }
+
+  const current = history.filter((record) => atSnapshot(record, snapshot));
+  for (const record of current) counts[record.result]++;
+  if (current.length === 0) {
+    return withNote({ ...base, status: "stale", latest: history[history.length - 1] }, [ignored]);
+  }
+
+  const latest = current[current.length - 1];
+  const status: EvidenceStatus =
+    latest.result === "pass" ? "current-pass" : latest.result === "fail" ? "current-fail" : latest.result;
+  return withNote({ ...base, status, latest }, [snapshot.commit === null ? NOTE_COMMIT_UNKNOWN : null, ignored]);
+}
+
+function withNote(evidence: Omit<VerificationEvidence, "note">, notes: (string | null)[]): VerificationEvidence {
+  const present = notes.filter((note): note is string => note !== null);
+  const { verificationId, method, status, latest, history, atSnapshot: counts, outOfScope } = evidence;
+  return {
+    verificationId,
+    method,
+    status,
+    ...(present.length > 0 ? { note: present.join("; ") } : {}),
+    latest,
+    history,
+    atSnapshot: counts,
+    outOfScope,
+  };
+}