heyspex 0.1.0

package/dev-factory/lib/sufficiency-score.mjs

@@ -0,0 +1,441 @@
+#!/usr/bin/env node
+/**
+ * sufficiency-score.mjs — score a capture or research doc against the
+ * evidence-backed "minimal-sufficient" element set before an autonomous run
+ * trusts it as input. [[HS-9YK8-TASK-DB3O]] · [[HS-DOU7-3EFV-EPIC-capture-and-research-quality-gate-score]]
+ *
+ * This is the pure JUDGMENT core of the quality gate — it does not wire into
+ * Phase-1 pipeline entry or the capture-promotion flow (separate follow-on
+ * tasks), does not do retry-or-quorum over stochastic verdicts, and does not
+ * calibrate/version the pass bar from the ablation harnesses. It just scores
+ * one document, once, deterministically, from its frontmatter + body text.
+ *
+ * Element set + weights come straight from [[HS-B3KK-1L9W-RES-minimal-sufficient-capture-shape-for-age]]
+ * (cross-checked against dream-loop ablation iterations 7-14, dev-factory/DREAM-LOOP.md):
+ *   - `example`      (0.30) — the PROVEN load-bearing element (iteration 14: a
+ *                     real example's presence/absence measurably changed what
+ *                     survived into L3 research). Highest weight.
+ *   - `core-idea`    (0.30) — the foundational goal/problem statement; machine
+ *                     stages pass on core idea alone (iterations 8-9, 11).
+ *   - `done-criteria`(0.20) — the one element every surveyed external spec
+ *                     format requires in some notation (EARS / Given-When-Then
+ *                     / Verification / our own "Acceptance criteria (binary)");
+ *                     not yet a coded capture section, but present on tasks/epics.
+ *   - `kind`         (0.10) — capture routing metadata (still the default value
+ *                     when unset — mirrors capture-vault.ts's CAPTURE_STATUS
+ *                     default logic, `fm.kind ?? "idea"`).
+ *   - `user-stories` (0.10) — OPTIONAL formalization (research: no measured
+ *                     machine-stage effect; Kiro/Spec-Kit require it, OpenAI/
+ *                     Anthropic guidance does not). Only scored when the doc
+ *                     is feature-shaped; excluded (weight redistributed) for
+ *                     research docs and non-feature capture kinds.
+ *
+ * Weights of the elements that DO apply to a given doc are renormalized to
+ * sum to 1, so `score` always lands in [0, 1] regardless of which elements
+ * were applicable. This is the scoring RUBRIC only — the pass bar
+ * (`SUFFICIENCY_THRESHOLD`) is a single overridable constant, not the
+ * versioned/calibrated bar the epic describes; that calibration is a
+ * separate follow-on task.
+ *
+ * Pure string/object logic, no network, no fs. `--self-test` covers every
+ * branch + a sufficient/insufficient/malformed fixture (run:
+ * `node dev-factory/lib/sufficiency-score.mjs --self-test`).
+ */
+
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { readFileSync, existsSync } from "node:fs";
+import { join, dirname } from "node:path";
+
+const HERE = dirname(fileURLToPath(import.meta.url));
+const THRESHOLD_PATH = join(HERE, "sufficiency-threshold.json");
+
+/**
+ * Read the versioned threshold from disk if it exists, else fallback to 0.7.
+ */
+function loadThreshold() {
+  try {
+    if (existsSync(THRESHOLD_PATH)) {
+      const raw = readFileSync(THRESHOLD_PATH, "utf8");
+      const parsed = JSON.parse(raw);
+      if (
+        parsed &&
+        typeof parsed.threshold === "number" &&
+        parsed.threshold >= 0 &&
+        parsed.threshold <= 1
+      ) {
+        return parsed.threshold;
+      }
+    }
+  } catch {
+    // ignore
+  }
+  return 0.7; // fallback
+}
+
+/** Overridable pass bar (may be read from versioned file). */
+export const SUFFICIENCY_THRESHOLD = loadThreshold();
+
+/** capture `kind:` default at creation — mirrors capture-vault.ts's `CAPTURE_KIND_DEFAULT` (kept in sync by hand; no cross-import across the .mjs/.ts boundary — see task report). */
+const CAPTURE_KIND_DEFAULT_VALUE = "idea";
+
+/** Capture `kind:` values that are explicitly NOT feature-shaped work (per entity-model.md capture vocabulary). */
+const NON_FEATURE_CAPTURE_KINDS = new Set([
+  "bug-fix",
+  "clip",
+  "reference",
+  "question",
+  "spec-import",
+]);
+
+/** Body text that signals a specific non-idea capture kind (bug-fix / clip / reference / question). */
+// Content → non-idea-kind signals, MIRRORED from vendor/heyspex-mcp/
+// capture-kind-suggest.mjs (guessCaptureKindFromText) — kept dependency-free
+// here (and in the hand-synced sufficiency-score.ts) rather than imported, per
+// the HS-3J5A builder brief. Only kinds with a reliable lexical tell are
+// signaled (bug-fix strong/weak+code-shape, clip = URL-only body, question =
+// single short question); reference has no reliable tell and is never flagged.
+const KIND_SIGNAL_BUG_FIX_RE =
+  /\b(stack ?trace|traceback|regression|silently (?:fails?|broke|breaks?)|doesn'?t work|does not work|no longer works|fails? to|null pointer|undefined is not a function|cannot read propert(?:y|ies) of (?:null|undefined)|this is (?:itself )?a bug|reproduc(?:e|es|ed|ible)|(?:type|reference|syntax)error\b|unhandled exception|threw an exception)\b/i;
+const KIND_SIGNAL_BUG_FIX_WEAK_RE = /\b(error|exception|broken|crash(?:es|ed|ing)?|bug)\b/i;
+const KIND_SIGNAL_CODE_SHAPE_RE =
+  /(\bat\s+\S+\(.*:\d+:\d+\)|\.(?:mjs|ts|tsx|js|jsx|cs|py):\d+|Traceback \(most recent call last\)|Exception in thread)/;
+const KIND_SIGNAL_URL_ONLY_RE = /^\s*<?(https?:\/\/\S+)>?\s*$/i;
+const KIND_SIGNAL_QUESTION_ONLY_RE =
+  /^\s*(?:why|how|what|when|where|who|which|should|is|are|does|do|can)\b[^\n]{0,240}\?\s*$/i;
+
+const ELEMENT_WEIGHTS = Object.freeze({
+  example: 0.3,
+  "core-idea": 0.3,
+  "done-criteria": 0.2,
+  kind: 0.1,
+  "user-stories": 0.1,
+});
+
+// Section-content detectors. Generic markdown-heading matches so the same
+// scorer works on a capture body (`# Core Idea`, `## Example`, `## User
+// Stories`) and a research doc body (`## Executive Summary`, `## Sources`).
+const CORE_IDEA_RE = /#[ \t]*Core[ \t]+Idea[ \t]*\n([\s\S]*?)(?=\n##?\s|$)/i;
+const EXEC_SUMMARY_RE =
+  /##[ \t]*Executive[ \t]+Summary[ \t]*\n([\s\S]*?)(?=\n##\s|$)/i;
+const EXAMPLE_RE = /##[ \t]*Example[ \t]*\n([\s\S]*?)(?=\n##\s|$)/i;
+const USER_STORIES_RE =
+  /##[ \t]*User[ \t]+Stories[ \t]*\n([\s\S]*?)(?=\n##\s|$)/i;
+// "Done When" is the research-recommended candidate heading; "Acceptance
+// criteria" / "Definition of Done" are the headings already live on tasks/epics.
+const DONE_CRITERIA_RE =
+  /##[ \t]*(?:Done[ \t]+When|Acceptance[ \t]+Criteria(?:\s*\([^)\n]*\))?|Definition[ \t]+of[ \t]+Done)[ \t]*\n([\s\S]*?)(?=\n##\s|$)/i;
+
+/** "As a &lt;role&gt;, I want …" — a user story with a real, named role. */
+const USER_STORY_WITH_ROLE_RE = /\bas\s+an?\s+\S+/i;
+/**
+ * A real (not AI-illustrative) Example is labeled by provenance
+ * (capture-guardrails §Example). Tolerates markdown emphasis around the label
+ * (`**Real example:**`, `_Real…_`, `**_Real…_**`) — kept in sync with
+ * capture-vault.ts / sufficiency-score.ts; part of the
+ * [[HS-CAIO-NR43-DIAG-promote-sufficiency-gate-demands-kind]] fix (a caller
+ * following the gate's own instruction to label real examples must not be
+ * marked insufficient for bolding it).
+ */
+const REAL_EXAMPLE_LABEL_RE = /^\s*[*_]*\s*real\b/i;
+/** At least one binary/checklist criterion — `- [ ]` / `- [x]` — not just a bare header. */
+const CHECKLIST_ITEM_RE = /^[ \t]*[-*][ \t]+\[[ xX]\]/m;
+
+function asString(value) {
+  return typeof value === "string" ? value : "";
+}
+
+function asRecord(value) {
+  return value && typeof value === "object" && !Array.isArray(value)
+    ? value
+    : {};
+}
+
+function matchSection(body, re) {
+  const m = asString(body).replace(/\r\n/g, "\n").match(re);
+  return m ? m[1].trim() : "";
+}
+
+/** Clear core idea: a `# Core Idea` section (capture) or `## Executive Summary` (research doc) with real content. */
+function hasClearCoreIdea(body) {
+  const core = matchSection(body, CORE_IDEA_RE);
+  if (core.length >= 3) return true;
+  const summary = matchSection(body, EXEC_SUMMARY_RE);
+  return summary.length >= 3;
+}
+
+/** Real example: an `## Example` section whose content carries the "Real …" provenance label. */
+function hasRealExample(body) {
+  const example = matchSection(body, EXAMPLE_RE);
+  return example.length > 0 && REAL_EXAMPLE_LABEL_RE.test(example);
+}
+
+/**
+ * Kind set: frontmatter.kind is present and not contradicted by the body.
+ * Explicitly storing `idea` is sufficient unless the content signals a
+ * specific non-idea kind (bug-fix / clip / reference / question) — in that
+ * case the kind metadata is inadequate and the element is marked missing.
+ */
+function hasAdequateKind(frontmatter, body) {
+  const kind = frontmatter.kind;
+  if (typeof kind !== "string" || kind.trim().length === 0) return false;
+  const normalized = kind.trim();
+  if (normalized === CAPTURE_KIND_DEFAULT_VALUE) {
+    return !signalsNonIdeaKind(body);
+  }
+  return true;
+}
+
+/** Does the body text signal a specific non-idea capture kind? Mirrors
+ *  guessCaptureKindFromText in capture-kind-suggest.mjs. */
+function signalsNonIdeaKind(body) {
+  const text = asString(body).trim();
+  if (!text) return false;
+  if (KIND_SIGNAL_BUG_FIX_RE.test(text)) return true;
+  if (KIND_SIGNAL_BUG_FIX_WEAK_RE.test(text) && KIND_SIGNAL_CODE_SHAPE_RE.test(text)) return true;
+  if (KIND_SIGNAL_URL_ONLY_RE.test(text)) return true;
+  if (KIND_SIGNAL_QUESTION_ONLY_RE.test(text) && !text.includes("\n\n")) return true;
+  return false;
+}
+
+/** Done-criteria: a Done-When/Acceptance-Criteria/Definition-of-Done section with at least one checklist item. */
+function hasDoneCriteria(body) {
+  const section = matchSection(body, DONE_CRITERIA_RE);
+  if (!section) return false;
+  return CHECKLIST_ITEM_RE.test(section) || section.length >= 10;
+}
+
+/** User stories with a real role: an `## User Stories` section naming at least one "As a …" actor. */
+function hasUserStoriesWithRole(body) {
+  const stories = matchSection(body, USER_STORIES_RE);
+  return stories.length > 0 && USER_STORY_WITH_ROLE_RE.test(stories);
+}
+
+/**
+ * Is this doc feature-shaped (does "user stories" even apply)? Research docs
+ * report findings, not a feature to build, so they're excluded. Capture kinds
+ * that are explicitly non-feature (bug-fix / clip / reference / question /
+ * spec-import) are excluded too. Everything else (default `idea`, tasks,
+ * epics, unset) is treated as feature-shaped by default.
+ */
+function isFeatureShaped(frontmatter) {
+  const type = asString(frontmatter.type);
+  if (type === "research") return false;
+  const kind = asString(frontmatter.kind).trim();
+  if (NON_FEATURE_CAPTURE_KINDS.has(kind)) return false;
+  return true;
+}
+
+/**
+ * Score a capture or research doc against the evidence-backed minimal-
+ * sufficient element set.
+ *
+ * @param {{ frontmatter?: Record<string, unknown>, body?: string }} doc
+ * @param {number} [threshold] overridable pass bar — defaults to SUFFICIENCY_THRESHOLD.
+ * @returns {{ score: number, missing: string[], pass: boolean }}
+ */
+export function scoreSufficiency(doc, threshold = SUFFICIENCY_THRESHOLD) {
+  const frontmatter = asRecord(doc && doc.frontmatter);
+  const body = asString(doc && doc.body);
+
+  const checks = {
+    "core-idea": hasClearCoreIdea(body),
+    example: hasRealExample(body),
+    kind: hasAdequateKind(frontmatter, body),
+    "done-criteria": hasDoneCriteria(body),
+  };
+  if (isFeatureShaped(frontmatter)) {
+    checks["user-stories"] = hasUserStoriesWithRole(body);
+  }
+
+  const applicable = Object.keys(checks);
+  const totalWeight =
+    applicable.reduce((sum, key) => sum + (ELEMENT_WEIGHTS[key] || 0), 0) || 1;
+
+  const missing = [];
+  let earned = 0;
+  for (const key of applicable) {
+    const normalizedWeight = (ELEMENT_WEIGHTS[key] || 0) / totalWeight;
+    if (checks[key]) {
+      earned += normalizedWeight;
+    } else {
+      missing.push(key);
+    }
+  }
+
+  const score = Math.max(0, Math.min(1, earned));
+  return { score, missing, pass: score >= threshold };
+}
+
+// ── Self-test (pure) ─────────────────────────────────────────────────────────────
+function runSelfTest() {
+  let ok = true;
+  const check = (name, cond) => {
+    console.log(`  ${cond ? "✓" : "✗"} ${name}`);
+    if (!cond) ok = false;
+  };
+  console.log("== lib/sufficiency-score.mjs self-test ==\n");
+
+  // AC1: exports scoreSufficiency.
+  check(
+    "exports scoreSufficiency as a function",
+    typeof scoreSufficiency === "function",
+  );
+
+  // Sufficient fixture: a capture with every element present.
+  const sufficientCapture = {
+    frontmatter: { type: "capture", kind: "bug-fix" },
+    body: [
+      "# Core Idea",
+      "",
+      "The export button silently no-ops on the second rapid click.",
+      "",
+      "## Example",
+      "",
+      "Real example: click Export twice fast on a 200-row report; the second click does nothing, no error, no toast.",
+      "",
+      "## User Stories",
+      "",
+      "As a support agent, I want duplicate exports blocked, so that tickets don't double-file.",
+      "",
+      "## Acceptance criteria (binary)",
+      "",
+      "- [ ] AC1: A second click within 500ms is a no-op.",
+      "- [ ] AC2: The button re-enables after the first export completes.",
+    ].join("\n"),
+  };
+  const sufficientResult = scoreSufficiency(sufficientCapture);
+  // AC2: score in [0,1] + missing array.
+  check(
+    "AC2: sufficient fixture score is in [0,1]",
+    sufficientResult.score >= 0 && sufficientResult.score <= 1,
+  );
+  check(
+    "AC2: sufficient fixture returns a missing array",
+    Array.isArray(sufficientResult.missing),
+  );
+  // AC3: fully-present doc passes with nothing missing.
+  check(
+    "AC3: sufficient fixture has score >= threshold",
+    sufficientResult.score >= SUFFICIENCY_THRESHOLD,
+  );
+  check(
+    "AC3: sufficient fixture pass === true",
+    sufficientResult.pass === true,
+  );
+  check(
+    "AC3: sufficient fixture has no missing elements",
+    sufficientResult.missing.length === 0,
+  );
+
+  // Insufficient fixture: default kind, no example, no user stories, no done-criteria.
+  const insufficientCapture = {
+    frontmatter: { type: "capture", kind: "idea" },
+    body: "# Core Idea\n\nSomething should be better.\n",
+  };
+  const insufficientResult = scoreSufficiency(insufficientCapture);
+  check(
+    "AC3: insufficient fixture pass === false",
+    insufficientResult.pass === false,
+  );
+  check(
+    "AC1: explicit idea kind is not missing when content doesn't signal another kind",
+    !insufficientResult.missing.includes("kind"),
+  );
+  check(
+    "AC3: insufficient fixture lists the missing elements",
+    ["example", "done-criteria", "user-stories"].every((k) =>
+      insufficientResult.missing.includes(k),
+    ),
+  );
+  check(
+    "AC3: insufficient fixture does NOT list core-idea (it's present)",
+    !insufficientResult.missing.includes("core-idea"),
+  );
+
+  // AC2: content signaling a non-idea kind while stored kind is idea fails the kind element.
+  const contradictoryKindCapture = {
+    frontmatter: { type: "capture", kind: "idea" },
+    body: "# Core Idea\n\nThe bug-fix needs a regression test.\n",
+  };
+  const contradictoryKindResult = scoreSufficiency(contradictoryKindCapture);
+  check(
+    "AC2: content signaling bug-fix with stored idea fails kind",
+    contradictoryKindResult.missing.includes("kind"),
+  );
+
+  // AC4: malformed / missing fields never throw.
+  let threwOnMalformed = false;
+  const malformedInputs = [
+    undefined,
+    null,
+    {},
+    { frontmatter: null, body: null },
+    { frontmatter: "not-an-object", body: 42 },
+    { frontmatter: [], body: [] },
+  ];
+  for (const input of malformedInputs) {
+    try {
+      const r = scoreSufficiency(input);
+      if (
+        typeof r.score !== "number" ||
+        !Array.isArray(r.missing) ||
+        typeof r.pass !== "boolean"
+      ) {
+        threwOnMalformed = true;
+      }
+    } catch {
+      threwOnMalformed = true;
+    }
+  }
+  check(
+    "AC4: malformed/missing fields never throw and still return a well-shaped result",
+    !threwOnMalformed,
+  );
+  check(
+    "AC4: a fully-empty doc is scored as insufficient, not crashed",
+    scoreSufficiency({}).pass === false,
+  );
+
+  // Research-doc shape: user-stories excluded (not feature-shaped), example/done-criteria still apply.
+  const researchDoc = {
+    frontmatter: { type: "research" },
+    body: [
+      "## Executive Summary",
+      "",
+      "External evidence and our own ablations converge on the same five-element answer.",
+      "",
+      "## Sources",
+      "",
+      "- [Some source](https://example.com)",
+    ].join("\n"),
+  };
+  const researchResult = scoreSufficiency(researchDoc);
+  check(
+    "research doc: user-stories excluded from missing (not feature-shaped)",
+    !researchResult.missing.includes("user-stories"),
+  );
+  check(
+    "research doc: core-idea satisfied via Executive Summary",
+    !researchResult.missing.includes("core-idea"),
+  );
+
+  // Threshold is a single named, overridable constant.
+  check(
+    "threshold is overridable via the second argument",
+    scoreSufficiency(insufficientCapture, 0).pass === true,
+  );
+  check(
+    "SUFFICIENCY_THRESHOLD is exported and in (0,1]",
+    SUFFICIENCY_THRESHOLD > 0 && SUFFICIENCY_THRESHOLD <= 1,
+  );
+
+  console.log(`\n${ok ? "✓ self-test PASS" : "✗ self-test FAIL"}`);
+  return ok ? 0 : 1;
+}
+
+if (
+  process.argv[1] &&
+  pathToFileURL(process.argv[1]).href === import.meta.url
+) {
+  if (process.argv.includes("--self-test")) process.exit(runSelfTest());
+}