@forwardimpact/libwiki 0.2.25 → 0.2.27

package/src/secret-gate.js

@@ -0,0 +1,177 @@
+/**
+ * Fail-closed secret gate for the wiki push path. Runs gitleaks over the
+ * commit range a push introduces and reports a clean / finding /
+ * scanner-absent verdict. The wiki has no destination-side secret control (a
+ * GitHub Wiki repo runs no Actions and is excluded from GitHub
+ * secret-scanning), so this is the only place a content backstop can live.
+ *
+ * The module never throws on a scanner result: a missing or erroring scanner
+ * resolves to `scanner-absent` so the caller fails closed rather than treating
+ * an error as clean. Findings carry only a location (`file:line:rule`) — never
+ * the matched secret value, so an audit record built from them cannot itself
+ * leak.
+ */
+
+import { isoTimestamp } from "@forwardimpact/libutil";
+import { createLogger } from "@forwardimpact/libtelemetry";
+
+/** The gitleaks binary name resolved on PATH; provisioning is an operator concern (see wiki-operations guide). */
+const GITLEAKS = "gitleaks";
+
+/**
+ * Scan the commit range a push introduces for secrets, fail closed.
+ *
+ * Probes `gitleaks version` first; an unresolvable binary short-circuits to
+ * `scanner-absent`. Then runs `gitleaks detect` over `range` expressed as
+ * `git log` options, reading the JSON report from stdout. Exit codes follow
+ * gitleaks' documented contract: `0` clean, `1` leaks found, any other
+ * non-zero an invocation error (treated as `scanner-absent` — fail closed, an
+ * error is never reported as clean).
+ *
+ * @param {object} args
+ * @param {import('@forwardimpact/libutil/runtime').Runtime} args.runtime - Provides `subprocess.run`.
+ * @param {string} args.wikiDir - The wiki clone directory to scan.
+ * @param {string} args.range - A `git log` range (e.g. `origin/master..HEAD`).
+ * @returns {Promise<{status: "clean"|"finding"|"scanner-absent", findings?: Array<{file: string, line: number, rule: string}>}>}
+ */
+export async function scanPushWindow({ runtime, wikiDir, range }) {
+  const probe = await runtime.subprocess.run(GITLEAKS, ["version"], {
+    cwd: wikiDir,
+  });
+  if (probe.exitCode !== 0) return { status: "scanner-absent" };
+
+  const scan = await runtime.subprocess.run(
+    GITLEAKS,
+    [
+      "detect",
+      "--source",
+      wikiDir,
+      "--log-opts",
+      range,
+      "--report-format",
+      "json",
+      "--report-path",
+      "-",
+    ],
+    { cwd: wikiDir },
+  );
+
+  if (scan.exitCode === 0) return { status: "clean" };
+  if (scan.exitCode === 1) {
+    return { status: "finding", findings: parseFindings(scan.stdout) };
+  }
+  // Any other non-zero is an invocation/usage error, not a leak verdict:
+  // fail closed rather than risk reporting a broken scan as clean.
+  return { status: "scanner-absent" };
+}
+
+/**
+ * Parse a gitleaks JSON report into location-only findings. Reads only the
+ * file, line, and rule of each entry — never the matched secret value — so a
+ * record built from the result is secret-free by construction. A malformed or
+ * empty report yields an empty list.
+ *
+ * @param {string} stdout - The gitleaks JSON report.
+ * @returns {Array<{file: string, line: number, rule: string}>}
+ */
+function parseFindings(stdout) {
+  let report;
+  try {
+    report = JSON.parse(stdout);
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(report)) return [];
+  return report.map((entry) => ({
+    file: entry.File ?? "",
+    line: entry.StartLine ?? 0,
+    rule: entry.RuleID ?? "",
+  }));
+}
+
+/**
+ * Append one secret-free line to the wiki tree's `secret-overrides.log` and
+ * stage it (path-scoped) so it lands in the same push as the overridden
+ * content. The line records the override as a durable, inspectable audit
+ * trail: an ISO timestamp, the asserted operator identity (`git config
+ * user.email` — attribution of intent, NOT an authenticated identity), the
+ * override class, the reason, and for a finding its location. It never carries
+ * a matched secret value.
+ *
+ * @param {object} args
+ * @param {import('@forwardimpact/libutil/runtime').Runtime} args.runtime - Provides `fs` and `clock`.
+ * @param {import('@forwardimpact/libutil').GitClient} args.gitClient - Stages the log into the push.
+ * @param {string} args.wikiDir - The wiki clone directory.
+ * @param {"finding"|"scanner-absent"} args.klass - The override class.
+ * @param {string} args.reason - The operator-supplied reason for the override.
+ * @param {Array<{file: string, line: number, rule: string}>} [args.findings] - Locations for a finding override.
+ * @returns {Promise<{path: string}>} The relative path staged into the push.
+ */
+export async function appendOverrideRecord({
+  runtime,
+  gitClient,
+  wikiDir,
+  klass,
+  reason,
+  findings = [],
+}) {
+  const email =
+    (await gitClient.configGet("user.email", { cwd: wikiDir })) || "unknown";
+  const where =
+    klass === "finding"
+      ? findings.map((f) => `${f.file}:${f.line}:${f.rule}`).join(",") ||
+        "unspecified"
+      : "scanner-absent";
+  const ts = isoTimestamp(runtime.clock.now());
+  // Tab-separated, single line; the reason is collapsed so the record stays
+  // one inspectable row per override.
+  const line = `${ts}\t${email}\t${klass}\t${reason.replace(/\s+/g, " ").trim()}\t${where}\n`;
+  const logPath = `${wikiDir}/${OVERRIDE_LOG}`;
+  await runtime.fs.appendFile(logPath, line);
+  await gitClient.commitPaths(
+    `wiki: secret-gate override (${klass})`,
+    [OVERRIDE_LOG],
+    { cwd: wikiDir },
+  );
+  return { path: OVERRIDE_LOG };
+}
+
+/** The append-only audit log of break-glass overrides, in the wiki tree root. */
+export const OVERRIDE_LOG = "secret-overrides.log";
+
+/**
+ * Translate a `commitAndPush` security refusal into a command envelope,
+ * logging the cause and its break-glass procedure at error level (always
+ * surfaced, regardless of LOG_LEVEL). Returns `null` for any non-refusal
+ * result (clean / pushed / network "saved locally"), so a caller can fall
+ * through to its normal success handling. Shared by every command surface so
+ * the refusal message and exit code live in one place.
+ *
+ * @param {object} runtime - The runtime bag (the logger writes to `proc.stderr`).
+ * @param {{reason?: string, findings?: Array<{file: string, line: number, rule: string}>}} result - A `commitAndPush` result.
+ * @returns {{ok: false, code: 1}|null}
+ */
+export function refusalEnvelope(runtime, result) {
+  if (result.reason === "secret-detected") {
+    const where = (result.findings ?? [])
+      .map((f) => `${f.file}:${f.line}:${f.rule}`)
+      .join(", ");
+    createLogger("wiki", runtime).error(
+      "push",
+      `push blocked: secret detected in wiki content${where ? ` (${where})` : ""}; ` +
+        "the push was not attempted. After confirming a false positive, set " +
+        "FIT_WIKI_SECRET_OVERRIDE to a reason to override (audited).",
+    );
+    return { ok: false, code: 1 };
+  }
+  if (result.reason === "scanner-unavailable") {
+    createLogger("wiki", runtime).error(
+      "push",
+      "push blocked: the secret scanner (gitleaks) is unavailable; the push " +
+        "was not attempted. Install gitleaks, or set FIT_WIKI_SCANNER_ABSENT_OK " +
+        "to a reason to override (audited).",
+    );
+    return { ok: false, code: 1 };
+  }
+  return null;
+}

package/src/status.js

@@ -1,19 +1,46 @@
-// STATUS.md row ids may carry a `/<unit>` suffix denoting a
-// per-migration-unit sub-row of a master spec (`1370/libutil`, …). The master
-// `NNNN` row advances only when every sub-row reads `plan implemented`.
+// STATUS.md rows come in two kinds. A spec row's id is four digits with an
+// optional `/<unit>` suffix denoting a per-migration-unit sub-row of a master
+// spec (`1370/libutil`, …); the master `NNNN` row advances only when every
+// sub-row reads `plan implemented`. An experiment row's id is `exp:<issue>`
+// and the row carries four tab cells — `exp:<issue><TAB><state><TAB><pin>
+// <TAB><plan-ref>` — keying the merge-gate approval path for a spec-less
+// experiment PR. The `exp:` namespace cannot match the spec id's `^\d{4}`
+// anchor, so the two kinds never collide for any issue-number width.
 
-/** Matches a status-row id: four digits, optionally a `/<unit>` suffix. */
-export const STATUS_ID_REGEX = /^\d{4}(\/[a-z0-9-]+)?$/;
+/** Matches a status-row id: a four-digit spec id (optional `/<unit>`) or `exp:<issue>`. */
+export const STATUS_ID_REGEX = /^(\d{4}(\/[a-z0-9-]+)?|exp:\d+)$/;
 
 /**
- * Parse a status-row id into its master spec id and optional unit suffix.
- * @param {string} id - The id field of a STATUS.md row.
- * @returns {{ specId: string, unit: string|null }|null} Parsed parts, or null
- *   when the id does not match {@link STATUS_ID_REGEX}.
+ * Classify a status-row id into its kind and parts. Experiment rows are
+ * identified by an `exp:` id together with a four-cell row; the optional
+ * `cells` array supplies that count (a bare `exp:` id without four cells is
+ * not a valid row and yields null).
+ * @param {string} id - The id field (cell 0) of a STATUS.md row.
+ * @param {string[]} [cells] - The full tab-separated cells of the row, when
+ *   available. Required to classify an experiment row.
+ * @returns {(
+ *   {kind: "spec", specId: string, unit: string|null} |
+ *   {kind: "experiment", issue: string, state: string, pin: string, planRef: string} |
+ *   null
+ * )} Parsed parts, or null when the id/row does not match a known kind.
  */
-export function parseStatusRowId(id) {
+export function parseStatusRowId(id, cells) {
   if (typeof id !== "string" || !STATUS_ID_REGEX.test(id)) return null;
+  if (id.startsWith("exp:")) {
+    if (!Array.isArray(cells) || cells.length !== 4) return null;
+    return {
+      kind: "experiment",
+      issue: id.slice("exp:".length),
+      state: cells[1],
+      pin: cells[2],
+      planRef: cells[3],
+    };
+  }
   const slash = id.indexOf("/");
-  if (slash === -1) return { specId: id, unit: null };
-  return { specId: id.slice(0, slash), unit: id.slice(slash + 1) };
+  if (slash === -1) return { kind: "spec", specId: id, unit: null };
+  return {
+    kind: "spec",
+    specId: id.slice(0, slash),
+    unit: id.slice(slash + 1),
+  };
 }

package/src/util/agent-flag.js

@@ -0,0 +1,31 @@
+/**
+ * Resolve the required agent flag from frozen CLI options. Pure — reads no
+ * filesystem and no environment, so it runs before any state change. Returns
+ * `{ ok: true, agent }` when the flag is present, or
+ * `{ ok: false, code: 2, error }` when it is missing, where `error` names the
+ * missing flag and shows a corrected example invocation. The error never
+ * mentions an environment variable: `libwiki` carries no ambient agent
+ * identity, so there is no fallback to offer.
+ *
+ * @param {Record<string, unknown>} options - The frozen `ctx.options`.
+ * @param {{ command: string, flag?: string, example: string }} spec
+ *   `command` names the failing subcommand; `flag` is the option key prefix
+ *   (`--agent` by default, `--from` for `memo`); `example` is a correct
+ *   invocation shown verbatim in the error.
+ * @returns {{ ok: true, agent: string } | { ok: false, code: 2, error: string }}
+ */
+export function requireAgentFlag(
+  options,
+  { command, flag = "--agent", example },
+) {
+  const key = flag === "--from" ? "from" : "agent";
+  const agent = options[key];
+  if (!agent) {
+    return {
+      ok: false,
+      code: 2,
+      error: `${command} requires ${flag} <name>; e.g. ${example}`,
+    };
+  }
+  return { ok: true, agent };
+}

package/src/weekly-log.js

@@ -4,9 +4,14 @@ import { countLines, countWords } from "./budget.js";
 import {
   WEEKLY_LOG_LINE_BUDGET,
   WEEKLY_LOG_PART_NAME_RE,
+  WEEKLY_LOG_SEAM_RE,
   WEEKLY_LOG_WORD_BUDGET,
 } from "./constants.js";
 
+// Block seam inside a day-section: a `### ` heading at line start. The finer
+// grain rotation falls to when a lone day-section alone exceeds a budget.
+const BLOCK_SEAM_RE = /^### /;
+
 // ISO week computation lives in libutil's calendar util (the one place a
 // `new Date` is allowed); re-exported here for the existing public surface.
 export { isoWeek } from "@forwardimpact/libutil";
@@ -48,10 +53,11 @@ function defaultH1(agent, isoWeekStr) {
   return `# ${agentTitle(agent)} — ${isoWeekStr}\n`;
 }
 
-/** Describe a lone over-cap day-section as a residue at its part index. */
+/** Describe a lone over-cap section as a residue at its part index. The
+ * section label is a day date (`## ` seam) or a block heading (`### ` seam). */
 function residueOf(sec, partIndex, measure) {
   const { lines, words } = measure(sec.text);
-  return { section: sec.date, lines, words, partIndex };
+  return { section: sec.label, lines, words, partIndex };
 }
 
 /**
@@ -93,7 +99,7 @@ function packSections(sections, prologue, budget) {
   };
   for (const sec of sections) {
     if (overBudget(sec.text)) {
-      // Irreducible lone day-section: flush the open part, then seal it alone.
+      // Irreducible lone section: flush the open part, then seal it alone.
       flush();
       residue ??= residueOf(sec, partBodies.length, measure);
       partBodies.push(sec.text);
@@ -111,6 +117,37 @@ function packSections(sections, prologue, budget) {
   return { partBodies, residue };
 }
 
+/**
+ * Slice `body` at `seamRe` seam offsets into `{label, text}` sections, the
+ * label being the seam's full matched heading line (date or `### ` heading).
+ * Returns `{prologue, sections}` where the prologue is everything above the
+ * first seam (the whole body when there are no seams). Concatenating the
+ * prologue and every section's text reproduces `body` byte-for-byte.
+ */
+function findSections(body, seamRe) {
+  const re = new RegExp(seamRe.source, "gm");
+  const seams = [];
+  let match;
+  while ((match = re.exec(body)) !== null) {
+    const eol = body.indexOf("\n", match.index);
+    const headingLine = body.slice(match.index, eol === -1 ? body.length : eol);
+    // A captured group (the day seam's date) labels the section; otherwise the
+    // full heading line (a `### ` block heading) is the label.
+    const label = match[1] ?? headingLine;
+    seams.push({ offset: match.index, label });
+  }
+  if (seams.length === 0) return { prologue: body, sections: [] };
+  const prologue = body.slice(0, seams[0].offset);
+  const sections = seams.map((s, i) => ({
+    label: s.label,
+    text: body.slice(
+      s.offset,
+      i + 1 < seams.length ? seams[i + 1].offset : body.length,
+    ),
+  }));
+  return { prologue, sections };
+}
+
 /**
  * Split an over-budget weekly-log source at its `## YYYY-MM-DD` day-section
  * seams into an ordered list of conforming parts. Pure — no I/O.
@@ -123,9 +160,13 @@ function packSections(sections, prologue, budget) {
  * are greedily packed left-to-right under both the line- and word-budget, with
  * each candidate part measured H1-included so its own H1 is charged.
  *
- * When a single chunk alone exceeds a budget — a lone day-section, or the
- * whole prologue when the source has no day-sections — it is sealed as its own
- * (over-budget) part and named in `residue`; the rest still packs normally.
+ * When a lone day-section alone exceeds a budget, it is re-bisected at its
+ * `### ` block seams (one grain finer) and the resulting block-parts replace it,
+ * so a single over-cap day no longer forces a hand-split. Only a single `### `
+ * block that alone exceeds a budget — or an over-cap seamless prologue — remains
+ * an irreducible residue: it is sealed as its own (over-budget) part and named
+ * in `residue` (the residue's `section` then names the block heading, not a
+ * date); the rest still packs normally.
  *
  * @param {string} text - The full weekly-log source (H1 + body).
  * @param {string} agent - Agent profile id (e.g. "staff-engineer").
@@ -157,37 +198,68 @@ export function bisectWeeklyLog(text, agent, isoWeekStr) {
     };
   };
 
+  const budget = { overBudget, measure };
   // Locate the day-section seams (date at line-start, trailing suffix
   // tolerated, e.g. `## 2026-05-19 (third activation)`).
-  const seamRe = /^## (\d{4}-\d{2}-\d{2})/gm;
-  const seams = [];
-  let match;
-  while ((match = seamRe.exec(body)) !== null) {
-    seams.push({ offset: match.index, date: match[1] });
-  }
+  const { prologue, sections } = findSections(body, WEEKLY_LOG_SEAM_RE);
 
   // Zero day-sections: the whole body is the prologue and its own single part,
   // flagged as a residue when it alone exceeds a budget.
-  if (seams.length === 0) {
-    return finish([body], prologueResidue(body, { overBudget, measure }));
+  if (sections.length === 0) {
+    return finish([body], prologueResidue(body, budget));
   }
 
-  const prologue = body.slice(0, seams[0].offset);
-  const sections = seams.map((s, i) => ({
-    date: s.date,
-    text: body.slice(
-      s.offset,
-      i + 1 < seams.length ? seams[i + 1].offset : body.length,
-    ),
-  }));
-
-  const { partBodies, residue } = packSections(sections, prologue, {
-    overBudget,
-    measure,
-  });
+  const { partBodies, residue } = packSections(sections, prologue, budget);
+  // A lone day-section that alone exceeds a budget is the only residue
+  // `packSections` can produce here (the prologue rides part 1). Re-bisect that
+  // day at its `### ` block seams and splice the block-parts in for it; the
+  // surfaced residue becomes the inner one (an over-cap block) or null.
+  if (residue !== null) {
+    const sub = resplitDaySection(partBodies, residue, budget);
+    return finish(sub.partBodies, sub.residue);
+  }
   return finish(partBodies, residue);
 }
 
+/**
+ * Re-bisect the lone over-cap day-section that `packSections` flagged as the
+ * residue: split that part's body at its `### ` block seams and splice the
+ * resulting block-bodies into `partBodies` in its place. The day-section's body
+ * carries its own `## ` heading as a prologue above the first `### ` block, so
+ * that heading rides the first block-part. Returns the spliced `partBodies` and
+ * the inner residue (a single over-cap `### ` block, or null when every block
+ * now conforms), re-indexed to its position in the spliced list.
+ */
+function resplitDaySection(partBodies, residue, budget) {
+  const dayBody = partBodies[residue.partIndex];
+  // Only a day-section (its body opens with a `## ` heading) is re-split at the
+  // block grain; an over-cap seamless prologue has no day to descend into and
+  // stays the terminal residue.
+  if (!dayBody.startsWith("## ")) return { partBodies, residue };
+  const { prologue, sections } = findSections(dayBody, BLOCK_SEAM_RE);
+  // No `### ` block seam inside the day: nothing finer to cut — keep the day as
+  // the irreducible residue (criterion 3's terminal case at the day grain).
+  if (sections.length === 0) return { partBodies, residue };
+  const { partBodies: blockBodies, residue: blockResidue } = packSections(
+    sections,
+    prologue,
+    budget,
+  );
+  const spliced = [
+    ...partBodies.slice(0, residue.partIndex),
+    ...blockBodies,
+    ...partBodies.slice(residue.partIndex + 1),
+  ];
+  if (blockResidue === null) return { partBodies: spliced, residue: null };
+  return {
+    partBodies: spliced,
+    residue: {
+      ...blockResidue,
+      partIndex: residue.partIndex + blockResidue.partIndex,
+    },
+  };
+}
+
 /**
  * Stage every write to `${path}.tmp`, then commit by renaming each `leading`
  * write onto its path (tracked for rollback) and the `anchor` write LAST — the
@@ -261,11 +333,20 @@ function atomicSeal(filePath, parts, agent, isoWeekStr, fs) {
  * into one-or-more conforming parts), or `{status:"incomplete",parts,residue}`
  * (a lone day-section exceeds a budget and is named).
  *
- * @returns {{status: "noop"|"sealed"|"incomplete", fromPath: string, parts?: string[], residue?: {path: string, section: string, lines: number, words: number}}}
+ * A `noop` return carries a `reason` — `"missing"` (no file; no size measured),
+ * `"floor"` (header-only/empty body; nothing to seal), or `"under-budget"`
+ * (under both budgets without `--force`) — plus the measured `lines`/`words`
+ * for the two reasons that read the file, so the CLI guard need not re-read it.
+ * "Over budget" is decided here over *either* budget (lines or words), so a
+ * caller never needs `force: true` to seal a word-over/line-under log.
+ *
+ * @returns {{status: "noop"|"sealed"|"incomplete", reason?: "missing"|"floor"|"under-budget", lines?: number, words?: number, fromPath: string, parts?: string[], residue?: {path: string, section: string, lines: number, words: number}}}
  * @param {string} wikiRoot
  * @param {string} agent
  * @param {string} today - ISO date string.
- * @param {number} [appendLines=0]
+ * @param {{lines?: number, words?: number}} [delta={}] - Projected post-append
+ *   line/word delta; the trigger fires when the current file plus this delta
+ *   would breach either budget. Force-rotate callers pass `{}`.
  * @param {{force?: boolean}} [options]
  * @param {object} fs - Sync filesystem surface (`runtime.fsSync`).
  */
@@ -273,24 +354,52 @@ export function rotateIfOverBudget(
   wikiRoot,
   agent,
   today,
-  appendLines = 0,
+  delta = {},
   options = {},
   fs,
 ) {
   const filePath = weeklyLogPath(wikiRoot, agent, today);
   const { force = false } = options;
-  if (!fs.existsSync(filePath)) return { status: "noop", fromPath: filePath };
+  if (!fs.existsSync(filePath)) {
+    return { status: "noop", reason: "missing", fromPath: filePath };
+  }
   const text = fs.readFileSync(filePath, "utf-8");
+  const lines = countLines(text);
+  const words = countWords(text);
   // A header-only (or empty) log has nothing to seal. Without this floor,
   // force-rotating a freshly-reset main would mint an empty `(part 1 of 1)`
-  // file and reset the main again — once per invocation, forever.
+  // file and reset the main again — once per invocation, forever. The floor
+  // holds even under `--force`, so it is checked before the force branch.
   const nl = text.indexOf("\n");
   if ((nl === -1 ? "" : text.slice(nl + 1)).trim() === "") {
-    return { status: "noop", fromPath: filePath };
+    return {
+      status: "noop",
+      reason: "floor",
+      lines,
+      words,
+      fromPath: filePath,
+    };
   }
-  const current = countLines(text);
-  if (!force && current + appendLines <= WEEKLY_LOG_LINE_BUDGET) {
-    return { status: "noop", fromPath: filePath };
+  // Over either budget (lines or words), decided once here in core: a
+  // word-over/line-under log seals without `--force`. The projection folds in
+  // the caller's append delta so a pre-append rotate fires on the post-append
+  // size. The `noop`/`under-budget` arm carries the measured size so the CLI
+  // handler can report the resolved target without re-reading the file.
+  const { lines: dLines = 0, words: dWords = 0 } = delta;
+  const projectedLines = lines + dLines;
+  const projectedWords = words + dWords;
+  if (
+    !force &&
+    projectedLines <= WEEKLY_LOG_LINE_BUDGET &&
+    projectedWords <= WEEKLY_LOG_WORD_BUDGET
+  ) {
+    return {
+      status: "noop",
+      reason: "under-budget",
+      lines,
+      words,
+      fromPath: filePath,
+    };
   }
   const isoWeekStr = isoWeekString(today);
   const { parts, residue } = bisectWeeklyLog(text, agent, isoWeekStr);
@@ -388,7 +497,7 @@ export function rebisectOverBudgetPart(partPath, fs) {
   // surface a residue (synthesised from the file when the bisector did not name
   // one) so the caller's re-audit re-flags it.
   if (parts.length === 1) {
-    const seam = text.match(/^## (\d{4}-\d{2}-\d{2})/m);
+    const seam = text.match(new RegExp(WEEKLY_LOG_SEAM_RE.source, "m"));
     const r = residue ?? {
       section: seam ? seam[1] : "prologue",
       lines,