@forwardimpact/libwiki 0.2.25 → 0.2.26

package/src/integrity.js

@@ -0,0 +1,288 @@
+import { isoTimestamp } from "@forwardimpact/libutil";
+import { SESSION_GAP_MS } from "./constants.js";
+import { isLaneFile, enumerateLaneFiles } from "./lane-files.js";
+
+/**
+ * Normalize a content line for content-keyed presence: strip a trailing CR and
+ * trailing whitespace. Blank lines normalize to "" and are dropped by callers.
+ * @param {string} line
+ * @returns {string}
+ */
+export function normLine(line) {
+  return line.replace(/\r$/, "").replace(/\s+$/, "");
+}
+
+/**
+ * Parse `git diff --unified=0` text into per-file change records, attributing
+ * each `+`/`-` line to the file named by the most recent `+++ b/<path>` header.
+ * The `+++`/`---`/`@@` framing lines are not content. `/dev/null` targets
+ * (pure deletions) are kept as the home for their removed lines.
+ * @param {string} diffText
+ * @returns {Array<{home: string, added: string[], removed: string[]}>}
+ */
+export function parseDiff(diffText) {
+  const byHome = new Map();
+  let home = null;
+  for (const raw of diffText.split("\n")) {
+    if (raw.startsWith("+++ ")) {
+      home = stripDiffTarget(raw.slice(4));
+      continue;
+    }
+    if (home === null || isDiffFraming(raw)) continue;
+    const rec = byHome.get(home) ?? { home, added: [], removed: [] };
+    if (raw.startsWith("+")) rec.added.push(raw.slice(1));
+    else if (raw.startsWith("-")) rec.removed.push(raw.slice(1));
+    else continue;
+    byHome.set(home, rec);
+  }
+  return [...byHome.values()];
+}
+
+/** Whether a diff line is framing (`---`, `@@`, `diff`, `index`) rather than content. */
+function isDiffFraming(raw) {
+  return (
+    raw.startsWith("--- ") ||
+    raw.startsWith("@@") ||
+    raw.startsWith("diff ") ||
+    raw.startsWith("index ")
+  );
+}
+
+/** Strip a diff target's `a/`/`b/` prefix; `/dev/null` stays as-is. */
+function stripDiffTarget(target) {
+  const t = target.trim();
+  if (t === "/dev/null") return t;
+  return t.replace(/^[ab]\//, "");
+}
+
+/**
+ * Compose a window of change records (oldest→newest) into the surviving
+ * addition assertions, then return those absent from the tip. Content-keyed on
+ * `norm(line)`: a later own-lane deletion of an earlier-added line cancels its
+ * assertion (additions-only, own-deletions cancel). A surviving assertion is
+ * present iff its normalized line appears anywhere in `tipText`.
+ *
+ * @param {Array<{home: string, added: string[], removed: string[]}>} changes
+ *   Window changes, oldest→newest.
+ * @param {string} tipText - Concatenated text of the tip's in-scope files.
+ * @param {(line: string) => string} norm - Line normalizer.
+ * @returns {Array<{contentId: string, pushHome: string}>} Absent assertions.
+ */
+export function findAbsent(changes, tipText, norm) {
+  const asserted = composeAssertions(changes, norm);
+  const present = normalizedKeySet(tipText, norm);
+  const absent = [];
+  for (const [key, a] of asserted) {
+    if (!present.has(key)) absent.push(a);
+  }
+  return absent;
+}
+
+/** Compose window changes into surviving additions; a later removal cancels its key. */
+function composeAssertions(changes, norm) {
+  const asserted = new Map(); // norm(line) -> { contentId, pushHome }
+  for (const change of changes) {
+    for (const line of change.added) {
+      const key = norm(line);
+      if (key !== "" && !asserted.has(key)) {
+        asserted.set(key, { contentId: key, pushHome: change.home });
+      }
+    }
+    for (const line of change.removed) {
+      const key = norm(line);
+      if (key !== "") asserted.delete(key);
+    }
+  }
+  return asserted;
+}
+
+/** The set of non-blank normalized lines in `text`. */
+function normalizedKeySet(text, norm) {
+  const set = new Set();
+  for (const line of text.split("\n")) {
+    const key = norm(line);
+    if (key !== "") set.add(key);
+  }
+  return set;
+}
+
+/**
+ * Resolve the previous-session push set from lane-authored commits (newest
+ * first) by idle-gap. Tier 2 runs at boot before the current session has
+ * pushed, so the most recent contiguous run is the previous session. Vacuous
+ * only for empty history; the degenerate (content-unresolvable) case is raised
+ * by {@link sweepTier2}, not here.
+ *
+ * @param {Array<{sha: string, when: number}>} commits - Newest first.
+ * @param {number} gapMs - Idle-gap threshold (ms).
+ * @returns {{kind: "vacuous"} | {kind: "window", commits: Array<{sha: string, when: number}>}}
+ */
+export function previousSessionWindow(commits, gapMs) {
+  if (commits.length === 0) return { kind: "vacuous" };
+  const tipRun = [commits[0]];
+  for (let i = 1; i < commits.length; i++) {
+    // commits are newest-first; `when` is seconds, gap threshold is ms.
+    const gap = (commits[i - 1].when - commits[i].when) * 1000;
+    if (gap > gapMs) break;
+    tipRun.push(commits[i]);
+  }
+  return { kind: "window", commits: tipRun };
+}
+
+/**
+ * Build a detection record. `detectedAt` is the binding wall-clock stamp (ISO);
+ * an exposure figure derived from commit timestamps carries the labeled
+ * `commit-timestamp` fallback basis.
+ *
+ * @param {object} d
+ * @param {1|2} d.tier
+ * @param {string} d.contentId - The absent content's identity (normalized line).
+ * @param {string} d.pushHome - The content's push-time home path.
+ * @param {number|Date} d.now - Wall-clock (e.g. `runtime.clock.now()` ms).
+ * @param {number} [d.exposureSeconds] - Commit-timestamp-derived exposure.
+ * @returns {object}
+ */
+export function makeDetection({
+  tier,
+  contentId,
+  pushHome,
+  now,
+  exposureSeconds,
+}) {
+  const detection = {
+    tier,
+    contentId,
+    pushHome,
+    detectedAt: isoTimestamp(now),
+  };
+  if (exposureSeconds != null) {
+    detection.exposure = {
+      seconds: exposureSeconds,
+      basis: "commit-timestamp",
+    };
+  }
+  return detection;
+}
+
+/**
+ * Render detections to flow output text. Empty input renders the empty string
+ * (clean-path silence). One line per detection naming tier, push-time home, and
+ * content identity; exposure (when present) is labeled with its fallback basis.
+ * @param {object[]} detections
+ * @returns {string}
+ */
+export function renderDetections(detections) {
+  if (detections.length === 0) return "";
+  return (
+    detections
+      .map((d) => {
+        const exposure = d.exposure
+          ? ` exposure=${d.exposure.seconds}s (basis: ${d.exposure.basis})`
+          : "";
+        return `integrity[tier ${d.tier}]: absent content in ${d.pushHome} — "${d.contentId}" detected ${d.detectedAt}${exposure}`;
+      })
+      .join("\n") + "\n"
+  );
+}
+
+/**
+ * Tier-2 boot sweep: verify the lane's previous-session push set is still
+ * content-present at the fetched (rebased) origin tip, returning detections for
+ * any absence. Reads git and lane files from `wikiDir` (the rebased tree).
+ * Never writes, never throws into the flow (the caller wraps it).
+ *
+ * @param {object} ctx
+ * @param {import('@forwardimpact/libutil/runtime').Runtime} ctx.runtime
+ * @param {import('@forwardimpact/libutil').GitClient} ctx.gitClient
+ * @param {string} ctx.wikiDir - The wiki clone dir (git cwd and fs read-root).
+ * @param {string} ctx.agent - Lane agent id.
+ * @param {number} ctx.now - Wall-clock (ms).
+ * @returns {Promise<object[]>} Detections (possibly empty).
+ */
+export async function sweepTier2({ runtime, gitClient, wikiDir, agent, now }) {
+  const email = await gitClient.configGet("user.email", { cwd: wikiDir });
+  if (!email) {
+    // Lane identity unresolvable — never a silent vacuous pass.
+    return [
+      makeDetection({
+        tier: 2,
+        contentId: "<unresolvable: no author identity>",
+        pushHome: "-",
+        now,
+      }),
+    ];
+  }
+  const commits = await gitClient.logByAuthor(email, {
+    cwd: wikiDir,
+    ref: "HEAD",
+  });
+  const window = previousSessionWindow(commits, SESSION_GAP_MS);
+  if (window.kind === "vacuous") return [];
+
+  const detections = [];
+  const changes = [];
+  // Oldest→newest so own-deletion cancellation composes in commit order.
+  const ordered = [...window.commits].reverse();
+  for (const commit of ordered) {
+    const diff = await gitClient.diffRange(`${commit.sha}~1 ${commit.sha}`, {
+      cwd: wikiDir,
+    });
+    if (diff === null) {
+      detections.push(
+        makeDetection({
+          tier: 2,
+          contentId: `<unresolvable content: ${commit.sha}>`,
+          pushHome: "-",
+          now,
+        }),
+      );
+      continue;
+    }
+    for (const rec of parseDiff(diff)) {
+      if (isLaneFile(rec.home, agent))
+        changes.push({ ...rec, when: commit.when });
+    }
+  }
+
+  const tipText = enumerateLaneFiles(wikiDir, agent, runtime.fsSync)
+    .map((rel) => readFileOrEmpty(runtime.fsSync, wikiDir, rel))
+    .join("\n");
+
+  for (const absent of findAbsent(changes, tipText, normLine)) {
+    // Exposure runs from the LAST window commit that added this exact line
+    // (its most recent assertion at origin), not merely a same-home commit.
+    const when = lastAssertionTime(changes, absent.contentId);
+    const exposureSeconds =
+      when != null ? Math.round(now / 1000 - when) : undefined;
+    detections.push(
+      makeDetection({
+        tier: 2,
+        contentId: absent.contentId,
+        pushHome: absent.pushHome,
+        now,
+        exposureSeconds,
+      }),
+    );
+  }
+  return detections;
+}
+
+/** The `when` of the latest window change whose normalized additions include `contentId`. */
+function lastAssertionTime(changes, contentId) {
+  let when;
+  for (const change of changes) {
+    if (change.added.some((line) => normLine(line) === contentId)) {
+      when = change.when;
+    }
+  }
+  return when;
+}
+
+function readFileOrEmpty(fsSync, wikiDir, rel) {
+  const abs = `${wikiDir}/${rel}`;
+  try {
+    return fsSync.readFileSync(abs, "utf-8");
+  } catch {
+    return "";
+  }
+}

package/src/lane-files.js

@@ -0,0 +1,62 @@
+import path from "node:path";
+import { WEEKLY_LOG_NAME_RE, WEEKLY_LOG_PART_NAME_RE } from "./constants.js";
+
+const METRICS_CSV_RE = /^metrics\/[^/]+\/\d{4}\.csv$/;
+
+/**
+ * Whether a wiki-root-relative path is one of the lane's own files: the
+ * agent's summary (`<agent>.md`), a weekly log or sealed part
+ * (`<agent>-YYYY-Www.md`, `<agent>-YYYY-Www-partN.md`, matched on the captured
+ * agent token), or a metrics CSV (`metrics/<skill>/<year>.csv`). Metrics CSVs
+ * match by path for every agent; lane ownership of a metrics CSV is enforced by
+ * the tier-2 sweep's author filter at the commit level, not here.
+ *
+ * @param {string} relPath - Path relative to the wiki root (POSIX or native).
+ * @param {string} agent - Agent profile id (e.g. "staff-engineer").
+ * @returns {boolean}
+ */
+export function isLaneFile(relPath, agent) {
+  const rel = relPath.replace(/\\/g, "/");
+  const base = path.posix.basename(rel);
+  if (base === `${agent}.md`) return true;
+  for (const re of [WEEKLY_LOG_NAME_RE, WEEKLY_LOG_PART_NAME_RE]) {
+    const m = base.match(re);
+    if (m && m[1] === agent) return true;
+  }
+  return METRICS_CSV_RE.test(rel);
+}
+
+/**
+ * Enumerate the lane's own files present under `wikiRoot`: matching top-level
+ * summary and weekly-log files, plus every `metrics/<skill>/<year>.csv`.
+ * Returns wiki-root-relative POSIX paths.
+ *
+ * @param {string} wikiRoot
+ * @param {string} agent
+ * @param {object} fsSync - Sync filesystem surface (`runtime.fsSync`).
+ * @returns {string[]} Relative paths, in directory-read order.
+ */
+export function enumerateLaneFiles(wikiRoot, agent, fsSync) {
+  const out = [];
+  for (const entry of fsSync.readdirSync(wikiRoot)) {
+    if (entry !== "metrics" && isLaneFile(entry, agent)) out.push(entry);
+  }
+  out.push(...enumerateMetricsCsvs(wikiRoot, agent, fsSync));
+  return out;
+}
+
+/** Wiki-root-relative `metrics/<skill>/<year>.csv` paths matching the lane. */
+function enumerateMetricsCsvs(wikiRoot, agent, fsSync) {
+  const metricsDir = path.join(wikiRoot, "metrics");
+  if (!fsSync.existsSync(metricsDir)) return [];
+  const found = [];
+  for (const skill of fsSync.readdirSync(metricsDir)) {
+    const skillDir = path.join(metricsDir, skill);
+    if (!fsSync.statSync(skillDir).isDirectory()) continue;
+    for (const file of fsSync.readdirSync(skillDir)) {
+      const rel = `metrics/${skill}/${file}`;
+      if (isLaneFile(rel, agent)) found.push(rel);
+    }
+  }
+  return found;
+}

package/src/marker-scanner.js

@@ -20,6 +20,7 @@ function tryOpen(line, i) {
       kind: "xmr",
       metric: xmrMatch[1],
       csvPath: xmrMatch[2],
+      priorReadAnchor: xmrMatch[3] || null,
       openLine: i,
     };
   }
@@ -42,6 +43,7 @@ function closePair(open, i) {
       kind: "xmr",
       metric: open.metric,
       csvPath: open.csvPath,
+      priorReadAnchor: open.priorReadAnchor,
       openLine: open.openLine,
       closeLine: i,
     };

package/src/secret-gate.js

@@ -0,0 +1,177 @@
+/**
+ * Fail-closed secret gate for the wiki push path. Runs gitleaks over the
+ * commit range a push introduces and reports a clean / finding /
+ * scanner-absent verdict. The wiki has no destination-side secret control (a
+ * GitHub Wiki repo runs no Actions and is excluded from GitHub
+ * secret-scanning), so this is the only place a content backstop can live.
+ *
+ * The module never throws on a scanner result: a missing or erroring scanner
+ * resolves to `scanner-absent` so the caller fails closed rather than treating
+ * an error as clean. Findings carry only a location (`file:line:rule`) — never
+ * the matched secret value, so an audit record built from them cannot itself
+ * leak.
+ */
+
+import { isoTimestamp } from "@forwardimpact/libutil";
+import { createLogger } from "@forwardimpact/libtelemetry";
+
+/** The gitleaks binary name resolved on PATH; provisioning is an operator concern (see wiki-operations guide). */
+const GITLEAKS = "gitleaks";
+
+/**
+ * Scan the commit range a push introduces for secrets, fail closed.
+ *
+ * Probes `gitleaks version` first; an unresolvable binary short-circuits to
+ * `scanner-absent`. Then runs `gitleaks detect` over `range` expressed as
+ * `git log` options, reading the JSON report from stdout. Exit codes follow
+ * gitleaks' documented contract: `0` clean, `1` leaks found, any other
+ * non-zero an invocation error (treated as `scanner-absent` — fail closed, an
+ * error is never reported as clean).
+ *
+ * @param {object} args
+ * @param {import('@forwardimpact/libutil/runtime').Runtime} args.runtime - Provides `subprocess.run`.
+ * @param {string} args.wikiDir - The wiki clone directory to scan.
+ * @param {string} args.range - A `git log` range (e.g. `origin/master..HEAD`).
+ * @returns {Promise<{status: "clean"|"finding"|"scanner-absent", findings?: Array<{file: string, line: number, rule: string}>}>}
+ */
+export async function scanPushWindow({ runtime, wikiDir, range }) {
+  const probe = await runtime.subprocess.run(GITLEAKS, ["version"], {
+    cwd: wikiDir,
+  });
+  if (probe.exitCode !== 0) return { status: "scanner-absent" };
+
+  const scan = await runtime.subprocess.run(
+    GITLEAKS,
+    [
+      "detect",
+      "--source",
+      wikiDir,
+      "--log-opts",
+      range,
+      "--report-format",
+      "json",
+      "--report-path",
+      "-",
+    ],
+    { cwd: wikiDir },
+  );
+
+  if (scan.exitCode === 0) return { status: "clean" };
+  if (scan.exitCode === 1) {
+    return { status: "finding", findings: parseFindings(scan.stdout) };
+  }
+  // Any other non-zero is an invocation/usage error, not a leak verdict:
+  // fail closed rather than risk reporting a broken scan as clean.
+  return { status: "scanner-absent" };
+}
+
+/**
+ * Parse a gitleaks JSON report into location-only findings. Reads only the
+ * file, line, and rule of each entry — never the matched secret value — so a
+ * record built from the result is secret-free by construction. A malformed or
+ * empty report yields an empty list.
+ *
+ * @param {string} stdout - The gitleaks JSON report.
+ * @returns {Array<{file: string, line: number, rule: string}>}
+ */
+function parseFindings(stdout) {
+  let report;
+  try {
+    report = JSON.parse(stdout);
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(report)) return [];
+  return report.map((entry) => ({
+    file: entry.File ?? "",
+    line: entry.StartLine ?? 0,
+    rule: entry.RuleID ?? "",
+  }));
+}
+
+/**
+ * Append one secret-free line to the wiki tree's `secret-overrides.log` and
+ * stage it (path-scoped) so it lands in the same push as the overridden
+ * content. The line records the override as a durable, inspectable audit
+ * trail: an ISO timestamp, the asserted operator identity (`git config
+ * user.email` — attribution of intent, NOT an authenticated identity), the
+ * override class, the reason, and for a finding its location. It never carries
+ * a matched secret value.
+ *
+ * @param {object} args
+ * @param {import('@forwardimpact/libutil/runtime').Runtime} args.runtime - Provides `fs` and `clock`.
+ * @param {import('@forwardimpact/libutil').GitClient} args.gitClient - Stages the log into the push.
+ * @param {string} args.wikiDir - The wiki clone directory.
+ * @param {"finding"|"scanner-absent"} args.klass - The override class.
+ * @param {string} args.reason - The operator-supplied reason for the override.
+ * @param {Array<{file: string, line: number, rule: string}>} [args.findings] - Locations for a finding override.
+ * @returns {Promise<{path: string}>} The relative path staged into the push.
+ */
+export async function appendOverrideRecord({
+  runtime,
+  gitClient,
+  wikiDir,
+  klass,
+  reason,
+  findings = [],
+}) {
+  const email =
+    (await gitClient.configGet("user.email", { cwd: wikiDir })) || "unknown";
+  const where =
+    klass === "finding"
+      ? findings.map((f) => `${f.file}:${f.line}:${f.rule}`).join(",") ||
+        "unspecified"
+      : "scanner-absent";
+  const ts = isoTimestamp(runtime.clock.now());
+  // Tab-separated, single line; the reason is collapsed so the record stays
+  // one inspectable row per override.
+  const line = `${ts}\t${email}\t${klass}\t${reason.replace(/\s+/g, " ").trim()}\t${where}\n`;
+  const logPath = `${wikiDir}/${OVERRIDE_LOG}`;
+  await runtime.fs.appendFile(logPath, line);
+  await gitClient.commitPaths(
+    `wiki: secret-gate override (${klass})`,
+    [OVERRIDE_LOG],
+    { cwd: wikiDir },
+  );
+  return { path: OVERRIDE_LOG };
+}
+
+/** The append-only audit log of break-glass overrides, in the wiki tree root. */
+export const OVERRIDE_LOG = "secret-overrides.log";
+
+/**
+ * Translate a `commitAndPush` security refusal into a command envelope,
+ * logging the cause and its break-glass procedure at error level (always
+ * surfaced, regardless of LOG_LEVEL). Returns `null` for any non-refusal
+ * result (clean / pushed / network "saved locally"), so a caller can fall
+ * through to its normal success handling. Shared by every command surface so
+ * the refusal message and exit code live in one place.
+ *
+ * @param {object} runtime - The runtime bag (the logger writes to `proc.stderr`).
+ * @param {{reason?: string, findings?: Array<{file: string, line: number, rule: string}>}} result - A `commitAndPush` result.
+ * @returns {{ok: false, code: 1}|null}
+ */
+export function refusalEnvelope(runtime, result) {
+  if (result.reason === "secret-detected") {
+    const where = (result.findings ?? [])
+      .map((f) => `${f.file}:${f.line}:${f.rule}`)
+      .join(", ");
+    createLogger("wiki", runtime).error(
+      "push",
+      `push blocked: secret detected in wiki content${where ? ` (${where})` : ""}; ` +
+        "the push was not attempted. After confirming a false positive, set " +
+        "FIT_WIKI_SECRET_OVERRIDE to a reason to override (audited).",
+    );
+    return { ok: false, code: 1 };
+  }
+  if (result.reason === "scanner-unavailable") {
+    createLogger("wiki", runtime).error(
+      "push",
+      "push blocked: the secret scanner (gitleaks) is unavailable; the push " +
+        "was not attempted. Install gitleaks, or set FIT_WIKI_SCANNER_ABSENT_OK " +
+        "to a reason to override (audited).",
+    );
+    return { ok: false, code: 1 };
+  }
+  return null;
+}

package/src/util/agent-flag.js

@@ -0,0 +1,31 @@
+/**
+ * Resolve the required agent flag from frozen CLI options. Pure — reads no
+ * filesystem and no environment, so it runs before any state change. Returns
+ * `{ ok: true, agent }` when the flag is present, or
+ * `{ ok: false, code: 2, error }` when it is missing, where `error` names the
+ * missing flag and shows a corrected example invocation. The error never
+ * mentions an environment variable: `libwiki` carries no ambient agent
+ * identity, so there is no fallback to offer.
+ *
+ * @param {Record<string, unknown>} options - The frozen `ctx.options`.
+ * @param {{ command: string, flag?: string, example: string }} spec
+ *   `command` names the failing subcommand; `flag` is the option key prefix
+ *   (`--agent` by default, `--from` for `memo`); `example` is a correct
+ *   invocation shown verbatim in the error.
+ * @returns {{ ok: true, agent: string } | { ok: false, code: 2, error: string }}
+ */
+export function requireAgentFlag(
+  options,
+  { command, flag = "--agent", example },
+) {
+  const key = flag === "--from" ? "from" : "agent";
+  const agent = options[key];
+  if (!agent) {
+    return {
+      ok: false,
+      code: 2,
+      error: `${command} requires ${flag} <name>; e.g. ${example}`,
+    };
+  }
+  return { ok: true, agent };
+}