harnery 0.6.0 → 0.7.0

package/src/lib/docs-meta.ts

@@ -0,0 +1,44 @@
+import { readFileSync, statSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+import { parseFrontmatter } from "./docs-frontmatter.ts";
+
+export interface DocsMetadata {
+  path: string;
+  data: Record<string, unknown>;
+}
+
+/**
+ * Read the leading YAML frontmatter from a markdown file.
+ *
+ * Relative paths resolve from the host repo root so the command behaves the
+ * same no matter which directory invoked it.
+ */
+export function readDocsMetadata(repoRoot: string, inputPath: string): DocsMetadata {
+  const path = isAbsolute(inputPath) ? resolve(inputPath) : resolve(repoRoot, inputPath);
+  try {
+    if (!statSync(path).isFile()) throw new Error("not a file");
+  } catch {
+    throw new Error(`Documentation file not found: ${inputPath}`);
+  }
+
+  const parsed = parseFrontmatter(readFileSync(path, "utf8"));
+  if (parsed.raw === null) {
+    throw new Error(`Documentation file has no leading YAML frontmatter: ${inputPath}`);
+  }
+  if (Object.keys(parsed.data).length === 0) {
+    throw new Error(`Documentation frontmatter is empty or malformed: ${inputPath}`);
+  }
+  return { path, data: parsed.data };
+}
+
+/** Read one top-level metadata key, failing when the key is absent. */
+export function readDocsMetadataKey(
+  metadata: Record<string, unknown>,
+  key: string,
+  inputPath: string,
+): unknown {
+  if (!Object.hasOwn(metadata, key)) {
+    throw new Error(`Documentation frontmatter key '${key}' not found: ${inputPath}`);
+  }
+  return metadata[key];
+}

package/src/lib/docs-sweep.ts

@@ -1,5 +1,6 @@
 import { existsSync as __existsSyncForDocs } from "node:fs";
 import { resolve as __resolveForDocs } from "node:path";
+import { readDocStatus } from "./docs-frontmatter.ts";
 import { sh } from "./exec.ts";
 
 // Module-level docs context, initialized by initDocsContext() before any
@@ -21,7 +22,7 @@ function isSubmoduleInitialized(name: string): boolean {
   return __existsSyncForDocs(__resolveForDocs(REPO_ROOT, name, ".git"));
 }
 
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync, statSync } from "node:fs";
 import { join, relative } from "node:path";
 
 /**
@@ -34,6 +35,10 @@ import { join, relative } from "node:path";
  *
  * Audit files and issue files under `docs/audits/` are explicitly **not**
  * flagged for age; they're immutable records by design.
+ *
+ * Performance: ages come from one `git log --name-only` per repo (not one
+ * `git log` per file). A naive per-file spawn was ~1min+ on large hosts
+ * (thousands of topic docs) and looked hung when piped.
  */
 
 export interface SweepOpts {
@@ -65,12 +70,48 @@ const RUNBOOK_DAYS = 180;
 const TOPIC_DOC_DAYS = 365;
 const DECISIONS_DORMANT_DAYS = 180;
 
-/** Days since last git commit touching a file */
-async function lastCommitAgeDays(cwd: string, file: string): Promise<number | null> {
-  const result = await sh(`git log -1 --format=%aI -- "${file}"`, { cwd });
-  if (result.exitCode !== 0 || !result.stdout.trim()) return null;
-  const ms = Date.now() - new Date(result.stdout.trim()).getTime();
-  return Math.floor(ms / (1000 * 60 * 60 * 24));
+type AgeMap = Map<string, number>;
+
+/**
+ * Parse a `git log --format="COMMIT %aI" --name-only` body into
+ * repo-relative path → age in days. Newest commit wins.
+ * Exported for unit tests.
+ */
+export function parseDocsAgeLog(stdout: string, nowMs: number = Date.now()): AgeMap {
+  const ages: AgeMap = new Map();
+  let currentAge: number | null = null;
+  for (const line of stdout.split("\n")) {
+    if (line.startsWith("COMMIT ")) {
+      const iso = line.slice("COMMIT ".length).trim();
+      const ms = Date.parse(iso);
+      currentAge = Number.isNaN(ms) ? null : Math.floor((nowMs - ms) / (1000 * 60 * 60 * 24));
+      continue;
+    }
+    if (!line || currentAge == null) continue;
+    if (!line.endsWith(".md")) continue;
+    // First (newest) sighting wins
+    if (!ages.has(line)) ages.set(line, currentAge);
+  }
+  return ages;
+}
+
+/**
+ * One `git log --name-only` for the whole docs/ tree. Docs histories are
+ * small even without --since (a large host's full docs log is ~5k lines / <100ms),
+ * so we take the full history for accurate ages on old files.
+ */
+async function loadDocsAges(cwd: string): Promise<AgeMap> {
+  const result = await sh(`git log --format="COMMIT %aI" --name-only -- docs/`, {
+    cwd,
+    timeout: 120_000,
+  });
+  if (result.exitCode !== 0 || !result.stdout.trim()) return new Map();
+  return parseDocsAgeLog(result.stdout);
+}
+
+/** Age for a tracked path, or null if untracked / never under docs/. */
+function ageDays(ages: AgeMap, rel: string): number | null {
+  return ages.has(rel) ? ages.get(rel)! : null;
 }
 
 /** Days since ANY commit in the repo (measures repo activity) */
@@ -81,47 +122,35 @@ async function lastRepoCommitAgeDays(cwd: string): Promise<number | null> {
   return Math.floor(ms / (1000 * 60 * 60 * 24));
 }
 
-function readStatus(filePath: string): string | null {
-  try {
-    const content = readFileSync(filePath, "utf8");
-    const head = content.split("\n").slice(0, 20).join("\n");
-    const m = head.match(/\*\*Status:\*\*\s*([a-zA-Z][a-zA-Z-]*)/);
-    return m ? m[1]!.toLowerCase() : null;
-  } catch {
-    return null;
+function walkMdFiles(dir: string, skipReadme = false): string[] {
+  const out: string[] = [];
+  for (const entry of readdirSync(dir)) {
+    const full = join(dir, entry);
+    let st: ReturnType<typeof statSync> | undefined;
+    try {
+      st = statSync(full);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) {
+      out.push(...walkMdFiles(full, skipReadme));
+    } else if (entry.endsWith(".md") && !(skipReadme && entry === "README.md")) {
+      out.push(full);
+    }
   }
+  return out;
 }
 
-async function sweepPlans(repoName: string, repoPath: string, items: SweepItem[]): Promise<void> {
+function sweepPlans(repoName: string, repoPath: string, ages: AgeMap, items: SweepItem[]): void {
   const plansDir = join(repoPath, "docs", "plans");
   if (!existsSync(plansDir)) return;
 
-  const walk = (dir: string): string[] => {
-    const out: string[] = [];
-    for (const entry of readdirSync(dir)) {
-      const full = join(dir, entry);
-      let st: ReturnType<typeof statSync> | undefined;
-      try {
-        st = statSync(full);
-      } catch {
-        continue;
-      }
-      if (st.isDirectory()) {
-        out.push(...walk(full));
-      } else if (entry.endsWith(".md") && entry !== "README.md") {
-        out.push(full);
-      }
-    }
-    return out;
-  };
-
-  const files = walk(plansDir);
-  for (const full of files) {
+  for (const full of walkMdFiles(plansDir, true)) {
     const rel = relative(repoPath, full);
     const displayPath = join(repoName === "(root)" ? "" : repoName, rel);
     const isArchived = rel.includes("/archive/");
-    const status = readStatus(full);
-    const age = await lastCommitAgeDays(repoPath, rel);
+    const status = readDocStatus(full, "plan");
+    const age = ageDays(ages, rel);
     if (age == null) continue;
 
     if (!isArchived && status === "in-progress" && age > STALLED_PLAN_DAYS) {
@@ -145,7 +174,7 @@ async function sweepPlans(repoName: string, repoPath: string, items: SweepItem[]
   }
 }
 
-async function sweepIssues(repoName: string, repoPath: string, items: SweepItem[]): Promise<void> {
+function sweepIssues(repoName: string, repoPath: string, ages: AgeMap, items: SweepItem[]): void {
   const issuesDir = join(repoPath, "docs", "issues");
   if (!existsSync(issuesDir)) return;
 
@@ -154,9 +183,9 @@ async function sweepIssues(repoName: string, repoPath: string, items: SweepItem[
     const full = join(issuesDir, entry);
     const rel = join("docs", "issues", entry);
     const displayPath = join(repoName === "(root)" ? "" : repoName, rel);
-    const status = readStatus(full);
+    const status = readDocStatus(full, "issue");
     if (status !== "open") continue;
-    const age = await lastCommitAgeDays(repoPath, rel);
+    const age = ageDays(ages, rel);
     if (age == null || age <= OPEN_ISSUE_DAYS) continue;
     items.push({
       kind: "open-issue-cold",
@@ -168,38 +197,15 @@ async function sweepIssues(repoName: string, repoPath: string, items: SweepItem[
   }
 }
 
-async function sweepHandoffs(
-  repoName: string,
-  repoPath: string,
-  items: SweepItem[],
-): Promise<void> {
+function sweepHandoffs(repoName: string, repoPath: string, ages: AgeMap, items: SweepItem[]): void {
   const handoffsDir = join(repoPath, "docs", "handoffs");
   if (!existsSync(handoffsDir)) return;
 
-  const walk = (dir: string): string[] => {
-    const out: string[] = [];
-    for (const entry of readdirSync(dir)) {
-      const full = join(dir, entry);
-      let st: ReturnType<typeof statSync> | undefined;
-      try {
-        st = statSync(full);
-      } catch {
-        continue;
-      }
-      if (st.isDirectory()) {
-        out.push(...walk(full));
-      } else if (entry.endsWith(".md")) {
-        out.push(full);
-      }
-    }
-    return out;
-  };
-
-  for (const full of walk(handoffsDir)) {
+  for (const full of walkMdFiles(handoffsDir)) {
     const rel = relative(repoPath, full);
-    const status = readStatus(full);
+    const status = readDocStatus(full, "handoff");
     if (status !== "open") continue;
-    const age = await lastCommitAgeDays(repoPath, rel);
+    const age = ageDays(ages, rel);
     if (age == null || age <= COLD_HANDOFF_DAYS) continue;
     const displayPath = join(repoName === "(root)" ? "" : repoName, rel);
     items.push({
@@ -212,10 +218,10 @@ async function sweepHandoffs(
   }
 }
 
-async function sweepRunbook(repoName: string, repoPath: string, items: SweepItem[]): Promise<void> {
+function sweepRunbook(repoName: string, repoPath: string, ages: AgeMap, items: SweepItem[]): void {
   const runbook = join(repoPath, "docs", "runbook.md");
   if (!existsSync(runbook)) return;
-  const age = await lastCommitAgeDays(repoPath, "docs/runbook.md");
+  const age = ageDays(ages, "docs/runbook.md");
   if (age == null || age <= RUNBOOK_DAYS) return;
   const displayPath = join(repoName === "(root)" ? "" : repoName, "docs/runbook.md");
   items.push({
@@ -227,21 +233,27 @@ async function sweepRunbook(repoName: string, repoPath: string, items: SweepItem
   });
 }
 
-async function sweepTopicDocs(
+function sweepTopicDocs(
   repoName: string,
   repoPath: string,
+  ages: AgeMap,
   items: SweepItem[],
-): Promise<void> {
+): void {
   const docsDir = join(repoPath, "docs");
   if (!existsSync(docsDir)) return;
 
-  // Skip known date-stamped or lifecycle-managed dirs
+  // Skip known date-stamped or lifecycle-managed dirs (and vendor dumps).
+  // handoffs/inquiries are lifecycle-managed elsewhere; vendors match the
+  // docs freshness scanner's IGNORE_DIRS.
   const skipDirs = new Set([
     "audits",
     "issues",
     "plans",
     "changelogs",
     "emails", // parent-specific
+    "handoffs",
+    "inquiries",
+    "vendors",
   ]);
 
   for (const entry of readdirSync(docsDir)) {
@@ -255,26 +267,9 @@ async function sweepTopicDocs(
     }
     if (!st.isDirectory()) continue;
 
-    // For each topic dir, find .md files and check age
-    const walk = (dir: string): string[] => {
-      const out: string[] = [];
-      for (const f of readdirSync(dir)) {
-        const fp = join(dir, f);
-        let s: ReturnType<typeof statSync> | undefined;
-        try {
-          s = statSync(fp);
-        } catch {
-          continue;
-        }
-        if (s.isDirectory()) out.push(...walk(fp));
-        else if (f.endsWith(".md")) out.push(fp);
-      }
-      return out;
-    };
-
-    for (const full2 of walk(full)) {
+    for (const full2 of walkMdFiles(full)) {
       const rel = relative(repoPath, full2);
-      const age = await lastCommitAgeDays(repoPath, rel);
+      const age = ageDays(ages, rel);
       if (age == null || age <= TOPIC_DOC_DAYS) continue;
       const displayPath = join(repoName === "(root)" ? "" : repoName, rel);
       items.push({
@@ -291,11 +286,12 @@ async function sweepTopicDocs(
 async function sweepDecisions(
   repoName: string,
   repoPath: string,
+  ages: AgeMap,
   items: SweepItem[],
 ): Promise<void> {
   const decisions = join(repoPath, "docs", "decisions.md");
   if (!existsSync(decisions)) return;
-  const age = await lastCommitAgeDays(repoPath, "docs/decisions.md");
+  const age = ageDays(ages, "docs/decisions.md");
   const repoAge = await lastRepoCommitAgeDays(repoPath);
   if (age == null || repoAge == null) return;
   // Only flag if the repo is active (recent commits) but decisions haven't moved
@@ -322,13 +318,18 @@ export async function runSweep(opts: SweepOpts): Promise<SweepItem[]> {
   const filtered = filter ? targets.filter((t) => t.name === filter) : targets;
 
   const items: SweepItem[] = [];
-  for (const { name, path } of filtered) {
-    await sweepPlans(name, path, items);
-    await sweepIssues(name, path, items);
-    await sweepHandoffs(name, path, items);
-    await sweepRunbook(name, path, items);
-    await sweepTopicDocs(name, path, items);
-    await sweepDecisions(name, path, items);
+  // Load ages per repo in parallel — one git log each, not one per file.
+  const ageMaps = await Promise.all(filtered.map((t) => loadDocsAges(t.path)));
+
+  for (let i = 0; i < filtered.length; i++) {
+    const { name, path } = filtered[i]!;
+    const ages = ageMaps[i]!;
+    sweepPlans(name, path, ages, items);
+    sweepIssues(name, path, ages, items);
+    sweepHandoffs(name, path, ages, items);
+    sweepRunbook(name, path, ages, items);
+    sweepTopicDocs(name, path, ages, items);
+    await sweepDecisions(name, path, ages, items);
   }
 
   // Sort by severity proxy: oldest first within kind
@@ -343,6 +344,7 @@ export async function runSweep(opts: SweepOpts): Promise<SweepItem[]> {
  */
 export async function countColdHandoffs(): Promise<number> {
   const items: SweepItem[] = [];
-  await sweepHandoffs("(root)", REPO_ROOT, items);
+  const ages = await loadDocsAges(REPO_ROOT);
+  sweepHandoffs("(root)", REPO_ROOT, ages, items);
   return items.length;
 }

package/src/lib/format.ts

@@ -48,7 +48,7 @@ export function colorJson(value: unknown, indent = 0): string {
       // Class instances that define toJSON() (Big.js, Decimal.js, Date, etc.)
       // would render as their raw internal shape if we walked Object.keys
       // directly. Unwrap once so callers see the intended representation
-      // (Big.js → "1.97" instead of {s,e,c} from BigQuery NUMERIC fields).
+      // (e.g. Big.js → "1.97" instead of its internal {s,e,c} fields).
       const maybeJsonable = v as { toJSON?: () => unknown };
       if (typeof maybeJsonable.toJSON === "function") {
         return fmt(maybeJsonable.toJSON(), depth);
@@ -115,7 +115,7 @@ function stringify(val: unknown): string {
   if (val === null || val === undefined) return "NULL";
   if (typeof val === "object") {
     if (val instanceof Date) return val.toISOString();
-    // BigQuery returns { value: "..." } for some types
+    // Some data sources wrap a scalar as { value: "..." }; unwrap to the inner value.
     if ("value" in val && Object.keys(val).length === 1) {
       return String((val as { value: unknown }).value);
     }

package/src/lib/http/index.ts

@@ -1 +1,2 @@
 export { type FetchOptions, type FetchResult, fetchWithJar } from "./client.js";
+export * from "./request.ts";

package/src/lib/http/request.ts

@@ -0,0 +1,154 @@
+/**
+ * Retrying JSON-API request — the toolkit-tier HTTP primitive for host CLIs'
+ * vendor clients.
+ *
+ * Extracted from the first embedding host, where ten vendor clients carried
+ * byte-similar copies of the same loop: abort-controller timeout, retry on
+ * 429/5xx with exponential backoff + jitter, `Retry-After` honored when sane,
+ * vendor-specific error taxonomy applied by the caller. This module owns the
+ * loop; callers keep their auth headers and error classes:
+ *
+ *   const r = await requestWithRetries(url, {
+ *     method, body,
+ *     headers: { Authorization: `ApiKey ${key}`, Accept: "application/json" },
+ *     timeoutMs: this.timeoutMs,
+ *     maxRetries: this.maxRetries,
+ *     onResponse: ({ status }) => log(`${method} ${url} → ${status}`),
+ *     networkError: (msg) => new VendorError("network_error", msg),
+ *   });
+ *   if (!r.ok) throw makeHttpError(r.status, url, r.text, r.headers);
+ *
+ * Design choices, so they survive review:
+ *   - Terminal non-2xx responses RETURN (`ok: false`) rather than throw — the
+ *     vendor error taxonomy belongs to the caller, not this module.
+ *   - Only terminal NETWORK failures throw (after retries), because there is
+ *     no response to hand back; `networkError` lets the caller keep its class.
+ *   - The response body is always read (even on retried statuses) so keep-alive
+ *     sockets are released.
+ */
+
+export interface RetryingResponse {
+  /** `status` in the 2xx range. */
+  ok: boolean;
+  status: number;
+  /** Response body as text; callers handle JSON parsing. */
+  text: string;
+  url: string;
+  headers: Headers;
+}
+
+export interface RequestWithRetriesOptions {
+  /** HTTP method. Default GET. */
+  method?: string;
+  /** Extra headers. Content-Type defaults to application/json when a non-string body is given. */
+  headers?: Record<string, string>;
+  /**
+   * Request body. Strings, Uint8Array, FormData, Blob, and ReadableStream pass
+   * through untouched; any other value is JSON.stringify'd (with a
+   * Content-Type: application/json default).
+   */
+  body?: unknown;
+  /** Per-attempt timeout (AbortController). Default 30s. */
+  timeoutMs?: number;
+  /** Retries after the first attempt. Default 3. */
+  maxRetries?: number;
+  /** Which statuses to retry. Default: 429 and 5xx. */
+  shouldRetry?: (status: number) => boolean;
+  /** Delay before retry `attempt` (0-based). Default `backoffDelayMs`. */
+  delayMs?: (attempt: number, retryAfterSeconds?: number | null) => number;
+  /** Observability hook — fires once per received response (every attempt). */
+  onResponse?: (info: { method: string; url: string; status: number; attempt: number }) => void;
+  /**
+   * Wrap a terminal network failure (fetch threw on the last attempt) in the
+   * caller's error class. Default: a plain Error with the message.
+   */
+  networkError?: (message: string, url: string, retries: number) => Error;
+}
+
+/**
+ * Exponential backoff with jitter: 500ms · 2^attempt, capped at 30s, plus up
+ * to 250ms of jitter. A sane `Retry-After` (0 < s < 60) short-circuits the
+ * curve — the server knows better than the guess.
+ */
+export function backoffDelayMs(attempt: number, retryAfterSeconds?: number | null): number {
+  if (retryAfterSeconds && retryAfterSeconds > 0 && retryAfterSeconds < 60) {
+    return retryAfterSeconds * 1000 + Math.floor(Math.random() * 250);
+  }
+  const base = Math.min(30_000, 500 * 2 ** attempt);
+  return base + Math.floor(Math.random() * 250);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
+}
+
+export async function requestWithRetries(
+  url: string,
+  opts: RequestWithRetriesOptions = {},
+): Promise<RetryingResponse> {
+  const method = opts.method ?? "GET";
+  const timeoutMs = opts.timeoutMs ?? 30_000;
+  const maxRetries = opts.maxRetries ?? 3;
+  const shouldRetry = opts.shouldRetry ?? ((status: number) => status === 429 || status >= 500);
+  const delayMs = opts.delayMs ?? backoffDelayMs;
+
+  let attempt = 0;
+  for (;;) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+    const headers: Record<string, string> = { ...(opts.headers ?? {}) };
+    const init: RequestInit = { method, headers, signal: controller.signal };
+    if (opts.body !== undefined && opts.body !== null) {
+      const passthrough =
+        typeof opts.body === "string" ||
+        opts.body instanceof Uint8Array ||
+        opts.body instanceof FormData ||
+        opts.body instanceof Blob ||
+        opts.body instanceof ReadableStream;
+      if (passthrough) {
+        init.body = opts.body as BodyInit;
+      } else {
+        if (!Object.keys(headers).some((k) => k.toLowerCase() === "content-type")) {
+          headers["Content-Type"] = "application/json";
+        }
+        init.body = JSON.stringify(opts.body);
+      }
+    }
+
+    let res: Response;
+    try {
+      res = await fetch(url, init);
+    } catch (err: unknown) {
+      clearTimeout(timer);
+      if (attempt < maxRetries) {
+        await sleep(delayMs(attempt));
+        attempt++;
+        continue;
+      }
+      const msg = err instanceof Error ? err.message : String(err);
+      const message = `request failed after ${maxRetries} retries: ${msg} (URL: ${url})`;
+      throw opts.networkError ? opts.networkError(msg, url, maxRetries) : new Error(message);
+    }
+    clearTimeout(timer);
+
+    opts.onResponse?.({ method, url, status: res.status, attempt });
+
+    if (shouldRetry(res.status) && attempt < maxRetries) {
+      const retryAfter = Number(res.headers.get("retry-after"));
+      // Drain the body so the socket is reusable before we sleep.
+      await res.text().catch(() => undefined);
+      await sleep(delayMs(attempt, Number.isFinite(retryAfter) ? retryAfter : null));
+      attempt++;
+      continue;
+    }
+
+    return {
+      ok: res.ok,
+      status: res.status,
+      text: await res.text(),
+      url,
+      headers: res.headers,
+    };
+  }
+}