ashlr-plugin 1.36.4

package/scripts/hook-timings-report.ts

@@ -0,0 +1,554 @@
+#!/usr/bin/env bun
+/**
+ * ashlr hook-timings-report — reads ~/.ashlr/hook-timings.jsonl and renders
+ * a compact per-hook latency table with p50/p95/max and error/block rates.
+ *
+ * Exported surface:
+ *   readHookTimings(path?)        → HookTimingRecord[], fault-tolerant
+ *   computeAggregates(records, windowHours?) → HookAggregate[]
+ *
+ * CLI: `bun run scripts/hook-timings-report.ts [--hours N]`
+ *   Exits 0 always.
+ */
+
+import {
+  readHookTimingsDetailed,
+  type HookTimingReadQuality,
+  type HookTimingRecord,
+} from "./hook-timing-reader.ts";
+
+export { readHookTimingsDetailed } from "./hook-timing-reader.ts";
+export type { HookTimingReadQuality, HookTimingRecord } from "./hook-timing-reader.ts";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface HookAggregate {
+  hook: string;
+  calls: number;
+  p50: number;
+  p95: number;
+  max: number;
+  errorPct: number;
+  blockPct: number;
+  timeoutPct: number;
+}
+
+export type TrendClassification = "improved" | "regressed" | "stable" | "new";
+
+export interface HookTrend {
+  hook: string;
+  current: { mean: number; p50: number; p95: number };
+  compare: { mean: number; p50: number; p95: number };
+  deltaMs: number;   // absolute ms: current.p95 - compare.p95
+  deltaPct: number;  // percent change (negative = improved)
+  trend: TrendClassification;
+}
+
+// ---------------------------------------------------------------------------
+// Reading + parsing
+// ---------------------------------------------------------------------------
+
+/**
+ * Read and parse hook-timings.jsonl. Skips malformed lines silently.
+ * @param path Override the default ~/.ashlr/hook-timings.jsonl path.
+ */
+export function readHookTimings(path?: string): HookTimingRecord[] {
+  return readHookTimingsDetailed({ path }).records;
+}
+
+// ---------------------------------------------------------------------------
+// Aggregation
+// ---------------------------------------------------------------------------
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0]!;
+  const idx = (p / 100) * (sorted.length - 1);
+  const lo = Math.floor(idx);
+  const hi = Math.ceil(idx);
+  if (lo === hi) return sorted[lo]!;
+  const frac = idx - lo;
+  return Math.round(sorted[lo]! + frac * (sorted[hi]! - sorted[lo]!));
+}
+
+/**
+ * Filter records whose ts parses to a time in [from, to) (to = Infinity means open-ended).
+ */
+function filterByTime(
+  records: HookTimingRecord[],
+  from: number,
+  to: number = Infinity,
+): HookTimingRecord[] {
+  return records.filter((r) => {
+    const t = new Date(r.ts).getTime();
+    return !isNaN(t) && t >= from && t < to;
+  });
+}
+
+/** Group records by hook name. */
+function groupByHook<T extends HookTimingRecord>(recs: T[]): Map<string, T[]> {
+  const m = new Map<string, T[]>();
+  for (const rec of recs) {
+    const arr = m.get(rec.hook) ?? [];
+    arr.push(rec);
+    m.set(rec.hook, arr);
+  }
+  return m;
+}
+
+/** {mean, p50, p95} summary over an unsorted duration list. */
+function durationStats(durations: number[]): { mean: number; p50: number; p95: number } {
+  if (durations.length === 0) return { mean: 0, p50: 0, p95: 0 };
+  const sorted = [...durations].sort((a, b) => a - b);
+  const mean = Math.round(sorted.reduce((s, v) => s + v, 0) / sorted.length);
+  return { mean, p50: percentile(sorted, 50), p95: percentile(sorted, 95) };
+}
+
+/**
+ * Compute per-hook aggregates, optionally filtered to the last N hours.
+ * @param records Raw timing records.
+ * @param windowHours Window in hours (default: 24). Pass Infinity for all.
+ */
+export function computeAggregates(
+  records: HookTimingRecord[],
+  windowHours = 24,
+): HookAggregate[] {
+  const cutoff = Date.now() - windowHours * 3_600_000;
+  const inWindow = filterByTime(records, cutoff);
+
+  const aggregates: HookAggregate[] = [];
+  for (const [hook, recs] of groupByHook(inWindow)) {
+    const durations = recs.map((r) => r.durationMs).sort((a, b) => a - b);
+    const errors = recs.filter((r) => r.outcome === "error").length;
+    const blocks = recs.filter((r) => r.outcome === "block").length;
+    const timeouts = recs.filter((r) => r.outcome === "timeout").length;
+    aggregates.push({
+      hook,
+      calls: recs.length,
+      p50: percentile(durations, 50),
+      p95: percentile(durations, 95),
+      max: durations[durations.length - 1] ?? 0,
+      errorPct: (errors / recs.length) * 100,
+      blockPct: (blocks / recs.length) * 100,
+      timeoutPct: (timeouts / recs.length) * 100,
+    });
+  }
+
+  return aggregates.sort((a, b) => b.calls - a.calls);
+}
+
+/**
+ * Compute trend data comparing the current window vs the previous equal-length
+ * compare window that immediately precedes it.
+ */
+export function computeTrends(
+  records: HookTimingRecord[],
+  opts: { windowHours: number; compareHours: number },
+): HookTrend[] {
+  const { windowHours, compareHours } = opts;
+  const now = Date.now();
+  const currentCutoff = now - windowHours * 3_600_000;
+  const compareCutoff = currentCutoff - compareHours * 3_600_000;
+
+  const toDurations = (recs: HookTimingRecord[]) => recs.map((r) => r.durationMs);
+  const currentByHook = new Map(
+    [...groupByHook(filterByTime(records, currentCutoff))].map(([h, r]) => [h, toDurations(r)]),
+  );
+  const compareByHook = new Map(
+    [...groupByHook(filterByTime(records, compareCutoff, currentCutoff))].map(
+      ([h, r]) => [h, toDurations(r)],
+    ),
+  );
+
+  const trends: HookTrend[] = [];
+  for (const [hook, curDurations] of currentByHook) {
+    const curStats = durationStats(curDurations);
+    const cmpDurations = compareByHook.get(hook) ?? [];
+    const cmpStats = durationStats(cmpDurations);
+
+    const deltaMs = curStats.p95 - cmpStats.p95;
+    // When the compare window has no data for this hook, we can't compute a
+    // percentage. Previously this fell through to "stable" silently — which
+    // masked real regressions on hooks that only appeared in the current
+    // window. Tag them as "new" so callers (and the dashboard flag output)
+    // can surface them distinctly.
+    const hasCompareBaseline = cmpDurations.length > 0 && cmpStats.p95 > 0;
+    const deltaPct = hasCompareBaseline ? (deltaMs / cmpStats.p95) * 100 : 0;
+
+    let trend: TrendClassification;
+    if (!hasCompareBaseline) trend = "new";
+    else if (deltaPct <= -20) trend = "improved";
+    else if (deltaPct >= 20) trend = "regressed";
+    else trend = "stable";
+
+    trends.push({ hook, current: curStats, compare: cmpStats, deltaMs, deltaPct, trend });
+  }
+
+  return trends;
+}
+
+// ---------------------------------------------------------------------------
+// Rendering
+// ---------------------------------------------------------------------------
+
+function fmtMs(ms: number): string {
+  if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${ms}ms`;
+}
+
+function fmtPct(pct: number, hasOutcome: boolean): string {
+  if (!hasOutcome) return "—";
+  return `${pct.toFixed(1)}%`;
+}
+
+function fmtNum(n: number): string {
+  return n.toLocaleString("en-US");
+}
+
+function medianOverAll(aggregates: HookAggregate[]): number {
+  if (aggregates.length === 0) return 0;
+  const total = aggregates.reduce((s, a) => s + a.calls, 0);
+  if (total === 0) return 0;
+  // weighted median approximation: sort all p50s by call count and pick mid
+  const sorted = [...aggregates].sort((a, b) => a.p50 - b.p50);
+  let acc = 0;
+  const mid = total / 2;
+  for (const a of sorted) {
+    acc += a.calls;
+    if (acc >= mid) return a.p50;
+  }
+  return sorted[sorted.length - 1]!.p50;
+}
+
+function buildFlags(aggregates: HookAggregate[]): string[] {
+  const flags: string[] = [];
+  for (const a of aggregates) {
+    if (a.p95 > 200) {
+      flags.push(
+        `⚠ ${a.hook} p95 ${fmtMs(a.p95)} exceeds 200ms threshold — consider parallelization`,
+      );
+    }
+    if (a.p95 > 100 && a.p95 <= 200) {
+      flags.push(
+        `Flagging: ${a.hook} p95 > 100ms — consider parallelization.`,
+      );
+    }
+    if (a.max >= 1000) {
+      flags.push(
+        `Flag: ${a.hook} max ${fmtMs(a.max)} — investigate slow path.`,
+      );
+    }
+  }
+  return flags;
+}
+
+export function renderReport(
+  aggregates: HookAggregate[],
+  windowHours: number,
+  totalRecords: number,
+  trends?: HookTrend[],
+): string {
+  if (totalRecords === 0) {
+    return "ashlr hook timings: no records yet (enable hooks and trigger a few tool calls).";
+  }
+
+  const med = medianOverAll(aggregates);
+  const header = `ashlr hook timings · last ${windowHours}h · ${fmtNum(totalRecords)} records · median ${fmtMs(med)}`;
+
+  // Build trend lookup by hook name
+  const trendMap = new Map<string, HookTrend>();
+  if (trends) {
+    for (const t of trends) trendMap.set(t.hook, t);
+  }
+
+  // Column widths (must fit 80 cols total)
+  // With trends: hook(20) calls(7) p50(7) p95(7) trend(3) max(8) error%(8) block%(7) = 67 + spaces
+  // Without:     hook(22) calls(7) p50(7) p95(7)         max(8) error%(8) block%(7) = 66 + spaces
+  const hasTrends = trendMap.size > 0;
+  const COL_HOOK = hasTrends ? 20 : 22;
+  const colHead =
+    "hook".padEnd(COL_HOOK) +
+    "calls".padStart(7) +
+    "p50".padStart(7) +
+    "p95".padStart(7) +
+    (hasTrends ? "   " : "") +
+    "max".padStart(8) +
+    "error%".padStart(8) +
+    "block%".padStart(7);
+  const rule = "─".repeat(70);
+
+  const rows = aggregates.map((a) => {
+    const hookCol = a.hook.length > COL_HOOK
+      ? a.hook.slice(0, COL_HOOK - 1) + "…"
+      : a.hook.padEnd(COL_HOOK);
+    const hasErrors = a.calls > 0;
+    const hasBlocks = a.calls > 0;
+    let trendIndicator = "";
+    if (hasTrends) {
+      const t = trendMap.get(a.hook);
+      if (t?.trend === "improved") trendIndicator = " ↓ ";
+      else if (t?.trend === "regressed") trendIndicator = " ↑ ";
+      else if (t?.trend === "new") trendIndicator = " ★ ";  // no baseline to compare against
+      else trendIndicator = "   ";
+    }
+    return (
+      hookCol +
+      fmtNum(a.calls).padStart(7) +
+      fmtMs(a.p50).padStart(7) +
+      fmtMs(a.p95).padStart(7) +
+      trendIndicator +
+      fmtMs(a.max).padStart(8) +
+      fmtPct(a.errorPct, hasErrors).padStart(8) +
+      fmtPct(a.blockPct, hasBlocks).padStart(7)
+    );
+  });
+
+  const flags = buildFlags(aggregates);
+
+  const parts = [header, "", colHead, rule, ...rows];
+  if (flags.length > 0) {
+    parts.push("", ...flags);
+  }
+  return parts.join("\n");
+}
+
+/**
+ * Compact 4-5 line summary for embedding in `/ashlr-dashboard`.
+ * Returns empty string when records is empty.
+ */
+export function renderCompact(opts: {
+  records: HookTimingRecord[];
+  topN: number;
+  windowHours?: number;
+}): string {
+  const { records, topN, windowHours = 24 } = opts;
+  if (records.length === 0) return "";
+
+  const aggregates = computeAggregates(records, windowHours);
+  if (aggregates.length === 0) return "";
+
+  const totalCalls = aggregates.reduce((s, a) => s + a.calls, 0);
+  const med = medianOverAll(aggregates);
+
+  // Top N slowest by p95
+  const byP95 = [...aggregates].sort((a, b) => b.p95 - a.p95).slice(0, topN);
+
+  const lines: string[] = [];
+  lines.push(`Hook performance (last ${windowHours}h): ${fmtNum(totalCalls)} calls · median ${fmtMs(med)}`);
+  lines.push(`Top ${Math.min(topN, byP95.length)} slowest by p95:`);
+  for (const a of byP95) {
+    const hookCol = a.hook.length > 24 ? a.hook.slice(0, 23) + "…" : a.hook.padEnd(24);
+    lines.push(`  ${hookCol}  p95 ${fmtMs(a.p95).padStart(6)}  (${fmtNum(a.calls)} calls)`);
+  }
+  return lines.join("\n");
+}
+
+// ---------------------------------------------------------------------------
+// Public top-level builder
+// ---------------------------------------------------------------------------
+
+export function buildHookTimingsReport(opts: {
+  path?: string;
+  hours?: number;
+  compare?: boolean;
+  flagsOnly?: boolean;
+} = {}): string {
+  const hours = opts.hours ?? 24;
+  const requestedHours = opts.compare ? hours * 2 : hours;
+  const detailed = readHookTimingsDetailed({
+    path: opts.path,
+    sinceMs: Date.now() - requestedHours * 3_600_000,
+  });
+  const records = detailed.records;
+  const aggregates = computeAggregates(records, hours);
+  const totalRecords = aggregates.reduce((s, a) => s + a.calls, 0);
+
+  if (opts.flagsOnly) {
+    if (totalRecords === 0) return detailed.coverage === "partial" ? coverageWarning(detailed.quality) : "";
+    const flags = buildFlags(aggregates);
+    if (detailed.coverage === "partial") flags.unshift(coverageWarning(detailed.quality));
+    return flags.join("\n");
+  }
+
+  const trends = opts.compare
+    ? computeTrends(records, { windowHours: hours, compareHours: hours })
+    : undefined;
+
+  const report = renderReport(aggregates, hours, totalRecords, trends);
+  return detailed.coverage === "partial"
+    ? `${coverageWarning(detailed.quality)}\n${report}`
+    : report;
+}
+
+function coverageWarning(quality: HookTimingReadQuality): string {
+  const reasons: string[] = [];
+  if (quality.droppedThrough) reasons.push(`history dropped through ${quality.droppedThrough}`);
+  if (quality.sources.some((source) => source.raced)) reasons.push("rotation race");
+  if (quality.sources.some((source) => source.unreadable)) reasons.push("unreadable source");
+  if (quality.sources.some((source) => source.truncatedPrefix)) reasons.push("source over 16 MiB");
+  if (quality.sources.some((source) => source.truncatedTail)) reasons.push("torn tail");
+  if (quality.sources.some((source) => source.oversizedRows > 0)) reasons.push("oversized row");
+  if (quality.sources.some((source) => source.malformedRows > 0)) reasons.push("malformed row");
+  if (quality.metadataMalformed || quality.metadataUnreadable) reasons.push("metadata unavailable");
+  else if (quality.metadataPresent && quality.droppedThrough === null) reasons.push("drop time unknown");
+  const prefix = "Warning: partial hook-timing coverage";
+  if (reasons.length === 0) return `${prefix}.`;
+  const message = `${prefix} (${reasons.join(", ")}).`;
+  return message.length <= 80 ? message : `${message.slice(0, 77)}...`;
+}
+
+// ---------------------------------------------------------------------------
+// Doctor integration — surface hook health in /ashlr-doctor
+// ---------------------------------------------------------------------------
+
+export interface DoctorHookFinding {
+  status: "ok" | "warn" | "fail";
+  label: string;
+  detail: string;
+  fix?: string;
+}
+
+export interface DoctorHookHealthOpts {
+  /** Path override for tests. Default: ~/.ashlr/hook-timings.jsonl */
+  path?: string;
+  /** Window in hours. Default 24. */
+  hours?: number;
+  /** p95 (ms) above which a hook is yellow-flagged. Default 50. */
+  warnP95Ms?: number;
+  /** p95 (ms) above which a hook is red-flagged. Default 200. */
+  failP95Ms?: number;
+}
+
+/**
+ * Build the per-hook health lines used by /ashlr-doctor's "hook performance"
+ * section. Honors the v1.29 outcome enum (including "timeout") so a hook
+ * that's hitting the 2s safety net surfaces here rather than silently dying.
+ *
+ * Ranking rules:
+ *   - timeoutPct > 0 → fail (a hook that hangs is the worst outcome)
+ *   - errorPct > 5 → fail
+ *   - p95 ≥ failP95Ms → fail
+ *   - p95 ≥ warnP95Ms → warn
+ *   - otherwise → ok (but only top-3 ok rows are returned to keep doctor tight)
+ */
+export function getDoctorHookHealth(opts: DoctorHookHealthOpts = {}): {
+  findings: DoctorHookFinding[];
+  totalCalls: number;
+  coverage: "complete" | "partial";
+  quality: HookTimingReadQuality;
+} {
+  const hours = opts.hours ?? 24;
+  const warnMs = opts.warnP95Ms ?? 50;
+  const failMs = opts.failP95Ms ?? 200;
+  const detailed = readHookTimingsDetailed({
+    path: opts.path,
+    sinceMs: Date.now() - hours * 3_600_000,
+  });
+  const records = detailed.records;
+  const aggregates = computeAggregates(records, hours);
+  const totalCalls = aggregates.reduce((s, a) => s + a.calls, 0);
+
+  if (totalCalls === 0) {
+    const findings = detailed.coverage === "partial" ? [{
+      status: "warn" as const,
+      label: "hook timing coverage",
+      detail: coverageWarning(detailed.quality),
+    }] : [];
+    return { findings, totalCalls: 0, coverage: detailed.coverage, quality: detailed.quality };
+  }
+
+  const findings: DoctorHookFinding[] = [];
+  const okRows: DoctorHookFinding[] = [];
+
+  // Sort by p95 desc so the worst offenders come first.
+  const byP95 = [...aggregates].sort((a, b) => b.p95 - a.p95);
+
+  for (const a of byP95) {
+    const pieces = [
+      `${a.calls} call${a.calls === 1 ? "" : "s"}`,
+      `p50 ${a.p50}ms`,
+      `p95 ${a.p95}ms`,
+    ];
+    if (a.timeoutPct > 0) pieces.push(`timeout ${a.timeoutPct.toFixed(1)}%`);
+    if (a.errorPct > 0) pieces.push(`error ${a.errorPct.toFixed(1)}%`);
+
+    if (a.timeoutPct > 0) {
+      findings.push({
+        status: "fail",
+        label: a.hook,
+        detail: pieces.join(" · "),
+        fix: `hook hit the 2s safety net — set ASHLR_HOOK_TIMEOUT_MS to a higher value if expected, otherwise investigate ~/.ashlr/hook-errors.jsonl`,
+      });
+    } else if (a.errorPct > 5) {
+      findings.push({
+        status: "fail",
+        label: a.hook,
+        detail: pieces.join(" · "),
+        fix: `tail ~/.ashlr/hook-errors.jsonl | grep ${a.hook}`,
+      });
+    } else if (a.p95 >= failMs) {
+      findings.push({
+        status: "fail",
+        label: a.hook,
+        detail: pieces.join(" · "),
+        fix: `hook is slow at p95 — consider reducing work in /${a.hook}/ entry script`,
+      });
+    } else if (a.p95 >= warnMs) {
+      findings.push({
+        status: "warn",
+        label: a.hook,
+        detail: pieces.join(" · "),
+      });
+    } else {
+      okRows.push({
+        status: "ok",
+        label: a.hook,
+        detail: pieces.join(" · "),
+      });
+    }
+  }
+
+  // Keep the top-3 healthy rows for context, prefer most-called.
+  const okByCalls = [...okRows].sort((a, b) => {
+    const aAgg = aggregates.find((x) => x.hook === a.label);
+    const bAgg = aggregates.find((x) => x.hook === b.label);
+    return (bAgg?.calls ?? 0) - (aAgg?.calls ?? 0);
+  });
+  findings.push(...okByCalls.slice(0, 3));
+
+  if (detailed.coverage === "partial") {
+    findings.unshift({
+      status: "warn",
+      label: "hook timing coverage",
+      detail: coverageWarning(detailed.quality),
+    });
+  }
+  return { findings, totalCalls, coverage: detailed.coverage, quality: detailed.quality };
+}
+
+// ---------------------------------------------------------------------------
+// CLI entry
+// ---------------------------------------------------------------------------
+
+if (import.meta.main) {
+  let hours = 24;
+  const argv = process.argv.slice(2);
+  const hi = argv.indexOf("--hours");
+  if (hi !== -1 && argv[hi + 1]) {
+    const parsed = parseInt(argv[hi + 1]!, 10);
+    if (!isNaN(parsed) && parsed > 0) hours = parsed;
+  }
+  const compare = argv.includes("--compare");
+  const flagsOnly = argv.includes("--flags-only");
+
+  try {
+    const out = buildHookTimingsReport({ hours, compare, flagsOnly });
+    if (out) process.stdout.write(out + "\n");
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    process.stdout.write(`hook-timings-report failed: ${msg}\n`);
+  }
+  process.exit(0);
+}