@tech-leads-club/harness-toolkit 0.2.1

package/src/core/observability/observability.store.ts

@@ -0,0 +1,269 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { appendRecord, readTail } from "../../platform/fs-jsonl.ts";
+import { projectStateDir, runtimeSpoolPath, runtimeStateDir } from "../../platform/paths.ts";
+import { sanitizeSegment } from "../../platform/sanitize.ts";
+import type { AuditRecord, ObsEvent } from "./observability.types.ts";
+
+export type SessionRollup = {
+  session_id: string;
+  provider: string;
+  started_at: string;
+  updated_at: string;
+  models: Record<string, number>;
+  tools: Record<string, { ok: number; fail: number; ms: number }>;
+  subagents: Record<string, { count: number; models: Record<string, number> }>;
+  gates: { pass: number; fail: number };
+  denials: number;
+  prompts: number;
+  responses: number;
+  thoughts: number;
+  comped: number;
+  /**
+   * why: `byRule` exists because a count on its own is not actionable. "Seven interruptions" tells an operator
+   * nothing about which switch to reach for; "six from the paired posture, one from the catastrophic rule" does.
+   */
+  shell: { allow: number; ask: number; deny: number; byRule: Record<string, number> };
+  /**
+   * why: every refusal the harness made, shell and otherwise, keyed by the rule that made it. This is what turns
+   * "the harness interrupted me a lot" into "this rule interrupted you a lot", which is the only form an operator
+   * can act on ([/decisions/ad-027.md](/decisions/ad-027.md)).
+   */
+  railsByRule: Record<string, number>;
+  /** Per-gate pass/fail, so a single flaky gate is distinguishable from a codebase that does not build. */
+  gatesByName: Record<string, { pass: number; fail: number }>;
+  /**
+   * why: an operator reported thirty minutes and had no way to see where it went. The duration was already recorded
+   * on every gate outcome and surfaced nowhere. `worstMs` separates "one slow run" from "many runs"
+   * ([/decisions/ad-033.md](/decisions/ad-033.md)).
+   */
+  gateTime: Record<string, { runs: number; totalMs: number; worstMs: number; reused?: number }>;
+  /** Characters of prose the harness injected at session start. The cost side of every rail, in one number. */
+  injected_chars: number;
+  /**
+   * hazard: `injected_chars` was reported as the price paid on every turn. On a host that drops hook context it is
+   * paid never, and the durable rules file — which asks to be included on every request — was absent from the
+   * rollup. The number was wrong in both directions there ([/decisions/ad-050.md](/decisions/ad-050.md)).
+   */
+  durable_chars: number;
+  /** What the provider declares about delivering context from its session-start hook. */
+  hook_context_reliable: boolean;
+  mcp: Record<string, number>;
+  estimated_cost_usd: number;
+  cost_incomplete: boolean;
+  input_tokens: number;
+  output_tokens: number;
+  cost_alert_sent: boolean;
+};
+
+function safeMkdir(dir: string): boolean {
+  try {
+    mkdirSync(dir, { recursive: true });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export type SpoolStream = "obs" | "audit";
+
+export type SpoolEnvelope = {
+  repo: string;
+  project: string;
+  stream: SpoolStream;
+  record: unknown;
+};
+
+export function spoolEnvelope(root: string, stream: SpoolStream, record: unknown): SpoolEnvelope {
+  return { repo: root, project: basename(root), stream, record };
+}
+
+// hazard: best-effort by contract. The project's own state directory holds the authoritative copy, so a
+// read-only runtime home, a full disk or a concurrent writer must degrade to project-only recording
+// rather than change what the caller is told.
+export function appendSpoolRecord(root: string, stream: SpoolStream, record: unknown): boolean {
+  try {
+    if (!safeMkdir(runtimeStateDir())) {
+      return false;
+    }
+    appendRecord(runtimeSpoolPath(), spoolEnvelope(root, stream, record));
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function appendObsRecord(root: string, file: string, event: ObsEvent, spool = false): boolean {
+  if (!safeMkdir(projectStateDir(root))) {
+    return false;
+  }
+  try {
+    appendRecord(join(projectStateDir(root), file), event);
+  } catch {
+    return false;
+  }
+  if (spool) {
+    appendSpoolRecord(root, "obs", event);
+  }
+  return true;
+}
+
+export function appendAuditRecord(root: string, record: AuditRecord, spool = false): boolean {
+  if (!safeMkdir(projectStateDir(root))) {
+    return false;
+  }
+  try {
+    appendRecord(join(projectStateDir(root), "audit.jsonl"), record);
+  } catch {
+    return false;
+  }
+  if (spool) {
+    appendSpoolRecord(root, "audit", record);
+  }
+  return true;
+}
+
+function spoolLineTimestamp(line: string): number | null {
+  try {
+    const parsed = JSON.parse(line) as SpoolEnvelope;
+    const record = parsed.record as { ts?: unknown } | null;
+    const ts = typeof record?.ts === "string" ? Date.parse(record.ts) : Number.NaN;
+    return Number.isNaN(ts) ? null : ts;
+  } catch {
+    return null;
+  }
+}
+
+// hazard: rewriting a file every repository on the machine appends to can drop a line written between the
+// read and the rename. That is acceptable here and only here: the spool is a derived cross-repo view, and
+// the per-project records it mirrors are never touched.
+export function pruneSpool(retentionDays: number, now = Date.now()): number {
+  const path = runtimeSpoolPath();
+  if (!existsSync(path)) {
+    return 0;
+  }
+  const cutoff = now - retentionDays * 24 * 60 * 60 * 1000;
+  let lines: string[] = [];
+  try {
+    lines = readFileSync(path, "utf8")
+      .split("\n")
+      .filter((line) => line.trim().length > 0);
+  } catch {
+    return 0;
+  }
+  const kept = lines.filter((line) => {
+    const ts = spoolLineTimestamp(line);
+    return ts === null || ts >= cutoff;
+  });
+  if (kept.length === lines.length) {
+    return 0;
+  }
+  try {
+    writeFileSync(path, kept.length > 0 ? `${kept.join("\n")}\n` : "", "utf8");
+  } catch {
+    return 0;
+  }
+  return lines.length - kept.length;
+}
+
+export function readSignalEvents(root: string, file: string, limit = 200): ObsEvent[] {
+  try {
+    return readTail<ObsEvent>(join(projectStateDir(root), file), limit);
+  } catch {
+    return [];
+  }
+}
+
+function rollupPath(root: string, sessionKey: string): string {
+  return join(projectStateDir(root), "sessions", `${sanitizeSegment(sessionKey)}.json`);
+}
+
+function readJson<T>(path: string): T | null {
+  if (!existsSync(path)) {
+    return null;
+  }
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as T;
+  } catch {
+    return null;
+  }
+}
+
+export function newRollup(sessionKey: string, provider: string): SessionRollup {
+  const now = new Date().toISOString();
+  return {
+    session_id: sessionKey,
+    provider,
+    started_at: now,
+    updated_at: now,
+    models: {},
+    tools: {},
+    subagents: {},
+    gates: { pass: 0, fail: 0 },
+    denials: 0,
+    prompts: 0,
+    responses: 0,
+    thoughts: 0,
+    comped: 0,
+    shell: { allow: 0, ask: 0, deny: 0, byRule: {} },
+    railsByRule: {},
+    gatesByName: {},
+    gateTime: {},
+    injected_chars: 0,
+    durable_chars: 0,
+    // why: the pessimistic default. A rollup whose session start never ran claims no delivery rather than claiming
+    // one it did not observe.
+    hook_context_reliable: false,
+    mcp: {},
+    estimated_cost_usd: 0,
+    cost_incomplete: false,
+    input_tokens: 0,
+    output_tokens: 0,
+    cost_alert_sent: false,
+  };
+}
+
+export function loadRollup(root: string, sessionKey: string, provider: string): SessionRollup {
+  return readJson<SessionRollup>(rollupPath(root, sessionKey)) ?? newRollup(sessionKey, provider);
+}
+
+export function saveRollup(root: string, rollup: SessionRollup): boolean {
+  const dir = join(projectStateDir(root), "sessions");
+  if (!safeMkdir(dir)) {
+    return false;
+  }
+  rollup.updated_at = new Date().toISOString();
+  try {
+    writeFileSync(rollupPath(root, rollup.session_id), `${JSON.stringify(rollup, null, 2)}\n`, "utf8");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export function getRollup(root: string, sessionKey: string): SessionRollup | null {
+  return readJson<SessionRollup>(rollupPath(root, sessionKey));
+}
+
+export function pruneObs(root: string, retentionDays: number): void {
+  const dir = join(projectStateDir(root), "sessions");
+  if (!existsSync(dir)) {
+    return;
+  }
+  const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1000;
+  let names: string[] = [];
+  try {
+    names = readdirSync(dir);
+  } catch {
+    return;
+  }
+  for (const name of names) {
+    const full = join(dir, name);
+    const data = readJson<SessionRollup>(full);
+    if (data && Date.parse(data.updated_at) < cutoff) {
+      try {
+        unlinkSync(full);
+      } catch {}
+    }
+  }
+}

package/src/core/observability/observability.types.ts

@@ -0,0 +1,193 @@
+import type { HarnessEventKind } from "../../contracts/harness-event.ts";
+
+export type ObsLevel = "signal" | "debug";
+
+export type ObsKind =
+  | "session.start"
+  | "session.end"
+  | "generation.start"
+  | "generation.end"
+  | "tool.start"
+  | "tool.end"
+  | "tool.fail"
+  | "shell.start"
+  | "shell.end"
+  | "mcp.start"
+  | "mcp.end"
+  | "subagent.start"
+  | "subagent.end"
+  | "file.edit"
+  | "file.read"
+  | "prompt.submit"
+  | "agent.response"
+  | "agent.thought"
+  | "compact"
+  | "gate.outcome"
+  | "cost.turn"
+  | "cost.session_alert"
+  | "ship.claim"
+  | "policy.deny"
+  // why: an observation is not a refusal. Sharing the kind would inflate the denial count with readings the
+  // operator never lived through, which is the opposite of what the rail exists for.
+  | "policy.observe";
+
+export type CostPool = "provider_native" | "other" | "auto" | "unknown";
+
+export type CostSource = "provider" | "litellm" | "override" | "billed" | "missing";
+
+export type ObsEvent = {
+  schema: "harness.observability.v1";
+  provider: string;
+  kind: ObsKind;
+  level: ObsLevel;
+  ts: string;
+  trace_id: string;
+  span_id: string;
+  parent_span_id?: string;
+  session_id?: string;
+  model?: string;
+  attrs: Record<string, unknown>;
+  // invariant: every field here is written by `usageGenAi`. `operation_name`, `provider_name`, `request_model`,
+  // `reasoning_tokens` and `outcome` were declared on the shipped schema and populated by nothing, which is the
+  // same defect as a consumer without a producer pointed the other way ([/decisions/ad-041.md](/decisions/ad-041.md)).
+  gen_ai?: {
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_read_tokens?: number;
+    cache_write_tokens?: number;
+    cost_usd?: number | null;
+    cost_source?: CostSource;
+    cost_pool?: CostPool;
+    duration_ms?: number;
+  };
+};
+
+export type AuditRecord = {
+  ts: string;
+  event: string;
+  payload: unknown;
+};
+
+export type ObservabilityConfig = {
+  enabled: boolean;
+  signalPath: "obs.jsonl";
+  debugPath: "debug.jsonl";
+  debugEnabled: boolean;
+  includePayloads: boolean;
+  maxAttrChars: number;
+  sessionCostAlertUsd: number | null;
+  retentionDays: number;
+  maxSignalEvents: number;
+  globalSpool: boolean;
+};
+
+export const DEFAULT_OBS: ObservabilityConfig = {
+  enabled: true,
+  signalPath: "obs.jsonl",
+  debugPath: "debug.jsonl",
+  debugEnabled: false,
+  includePayloads: false,
+  maxAttrChars: 500,
+  sessionCostAlertUsd: 5,
+  retentionDays: 14,
+  maxSignalEvents: 50_000,
+  globalSpool: false,
+};
+
+export const SIGNAL_KINDS = new Set<ObsKind>([
+  "session.start",
+  "session.end",
+  "generation.end",
+  "tool.fail",
+  "subagent.start",
+  "subagent.end",
+  "prompt.submit",
+  "compact",
+  "gate.outcome",
+  "cost.turn",
+  "cost.session_alert",
+  "ship.claim",
+  "policy.deny",
+  "policy.observe",
+]);
+
+export const LIVE_ALLOWLIST = new Set<ObsKind>([
+  "session.start",
+  "session.end",
+  "generation.end",
+  "tool.fail",
+  "shell.end",
+  "subagent.start",
+  "subagent.end",
+  "gate.outcome",
+  "cost.turn",
+  "cost.session_alert",
+  "ship.claim",
+  "policy.deny",
+  "compact",
+  "prompt.submit",
+]);
+
+export function resolveObsLevel(
+  kind: ObsKind,
+  attrs: Record<string, unknown> = {},
+  forceDebug = false,
+): ObsLevel {
+  if (forceDebug) {
+    return "debug";
+  }
+  // hazard: this branch used to key on `shell.end` alone, and a command that was denied or asked about never
+  // reaches an "after" event — so the only permission it could ever grade was `allow`, and `rollup.shell.ask`
+  // could not have moved even once the attribute was written. The decision is made at `shell.start`; that is the
+  // only phase where a non-allow permission exists ([/decisions/ad-026.md](/decisions/ad-026.md)).
+  if (kind === "shell.end" || kind === "shell.start") {
+    const permission = String(attrs.permission ?? "allow");
+    return permission === "allow" ? "debug" : "signal";
+  }
+  if (kind === "mcp.end") {
+    const outcome = String(attrs.outcome ?? attrs.status ?? "success");
+    return outcome === "error" || outcome === "fail" || outcome === "denied" ? "signal" : "debug";
+  }
+  return SIGNAL_KINDS.has(kind) ? "signal" : "debug";
+}
+
+export const EVENT_KIND_TO_OBS_KIND: Record<HarnessEventKind, ObsKind> = {
+  "session.start": "session.start",
+  "session.end": "session.end",
+  "prompt.submit": "prompt.submit",
+  "tool.before": "tool.start",
+  "tool.after": "tool.end",
+  "tool.failure": "tool.fail",
+  "shell.before": "shell.start",
+  "shell.after": "shell.end",
+  "mcp.before": "mcp.start",
+  "mcp.after": "mcp.end",
+  "read.before": "file.read",
+  "edit.after": "file.edit",
+  "subagent.start": "subagent.start",
+  "subagent.stop": "subagent.end",
+  stop: "generation.end",
+  "compact.before": "compact",
+  "response.after": "agent.response",
+  "thought.after": "agent.thought",
+};
+
+const SECRET_KEY = /(token|secret|password|api[_-]?key|authorization|credential|private[_-]?key)/i;
+const SECRET_VALUE = /\b(ghp_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/g;
+
+export function redactDeep(value: unknown): unknown {
+  if (typeof value === "string") {
+    return value.replace(SECRET_VALUE, "[REDACTED]");
+  }
+  if (Array.isArray(value)) {
+    return value.map(redactDeep);
+  }
+  if (value && typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [key, nested] of Object.entries(value as Record<string, unknown>)) {
+      out[key] = SECRET_KEY.test(key) ? "[REDACTED]" : redactDeep(nested);
+    }
+    return out;
+  }
+  return value;
+}

package/src/core/observability/observability.why.ts

@@ -0,0 +1,206 @@
+import { type ColorName, PLAIN, type Style, SYMBOLS } from "../../platform/style.ts";
+import type { ObsEvent } from "./observability.types.ts";
+
+/**
+ * why: an operator watching an agent cannot tell a harness decision from model behaviour, because a hook answers
+ * the host and the host decides whether to show it. `obs report` is a session rollup and `obs live` is a firehose;
+ * neither answers "did the harness just do that?" ([/decisions/ad-062.md](/decisions/ad-062.md)).
+ *
+ * invariant: derived from the two obs planes and nothing else. A second producer would be a second index, and the
+ * planes are already the record of record.
+ */
+export type HarnessDecision = {
+  ts: string;
+  /** What the harness was answering — the event kind, or the gate's name. */
+  about: string;
+  /** What it did: deny, ask, allow, pass, fail, or context. */
+  verdict: string;
+  /** The rule responsible, or `null` when the record carries none. */
+  rule: string | null;
+  /** One line of what it was about — a command, a tool, a size. Never the model's words. */
+  detail: string;
+};
+
+const NONE = new Set(["none", "", "undefined"]);
+
+function attr(event: ObsEvent, name: string): string | undefined {
+  const value = (event.attrs as Record<string, unknown> | undefined)?.[name];
+  return value === undefined || value === null ? undefined : String(value);
+}
+
+function ruleOf(event: ObsEvent): string | null {
+  const raw = attr(event, "rule");
+  return raw === undefined || NONE.has(raw) ? null : raw;
+}
+
+function truncate(text: string | undefined, max = 90): string {
+  if (!text) {
+    return "";
+  }
+  const flat = text.replace(/\s+/g, " ").trim();
+  return flat.length <= max ? flat : `${flat.slice(0, max - 1)}…`;
+}
+
+/**
+ * invariant: only records that represent a decision. Everything else in the planes is activity — a turn doing
+ * work is not the harness doing something, and listing it would bury the four lines that matter.
+ */
+export function decisionsFrom(events: readonly ObsEvent[], sessionKey?: string): HarnessDecision[] {
+  const out: HarnessDecision[] = [];
+  for (const event of events) {
+    if (sessionKey !== undefined && event.session_id !== sessionKey) {
+      continue;
+    }
+    if (event.kind === "policy.deny") {
+      out.push({
+        ts: event.ts,
+        about: attr(event, "event") ?? "tool",
+        verdict: attr(event, "permission") ?? "deny",
+        rule: ruleOf(event),
+        detail: truncate(attr(event, "tool_name")),
+      });
+      continue;
+    }
+    if (event.kind === "shell.start") {
+      out.push({
+        ts: event.ts,
+        about: "shell",
+        verdict: attr(event, "permission") ?? "allow",
+        rule: ruleOf(event),
+        detail: truncate(attr(event, "command")),
+      });
+      continue;
+    }
+    if (event.kind === "gate.outcome") {
+      out.push({
+        ts: event.ts,
+        about: `gate ${attr(event, "gate") ?? "?"}`,
+        verdict: attr(event, "passed") === "true" ? "pass" : "fail",
+        rule: null,
+        detail: truncate(attr(event, "scoped_env") === "none" ? "" : `env: ${attr(event, "scoped_env")}`),
+      });
+      continue;
+    }
+    // hazard: observation mode exists to produce a reading an operator acts on, and no command showed one. The
+    // orphan report found it on its first run ([/decisions/ad-065.md](/decisions/ad-065.md)).
+    if (event.kind === "policy.observe") {
+      out.push({
+        ts: event.ts,
+        about: `observe ${attr(event, "rail") ?? attr(event, "rule") ?? "?"}`,
+        verdict: attr(event, "reading") ?? "observed",
+        rule: ruleOf(event),
+        detail: truncate(attr(event, "violations") ? `${attr(event, "violations")} violation(s)` : ""),
+      });
+      continue;
+    }
+    if (event.kind === "cost.session_alert") {
+      out.push({
+        ts: event.ts,
+        about: "cost alert",
+        verdict: "alert",
+        rule: null,
+        detail: truncate(`$${attr(event, "estimated_cost_usd") ?? "?"} passed the session threshold`),
+      });
+      continue;
+    }
+    if (event.kind === "session.start") {
+      out.push({
+        ts: event.ts,
+        about: "session start",
+        verdict: "context",
+        rule: null,
+        detail: truncate(`${attr(event, "injected_chars") ?? "0"} chars injected`),
+      });
+    }
+  }
+  return out.sort((a, b) => (a.ts === b.ts ? 0 : a.ts < b.ts ? 1 : -1));
+}
+
+export const NOTHING_WAS_THE_HARNESS = [
+  "No harness decision in this window.",
+  "Whatever you just saw was the model, not a rail — the harness allowed everything it was asked about.",
+].join("\n");
+
+export type WhySummary = { denied: number; asked: number; allowed: number; other: number };
+
+export function summarise(decisions: readonly HarnessDecision[]): WhySummary {
+  const tally = { denied: 0, asked: 0, allowed: 0, other: 0 };
+  for (const decision of decisions) {
+    if (decision.verdict === "deny") {
+      tally.denied += 1;
+    } else if (decision.verdict === "ask") {
+      tally.asked += 1;
+    } else if (decision.verdict === "allow") {
+      tally.allowed += 1;
+    } else {
+      tally.other += 1;
+    }
+  }
+  return tally;
+}
+
+/**
+ * why: the empty case is the feature. "Nothing here was the harness" is the sentence an operator cannot get from
+ * any other command, and printing an empty table instead would leave them exactly as unsure as before.
+ */
+export function whyText(
+  decisions: readonly HarnessDecision[],
+  style: Style = PLAIN,
+  now = new Date(),
+): string {
+  if (decisions.length === 0) {
+    return [
+      style.paint("success", `${SYMBOLS.check} ${NOTHING_WAS_THE_HARNESS.split("\n")[0]}`),
+      style.dim(NOTHING_WAS_THE_HARNESS.split("\n")[1] ?? ""),
+    ].join("\n");
+  }
+
+  const tally = summarise(decisions);
+  const today = now.toISOString().slice(0, 10);
+  const parts = [
+    tally.denied > 0 ? style.paint("error", `${tally.denied} denied`) : "",
+    tally.asked > 0 ? style.paint("warning", `${tally.asked} asked`) : "",
+    tally.allowed > 0 ? style.paint("success", `${tally.allowed} allowed`) : "",
+    tally.other > 0 ? style.dim(`${tally.other} other`) : "",
+  ].filter(Boolean);
+
+  const lines = decisions.map((decision) => {
+    // why: the date appears only when the record is not from today. Ten identical timestamps with no date was
+    // the reading that made the first version unreadable — it looked like one burst and spanned three days.
+    const day = decision.ts.slice(0, 10);
+    const when = day === today ? decision.ts.slice(11, 19) : `${day} ${decision.ts.slice(11, 19)}`;
+    const verdictColor: ColorName =
+      decision.verdict === "deny" ? "error" : decision.verdict === "ask" ? "warning" : "success";
+    // why: the `rule=` prefix survives because it is what every denial message prints and what the
+    // troubleshooting page tells people to grep for. Colour is not available when the output is piped.
+    const rule =
+      decision.rule === null ? style.dim("rule=unattributed") : style.paint("info", `rule=${decision.rule}`);
+    const head = [
+      style.dim(when.padEnd(day === today ? 8 : 19)),
+      style.paint("textMuted", decision.about.padEnd(14)),
+      style.paint(verdictColor, decision.verdict.padEnd(7)),
+      rule,
+    ].join(" ");
+    return decision.detail ? `${head}\n${" ".repeat(4)}${style.dim(SYMBOLS.bar)} ${decision.detail}` : head;
+  });
+
+  return [
+    style.heading(`LAST ${decisions.length} HARNESS DECISION${decisions.length === 1 ? "" : "S"}`),
+    `   ${parts.join(style.dim(` ${SYMBOLS.bar} `))}`,
+    "",
+    ...lines,
+    "",
+    style.footer("newest first  ·  tlc harness why 30 widens the window  ·  --json for the records"),
+  ].join("\n");
+}
+
+// invariant: what `decisionsFrom` branches on, declared so `check-obs-contract` can verify both sides. A kind
+// added to the switch and not to this set is a consumer the contract cannot see.
+export const WHY_KINDS = [
+  "policy.deny",
+  "shell.start",
+  "gate.outcome",
+  "session.start",
+  "policy.observe",
+  "cost.session_alert",
+] as const;