@polymorphism-tech/morph-spec 8.37.1 → 8.40.0

package/src/lib/telemetry/metrics.js

@@ -0,0 +1,386 @@
+/**
+ * Loop measurements over the harness event stream: turns, tool calls split by
+ * orchestrator and sub-agent, time per command class, identical reruns, file
+ * rereads, feature-scope verify count, git calls per task and whether the
+ * reuse validator ran. Computed the same way on every window so a later
+ * window compares with an earlier one.
+ *
+ * Every number degrades to null ("not measurable here") when the stream lacks
+ * the field it needs — a v1 event never counts as a zero.
+ *
+ * A rerun is the same producer command (what runs before the output filter:
+ * `dotnet test | tail -25` then `| tail -20` is one test run twice) in the same
+ * feature within 10 minutes of its previous run. `rate` counts the repeats;
+ * `burstRate` counts every run that belongs to a burst, the first included.
+ */
+
+import { ingestedBytes } from './telemetry-reader.js';
+import { producerOf } from '../../../framework/hooks/shared/command-tokens.js';
+import { isFailedToolCall } from '../../../framework/hooks/shared/telemetry-logger.js';
+
+export const RERUN_WINDOW_MS = 10 * 60 * 1000;
+
+const RERUN_CLASSES = new Set(['test', 'build', 'verify']);
+const READ_ONLY = new Set(['grep', 'rg', 'sed', 'ls', 'cat', 'find', 'head', 'tail', 'wc', 'tree', 'awk']);
+const TOOL_EVENTS = new Set(['PostToolUse', 'PostToolUseFailure']);
+const NO_FEATURE = '\u0000';
+
+/**
+ * The command without quoted text and heredoc bodies: `morph-spec verify`
+ * inside a commit message is not a verify run.
+ */
+function executedText(command) {
+  const beforeHeredoc = command.split('<<', 1)[0];
+  return beforeHeredoc.replace(/"(?:[^"\\]|\\.)*"|'[^']*'/g, (q) => q[0] + ' '.repeat(q.length - 2) + q[0]);
+}
+
+/**
+ * Kind (fine) and class (coarse) of a shell command. Searched anywhere in the
+ * command, most specific first, so `cd x && dotnet test` is a test.
+ *
+ * @param {string} command
+ * @returns {{ kind: string, klass: string }}
+ */
+export function classifyCommand(command) {
+  const cmd = String(command || '');
+  const verify = /\bmorph-spec(?:\.js)?\s+verify\s+[^\s;&|)]+(\s+T\d+\b)?/.exec(executedText(cmd));
+  if (verify) return { kind: verify[1] ? 'verify-task' : 'verify-feature', klass: 'verify' };
+  if (/\bdotnet\s+test\b/.test(cmd)) {
+    if (/--list-tests\b/.test(cmd)) return { kind: 'dotnet-test-list', klass: 'test' };
+    return { kind: /--filter\b/.test(cmd) ? 'dotnet-test-filtered' : 'dotnet-test-full', klass: 'test' };
+  }
+  if (/\b(?:(?:npm|pnpm|yarn)\s+(?:run\s+)?test|vitest|jest|node\s+--test|run-tests\.mjs)\b/.test(cmd)) {
+    return { kind: 'js-test', klass: 'test' };
+  }
+  if (/\b(?:dotnet\s+build|(?:npm|pnpm|yarn)\s+(?:run\s+)?build|next\s+build|tsc)\b/.test(cmd)) {
+    return { kind: 'build', klass: 'build' };
+  }
+  if (/\b(?:morph-spec\s+e2e|playwright\s+test|docker\s+compose\s+up)\b/.test(cmd)) return { kind: 'e2e', klass: 'e2e' };
+  const git = /\bgit\s+(?:-C\s+(?:"[^"]*"|\S+)\s+)?([a-z][a-z-]*)/.exec(cmd);
+  if (git) return { kind: `git-${git[1]}`, klass: 'git' };
+  const morph = /\bmorph-spec\s+([a-z][a-z-]*)/.exec(cmd);
+  if (morph) return { kind: `morph-spec-${morph[1]}`, klass: 'morph-spec' };
+  const first = cmd.trim().split(/\s+/, 1)[0];
+  if (READ_ONLY.has(first)) return { kind: first, klass: 'read' };
+  return { kind: 'other', klass: 'other' };
+}
+
+/** `git status` and `git diff` occurrences — `git status && git diff` is two calls. */
+function gitCalls(command) {
+  const text = executedText(String(command || ''));
+  const count = (sub) => (text.match(new RegExp(`\\bgit\\s+(?:-C\\s+(?:"[^"]*"|\\S+)\\s+)?${sub}\\b`, 'g')) || []).length;
+  return { status: count('status'), diff: count('diff') };
+}
+
+function median(values) {
+  const v = values.filter((x) => Number.isFinite(x)).sort((a, b) => a - b);
+  if (!v.length) return null;
+  const mid = Math.floor(v.length / 2);
+  return v.length % 2 ? v[mid] : (v[mid - 1] + v[mid]) / 2;
+}
+
+function rate(part, whole) {
+  return whole ? Math.round((part / whole) * 1000) / 1000 : null;
+}
+
+function isV2(e) {
+  return Number.isInteger(e.schemaVersion) && e.schemaVersion >= 2;
+}
+
+function whoOf(e) {
+  if (e.agentId) return 'subagent';
+  return isV2(e) ? 'orchestrator' : 'unknown';
+}
+
+function newCounters() {
+  return {
+    toolCalls: { total: 0, orchestrator: 0, subagent: 0, unknown: 0 },
+    turns: { total: 0, orchestrator: 0, subagent: 0, unknown: 0 },
+    timeByClass: {},
+    reruns: { runs: 0, reruns: 0, inBursts: 0 },
+    git: { status: 0, diff: 0 },
+    reads: { reads: 0, bytes: 0, redundantBytes: 0, windowed: 0 },
+  };
+}
+
+function addTime(counters, klass, durationMs) {
+  const slot = counters.timeByClass[klass] ||= { runs: 0, measuredMs: 0, unmeasured: 0 };
+  slot.runs += 1;
+  if (Number.isFinite(durationMs)) slot.measuredMs += durationMs;
+  else slot.unmeasured += 1;
+}
+
+function finishCounters(c, turnsObserved) {
+  return {
+    toolCalls: c.toolCalls,
+    turns: turnsObserved ? c.turns : null,
+    timeByClass: c.timeByClass,
+    reruns: { ...c.reruns, rate: rate(c.reruns.reruns, c.reruns.runs), burstRate: rate(c.reruns.inBursts, c.reruns.runs) },
+    git: c.git,
+    rereads: { ...c.reads, rate: rate(c.reads.redundantBytes, c.reads.bytes) },
+  };
+}
+
+/**
+ * Why a command ran again within the window. Edits in between and a changed
+ * output filter need no exit code; the pass/fail split does, so a masked or v1
+ * exit code falls back to "after an edit or not".
+ */
+function rerunClass(prev, current, editsBetween) {
+  if (isV2(prev) && prev.outcome === 'timeout') return 'timeout';
+  if (editsBetween === 0 && prev.command !== current.command) return 'refilter';
+  if (!isV2(prev) || !isV2(current) || prev.exitMasked || current.exitMasked) {
+    return editsBetween > 0 ? 'unknown-after-edit' : 'unknown-no-edit';
+  }
+  const prevFailed = prev.outcome === 'error' || (Number.isInteger(prev.exitCode) && prev.exitCode !== 0);
+  const nowFailed = current.outcome === 'error' || (Number.isInteger(current.exitCode) && current.exitCode !== 0);
+  if (prevFailed) {
+    if (editsBetween > 0) return 'fix-loop';
+    return nowFailed ? 'retry' : 'flake';
+  }
+  return editsBetween > 0 ? 're-verify' : 'redundant';
+}
+
+function reuseStateOf(e) {
+  const skipped = (e.nodes?.validators?.skipped || []).find((s) => s?.validator === 'reuse');
+  if (skipped) return { state: 'skipped', reason: skipped.reason || null };
+  if ((e.nodes?.validators?.ran || []).includes('reuse')) return { state: 'ran', reason: null };
+  return { state: 'absent', reason: null };
+}
+
+/**
+ * @param {Object[]} events - deduplicated, time-ordered (loadTelemetry)
+ * @param {{ feature?: string, top?: number }} [opts]
+ * @returns {Object} see framework/docs/CLI.md, "cost --metrics"
+ */
+export function computeMetrics(events, opts = {}) {
+  const top = Number.isInteger(opts.top) && opts.top > 0 ? opts.top : 10;
+  const scoped = (events || []).filter((e) => e && typeof e === 'object' && (!opts.feature || e.feature === opts.feature));
+
+  const global = newCounters();
+  const byFeature = new Map();
+  const openTask = new Map();
+  const tasks = [];
+  const lastRun = new Map();
+  const editCount = new Map();
+  const readsSeen = new Set();
+  const files = new Map();
+  const rerunClasses = {};
+  const rerunByKind = {};
+  const verifyByFeature = {};
+  const fullSuite = { runs: 0, withBuild: 0, byPhase: {} };
+  const lastFingerprint = new Map();
+  const unchangedRepeats = {};
+  const reuse = { runs: 0, byTree: {}, skipReasons: {} };
+  const tasksOf = new Map();
+  let callsInTaskWindows = 0;
+  let v2 = 0;
+  let legacy = 0;
+  let turnsObserved = false;
+  let from = null;
+  let to = null;
+
+  const featureCounters = (key) => {
+    if (!byFeature.has(key)) byFeature.set(key, newCounters());
+    return byFeature.get(key);
+  };
+  const closeTask = (key, endTs, status) => {
+    const task = openTask.get(key);
+    if (!task) return;
+    openTask.delete(key);
+    task.endTs = endTs;
+    task.status = status;
+    task.open = endTs === null;
+    tasks.push(task);
+  };
+
+  for (const e of scoped) {
+    if (isV2(e)) v2 += 1; else legacy += 1;
+    if (e.ts) {
+      from ??= e.ts;
+      to = e.ts;
+    }
+    const fkey = e.feature || NO_FEATURE;
+
+    if (e.event === 'TaskStart' && e.task) {
+      if (!tasksOf.has(fkey)) tasksOf.set(fkey, new Set());
+      tasksOf.get(fkey).add(e.task);
+      closeTask(fkey, null, null);
+      openTask.set(fkey, { feature: e.feature ?? null, task: e.task, startTs: e.ts, endTs: null, status: null, open: true, c: newCounters() });
+      continue;
+    }
+    if (e.event === 'TaskEnd' && e.task) {
+      if (openTask.get(fkey)?.task === e.task) closeTask(fkey, e.ts, e.status ?? null);
+      continue;
+    }
+
+    const sinks = [global, featureCounters(fkey)];
+    const task = openTask.get(fkey);
+    if (task) sinks.push(task.c);
+
+    if (e.event === 'ToolBatch') {
+      turnsObserved = true;
+      for (const c of sinks) { c.turns.total += 1; c.turns[whoOf(e)] += 1; }
+      continue;
+    }
+
+    if (e.event === 'Verify') {
+      if (e.scope === 'feature' && e.fingerprint && e.persisted !== false) {
+        if (lastFingerprint.get(fkey) === e.fingerprint) {
+          unchangedRepeats[e.feature || '(sem feature)'] = (unchangedRepeats[e.feature || '(sem feature)'] || 0) + 1;
+        }
+        lastFingerprint.set(fkey, e.fingerprint);
+      }
+      reuse.runs += 1;
+      const tree = e.tree || 'unknown';
+      const { state, reason } = reuseStateOf(e);
+      const slot = reuse.byTree[tree] ||= { ran: 0, skipped: 0, absent: 0 };
+      slot[state] += 1;
+      if (reason) reuse.skipReasons[reason] = (reuse.skipReasons[reason] || 0) + 1;
+      continue;
+    }
+
+    if (!TOOL_EVENTS.has(e.event) || !e.tool) continue;
+    for (const c of sinks) { c.toolCalls.total += 1; c.toolCalls[whoOf(e)] += 1; }
+    if (task) callsInTaskWindows += 1;
+
+    if (e.tool === 'Write' || e.tool === 'Edit') {
+      if (!isFailedToolCall(e)) editCount.set(fkey, (editCount.get(fkey) || 0) + 1);
+      continue;
+    }
+
+    if (e.tool === 'Read') {
+      const bytes = ingestedBytes(e);
+      if (bytes === null || !e.target) continue;
+      const window = `${fkey}\u0000${e.target}\u0000${e.offset || 1}\u0000${e.limit || 0}`;
+      const redundant = readsSeen.has(window) ? bytes : 0;
+      readsSeen.add(window);
+      for (const c of sinks) {
+        c.reads.reads += 1;
+        c.reads.bytes += bytes;
+        c.reads.redundantBytes += redundant;
+        if (Number.isFinite(e.bytesRead)) c.reads.windowed += 1;
+      }
+      const fileKey = `${fkey}\u0000${e.target}`;
+      const file = files.get(fileKey) || { feature: e.feature ?? null, file: e.target, reads: 0, bytes: 0, redundantBytes: 0 };
+      file.reads += 1;
+      file.bytes += bytes;
+      file.redundantBytes += redundant;
+      files.set(fileKey, file);
+      continue;
+    }
+
+    if (e.tool !== 'Bash' && e.tool !== 'PowerShell') continue;
+    const { kind, klass } = classifyCommand(e.command);
+    const git = gitCalls(e.command);
+    for (const c of sinks) {
+      addTime(c, klass, e.durationMs);
+      c.git.status += git.status;
+      c.git.diff += git.diff;
+    }
+
+    if (kind === 'dotnet-test-full') {
+      const rebuilds = !/--no-build\b/.test(e.command);
+      const phase = fullSuite.byPhase[e.phase || '-'] ||= { runs: 0, withBuild: 0 };
+      fullSuite.runs += 1;
+      phase.runs += 1;
+      if (rebuilds) { fullSuite.withBuild += 1; phase.withBuild += 1; }
+    }
+
+    if (klass === 'verify') {
+      const slot = verifyByFeature[e.feature || '(sem feature)'] ||= { feature: 0, task: 0 };
+      slot[kind === 'verify-feature' ? 'feature' : 'task'] += 1;
+    }
+
+    if (!RERUN_CLASSES.has(klass) || typeof e.command !== 'string') continue;
+    const runKey = `${fkey}\u0000${producerOf(e.command)}`;
+    const prev = lastRun.get(runKey);
+    const edits = editCount.get(fkey) || 0;
+    const t = Date.parse(e.ts ?? '');
+    const isRerun = Boolean(prev) && Number.isFinite(t) && t - prev.t <= RERUN_WINDOW_MS;
+    const kindSlot = rerunByKind[kind] ||= { runs: 0, reruns: 0, inBursts: 0 };
+    kindSlot.runs += 1;
+    for (const c of sinks) c.reruns.runs += 1;
+    const run = { t: Number.isFinite(t) ? t : 0, edits, event: e, sinks, kindSlot, inBurst: false };
+    if (isRerun) {
+      kindSlot.reruns += 1;
+      for (const c of sinks) c.reruns.reruns += 1;
+      // A burst counts its first run too; that one is only known once the
+      // second arrives, so it is credited to where it ran, late.
+      for (const r of prev.inBurst ? [run] : [prev, run]) {
+        r.inBurst = true;
+        r.kindSlot.inBursts += 1;
+        for (const c of r.sinks) c.reruns.inBursts += 1;
+      }
+      const why = rerunClass(prev.event, e, edits - prev.edits);
+      rerunClasses[why] = (rerunClasses[why] || 0) + 1;
+    }
+    lastRun.set(runKey, run);
+  }
+
+  for (const key of [...openTask.keys()]) closeTask(key, null, null);
+  tasks.sort((a, b) => String(a.startTs).localeCompare(String(b.startTs)));
+
+  const taskRows = tasks.map((t) => ({
+    feature: t.feature,
+    task: t.task,
+    startTs: t.startTs,
+    endTs: t.endTs,
+    open: t.open,
+    status: t.status,
+    ...finishCounters(t.c, turnsObserved),
+  }));
+
+  const featureRows = [...byFeature.entries()]
+    .filter(([key]) => key !== NO_FEATURE)
+    .map(([feature, c]) => ({
+      feature,
+      ...finishCounters(c, turnsObserved),
+      verify: verifyByFeature[feature] || { feature: 0, task: 0 },
+    }))
+    .sort((a, b) => b.toolCalls.total - a.toolCalls.total || a.feature.localeCompare(b.feature));
+
+  // A task window only holds what ran between its TaskStart and TaskEnd, and
+  // a parallel batch or a late `in_progress` leaves most calls outside any
+  // window. Dividing each feature's totals by its task count needs no window.
+  const perTaskOfFeature = (pick) => median(featureRows
+    .filter((f) => tasksOf.has(f.feature))
+    .map((f) => Math.round((pick(f) / tasksOf.get(f.feature).size) * 10) / 10));
+  const byKind = {};
+  for (const [kind, slot] of Object.entries(rerunByKind)) {
+    byKind[kind] = { ...slot, rate: rate(slot.reruns, slot.runs), burstRate: rate(slot.inBursts, slot.runs) };
+  }
+
+  return {
+    scope: { feature: opts.feature ?? null },
+    window: { from, to, events: scoped.length },
+    coverage: { v2, legacy },
+    ...finishCounters(global, turnsObserved),
+    rerunsByKind: byKind,
+    rerunClasses,
+    topRereads: [...files.values()]
+      .filter((f) => f.redundantBytes > 0)
+      .sort((a, b) => b.redundantBytes - a.redundantBytes || a.file.localeCompare(b.file))
+      .slice(0, top),
+    verify: {
+      byFeature: verifyByFeature,
+      featureScopeMedian: median(featureRows.map((f) => f.verify.feature).filter((n) => n > 0)),
+      unchangedRepeats,
+    },
+    gitPerTaskMedian: {
+      status: median(taskRows.map((t) => t.git.status)),
+      diff: median(taskRows.map((t) => t.git.diff)),
+    },
+    turnsPerTaskMedian: turnsObserved ? median(taskRows.map((t) => t.turns.total)) : null,
+    taskWindowCoverage: rate(callsInTaskWindows, global.toolCalls.total),
+    perTaskByFeature: {
+      git: perTaskOfFeature((f) => f.git.status + f.git.diff),
+      toolCalls: perTaskOfFeature((f) => f.toolCalls.total),
+      turns: turnsObserved ? perTaskOfFeature((f) => f.turns.total) : null,
+    },
+    dotnetFullSuite: fullSuite,
+    reuse,
+    features: featureRows,
+    tasks: taskRows,
+  };
+}