mira-harness 0.2.1 → 0.2.3

package/README.md

@@ -106,6 +106,7 @@ a "typing…" fallback for a slow bot — replies run 5–62s) and capture, per
 | `watch` | Live-tail @mira's messages (observe-only). `--peer` |
 | `report` | Distill the run log into Markdown. `--in --out --category` |
 | `stats` | At-a-glance dashboard: totals, latency records, sparkline. `--in --category --json` |
+| `diff` | Compare two run logs for @mira behavioral drift (exit 1 on a regression). `--json --no-fail` |
 
 Run `mira-harness --help` (or `<command> --help`) for full options.
 
@@ -159,6 +160,19 @@ Give a probe an optional `expect` block and `loop` grades it ✓/✗. The checks
 Probes without `expect` stay observe-only (informational). `loop` **exits non-zero** if any
 graded probe fails — so it drops straight into CI. Add `--no-fail` to report without failing.
 
+### Drift detection
+
+`diff` compares two run logs and flags how @mira's behavior **changed** (structural, not
+exact text). **Regressions** — an assertion that flipped ✓→✗, a probe that now times out, a
+>2× latency blow-up — exit non-zero; surface changes (buttons / links / media) and
+improvements are reported but pass. Snapshot a baseline, re-run later, diff:
+
+```bash
+MIRA_RUNS_FILE=baseline.jsonl mira-harness loop --category core   # snapshot a baseline
+mira-harness loop --category core                                # a later run -> mira-runs.jsonl
+mira-harness diff baseline.jsonl                                 # vs the current run log
+```
+
 ## Use as a library
 
 The CLI is a thin frontend over an exported core:

package/dist/cli.js

@@ -430,6 +430,263 @@ function listCatalog(opts = {}) {
   }
 }
 
+// src/commands/diff.ts
+import { resolve as resolve4 } from "node:path";
+
+// src/log.ts
+import { appendFile, mkdir } from "node:fs/promises";
+import { dirname, resolve as resolve2 } from "node:path";
+var RUNS_FILE = resolve2(process.cwd(), process.env.MIRA_RUNS_FILE ?? "mira-runs.jsonl");
+async function appendRun(result, meta = {}) {
+  await mkdir(dirname(RUNS_FILE), { recursive: true });
+  const record = { ...meta, ...result };
+  await appendFile(RUNS_FILE, `${JSON.stringify(record)}
+`, "utf8");
+}
+
+// src/commands/report.ts
+import { readFileSync as readFileSync2, writeFileSync } from "node:fs";
+import { resolve as resolve3 } from "node:path";
+function loadRunRecords(inFile, category) {
+  const path = inFile ? resolve3(process.cwd(), inFile) : RUNS_FILE;
+  const raw = readFileSync2(path, "utf8");
+  const records = [];
+  for (const line of raw.split(`
+`)) {
+    const t = line.trim();
+    if (!t)
+      continue;
+    try {
+      records.push(JSON.parse(t));
+    } catch {}
+  }
+  return category ? records.filter((r) => r.category === category) : records;
+}
+function truncate(s, n) {
+  const flat = s.replace(/\s+/g, " ").trim();
+  return flat.length > n ? `${flat.slice(0, n - 1)}…` : flat;
+}
+function cell(s) {
+  return s.replace(/\|/g, "\\|");
+}
+function signals(r) {
+  const btns = r.messages.reduce((n, m) => n + m.buttons.length, 0);
+  const links = r.messages.reduce((n, m) => n + m.links.length, 0);
+  const media = r.messages.filter((m) => m.media).map((m) => m.media?.kind);
+  const edits = r.messages.reduce((n, m) => n + m.editCount, 0);
+  return [
+    btns ? `${btns}btn` : "",
+    links ? `${links}link` : "",
+    media.length ? `media=${media.join(",")}` : "",
+    edits ? `${edits}edit` : ""
+  ].filter(Boolean).join(" ") || "—";
+}
+function gist(r) {
+  const first = r.messages.find((m) => m.text)?.text ?? (r.messages.length ? "(non-text)" : "(no reply)");
+  return truncate(first, 90);
+}
+function verdictCell(r) {
+  if (!r.assert)
+    return "—";
+  if (r.assert.ok)
+    return "✅";
+  const failed = r.assert.checks.filter((ch) => !ch.ok).map((ch) => ch.name).join(", ");
+  return `❌ ${cell(failed)}`;
+}
+function table(rows) {
+  const head = `| Probe | Test | Sent | Reply (gist) | Signals | First reply | Settled |
+` + "|---|---|---|---|---|---|---|";
+  const body = rows.map((r) => {
+    const id = r.probeId ?? "—";
+    const sent = cell(truncate(r.sent, 40));
+    const reply = r.timedOut ? "**TIMEOUT**" : cell(gist(r));
+    const first = r.firstReplyMs === null ? "—" : `${(r.firstReplyMs / 1000).toFixed(1)}s`;
+    const total = `${(r.totalMs / 1000).toFixed(1)}s`;
+    return `| \`${cell(id)}\` | ${verdictCell(r)} | ${sent} | ${reply} | ${cell(signals(r))} | ${first} | ${total} |`;
+  }).join(`
+`);
+  return `${head}
+${body}`;
+}
+function buildReport(records) {
+  const replied = records.filter((r) => !r.timedOut);
+  const latencies = replied.map((r) => r.firstReplyMs ?? 0).filter((n) => n > 0);
+  const avg = latencies.length ? latencies.reduce((a, b) => a + b, 0) / latencies.length : 0;
+  const max = latencies.length ? Math.max(...latencies) : 0;
+  const graded = records.filter((r) => r.assert);
+  const passed = graded.filter((r) => r.assert?.ok);
+  const out = [];
+  out.push("# Mira(@mira) probe report (auto-generated)");
+  out.push("");
+  out.push("> Distilled from the run log by `mira-harness report`.");
+  out.push("");
+  out.push(`**${records.length}** probes · **${replied.length}** replied · **${records.length - replied.length}** timed out · ` + (graded.length ? `**${passed.length}/${graded.length}** assertions passed · ` : "") + `first-reply avg **${(avg / 1000).toFixed(1)}s** / max **${(max / 1000).toFixed(1)}s**.`);
+  out.push("");
+  const builtin = CATEGORIES.filter((c2) => records.some((r) => r.category === c2));
+  const custom = [];
+  for (const r of records) {
+    if (r.category && !CATEGORIES.includes(r.category) && !custom.includes(r.category)) {
+      custom.push(r.category);
+    }
+  }
+  for (const cat of [...builtin, ...custom]) {
+    out.push(`## ${cat}`, "", table(records.filter((r) => r.category === cat)), "");
+  }
+  const uncategorized = records.filter((r) => !r.category);
+  if (uncategorized.length)
+    out.push("## (uncategorized)", "", table(uncategorized), "");
+  return out.join(`
+`);
+}
+function renderReport(inFile, category) {
+  const records = loadRunRecords(inFile, category);
+  if (!records.length) {
+    throw new Error(category ? `no probes in category "${category}" in the run log.` : "run log is empty.");
+  }
+  return buildReport(records);
+}
+function report(opts) {
+  let md;
+  try {
+    md = renderReport(opts.in, opts.category);
+  } catch (e) {
+    if (e?.code === "ENOENT") {
+      const path = opts.in ? resolve3(process.cwd(), opts.in) : RUNS_FILE;
+      console.error(`no run log at ${path} — run \`mira-harness send\` or \`mira-harness loop\` first.`);
+    } else {
+      console.error(e instanceof Error ? e.message : String(e));
+    }
+    process.exit(1);
+  }
+  if (opts.out) {
+    const dest = resolve3(process.cwd(), opts.out);
+    writeFileSync(dest, `${md}
+`, "utf8");
+    console.error(`wrote report to ${opts.out}`);
+  } else {
+    console.log(md);
+  }
+}
+
+// src/commands/diff.ts
+function lastByProbe(records) {
+  const m = new Map;
+  for (const r of records) {
+    if (r.probeId)
+      m.set(r.probeId, r);
+  }
+  return m;
+}
+function signals2(r) {
+  const buttons = r.messages.reduce((n, m) => n + m.buttons.length, 0);
+  const links = r.messages.reduce((n, m) => n + m.links.length, 0);
+  const media = r.messages.map((m) => m.media?.kind).filter((k) => k !== undefined).join(",");
+  return { buttons, links, media };
+}
+function diffRuns(baseline, current) {
+  const base = lastByProbe(baseline);
+  const cur = lastByProbe(current);
+  const items = [];
+  for (const id of [...new Set([...base.keys(), ...cur.keys()])].sort()) {
+    const b = base.get(id);
+    const c2 = cur.get(id);
+    if (b && !c2) {
+      items.push({ probeId: id, kind: "removed", what: "in baseline, not in current" });
+      continue;
+    }
+    if (c2 && !b) {
+      items.push({ probeId: id, kind: "added", what: "new in current (not in baseline)" });
+      continue;
+    }
+    if (!b || !c2)
+      continue;
+    if (!b.timedOut && c2.timedOut)
+      items.push({ probeId: id, kind: "regression", what: "now times out (was replying)" });
+    else if (b.timedOut && !c2.timedOut)
+      items.push({ probeId: id, kind: "improvement", what: "now replies (was timing out)" });
+    const ba = b.assert;
+    const ca = c2.assert;
+    if (ba && ca && ba.ok !== ca.ok) {
+      if (ba.ok && !ca.ok) {
+        const failed = ca.checks.filter((x) => !x.ok).map((x) => x.name).join(", ");
+        items.push({ probeId: id, kind: "regression", what: `assertion now FAILS (${failed})` });
+      } else {
+        items.push({ probeId: id, kind: "improvement", what: "assertion now passes" });
+      }
+    }
+    if (b.firstReplyMs !== null && c2.firstReplyMs !== null) {
+      const bms = b.firstReplyMs;
+      const cms = c2.firstReplyMs;
+      if (cms > bms * 2 && cms - bms > 5000) {
+        items.push({
+          probeId: id,
+          kind: "regression",
+          what: `first reply ${(bms / 1000).toFixed(1)}s -> ${(cms / 1000).toFixed(1)}s (>2x slower)`
+        });
+      }
+    }
+    const bs = signals2(b);
+    const cs = signals2(c2);
+    if (bs.buttons !== cs.buttons)
+      items.push({ probeId: id, kind: "change", what: `buttons ${bs.buttons} -> ${cs.buttons}` });
+    if (bs.links !== cs.links)
+      items.push({ probeId: id, kind: "change", what: `links ${bs.links} -> ${cs.links}` });
+    if (bs.media !== cs.media)
+      items.push({ probeId: id, kind: "change", what: `media [${bs.media}] -> [${cs.media}]` });
+  }
+  return items;
+}
+function load(path, label) {
+  try {
+    return loadRunRecords(path);
+  } catch (e) {
+    const shown = path ? resolve4(process.cwd(), path) : RUNS_FILE;
+    if (e?.code === "ENOENT")
+      console.error(`${label}: no run log at ${shown}`);
+    else
+      console.error(`${label}: ${e instanceof Error ? e.message : String(e)}`);
+    return null;
+  }
+}
+function diff(opts) {
+  const base = load(opts.baseline, "baseline");
+  const cur = load(opts.current, "current");
+  if (!base || !cur)
+    process.exit(1);
+  if (!base.length)
+    note(c.yellow(`baseline ${opts.baseline} has no records.`));
+  if (!cur.length)
+    note(c.yellow(`current ${opts.current ?? "(run log)"} has no records.`));
+  const items = diffRuns(base, cur);
+  const regressions = items.filter((i) => i.kind === "regression");
+  if (opts.json) {
+    console.log(JSON.stringify({ regressions: regressions.length, drift: items.length, items }, null, 2));
+    if (regressions.length && !opts.noFail)
+      process.exit(1);
+    return;
+  }
+  note(c.bold(`drift  ${opts.baseline} -> ${opts.current ?? "(current run log)"}`));
+  if (!items.length) {
+    note(c.green("  no drift detected."));
+    return;
+  }
+  const paint = {
+    regression: (s) => c.red(s),
+    improvement: (s) => c.green(s),
+    change: (s) => c.dim(s),
+    added: (s) => c.dim(s),
+    removed: (s) => c.dim(s)
+  };
+  for (const i of items) {
+    note(`  ${paint[i.kind](i.kind.padEnd(11))} ${c.cyan(i.probeId)} — ${i.what}`);
+  }
+  note(regressions.length ? c.red(`
+${regressions.length} regression(s).`) : c.green(`
+no regressions.`));
+  if (regressions.length && !opts.noFail)
+    process.exit(1);
+}
+
 // src/client.ts
 import { Api, TelegramClient } from "telegram";
 import { EditedMessage } from "telegram/events/EditedMessage.js";
@@ -602,7 +859,7 @@ function buildCollector(peer, client, t, sent, opts) {
   const collected = new Map;
   const start = Date.now();
   let firstReplyMs = null;
-  return new Promise((resolve2) => {
+  return new Promise((resolve5) => {
     let settleTimer;
     let firstTimer;
     let hardTimer;
@@ -632,7 +889,7 @@ function buildCollector(peer, client, t, sent, opts) {
     const finish = () => {
       cleanup();
       const messages = [...collected.values()].sort((a, b) => a.msg.id - b.msg.id).map(({ msg, editCount }) => extractMessage(msg, editCount));
-      resolve2({
+      resolve5({
         peer,
         sent,
         messages,
@@ -797,19 +1054,6 @@ This string grants full access to your account. Never commit it.
 
 // src/commands/loop.ts
 import { existsSync } from "node:fs";
-
-// src/log.ts
-import { appendFile, mkdir } from "node:fs/promises";
-import { dirname, resolve as resolve2 } from "node:path";
-var RUNS_FILE = resolve2(process.cwd(), process.env.MIRA_RUNS_FILE ?? "mira-runs.jsonl");
-async function appendRun(result, meta = {}) {
-  await mkdir(dirname(RUNS_FILE), { recursive: true });
-  const record = { ...meta, ...result };
-  await appendFile(RUNS_FILE, `${JSON.stringify(record)}
-`, "utf8");
-}
-
-// src/commands/loop.ts
 var STOP_FILE = "STOP_MIRA";
 var DEFAULT_GAP_MS = 15000;
 var SLOW = { firstReplyTimeoutMs: 90000, maxMs: 240000, typingGraceMs: 90000 };
@@ -952,130 +1196,6 @@ async function loop(opts) {
   }
 }
 
-// src/commands/report.ts
-import { readFileSync as readFileSync2, writeFileSync } from "node:fs";
-import { resolve as resolve3 } from "node:path";
-function loadRunRecords(inFile, category) {
-  const path = inFile ? resolve3(process.cwd(), inFile) : RUNS_FILE;
-  const raw = readFileSync2(path, "utf8");
-  const records = [];
-  for (const line of raw.split(`
-`)) {
-    const t = line.trim();
-    if (!t)
-      continue;
-    try {
-      records.push(JSON.parse(t));
-    } catch {}
-  }
-  return category ? records.filter((r) => r.category === category) : records;
-}
-function truncate(s, n) {
-  const flat = s.replace(/\s+/g, " ").trim();
-  return flat.length > n ? `${flat.slice(0, n - 1)}…` : flat;
-}
-function cell(s) {
-  return s.replace(/\|/g, "\\|");
-}
-function signals(r) {
-  const btns = r.messages.reduce((n, m) => n + m.buttons.length, 0);
-  const links = r.messages.reduce((n, m) => n + m.links.length, 0);
-  const media = r.messages.filter((m) => m.media).map((m) => m.media?.kind);
-  const edits = r.messages.reduce((n, m) => n + m.editCount, 0);
-  return [
-    btns ? `${btns}btn` : "",
-    links ? `${links}link` : "",
-    media.length ? `media=${media.join(",")}` : "",
-    edits ? `${edits}edit` : ""
-  ].filter(Boolean).join(" ") || "—";
-}
-function gist(r) {
-  const first = r.messages.find((m) => m.text)?.text ?? (r.messages.length ? "(non-text)" : "(no reply)");
-  return truncate(first, 90);
-}
-function verdictCell(r) {
-  if (!r.assert)
-    return "—";
-  if (r.assert.ok)
-    return "✅";
-  const failed = r.assert.checks.filter((ch) => !ch.ok).map((ch) => ch.name).join(", ");
-  return `❌ ${cell(failed)}`;
-}
-function table(rows) {
-  const head = `| Probe | Test | Sent | Reply (gist) | Signals | First reply | Settled |
-` + "|---|---|---|---|---|---|---|";
-  const body = rows.map((r) => {
-    const id = r.probeId ?? "—";
-    const sent = cell(truncate(r.sent, 40));
-    const reply = r.timedOut ? "**TIMEOUT**" : cell(gist(r));
-    const first = r.firstReplyMs === null ? "—" : `${(r.firstReplyMs / 1000).toFixed(1)}s`;
-    const total = `${(r.totalMs / 1000).toFixed(1)}s`;
-    return `| \`${cell(id)}\` | ${verdictCell(r)} | ${sent} | ${reply} | ${cell(signals(r))} | ${first} | ${total} |`;
-  }).join(`
-`);
-  return `${head}
-${body}`;
-}
-function buildReport(records) {
-  const replied = records.filter((r) => !r.timedOut);
-  const latencies = replied.map((r) => r.firstReplyMs ?? 0).filter((n) => n > 0);
-  const avg = latencies.length ? latencies.reduce((a, b) => a + b, 0) / latencies.length : 0;
-  const max = latencies.length ? Math.max(...latencies) : 0;
-  const graded = records.filter((r) => r.assert);
-  const passed = graded.filter((r) => r.assert?.ok);
-  const out = [];
-  out.push("# Mira(@mira) probe report (auto-generated)");
-  out.push("");
-  out.push("> Distilled from the run log by `mira-harness report`.");
-  out.push("");
-  out.push(`**${records.length}** probes · **${replied.length}** replied · **${records.length - replied.length}** timed out · ` + (graded.length ? `**${passed.length}/${graded.length}** assertions passed · ` : "") + `first-reply avg **${(avg / 1000).toFixed(1)}s** / max **${(max / 1000).toFixed(1)}s**.`);
-  out.push("");
-  const builtin = CATEGORIES.filter((c2) => records.some((r) => r.category === c2));
-  const custom = [];
-  for (const r of records) {
-    if (r.category && !CATEGORIES.includes(r.category) && !custom.includes(r.category)) {
-      custom.push(r.category);
-    }
-  }
-  for (const cat of [...builtin, ...custom]) {
-    out.push(`## ${cat}`, "", table(records.filter((r) => r.category === cat)), "");
-  }
-  const uncategorized = records.filter((r) => !r.category);
-  if (uncategorized.length)
-    out.push("## (uncategorized)", "", table(uncategorized), "");
-  return out.join(`
-`);
-}
-function renderReport(inFile, category) {
-  const records = loadRunRecords(inFile, category);
-  if (!records.length) {
-    throw new Error(category ? `no probes in category "${category}" in the run log.` : "run log is empty.");
-  }
-  return buildReport(records);
-}
-function report(opts) {
-  let md;
-  try {
-    md = renderReport(opts.in, opts.category);
-  } catch (e) {
-    if (e?.code === "ENOENT") {
-      const path = opts.in ? resolve3(process.cwd(), opts.in) : RUNS_FILE;
-      console.error(`no run log at ${path} — run \`mira-harness send\` or \`mira-harness loop\` first.`);
-    } else {
-      console.error(e instanceof Error ? e.message : String(e));
-    }
-    process.exit(1);
-  }
-  if (opts.out) {
-    const dest = resolve3(process.cwd(), opts.out);
-    writeFileSync(dest, `${md}
-`, "utf8");
-    console.error(`wrote report to ${opts.out}`);
-  } else {
-    console.log(md);
-  }
-}
-
 // src/commands/send.ts
 import { existsSync as existsSync2 } from "node:fs";
 var STOP_FILE2 = "STOP_MIRA";
@@ -1123,7 +1243,7 @@ async function send(rawMessage, opts = {}) {
 }
 
 // src/commands/stats.ts
-import { resolve as resolve4 } from "node:path";
+import { resolve as resolve5 } from "node:path";
 var SPARK = "▁▂▃▄▅▆▇█";
 function sparkline(values) {
   if (!values.length)
@@ -1162,7 +1282,7 @@ function stats(opts = {}) {
     records = loadRunRecords(opts.in, opts.category);
   } catch (e) {
     if (e?.code === "ENOENT") {
-      const path = opts.in ? resolve4(process.cwd(), opts.in) : RUNS_FILE;
+      const path = opts.in ? resolve5(process.cwd(), opts.in) : RUNS_FILE;
       console.error(`no run log at ${path} — run \`mira-harness send\` or \`mira-harness loop\` first.`);
     } else {
       console.error(e instanceof Error ? e.message : String(e));
@@ -1253,11 +1373,11 @@ async function watch(opts = {}) {
   const client = await connect(session);
   const unsubscribe = await subscribe(client, peer, (m, kind) => note(line(peer, m, kind)));
   note(c.bold(`watching @${peer} — Ctrl-C to stop`));
-  await new Promise((resolve5) => {
+  await new Promise((resolve6) => {
     process.once("SIGINT", () => {
       note(c.dim(`
 stopping…`));
-      resolve5();
+      resolve6();
     });
   });
   unsubscribe();
@@ -1338,6 +1458,9 @@ program.command("report").description("Distill the run log (JSONL) into a Markdo
 program.command("stats").description("At-a-glance run-log dashboard: totals, latency records, sparkline").option("--in <file>", "input JSONL (default: the run log)").option("-c, --category <category>", "only include probes from this category").option("--json", "output a JSON summary instead of the colored dashboard", false).action((opts) => {
   stats({ in: opts.in, category: opts.category, json: opts.json });
 });
+program.command("diff").description("Compare two run logs for @mira behavioral drift (exit 1 on a regression)").argument("<baseline>", "baseline run log (JSONL)").argument("[current]", "current run log (JSONL); defaults to the run log").option("--json", "output the drift as JSON", false).option("--no-fail", "report regressions but still exit 0").action((baseline, current, opts) => {
+  diff({ baseline, current, json: opts.json, noFail: !opts.fail });
+});
 program.addHelpText("after", `
 Examples:
   $ mira-harness login
@@ -1352,6 +1475,7 @@ Examples:
   $ mira-harness watch
   $ mira-harness report --category core --out report.md
   $ mira-harness stats
+  $ mira-harness diff baseline.jsonl
 `);
 if (process.argv.length <= 2) {
   banner(version);

package/llms.txt

@@ -24,6 +24,7 @@ so the only programmatic path is a userbot (a real user account over MTProto / G
 - `watch` — live-tail @mira's messages, observe-only. Flag: `--peer`.
 - `report` — distill the JSONL run log into Markdown. Flags: `--in <file> --out <file> --category`.
 - `stats` — at-a-glance run-log dashboard: totals, first-reply latency records (fastest / median / p95 / slowest), an ASCII sparkline, and a per-category breakdown. Flags: `--in <file> --category --json`.
+- `diff <baseline> [current]` — compare two run logs for @mira behavioral drift (matches probes by id; structural, not exact-text). Regressions (assertion ✓→✗, new timeout, >2x latency) **exit non-zero**; surface changes / improvements are reported. `current` defaults to the run log. Flags: `--json --no-fail`.
 - Help: `mira-harness --help` or `<command> --help`.
 
 ## MCP server (bin: mira-harness-mcp, entry dist/mcp.js)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mira-harness",
-  "version": "0.2.1",
+  "version": "0.2.3",
   "description": "A CLI + MCP dev-tool for communicating with the @mira Telegram bot — capture its full reply (buttons/links/media) and run a self-driving experiment catalog.",
   "type": "module",
   "bin": {