mira-harness 0.2.7 → 0.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/cli.js +46 -26
- package/dist/mcp.js +461 -5
- package/llms.txt +3 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -209,6 +209,9 @@ Claude/agent can probe @mira directly via tools.
|
|
|
209
209
|
| `mira_loop` | `category?`, `max?`, `peer?`, `gapMs?`, `settleMs?`, `timeoutMs?`, `catalogFile?` | run the catalog, **observe-only** (never clicks / spends credits) |
|
|
210
210
|
| `mira_catalog` | `category?`, `catalogFile?` | list the catalog (no network) |
|
|
211
211
|
| `mira_report` | `inFile?`, `category?` | run log → Markdown |
|
|
212
|
+
| `mira_stats` | `inFile?`, `category?` | run-log summary: totals, reply rate, latency, by-category (JSON) |
|
|
213
|
+
| `mira_diff` | `baseline`, `current?` | drift between two run logs (regressions / changes) |
|
|
214
|
+
| `mira_assert` | `inFile?`, `catalogFile?`, `category?` | re-grade a saved run log offline (no network) |
|
|
212
215
|
| `mira_doctor` | — | env / session / connectivity check |
|
|
213
216
|
|
|
214
217
|
Register it (local build — run `bun run build` first):
|
package/dist/cli.js
CHANGED
|
@@ -561,6 +561,19 @@ function report(opts) {
|
|
|
561
561
|
}
|
|
562
562
|
|
|
563
563
|
// src/commands/assert.ts
|
|
564
|
+
function assertSummary(records, source) {
|
|
565
|
+
const byId = new Map(source.map((p) => [p.id, p]));
|
|
566
|
+
const results = [];
|
|
567
|
+
for (const r of records) {
|
|
568
|
+
if (!r.probeId)
|
|
569
|
+
continue;
|
|
570
|
+
const p = byId.get(r.probeId);
|
|
571
|
+
if (!p?.expect)
|
|
572
|
+
continue;
|
|
573
|
+
results.push({ id: r.probeId, verdict: evaluate(p.expect, r) });
|
|
574
|
+
}
|
|
575
|
+
return { graded: results.length, passed: results.filter((g) => g.verdict.ok).length, results };
|
|
576
|
+
}
|
|
564
577
|
function assertLog(opts = {}) {
|
|
565
578
|
let records;
|
|
566
579
|
try {
|
|
@@ -575,29 +588,20 @@ function assertLog(opts = {}) {
|
|
|
575
588
|
process.exit(1);
|
|
576
589
|
}
|
|
577
590
|
const source = opts.catalog ? loadCatalog(opts.catalog) : CATALOG;
|
|
578
|
-
const
|
|
579
|
-
const
|
|
580
|
-
for (const r of records) {
|
|
581
|
-
if (!r.probeId)
|
|
582
|
-
continue;
|
|
583
|
-
const p = byId.get(r.probeId);
|
|
584
|
-
if (!p?.expect)
|
|
585
|
-
continue;
|
|
586
|
-
graded.push({ id: r.probeId, verdict: evaluate(p.expect, r) });
|
|
587
|
-
}
|
|
588
|
-
const failed = graded.filter((g) => !g.verdict.ok);
|
|
591
|
+
const { graded, passed, results } = assertSummary(records, source);
|
|
592
|
+
const failed = results.filter((g) => !g.verdict.ok);
|
|
589
593
|
if (opts.json) {
|
|
590
|
-
console.log(JSON.stringify({ graded
|
|
594
|
+
console.log(JSON.stringify({ graded, passed, results }, null, 2));
|
|
591
595
|
if (failed.length && !opts.noFail)
|
|
592
596
|
process.exit(1);
|
|
593
597
|
return;
|
|
594
598
|
}
|
|
595
599
|
note(c.bold(`assert ${opts.catalog ?? "(built-in catalog)"} × ${opts.in ?? "(run log)"}`));
|
|
596
|
-
if (!graded
|
|
600
|
+
if (!graded) {
|
|
597
601
|
note(c.dim(" no records matched a probe with `expect` — nothing to grade."));
|
|
598
602
|
return;
|
|
599
603
|
}
|
|
600
|
-
for (const g of
|
|
604
|
+
for (const g of results) {
|
|
601
605
|
note(` ${g.verdict.ok ? c.green("✓") : c.red("✗")} ${c.cyan(g.id)}`);
|
|
602
606
|
if (!g.verdict.ok) {
|
|
603
607
|
for (const ch of g.verdict.checks.filter((x) => !x.ok))
|
|
@@ -605,7 +609,7 @@ function assertLog(opts = {}) {
|
|
|
605
609
|
}
|
|
606
610
|
}
|
|
607
611
|
note((failed.length ? c.red : c.green)(`
|
|
608
|
-
${
|
|
612
|
+
${passed}/${graded} assertions passed.`));
|
|
609
613
|
if (failed.length && !opts.noFail)
|
|
610
614
|
process.exit(1);
|
|
611
615
|
}
|
|
@@ -1394,6 +1398,29 @@ function latencyStats(records) {
|
|
|
1394
1398
|
max: latencies[latencies.length - 1] ?? 0
|
|
1395
1399
|
};
|
|
1396
1400
|
}
|
|
1401
|
+
function statsSummary(records) {
|
|
1402
|
+
const replied = records.filter((r) => !r.timedOut);
|
|
1403
|
+
const lat = latencyStats(records);
|
|
1404
|
+
const graded = records.filter((r) => r.assert);
|
|
1405
|
+
const passed = graded.filter((r) => r.assert?.ok);
|
|
1406
|
+
const byCat = new Map;
|
|
1407
|
+
for (const r of records) {
|
|
1408
|
+
const key = r.category ?? "(uncategorized)";
|
|
1409
|
+
const e = byCat.get(key) ?? { total: 0, replied: 0 };
|
|
1410
|
+
e.total += 1;
|
|
1411
|
+
if (!r.timedOut)
|
|
1412
|
+
e.replied += 1;
|
|
1413
|
+
byCat.set(key, e);
|
|
1414
|
+
}
|
|
1415
|
+
return {
|
|
1416
|
+
probes: records.length,
|
|
1417
|
+
replied: replied.length,
|
|
1418
|
+
timedOut: records.length - replied.length,
|
|
1419
|
+
assertions: graded.length ? { graded: graded.length, passed: passed.length } : null,
|
|
1420
|
+
latencyMs: lat.count ? { min: lat.min, median: lat.median, p95: lat.p95, max: lat.max } : null,
|
|
1421
|
+
byCategory: Object.fromEntries([...byCat].map(([k, v]) => [k, v]))
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
1397
1424
|
function stats(opts = {}) {
|
|
1398
1425
|
let records;
|
|
1399
1426
|
try {
|
|
@@ -1411,6 +1438,10 @@ function stats(opts = {}) {
|
|
|
1411
1438
|
console.error(opts.category ? `no probes in category "${opts.category}" in the run log.` : "run log is empty — run `mira-harness send` or `mira-harness loop` first.");
|
|
1412
1439
|
process.exit(1);
|
|
1413
1440
|
}
|
|
1441
|
+
if (opts.json) {
|
|
1442
|
+
console.log(JSON.stringify(statsSummary(records), null, 2));
|
|
1443
|
+
return;
|
|
1444
|
+
}
|
|
1414
1445
|
const replied = records.filter((r) => !r.timedOut);
|
|
1415
1446
|
const timedOut = records.length - replied.length;
|
|
1416
1447
|
const lat = latencyStats(records);
|
|
@@ -1426,17 +1457,6 @@ function stats(opts = {}) {
|
|
|
1426
1457
|
byCat.set(key, e);
|
|
1427
1458
|
}
|
|
1428
1459
|
const series = [...records].sort((a, b) => a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0).filter((r) => !r.timedOut && r.firstReplyMs !== null && r.firstReplyMs > 0).map((r) => r.firstReplyMs);
|
|
1429
|
-
if (opts.json) {
|
|
1430
|
-
console.log(JSON.stringify({
|
|
1431
|
-
probes: records.length,
|
|
1432
|
-
replied: replied.length,
|
|
1433
|
-
timedOut,
|
|
1434
|
-
assertions: graded.length ? { graded: graded.length, passed: passed.length } : null,
|
|
1435
|
-
latencyMs: lat.count ? { min: lat.min, median: lat.median, p95: lat.p95, max: lat.max } : null,
|
|
1436
|
-
byCategory: Object.fromEntries([...byCat].map(([k, v]) => [k, v]))
|
|
1437
|
-
}, null, 2));
|
|
1438
|
-
return;
|
|
1439
|
-
}
|
|
1440
1460
|
const rate = (n, total) => `${total ? Math.round(n / total * 100) : 0}%`;
|
|
1441
1461
|
note(c.bold(`\uD83D\uDCCA mira stats${opts.category ? ` [${opts.category}]` : ""}`));
|
|
1442
1462
|
note(` ${c.cyan(String(records.length))} probes · ${c.green(`${replied.length} replied`)} · ` + (timedOut ? c.yellow(`${timedOut} timed out`) : c.dim("0 timed out")) + c.dim(` (${rate(replied.length, records.length)} reply rate)`));
|
package/dist/mcp.js
CHANGED
|
@@ -598,9 +598,8 @@ async function subscribe(client, peer, onMessage) {
|
|
|
598
598
|
};
|
|
599
599
|
}
|
|
600
600
|
|
|
601
|
-
// src/commands/
|
|
602
|
-
import {
|
|
603
|
-
import { resolve as resolve3 } from "node:path";
|
|
601
|
+
// src/commands/assert.ts
|
|
602
|
+
import { resolve as resolve4 } from "node:path";
|
|
604
603
|
|
|
605
604
|
// src/log.ts
|
|
606
605
|
import { appendFile, mkdir } from "node:fs/promises";
|
|
@@ -613,7 +612,116 @@ async function appendRun(result, meta = {}) {
|
|
|
613
612
|
`, "utf8");
|
|
614
613
|
}
|
|
615
614
|
|
|
615
|
+
// src/ui.ts
|
|
616
|
+
import pc from "picocolors";
|
|
617
|
+
var c = pc;
|
|
618
|
+
function note(msg) {
|
|
619
|
+
process.stderr.write(`${msg}
|
|
620
|
+
`);
|
|
621
|
+
}
|
|
622
|
+
function center(s, width) {
|
|
623
|
+
const pad = Math.max(0, width - s.length);
|
|
624
|
+
const left = Math.floor(pad / 2);
|
|
625
|
+
return " ".repeat(left) + s + " ".repeat(pad - left);
|
|
626
|
+
}
|
|
627
|
+
function mascot(mood = "neutral") {
|
|
628
|
+
const faces = {
|
|
629
|
+
neutral: { top: "* . *", eyes: "o o", mouth: "~" },
|
|
630
|
+
happy: { top: "* . *", eyes: "^ ^", mouth: "\\_/" },
|
|
631
|
+
sleepy: { top: "z . z", eyes: "- -", mouth: "~" },
|
|
632
|
+
sad: { top: " ", eyes: "x x", mouth: "_" }
|
|
633
|
+
};
|
|
634
|
+
const f = faces[mood];
|
|
635
|
+
return [` ${f.top}`, " .-------.", ` | ${f.eyes} |`, ` | ${center(f.mouth, 5)} |`, " '-------'"];
|
|
636
|
+
}
|
|
637
|
+
var WORDMARK = [
|
|
638
|
+
" __ __ ___ ____ _",
|
|
639
|
+
"| \\/ |_ _| _ \\ / \\",
|
|
640
|
+
"| |\\/| || || |_) | / _ \\",
|
|
641
|
+
"| | | || || _ < / ___ \\",
|
|
642
|
+
"|_| |_|___|_| \\_\\/_/ \\_\\"
|
|
643
|
+
];
|
|
644
|
+
var TIPS = [
|
|
645
|
+
'echo "What can you do?" | mira-harness send — pipe a probe from stdin',
|
|
646
|
+
"mira-harness loop --list — preview the probes without sending anything",
|
|
647
|
+
"touch STOP_MIRA — instant kill switch; blocks every send until removed",
|
|
648
|
+
"mira-harness loop --catalog ./my.json — run your own probe catalog",
|
|
649
|
+
"mira-harness report --out report.md — distill the run log into Markdown",
|
|
650
|
+
"mira-harness stats — latency records + an ASCII sparkline of your runs",
|
|
651
|
+
"mira-harness watch — live-tail @mira while you poke it by hand",
|
|
652
|
+
"MIRA_NO_BANNER=1 hides this banner · --quiet hides all decoration"
|
|
653
|
+
];
|
|
654
|
+
function banner(version, opts = {}) {
|
|
655
|
+
if (opts.quiet || process.env.MIRA_NO_BANNER || !process.stderr.isTTY || !process.stdout.isTTY)
|
|
656
|
+
return;
|
|
657
|
+
const tip = TIPS[new Date().getDate() % TIPS.length];
|
|
658
|
+
const lines = [
|
|
659
|
+
...mascot("neutral").map((l) => c.magenta(l)),
|
|
660
|
+
...WORDMARK.map((l) => c.cyan(l)),
|
|
661
|
+
c.dim(`mira-harness v${version} · drive @mira from your terminal`),
|
|
662
|
+
c.dim(`tip: ${tip}`)
|
|
663
|
+
];
|
|
664
|
+
process.stderr.write(`
|
|
665
|
+
${lines.map((l) => ` ${l}`).join(`
|
|
666
|
+
`)}
|
|
667
|
+
|
|
668
|
+
`);
|
|
669
|
+
}
|
|
670
|
+
var VERBS = [
|
|
671
|
+
"Summoning",
|
|
672
|
+
"Consulting the chain",
|
|
673
|
+
"Poking the bot",
|
|
674
|
+
"Nudging the oracle",
|
|
675
|
+
"Pondering",
|
|
676
|
+
"Channeling TON",
|
|
677
|
+
"Crunching tokens",
|
|
678
|
+
"Waiting on a reply"
|
|
679
|
+
];
|
|
680
|
+
var VERBS_SLOW = ["Still pondering", "Deep in thought", "Brewing a big one", "Hang tight", "Almost there"];
|
|
681
|
+
var SLOW_AFTER_MS = 30000;
|
|
682
|
+
var VERB_EVERY_MS = 3500;
|
|
683
|
+
async function withProgress(label, fn, quiet = false) {
|
|
684
|
+
if (quiet || !process.stderr.isTTY)
|
|
685
|
+
return fn();
|
|
686
|
+
const frames = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
|
|
687
|
+
const start = Date.now();
|
|
688
|
+
let i = 0;
|
|
689
|
+
const timer = setInterval(() => {
|
|
690
|
+
const elapsed = Date.now() - start;
|
|
691
|
+
const pool = elapsed >= SLOW_AFTER_MS ? VERBS_SLOW : VERBS;
|
|
692
|
+
const verb = pool[Math.floor(elapsed / VERB_EVERY_MS) % pool.length];
|
|
693
|
+
const s = (elapsed / 1000).toFixed(0);
|
|
694
|
+
process.stderr.write(`\r${pc.cyan(frames[i++ % frames.length])} ${verb} ${pc.dim(`· ${label} · ${s}s`)} `);
|
|
695
|
+
}, 100);
|
|
696
|
+
try {
|
|
697
|
+
return await fn();
|
|
698
|
+
} finally {
|
|
699
|
+
clearInterval(timer);
|
|
700
|
+
process.stderr.write("\r\x1B[K");
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
function oscSafe(s) {
|
|
704
|
+
return s.replace(/[\x00-\x1f\x7f]/g, " ").trim();
|
|
705
|
+
}
|
|
706
|
+
function notify(message, opts = {}) {
|
|
707
|
+
if (opts.quiet || process.env.MIRA_NO_NOTIFY || !process.stderr.isTTY)
|
|
708
|
+
return;
|
|
709
|
+
process.stderr.write(`\x1B]9;${oscSafe(message)}\x07`);
|
|
710
|
+
}
|
|
711
|
+
function setTitle(title) {
|
|
712
|
+
if (process.env.MIRA_NO_TITLE || !process.stderr.isTTY)
|
|
713
|
+
return;
|
|
714
|
+
process.stderr.write(`\x1B]0;${oscSafe(title)}\x07`);
|
|
715
|
+
}
|
|
716
|
+
function clearTitle() {
|
|
717
|
+
if (!process.stderr.isTTY)
|
|
718
|
+
return;
|
|
719
|
+
process.stderr.write("\x1B]0;\x07");
|
|
720
|
+
}
|
|
721
|
+
|
|
616
722
|
// src/commands/report.ts
|
|
723
|
+
import { readFileSync as readFileSync2, writeFileSync } from "node:fs";
|
|
724
|
+
import { resolve as resolve3 } from "node:path";
|
|
617
725
|
function isRunRecord(x) {
|
|
618
726
|
return typeof x === "object" && x !== null && Array.isArray(x.messages) && typeof x.sent === "string";
|
|
619
727
|
}
|
|
@@ -694,7 +802,7 @@ function buildReport(records) {
|
|
|
694
802
|
out.push("");
|
|
695
803
|
out.push(`**${records.length}** probes · **${replied.length}** replied · **${records.length - replied.length}** timed out · ` + (graded.length ? `**${passed.length}/${graded.length}** assertions passed · ` : "") + `first-reply avg **${(avg / 1000).toFixed(1)}s** / max **${(max / 1000).toFixed(1)}s**.`);
|
|
696
804
|
out.push("");
|
|
697
|
-
const builtin = CATEGORIES.filter((
|
|
805
|
+
const builtin = CATEGORIES.filter((c2) => records.some((r) => r.category === c2));
|
|
698
806
|
const custom = [];
|
|
699
807
|
for (const r of records) {
|
|
700
808
|
if (r.category && !CATEGORIES.includes(r.category) && !custom.includes(r.category)) {
|
|
@@ -740,6 +848,296 @@ function report(opts) {
|
|
|
740
848
|
}
|
|
741
849
|
}
|
|
742
850
|
|
|
851
|
+
// src/commands/assert.ts
|
|
852
|
+
function assertSummary(records, source) {
|
|
853
|
+
const byId = new Map(source.map((p) => [p.id, p]));
|
|
854
|
+
const results = [];
|
|
855
|
+
for (const r of records) {
|
|
856
|
+
if (!r.probeId)
|
|
857
|
+
continue;
|
|
858
|
+
const p = byId.get(r.probeId);
|
|
859
|
+
if (!p?.expect)
|
|
860
|
+
continue;
|
|
861
|
+
results.push({ id: r.probeId, verdict: evaluate(p.expect, r) });
|
|
862
|
+
}
|
|
863
|
+
return { graded: results.length, passed: results.filter((g) => g.verdict.ok).length, results };
|
|
864
|
+
}
|
|
865
|
+
function assertLog(opts = {}) {
|
|
866
|
+
let records;
|
|
867
|
+
try {
|
|
868
|
+
records = loadRunRecords(opts.in, opts.category);
|
|
869
|
+
} catch (e) {
|
|
870
|
+
if (e?.code === "ENOENT") {
|
|
871
|
+
const path = opts.in ? resolve4(process.cwd(), opts.in) : RUNS_FILE;
|
|
872
|
+
console.error(`no run log at ${path} — run \`mira-harness send\` or \`mira-harness loop\` first.`);
|
|
873
|
+
} else {
|
|
874
|
+
console.error(e instanceof Error ? e.message : String(e));
|
|
875
|
+
}
|
|
876
|
+
process.exit(1);
|
|
877
|
+
}
|
|
878
|
+
const source = opts.catalog ? loadCatalog(opts.catalog) : CATALOG;
|
|
879
|
+
const { graded, passed, results } = assertSummary(records, source);
|
|
880
|
+
const failed = results.filter((g) => !g.verdict.ok);
|
|
881
|
+
if (opts.json) {
|
|
882
|
+
console.log(JSON.stringify({ graded, passed, results }, null, 2));
|
|
883
|
+
if (failed.length && !opts.noFail)
|
|
884
|
+
process.exit(1);
|
|
885
|
+
return;
|
|
886
|
+
}
|
|
887
|
+
note(c.bold(`assert ${opts.catalog ?? "(built-in catalog)"} × ${opts.in ?? "(run log)"}`));
|
|
888
|
+
if (!graded) {
|
|
889
|
+
note(c.dim(" no records matched a probe with `expect` — nothing to grade."));
|
|
890
|
+
return;
|
|
891
|
+
}
|
|
892
|
+
for (const g of results) {
|
|
893
|
+
note(` ${g.verdict.ok ? c.green("✓") : c.red("✗")} ${c.cyan(g.id)}`);
|
|
894
|
+
if (!g.verdict.ok) {
|
|
895
|
+
for (const ch of g.verdict.checks.filter((x) => !x.ok))
|
|
896
|
+
note(c.dim(` ✗ ${ch.name}: ${ch.detail}`));
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
note((failed.length ? c.red : c.green)(`
|
|
900
|
+
${passed}/${graded} assertions passed.`));
|
|
901
|
+
if (failed.length && !opts.noFail)
|
|
902
|
+
process.exit(1);
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// src/commands/diff.ts
|
|
906
|
+
import { resolve as resolve5 } from "node:path";
|
|
907
|
+
function lastByProbe(records) {
|
|
908
|
+
const m = new Map;
|
|
909
|
+
for (const r of records) {
|
|
910
|
+
if (r.probeId)
|
|
911
|
+
m.set(r.probeId, r);
|
|
912
|
+
}
|
|
913
|
+
return m;
|
|
914
|
+
}
|
|
915
|
+
function signals2(r) {
|
|
916
|
+
const buttons = r.messages.reduce((n, m) => n + m.buttons.length, 0);
|
|
917
|
+
const links = r.messages.reduce((n, m) => n + m.links.length, 0);
|
|
918
|
+
const media = r.messages.map((m) => m.media?.kind).filter((k) => k !== undefined).join(",");
|
|
919
|
+
return { buttons, links, media };
|
|
920
|
+
}
|
|
921
|
+
function diffRuns(baseline, current) {
|
|
922
|
+
const base = lastByProbe(baseline);
|
|
923
|
+
const cur = lastByProbe(current);
|
|
924
|
+
const items = [];
|
|
925
|
+
for (const id of [...new Set([...base.keys(), ...cur.keys()])].sort()) {
|
|
926
|
+
const b = base.get(id);
|
|
927
|
+
const c2 = cur.get(id);
|
|
928
|
+
if (b && !c2) {
|
|
929
|
+
items.push({ probeId: id, kind: "removed", what: "in baseline, not in current" });
|
|
930
|
+
continue;
|
|
931
|
+
}
|
|
932
|
+
if (c2 && !b) {
|
|
933
|
+
items.push({ probeId: id, kind: "added", what: "new in current (not in baseline)" });
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
if (!b || !c2)
|
|
937
|
+
continue;
|
|
938
|
+
if (!b.timedOut && c2.timedOut)
|
|
939
|
+
items.push({ probeId: id, kind: "regression", what: "now times out (was replying)" });
|
|
940
|
+
else if (b.timedOut && !c2.timedOut)
|
|
941
|
+
items.push({ probeId: id, kind: "improvement", what: "now replies (was timing out)" });
|
|
942
|
+
const ba = b.assert;
|
|
943
|
+
const ca = c2.assert;
|
|
944
|
+
if (ba && ca && ba.ok !== ca.ok) {
|
|
945
|
+
if (ba.ok && !ca.ok) {
|
|
946
|
+
const failed = ca.checks.filter((x) => !x.ok).map((x) => x.name).join(", ");
|
|
947
|
+
items.push({ probeId: id, kind: "regression", what: `assertion now FAILS (${failed})` });
|
|
948
|
+
} else {
|
|
949
|
+
items.push({ probeId: id, kind: "improvement", what: "assertion now passes" });
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
if (b.firstReplyMs !== null && c2.firstReplyMs !== null) {
|
|
953
|
+
const bms = b.firstReplyMs;
|
|
954
|
+
const cms = c2.firstReplyMs;
|
|
955
|
+
if (cms > bms * 2 && cms - bms > 5000) {
|
|
956
|
+
items.push({
|
|
957
|
+
probeId: id,
|
|
958
|
+
kind: "regression",
|
|
959
|
+
what: `first reply ${(bms / 1000).toFixed(1)}s -> ${(cms / 1000).toFixed(1)}s (>2x slower)`
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
const bs = signals2(b);
|
|
964
|
+
const cs = signals2(c2);
|
|
965
|
+
if (bs.buttons !== cs.buttons)
|
|
966
|
+
items.push({ probeId: id, kind: "change", what: `buttons ${bs.buttons} -> ${cs.buttons}` });
|
|
967
|
+
if (bs.links !== cs.links)
|
|
968
|
+
items.push({ probeId: id, kind: "change", what: `links ${bs.links} -> ${cs.links}` });
|
|
969
|
+
if (bs.media !== cs.media)
|
|
970
|
+
items.push({ probeId: id, kind: "change", what: `media [${bs.media}] -> [${cs.media}]` });
|
|
971
|
+
}
|
|
972
|
+
return items;
|
|
973
|
+
}
|
|
974
|
+
function load(path, label) {
|
|
975
|
+
try {
|
|
976
|
+
return loadRunRecords(path);
|
|
977
|
+
} catch (e) {
|
|
978
|
+
const shown = path ? resolve5(process.cwd(), path) : RUNS_FILE;
|
|
979
|
+
if (e?.code === "ENOENT")
|
|
980
|
+
console.error(`${label}: no run log at ${shown}`);
|
|
981
|
+
else
|
|
982
|
+
console.error(`${label}: ${e instanceof Error ? e.message : String(e)}`);
|
|
983
|
+
return null;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
function diff(opts) {
|
|
987
|
+
const base = load(opts.baseline, "baseline");
|
|
988
|
+
const cur = load(opts.current, "current");
|
|
989
|
+
if (!base || !cur)
|
|
990
|
+
process.exit(1);
|
|
991
|
+
if (!base.length)
|
|
992
|
+
note(c.yellow(`baseline ${opts.baseline} has no records.`));
|
|
993
|
+
if (!cur.length)
|
|
994
|
+
note(c.yellow(`current ${opts.current ?? "(run log)"} has no records.`));
|
|
995
|
+
const items = diffRuns(base, cur);
|
|
996
|
+
const regressions = items.filter((i) => i.kind === "regression");
|
|
997
|
+
if (opts.json) {
|
|
998
|
+
console.log(JSON.stringify({ regressions: regressions.length, drift: items.length, items }, null, 2));
|
|
999
|
+
if (regressions.length && !opts.noFail)
|
|
1000
|
+
process.exit(1);
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
note(c.bold(`drift ${opts.baseline} -> ${opts.current ?? "(current run log)"}`));
|
|
1004
|
+
if (!items.length) {
|
|
1005
|
+
note(c.green(" no drift detected."));
|
|
1006
|
+
return;
|
|
1007
|
+
}
|
|
1008
|
+
const paint = {
|
|
1009
|
+
regression: (s) => c.red(s),
|
|
1010
|
+
improvement: (s) => c.green(s),
|
|
1011
|
+
change: (s) => c.dim(s),
|
|
1012
|
+
added: (s) => c.dim(s),
|
|
1013
|
+
removed: (s) => c.dim(s)
|
|
1014
|
+
};
|
|
1015
|
+
for (const i of items) {
|
|
1016
|
+
note(` ${paint[i.kind](i.kind.padEnd(11))} ${c.cyan(i.probeId)} — ${i.what}`);
|
|
1017
|
+
}
|
|
1018
|
+
note(regressions.length ? c.red(`
|
|
1019
|
+
${regressions.length} regression(s).`) : c.green(`
|
|
1020
|
+
no regressions.`));
|
|
1021
|
+
if (regressions.length && !opts.noFail)
|
|
1022
|
+
process.exit(1);
|
|
1023
|
+
}
|
|
1024
|
+
|
|
1025
|
+
// src/commands/stats.ts
|
|
1026
|
+
import { resolve as resolve6 } from "node:path";
|
|
1027
|
+
var SPARK = "▁▂▃▄▅▆▇█";
|
|
1028
|
+
function sparkline(values) {
|
|
1029
|
+
if (!values.length)
|
|
1030
|
+
return "";
|
|
1031
|
+
let min = Number.POSITIVE_INFINITY;
|
|
1032
|
+
let max = Number.NEGATIVE_INFINITY;
|
|
1033
|
+
for (const v of values) {
|
|
1034
|
+
if (v < min)
|
|
1035
|
+
min = v;
|
|
1036
|
+
if (v > max)
|
|
1037
|
+
max = v;
|
|
1038
|
+
}
|
|
1039
|
+
const range = max - min || 1;
|
|
1040
|
+
return values.map((v) => SPARK[Math.min(SPARK.length - 1, Math.floor((v - min) / range * SPARK.length))]).join("");
|
|
1041
|
+
}
|
|
1042
|
+
function percentile(sortedAsc, p) {
|
|
1043
|
+
if (!sortedAsc.length)
|
|
1044
|
+
return 0;
|
|
1045
|
+
const idx = Math.min(sortedAsc.length - 1, Math.floor(p / 100 * sortedAsc.length));
|
|
1046
|
+
return sortedAsc[idx] ?? 0;
|
|
1047
|
+
}
|
|
1048
|
+
var secs = (ms) => `${(ms / 1000).toFixed(1)}s`;
|
|
1049
|
+
function latencyStats(records) {
|
|
1050
|
+
const latencies = records.filter((r) => !r.timedOut && r.firstReplyMs !== null && r.firstReplyMs > 0).map((r) => r.firstReplyMs).sort((a, b) => a - b);
|
|
1051
|
+
return {
|
|
1052
|
+
count: latencies.length,
|
|
1053
|
+
min: latencies[0] ?? 0,
|
|
1054
|
+
median: percentile(latencies, 50),
|
|
1055
|
+
p95: percentile(latencies, 95),
|
|
1056
|
+
max: latencies[latencies.length - 1] ?? 0
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
function statsSummary(records) {
|
|
1060
|
+
const replied = records.filter((r) => !r.timedOut);
|
|
1061
|
+
const lat = latencyStats(records);
|
|
1062
|
+
const graded = records.filter((r) => r.assert);
|
|
1063
|
+
const passed = graded.filter((r) => r.assert?.ok);
|
|
1064
|
+
const byCat = new Map;
|
|
1065
|
+
for (const r of records) {
|
|
1066
|
+
const key = r.category ?? "(uncategorized)";
|
|
1067
|
+
const e = byCat.get(key) ?? { total: 0, replied: 0 };
|
|
1068
|
+
e.total += 1;
|
|
1069
|
+
if (!r.timedOut)
|
|
1070
|
+
e.replied += 1;
|
|
1071
|
+
byCat.set(key, e);
|
|
1072
|
+
}
|
|
1073
|
+
return {
|
|
1074
|
+
probes: records.length,
|
|
1075
|
+
replied: replied.length,
|
|
1076
|
+
timedOut: records.length - replied.length,
|
|
1077
|
+
assertions: graded.length ? { graded: graded.length, passed: passed.length } : null,
|
|
1078
|
+
latencyMs: lat.count ? { min: lat.min, median: lat.median, p95: lat.p95, max: lat.max } : null,
|
|
1079
|
+
byCategory: Object.fromEntries([...byCat].map(([k, v]) => [k, v]))
|
|
1080
|
+
};
|
|
1081
|
+
}
|
|
1082
|
+
function stats(opts = {}) {
|
|
1083
|
+
let records;
|
|
1084
|
+
try {
|
|
1085
|
+
records = loadRunRecords(opts.in, opts.category);
|
|
1086
|
+
} catch (e) {
|
|
1087
|
+
if (e?.code === "ENOENT") {
|
|
1088
|
+
const path = opts.in ? resolve6(process.cwd(), opts.in) : RUNS_FILE;
|
|
1089
|
+
console.error(`no run log at ${path} — run \`mira-harness send\` or \`mira-harness loop\` first.`);
|
|
1090
|
+
} else {
|
|
1091
|
+
console.error(e instanceof Error ? e.message : String(e));
|
|
1092
|
+
}
|
|
1093
|
+
process.exit(1);
|
|
1094
|
+
}
|
|
1095
|
+
if (!records.length) {
|
|
1096
|
+
console.error(opts.category ? `no probes in category "${opts.category}" in the run log.` : "run log is empty — run `mira-harness send` or `mira-harness loop` first.");
|
|
1097
|
+
process.exit(1);
|
|
1098
|
+
}
|
|
1099
|
+
if (opts.json) {
|
|
1100
|
+
console.log(JSON.stringify(statsSummary(records), null, 2));
|
|
1101
|
+
return;
|
|
1102
|
+
}
|
|
1103
|
+
const replied = records.filter((r) => !r.timedOut);
|
|
1104
|
+
const timedOut = records.length - replied.length;
|
|
1105
|
+
const lat = latencyStats(records);
|
|
1106
|
+
const graded = records.filter((r) => r.assert);
|
|
1107
|
+
const passed = graded.filter((r) => r.assert?.ok);
|
|
1108
|
+
const byCat = new Map;
|
|
1109
|
+
for (const r of records) {
|
|
1110
|
+
const key = r.category ?? "(uncategorized)";
|
|
1111
|
+
const e = byCat.get(key) ?? { total: 0, replied: 0 };
|
|
1112
|
+
e.total += 1;
|
|
1113
|
+
if (!r.timedOut)
|
|
1114
|
+
e.replied += 1;
|
|
1115
|
+
byCat.set(key, e);
|
|
1116
|
+
}
|
|
1117
|
+
const series = [...records].sort((a, b) => a.ts < b.ts ? -1 : a.ts > b.ts ? 1 : 0).filter((r) => !r.timedOut && r.firstReplyMs !== null && r.firstReplyMs > 0).map((r) => r.firstReplyMs);
|
|
1118
|
+
const rate = (n, total) => `${total ? Math.round(n / total * 100) : 0}%`;
|
|
1119
|
+
note(c.bold(`\uD83D\uDCCA mira stats${opts.category ? ` [${opts.category}]` : ""}`));
|
|
1120
|
+
note(` ${c.cyan(String(records.length))} probes · ${c.green(`${replied.length} replied`)} · ` + (timedOut ? c.yellow(`${timedOut} timed out`) : c.dim("0 timed out")) + c.dim(` (${rate(replied.length, records.length)} reply rate)`));
|
|
1121
|
+
if (graded.length) {
|
|
1122
|
+
const allPass = passed.length === graded.length;
|
|
1123
|
+
note(` ${(allPass ? c.green : c.yellow)(`${passed.length}/${graded.length}`)} assertions passed` + c.dim(` (${rate(passed.length, graded.length)})`));
|
|
1124
|
+
}
|
|
1125
|
+
if (lat.count) {
|
|
1126
|
+
note("");
|
|
1127
|
+
note(c.bold(" first-reply latency"));
|
|
1128
|
+
note(` \uD83C\uDFC6 fastest ${c.green(secs(lat.min))} median ${c.cyan(secs(lat.median))} ` + `p95 ${c.cyan(secs(lat.p95))} slowest ${c.yellow(secs(lat.max))}`);
|
|
1129
|
+
if (series.length > 1)
|
|
1130
|
+
note(` ${c.magenta(sparkline(series))} ${c.dim(`(${series.length} replies, oldest→newest)`)}`);
|
|
1131
|
+
} else {
|
|
1132
|
+
note(c.dim(" no first-reply times recorded yet."));
|
|
1133
|
+
}
|
|
1134
|
+
note("");
|
|
1135
|
+
note(c.bold(" by category"));
|
|
1136
|
+
for (const [cat, v] of byCat) {
|
|
1137
|
+
note(` ${c.cyan(cat.padEnd(16))} ${String(v.total).padStart(3)} probes ${c.dim(`${rate(v.replied, v.total)} replied`)}`);
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
|
|
743
1141
|
// src/version.ts
|
|
744
1142
|
import { readFileSync as readFileSync3 } from "node:fs";
|
|
745
1143
|
import { dirname as dirname2, join } from "node:path";
|
|
@@ -803,7 +1201,7 @@ server.registerTool("mira_send", {
|
|
|
803
1201
|
if (timeoutMs !== undefined)
|
|
804
1202
|
collect.firstReplyTimeoutMs = timeoutMs;
|
|
805
1203
|
try {
|
|
806
|
-
const result = await withClient((
|
|
1204
|
+
const result = await withClient((c2) => sendAndCollect(c2, tgEnv.miraPeer, message, collect));
|
|
807
1205
|
await appendRun(result);
|
|
808
1206
|
return json(result);
|
|
809
1207
|
} catch (e) {
|
|
@@ -912,6 +1310,64 @@ server.registerTool("mira_report", {
|
|
|
912
1310
|
return errorText(msg(e));
|
|
913
1311
|
}
|
|
914
1312
|
});
|
|
1313
|
+
server.registerTool("mira_stats", {
|
|
1314
|
+
title: "Summarize the run log",
|
|
1315
|
+
description: "Roll the run log up into totals, reply rate, assertion pass count, first-reply latency (min/median/p95/max) and a per-category breakdown. Read-only — sends nothing.",
|
|
1316
|
+
inputSchema: {
|
|
1317
|
+
inFile: z3.string().optional().describe("input JSONL path (default: the run log)"),
|
|
1318
|
+
category: z3.string().optional().describe("only include probes from this category")
|
|
1319
|
+
}
|
|
1320
|
+
}, async ({ inFile, category }) => {
|
|
1321
|
+
let records;
|
|
1322
|
+
try {
|
|
1323
|
+
records = loadRunRecords(inFile, category);
|
|
1324
|
+
} catch (e) {
|
|
1325
|
+
return errorText(msg(e));
|
|
1326
|
+
}
|
|
1327
|
+
if (!records.length) {
|
|
1328
|
+
return errorText(category ? `no probes in category "${category}" in the run log.` : "run log is empty — run mira_send or mira_loop first.");
|
|
1329
|
+
}
|
|
1330
|
+
return json(statsSummary(records));
|
|
1331
|
+
});
|
|
1332
|
+
server.registerTool("mira_diff", {
|
|
1333
|
+
title: "Diff two run logs for @mira drift",
|
|
1334
|
+
description: "Compare two run logs probe-by-probe and report behavioral drift (regressions, improvements, surface changes). Read-only — sends nothing.",
|
|
1335
|
+
inputSchema: {
|
|
1336
|
+
baseline: z3.string().describe("baseline run-log path to compare against"),
|
|
1337
|
+
current: z3.string().optional().describe("current run-log path (default: the run log)")
|
|
1338
|
+
}
|
|
1339
|
+
}, async ({ baseline, current }) => {
|
|
1340
|
+
let base;
|
|
1341
|
+
let cur;
|
|
1342
|
+
try {
|
|
1343
|
+
base = loadRunRecords(baseline);
|
|
1344
|
+
cur = loadRunRecords(current);
|
|
1345
|
+
} catch (e) {
|
|
1346
|
+
return errorText(msg(e));
|
|
1347
|
+
}
|
|
1348
|
+
const items = diffRuns(base, cur);
|
|
1349
|
+
const regressions = items.filter((i) => i.kind === "regression").length;
|
|
1350
|
+
return json({ regressions, drift: items.length, items });
|
|
1351
|
+
});
|
|
1352
|
+
server.registerTool("mira_assert", {
|
|
1353
|
+
title: "Re-grade a run log offline",
|
|
1354
|
+
description: "Re-apply a catalog's `expect` assertions to a SAVED run log (no network) and return per-probe verdicts plus a pass count. Read-only — sends nothing.",
|
|
1355
|
+
inputSchema: {
|
|
1356
|
+
inFile: z3.string().optional().describe("input JSONL path (default: the run log)"),
|
|
1357
|
+
catalogFile: z3.string().optional().describe("custom catalog JSON file (default: the built-in catalog)"),
|
|
1358
|
+
category: z3.string().optional().describe("only include probes from this category")
|
|
1359
|
+
}
|
|
1360
|
+
}, async ({ inFile, catalogFile, category }) => {
|
|
1361
|
+
let records;
|
|
1362
|
+
let source;
|
|
1363
|
+
try {
|
|
1364
|
+
records = loadRunRecords(inFile, category);
|
|
1365
|
+
source = catalogFile ? loadCatalog(catalogFile) : CATALOG;
|
|
1366
|
+
} catch (e) {
|
|
1367
|
+
return errorText(msg(e));
|
|
1368
|
+
}
|
|
1369
|
+
return json(assertSummary(records, source));
|
|
1370
|
+
});
|
|
915
1371
|
server.registerTool("mira_doctor", {
|
|
916
1372
|
title: "Preflight checks",
|
|
917
1373
|
description: "Check env, session, connectivity and @mira resolution (read-only — sends nothing)."
|
package/llms.txt
CHANGED
|
@@ -35,6 +35,9 @@ A second frontend over the same core, for agents. Tools:
|
|
|
35
35
|
- `mira_loop` { category?, max?, peer?, gapMs?, settleMs?, timeoutMs?, catalogFile? } — run the catalog, **observe-only** (never clicks / spends credits).
|
|
36
36
|
- `mira_catalog` { category?, catalogFile? } — list the catalog (no network).
|
|
37
37
|
- `mira_report` { inFile?, category? } — run log → Markdown.
|
|
38
|
+
- `mira_stats` { inFile?, category? } — run-log summary: totals, reply rate, latency, by-category (JSON, no network).
|
|
39
|
+
- `mira_diff` { baseline, current? } — drift between two run logs: regressions / changes (JSON, no network).
|
|
40
|
+
- `mira_assert` { inFile?, catalogFile?, category? } — re-grade a saved run log against the catalog's `expect` (JSON, no network).
|
|
38
41
|
- `mira_doctor` — env / session / connectivity check (no arguments).
|
|
39
42
|
Credentials come from the MCP `env` block or a `.env` in the server's cwd.
|
|
40
43
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "mira-harness",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.8",
|
|
4
4
|
"description": "A CLI + MCP dev-tool for communicating with the @mira Telegram bot — capture its full reply (buttons/links/media) and run a self-driving experiment catalog.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|