mira-harness 0.2.0 → 0.2.2

package/README.md

@@ -101,11 +101,12 @@ a "typing…" fallback for a slow bot — replies run 5–62s) and capture, per
 | `login` | One-time interactive login → prints `TG_SESSION` |
 | `doctor` | Check `.env` / session / connectivity / @mira resolution (read-only) |
 | `send [message...]` | One probe → full reply as JSON (message via arg or stdin). `--quiet --settle --timeout --no-log` |
-| `loop` | Run the catalog paced. `--category --max --confirm --peer --gap --settle --timeout --list --catalog --quiet` |
+| `loop` | Run the catalog paced; grades `expect` probes (exit 1 on failure). `--category --max --confirm --peer --gap --settle --timeout --list --catalog --no-fail --quiet` |
 | `catalog` | List the catalog (no sends). `--category --catalog --json` |
 | `watch` | Live-tail @mira's messages (observe-only). `--peer` |
 | `report` | Distill the run log into Markdown. `--in --out --category` |
 | `stats` | At-a-glance dashboard: totals, latency records, sparkline. `--in --category --json` |
+| `diff` | Compare two run logs for @mira behavioral drift (exit 1 on a regression). `--json --no-fail` |
 
 Run `mira-harness --help` (or `<command> --help`) for full options.
 
@@ -130,13 +131,48 @@ per-feature flags — `MIRA_NO_BANNER=1` (mascot + tip), `MIRA_NO_NOTIFY=1` (com
 The built-in catalog (27 probes: `core` / `skills` / `generation` / `wallet`) is just a
 default. Point `--catalog <file.json>` (CLI) or `catalogFile` (MCP) at your own probe set
 to probe any bot — each entry needs `id` + `send` (`category` / `hypothesis` / `slow` /
-`confirm` / `note` optional). See [`examples/catalog.sample.json`](examples/catalog.sample.json):
+`confirm` / `note` / `expect` optional). See [`examples/catalog.sample.json`](examples/catalog.sample.json):
 
 ```bash
 mira-harness loop --catalog ./examples/catalog.sample.json
 mira-harness catalog --catalog ./examples/catalog.sample.json --json
 ```
 
+### Assertions (PASS/FAIL)
+
+Give a probe an optional `expect` block and `loop` grades it ✓/✗. The checks are **structural**
+— @mira is an LLM (non-deterministic), so exact-text matches would flake:
+
+| Check | Means |
+|---|---|
+| `replies: true` | a reply arrived (no timeout) |
+| `textMatches: "<regex>"` | some message text matches (case-insensitive) |
+| `minButtons` / `minLinks` | at least N inline buttons / links across messages |
+| `hasWebApp: true` | a Mini App (`web_app` / startapp) "Launch" button is present |
+| `media: "photo"` | a message carries media of that kind (`photo`/`video`/`audio`/…) |
+| `maxFirstReplyMs` | first-reply latency within the bound |
+| `json: true` | the first message text parses as JSON |
+
+```json
+{ "id": "json-strict", "send": "Reply with ONLY {\"ok\":true}", "expect": { "json": true } }
+```
+
+Probes without `expect` stay observe-only (informational). `loop` **exits non-zero** if any
+graded probe fails — so it drops straight into CI. Add `--no-fail` to report without failing.
+
+### Drift detection
+
+`diff` compares two run logs and flags how @mira's behavior **changed** (structural, not
+exact text). **Regressions** — an assertion that flipped ✓→✗, a probe that now times out, a
+>2× latency blow-up — exit non-zero; surface changes (buttons / links / media) and
+improvements are reported but pass. Snapshot a baseline, re-run later, diff:
+
+```bash
+MIRA_RUNS_FILE=baseline.jsonl mira-harness loop --category core   # snapshot a baseline
+mira-harness loop --category core                                # a later run -> mira-runs.jsonl
+mira-harness diff baseline.jsonl                                 # vs the current run log
+```
+
 ## Use as a library
 
 The CLI is a thin frontend over an exported core:

package/dist/assert.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Probe assertions — turn an observed reply into a PASS/FAIL verdict.
+ *
+ * A probe's optional `expect` declares machine-checkable expectations; `evaluate`
+ * runs them against the captured ProbeResult. @mira is an LLM (non-deterministic),
+ * so the checks are deliberately STRUCTURAL and loose — a reply arrived, >= N
+ * links, valid JSON, a latency bound — never exact text, which would flake.
+ *
+ * Probes WITHOUT `expect` are not graded: they stay observe-only / informational.
+ */
+import { z } from "zod";
+import type { ProbeResult } from "./capture.js";
+export declare const ExpectSchema: z.ZodObject<{
+    replies: z.ZodOptional<z.ZodBoolean>;
+    textMatches: z.ZodOptional<z.ZodString>;
+    minButtons: z.ZodOptional<z.ZodNumber>;
+    minLinks: z.ZodOptional<z.ZodNumber>;
+    hasWebApp: z.ZodOptional<z.ZodBoolean>;
+    media: z.ZodOptional<z.ZodEnum<{
+        photo: "photo";
+        video: "video";
+        audio: "audio";
+        document: "document";
+        webpage: "webpage";
+        other: "other";
+    }>>;
+    maxFirstReplyMs: z.ZodOptional<z.ZodNumber>;
+    json: z.ZodOptional<z.ZodBoolean>;
+}, z.core.$strip>;
+export type Expect = z.infer<typeof ExpectSchema>;
+export interface Check {
+    name: string;
+    ok: boolean;
+    detail: string;
+}
+export interface Verdict {
+    ok: boolean;
+    checks: Check[];
+}
+/** Run a probe's expectations against its captured result. Pure — no network. */
+export declare function evaluate(expect: Expect, result: ProbeResult): Verdict;

package/dist/catalog.d.ts

@@ -1,3 +1,4 @@
+import { type Expect } from "./assert.js";
 /** Built-in categories. Custom catalogs may use any category string. */
 export type ProbeCategory = "core" | "skills" | "generation" | "wallet";
 export interface Probe {
@@ -15,6 +16,8 @@ export interface Probe {
      */
     confirm?: boolean;
     note?: string;
+    /** Optional machine-checkable expectations — graded by `loop` (PASS/FAIL). */
+    expect?: Expect;
 }
 export declare const CATEGORIES: ProbeCategory[];
 /**