@matt82198/aesop 0.1.0-beta.5 → 0.1.0

package/skills/CLAUDE.md

@@ -1,3 +1,4 @@
 # skills/ — Orchestration and priming skills
 
 - **power/** — /power skill: prime the filesystem brain (load rules, dispatch model, memory, state) and report system health — see power/SKILL.md
+- **healthcheck/** — /healthcheck skill: check fleet health — one colored ball (heartbeats, alerts, orchestrator, tracker items) — see healthcheck/SKILL.md

package/state_store/ingest.py

@@ -15,10 +15,27 @@ def ingest_tracker_json(api, tracker_json_path: str, actor: str = "migration") -
     """Append one ``item_created`` per item in ``tracker_json_path``.
 
     Returns the number of items ingested.
+    Validates that each item is a dict; rejects malformed events with a clear error.
     """
     with open(tracker_json_path, "r", encoding="utf-8") as fh:
         data = json.load(fh)
+
+    if not isinstance(data, dict):
+        raise TypeError(
+            f"tracker.json root must be a dict, got {type(data).__name__}"
+        )
+
     items = data.get("items", [])
-    for item in items:
+    if not isinstance(items, list):
+        raise TypeError(
+            f"tracker.json 'items' field must be a list, got {type(items).__name__}"
+        )
+
+    for i, item in enumerate(items):
+        if not isinstance(item, dict):
+            raise TypeError(
+                f"tracker.json item at index {i} must be a dict, got {type(item).__name__}"
+            )
         api.append("tracker", "item_created", item, actor)
+
     return len(items)

package/state_store/projections.py

@@ -12,16 +12,35 @@
 
 Unknown ids on update/archive and unknown event types are ignored, keeping the
 fold tolerant of partial/legacy streams.
+
+Snapshots enable O(n) tail-replay (instead of replaying full log each time):
+  - save_snapshot(store, stream, event_version, projection): persist materialized state
+  - project_tracker_with_snapshot(store, stream, events): load snapshot, fold tail events
 """
 from __future__ import annotations
 
 TRACKER_VERSION = 1
 
 
-def project_tracker(events: list) -> dict:
-    """Return the current tracker state projected from ``events``."""
-    order = []      # item ids in first-seen order (stable output ordering)
-    items = {}      # id -> item dict
+def _fold_events(events: list, order: list | None = None, items: dict | None = None) -> tuple:
+    """Fold events into (order, items) state.
+
+    Helper used by both project_tracker and project_tracker_with_snapshot to avoid
+    duplicating the folding logic. Mutates order and items in place.
+
+    Args:
+        events: list of event dicts
+        order: existing order list (item ids in first-seen order); mutated in place
+        items: existing items dict (id -> item dict); mutated in place
+
+    Returns:
+        (order, items) tuple with accumulated state
+    """
+    if order is None:
+        order = []
+    if items is None:
+        items = {}
+
     for ev in events:
         etype = ev.get("type")
         payload = ev.get("payload") or {}
@@ -49,4 +68,59 @@ def project_tracker(events: list) -> dict:
                     merged["completed_at"] = payload["completed_at"]
                 items[iid] = merged
         # any other event type is ignored
+    return (order, items)
+
+
+def project_tracker(events: list) -> dict:
+    """Return the current tracker state projected from ``events``.
+
+    This is the original full-replay projection. For production use with snapshots,
+    prefer project_tracker_with_snapshot() which enables tail-replay.
+    """
+    order, items = _fold_events(events)
+    return {"version": TRACKER_VERSION, "items": [items[i] for i in order]}
+
+
+def project_tracker_with_snapshot(store, stream: str, events: list) -> dict:
+    """Project tracker state using snapshots for O(n) tail-replay instead of O(n²) full replay.
+
+    If a valid snapshot exists for the stream, starts from the snapshot state and folds
+    only events after the snapshot's version. If snapshot is missing or corrupt, falls
+    back to full replay from event 1 (graceful degradation).
+
+    Args:
+        store: EventStore instance with read_snapshot() method
+        stream: stream name (e.g., "tracker")
+        events: list of all events for the stream (from store.read(stream))
+
+    Returns:
+        dict: {"version": 1, "items": [...]} projection, identical to project_tracker()
+    """
+    snapshot = store.read_snapshot(stream)
+
+    if snapshot is not None:
+        # Snapshot exists and is valid; start from it and fold tail events
+        snapshot_version, projection, _ = snapshot
+        order = [item["id"] for item in projection.get("items", [])]
+        items = {item["id"]: dict(item) for item in projection.get("items", [])}
+
+        # Fold only events after the snapshot
+        tail_events = [ev for ev in events if ev.get("version", 0) > snapshot_version]
+        order, items = _fold_events(tail_events, order, items)
+    else:
+        # No valid snapshot; do full replay
+        order, items = _fold_events(events)
+
     return {"version": TRACKER_VERSION, "items": [items[i] for i in order]}
+
+
+def save_snapshot(store, stream: str, event_version: int, projection: dict) -> None:
+    """Persist a materialized projection snapshot.
+
+    Args:
+        store: EventStore instance with save_snapshot() method
+        stream: stream name (e.g., "tracker")
+        event_version: the per-stream event version this snapshot was computed through
+        projection: the materialized state dict (e.g. full tracker projection)
+    """
+    store.save_snapshot(stream, event_version, projection)

package/state_store/store.py

@@ -10,8 +10,10 @@ Stdlib only (sqlite3, json, time) per aesop's no-external-deps invariant.
 """
 from __future__ import annotations
 
+import hashlib
 import json
 import sqlite3
+import sys
 import time
 
 
@@ -48,6 +50,22 @@ class EventStore:
                 "CREATE INDEX IF NOT EXISTS idx_events_stream_version "
                 "ON events(stream, version)"
             )
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS snapshots (
+                    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+                    ts           REAL    NOT NULL,
+                    stream       TEXT    NOT NULL,
+                    event_version INTEGER NOT NULL,
+                    projection   TEXT    NOT NULL,
+                    checksum     TEXT    NOT NULL
+                )
+                """
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_snapshots_stream_version "
+                "ON snapshots(stream, event_version)"
+            )
             conn.commit()
         finally:
             conn.close()
@@ -91,12 +109,89 @@ class EventStore:
                 "SELECT id, ts, actor, stream, type, payload, version FROM events " + clause,
                 params,
             )
-            return [
-                {
-                    "id": r[0], "ts": r[1], "actor": r[2], "stream": r[3],
-                    "type": r[4], "payload": json.loads(r[5]), "version": r[6],
-                }
-                for r in cur.fetchall()
-            ]
+            rows = []
+            for r in cur.fetchall():
+                try:
+                    payload = json.loads(r[5])
+                    rows.append({
+                        "id": r[0], "ts": r[1], "actor": r[2], "stream": r[3],
+                        "type": r[4], "payload": payload, "version": r[6],
+                    })
+                except json.JSONDecodeError as e:
+                    # Log corrupt event (stream id + sequence) to stderr and skip
+                    print(
+                        f"WARNING: corrupt JSON payload in stream={r[3]} id={r[0]}: {e}",
+                        file=sys.stderr,
+                    )
+            return rows
+        finally:
+            conn.close()
+
+    def save_snapshot(self, stream: str, event_version: int, projection: dict) -> None:
+        """Save a materialized projection snapshot at a specific event version.
+
+        The snapshot enables tail-replay: instead of replaying from event 1,
+        project() can resume from this snapshot's version and fold only newer events.
+
+        Args:
+            stream: the stream name (e.g. "tracker")
+            event_version: the per-stream event version this snapshot was computed through
+            projection: the materialized state dict (e.g. the full tracker projection)
+        """
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("PRAGMA busy_timeout=5000")
+            projection_json = json.dumps(projection, separators=(",", ":"), sort_keys=True)
+            checksum = hashlib.sha256(projection_json.encode()).hexdigest()
+            conn.execute(
+                "INSERT INTO snapshots (ts, stream, event_version, projection, checksum) "
+                "VALUES (?, ?, ?, ?, ?)",
+                (time.time(), stream, event_version, projection_json, checksum),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def read_snapshot(self, stream: str) -> tuple | None:
+        """Read the most recent snapshot for a stream.
+
+        Returns:
+            A tuple (event_version, projection_dict, checksum) if a valid snapshot exists,
+            or None if no snapshot found.
+
+        On corrupt/unreadable snapshot, logs a warning and returns None (falls back to
+        full replay).
+        """
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("PRAGMA busy_timeout=5000")
+            cur = conn.execute(
+                "SELECT event_version, projection, checksum FROM snapshots "
+                "WHERE stream = ? ORDER BY event_version DESC LIMIT 1",
+                (stream,),
+            )
+            row = cur.fetchone()
+            if row is None:
+                return None
+            event_version, projection_json, checksum = row
+            try:
+                projection = json.loads(projection_json)
+                # Verify checksum
+                computed_checksum = hashlib.sha256(projection_json.encode()).hexdigest()
+                if computed_checksum != checksum:
+                    print(
+                        f"WARNING: snapshot checksum mismatch for stream={stream} "
+                        f"version={event_version}; falling back to full replay",
+                        file=sys.stderr,
+                    )
+                    return None
+                return (event_version, projection, checksum)
+            except json.JSONDecodeError as e:
+                print(
+                    f"WARNING: corrupt JSON in snapshot for stream={stream} "
+                    f"version={event_version}: {e}; falling back to full replay",
+                    file=sys.stderr,
+                )
+                return None
         finally:
             conn.close()

package/tools/CLAUDE.md

@@ -4,36 +4,43 @@ Local-only Python (stdlib only, no external deps), bash (POSIX, CRLF-safe). Neve
 
 ## FILES
 
-- `lock.mjs` — Fail-closed atomic lock acquisition (exponential backoff + stale-lock detection)
-- `secret_scan.py` — Pre-push secret/credential detection gate
-- `scanner_selftest.py` — Regression harness for secret_scan.py
-- `prepublish_scan.py` — Pre-publish full history + staged-changes scan gate
-- `metrics_gate.py` — PR gate for hard numeric claims in markdown
-- `self_stats.py` — Git-derived metrics counter + README block generator
-- `verify_dash.py` — Browser proof for realtime SSE dashboard
-- `verify_submit_encoding.py` — Browser proof for /submit UTF-8 inbox bootstrap
-- `ci_merge_wait.py` — CI-gated merge helper (polls gh pr view until SUCCESS)
 - `alert_bridge.py` — Slack/Discord webhook bridge for SECURITY-ALERTS
-- `proposals.mjs` — Proposal lifecycle manager (list/accept/reject)
-- `power_selftest.py` — Health check harness for /power bootstrap
-- `healthcheck.py` — Fleet health aggregator (heartbeat/alert/orchestrator status)
+- `bench_runner.py` — Held-out benchmark runner + scorer with accuracy + cost axis (offline mock runner; pluggable Haiku/Sonnet/Opus runners return text or (text, usage))
 - `buildlog.py` — Uniform BUILDLOG.md appender
+- `ci_merge_wait.py` — CI-gated merge helper (polls gh pr view until SUCCESS)
+- `common.py` — Shared utilities (state directory resolution, heartbeat staleness checks)
+- `cost_ceiling.py` — Cost-ceiling checker: trips the HALT kill-switch when configured token limits are exceeded
 - `ensure_state.py` — Scaffold STATE.md and BUILDLOG.md templates
+- `eod_sweep.py` — End-of-day safety check (dirty trees, unpushed commits)
 - `fleet_ledger.py` — Append-only cost ledger with harvest/rotate
+- `fleet_prompt_extractor.py` — Extract and deduplicate Agent/Task spawn prompts
+- `halt.py` — Kill-switch: writes/reads/clears the `.HALT` sentinel that daemons/dispatch check to stop the fleet
+- `healthcheck.py` — Fleet health aggregator (heartbeat/alert/orchestrator status)
 - `heartbeat.py` — Single-instance loop liveness registry
-- `stall_check.py` — Automated agent transcript stall detector
 - `inbox_drain.py` — Drain UI inbox submissions
+- `launch_tui.py` — Spawn bash TUI script in detached terminal
+- `lock.mjs` — Fail-closed atomic lock acquisition (exponential backoff + stale-lock detection)
+- `metrics_gate.py` — PR gate for hard numeric claims in markdown
 - `orchestrator_status.py` — Atomic orchestrator status updates
+- `power_selftest.py` — Health check harness for /power bootstrap
+- `prepublish_scan.py` — Pre-publish full history + staged-changes scan gate
+- `proposals.mjs` — Proposal lifecycle manager (list/accept/reject)
+- `reconcile.py` — Detect/resolve drift between git STATE.md (phase) and the state_store projection (git-authoritative; --resolve appends to SQLite only)
 - `reconstitute.sh` — Clone/fetch repos from config with security validation
-- `eod_sweep.py` — End-of-day safety check (dirty trees, unpushed commits)
-- `agent-forensics.sh` — Incident forensics / behavior reconstruction
-- `launch_tui.py` — Spawn bash TUI script in detached terminal
 - `rotate_logs.py` — Log rotation utility with size/line thresholds
+- `scanner_selftest.py` — Regression harness for secret_scan.py
+- `secret_scan.py` — Pre-push secret/credential detection gate
+- `self_stats.py` — Git-derived metrics counter + README block generator
 - `session_usage_summary.py` — Aggregate token usage across session transcripts
+- `stall_check.py` — Automated agent transcript stall detector
+- `svg_to_png.mjs` — Rasterize SVG to PNG via @resvg/resvg-js (with lazy import error handling)
 - `transcript_replay.py` — Replay post-commit edits from transcripts to recover work
 - `transcript_timeline.py` — Extract Write/Edit/Read timeline from transcripts
-- `fleet_prompt_extractor.py` — Extract and deduplicate Agent/Task spawn prompts
-- `svg_to_png.mjs` — Rasterize SVG to PNG via @resvg/resvg-js (with lazy import error handling)
+- `verify_dash.py` — Browser proof for realtime SSE dashboard
+- `verify_submit_encoding.py` — Browser proof for /submit UTF-8 inbox bootstrap
+- `verify_prboard.py` — Browser proof for the Wave PR Board (/api/wave/prs), gh stubbed via AESOP_GH_BIN
+- `verify_agent_inspector.py` — Browser proof for the Agent Inspector drawer (/api/agent?id=), agents + transcript stubbed via a temp AESOP_ROOT
+- `agent-forensics.sh` — Incident forensics / behavior reconstruction
 
 ## secret_scan.py — Pre-push secret/credential detection gate
 

package/tools/alert_bridge.py

@@ -30,6 +30,11 @@ import urllib.request
 import urllib.error
 from pathlib import Path
 
+try:
+    from common import check_heartbeat_staleness as check_hb_staleness
+except ImportError:
+    from tools.common import check_heartbeat_staleness as check_hb_staleness
+
 
 # ==============================================================================
 # Configuration Loading
@@ -181,20 +186,19 @@ def check_heartbeat_staleness(state_root, stall_threshold_s):
         return False, None
 
     hb_file = state_root / ".watchdog-heartbeat"
-    if not hb_file.exists():
-        return True, "Watchdog heartbeat missing"
+    is_stale, age_seconds, info = check_hb_staleness(hb_file, stall_threshold_s)
 
-    try:
-        timestamp = int(hb_file.read_text(encoding="utf-8").strip())
-    except (ValueError, IOError):
-        return True, "Watchdog heartbeat unreadable"
+    if not is_stale:
+        return False, None
 
-    age_seconds = int(time.time()) - timestamp
-    if age_seconds >= stall_threshold_s:
+    # Map the common function's result to our return format
+    if age_seconds == 0:
+        # File missing/empty/unreadable
+        return True, info
+    else:
+        # File exists but is stale
         return True, f"Watchdog heartbeat stale ({age_seconds}s >= {stall_threshold_s}s)"
 
-    return False, None
-
 
 # ==============================================================================
 # Payload Formatting