@matt82198/aesop 0.1.0-beta.3 → 0.1.0-beta.5

package/state_store/export.py

@@ -0,0 +1,19 @@
+"""state_store.export — render a projection back into git-tracked JSON.
+
+In the DB-source-of-truth design git stops being *read* for state; this job
+renders human-readable, diffable snapshots (e.g. tracker.json) FROM the
+projections for durability and review. The JSON style (indent=2, ascii-escaped)
+matches the existing state/tracker.json so a future cutover produces
+minimal-diff snapshots.
+"""
+from __future__ import annotations
+
+import json
+
+
+def export_tracker(api, out_path: str) -> None:
+    """Write the tracker projection to ``out_path`` as pretty JSON."""
+    projection = api.project("tracker")
+    with open(out_path, "w", encoding="utf-8") as fh:
+        json.dump(projection, fh, indent=2)
+        fh.write("\n")

package/state_store/ingest.py

@@ -0,0 +1,24 @@
+"""state_store.ingest — backfill events from an existing tracker.json.
+
+Migration path for the DB-source-of-truth cutover: read the current
+tracker.json and emit one ``item_created`` event per item (full payload) so the
+event store reproduces the current state. History is not reconstructed (only
+final state exists on disk), but the projection round-trips to the same items,
+so an export re-renders the identical tracker.json.
+"""
+from __future__ import annotations
+
+import json
+
+
+def ingest_tracker_json(api, tracker_json_path: str, actor: str = "migration") -> int:
+    """Append one ``item_created`` per item in ``tracker_json_path``.
+
+    Returns the number of items ingested.
+    """
+    with open(tracker_json_path, "r", encoding="utf-8") as fh:
+        data = json.load(fh)
+    items = data.get("items", [])
+    for item in items:
+        api.append("tracker", "item_created", item, actor)
+    return len(items)

package/state_store/projections.py

@@ -0,0 +1,52 @@
+"""state_store.projections — fold tracker events into current state.
+
+``project_tracker`` reconstructs the tracker.json shape
+(``{"version": 1, "items": [...]}``) from the ``tracker`` event stream:
+
+  - ``item_created``  : payload is the full item dict; establishes the item
+                        (kept in first-seen order).
+  - ``item_updated``  : payload is ``{"id": <id>, ...partial fields}``; merges
+                        the given fields onto the existing item.
+  - ``item_archived`` : payload is ``{"id": <id>}`` (+ optional
+                        ``completed_at``); sets ``status`` to ``"archived"``.
+
+Unknown ids on update/archive and unknown event types are ignored, keeping the
+fold tolerant of partial/legacy streams.
+"""
+from __future__ import annotations
+
+TRACKER_VERSION = 1
+
+
+def project_tracker(events: list) -> dict:
+    """Return the current tracker state projected from ``events``."""
+    order = []      # item ids in first-seen order (stable output ordering)
+    items = {}      # id -> item dict
+    for ev in events:
+        etype = ev.get("type")
+        payload = ev.get("payload") or {}
+        if etype == "item_created":
+            iid = payload.get("id")
+            if iid is None:
+                continue
+            if iid not in items:
+                order.append(iid)
+            items[iid] = dict(payload)
+        elif etype == "item_updated":
+            iid = payload.get("id")
+            if iid in items:
+                merged = dict(items[iid])
+                for key, value in payload.items():
+                    if key != "id":
+                        merged[key] = value
+                items[iid] = merged
+        elif etype == "item_archived":
+            iid = payload.get("id")
+            if iid in items:
+                merged = dict(items[iid])
+                merged["status"] = "archived"
+                if "completed_at" in payload:
+                    merged["completed_at"] = payload["completed_at"]
+                items[iid] = merged
+        # any other event type is ignored
+    return {"version": TRACKER_VERSION, "items": [items[i] for i in order]}

package/state_store/store.py

@@ -0,0 +1,102 @@
+"""state_store.store — SQLite-backed, append-only, concurrency-safe event log.
+
+Durable substrate for aesop's event-sourced state layer (the DB-source-of-truth
+design; git becomes a rendered export, not the coordination layer). Backend is
+SQLite in WAL mode so many readers and serialized writers share one file; the
+same interface is meant to swap to Postgres behind ``state_store.api.StateAPI``
+for team scale without touching call sites.
+
+Stdlib only (sqlite3, json, time) per aesop's no-external-deps invariant.
+"""
+from __future__ import annotations
+
+import json
+import sqlite3
+import time
+
+
+class EventStore:
+    """Append-only event log stored at ``db_path``.
+
+    Safe under concurrent appends from multiple threads AND multiple EventStore
+    instances on the same file: every call opens its own connection with
+    ``PRAGMA busy_timeout`` and assigns the per-stream version inside a
+    ``BEGIN IMMEDIATE`` transaction, so the read-max-version-then-insert is
+    atomic and two writers can never collide or duplicate a version.
+    """
+
+    def __init__(self, db_path: str):
+        self.db_path = db_path
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.execute("PRAGMA busy_timeout=5000")
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS events (
+                    id      INTEGER PRIMARY KEY AUTOINCREMENT,
+                    ts      REAL    NOT NULL,
+                    actor   TEXT    NOT NULL,
+                    stream  TEXT    NOT NULL,
+                    type    TEXT    NOT NULL,
+                    payload TEXT    NOT NULL,
+                    version INTEGER NOT NULL
+                )
+                """
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS idx_events_stream_version "
+                "ON events(stream, version)"
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def append(self, stream: str, event_type: str, payload: dict, actor: str = "system") -> int:
+        """Append one event to ``stream``; return its new per-stream version (1-based)."""
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("PRAGMA busy_timeout=5000")
+            # BEGIN IMMEDIATE takes the write lock up front so the
+            # read-max-then-insert below is atomic under contention.
+            conn.execute("BEGIN IMMEDIATE")
+            row = conn.execute(
+                "SELECT COALESCE(MAX(version), 0) FROM events WHERE stream = ?",
+                (stream,),
+            ).fetchone()
+            version = row[0] + 1
+            conn.execute(
+                "INSERT INTO events (ts, actor, stream, type, payload, version) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                (time.time(), actor, stream, event_type, json.dumps(payload), version),
+            )
+            conn.commit()
+            return version
+        finally:
+            conn.close()
+
+    def read(self, stream: str) -> list:
+        """Return all events for ``stream`` ascending by version (empty if none)."""
+        return self._rows("WHERE stream = ? ORDER BY version ASC", (stream,))
+
+    def read_all(self) -> list:
+        """Return all events across all streams ascending by id."""
+        return self._rows("ORDER BY id ASC", ())
+
+    def _rows(self, clause: str, params) -> list:
+        conn = sqlite3.connect(self.db_path)
+        try:
+            conn.execute("PRAGMA busy_timeout=5000")
+            cur = conn.execute(
+                "SELECT id, ts, actor, stream, type, payload, version FROM events " + clause,
+                params,
+            )
+            return [
+                {
+                    "id": r[0], "ts": r[1], "actor": r[2], "stream": r[3],
+                    "type": r[4], "payload": json.loads(r[5]), "version": r[6],
+                }
+                for r in cur.fetchall()
+            ]
+        finally:
+            conn.close()

package/tools/CLAUDE.md

@@ -0,0 +1,72 @@
+# tools/ — Build utilities
+
+Local-only Python (stdlib only, no external deps), bash (POSIX, CRLF-safe). Never print secrets — report by pattern name/location only.
+
+## FILES
+
+- `lock.mjs` — Fail-closed atomic lock acquisition (exponential backoff + stale-lock detection)
+- `secret_scan.py` — Pre-push secret/credential detection gate
+- `scanner_selftest.py` — Regression harness for secret_scan.py
+- `prepublish_scan.py` — Pre-publish full history + staged-changes scan gate
+- `metrics_gate.py` — PR gate for hard numeric claims in markdown
+- `self_stats.py` — Git-derived metrics counter + README block generator
+- `verify_dash.py` — Browser proof for realtime SSE dashboard
+- `verify_submit_encoding.py` — Browser proof for /submit UTF-8 inbox bootstrap
+- `ci_merge_wait.py` — CI-gated merge helper (polls gh pr view until SUCCESS)
+- `alert_bridge.py` — Slack/Discord webhook bridge for SECURITY-ALERTS
+- `proposals.mjs` — Proposal lifecycle manager (list/accept/reject)
+- `power_selftest.py` — Health check harness for /power bootstrap
+- `healthcheck.py` — Fleet health aggregator (heartbeat/alert/orchestrator status)
+- `buildlog.py` — Uniform BUILDLOG.md appender
+- `ensure_state.py` — Scaffold STATE.md and BUILDLOG.md templates
+- `fleet_ledger.py` — Append-only cost ledger with harvest/rotate
+- `heartbeat.py` — Single-instance loop liveness registry
+- `stall_check.py` — Automated agent transcript stall detector
+- `inbox_drain.py` — Drain UI inbox submissions
+- `orchestrator_status.py` — Atomic orchestrator status updates
+- `reconstitute.sh` — Clone/fetch repos from config with security validation
+- `eod_sweep.py` — End-of-day safety check (dirty trees, unpushed commits)
+- `agent-forensics.sh` — Incident forensics / behavior reconstruction
+- `launch_tui.py` — Spawn bash TUI script in detached terminal
+- `rotate_logs.py` — Log rotation utility with size/line thresholds
+- `session_usage_summary.py` — Aggregate token usage across session transcripts
+- `transcript_replay.py` — Replay post-commit edits from transcripts to recover work
+- `transcript_timeline.py` — Extract Write/Edit/Read timeline from transcripts
+- `fleet_prompt_extractor.py` — Extract and deduplicate Agent/Task spawn prompts
+- `svg_to_png.mjs` — Rasterize SVG to PNG via @resvg/resvg-js (with lazy import error handling)
+
+## secret_scan.py — Pre-push secret/credential detection gate
+
+Scans staged/history/paths for secrets by regex pattern and credential filenames; blocks pushes on findings.
+
+- `secret_scan.py --staged [--repo PATH]` — scan git staged files
+- `secret_scan.py --history [--repo PATH]` — scan all git history blobs
+- `secret_scan.py PATH [PATH...]` — scan files/dirs directly (recurse)
+
+Exit: 0=clean, 1=findings, 2=usage error. Output masks secrets as `xxxx...`.
+
+**Pragma escape** (pattern findings only; credential filenames always fatal):
+```
+# secretscan: allow-pattern-docs
+```
+Mark file's first 10 lines to report rule-based findings as ALLOWED-DOC (non-fatal). Use only for deliberate pattern documentation.
+
+## agent-forensics.sh — Incident forensics / behavior reconstruction
+
+Read-only git plumbing; reconstructs agent behavior snapshot or diffs behavior-controlling files.
+
+- `bash tools/agent-forensics.sh <commit>` — print commit header, rules snapshot, CLAUDE.md, STATE.md, last 30 lines of BUILDLOG.md
+- `bash tools/agent-forensics.sh --diff <commitA> <commitB>` — diff CLAUDE.md, STATE.md, docs/, monitor/CHARTER.md, hooks/ between commits
+
+Exit: 0=success, 1=error (never raw git traces). Requires: git, head, tail, wc, grep.
+
+
+## Invariants
+
+- **Dependency-light**: Python tools must work on base Python 3 (no pip installs).
+- **CRLF-safe shell**: no line continuations in .sh scripts; Git Bash + Linux compatible.
+- **Never print secrets**: mask as pattern name + masked value only.
+- **Config-driven paths**: heartbeat/ledger/logs use AESOP_STATE_ROOT env var (default ./state) or CLI args; no hardcoded personal paths.
+- **Fragment-assembled secrets in tests**: scanner_selftest.py concatenates dummy secrets at runtime so pattern text never appears contiguously (self-scan invariant).
+- **verify_*.py are mandatory CI gates**: verify_dash.py and verify_submit_encoding.py are required pre-push gates; use `--allow-skip` only in truly browserless environments (CI must run both).
+- **lock.mjs is the ONLY lock implementation**: never reimplement locking in proposals.mjs or elsewhere; all proposals/state updates must use fail-closed lock.mjs with exponential backoff + stale-lock breaking.

package/tools/agent-forensics.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+# agent-forensics.sh — incident forensics tool
+# Reconstruct agent behavior as of a commit using git plumbing.
+# Usage:
+#   bash tools/agent-forensics.sh <commit>
+#   bash tools/agent-forensics.sh --diff <commitA> <commitB>
+#
+# Prints:
+#   - Commit header (hash, date, subject)
+#   - Rules snapshot (docs/CARDINAL-RULES.md)
+#   - CLAUDE.md (if present)
+#   - STATE.md (if present)
+#   - Last 30 lines of BUILDLOG.md (if tracked)
+#   - For --diff: behavior changes between two commits
+#
+# Exit codes: 0 on success, 1 on error (never raw git stack traces).
+# Requires: git, head, tail, wc, grep
+# CRLF-safe: no line continuations, Git Bash + Linux compatible.
+
+set -e
+
+show_error() {
+  printf 'Error: %s\n' "$1" >&2
+  exit 1
+}
+
+show_usage() {
+  cat <<'EOF'
+Usage:
+  bash tools/agent-forensics.sh <commit>              Forensics at <commit>
+  bash tools/agent-forensics.sh --diff <commitA> <commitB>   Behavior diff
+EOF
+  exit 1
+}
+
+# --- Commit header (hash, date, subject) ---
+print_commit_header() {
+  local commit="$1"
+
+  # Verify commit exists
+  if ! git cat-file -t "$commit" >/dev/null 2>&1; then
+    show_error "unknown commit: $commit"
+  fi
+
+  local hash
+  local date
+  local subject
+
+  hash=$(git rev-parse "$commit" 2>/dev/null || show_error "commit parse failed: $commit")
+  date=$(git log -1 --format=%ai "$commit" 2>/dev/null || show_error "failed to read commit date")
+  subject=$(git log -1 --format=%s "$commit" 2>/dev/null || show_error "failed to read commit subject")
+
+  printf '# Forensics @ %s\n\n' "$hash"
+  printf 'Commit:  %s\n' "$hash"
+  printf 'Date:    %s\n' "$date"
+  printf 'Subject: %s\n\n' "$subject"
+}
+
+# --- Print first ~40 lines of docs/CARDINAL-RULES.md ---
+print_rules() {
+  local commit="$1"
+
+  printf '## Rules (docs/CARDINAL-RULES.md)\n\n'
+
+  if ! git cat-file -e "$commit:docs/CARDINAL-RULES.md" 2>/dev/null; then
+    printf '(file not present at this commit)\n\n'
+    return
+  fi
+
+  git show "$commit:docs/CARDINAL-RULES.md" 2>/dev/null | head -40
+
+  # Check if file was truncated
+  local total_lines
+  total_lines=$(git show "$commit:docs/CARDINAL-RULES.md" 2>/dev/null | wc -l)
+  if [ "$total_lines" -gt 40 ]; then
+    printf '\n[... %d more lines; see git show %s:docs/CARDINAL-RULES.md]\n' "$((total_lines - 40))" "$commit"
+  fi
+  printf '\n\n'
+}
+
+# --- Print CLAUDE.md if present ---
+print_claude_md() {
+  local commit="$1"
+
+  printf '## CLAUDE.md\n\n'
+
+  if ! git cat-file -e "$commit:CLAUDE.md" 2>/dev/null; then
+    printf '(not present at this commit)\n\n'
+    return
+  fi
+
+  git show "$commit:CLAUDE.md" 2>/dev/null
+  printf '\n\n'
+}
+
+# --- Print STATE.md if present ---
+print_state_md() {
+  local commit="$1"
+
+  printf '## STATE.md\n\n'
+
+  if ! git cat-file -e "$commit:STATE.md" 2>/dev/null; then
+    printf '(not present at this commit)\n\n'
+    return
+  fi
+
+  git show "$commit:STATE.md" 2>/dev/null
+  printf '\n\n'
+}
+
+# --- Print last 30 lines of BUILDLOG.md ---
+print_buildlog() {
+  local commit="$1"
+
+  printf '## BUILDLOG.md (last 30 lines)\n\n'
+
+  if ! git cat-file -e "$commit:BUILDLOG.md" 2>/dev/null; then
+    printf '(not tracked at this commit)\n\n'
+    return
+  fi
+
+  git show "$commit:BUILDLOG.md" 2>/dev/null | tail -30
+  printf '\n\n'
+}
+
+# --- Diff behavior between two commits ---
+diff_behavior() {
+  local commitA="$1"
+  local commitB="$2"
+
+  # Verify both commits exist
+  if ! git cat-file -t "$commitA" >/dev/null 2>&1; then
+    show_error "unknown commit: $commitA"
+  fi
+  if ! git cat-file -t "$commitB" >/dev/null 2>&1; then
+    show_error "unknown commit: $commitB"
+  fi
+
+  printf '# Behavior Diff\n\n'
+  printf '%s -> %s\n\n' "$(git rev-parse "$commitA")" "$(git rev-parse "$commitB")"
+
+  # Files that affect behavior: rules, CLAUDE.md, STATE.md, hooks, monitor/CHARTER.md
+  local behavior_files
+  behavior_files="CLAUDE.md STATE.md docs/CARDINAL-RULES.md docs/DISPATCH-MODEL.md docs/GOVERNANCE.md hooks/ monitor/CHARTER.md"
+
+  printf '## Changes in behavior-controlling files\n\n'
+
+  local has_changes=0
+  for file in $behavior_files; do
+    if git diff "$commitA" "$commitB" --name-only -- "$file" 2>/dev/null | grep -q .; then
+      has_changes=1
+      printf '### %s\n\n' "$file"
+      git diff "$commitA" "$commitB" -- "$file" 2>/dev/null | head -100
+      printf '\n'
+    fi
+  done
+
+  if [ "$has_changes" -eq 0 ]; then
+    printf '(no changes in behavior-controlling files)\n\n'
+  fi
+}
+
+# --- Main dispatch ---
+case "${1:-}" in
+  --diff)
+    if [ $# -lt 3 ]; then
+      show_error "--diff requires two commits: --diff <commitA> <commitB>"
+    fi
+    diff_behavior "$2" "$3"
+    ;;
+  --help|-h)
+    show_usage
+    ;;
+  '')
+    show_usage
+    ;;
+  *)
+    print_commit_header "$1"
+    print_rules "$1"
+    print_claude_md "$1"
+    print_state_md "$1"
+    print_buildlog "$1"
+    ;;
+esac
+
+exit 0