@matt82198/aesop 0.1.0 → 0.2.0

package/tools/wave_resume.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+Mid-wave recovery: classify completed vs remaining items from workflow journal.
+
+Given a workflow run's journal.jsonl and its worktree, parses journal to identify
+which items have already COMPLETED (files written + tests green) vs which remain,
+enabling resume from the last good phase instead of full re-run.
+
+Usage:
+  wave_resume.py --journal <path> --workdir <worktree> [--json]
+
+Inputs:
+  --journal PATH     Path to journal.jsonl (append-only log of task results)
+  --workdir PATH     Path to worktree root
+  --json             Output as JSON (default: human-readable summary)
+
+Output:
+  {
+    "completed": ["item-slug-1", "item-slug-2"],
+    "remaining": ["item-slug-3", "item-slug-4"],
+    "resume_hint": "Resume from item: item-slug-3"
+  }
+
+Logic:
+  1. Parse journal.jsonl (each line = JSON task result)
+  2. For each unique slug with status='completed':
+     - Verify all reported 'files' exist in workdir
+     - If all files exist, classify as completed
+     - Otherwise classify as remaining
+  3. Emit resume plan (completed, remaining, hint)
+
+Read-only: never mutates workdir or journal. Stdlib-only, Windows-safe paths.
+"""
+
+import json
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+
+def load_journal(journal_path: str) -> List[Dict[str, Any]]:
+    """
+    Load and parse journal.jsonl file.
+
+    Each line is expected to be valid JSON representing a task result.
+    Malformed JSON lines are silently skipped.
+
+    Args:
+        journal_path: Path to journal.jsonl file
+
+    Returns:
+        List of parsed JSON records (in order), or empty list if file missing/unreadable
+    """
+    journal = []
+    path = Path(journal_path)
+
+    if not path.exists():
+        return journal
+
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        record = json.loads(line)
+                        journal.append(record)
+                    except json.JSONDecodeError:
+                        # Silently skip malformed JSON lines
+                        continue
+    except (IOError, OSError):
+        # File unreadable; return empty list
+        pass
+
+    return journal
+
+
+def verify_files_exist(workdir: str, files: List[str]) -> bool:
+    """
+    Check if all files in the list exist under workdir.
+
+    Args:
+        workdir: Path to worktree root
+        files: List of relative file paths to verify
+
+    Returns:
+        True if all files exist (or list is empty), False if any missing
+    """
+    workdir_path = Path(workdir)
+
+    if not files:
+        # Empty list is considered valid (no files to verify)
+        return True
+
+    for file_path in files:
+        # Construct full path and check existence
+        full_path = workdir_path / file_path
+
+        # Normalize path to handle platform differences
+        try:
+            # Resolve to absolute path for safety
+            resolved = full_path.resolve()
+            if not resolved.exists():
+                return False
+        except (OSError, RuntimeError):
+            # Path resolution failed (e.g., invalid characters, symlink loops)
+            return False
+
+    return True
+
+
+def classify_items(
+    journal: List[Dict[str, Any]], workdir: str
+) -> Dict[str, Any]:
+    """
+    Classify items as completed vs remaining based on journal + file verification.
+
+    For each unique slug in journal:
+    - If status='completed' AND all reported files exist in workdir: completed
+    - Otherwise: remaining
+
+    Args:
+        journal: List of task result records from load_journal()
+        workdir: Path to worktree root
+
+    Returns:
+        {
+            "completed": [list of slugs],
+            "remaining": [list of slugs],
+            "resume_hint": "Resume from item: <first-remaining-slug>"
+        }
+    """
+    completed = []
+    remaining = []
+    processed_slugs = set()
+
+    for record in journal:
+        slug = record.get("slug")
+
+        # Skip records without slug or already-processed slugs
+        if not slug or slug in processed_slugs:
+            continue
+
+        processed_slugs.add(slug)
+
+        status = record.get("status", "unknown")
+        files = record.get("files", [])
+
+        # An item is completed if:
+        # 1. status == 'completed'
+        # 2. files list is non-empty AND all files exist
+        if status == "completed" and files and verify_files_exist(workdir, files):
+            completed.append(slug)
+        else:
+            remaining.append(slug)
+
+    # Generate resume hint
+    if remaining:
+        resume_hint = f"Resume from item: {remaining[0]}"
+    else:
+        resume_hint = "All items completed"
+
+    return {
+        "completed": completed,
+        "remaining": remaining,
+        "resume_hint": resume_hint,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Mid-wave recovery: classify completed vs remaining items",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "--journal",
+        required=True,
+        help="Path to journal.jsonl file",
+    )
+    parser.add_argument(
+        "--workdir",
+        required=True,
+        help="Path to worktree directory",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON (default: human-readable)",
+    )
+
+    args = parser.parse_args()
+
+    # Load and classify
+    journal = load_journal(args.journal)
+    result = classify_items(journal, args.workdir)
+
+    # Output
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"Completed: {len(result['completed'])} items")
+        print(f"Remaining: {len(result['remaining'])} items")
+        if result["completed"]:
+            print(f"  Completed: {', '.join(result['completed'])}")
+        if result["remaining"]:
+            print(f"  Remaining: {', '.join(result['remaining'])}")
+        print(result["resume_hint"])
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/wave_templates.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""Wave template manager: load and instantiate preset manifests for common project types.
+
+Provides reusable wave-manifest scaffolds for bootstrapping first waves:
+  - SaaS: API + frontend + ops (typical 3-tier)
+  - Data: pipeline + analytics + infra (ETL/analytics pattern)
+  - Library: core + tests + docs (reusable module pattern)
+
+Each preset is a JSON file with template variables ({project_name}, {base_dir})
+that are substituted during instantiation.
+
+Usage (CLI):
+  python tools/wave_templates.py <preset> --project-name my-app --base-dir /workspace
+
+Usage (Python API):
+  from wave_templates import load_preset, instantiate_template
+  preset = load_preset("saas")
+  manifest = instantiate_template(preset, project_name="my-app", base_dir="/workspace")
+
+Invariants:
+  - Presets live in templates/wave-presets/ (git-tracked, editable)
+  - No file ownership overlap within a manifest (preflight guard)
+  - All paths use forward slashes (POSIX) for cross-platform compatibility
+  - UTF-8 encoding, LF line endings (Linux parity)
+  - Manifests are fully resolved JSON (no further substitution by the wave engine)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+
+# Presets directory (relative to this file's location).
+TOOLS_DIR = Path(__file__).resolve().parent
+REPO_ROOT = TOOLS_DIR.parent
+PRESETS_DIR = REPO_ROOT / "templates" / "wave-presets"
+
+
+def load_preset(preset_name: str) -> Dict[str, Any]:
+    """Load a preset manifest template by name.
+
+    Args:
+        preset_name: preset identifier (e.g., "saas", "data", "library")
+
+    Returns:
+        dict with preset template (contains {project_name}, {base_dir} placeholders)
+
+    Raises:
+        FileNotFoundError: if preset file does not exist
+        json.JSONDecodeError: if preset JSON is invalid
+    """
+    preset_file = PRESETS_DIR / f"{preset_name}.json"
+    if not preset_file.exists():
+        raise FileNotFoundError(f"Preset not found: {preset_file}")
+
+    with open(preset_file, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def instantiate_template(
+    preset: Dict[str, Any],
+    project_name: str,
+    base_dir: str,
+) -> Dict[str, Any]:
+    """Instantiate a preset template with project-specific values.
+
+    Replaces placeholder strings {project_name} and {base_dir} throughout
+    the preset. Also derives a wave_id and wave_description if not provided.
+
+    Args:
+        preset: template dict from load_preset()
+        project_name: user's project/app name (e.g., "payment-api")
+        base_dir: root working directory (e.g., "/workspace/my-app")
+
+    Returns:
+        dict with fully resolved manifest (ready to pass to wave engine)
+    """
+    # Deep copy to avoid mutating the original preset.
+    manifest = json.loads(json.dumps(preset))
+
+    # Utility to substitute placeholders recursively.
+    def substitute(obj):
+        if isinstance(obj, str):
+            return obj.replace("{project_name}", project_name).replace("{base_dir}", base_dir)
+        elif isinstance(obj, dict):
+            return {k: substitute(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [substitute(item) for item in obj]
+        else:
+            return obj
+
+    manifest = substitute(manifest)
+
+    # Add wave metadata if not already present.
+    if "wave_id" not in manifest:
+        manifest["wave_id"] = f"wave-{project_name}".lower().replace(" ", "-")
+    if "wave_description" not in manifest:
+        manifest["wave_description"] = f"Bootstrap wave for {project_name}"
+
+    return manifest
+
+
+def validate_manifest(manifest: Dict[str, Any]) -> None:
+    """Validate manifest schema and invariants.
+
+    Checks:
+      - items array exists and is non-empty
+      - each item has required fields (slug, prompt, ownsFiles)
+      - no file ownership overlap
+      - no placeholder strings remain
+
+    Args:
+        manifest: the manifest dict to validate
+
+    Raises:
+        ValueError: if validation fails
+    """
+    if "items" not in manifest:
+        raise ValueError("Manifest missing 'items' array")
+
+    items = manifest["items"]
+    if not isinstance(items, list) or len(items) == 0:
+        raise ValueError("'items' must be a non-empty list")
+
+    # Check required fields and validate structure.
+    required_fields = {"slug", "prompt", "ownsFiles"}
+    owner_map = {}
+    conflicts = []
+
+    for item in items:
+        if not isinstance(item, dict):
+            raise ValueError(f"Item must be a dict, got {type(item)}")
+
+        # Check required fields.
+        for field in required_fields:
+            if field not in item:
+                raise ValueError(f"Item {item.get('slug', 'unknown')} missing '{field}'")
+
+        slug = item["slug"]
+        if not isinstance(slug, str) or not slug:
+            raise ValueError(f"Item slug must be non-empty string, got {slug}")
+
+        # Check ownsFiles.
+        owned_files = item.get("ownsFiles", [])
+        if not isinstance(owned_files, list) or not owned_files:
+            raise ValueError(f"Item {slug} must have non-empty ownsFiles list")
+
+        # Track file ownership.
+        for f in owned_files:
+            if f in owner_map:
+                conflicts.append((f, owner_map[f], slug))
+            else:
+                owner_map[f] = slug
+
+    if conflicts:
+        raise ValueError(f"File ownership overlap: {conflicts}")
+
+    # Check no placeholders remain.
+    manifest_str = json.dumps(manifest)
+    if "{project_name}" in manifest_str or "{base_dir}" in manifest_str:
+        raise ValueError("Manifest contains unsubstituted placeholders")
+
+
+def main():
+    """CLI entry point: load preset and output resolved manifest."""
+    parser = argparse.ArgumentParser(
+        description="Load and instantiate a wave manifest preset"
+    )
+    parser.add_argument(
+        "preset",
+        help="preset name: saas, data, or library"
+    )
+    parser.add_argument(
+        "--project-name",
+        required=True,
+        help="project/app name (e.g., 'payment-api')"
+    )
+    parser.add_argument(
+        "--base-dir",
+        required=True,
+        help="base working directory (e.g., '/workspace/my-app')"
+    )
+    parser.add_argument(
+        "--output",
+        default=None,
+        help="output file (default: stdout)"
+    )
+
+    args = parser.parse_args()
+
+    try:
+        # Load and instantiate.
+        preset = load_preset(args.preset)
+        manifest = instantiate_template(preset, args.project_name, args.base_dir)
+
+        # Validate.
+        validate_manifest(manifest)
+
+        # Output.
+        output_json = json.dumps(manifest, indent=2)
+        if args.output:
+            output_path = Path(args.output)
+            output_path.write_text(output_json, encoding="utf-8")
+            print(f"Manifest written to {output_path}", file=sys.stderr)
+        else:
+            print(output_json)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/ui/agents.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 """Aesop UI — agent transcript reading + path-traversal-safe id handling (wave-9 split)."""
 import json
+import os
 import re
 import subprocess
 import sys
+import time
 from pathlib import Path
 
 import config
@@ -59,6 +61,11 @@ def sanitize_agents_for_broadcast(agents):
 def get_fleet_agents():
     """Detect running subagents by calling dash-extra.mjs --json.
 
+    Fallback (test-only): when AESOP_PROOF_FIXTURES=1 is set, read fixture agents from
+    _collector.json if dash-extra.mjs returns no agents (used by browser proofs that
+    don't have running agent processes). This env var is NEVER set in production, so
+    the fallback is safely gated.
+
     dash-extra.mjs truncates agent ids to 13 characters for display. With enough
     concurrently-active agents, two distinct agents can share the same 13-char
     prefix and collide onto the same id. The dashboard keys DOM rows (and the
@@ -71,23 +78,36 @@ def get_fleet_agents():
     try:
         # Call the working detector (dash-extra.mjs) with --json flag
         dash_extra_path = config.AESOP_ROOT / "dash" / "dash-extra.mjs"
-        if not dash_extra_path.exists():
-            return agents
-        result = subprocess.run(
-            ["node", str(dash_extra_path), "--json"],
-            capture_output=True,
-            text=True,
-            encoding='utf-8',
-            errors='replace',
-            timeout=5
-        )
-        if result.returncode == 0 and result.stdout:
-            agents = json.loads(result.stdout.strip())
+        if dash_extra_path.exists():
+            result = subprocess.run(
+                ["node", str(dash_extra_path), "--json"],
+                capture_output=True,
+                text=True,
+                encoding='utf-8',
+                errors='replace',
+                timeout=5
+            )
+            if result.returncode == 0 and result.stdout:
+                agents = json.loads(result.stdout.strip())
     except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError):
         pass
     except Exception:
         pass
 
+    # Fallback (test-only): if no agents detected by dash-extra.mjs AND
+    # AESOP_PROOF_FIXTURES is set, try reading from _collector.json
+    # (used by browser proofs that create fixture agents in state/_collector.json).
+    # This is gated by an explicit env var so it never fires in production.
+    if not agents and os.getenv("AESOP_PROOF_FIXTURES"):
+        try:
+            collector_json = config.STATE_DIR / "_collector.json"
+            if collector_json.exists():
+                with open(collector_json, encoding='utf-8') as f:
+                    data = json.load(f)
+                    agents = data.get("agents", [])
+        except (json.JSONDecodeError, OSError, KeyError):
+            pass
+
     seen = {}
     for a in agents:
         if not isinstance(a, dict):
@@ -102,6 +122,12 @@ def get_fleet_agents():
 
 _AGENT_ID_FORBIDDEN = re.compile(r'\.\.|[/\\*?\[\]]')
 
+# Fingerprint caching: prevent expensive recursive glob on every collector tick (~1Hz).
+# Cache the fingerprint for N seconds, recompute only when window expires.
+# This keeps the cache-busting purpose (detect real changes), but throttles the cost.
+_FINGERPRINT_CACHE = {"value": None, "expires": 0.0}
+_FINGERPRINT_CACHE_TTL = 5.0  # seconds; can be overridden by tests
+
 # Transcript-tail bounds (defense against loading a whole multi-MB transcript
 # into memory and against emitting an unbounded payload to the browser).
 TRANSCRIPT_TAIL_LINES = 40          # last N NDJSON lines rendered in the drawer
@@ -457,12 +483,15 @@ def extract_agent_dispatch_prompt(agent_id):
         print(f"[extract_agent_dispatch_prompt] Uncaught exception: {e}", file=sys.stderr)
         return {"error": "Failed to extract dispatch prompt"}
 
-def _transcripts_fingerprint():
-    """Cheap fs-stat-only fingerprint of the transcripts tree.
+def _transcripts_fingerprint_uncached():
+    """Cheap fs-stat-only fingerprint of the transcripts tree (no caching).
 
     Used to decide whether it's worth re-invoking `node dash-extra.mjs` (which is
     comparatively expensive: process spawn + re-parsing every agent transcript).
     Only file count + max mtime — no file content is read.
+
+    This is the raw implementation; see _transcripts_fingerprint() for the
+    cache-throttled wrapper.
     """
     try:
         if not config.TRANSCRIPTS_ROOT.exists():
@@ -480,3 +509,28 @@ def _transcripts_fingerprint():
         return (count, latest)
     except Exception:
         return (0, 0.0)
+
+
+def _transcripts_fingerprint():
+    """Cached wrapper around _transcripts_fingerprint_uncached().
+
+    Throttles fingerprint recomputation to at most once per _FINGERPRINT_CACHE_TTL
+    seconds (default 5s, configurable). Returns cached value until window expires.
+    This keeps the cache-busting purpose (detect real agent changes) but prevents
+    the expensive recursive glob from running on every collector tick (~1Hz).
+
+    Returns:
+        tuple: (file_count, latest_mtime) — same format as uncached version.
+    """
+    global _FINGERPRINT_CACHE
+    now = time.time()
+
+    # If cache is still valid, return cached value
+    if _FINGERPRINT_CACHE["value"] is not None and now < _FINGERPRINT_CACHE["expires"]:
+        return _FINGERPRINT_CACHE["value"]
+
+    # Cache expired or uninitialized; recompute
+    value = _transcripts_fingerprint_uncached()
+    _FINGERPRINT_CACHE["value"] = value
+    _FINGERPRINT_CACHE["expires"] = now + _FINGERPRINT_CACHE_TTL
+    return value

package/ui/collectors.py

@@ -371,61 +371,6 @@ def save_tracker(tracker):
             print(f"[tracker] Failed to unlink temp file: {ue}", file=sys.stderr)
         raise
 
-def migrate_tracker_from_backlog():
-    """One-time idempotent migration: AUDIT-BACKLOG.md -> tracker.json."""
-    if config.TRACKER_FILE.exists():
-        return load_tracker()
-
-    backlog_data = parse_audit_backlog()
-    if not backlog_data.get("tiers"):
-        return {"version": 1, "items": []}
-
-    items = []
-    for tier_data in backlog_data["tiers"]:
-        priority = tier_data["tier"]
-
-        for backlog_item in tier_data.get("items", []):
-            status_glyph = backlog_item["status"]
-
-            if status_glyph == "✅":
-                status, lane = "done", "done"
-                tags = []
-            elif status_glyph == "🔵":
-                status, lane = "in-progress", "in-progress"
-                tags = []
-            elif status_glyph == "⏸":
-                status, lane = "todo", "proposed"
-                tags = ["needs-decision"]
-            else:
-                status, lane = "todo", "ranked"
-                tags = []
-
-            title = backlog_item.get("title", "")
-            tag_prefix = backlog_item.get("tag", "")
-            if tag_prefix:
-                tag_value = tag_prefix.strip("[]")
-                if tag_value and tag_value not in tags:
-                    tags.insert(0, tag_value)
-
-            item = {
-                "id": secrets.token_hex(6),
-                "title": title,
-                "priority": priority,
-                "status": status,
-                "lane": lane,
-                "source": "audit-backlog-migration",
-                "tags": tags,
-                "notes": None,
-                "pr_link": None,
-                "created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
-                "completed_at": None
-            }
-            items.append(item)
-
-    tracker = {"version": 1, "items": items}
-    save_tracker(tracker)
-    return tracker
-
 def get_tracker_items(status=None, priority=None):
     """Retrieve tracker items with optional filters."""
     tracker = load_tracker()

package/ui/config.py

@@ -97,8 +97,13 @@ def reload():
     ORCH_STATUS_FILE = STATE_DIR / "orchestrator-status.json"
 
     # Wave-14 dashboard rewrite (plan D3): built frontend + cost ledger paths.
-    # WEB_DIST: the committed Vite build output served at / and /assets/*.
-    WEB_DIST = AESOP_ROOT / "ui" / "web" / "dist"
+    # WEB_DIST: env AESOP_WEB_DIST > AESOP_ROOT/ui/web/dist (test override for fixtures).
+    WEB_DIST = Path(
+        os.getenv(
+            "AESOP_WEB_DIST",
+            str(AESOP_ROOT / "ui" / "web" / "dist")
+        )
+    )
     # LEDGER_FILE: outcomes ledger parsed by ui/cost.py; sse.py mtime-gates on it.
     LEDGER_FILE = STATE_DIR / "ledger" / "OUTCOMES-LEDGER.md"