@matt82198/aesop 0.1.0 → 0.3.1

package/tools/wave_resume.py

@@ -0,0 +1,215 @@
+#!/usr/bin/env python3
+"""
+Mid-wave recovery: classify completed vs remaining items from workflow journal.
+
+Given a workflow run's journal.jsonl and its worktree, parses journal to identify
+which items have already COMPLETED (files written + tests green) vs which remain,
+enabling resume from the last good phase instead of full re-run.
+
+Usage:
+  wave_resume.py --journal <path> --workdir <worktree> [--json]
+
+Inputs:
+  --journal PATH     Path to journal.jsonl (append-only log of task results)
+  --workdir PATH     Path to worktree root
+  --json             Output as JSON (default: human-readable summary)
+
+Output:
+  {
+    "completed": ["item-slug-1", "item-slug-2"],
+    "remaining": ["item-slug-3", "item-slug-4"],
+    "resume_hint": "Resume from item: item-slug-3"
+  }
+
+Logic:
+  1. Parse journal.jsonl (each line = JSON task result)
+  2. For each unique slug with status='completed':
+     - Verify all reported 'files' exist in workdir
+     - If all files exist, classify as completed
+     - Otherwise classify as remaining
+  3. Emit resume plan (completed, remaining, hint)
+
+Read-only: never mutates workdir or journal. Stdlib-only, Windows-safe paths.
+"""
+
+import json
+import argparse
+import sys
+from pathlib import Path
+from typing import List, Dict, Any, Optional
+
+
+def load_journal(journal_path: str) -> List[Dict[str, Any]]:
+    """
+    Load and parse journal.jsonl file.
+
+    Each line is expected to be valid JSON representing a task result.
+    Malformed JSON lines are silently skipped.
+
+    Args:
+        journal_path: Path to journal.jsonl file
+
+    Returns:
+        List of parsed JSON records (in order), or empty list if file missing/unreadable
+    """
+    journal = []
+    path = Path(journal_path)
+
+    if not path.exists():
+        return journal
+
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        record = json.loads(line)
+                        journal.append(record)
+                    except json.JSONDecodeError:
+                        # Silently skip malformed JSON lines
+                        continue
+    except (IOError, OSError):
+        # File unreadable; return empty list
+        pass
+
+    return journal
+
+
+def verify_files_exist(workdir: str, files: List[str]) -> bool:
+    """
+    Check if all files in the list exist under workdir.
+
+    Args:
+        workdir: Path to worktree root
+        files: List of relative file paths to verify
+
+    Returns:
+        True if all files exist (or list is empty), False if any missing
+    """
+    workdir_path = Path(workdir)
+
+    if not files:
+        # Empty list is considered valid (no files to verify)
+        return True
+
+    for file_path in files:
+        # Construct full path and check existence
+        full_path = workdir_path / file_path
+
+        # Normalize path to handle platform differences
+        try:
+            # Resolve to absolute path for safety
+            resolved = full_path.resolve()
+            if not resolved.exists():
+                return False
+        except (OSError, RuntimeError):
+            # Path resolution failed (e.g., invalid characters, symlink loops)
+            return False
+
+    return True
+
+
+def classify_items(
+    journal: List[Dict[str, Any]], workdir: str
+) -> Dict[str, Any]:
+    """
+    Classify items as completed vs remaining based on journal + file verification.
+
+    For each unique slug in journal:
+    - If status='completed' AND all reported files exist in workdir: completed
+    - Otherwise: remaining
+
+    Args:
+        journal: List of task result records from load_journal()
+        workdir: Path to worktree root
+
+    Returns:
+        {
+            "completed": [list of slugs],
+            "remaining": [list of slugs],
+            "resume_hint": "Resume from item: <first-remaining-slug>"
+        }
+    """
+    completed = []
+    remaining = []
+    processed_slugs = set()
+
+    for record in journal:
+        slug = record.get("slug")
+
+        # Skip records without slug or already-processed slugs
+        if not slug or slug in processed_slugs:
+            continue
+
+        processed_slugs.add(slug)
+
+        status = record.get("status", "unknown")
+        files = record.get("files", [])
+
+        # An item is completed if:
+        # 1. status == 'completed'
+        # 2. files list is non-empty AND all files exist
+        if status == "completed" and files and verify_files_exist(workdir, files):
+            completed.append(slug)
+        else:
+            remaining.append(slug)
+
+    # Generate resume hint
+    if remaining:
+        resume_hint = f"Resume from item: {remaining[0]}"
+    else:
+        resume_hint = "All items completed"
+
+    return {
+        "completed": completed,
+        "remaining": remaining,
+        "resume_hint": resume_hint,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Mid-wave recovery: classify completed vs remaining items",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=__doc__,
+    )
+    parser.add_argument(
+        "--journal",
+        required=True,
+        help="Path to journal.jsonl file",
+    )
+    parser.add_argument(
+        "--workdir",
+        required=True,
+        help="Path to worktree directory",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON (default: human-readable)",
+    )
+
+    args = parser.parse_args()
+
+    # Load and classify
+    journal = load_journal(args.journal)
+    result = classify_items(journal, args.workdir)
+
+    # Output
+    if args.json:
+        print(json.dumps(result, indent=2))
+    else:
+        print(f"Completed: {len(result['completed'])} items")
+        print(f"Remaining: {len(result['remaining'])} items")
+        if result["completed"]:
+            print(f"  Completed: {', '.join(result['completed'])}")
+        if result["remaining"]:
+            print(f"  Remaining: {', '.join(result['remaining'])}")
+        print(result["resume_hint"])
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/tools/wave_templates.py

@@ -0,0 +1,340 @@
+#!/usr/bin/env python3
+"""Wave template manager: load and instantiate preset manifests for common project types.
+
+Provides reusable wave-manifest scaffolds for bootstrapping first waves:
+  - SaaS: API + frontend + ops (typical 3-tier)
+  - Data: pipeline + analytics + infra (ETL/analytics pattern)
+  - Library: core + tests + docs (reusable module pattern)
+
+Each preset is a JSON file with template variables ({project_name}, {base_dir})
+that are substituted during instantiation.
+
+Usage (CLI):
+  python tools/wave_templates.py <preset> --project-name my-app --base-dir /workspace
+
+Usage (Python API):
+  from wave_templates import load_preset, instantiate_template
+  preset = load_preset("saas")
+  manifest = instantiate_template(preset, project_name="my-app", base_dir="/workspace")
+
+Invariants:
+  - Presets live in templates/wave-presets/ (git-tracked, editable)
+  - No file ownership overlap within a manifest (preflight guard)
+  - All paths use forward slashes (POSIX) for cross-platform compatibility
+  - UTF-8 encoding, LF line endings (Linux parity)
+  - Manifests are fully resolved JSON (no further substitution by the wave engine)
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Dict, Any, List, Optional, Tuple
+
+# Presets directory (relative to this file's location).
+TOOLS_DIR = Path(__file__).resolve().parent
+REPO_ROOT = TOOLS_DIR.parent
+PRESETS_DIR = REPO_ROOT / "templates" / "wave-presets"
+
+
+def load_preset(preset_name: str) -> Dict[str, Any]:
+    """Load a preset manifest template by name.
+
+    Args:
+        preset_name: preset identifier (e.g., "saas", "data", "library")
+
+    Returns:
+        dict with preset template (contains {project_name}, {base_dir} placeholders)
+
+    Raises:
+        FileNotFoundError: if preset file does not exist
+        json.JSONDecodeError: if preset JSON is invalid
+    """
+    preset_file = PRESETS_DIR / f"{preset_name}.json"
+    if not preset_file.exists():
+        raise FileNotFoundError(f"Preset not found: {preset_file}")
+
+    with open(preset_file, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def instantiate_template(
+    preset: Dict[str, Any],
+    project_name: str,
+    base_dir: str,
+) -> Dict[str, Any]:
+    """Instantiate a preset template with project-specific values.
+
+    Replaces placeholder strings {project_name} and {base_dir} throughout
+    the preset. Also derives a wave_id and wave_description if not provided.
+
+    Args:
+        preset: template dict from load_preset()
+        project_name: user's project/app name (e.g., "payment-api")
+        base_dir: root working directory (e.g., "/workspace/my-app")
+
+    Returns:
+        dict with fully resolved manifest (ready to pass to wave engine)
+    """
+    # Deep copy to avoid mutating the original preset.
+    manifest = json.loads(json.dumps(preset))
+
+    # Utility to substitute placeholders recursively.
+    def substitute(obj):
+        if isinstance(obj, str):
+            return obj.replace("{project_name}", project_name).replace("{base_dir}", base_dir)
+        elif isinstance(obj, dict):
+            return {k: substitute(v) for k, v in obj.items()}
+        elif isinstance(obj, list):
+            return [substitute(item) for item in obj]
+        else:
+            return obj
+
+    manifest = substitute(manifest)
+
+    # Add wave metadata if not already present.
+    if "wave_id" not in manifest:
+        manifest["wave_id"] = f"wave-{project_name}".lower().replace(" ", "-")
+    if "wave_description" not in manifest:
+        manifest["wave_description"] = f"Bootstrap wave for {project_name}"
+
+    return manifest
+
+
+def validate_manifest(
+    manifest: Dict[str, Any],
+    allow_placeholders: bool = True,
+    require_testcmd: bool = True
+) -> None:
+    """Validate manifest schema and invariants.
+
+    Checks:
+      - items array exists and is non-empty
+      - each item has required fields (slug, prompt, ownsFiles, and optionally testCmd)
+      - no file ownership overlap (per-manifest)
+      - optionally: no placeholder strings remain (for instantiated manifests)
+
+    Args:
+        manifest: the manifest dict to validate
+        allow_placeholders: if False, reject unsubstituted placeholders (default: True,
+                           for backward compatibility with presets). Set to False for
+                           instantiated manifests.
+        require_testcmd: if True, require testCmd field (default: True, since the wave
+                        engine needs it). Set to False to validate presets only.
+
+    Raises:
+        ValueError: if validation fails with detailed error per item
+    """
+    if "items" not in manifest:
+        raise ValueError("Manifest missing 'items' array")
+
+    items = manifest["items"]
+    if not isinstance(items, list) or len(items) == 0:
+        raise ValueError("'items' must be a non-empty list")
+
+    # Core required fields (always required).
+    required_core_fields = {"slug", "prompt", "ownsFiles"}
+    # Optional fields for instantiated manifests.
+    optional_fields = {"testCmd", "workDir"} if require_testcmd else set()
+    required_fields = required_core_fields | ({"testCmd"} if require_testcmd else set())
+
+    owner_map = {}
+    conflicts = []
+    errors = []
+
+    for item_idx, item in enumerate(items):
+        if not isinstance(item, dict):
+            errors.append(f"Item {item_idx}: must be a dict, got {type(item)}")
+            continue
+
+        # Check required core fields.
+        for field in required_core_fields:
+            if field not in item:
+                errors.append(f"Item {item_idx} ({item.get('slug', 'unknown')}): missing required field '{field}'")
+
+        # Check required testCmd if flag is set.
+        if require_testcmd and "testCmd" not in item:
+            errors.append(f"Item {item_idx} ({item.get('slug', 'unknown')}): missing required field 'testCmd'")
+
+        slug = item.get("slug")
+        if slug is not None:
+            if not isinstance(slug, str) or not slug:
+                errors.append(f"Item {item_idx}: slug must be non-empty string, got {slug}")
+
+        # Check ownsFiles.
+        owned_files = item.get("ownsFiles", [])
+        if not isinstance(owned_files, list):
+            errors.append(f"Item {item_idx} ({slug}): ownsFiles must be a list")
+            continue
+        if len(owned_files) == 0:
+            errors.append(f"Item {item_idx} ({slug}): ownsFiles must be non-empty")
+            continue
+
+        # Track file ownership (within this manifest only).
+        for f in owned_files:
+            if f in owner_map:
+                conflicts.append((f, owner_map[f], slug))
+            else:
+                owner_map[f] = slug
+
+    # Report all collected errors first.
+    if errors:
+        error_msg = "Manifest validation failed:\n  " + "\n  ".join(errors)
+        raise ValueError(error_msg)
+
+    if conflicts:
+        conflict_msg = "File ownership overlap (within manifest):\n  " + "\n  ".join(
+            [f"{f!r}: owned by both {o1!r} and {o2!r}" for f, o1, o2 in conflicts]
+        )
+        raise ValueError(conflict_msg)
+
+    # Check no placeholders remain (if instantiated manifest).
+    if not allow_placeholders:
+        manifest_str = json.dumps(manifest)
+        if "{project_name}" in manifest_str or "{base_dir}" in manifest_str:
+            raise ValueError("Manifest contains unsubstituted placeholders (not fully instantiated)")
+
+
+def validate_presets(preset_names: List[str], output_json: bool = False) -> Tuple[bool, List[str]]:
+    """Validate one or more presets.
+
+    Args:
+        preset_names: list of preset names to validate (e.g., ["saas", "data", "library"])
+
+    Returns:
+        tuple (success: bool, errors: List[str])
+          - success is True if all presets validate clean
+          - errors is a list of formatted error messages per preset/item
+    """
+    errors = []
+    all_valid = True
+    results = {}
+
+    for preset_name in preset_names:
+        try:
+            preset = load_preset(preset_name)
+            # Validate the preset: allow placeholders (presets have them), require testCmd (wave engine needs it)
+            validate_manifest(preset, allow_placeholders=True, require_testcmd=True)
+            results[preset_name] = {"valid": True, "errors": []}
+            print(f"✓ {preset_name}: valid", file=sys.stderr)
+        except FileNotFoundError as e:
+            errors.append(f"✗ {preset_name}: {e}")
+            all_valid = False
+            results[preset_name] = {"valid": False, "errors": [str(e)]}
+        except ValueError as e:
+            errors.append(f"✗ {preset_name}: {e}")
+            all_valid = False
+            results[preset_name] = {"valid": False, "errors": [str(e)]}
+        except Exception as e:
+            errors.append(f"✗ {preset_name}: unexpected error: {e}")
+            all_valid = False
+            results[preset_name] = {"valid": False, "errors": [str(e)]}
+
+    if output_json:
+        print(json.dumps({"ok": all_valid, "templates": results}))
+    else:
+        # Print errors if any
+        for error in errors:
+            print(error, file=sys.stderr)
+
+    return all_valid, errors
+
+
+def main():
+    """CLI entry point: load preset, instantiate, validate, or validate presets."""
+    parser = argparse.ArgumentParser(
+        description="Wave manifest preset manager: load, instantiate, and validate presets"
+    )
+
+    # Create subparsers for different commands
+    subparsers = parser.add_subparsers(dest="command", help="command to run")
+
+    # Subcommand: validate
+    validate_parser = subparsers.add_parser(
+        "validate",
+        help="validate preset template(s) for completeness"
+    )
+    validate_parser.add_argument(
+        "--template",
+        choices=["saas", "data", "library", "all"],
+        default="all",
+        help="which preset(s) to validate (default: all)"
+    )
+    validate_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="output results in JSON format"
+    )
+
+    # Subcommand: instantiate
+    inst_parser = subparsers.add_parser(
+        "instantiate",
+        help="load and instantiate a preset manifest"
+    )
+    inst_parser.add_argument(
+        "preset",
+        help="preset name: saas, data, or library"
+    )
+    inst_parser.add_argument(
+        "--project-name",
+        required=True,
+        help="project/app name (e.g., 'payment-api')"
+    )
+    inst_parser.add_argument(
+        "--base-dir",
+        required=True,
+        help="base working directory (e.g., '/workspace/my-app')"
+    )
+    inst_parser.add_argument(
+        "--output",
+        default=None,
+        help="output file (default: stdout)"
+    )
+
+    args = parser.parse_args()
+
+    try:
+        if args.command == "validate":
+            # Validate preset(s)
+            if args.template == "all":
+                presets = ["saas", "data", "library"]
+            else:
+                presets = [args.template]
+
+            success, errors = validate_presets(presets, output_json=args.json)
+
+            if not success:
+                sys.exit(1)
+            else:
+                print(f"\nAll {len(presets)} preset(s) validated successfully.", file=sys.stderr)
+                sys.exit(0)
+
+        elif args.command == "instantiate":
+            # Instantiate preset
+            preset = load_preset(args.preset)
+            manifest = instantiate_template(preset, args.project_name, args.base_dir)
+
+            # Validate the instantiated manifest (disallow placeholders, require testCmd)
+            validate_manifest(manifest, allow_placeholders=False, require_testcmd=True)
+
+            # Output.
+            output_json = json.dumps(manifest, indent=2)
+            if args.output:
+                output_path = Path(args.output)
+                output_path.write_text(output_json, encoding="utf-8")
+                print(f"Manifest written to {output_path}", file=sys.stderr)
+            else:
+                print(output_json)
+
+        else:
+            # No command specified - print help
+            parser.print_help()
+            sys.exit(0)
+
+    except Exception as e:
+        print(f"Error: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/ui/agents.py

@@ -1,9 +1,11 @@
 #!/usr/bin/env python3
 """Aesop UI — agent transcript reading + path-traversal-safe id handling (wave-9 split)."""
 import json
+import os
 import re
 import subprocess
 import sys
+import time
 from pathlib import Path
 
 import config
@@ -59,6 +61,11 @@ def sanitize_agents_for_broadcast(agents):
 def get_fleet_agents():
     """Detect running subagents by calling dash-extra.mjs --json.
 
+    Fallback (test-only): when AESOP_PROOF_FIXTURES=1 is set, read fixture agents from
+    _collector.json if dash-extra.mjs returns no agents (used by browser proofs that
+    don't have running agent processes). This env var is NEVER set in production, so
+    the fallback is safely gated.
+
     dash-extra.mjs truncates agent ids to 13 characters for display. With enough
     concurrently-active agents, two distinct agents can share the same 13-char
     prefix and collide onto the same id. The dashboard keys DOM rows (and the
@@ -71,23 +78,36 @@ def get_fleet_agents():
     try:
         # Call the working detector (dash-extra.mjs) with --json flag
         dash_extra_path = config.AESOP_ROOT / "dash" / "dash-extra.mjs"
-        if not dash_extra_path.exists():
-            return agents
-        result = subprocess.run(
-            ["node", str(dash_extra_path), "--json"],
-            capture_output=True,
-            text=True,
-            encoding='utf-8',
-            errors='replace',
-            timeout=5
-        )
-        if result.returncode == 0 and result.stdout:
-            agents = json.loads(result.stdout.strip())
+        if dash_extra_path.exists():
+            result = subprocess.run(
+                ["node", str(dash_extra_path), "--json"],
+                capture_output=True,
+                text=True,
+                encoding='utf-8',
+                errors='replace',
+                timeout=5
+            )
+            if result.returncode == 0 and result.stdout:
+                agents = json.loads(result.stdout.strip())
     except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError):
         pass
     except Exception:
         pass
 
+    # Fallback (test-only): if no agents detected by dash-extra.mjs AND
+    # AESOP_PROOF_FIXTURES is set, try reading from _collector.json
+    # (used by browser proofs that create fixture agents in state/_collector.json).
+    # This is gated by an explicit env var so it never fires in production.
+    if not agents and os.getenv("AESOP_PROOF_FIXTURES"):
+        try:
+            collector_json = config.STATE_DIR / "_collector.json"
+            if collector_json.exists():
+                with open(collector_json, encoding='utf-8') as f:
+                    data = json.load(f)
+                    agents = data.get("agents", [])
+        except (json.JSONDecodeError, OSError, KeyError):
+            pass
+
     seen = {}
     for a in agents:
         if not isinstance(a, dict):
@@ -102,6 +122,12 @@ def get_fleet_agents():
 
 _AGENT_ID_FORBIDDEN = re.compile(r'\.\.|[/\\*?\[\]]')
 
+# Fingerprint caching: prevent expensive recursive glob on every collector tick (~1Hz).
+# Cache the fingerprint for N seconds, recompute only when window expires.
+# This keeps the cache-busting purpose (detect real changes), but throttles the cost.
+_FINGERPRINT_CACHE = {"value": None, "expires": 0.0}
+_FINGERPRINT_CACHE_TTL = 5.0  # seconds; can be overridden by tests
+
 # Transcript-tail bounds (defense against loading a whole multi-MB transcript
 # into memory and against emitting an unbounded payload to the browser).
 TRANSCRIPT_TAIL_LINES = 40          # last N NDJSON lines rendered in the drawer
@@ -457,12 +483,15 @@ def extract_agent_dispatch_prompt(agent_id):
         print(f"[extract_agent_dispatch_prompt] Uncaught exception: {e}", file=sys.stderr)
         return {"error": "Failed to extract dispatch prompt"}
 
-def _transcripts_fingerprint():
-    """Cheap fs-stat-only fingerprint of the transcripts tree.
+def _transcripts_fingerprint_uncached():
+    """Cheap fs-stat-only fingerprint of the transcripts tree (no caching).
 
     Used to decide whether it's worth re-invoking `node dash-extra.mjs` (which is
     comparatively expensive: process spawn + re-parsing every agent transcript).
     Only file count + max mtime — no file content is read.
+
+    This is the raw implementation; see _transcripts_fingerprint() for the
+    cache-throttled wrapper.
     """
     try:
         if not config.TRANSCRIPTS_ROOT.exists():
@@ -480,3 +509,28 @@ def _transcripts_fingerprint():
         return (count, latest)
     except Exception:
         return (0, 0.0)
+
+
+def _transcripts_fingerprint():
+    """Cached wrapper around _transcripts_fingerprint_uncached().
+
+    Throttles fingerprint recomputation to at most once per _FINGERPRINT_CACHE_TTL
+    seconds (default 5s, configurable). Returns cached value until window expires.
+    This keeps the cache-busting purpose (detect real agent changes) but prevents
+    the expensive recursive glob from running on every collector tick (~1Hz).
+
+    Returns:
+        tuple: (file_count, latest_mtime) — same format as uncached version.
+    """
+    global _FINGERPRINT_CACHE
+    now = time.time()
+
+    # If cache is still valid, return cached value
+    if _FINGERPRINT_CACHE["value"] is not None and now < _FINGERPRINT_CACHE["expires"]:
+        return _FINGERPRINT_CACHE["value"]
+
+    # Cache expired or uninitialized; recompute
+    value = _transcripts_fingerprint_uncached()
+    _FINGERPRINT_CACHE["value"] = value
+    _FINGERPRINT_CACHE["expires"] = now + _FINGERPRINT_CACHE_TTL
+    return value