@matt82198/aesop 0.1.0-beta.1

package/tools/secret_scan.py

@@ -0,0 +1,363 @@
+#!/usr/bin/env python3
+# secretscan: allow-pattern-docs
+"""
+secret_scan.py — Pre-push secret/credential detection gate.
+
+Modes:
+  secret_scan.py --staged [--repo PATH]  Scan git staged files (default repo=cwd)
+  secret_scan.py --history [--repo PATH] Scan all blobs in git history
+  secret_scan.py PATH [PATH...]          Scan files/dirs directly (recurse dirs)
+
+Exit codes: 0=clean, 1=findings, 2=usage error
+Output: one line per finding or summary (never prints full secrets)
+
+Pragma escape hatch (defensive only):
+  If the literal string 'secretscan: allow-pattern-docs' appears in a file's first 10 lines
+  (any comment syntax: #, //, <!-- -->), pattern-based findings in that file are reported
+  as ALLOWED-DOC and do NOT cause exit 1. Filename-based findings (credential files like
+  .credentials.json, *.pem, id_rsa*, *.p12) stay fatal regardless. Use only for deliberate
+  pattern documentation; the pragma appears in git diffs and is a reviewable act.
+
+NOTE: Public repo version has NO vault allowlist. All secrets block unless permitted by pragma.
+"""
+
+import argparse
+import os
+import re
+import sys
+import subprocess
+from pathlib import Path
+
+
+# Regex patterns for secret detection (case-insensitive where sensible)
+PATTERNS = {
+    "pem_private_key": (r"-----BEGIN .* PRIVATE KEY-----", re.IGNORECASE),
+    "aws_access_key": (r"AKIA[0-9A-Z]{16}", 0),
+    "aws_secret_pattern": (
+        r"aws[_-]?secret[_-]?access[_-]?key\s*[:=]\s*[^\s\$\<\{]",
+        re.IGNORECASE,
+    ),
+    "github_token": (
+        r"(ghp_|gho_|ghu_|ghs_|ghr_|github_pat_)[A-Za-z0-9_]{20,}",
+        0,
+    ),
+    "slack_token": (r"xox[baprs]-[A-Za-z0-9-]{10,}", 0),
+    "openai_anthropic_key": (r"sk-[A-Za-z0-9_\-]{20,}", 0),
+    "generic_secret_assignment": (
+        r"(password|passwd|secret|api[_-]?key|token|authorization)\s*[:=]\s*[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']",
+        re.IGNORECASE,
+    ),
+    "connection_string": (r"://[^:]+:[^@/\s]+@(?!localhost(?:[:/]|$)|127\.0\.0\.1(?:[:/]|$)|example\.com(?:[:/]|$))[^\s]+", 0),
+    "env_access": (
+        r"(?i:(?:os\.getenv|os\.environ|System\.getenv|process\.env)\s*[\[\(][\"']?[A-Z_]*(?:password|secret|api[_-]?key|token|auth|key)[A-Z_0-9]*[\"']?[\)\]])",
+        0,
+    ),
+    "env_assignment": (
+        r"(?i:[A-Z_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API[_-]?KEY|PRIVATE|CREDENTIAL|AUTH)[A-Z_0-9]*\s*=\s*)(?!.*(?:xxx|changeme|your-|example))[^\s].{8,}",
+        0,
+    ),
+}
+
+# File patterns that look like credentials (case-insensitive)
+CREDENTIAL_FILENAMES = [
+    r"\.credentials.*",
+    r".*token.*",
+    r".*\.pem$",
+    r".*\.p12$",
+    r"id_rsa.*",
+]
+
+# Placeholders that don't count as secrets
+PLACEHOLDERS = {"xxx", "changeme", "your-key-here", "example", "test", "demo"}
+
+
+def has_pragma(filepath):
+    """Check if file has 'secretscan: allow-pattern-docs' pragma in first 10 lines."""
+    try:
+        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+            for i, line in enumerate(f):
+                if i >= 10:  # Only check first 10 lines
+                    break
+                if "secretscan: allow-pattern-docs" in line:
+                    return True
+    except Exception:
+        pass
+    return False
+
+
+def is_binary_file(filepath):
+    """Check if file is binary (contains null bytes)."""
+    try:
+        with open(filepath, "rb") as f:
+            return b"\x00" in f.read(8192)
+    except Exception:
+        return True
+
+
+def should_skip_file(filepath):
+    """Check if file should be skipped (binary, too large, in .git/)."""
+    if ".git" in filepath.parts or ".git" in str(filepath):
+        return True
+
+    try:
+        stat = filepath.stat()
+        if stat.st_size > 1024 * 1024:  # >1MB
+            return True
+        if is_binary_file(filepath):
+            return True
+    except Exception:
+        return True
+
+    return False
+
+
+def is_placeholder(value):
+    """Check if a string is a placeholder value (word-boundary aware)."""
+    lower_val = value.lower()
+    for p in PLACEHOLDERS:
+        if re.search(r'\b' + re.escape(p) + r'\b(?!\.)', lower_val):
+            return True
+    # Also check for template syntax
+    return bool(re.search(r"<[^>]*>|\$\{[^}]*\}", value))
+
+
+def mask_secret(secret_str):
+    """Mask secret: show first 4 chars + *** (or *** if <4 chars)."""
+    if len(secret_str) <= 4:
+        return "***"
+    return f"{secret_str[:4]}***"
+
+
+def is_env_file(filepath):
+    r"""Check if file is .env-like (basename matches ^\.env(\..*)?$ or *.env or *.properties)."""
+    name = filepath.name.lower()
+    return bool(
+        re.match(r"^\.env(\..*)?$", name) or
+        name.endswith(".env") or
+        name.endswith(".properties")
+    )
+
+
+def scan_file(filepath):
+    """
+    Scan a single file for secrets.
+    Returns list of (line_num, rule, match_str, is_fatal).
+    is_fatal=True for credential filenames; is_fatal=False if pragma present and rule-based.
+    """
+    findings = []
+
+    if should_skip_file(filepath):
+        return findings
+
+    # Check for pragma (applies only to rule-based findings, not filename findings)
+    has_file_pragma = has_pragma(filepath)
+
+    # Check if filename matches credential patterns (always fatal, pragma does NOT apply)
+    filename = filepath.name.lower()
+    for pattern in CREDENTIAL_FILENAMES:
+        if re.match(pattern, filename, re.IGNORECASE):
+            findings.append(
+                (0, "credential_filename", f"File name matches credential pattern: {filepath.name}", True)
+            )
+            break
+
+    try:
+        with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
+            for line_num, line in enumerate(f, start=1):
+                for rule_name, (pattern, flags) in PATTERNS.items():
+                    # Skip env_assignment rule if not an .env-like file
+                    if rule_name == "env_assignment" and not is_env_file(filepath):
+                        continue
+
+                    matches = re.finditer(pattern, line, flags)
+                    for match in matches:
+                        match_str = match.group(0)
+
+                        # Skip if it's a placeholder
+                        if is_placeholder(match_str):
+                            continue
+
+                        # Mark as non-fatal if pragma present (rule-based only)
+                        is_fatal = not has_file_pragma
+                        findings.append((line_num, rule_name, match_str, is_fatal))
+
+    except Exception:
+        pass
+
+    return findings
+
+
+def get_staged_files(repo_path):
+    """Get list of staged files from git repo."""
+    try:
+        result = subprocess.run(
+            ["git", "diff", "--cached", "--name-only"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        )
+        if result.returncode != 0:
+            return []
+        return [
+            Path(repo_path) / f
+            for f in result.stdout.strip().split("\n")
+            if f.strip()
+        ]
+    except Exception:
+        return []
+
+
+def get_history_files(repo_path):
+    """Get all file contents from git history via git log -p."""
+    files_content = []
+    try:
+        # Use git log -p to get full diff history
+        result = subprocess.run(
+            ["git", "log", "--all", "-p", "--reverse"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            timeout=60,
+        )
+        if result.returncode == 0:
+            # Parse the git log output: each file appears as +++ b/path/to/file followed by its content
+            current_file = None
+            current_content = []
+            for line in result.stdout.split("\n"):
+                if line.startswith("+++ b/"):
+                    if current_file and current_content:
+                        files_content.append((current_file, "\n".join(current_content)))
+                    current_file = line[6:]  # Remove "+++ b/"
+                    current_content = []
+                elif current_file and line.startswith("+") and not line.startswith("+++"):
+                    # Content line (added), strip the leading +
+                    current_content.append(line[1:])
+                elif current_file and line.startswith(" "):
+                    # Context line, keep it as-is (strip leading space)
+                    current_content.append(line[1:])
+            if current_file and current_content:
+                files_content.append((current_file, "\n".join(current_content)))
+    except Exception:
+        pass
+    return files_content
+
+
+def scan_paths(paths):
+    """Recursively scan paths (files and directories)."""
+    files_to_scan = []
+    for path_str in paths:
+        path = Path(path_str).resolve()
+        if path.is_file():
+            files_to_scan.append(path)
+        elif path.is_dir():
+            files_to_scan.extend(path.rglob("*"))
+    return [p for p in files_to_scan if p.is_file()]
+
+
+def scan_content(content):
+    """Scan raw content for secrets (used by history scanning)."""
+    findings = []
+    for line_num, line in enumerate(content.split("\n"), start=1):
+        for rule_name, (pattern, flags) in PATTERNS.items():
+            matches = re.finditer(pattern, line, flags)
+            for match in matches:
+                match_str = match.group(0)
+                if is_placeholder(match_str):
+                    continue
+                findings.append((line_num, rule_name, match_str, True))
+    return findings
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--staged",
+        action="store_true",
+        help="Scan git staged files (requires --repo or uses cwd)",
+    )
+    parser.add_argument(
+        "--history",
+        action="store_true",
+        help="Scan all blobs in git history (requires --repo or uses cwd)",
+    )
+    parser.add_argument(
+        "--repo",
+        default=os.getcwd(),
+        help="Git repo path (default: current directory)",
+    )
+    parser.add_argument(
+        "paths", nargs="*", help="Paths to scan directly (files or directories)"
+    )
+
+    args = parser.parse_args()
+
+    # Validate usage
+    if sum([args.staged, args.history, bool(args.paths)]) != 1:
+        print("ERROR: Use exactly one of --staged, --history, or path arguments", file=sys.stderr)
+        sys.exit(2)
+
+    # Collect files to scan
+    if args.staged:
+        files = get_staged_files(args.repo)
+    elif args.history:
+        # History mode: scan all files from git log
+        history_files = get_history_files(args.repo)
+        files = []  # We'll handle history differently
+    else:
+        files = scan_paths(args.paths)
+
+    # Scan files
+    all_findings = []
+    fatal_findings = []
+    allowed_doc_count = 0
+
+    if args.history:
+        # History scanning mode
+        for filepath, content in history_files:
+            findings = scan_content(content)
+            for line_num, rule, match_str, is_fatal in findings:
+                all_findings.append((filepath, line_num, rule, match_str, is_fatal))
+                if is_fatal:
+                    fatal_findings.append((filepath, line_num, rule, match_str))
+    else:
+        # Regular file scanning mode
+        for filepath in files:
+            findings = scan_file(filepath)
+            for line_num, rule, match_str, is_fatal in findings:
+                all_findings.append((filepath, line_num, rule, match_str, is_fatal))
+                if is_fatal:
+                    fatal_findings.append((filepath, line_num, rule, match_str))
+                else:
+                    allowed_doc_count += 1
+
+    # Output findings
+    for filepath, line_num, rule, match_str, is_fatal in all_findings:
+        masked = mask_secret(match_str)
+        if is_fatal:
+            print(f"HIGH {filepath}:{line_num} {rule} ({masked})")
+        else:
+            print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
+
+    # Summary and exit
+    file_count = len(files) if not args.history else len(set(f for f, _, _, _, _ in all_findings))
+
+    if len(fatal_findings) == 0:
+        if allowed_doc_count == 0:
+            if args.history:
+                print(f"CLEAN: git history scanned")
+            else:
+                print(f"CLEAN: {file_count} files scanned")
+        else:
+            pragma_file_count = len(set(f for f, _, _, _, is_fatal in all_findings if not is_fatal))
+            print(f"CLEAN: scanned ({allowed_doc_count} allowed-doc findings in {pragma_file_count} pragma files)")
+        sys.exit(0)
+    else:
+        print(f"FOUND: {len(fatal_findings)} secret(s)")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

package/ui/README.md

@@ -0,0 +1,141 @@
+# Aesop Web Dashboard
+
+A lightweight, self-contained web dashboard for real-time fleet monitoring. Replaces the bash TUI with a modern, responsive HTML interface that displays heartbeats, active agents, security alerts, and inbox status.
+
+## Launch
+
+```bash
+export AESOP_ROOT=$HOME/aesop
+python $AESOP_ROOT/ui/serve.py
+```
+
+Prints `Dashboard: http://localhost:8770` and serves until Ctrl-C.
+
+### Configuration
+
+The dashboard reads configuration from `aesop.config.json` if present. Key paths:
+
+- **`AESOP_ROOT`** (env var): Path to aesop installation (default: `$HOME/aesop`)
+- **`PORT`** (env var): Dashboard port (default: `8770`)
+- **`AESOP_TRANSCRIPTS_ROOT`** (env var): Path to Claude session transcripts (default: `~/.claude/projects`)
+
+Example with custom paths:
+
+```bash
+export AESOP_ROOT=/opt/aesop
+export AESOP_TRANSCRIPTS_ROOT=/home/user/.claude/projects
+export PORT=9000
+python $AESOP_ROOT/ui/serve.py
+```
+
+## Features
+
+### Header Status
+- **Watchdog Status**: Daemon heartbeat age + ALIVE/STALE (300s threshold)
+- **Monitor Status**: Orchestration monitor liveness + age (3600s threshold)
+- **Security Alerts**: Count of unreviewed alerts
+- **Running Agents**: Number of active subagents (<2 min old)
+
+### Panels
+
+1. **Inbox** (top): Text input → Send button appends to `state/ui-inbox.md`. Orchestrator reads each turn.
+2. **Fleet Agents**: Currently-running subagents with status and hint text (refreshed every 3s)
+3. **Repos Status**: Repository sync status from `state/.watchdog-repos.json`
+4. **Recent Events**: Last 8 lines from `state/FLEET-BACKUP.log`
+5. **Security Alerts**: Unreviewed lines from `scan/SECURITY-ALERTS.log` (skips NOTE:/RESOLVED-FP)
+6. **Main-Thread Prompts**: Last ~12 messages from newest session JSONL in transcripts root
+
+### Agent Details
+
+Click any agent row in the **Fleet Agents** panel to expand and view full dispatch details:
+
+- **Dispatcher**: Who launched this agent (main thread or parent agent)
+- **Model**: LLM model used for the agent
+- **Messages**: Total NDJSON message count in transcript
+- **Prompt**: Full dispatch prompt (expandable container with scroll)
+
+#### API: `/agent?id=<agent_id>`
+
+GET endpoint returns JSON with agent metadata:
+
+```json
+{
+  "id": "a77b995bcdb9",
+  "dispatch_prompt": "full prompt text here...",
+  "dispatcher": "main thread",
+  "model": "haiku",
+  "message_count": 42,
+  "first_seen": 1720700000,
+  "last_activity": 1720700300
+}
+```
+
+**Note**: Agent IDs are prefix-matched via glob (truncated IDs in the dashboard resolve to full transcripts on disk).
+
+### Inbox Contract
+
+POST `/submit` appends timestamped line to `state/ui-inbox.md`:
+
+```markdown
+# UI Inbox — orchestrator reads each turn / on /power
+
+- [2026-07-11T14:23:45.123456] user's task here
+- [2026-07-11T14:24:12.654321] another task
+```
+
+## Requirements
+
+- **Python 3.10+**: stdlib-only (no external dependencies)
+- **Node.js v18+** (optional): required for agent detection via `dash/dash-extra.mjs`
+
+## Robustness
+
+- Every data source is wrapped with exception handling
+- Missing or locked files → placeholder text (never 500 error)
+- `/data` endpoint always returns valid JSON
+- Auto-refresh every 3s via client-side fetch
+
+## Path Configuration
+
+If you have a non-standard setup, set paths in `aesop.config.json`:
+
+```json
+{
+  "state_root": "/path/to/state",
+  "scan_root": "/path/to/scan",
+  "transcripts_root": "/path/to/transcripts"
+}
+```
+
+Or use environment variables:
+
+```bash
+export AESOP_TRANSCRIPTS_ROOT=/my/custom/transcripts/path
+python ui/serve.py
+```
+
+## Troubleshooting
+
+### "Agents panel unavailable"
+- Ensure `node` is on PATH and `dash/dash-extra.mjs` exists
+- Install Node.js v18+
+
+### Dashboard shows no data
+- Verify `AESOP_ROOT` is set correctly: `echo $AESOP_ROOT`
+- Check that `state/` directory exists and is readable
+- Verify `aesop.config.json` is valid JSON (if present)
+
+### Can't connect to http://localhost:8770
+- Check port is not in use: `lsof -i :8770` (macOS/Linux) or `netstat -ano | findstr :8770` (Windows)
+- Try a different port: `export PORT=9000`
+
+## Integration with Claude Code
+
+To display main-thread transcripts, ensure `AESOP_TRANSCRIPTS_ROOT` points to your Claude session directory. Example:
+
+```bash
+export AESOP_TRANSCRIPTS_ROOT=$HOME/.claude/projects/my-project
+python $AESOP_ROOT/ui/serve.py
+```
+
+See the main Aesop README for full orchestration setup.