@matt82198/aesop 0.2.0 → 0.3.1

package/tools/secret_scan.py

@@ -123,12 +123,41 @@ FATAL_RULES = {
     "connection_string",
 }
 
+# USER-APPROVED EXEMPTION (2026-07-22): redaction-pattern source files.
+# A regex whose PURPOSE is redacting connection strings necessarily contains a
+# connection-string-SHAPED pattern in its source text. connection_string
+# findings in these exact repo-relative paths are downgraded to non-fatal
+# ALLOWED-REDACTION-SOURCE (still reported, never silent). Every other rule
+# stays fully fatal in these files; connection_string stays fatal everywhere
+# else, including pragma'd files. Additions to this set are a security-gate
+# change and require explicit user approval.
+# Residual risk (accepted): matching is by trailing path components, so a
+# nested copy at */bench/sample_transcripts_judgment.py inherits the
+# exemption for this ONE rule; all other rules still apply there.
+REDACTION_SOURCE_FILES = {"bench/sample_transcripts_judgment.py"}
+
+
+def _is_redaction_source(scan_path):
+    """True when scan_path's trailing components match an exempted file."""
+    if not scan_path:
+        return False
+    parts = str(scan_path).replace("\\", "/").split("/")
+    for entry in REDACTION_SOURCE_FILES:
+        entry_parts = entry.split("/")
+        if parts[-len(entry_parts):] == entry_parts:
+            return True
+    return False
+
 
-def _classify_finding(rule_name, has_file_pragma):
+def _classify_finding(rule_name, has_file_pragma, scan_path=None):
     """Shared fatal/softened decision, used by every scan_* variant (disk-file,
     blob, large-file-chunked) so the pragma contract can't drift between them.
-    FATAL_RULES are always fatal. SOFTENED_BY_PRAGMA rules are fatal unless the
-    pragma is present. Everything else (e.g. env_assignment) is always fatal."""
+    FATAL_RULES are always fatal — except the single user-approved
+    connection_string downgrade for REDACTION_SOURCE_FILES (see above).
+    SOFTENED_BY_PRAGMA rules are fatal unless the pragma is present.
+    Everything else (e.g. env_assignment) is always fatal."""
+    if rule_name == "connection_string" and _is_redaction_source(scan_path):
+        return False
     if rule_name in FATAL_RULES:
         return True
     if rule_name in SOFTENED_BY_PRAGMA:
@@ -325,7 +354,7 @@ def scan_file(filepath):
                                 if is_placeholder(match_str):
                                     continue
 
-                                is_fatal = _classify_finding(rule_name, has_file_pragma)
+                                is_fatal = _classify_finding(rule_name, has_file_pragma, filepath)
                                 findings.append((line_num, rule_name, match_str, is_fatal))
 
         else:
@@ -345,7 +374,7 @@ def scan_file(filepath):
                             if is_placeholder(match_str):
                                 continue
 
-                            is_fatal = _classify_finding(rule_name, has_file_pragma)
+                            is_fatal = _classify_finding(rule_name, has_file_pragma, filepath)
                             findings.append((line_num, rule_name, match_str, is_fatal))
 
     except (OSError, PermissionError, FileNotFoundError, IOError) as e:
@@ -438,7 +467,7 @@ def scan_blob(label, content_bytes):
                 match_str = match.group(0)
                 if is_placeholder(match_str):
                     continue
-                is_fatal = _classify_finding(rule_name, has_file_pragma)
+                is_fatal = _classify_finding(rule_name, has_file_pragma, label)
                 findings.append((line_num, rule_name, match_str, is_fatal))
 
     return findings
@@ -500,6 +529,7 @@ def get_staged_files(repo_path):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
             timeout=10,
         )
     except Exception as e:
@@ -527,6 +557,7 @@ def get_range_files(repo_path, commit_range):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
             timeout=10,
         )
     except Exception as e:
@@ -555,6 +586,8 @@ def get_history_files(repo_path):
             cwd=repo_path,
             capture_output=True,
             text=True,
+            encoding='utf-8',
+            errors='replace',
             timeout=60,
         )
     except Exception as e:
@@ -602,14 +635,14 @@ def _git_committable_set(dir_path):
     try:
         top = subprocess.run(
             ["git", "rev-parse", "--show-toplevel"],
-            cwd=str(dir_path), capture_output=True, text=True, timeout=10,
+            cwd=str(dir_path), capture_output=True, text=True, encoding='utf-8', timeout=10,
         )
         if top.returncode != 0:
             return None
         toplevel = Path(top.stdout.strip())
         listing = subprocess.run(
             ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"],
-            cwd=str(toplevel), capture_output=True, text=True, timeout=60,
+            cwd=str(toplevel), capture_output=True, text=True, encoding='utf-8', timeout=60,
         )
         if listing.returncode != 0:
             return None
@@ -650,8 +683,11 @@ def scan_paths(paths):
     return [p for p in files_to_scan if p.is_file()]
 
 
-def scan_content(content):
-    """Scan raw content for secrets (used by history scanning)."""
+def scan_content(content, scan_path=None):
+    """Scan raw content for secrets (used by history scanning). scan_path
+    (the blob's repo-relative path, when known) feeds the REDACTION_SOURCE_FILES
+    exemption so committed redaction-regex source does not turn every future
+    --history / prepublish scan permanently red."""
     findings = []
     for line_num, line in enumerate(content.split("\n"), start=1):
         for rule_name, (pattern, flags) in PATTERNS.items():
@@ -660,7 +696,8 @@ def scan_content(content):
                 match_str = match.group(0)
                 if is_placeholder(match_str):
                     continue
-                findings.append((line_num, rule_name, match_str, True))
+                is_fatal = _classify_finding(rule_name, False, scan_path)
+                findings.append((line_num, rule_name, match_str, is_fatal))
     return findings
 
 
@@ -703,6 +740,7 @@ def main():
     all_findings = []
     fatal_findings = []
     allowed_doc_count = 0
+    allowed_redaction_count = 0
     file_count = 0
 
     if args.staged:
@@ -731,7 +769,10 @@ def main():
                 if is_fatal:
                     fatal_findings.append((label, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     elif args.range:
         # Scan the COMMITTED blob at the TIP of the range for each changed
@@ -761,7 +802,10 @@ def main():
                 if is_fatal:
                     fatal_findings.append((label, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     elif args.history:
         # History scanning mode: unaffected by the blob-scan fix above, since
@@ -775,7 +819,7 @@ def main():
 
         file_count = len(set(f for f, _ in history_files))
         for filepath, content in history_files:
-            findings = scan_content(content)
+            findings = scan_content(content, scan_path=filepath)
             for line_num, rule, match_str, is_fatal in findings:
                 all_findings.append((filepath, line_num, rule, match_str, is_fatal))
                 if is_fatal:
@@ -796,7 +840,10 @@ def main():
                 if is_fatal:
                     fatal_findings.append((filepath, line_num, rule, match_str))
                 else:
-                    allowed_doc_count += 1
+                    if rule == "connection_string":
+                        allowed_redaction_count += 1
+                    else:
+                        allowed_doc_count += 1
 
     # Output findings
     for filepath, line_num, rule, match_str, is_fatal in all_findings:
@@ -804,18 +851,26 @@ def main():
         if is_fatal:
             print(f"HIGH {filepath}:{line_num} {rule} ({masked})")
         else:
-            print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
+            if rule == "connection_string":
+                print(f"ALLOWED-REDACTION-SOURCE {filepath}:{line_num} {rule} ({masked})")
+            else:
+                print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
 
     # Summary and exit
     if len(fatal_findings) == 0:
-        if allowed_doc_count == 0:
+        if allowed_doc_count == 0 and allowed_redaction_count == 0:
             if args.history:
                 print(f"CLEAN: git history scanned")
             else:
                 print(f"CLEAN: {file_count} files scanned")
         else:
             pragma_file_count = len(set(f for f, _, _, _, is_fatal in all_findings if not is_fatal))
-            print(f"CLEAN: scanned ({allowed_doc_count} allowed-doc findings in {pragma_file_count} pragma files)")
+            parts = []
+            if allowed_doc_count:
+                parts.append(f"{allowed_doc_count} allowed-doc findings in {pragma_file_count} pragma files")
+            if allowed_redaction_count:
+                parts.append(f"{allowed_redaction_count} allowed-redaction-source findings")
+            print(f"CLEAN: scanned ({'; '.join(parts)})")
         sys.exit(0)
     else:
         print(f"FOUND: {len(fatal_findings)} secret(s)")

package/tools/stall_check.py

@@ -4,6 +4,7 @@ Automated silent-hang detection for agent transcripts.
 
 Usage:
   stall_check.py [--transcripts-root DIR] [--threshold-seconds SEC] [--json] [--exit-nonzero-on-stall]
+                 [--active-from DIR] [--emit-recovery] [--recovery-dir DIR]
 
 Options:
   --transcripts-root DIR       Root directory to scan for agent-*.jsonl transcripts.
@@ -12,11 +13,24 @@ Options:
                                Transcripts older than this are flagged as stalled.
   --json                       Output as JSON list of {agent_id, age_seconds, stalled, last_mtime}.
   --exit-nonzero-on-stall      Exit 1 if any agent is detected as stalled; default exit 0 always.
+  --active-from DIR            Optional. When set, an agent is ACTIVE only if a matching task/status
+                               file exists in DIR (named <agent_id>.task or <agent_id>.status).
+                               STALLED = stale mtime AND active. Default behavior (no flag):
+                               STALLED = stale mtime only.
+  --emit-recovery              When set, emit recovery advisory JSON for each STALLED agent.
+                               Advisory includes: agent, verdict, age_s, suggested_action (list).
+                               Output as JSON blocks (one per STALLED agent).
+  --recovery-dir DIR           Optional; only used with --emit-recovery. When set, write one
+                               recovery-<agent>.json file per STALLED agent to DIR (idempotent).
+                               No files written when none stalled.
 
 Behavior:
   - Walks transcripts-root for files matching agent-*.jsonl.
   - For each file, computes age = now - file mtime (seconds).
-  - Reports agents as stalled if age > threshold-seconds.
+  - Without --active-from: Reports agents as stalled if age > threshold-seconds (legacy).
+  - With --active-from: Reports agents as stalled if age > threshold-seconds AND active file exists.
+  - With --emit-recovery: Emits JSON advisory for each STALLED agent (to stdout).
+  - With --recovery-dir: Additionally writes recovery-<agent>.json files (idempotent, overwrite).
   - Default output: human-readable table; add --json for structured output.
   - Gracefully reports "no transcripts found" if root is missing or empty.
   - Exit code: 0 always (unless --exit-nonzero-on-stall specified and stalls detected).
@@ -27,9 +41,50 @@ import os
 import time
 import json
 import argparse
+import re
 from pathlib import Path
 
 
+def sanitize_agent_id(agent_id, filename):
+    """Validate and sanitize agent ID to prevent path traversal (CWE-22).
+
+    Args:
+        agent_id: Extracted agent ID from filename
+        filename: Original filename for warning message
+
+    Returns:
+        Sanitized agent_id if valid, None if invalid (caller should skip entry)
+    """
+    # Allowlist: only alphanumeric, underscore, dash
+    if not re.match(r"^[A-Za-z0-9_-]+$", agent_id):
+        # Log warning with filename only, never echo raw agent_id into a path
+        sys.stderr.write(f"Warning: Skipping transcript with invalid agent ID in file: {filename}\n")
+        return None
+    return agent_id
+
+
+def verify_path_containment(resolved_path, base_dir):
+    """Verify that a path is contained within a base directory (defense-in-depth).
+
+    Args:
+        resolved_path: Resolved absolute path to verify
+        base_dir: Base directory that path must be contained in
+
+    Returns:
+        True if path is safely contained, False if escape attempted
+    """
+    # Resolve BOTH sides: an unresolved 8.3 short-form candidate compared
+    # against a resolved long-form base raises ValueError and reads as an
+    # escape (windows-runner regression: stale+active reported ok).
+    resolved_base = Path(base_dir).resolve()
+    try:
+        Path(resolved_path).resolve().relative_to(resolved_base)
+        return True
+    except ValueError:
+        # Path is outside base_dir
+        return False
+
+
 def get_transcripts_root():
     """Resolve transcripts root from env var or default to ~/.claude/projects."""
     if os.environ.get("AESOP_TRANSCRIPTS_ROOT"):
@@ -38,10 +93,44 @@ def get_transcripts_root():
     return Path.home() / ".claude" / "projects"
 
 
-def scan_transcripts(transcripts_root, threshold_seconds):
+def is_agent_active(agent_id, active_from_dir):
+    """Check if an agent is active by looking for task/status files.
+
+    Args:
+        agent_id: Agent identifier (e.g., 'abc123') - must be pre-sanitized
+        active_from_dir: Directory to scan for <agent_id>.task or <agent_id>.status files
+
+    Returns:
+        True if any matching task/status file exists, False otherwise.
+    """
+    if not active_from_dir:
+        return None  # Flag not provided
+
+    active_from_path = Path(active_from_dir)
+    if not active_from_path.exists():
+        return False
+
+    # Check for <agent_id>.task or <agent_id>.status files with containment verification
+    for pattern in [f"{agent_id}.task", f"{agent_id}.status"]:
+        candidate_path = (active_from_path / pattern)
+        # Defense-in-depth: verify path is contained in active_from_path
+        if verify_path_containment(candidate_path, active_from_path):
+            if candidate_path.exists():
+                return True
+
+    return False
+
+
+def scan_transcripts(transcripts_root, threshold_seconds, active_from_dir=None):
     """Scan transcripts root for agent-*.jsonl files and compute staleness.
 
-    Returns: list of dicts {agent, transcript, mtime_age_s, verdict, suggested_action, last_mtime (ISO)}
+    Args:
+        transcripts_root: Root directory to scan
+        threshold_seconds: Staleness threshold in seconds
+        active_from_dir: Optional. When set, agent is ACTIVE only if task/status file exists.
+                        STALLED = stale mtime AND active. Default (None): STALLED = stale mtime only.
+
+    Returns: list of dicts {agent, transcript, mtime_age_s, verdict, suggested_action, last_mtime (ISO), active}
     """
     transcripts_root = Path(transcripts_root)
 
@@ -66,16 +155,35 @@ def scan_transcripts(transcripts_root, threshold_seconds):
         # Extract agent_id from filename (e.g., agent-abc123.jsonl -> abc123)
         agent_id = jsonl_file.stem.replace("agent-", "")
 
+        # Sanitize agent_id to prevent path traversal (CWE-22)
+        agent_id = sanitize_agent_id(agent_id, str(jsonl_file))
+        if agent_id is None:
+            # Invalid agent_id, skip this entry
+            continue
+
+        # Check active status if flag is provided
+        active = is_agent_active(agent_id, active_from_dir)
+
         # Determine verdict
         if age_seconds <= STALE_THRESHOLD:
             verdict = "ok"
             suggested_action = None
         elif age_seconds <= DEAD_THRESHOLD:
-            verdict = "stale"
-            suggested_action = "monitor for progress or investigate why transcript is stalled"
+            # If active_from_dir is set, only stale if also active
+            if active_from_dir is not None and not active:
+                verdict = "ok"
+                suggested_action = None
+            else:
+                verdict = "stale"
+                suggested_action = "monitor for progress or investigate why transcript is stalled"
         else:
-            verdict = "dead"
-            suggested_action = "investigate immediately; agent may be hung or crashed"
+            # If active_from_dir is set, only dead if also active
+            if active_from_dir is not None and not active:
+                verdict = "ok"
+                suggested_action = None
+            else:
+                verdict = "dead"
+                suggested_action = "investigate immediately; agent may be hung or crashed"
 
         results.append({
             "agent": agent_id,
@@ -84,11 +192,107 @@ def scan_transcripts(transcripts_root, threshold_seconds):
             "verdict": verdict,
             "suggested_action": suggested_action,
             "last_mtime": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(mtime)),
+            "active": active if active_from_dir is not None else None,
         })
 
     return results
 
 
+def emit_recovery_advisories(results):
+    """Emit recovery advisory JSON blocks for stalled agents.
+
+    Yields JSON strings (one per stalled agent) with advisory structure:
+    {
+        "agent": <agent_id>,
+        "verdict": <verdict>,
+        "age_s": <seconds>,
+        "suggested_action": [<ordered list of actions>]
+    }
+    """
+    if not results:
+        return
+
+    stalled_entries = [r for r in results if r["verdict"] in ("stale", "dead")]
+
+    for entry in stalled_entries:
+        # Build ordered list of suggested actions
+        actions = []
+
+        if entry["verdict"] == "stale":
+            actions.append("SendMessage resume with scope recap")
+            actions.append("TaskStop + relaunch from journal")
+            actions.append("inspect transcript tail")
+        elif entry["verdict"] == "dead":
+            actions.append("TaskStop immediately")
+            actions.append("inspect transcript for crash/error")
+            actions.append("relaunch from last known-good checkpoint")
+
+        advisory = {
+            "agent": entry["agent"],
+            "verdict": entry["verdict"],
+            "age_s": entry["mtime_age_s"],
+            "suggested_action": actions,
+        }
+
+        yield json.dumps(advisory)
+
+
+def write_recovery_files(results, recovery_dir):
+    """Write recovery-<agent>.json files for each stalled agent (idempotent).
+
+    Args:
+        results: List of scan results
+        recovery_dir: Directory to write recovery files to
+
+    Returns:
+        Count of files written
+    """
+    if not results or not recovery_dir:
+        return 0
+
+    recovery_path = Path(recovery_dir)
+    recovery_path.mkdir(parents=True, exist_ok=True)
+
+    files_written = 0
+    stalled_entries = [r for r in results if r["verdict"] in ("stale", "dead")]
+
+    for entry in stalled_entries:
+        # Build ordered list of suggested actions
+        actions = []
+
+        if entry["verdict"] == "stale":
+            actions.append("SendMessage resume with scope recap")
+            actions.append("TaskStop + relaunch from journal")
+            actions.append("inspect transcript tail")
+        elif entry["verdict"] == "dead":
+            actions.append("TaskStop immediately")
+            actions.append("inspect transcript for crash/error")
+            actions.append("relaunch from last known-good checkpoint")
+
+        advisory = {
+            "agent": entry["agent"],
+            "verdict": entry["verdict"],
+            "age_s": entry["mtime_age_s"],
+            "suggested_action": actions,
+        }
+
+        recovery_file = recovery_path / f"recovery-{entry['agent']}.json"
+
+        # Defense-in-depth: verify recovery_file is contained in recovery_path
+        if not verify_path_containment(recovery_file, recovery_path):
+            sys.stderr.write(f"Warning: Recovery file path escape detected, skipping: {entry['agent']}\n")
+            continue
+
+        try:
+            recovery_file.write_text(json.dumps(advisory, indent=2), encoding='utf-8')
+            files_written += 1
+        except Exception as e:
+            # Fail-open: log error but continue
+            sys.stderr.write(f"Warning: Failed to write {recovery_file}: {e}\n")
+
+    return files_written
+
+
 def print_human_table(results):
     """Print results as a human-readable table."""
     if not results:
@@ -140,6 +344,21 @@ def main():
         action="store_true",
         help="Exit 1 if any agent is stalled (default: always exit 0)",
     )
+    parser.add_argument(
+        "--active-from",
+        default=None,
+        help="Optional. Directory to scan for task/status files. Agent is ACTIVE only if file exists.",
+    )
+    parser.add_argument(
+        "--emit-recovery",
+        action="store_true",
+        help="Emit recovery advisory JSON blocks to stdout for each STALLED agent",
+    )
+    parser.add_argument(
+        "--recovery-dir",
+        default=None,
+        help="Optional; only used with --emit-recovery. Write recovery-<agent>.json files to this directory",
+    )
 
     args = parser.parse_args()
 
@@ -147,16 +366,28 @@ def main():
     transcripts_root = args.transcripts_root if args.transcripts_root else get_transcripts_root()
 
     # Scan transcripts
-    results = scan_transcripts(transcripts_root, args.threshold_seconds)
-
-    # Output
-    if args.json:
-        print_json_output(results)
-    else:
-        if results is None:
-            print("no transcripts found")
+    results = scan_transcripts(transcripts_root, args.threshold_seconds, args.active_from)
+
+    # Emit recovery advisories if requested
+    if args.emit_recovery and results:
+        for advisory_json in emit_recovery_advisories(results):
+            print(advisory_json)
+
+    # Write recovery files if requested
+    if args.emit_recovery and args.recovery_dir and results:
+        write_recovery_files(results, args.recovery_dir)
+
+    # Output (human-readable or JSON)
+    # Note: if --emit-recovery was used, recovery advisories already printed above
+    if not args.emit_recovery:
+        # Only print human-readable/JSON if we're not emitting recovery advisories
+        if args.json:
+            print_json_output(results)
         else:
-            print_human_table(results)
+            if results is None:
+                print("no transcripts found")
+            else:
+                print_human_table(results)
 
     # Determine exit code
     exit_code = 0