@matt82198/aesop 0.1.0 → 0.2.0

package/tools/claudemd_lint.py

@@ -0,0 +1,359 @@
+#!/usr/bin/env python3
+"""CLAUDE.md linter — dogfoods the scope-min invariant.
+
+For each */CLAUDE.md in a repo:
+1. DOC-POINTER check — every referenced path ending .md/.py/.sh/.mjs that looks like a
+   REPO file (relative, not a runtime artifact) must exist. Distinguishes real repo-doc
+   pointers from legitimate references to runtime artifacts (state/**, *heartbeat*,
+   BRIEF.md, PROPOSALS.md, BUILDLOG.md, MEMORY.md, STATE.md, OUTCOMES-LEDGER.md, tracker.json).
+2. TEST-CMD check — any `npm run <script>` cited must exist in package.json scripts.
+   Flags `pytest` if the repo uses unittest (grep package.json test:py).
+3. Optional — flags files over --max-lines (default 150).
+
+Exit: 0=clean, 1=findings. Supports --json flag.
+"""
+
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Dict, List, Tuple
+
+# Runtime artifact allowlist — these are correctly absent from the tree
+RUNTIME_ARTIFACTS = {
+    # State/control files
+    "state",  # state/ directory
+    "BRIEF.md",
+    "PROPOSALS.md",
+    "BUILDLOG.md",
+    "MEMORY.md",
+    "STATE.md",
+    "OUTCOMES-LEDGER.md",
+    "tracker.json",
+    "ACTIONS.log",
+    ".monitor-heartbeat",
+    ".signal-state.json",
+    ".HALT",
+    ".git",
+    "node_modules",
+}
+
+# Patterns that indicate a runtime artifact
+RUNTIME_PATTERNS = [
+    r"^\.\.?/state/",  # state/ directory
+    r"heartbeat",  # *heartbeat*, .monitor-heartbeat, etc.
+    r"^BRIEF\.md$",
+    r"^PROPOSALS\.md$",
+    r"^BUILDLOG\.md$",
+    r"^MEMORY\.md$",
+    r"^STATE\.md$",
+    r"^CLAUDE\.md$",
+    r"^SKILL\.md$",
+    r"^OUTCOMES-LEDGER\.md$",
+    r"^tracker\.json$",
+    r"^ACTIONS\.log$",
+    r"^\./state/",
+    r"^state/",
+    # Allow these control files in compound refs like "CLAUDE.md/STATE.md"
+    r"CLAUDE\.md(?:/|$)",
+    r"STATE\.md(?:/|$)",
+    r"SKILL\.md(?:/|$)",
+]
+
+
+def is_runtime_artifact(ref: str) -> bool:
+    """Check if a reference is a legitimate runtime artifact."""
+    for pattern in RUNTIME_PATTERNS:
+        if re.search(pattern, ref, re.IGNORECASE):
+            return True
+    return False
+
+
+def extract_path_references(text: str) -> List[str]:
+    """Extract all references to paths ending in .md/.py/.sh/.mjs.
+
+    Filters out:
+    - Example paths starting with /path/to/
+    - Environment variable references (VAR_NAME/...)
+    - Absolute paths starting with /
+    - Home directory references (~/.../...)
+    - Non-relative paths
+    - Glob patterns (*.something)
+    - File type descriptions like ".py/.mjs"
+    - Hidden directory references like .claude/ (not repo structure)
+    """
+    # Match paths: relative or starting with ./, alphanumeric, /, -, _
+    # Also match inline code references like `path/file.md`
+    # Note: intentionally NOT matching patterns like "*.test.mjs" (glob)
+    pattern = r"(?:[`'\"])?([a-zA-Z0-9_.][a-zA-Z0-9_./\-]*\.(?:md|py|sh|mjs))(?:[`'\"])?"
+    matches = re.finditer(pattern, text)
+    refs = set()
+    for match in matches:
+        ref = match.group(1)
+
+        # Filter out false positives
+        if len(ref) <= 2:
+            continue
+
+        # Skip glob patterns (starting with *)
+        if ref.startswith("*"):
+            continue
+
+        # Skip absolute paths
+        if ref.startswith("/"):
+            continue
+
+        # Skip home directory references (~/...)
+        if "~/" in ref:
+            continue
+
+        # Skip hidden directories that don't look like repo structure (./.something/...)
+        # These are typically home dir refs like .claude/, .config/, etc.
+        if ref.startswith(".") and "/" in ref:
+            # Allow only ./ for current dir refs
+            if not ref.startswith("./"):
+                continue
+
+        # Skip example paths
+        if "/path/to/" in ref or ref.startswith("path/to/"):
+            continue
+
+        # Skip env var references (ALLCAPS_NAME/...)
+        if re.match(r"^[A-Z_]+/", ref):
+            continue
+
+        # Skip file type descriptions like ".py/.mjs" (multiple dots in non-path context)
+        if ref.count(".") > 2:
+            continue
+
+        # Skip references that don't have / (not a path)
+        if "/" not in ref:
+            continue
+
+        refs.add(ref)
+
+    return sorted(refs)
+
+
+def extract_npm_scripts(text: str) -> List[str]:
+    """Extract all `npm run <script>` references."""
+    pattern = r"npm\s+run\s+([a-zA-Z0-9:_\-]+)"
+    matches = re.finditer(pattern, text)
+    scripts = set()
+    for match in matches:
+        scripts.add(match.group(1))
+    return sorted(scripts)
+
+
+def get_package_scripts(repo_root: Path) -> Dict[str, str]:
+    """Load scripts from every package.json in the repo (root + nested, e.g. ui/web).
+
+    A multi-package repo (aesop has ui/web/package.json for the frontend) means a
+    domain doc may legitimately cite a script that lives in a nested package — union
+    them so the linter doesn't false-positive on ui/CLAUDE.md's `npm run build`/`dev`.
+    """
+    scripts: Dict[str, str] = {}
+    for pkg_path in repo_root.rglob("package.json"):
+        # skip dependencies' package.json
+        if "node_modules" in pkg_path.parts:
+            continue
+        try:
+            with open(pkg_path) as f:
+                pkg = json.load(f)
+            scripts.update(pkg.get("scripts", {}))
+        except (json.JSONDecodeError, IOError):
+            continue
+    return scripts
+
+
+def check_test_cmd_match(repo_root: Path) -> Tuple[bool, str]:
+    """Check if repo uses unittest (test:py in package.json uses 'unittest').
+
+    Returns: (is_using_unittest, test_cmd_value)
+    """
+    scripts = get_package_scripts(repo_root)
+    test_py = scripts.get("test:py", "")
+    is_unittest = "unittest" in test_py
+    return is_unittest, test_py
+
+
+def lint_claudemd(
+    claudemd_path: Path,
+    repo_root: Path,
+    max_lines: int = 150,
+) -> List[Dict[str, str]]:
+    """Lint a single CLAUDE.md file.
+
+    Returns list of findings, each a dict with 'type', 'line', 'message'.
+    """
+    findings = []
+
+    try:
+        content = claudemd_path.read_text(encoding="utf-8")
+    except (IOError, UnicodeDecodeError) as e:
+        return [{
+            "type": "file-read-error",
+            "line": "0",
+            "message": f"Failed to read {claudemd_path.relative_to(repo_root)}: {e}",
+        }]
+
+    lines = content.split("\n")
+
+    # Per-file oversize allowance: ui/CLAUDE.md is the documented dense-domain
+    # exception (lossless-verified, probe-passed at ~197 lines). Mirrors the same
+    # allowance in ~/scripts/compliance_check.py so the two gates agree.
+    ALLOWED_OVERSIZE = {"ui/CLAUDE.md": 210}  # grew with the dispatch-visibility route/panel (rc.7); still lossless-verified
+    rel = str(claudemd_path.relative_to(repo_root)).replace("\\", "/")
+    effective_max = ALLOWED_OVERSIZE.get(rel, max_lines)
+
+    # Check line count
+    if len(lines) > effective_max:
+        findings.append({
+            "type": "line-count",
+            "line": str(len(lines)),
+            "message": f"{claudemd_path.relative_to(repo_root)}: "
+                       f"{len(lines)} lines exceeds max {effective_max}",
+        })
+
+    # Check if content endorses pytest but repo uses unittest
+    # Exclude false positives where pytest is mentioned in passing or explicitly excluded
+    is_unittest, _ = check_test_cmd_match(repo_root)
+    if is_unittest:
+        content_lower = content.lower()
+        # Check for pytest endorsement (not just mention)
+        pytest_mentioned = "pytest" in content_lower
+        # Check for exclusion phrases that indicate pytest is NOT used
+        pytest_excluded = any(phrase in content_lower for phrase in [
+            "not pytest",
+            "not use pytest",
+            "don't use pytest",
+            "do not use pytest",
+            "uses unittest",
+            "use unittest",
+            "unittest, not pytest",
+            "-m unittest",
+        ])
+        # Flag only if pytest is mentioned AND not explicitly excluded
+        if pytest_mentioned and not pytest_excluded:
+            findings.append({
+                "type": "pytest-vs-unittest",
+                "line": "?",
+                "message": f"{claudemd_path.relative_to(repo_root)}: "
+                           f"mentions 'pytest' but repo uses unittest (test:py)",
+            })
+
+    # DOC-POINTER check: find file references
+    path_refs = extract_path_references(content)
+
+    # Get the directory of the CLAUDE.md file for relative resolution
+    claudemd_dir = claudemd_path.parent
+
+    for ref in path_refs:
+        # Skip runtime artifacts
+        if is_runtime_artifact(ref):
+            continue
+
+        # Try to resolve relative to the CLAUDE.md file's directory first
+        target = claudemd_dir / ref
+        if not target.exists():
+            # Fall back to repo root resolution
+            target = repo_root / ref
+            if not target.exists():
+                findings.append({
+                    "type": "phantom-path",
+                    "line": "?",
+                    "message": f"{claudemd_path.relative_to(repo_root)}: "
+                               f"references non-existent '{ref}'",
+                })
+
+    # TEST-CMD check: npm run scripts
+    npm_scripts = extract_npm_scripts(content)
+    available_scripts = get_package_scripts(repo_root)
+
+    for script in npm_scripts:
+        if script not in available_scripts:
+            findings.append({
+                "type": "missing-npm-script",
+                "line": "?",
+                "message": f"{claudemd_path.relative_to(repo_root)}: "
+                           f"npm run '{script}' not in package.json scripts",
+            })
+
+    return findings
+
+
+def main():
+    """Main entry point."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Lint CLAUDE.md files for integrity"
+    )
+    parser.add_argument(
+        "--root",
+        type=Path,
+        default=Path.cwd(),
+        help="Repository root (default: cwd)",
+    )
+    parser.add_argument(
+        "--max-lines",
+        type=int,
+        default=150,
+        help="Maximum lines per CLAUDE.md (default: 150)",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Output as JSON",
+    )
+
+    args = parser.parse_args()
+    repo_root = args.root.resolve()
+
+    if not repo_root.exists():
+        print(f"Error: repo root {repo_root} does not exist", file=sys.stderr)
+        sys.exit(1)
+
+    # Find all CLAUDE.md files (recursive, with exclusions for common junk dirs)
+    # Exclude: node_modules, .git, dist, worktrees (sibling dirs), .pytest_cache, __pycache__
+    claudemd_files = []
+
+    # Use rglob to find all CLAUDE.md files at any depth
+    for claudemd_path in repo_root.rglob("CLAUDE.md"):
+        # Exclude paths in problematic directories
+        parts = claudemd_path.parts
+        if any(part in {"node_modules", ".git", "dist", ".pytest_cache", "__pycache__"} for part in parts):
+            continue
+        # Exclude worktree paths (parent directory sibling paths like ../aesop-wt-*)
+        # This is already handled by only searching within repo_root
+        claudemd_files.append(claudemd_path)
+
+    claudemd_files = sorted(set(claudemd_files))
+
+    all_findings = []
+    for claudemd_path in claudemd_files:
+        findings = lint_claudemd(claudemd_path, repo_root, args.max_lines)
+        all_findings.extend(findings)
+
+    if args.json:
+        output = {
+            "findings": all_findings,
+            "count": len(all_findings),
+            "repo_root": str(repo_root),
+        }
+        print(json.dumps(output, indent=2))
+    else:
+        if all_findings:
+            for i, finding in enumerate(all_findings, 1):
+                print(
+                    f"{i}. [{finding['type']}] {finding['message']} "
+                    f"(line {finding['line']})"
+                )
+        else:
+            print("[OK] No issues found")
+
+    sys.exit(1 if all_findings else 0)
+
+
+if __name__ == "__main__":
+    main()

package/tools/common.py

@@ -6,14 +6,26 @@ Functions:
   get_state_dir() -> Path
     Resolve state directory from AESOP_STATE_ROOT env var or default to ./state
 
+  get_state_db_path() -> Path
+    Return the canonical SQLite DB path for the event store.
+
   check_heartbeat_staleness(hb_file, threshold_s) -> (is_stale, age_s, info)
     Check if a heartbeat file is stale and return staleness, age, and descriptive info
+
+Constants:
+  STATE_DB_FILENAME: The canonical filename for the event-sourced state DB.
+    Multi-instance coordination requires all instances to point to the same file.
 """
 
 import os
 import time
 from pathlib import Path
 
+# Canonical filename for the event-sourced state database.
+# Multi-instance requires all instances (including reconcile.py, ui/collectors.py, etc.)
+# to point at the SAME shared file. Previously inconsistent (tracker_events.db vs events.db).
+STATE_DB_FILENAME = "tracker_events.db"
+
 
 def get_state_dir():
     """Resolve state directory from env var or current working directory.
@@ -28,6 +40,18 @@ def get_state_dir():
     return Path.cwd() / "state"
 
 
+def get_state_db_path():
+    """Return the canonical SQLite DB path for the event store.
+
+    Multi-instance coordination requires all orchestrators to point at the
+    SAME shared database file. This function centralizes the DB path resolution.
+
+    Returns:
+        Path: The canonical path to the state database (state/tracker_events.db).
+    """
+    return get_state_dir() / STATE_DB_FILENAME
+
+
 def check_heartbeat_staleness(hb_file, threshold_s):
     """Check if a heartbeat file is stale.
 
@@ -41,8 +65,13 @@ def check_heartbeat_staleness(hb_file, threshold_s):
           age_s (int): Age in seconds (0 if file missing/unreadable)
           info (str or None): Descriptive message if stale/missing, None if fresh
     """
-    if not hb_file.exists():
-        return True, 0, "Heartbeat file missing"
+    try:
+        if not hb_file.exists():
+            return True, 0, "Heartbeat file missing"
+    except OSError:
+        # Parent dir unreadable (permissions) — cannot verify, report stale
+        # (fail-closed, per the documented contract: unreadable => stale)
+        return True, 0, "Heartbeat file unreadable"
 
     try:
         content = hb_file.read_text(encoding="utf-8").strip()
@@ -54,7 +83,14 @@ def check_heartbeat_staleness(hb_file, threshold_s):
         return True, 0, "Heartbeat file unreadable"
 
     age_seconds = int(time.time()) - timestamp
-    age_seconds = max(0, age_seconds)  # Never negative
+
+    # Check for future-dated timestamp (clock skew beyond tolerance)
+    # More than 120s in the future is treated as stale, not clamped-to-fresh
+    if age_seconds < -120:
+        return True, 0, "Heartbeat timestamp in future (clock skew)"
+
+    # Clamp small negative ages to 0 (normal clock skew recovery)
+    age_seconds = max(0, age_seconds)
 
     if age_seconds >= threshold_s:
         return True, age_seconds, f"Heartbeat stale ({age_seconds}s >= {threshold_s}s)"

package/tools/cost_ceiling.py

@@ -125,38 +125,64 @@ def read_ledger_total_tokens(state_dir, period="wave"):
 def check(spent=None, period="wave", config=None, state_dir=None, trip=True):
     """Check spend against the configured ceiling for `period`.
 
-    Returns a dict: {"period", "ceiling", "spent", "exceeded", "tripped"}.
+    Returns a dict: {"period", "ceiling", "spent", "exceeded", "tripped", "reason"}.
+
+    Distinctions:
+    - Genuine ceiling breach (ceiling is configured, spent >= ceiling): when trip=True,
+      writes persistent .HALT sentinel via halt.halt() and sets tripped=True.
+    - Exception during spend computation (ledger read/create failure, etc.): signals
+      abort of current wave (exceeded=True) but does NOT write persistent sentinel
+      (tripped=False). This preserves fleet availability across transient I/O errors
+      (e.g., momentary file lock, disk full) while still aborting the current wave.
     """
-    if config is None:
-        config = load_config()
-    if state_dir is None:
-        state_dir = halt.resolve_state_dir(config=config)
-    state_dir = Path(state_dir)
-
-    ceiling = get_ceiling(config, period)
-
-    if spent is None:
-        spent = read_ledger_total_tokens(state_dir, period=period)
-    spent = int(spent)
-
-    exceeded = ceiling is not None and spent >= ceiling
-    tripped = False
-
-    if exceeded and trip:
-        reason = (
-            f"cost ceiling exceeded: {period} spend {spent} tokens >= "
-            f"ceiling {ceiling} tokens"
-        )
-        halt.halt(reason, state_dir=state_dir)
-        tripped = True
-
-    return {
-        "period": period,
-        "ceiling": ceiling,
-        "spent": spent,
-        "exceeded": exceeded,
-        "tripped": tripped,
-    }
+    try:
+        if config is None:
+            config = load_config()
+        if state_dir is None:
+            state_dir = halt.resolve_state_dir(config=config)
+        state_dir = Path(state_dir)
+
+        ceiling = get_ceiling(config, period)
+
+        if spent is None:
+            spent = read_ledger_total_tokens(state_dir, period=period)
+        spent = int(spent)
+
+        exceeded = ceiling is not None and spent >= ceiling
+        tripped = False
+
+        if exceeded and trip:
+            reason = (
+                f"cost ceiling exceeded: {period} spend {spent} tokens >= "
+                f"ceiling {ceiling} tokens"
+            )
+            halt.halt(reason, state_dir=state_dir)
+            tripped = True
+
+        return {
+            "period": period,
+            "ceiling": ceiling,
+            "spent": spent,
+            "exceeded": exceeded,
+            "tripped": tripped,
+            "reason": None,
+        }
+    except Exception as e:
+        # Exception during spend computation (e.g., ledger read/create failure,
+        # file lock, transient I/O error). This is fail-SAFE: abort the current
+        # wave (exceeded=True) to prevent runaway work, but do NOT write the
+        # persistent .HALT sentinel (tripped=False). The distinction ensures a
+        # transient I/O hiccup does not permanently wedge the fleet.
+        reason_text = f"cost_check_error: {type(e).__name__}: {str(e)[:100]}"
+        return {
+            "period": period,
+            "ceiling": None,
+            "spent": None,
+            "exceeded": True,
+            "tripped": False,
+            "error": str(e),
+            "reason": reason_text,
+        }
 
 
 def main(argv=None):
@@ -172,6 +198,12 @@ def main(argv=None):
 
     result = check(spent=args.spent, period=args.period)
 
+    # Check for errors FIRST (fail-closed): exception during check() means abort the wave
+    if "error" in result:
+        reason = result.get("reason", result["error"])
+        print(f"[cost_ceiling] FATAL: {reason}", file=sys.stderr)
+        return 1
+
     if result["ceiling"] is None:
         print(f"[cost_ceiling] no {args.period} ceiling configured — skipping (spent={result['spent']})")
         return 0