@sentry/warden 0.14.0 → 0.15.0

package/skills/warden-sweep/scripts/scan.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 # /// script
 # requires-python = ">=3.9"
+# dependencies = ["tomli; python_version < '3.11'"]
 # ///
 """
 Warden Sweep: Scan phase.
@@ -28,12 +29,19 @@ import os
 import secrets
 import subprocess
 import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from datetime import datetime, timezone
 from pathlib import Path
 from typing import Any
 
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore[no-redefine]
+
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from _utils import run_cmd  # noqa: E402
+from _utils import ensure_github_label, run_cmd  # noqa: E402
 
 
 SUPPORTED_EXTENSIONS = {
@@ -66,22 +74,6 @@ def create_sweep_dir(sweep_dir: str) -> None:
         os.makedirs(os.path.join(sweep_dir, subdir), exist_ok=True)
 
 
-def create_warden_label() -> None:
-    """Create the warden label on GitHub (idempotent)."""
-    try:
-        subprocess.run(
-            [
-                "gh", "label", "create", "warden",
-                "--color", "5319E7",
-                "--description", "Automated fix from Warden Sweep",
-            ],
-            capture_output=True,
-            timeout=15,
-        )
-    except (subprocess.TimeoutExpired, FileNotFoundError):
-        pass
-
-
 def write_manifest(sweep_dir: str, run_id: str) -> None:
     """Write the initial manifest.json."""
     repo = "unknown"
@@ -101,6 +93,7 @@ def write_manifest(sweep_dir: str, run_id: str) -> None:
         "phases": {
             "scan": "pending",
             "verify": "pending",
+            "issue": "pending",
             "patch": "pending",
             "organize": "pending",
         },
@@ -112,84 +105,16 @@ def write_manifest(sweep_dir: str, run_id: str) -> None:
         f.write("\n")
 
 
-def _strip_toml_inline_comment(line: str) -> str:
-    """Strip inline TOML comments (# outside of quoted strings)."""
-    in_quote = False
-    quote_char = ""
-    for i, ch in enumerate(line):
-        if in_quote:
-            if ch == quote_char:
-                in_quote = False
-        elif ch in ('"', "'"):
-            in_quote = True
-            quote_char = ch
-        elif ch == "#":
-            return line[:i].rstrip()
-    return line
-
-
-def _toml_array_to_json(value: str) -> str:
-    """Convert a TOML array string to JSON-compatible format.
-
-    Handles TOML single-quoted strings and trailing commas.
-    Inline comments should be stripped before calling this function.
-    """
-    import re
-    # Replace single-quoted strings with double-quoted (TOML literal strings)
-    value = re.sub(r"'([^']*)'", r'"\1"', value)
-    # Strip trailing comma before closing bracket
-    value = re.sub(r",\s*]", "]", value)
-    return value
-
-
 def load_ignore_paths() -> list[str]:
     """Load ignorePaths from warden.toml defaults if present."""
-    try:
-        # Try to parse warden.toml for defaults.ignorePaths
-        toml_path = "warden.toml"
-        if not os.path.exists(toml_path):
-            return []
-
-        with open(toml_path) as f:
-            content = f.read()
-
-        # Simple TOML parsing for ignorePaths in [defaults] section
-        in_defaults = False
-        collecting_value = False
-        value_parts: list[str] = []
-        for line in content.splitlines():
-            stripped = line.strip()
-            if collecting_value:
-                # Skip TOML comment lines inside multiline arrays
-                if stripped.startswith("#"):
-                    continue
-                # Strip inline comments before accumulating
-                stripped = _strip_toml_inline_comment(stripped)
-                value_parts.append(stripped)
-                combined = "".join(value_parts)
-                if combined.count("[") <= combined.count("]"):
-                    try:
-                        return json.loads(_toml_array_to_json(combined))
-                    except json.JSONDecodeError:
-                        return []
-                continue
-            if stripped == "[defaults]":
-                in_defaults = True
-                continue
-            if stripped.startswith("[") and stripped != "[defaults]":
-                in_defaults = False
-                continue
-            if in_defaults and stripped.startswith("ignorePaths"):
-                _, _, value = stripped.partition("=")
-                value = _strip_toml_inline_comment(value.strip())
-                if not value:
-                    continue
-                try:
-                    return json.loads(_toml_array_to_json(value))
-                except json.JSONDecodeError:
-                    value_parts = [value]
-                    collecting_value = True
+    toml_path = "warden.toml"
+    if not os.path.exists(toml_path):
         return []
+    try:
+        with open(toml_path, "rb") as f:
+            config = tomllib.load(f)
+        paths = config.get("defaults", {}).get("ignorePaths", [])
+        return paths if isinstance(paths, list) else []
     except Exception:
         return []
 
@@ -246,7 +171,7 @@ def enumerate_files(
 ) -> list[str]:
     """Enumerate files to scan using git ls-files, filtered by extension."""
     if specific_files:
-        return specific_files
+        return [f for f in specific_files if not should_ignore(f, ignore_patterns)]
 
     result = run_cmd(["git", "ls-files"])
     if result.returncode != 0:
@@ -301,19 +226,22 @@ def log_path_for_file(sweep_dir: str, file_path: str) -> str:
 
 
 def scan_file(
-    file_path: str, log_file: str, timeout: int = 300
+    file_path: str, log_file: str, timeout: int = 600, skill: str | None = None
 ) -> dict[str, Any]:
     """Run warden on a single file. Returns scan-index entry."""
     try:
+        cmd = [
+            "warden", file_path,
+            "--json", "--log",
+            "--min-confidence", "off",
+            "--fail-on", "off",
+            "--quiet",
+            "--output", log_file,
+        ]
+        if skill:
+            cmd.extend(["--skill", skill])
         result = subprocess.run(
-            [
-                "warden", file_path,
-                "--json", "--log",
-                "--min-confidence", "off",
-                "--fail-on", "off",
-                "--quiet",
-                "--output", log_file,
-            ],
+            cmd,
             capture_output=True,
             text=True,
             timeout=timeout,
@@ -350,9 +278,9 @@ def scan_file(
                     record = json.loads(line)
                     if record.get("type") == "summary":
                         continue
-                    skill = record.get("skill", "")
-                    if skill:
-                        skills.add(skill)
+                    record_skill = record.get("skill", "")
+                    if record_skill:
+                        skills.add(record_skill)
                     findings = record.get("findings", [])
                     finding_count += len(findings)
                 except json.JSONDecodeError:
@@ -482,6 +410,10 @@ def main() -> None:
         "--sweep-dir",
         help="Resume into an existing sweep directory",
     )
+    parser.add_argument(
+        "--skill",
+        help="Run only this skill (passed through to warden --skill)",
+    )
     args = parser.parse_args()
 
     # Check dependencies
@@ -510,7 +442,7 @@ def main() -> None:
     if not os.path.exists(manifest_path):
         write_manifest(sweep_dir, run_id)
 
-    create_warden_label()
+    ensure_github_label("warden", "5319E7", "Automated fix from Warden Sweep")
 
     # Enumerate files
     ignore_patterns = load_ignore_paths()
@@ -542,30 +474,41 @@ def main() -> None:
             file=sys.stderr,
         )
 
-    # Scan remaining files
+    # Scan remaining files concurrently
     scanned = already_done
+    index_lock = threading.Lock()
 
-    for i, file_path in enumerate(remaining, start=1):
+    def _scan_and_record(file_path: str) -> dict[str, Any]:
         log_file = log_path_for_file(sweep_dir, file_path)
-        entry = scan_file(file_path, log_file)
+        entry = scan_file(file_path, log_file, skill=args.skill)
 
-        # Append to scan-index.jsonl
-        with open(scan_index_path, "a") as f:
-            f.write(json.dumps(entry) + "\n")
+        with index_lock:
+            with open(scan_index_path, "a") as f:
+                f.write(json.dumps(entry) + "\n")
 
-        scanned += 1
-        if entry["status"] == "error":
-            print(
-                f"[{scanned}/{total}] {file_path} (ERROR: {entry.get('error', 'unknown')})",
-                file=sys.stderr,
-            )
-        else:
-            count = entry.get("findingCount", 0)
-            suffix = f"({count} finding{'s' if count != 1 else ''})" if count > 0 else ""
-            print(
-                f"[{scanned}/{total}] {file_path} {suffix}".rstrip(),
-                file=sys.stderr,
-            )
+        return entry
+
+    with ThreadPoolExecutor(max_workers=4) as pool:
+        futures = {
+            pool.submit(_scan_and_record, fp): fp for fp in remaining
+        }
+        for future in as_completed(futures):
+            entry = future.result()
+            scanned += 1
+            file_path = entry.get("file", futures[future])
+            if entry["status"] == "error":
+                label = "TIMEOUT" if entry.get("error") == "timeout" else "ERROR"
+                print(
+                    f"[{scanned}/{total}] {file_path} ({label}: {entry.get('error', 'unknown')})",
+                    file=sys.stderr,
+                )
+            else:
+                count = entry.get("findingCount", 0)
+                suffix = f"({count} finding{'s' if count != 1 else ''})" if count > 0 else ""
+                print(
+                    f"[{scanned}/{total}] {file_path} {suffix}".rstrip(),
+                    file=sys.stderr,
+                )
 
     # Extract findings
     script_dir = os.path.dirname(os.path.abspath(__file__))
@@ -578,6 +521,7 @@ def main() -> None:
     # so that resumed scans don't include stale errors for files that later succeeded.
     # Scope to current file list so counts stay consistent with `scanned`.
     files_set = set(files)
+    timeouts: list[dict[str, Any]] = []
     errors: list[dict[str, Any]] = []
     if os.path.exists(scan_index_path):
         last_status: dict[str, dict[str, Any]] = {}
@@ -595,36 +539,44 @@ def main() -> None:
                     continue
         for entry in last_status.values():
             if entry.get("status") == "error":
-                errors.append({
+                item = {
                     "file": entry.get("file", ""),
                     "error": entry.get("error", "unknown"),
                     "exitCode": entry.get("exitCode", -1),
-                })
+                }
+                if entry.get("error") == "timeout":
+                    timeouts.append(item)
+                else:
+                    errors.append(item)
+
+    total_failed = len(timeouts) + len(errors)
 
     # Output JSON summary
     output = {
         "runId": run_id,
         "sweepDir": sweep_dir,
-        "filesScanned": scanned - len(errors),
+        "filesScanned": scanned - total_failed,
+        "filesTimedOut": len(timeouts),
         "filesErrored": len(errors),
         "totalFindings": len(findings),
         "bySeverity": by_severity,
         "findingsPath": os.path.join(sweep_dir, "data", "all-findings.jsonl"),
         "findings": findings,
+        "timeouts": timeouts,
         "errors": errors,
     }
 
     print(json.dumps(output, indent=2))
 
     # Fatal only if every file across all runs errored (no successful scans at all)
-    successful = scanned - len(errors)
+    successful = scanned - total_failed
     if successful == 0 and scanned > 0:
         update_manifest_phase(sweep_dir, "scan", "error")
         sys.exit(1)
 
     update_manifest_phase(sweep_dir, "scan", "complete")
 
-    if len(errors) > 0:
+    if total_failed > 0:
         sys.exit(2)