@sentry/warden 0.13.0 → 0.15.0

package/skills/warden-sweep/scripts/generate_report.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""
+Generate summary.md and report.json from a completed sweep.
+
+Usage:
+    python generate_report.py <sweep-dir>
+
+Reads the data/ subdirectory for all-findings.jsonl, verified.jsonl,
+rejected.jsonl, patches.jsonl, and security/index.jsonl, then produces:
+  - <sweep-dir>/summary.md
+  - <sweep-dir>/data/report.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from typing import Any
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from _utils import read_json, read_jsonl, severity_badge  # noqa: E402
+
+
+def generate_summary_md(
+    manifest: dict[str, Any],
+    scan_index: list[dict[str, Any]],
+    all_findings: list[dict[str, Any]],
+    verified: list[dict[str, Any]],
+    rejected: list[dict[str, Any]],
+    patches: list[dict[str, Any]],
+    security_index: list[dict[str, Any]],
+) -> str:
+    """Generate the summary.md content."""
+    run_id = manifest.get("runId", "unknown")
+    started_at = manifest.get("startedAt", "unknown")
+    repo = manifest.get("repo", "unknown")
+    completed_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    files_scanned = sum(1 for e in scan_index if e.get("status") == "complete")
+    files_timed_out = sum(
+        1 for e in scan_index
+        if e.get("status") == "error" and e.get("error") == "timeout"
+    )
+    files_errored = sum(
+        1 for e in scan_index
+        if e.get("status") == "error" and e.get("error") != "timeout"
+    )
+
+    prs_created = sum(1 for p in patches if p.get("status") == "created")
+    prs_failed = sum(1 for p in patches if p.get("status") == "error")
+
+    # Severity breakdown of verified findings
+    by_severity: dict[str, int] = {}
+    for f in verified:
+        sev = f.get("severity", "info")
+        by_severity[sev] = by_severity.get(sev, 0) + 1
+
+    lines = [
+        f"# Warden Sweep: `{run_id}`",
+        "",
+        f"**Repo**: {repo}",
+        f"**Started**: {started_at}",
+        f"**Completed**: {completed_at}",
+        "",
+        "## Stats",
+        "",
+        f"| Metric | Count |",
+        f"|--------|-------|",
+        f"| Files scanned | {files_scanned} |",
+        f"| Files timed out | {files_timed_out} |",
+        f"| Files errored | {files_errored} |",
+        f"| Total findings | {len(all_findings)} |",
+        f"| Verified | {len(verified)} |",
+        f"| Rejected | {len(rejected)} |",
+        f"| PRs created | {prs_created} |",
+        f"| PRs failed | {prs_failed} |",
+        f"| Security findings | {len(security_index)} |",
+        "",
+    ]
+
+    if by_severity:
+        lines.append("### By Severity")
+        lines.append("")
+        for sev in ["critical", "high", "medium", "low", "info"]:
+            count = by_severity.get(sev, 0)
+            if count > 0:
+                lines.append(f"- {severity_badge(sev)}: {count}")
+        lines.append("")
+
+    # Security callout
+    if security_index:
+        lines.append("## Security Findings")
+        lines.append("")
+        lines.append("The following findings are security-related and may need priority review:")
+        lines.append("")
+        lines.append("| ID | Severity | Skill | File | Title |")
+        lines.append("|----|----------|-------|------|-------|")
+        for sf in security_index:
+            fid = sf.get("findingId", "")
+            sev = severity_badge(sf.get("severity", "info"))
+            skill = sf.get("skill", "")
+            filepath = sf.get("file", "")
+            title = sf.get("title", "")
+            lines.append(f"| `{fid}` | {sev} | {skill} | `{filepath}` | {title} |")
+        lines.append("")
+
+    # Verified findings table
+    if verified:
+        lines.append("## Verified Findings")
+        lines.append("")
+        lines.append("| ID | Severity | Skill | File | Title | PR |")
+        lines.append("|----|----------|-------|------|-------|-----|")
+
+        # Build patches lookup
+        pr_lookup: dict[str, str] = {}
+        for p in patches:
+            if p.get("status") == "created" and p.get("findingId"):
+                pr_lookup[p["findingId"]] = p.get("prUrl", "")
+
+        for f in verified:
+            fid = f.get("findingId", "")
+            sev = severity_badge(f.get("severity", "info"))
+            skill = f.get("skill", "")
+            filepath = f.get("file", "")
+            title = f.get("title", "")
+            pr_url = pr_lookup.get(fid, "")
+            pr_link = f"[PR]({pr_url})" if pr_url else "-"
+            lines.append(f"| `{fid}` | {sev} | {skill} | `{filepath}` | {title} | {pr_link} |")
+        lines.append("")
+
+    # Rejected findings summary
+    if rejected:
+        lines.append(f"## Rejected Findings ({len(rejected)})")
+        lines.append("")
+        lines.append("These findings were evaluated and determined to be false positives.")
+        lines.append("See `data/rejected.jsonl` for details.")
+        lines.append("")
+
+    lines.append("---")
+    lines.append(f"*Generated by Warden Sweep `{run_id}`*")
+
+    return "\n".join(lines) + "\n"
+
+
+def generate_report_json(
+    manifest: dict[str, Any],
+    scan_index: list[dict[str, Any]],
+    all_findings: list[dict[str, Any]],
+    verified: list[dict[str, Any]],
+    rejected: list[dict[str, Any]],
+    patches: list[dict[str, Any]],
+    security_index: list[dict[str, Any]],
+) -> dict[str, Any]:
+    """Generate the report.json data."""
+    run_id = manifest.get("runId", "unknown")
+    completed_at = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    files_scanned = sum(1 for e in scan_index if e.get("status") == "complete")
+    files_timed_out = sum(
+        1 for e in scan_index
+        if e.get("status") == "error" and e.get("error") == "timeout"
+    )
+    files_errored = sum(
+        1 for e in scan_index
+        if e.get("status") == "error" and e.get("error") != "timeout"
+    )
+    prs_created = sum(1 for p in patches if p.get("status") == "created")
+    prs_failed = sum(1 for p in patches if p.get("status") == "error")
+
+    # Count verify errors (findings in all but not in verified or rejected)
+    verified_ids = {f["findingId"] for f in verified if "findingId" in f}
+    rejected_ids = {f["findingId"] for f in rejected if "findingId" in f}
+    all_ids = {f["findingId"] for f in all_findings if "findingId" in f}
+    verify_errors = len(all_ids - verified_ids - rejected_ids)
+
+    return {
+        "runId": run_id,
+        "completedAt": completed_at,
+        "scan": {
+            "filesScanned": files_scanned,
+            "filesTimedOut": files_timed_out,
+            "filesErrored": files_errored,
+            "totalFindings": len(all_findings),
+        },
+        "verify": {
+            "verified": len(verified),
+            "rejected": len(rejected),
+            "errors": verify_errors,
+        },
+        "patch": {
+            "prsCreated": prs_created,
+            "prsFailed": prs_failed,
+        },
+        "security": {
+            "count": len(security_index),
+        },
+        "prs": [
+            {
+                "findingId": p.get("findingId", ""),
+                "url": p.get("prUrl", ""),
+                "severity": next(
+                    (f.get("severity", "") for f in verified if f.get("findingId") == p.get("findingId")),
+                    "",
+                ),
+            }
+            for p in patches
+            if p.get("status") == "created"
+        ],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Generate sweep summary and report"
+    )
+    parser.add_argument("sweep_dir", help="Path to the sweep output directory")
+    args = parser.parse_args()
+
+    sweep_dir = args.sweep_dir
+    data_dir = os.path.join(sweep_dir, "data")
+
+    # Read inputs
+    manifest = read_json(os.path.join(data_dir, "manifest.json")) or {}
+    scan_index = read_jsonl(os.path.join(data_dir, "scan-index.jsonl"))
+    all_findings = read_jsonl(os.path.join(data_dir, "all-findings.jsonl"))
+    verified = read_jsonl(os.path.join(data_dir, "verified.jsonl"))
+    rejected = read_jsonl(os.path.join(data_dir, "rejected.jsonl"))
+    patches = read_jsonl(os.path.join(data_dir, "patches.jsonl"))
+    security_index = read_jsonl(os.path.join(sweep_dir, "security", "index.jsonl"))
+
+    # Generate summary.md
+    summary_md = generate_summary_md(
+        manifest, scan_index,
+        all_findings, verified, rejected, patches, security_index,
+    )
+    summary_path = os.path.join(sweep_dir, "summary.md")
+    with open(summary_path, "w") as f:
+        f.write(summary_md)
+
+    # Generate report.json
+    report = generate_report_json(
+        manifest, scan_index, all_findings,
+        verified, rejected, patches, security_index,
+    )
+    report_path = os.path.join(data_dir, "report.json")
+    with open(report_path, "w") as f:
+        json.dump(report, f, indent=2)
+        f.write("\n")
+
+    print(json.dumps({
+        "summaryPath": summary_path,
+        "reportPath": report_path,
+        "verified": len(verified),
+        "rejected": len(rejected),
+        "prsCreated": report["patch"]["prsCreated"],
+        "securityFindings": len(security_index),
+    }))
+
+
+if __name__ == "__main__":
+    main()

package/skills/warden-sweep/scripts/index_prs.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# ///
+"""
+Warden Sweep: Index existing PRs for deduplication.
+
+Fetches open warden-labeled PRs via gh, identifies file overlap with
+verified findings, and caches diffs for overlapping PRs.
+
+Usage:
+    uv run index_prs.py <sweep-dir>
+
+Stdout: JSON summary (for LLM consumption)
+Stderr: Progress lines
+
+Side effects:
+    - Creates data/existing-prs.json
+    - Creates data/pr-diffs/<number>.diff for overlapping PRs
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+from typing import Any
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from _utils import read_jsonl, run_cmd  # noqa: E402
+
+
+def fetch_warden_prs(sweep_dir: str) -> list[dict[str, Any]]:
+    """Fetch open PRs with the warden label."""
+    result = run_cmd(
+        [
+            "gh", "pr", "list",
+            "--label", "warden",
+            "--state", "open",
+            "--json", "number,title,url,files",
+            "--limit", "100",
+        ],
+        timeout=30,
+    )
+
+    if result.returncode != 0:
+        print(f"Warning: gh pr list failed: {result.stderr}", file=sys.stderr)
+        return []
+
+    try:
+        prs = json.loads(result.stdout)
+    except json.JSONDecodeError:
+        print("Warning: Failed to parse gh pr list output", file=sys.stderr)
+        return []
+
+    # Save raw PR data
+    prs_path = os.path.join(sweep_dir, "data", "existing-prs.json")
+    with open(prs_path, "w") as f:
+        json.dump(prs, f, indent=2)
+        f.write("\n")
+
+    return prs
+
+
+def build_file_index(
+    prs: list[dict[str, Any]],
+) -> dict[str, list[dict[str, Any]]]:
+    """Build a file-to-PR lookup from the PR list."""
+    index: dict[str, list[dict[str, Any]]] = {}
+
+    for pr in prs:
+        pr_info = {
+            "number": pr.get("number"),
+            "title": pr.get("title", ""),
+            "url": pr.get("url", ""),
+        }
+        files = pr.get("files") or []
+        for file_entry in files:
+            # gh returns files as objects with "path" key
+            if isinstance(file_entry, dict):
+                path = file_entry.get("path", "")
+            else:
+                path = str(file_entry)
+            if path:
+                index.setdefault(path, []).append(pr_info)
+
+    return index
+
+
+def get_verified_files(sweep_dir: str) -> set[str]:
+    """Get the set of files that have verified findings."""
+    verified_path = os.path.join(sweep_dir, "data", "verified.jsonl")
+    entries = read_jsonl(verified_path)
+    return {e.get("file", "") for e in entries if e.get("file")}
+
+
+def fetch_pr_diff(pr_number: int, sweep_dir: str) -> bool:
+    """Fetch and cache a PR diff. Returns True on success."""
+    diff_path = os.path.join(
+        sweep_dir, "data", "pr-diffs", f"{pr_number}.diff"
+    )
+
+    # Skip if already cached
+    if os.path.exists(diff_path):
+        return True
+
+    result = run_cmd(
+        ["gh", "pr", "diff", str(pr_number)],
+        timeout=30,
+    )
+
+    if result.returncode != 0:
+        print(
+            f"Warning: Failed to fetch diff for PR #{pr_number}: {result.stderr}",
+            file=sys.stderr,
+        )
+        return False
+
+    with open(diff_path, "w") as f:
+        f.write(result.stdout)
+
+    return True
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Warden Sweep: Index existing PRs for dedup"
+    )
+    parser.add_argument("sweep_dir", help="Path to the sweep directory")
+    args = parser.parse_args()
+
+    sweep_dir = args.sweep_dir
+
+    if not os.path.isdir(sweep_dir):
+        print(
+            json.dumps({"error": f"Sweep directory not found: {sweep_dir}"}),
+            file=sys.stdout,
+        )
+        sys.exit(1)
+
+    # Ensure pr-diffs directory exists
+    os.makedirs(os.path.join(sweep_dir, "data", "pr-diffs"), exist_ok=True)
+
+    # Fetch open warden PRs
+    print("Fetching open warden-labeled PRs...", file=sys.stderr)
+    prs = fetch_warden_prs(sweep_dir)
+    print(f"Found {len(prs)} open warden PR(s)", file=sys.stderr)
+
+    # Build file index
+    file_index = build_file_index(prs)
+
+    # Find overlap with verified findings
+    verified_files = get_verified_files(sweep_dir)
+    overlapping_prs: set[int] = set()
+
+    for vfile in verified_files:
+        if vfile in file_index:
+            for pr_info in file_index[vfile]:
+                overlapping_prs.add(pr_info["number"])
+
+    # Fetch diffs for overlapping PRs
+    diffs_cached = 0
+    for pr_number in sorted(overlapping_prs):
+        print(f"Caching diff for PR #{pr_number}...", file=sys.stderr)
+        if fetch_pr_diff(pr_number, sweep_dir):
+            diffs_cached += 1
+
+    # Build output file index (only for files that have verified findings)
+    output_file_index: dict[str, list[dict[str, Any]]] = {}
+    for vfile in verified_files:
+        if vfile in file_index:
+            output_file_index[vfile] = file_index[vfile]
+
+    # Output summary
+    output = {
+        "totalPRs": len(prs),
+        "overlappingPRs": len(overlapping_prs),
+        "fileIndex": output_file_index,
+        "diffsCached": diffs_cached,
+    }
+
+    print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+    main()