mindforge-cc 11.8.2 → 11.9.0

package/.mindforge/skills/osint-investigation/scripts/build_findings.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+"""Build a structured findings.json with evidence chains (stdlib-only).
+
+Aggregates cross_links.csv (entity_resolution output) and an optional
+timing.json (timing_analysis output) into a single evidence-chain document.
+
+Output structure:
+    {
+      "metadata": {...},
+      "findings": [
+        {
+          "id": "F0001",
+          "title": "...",
+          "severity": "HIGH|MEDIUM|LOW",
+          "confidence": "high|medium|low",
+          "summary": "...",
+          "evidence": [
+            {"source": "cross_links.csv", "row": 12, "fields": {...}},
+            ...
+          ],
+          "sources": ["cross_links.csv", "timing.json"]
+        }
+      ]
+    }
+
+Every finding traces to specific source rows. No naked claims.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from collections import defaultdict
+from pathlib import Path
+
+CONFIDENCE_ORDER = {"high": 0, "medium": 1, "low": 2}
+SEVERITY_ORDER = {"HIGH": 0, "MEDIUM": 1, "LOW": 2}
+
+
+def _read_cross_links(path: str) -> list[dict[str, str]]:
+    with open(path, newline="", encoding="utf-8") as fh:
+        return list(csv.DictReader(fh))
+
+
+def build_findings(
+    cross_links_path: str,
+    timing_path: str | None = None,
+    out_path: str = "findings.json",
+    bundled_threshold: int = 3,
+) -> dict:
+    findings: list[dict] = []
+    next_id = 1
+
+    # 1. Match-based findings, grouped by (left_normalized, right_normalized).
+    matches = _read_cross_links(cross_links_path)
+    grouped: dict[tuple[str, str], list[dict[str, str]]] = defaultdict(list)
+    for i, row in enumerate(matches):
+        row["__row__"] = str(i)
+        grouped[(row.get("left_normalized", ""), row.get("right_normalized", ""))].append(row)
+
+    for (left_norm, right_norm), rows in grouped.items():
+        if not left_norm or not right_norm:
+            continue
+        # Use the highest-confidence match for the finding's overall confidence.
+        best = min(rows, key=lambda r: CONFIDENCE_ORDER.get(r.get("confidence", "low"), 2))
+        finding_id = f"F{next_id:04d}"
+        next_id += 1
+        evidence = [
+            {
+                "source": "cross_links.csv",
+                "row": int(r["__row__"]),
+                "fields": {
+                    "match_type": r.get("match_type", ""),
+                    "confidence": r.get("confidence", ""),
+                    "left_name": r.get("left_name", ""),
+                    "right_name": r.get("right_name", ""),
+                    "overlap_ratio": r.get("overlap_ratio", ""),
+                    "shared_tokens": r.get("shared_tokens", ""),
+                },
+            }
+            for r in rows
+        ]
+        findings.append(
+            {
+                "id": finding_id,
+                "title": f"Entity match: {best.get('left_name', '')} ↔ {best.get('right_name', '')}",
+                "severity": "MEDIUM" if best.get("confidence") == "high" else "LOW",
+                "confidence": best.get("confidence", "low"),
+                "summary": (
+                    f"{len(rows)} cross-link record(s) tie "
+                    f"'{best.get('left_name', '')}' to "
+                    f"'{best.get('right_name', '')}' "
+                    f"(best tier: {best.get('match_type', '')})."
+                ),
+                "evidence": evidence,
+                "sources": ["cross_links.csv"],
+            }
+        )
+
+    # 2. Bundled-donations findings (if cross_links carries donor↔candidate pattern).
+    #    Heuristic: many distinct left names sharing the same right name.
+    by_right: dict[str, set[str]] = defaultdict(set)
+    by_right_rows: dict[str, list[dict[str, str]]] = defaultdict(list)
+    for r in matches:
+        right = r.get("right_normalized", "")
+        left_raw = r.get("left_name", "").strip()
+        if right and left_raw:
+            by_right[right].add(left_raw)
+            by_right_rows[right].append(r)
+    for right_norm, lefts in by_right.items():
+        if len(lefts) < bundled_threshold:
+            continue
+        rows = by_right_rows[right_norm]
+        right_raw = rows[0].get("right_name", "")
+        findings.append(
+            {
+                "id": f"F{next_id:04d}",
+                "title": f"Bundled cross-links: {len(lefts)} distinct left entities ↔ '{right_raw}'",
+                "severity": "HIGH",
+                "confidence": "medium",
+                "summary": (
+                    f"{len(lefts)} distinct left-side entities link to "
+                    f"'{right_raw}'. Pattern suggests coordinated relationship "
+                    f"(e.g. bundled donations, multi-vendor employer)."
+                ),
+                "evidence": [
+                    {
+                        "source": "cross_links.csv",
+                        "row": int(r.get("__row__", "0")),
+                        "fields": {
+                            "left_name": r.get("left_name", ""),
+                            "match_type": r.get("match_type", ""),
+                        },
+                    }
+                    for r in rows
+                ],
+                "sources": ["cross_links.csv"],
+            }
+        )
+        next_id += 1
+
+    # 3. Timing-based findings.
+    if timing_path and Path(timing_path).exists():
+        timing = json.loads(Path(timing_path).read_text())
+        for r in timing.get("results", []):
+            if not r.get("significant"):
+                continue
+            findings.append(
+                {
+                    "id": f"F{next_id:04d}",
+                    "title": (
+                        f"Donation timing significantly clusters near awards: "
+                        f"{r['donor']} ↔ {r['recipient']}"
+                    ),
+                    "severity": "HIGH" if r["p_value"] < 0.01 else "MEDIUM",
+                    "confidence": "medium",
+                    "summary": (
+                        f"Mean nearest-award distance {r['observed_mean_days']} days "
+                        f"(null {r['null_mean_days']} days). p={r['p_value']}, "
+                        f"effect size {r['effect_size_sd']} SD. "
+                        f"{r['n_donations']} donations, {r['n_award_dates']} awards."
+                    ),
+                    "evidence": [
+                        {
+                            "source": "timing.json",
+                            "row": None,
+                            "fields": r,
+                        }
+                    ],
+                    "sources": ["timing.json"],
+                }
+            )
+            next_id += 1
+
+    # Sort: severity → confidence → id.
+    findings.sort(
+        key=lambda f: (
+            SEVERITY_ORDER.get(f["severity"], 3),
+            CONFIDENCE_ORDER.get(f["confidence"], 3),
+            f["id"],
+        )
+    )
+
+    payload = {
+        "metadata": {
+            "n_findings": len(findings),
+            "cross_links_path": cross_links_path,
+            "timing_path": timing_path,
+            "bundled_threshold": bundled_threshold,
+        },
+        "findings": findings,
+    }
+    Path(out_path).write_text(json.dumps(payload, indent=2))
+    return payload
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--cross-links", required=True)
+    p.add_argument("--timing", help="Optional timing.json from timing_analysis.py")
+    p.add_argument("--out", default="findings.json")
+    p.add_argument(
+        "--bundled-threshold",
+        type=int,
+        default=3,
+        help="Minimum distinct left entities to flag as bundled (default 3)",
+    )
+    a = p.parse_args()
+
+    payload = build_findings(
+        cross_links_path=a.cross_links,
+        timing_path=a.timing,
+        out_path=a.out,
+        bundled_threshold=a.bundled_threshold,
+    )
+    print(f"Wrote {payload['metadata']['n_findings']} findings to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.mindforge/skills/osint-investigation/scripts/entity_resolution.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""Cross-source entity resolution (stdlib-only).
+
+Given two CSV files with name columns, find candidate matches using three
+tiers of normalization:
+
+  1. exact          — normalized strings equal
+  2. fuzzy          — sorted-token (word-bag) match
+  3. token_overlap  — >=60% Jaccard overlap on >=4-char tokens, >=2 shared
+
+Adapted from ShinMegamiBoson/OpenPlanter (MIT) but generalized: no Boston-
+specific record types, no contribution-code filters, no fixed schemas.
+
+Output CSV columns:
+    match_type, confidence, left_name, right_name,
+    left_normalized, right_normalized, left_row, right_row,
+    overlap_ratio, shared_tokens
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+from pathlib import Path
+
+# Allow running directly or as a module.
+sys.path.insert(0, str(Path(__file__).parent))
+from _normalize import (  # noqa: E402
+    normalize_name,
+    normalize_aggressive,
+    token_overlap_ratio,
+)
+
+CONFIDENCE = {
+    "exact": "high",
+    "fuzzy": "medium",
+    "token_overlap": "low",
+}
+
+
+def _read_csv(path: str, name_col: str) -> list[dict[str, str]]:
+    rows = []
+    with open(path, newline="", encoding="utf-8") as fh:
+        reader = csv.DictReader(fh)
+        if name_col not in (reader.fieldnames or []):
+            raise SystemExit(
+                f"Column {name_col!r} not in {path}. "
+                f"Available: {reader.fieldnames}"
+            )
+        for i, row in enumerate(reader):
+            row["__row__"] = str(i)
+            rows.append(row)
+    return rows
+
+
+def _build_index(rows: list[dict[str, str]], name_col: str):
+    """Index by exact-normalized and aggressive (sorted-token) form."""
+    exact: dict[str, list[dict[str, str]]] = {}
+    aggressive: dict[str, list[dict[str, str]]] = {}
+    for row in rows:
+        raw = row.get(name_col, "")
+        n = normalize_name(raw)
+        if n:
+            exact.setdefault(n, []).append(row)
+        a = normalize_aggressive(raw)
+        if a:
+            aggressive.setdefault(a, []).append(row)
+    return exact, aggressive
+
+
+def _emit(
+    out_rows: list[dict[str, str]],
+    seen: set[tuple],
+    match_type: str,
+    left_row: dict[str, str],
+    right_row: dict[str, str],
+    left_col: str,
+    right_col: str,
+    ratio: float = 0.0,
+    shared: int = 0,
+):
+    left_raw = left_row.get(left_col, "")
+    right_raw = right_row.get(right_col, "")
+    key = (
+        left_row["__row__"],
+        right_row["__row__"],
+        match_type,
+    )
+    if key in seen:
+        return
+    seen.add(key)
+    out_rows.append(
+        {
+            "match_type": match_type,
+            "confidence": CONFIDENCE[match_type],
+            "left_name": left_raw,
+            "right_name": right_raw,
+            "left_normalized": normalize_name(left_raw),
+            "right_normalized": normalize_name(right_raw),
+            "left_row": left_row["__row__"],
+            "right_row": right_row["__row__"],
+            "overlap_ratio": f"{ratio:.3f}" if ratio else "",
+            "shared_tokens": str(shared) if shared else "",
+        }
+    )
+
+
+def resolve(
+    left_path: str,
+    left_col: str,
+    right_path: str,
+    right_col: str,
+    out_path: str,
+    overlap_threshold: float = 0.60,
+    min_shared: int = 2,
+    skip_overlap: bool = False,
+) -> int:
+    left_rows = _read_csv(left_path, left_col)
+    right_rows = _read_csv(right_path, right_col)
+
+    right_exact, right_aggressive = _build_index(right_rows, right_col)
+
+    out_rows: list[dict[str, str]] = []
+    seen: set[tuple] = set()
+
+    # Pass 1+2: exact / fuzzy via index lookup.
+    for lrow in left_rows:
+        raw = lrow.get(left_col, "")
+        n = normalize_name(raw)
+        if not n:
+            continue
+        for rrow in right_exact.get(n, []):
+            _emit(out_rows, seen, "exact", lrow, rrow, left_col, right_col)
+        a = normalize_aggressive(raw)
+        if a:
+            for rrow in right_aggressive.get(a, []):
+                _emit(out_rows, seen, "fuzzy", lrow, rrow, left_col, right_col)
+
+    if not skip_overlap:
+        # Pass 3: token overlap (O(N*M) — expensive; allow opt-out).
+        for lrow in left_rows:
+            l_raw = lrow.get(left_col, "")
+            if not normalize_name(l_raw):
+                continue
+            for rrow in right_rows:
+                ratio, shared = token_overlap_ratio(
+                    l_raw, rrow.get(right_col, "")
+                )
+                if ratio >= overlap_threshold and shared >= min_shared:
+                    _emit(
+                        out_rows,
+                        seen,
+                        "token_overlap",
+                        lrow,
+                        rrow,
+                        left_col,
+                        right_col,
+                        ratio=ratio,
+                        shared=shared,
+                    )
+
+    fieldnames = [
+        "match_type",
+        "confidence",
+        "left_name",
+        "right_name",
+        "left_normalized",
+        "right_normalized",
+        "left_row",
+        "right_row",
+        "overlap_ratio",
+        "shared_tokens",
+    ]
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        writer = csv.DictWriter(fh, fieldnames=fieldnames)
+        writer.writeheader()
+        writer.writerows(out_rows)
+    return len(out_rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--left", required=True, help="Left CSV path")
+    p.add_argument(
+        "--left-name-col", required=True, help="Name column in left CSV"
+    )
+    p.add_argument("--right", required=True, help="Right CSV path")
+    p.add_argument(
+        "--right-name-col",
+        required=True,
+        help="Name column in right CSV",
+    )
+    p.add_argument("--out", required=True, help="Output CSV path")
+    p.add_argument(
+        "--overlap-threshold",
+        type=float,
+        default=0.60,
+        help="Jaccard overlap threshold for token_overlap tier (default 0.60)",
+    )
+    p.add_argument(
+        "--min-shared",
+        type=int,
+        default=2,
+        help="Minimum shared tokens for token_overlap tier (default 2)",
+    )
+    p.add_argument(
+        "--skip-overlap",
+        action="store_true",
+        help="Skip the O(N*M) token_overlap pass (much faster on large CSVs)",
+    )
+    args = p.parse_args()
+
+    count = resolve(
+        left_path=args.left,
+        left_col=args.left_name_col,
+        right_path=args.right,
+        right_col=args.right_name_col,
+        out_path=args.out,
+        overlap_threshold=args.overlap_threshold,
+        min_shared=args.min_shared,
+        skip_overlap=args.skip_overlap,
+    )
+    print(f"Wrote {count} match rows to {args.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.mindforge/skills/osint-investigation/scripts/fetch_courtlistener.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Search court records via CourtListener (Free Law Project).
+
+Covers ~10M federal and state court opinions, plus PACER docket data
+where available. Public REST API v4 supports anonymous read access for
+search; some endpoints require a token (free at courtlistener.com).
+
+Set COURTLISTENER_TOKEN to authenticate (raises rate limits).
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import os
+import sys
+import urllib.parse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _http import get_json  # noqa: E402
+
+BASE = "https://www.courtlistener.com/api/rest/v4/search/"
+
+COLUMNS = [
+    "case_name",
+    "court",
+    "court_id",
+    "date_filed",
+    "docket_number",
+    "judge",
+    "citation",
+    "result_type",
+    "snippet",
+    "absolute_url",
+]
+
+SEARCH_TYPES = {
+    "opinions": "o",       # Court opinions
+    "dockets": "r",        # PACER dockets (may require auth depending on coverage)
+    "oral": "oa",          # Oral arguments
+    "people": "p",         # Judges / people
+    "recap": "r",          # Same as dockets in v4
+}
+
+
+def fetch(
+    query: str,
+    search_type: str,
+    court: str | None,
+    date_from: str | None,
+    date_to: str | None,
+    token: str | None,
+    limit: int,
+    out_path: str,
+) -> int:
+    type_code = SEARCH_TYPES.get(search_type, search_type)
+    params = {
+        "q": query,
+        "type": type_code,
+    }
+    if court:
+        params["court"] = court
+    if date_from:
+        params["filed_after"] = date_from
+    if date_to:
+        params["filed_before"] = date_to
+    headers = {"Authorization": f"Token {token}"} if token else None
+
+    rows: list[dict[str, str]] = []
+    next_url: str | None = f"{BASE}?{urllib.parse.urlencode(params)}"
+    while next_url and len(rows) < limit:
+        try:
+            payload = get_json(next_url, headers=headers)
+        except Exception as e:  # noqa: BLE001
+            print(f"CourtListener error: {e}", file=sys.stderr)
+            break
+        if not isinstance(payload, dict):
+            break
+        results = payload.get("results", [])
+        for r in results:
+            if len(rows) >= limit:
+                break
+            rows.append(
+                {
+                    "case_name": r.get("caseName", "") or r.get("case_name", "") or "",
+                    "court": r.get("court", "") or "",
+                    "court_id": r.get("court_id", "") or "",
+                    "date_filed": (r.get("dateFiled", "") or r.get("date_filed", "") or "")[:10],
+                    "docket_number": r.get("docketNumber", "") or r.get("docket_number", "") or "",
+                    "judge": r.get("judge", "") or "",
+                    "citation": "; ".join(r.get("citation", []) or []) if isinstance(r.get("citation"), list) else (r.get("citation") or ""),
+                    "result_type": search_type,
+                    "snippet": (r.get("snippet", "") or "").replace("\n", " ")[:500],
+                    "absolute_url": (
+                        f"https://www.courtlistener.com{r.get('absolute_url', '')}"
+                        if r.get("absolute_url", "").startswith("/")
+                        else r.get("absolute_url", "")
+                    ),
+                }
+            )
+        next_url = payload.get("next")
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    if not rows:
+        print(
+            f"CourtListener: 0 results for type={search_type!r} q={query!r}. "
+            "Most private individuals don't appear in published court records "
+            "unless they were party to a federal or state appellate case.",
+            file=sys.stderr,
+        )
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--query", required=True, help="Search query (party name, case name, keyword)")
+    p.add_argument(
+        "--type",
+        default="opinions",
+        choices=list(SEARCH_TYPES.keys()),
+        help="Search type (default: opinions)",
+    )
+    p.add_argument("--court", help="Court ID filter (e.g. 'nysd' = SDNY, 'scotus' = Supreme Court)")
+    p.add_argument("--date-from", help="Filed-after date YYYY-MM-DD")
+    p.add_argument("--date-to", help="Filed-before date YYYY-MM-DD")
+    p.add_argument("--token", default=os.environ.get("COURTLISTENER_TOKEN"))
+    p.add_argument("--limit", type=int, default=100)
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    n = fetch(
+        query=a.query,
+        search_type=a.type,
+        court=a.court,
+        date_from=a.date_from,
+        date_to=a.date_to,
+        token=a.token,
+        limit=a.limit,
+        out_path=a.out,
+    )
+    print(f"Wrote {n} CourtListener rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())