mindforge-cc 11.8.2 → 11.9.0

package/.mindforge/skills/osint-investigation/scripts/fetch_gdelt.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env python3
+"""Search the GDELT 2.0 DOC API for news mentions.
+
+GDELT monitors world news in 100+ languages and indexes the full text.
+Free, anonymous, ~15-minute update frequency. Covers ~2015→present.
+
+Useful for surfacing news mentions of a person, company, or topic across
+international media — much wider net than Google News.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+import time
+import urllib.parse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _http import get_json  # noqa: E402
+
+BASE = "https://api.gdeltproject.org/api/v2/doc/doc"
+
+COLUMNS = [
+    "title",
+    "url",
+    "seen_date",
+    "domain",
+    "language",
+    "source_country",
+    "tone",
+    "social_image",
+]
+
+
+def fetch(
+    query: str,
+    mode: str,
+    timespan: str | None,
+    start_datetime: str | None,
+    end_datetime: str | None,
+    source_country: str | None,
+    source_lang: str | None,
+    limit: int,
+    out_path: str,
+) -> int:
+    params: dict[str, str] = {
+        "query": query,
+        "mode": mode,
+        "format": "json",
+        "maxrecords": str(min(limit, 250)),
+        "sort": "datedesc",
+    }
+    if timespan:
+        params["timespan"] = timespan
+    if start_datetime:
+        params["startdatetime"] = start_datetime.replace("-", "").replace(":", "").replace(" ", "")
+    if end_datetime:
+        params["enddatetime"] = end_datetime.replace("-", "").replace(":", "").replace(" ", "")
+    if source_country:
+        params["sourcecountry"] = source_country
+    if source_lang:
+        params["sourcelang"] = source_lang
+
+    url = f"{BASE}?{urllib.parse.urlencode(params)}"
+    payload: dict | list = {}
+    for attempt in range(3):
+        try:
+            payload = get_json(url)
+            break
+        except RuntimeError as e:
+            # GDELT requires 1 request per 5 seconds; back off and retry.
+            if "429" in str(e) and attempt < 2:
+                print(
+                    f"GDELT throttle hit; sleeping 6s before retry "
+                    f"(attempt {attempt + 1}/3)",
+                    file=sys.stderr,
+                )
+                time.sleep(6)
+                continue
+            print(f"GDELT error: {e}", file=sys.stderr)
+            payload = {}
+            break
+        except Exception as e:  # noqa: BLE001
+            print(f"GDELT error: {e}", file=sys.stderr)
+            payload = {}
+            break
+
+    rows: list[dict[str, str]] = []
+    if isinstance(payload, dict):
+        articles = payload.get("articles", []) or []
+        for a in articles[:limit]:
+            seen = (a.get("seendate") or "")
+            # GDELT format: 20260319T083000Z → 2026-03-19 08:30:00Z
+            if len(seen) == 16 and "T" in seen:
+                seen = f"{seen[0:4]}-{seen[4:6]}-{seen[6:8]} {seen[9:11]}:{seen[11:13]}:{seen[13:15]}Z"
+            rows.append(
+                {
+                    "title": (a.get("title") or "").replace("\n", " ").strip(),
+                    "url": a.get("url") or "",
+                    "seen_date": seen,
+                    "domain": a.get("domain") or "",
+                    "language": a.get("language") or "",
+                    "source_country": a.get("sourcecountry") or "",
+                    "tone": str(a.get("tone") or ""),
+                    "social_image": a.get("socialimage") or "",
+                }
+            )
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    if not rows:
+        print(
+            f"GDELT: 0 articles for query={query!r}. "
+            "GDELT indexes ~2015→present. Try widening the timespan or "
+            "checking the query syntax (https://blog.gdeltproject.org/gdelt-doc-2-0-api-debuts/).",
+            file=sys.stderr,
+        )
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--query", required=True, help='Search query (supports GDELT operators: quoted phrases, AND/OR/NOT, sourcecountry:, theme:)')
+    p.add_argument(
+        "--mode",
+        default="ArtList",
+        choices=["ArtList", "ImageCollage", "TimelineVol", "TimelineTone", "ToneChart"],
+        help="GDELT mode (default ArtList for article list)",
+    )
+    p.add_argument(
+        "--timespan",
+        help="Relative window: e.g. '1d', '1w', '1m', '3m', '1y' (overrides start/end)",
+    )
+    p.add_argument("--start", help="Absolute start YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS")
+    p.add_argument("--end", help="Absolute end YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS")
+    p.add_argument("--source-country", help="2-letter source country (e.g. US, UK)")
+    p.add_argument("--source-lang", help="Source language (e.g. English, Spanish)")
+    p.add_argument("--limit", type=int, default=100)
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    n = fetch(
+        query=a.query,
+        mode=a.mode,
+        timespan=a.timespan,
+        start_datetime=a.start,
+        end_datetime=a.end,
+        source_country=a.source_country,
+        source_lang=a.source_lang,
+        limit=a.limit,
+        out_path=a.out,
+    )
+    print(f"Wrote {n} GDELT article rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.mindforge/skills/osint-investigation/scripts/fetch_icij_offshore.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""Search ICIJ Offshore Leaks via the bulk CSV database.
+
+The old reconcile endpoint (https://offshoreleaks.icij.org/reconcile) returns
+404 — ICIJ has removed it. The remaining stable access path is the public
+bulk download:
+
+    https://offshoreleaks-data.icij.org/offshoreleaks/csv/full-oldb.LATEST.zip
+
+~70 MB, ~6 CSVs inside (nodes-entities, nodes-officers, nodes-intermediaries,
+nodes-addresses, relationships, ...). We cache it under
+$HERMES_OSINT_CACHE/icij/ (default: ~/.cache/hermes-osint/icij/) and search
+locally so the agent doesn't re-download for every query.
+
+Output CSV columns match the original `fetch_icij_offshore.py` contract.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import io
+import os
+import re
+import sys
+import time
+import urllib.request
+import zipfile
+from pathlib import Path
+
+BULK_URL = "https://offshoreleaks-data.icij.org/offshoreleaks/csv/full-oldb.LATEST.zip"
+
+COLUMNS = [
+    "node_id",
+    "name",
+    "node_type",
+    "country_codes",
+    "countries",
+    "jurisdiction",
+    "incorporation_date",
+    "inactivation_date",
+    "source",
+    "entity_url",
+    "connections",
+]
+
+
+def _cache_dir() -> Path:
+    base = os.environ.get("HERMES_OSINT_CACHE")
+    if base:
+        return Path(base) / "icij"
+    return Path.home() / ".cache" / "hermes-osint" / "icij"
+
+
+def _download(dest: Path, force: bool = False) -> Path:
+    """Download (or reuse cached) ICIJ bulk ZIP."""
+    dest.mkdir(parents=True, exist_ok=True)
+    zip_path = dest / "full-oldb.zip"
+    if zip_path.exists() and not force:
+        # Re-check age: refetch if older than 30 days.
+        age_days = (time.time() - zip_path.stat().st_mtime) / 86400
+        if age_days < 30:
+            return zip_path
+    print(f"Downloading ICIJ bulk database (~70 MB) to {zip_path}", file=sys.stderr)
+    req = urllib.request.Request(
+        BULK_URL,
+        headers={"User-Agent": "hermes-agent osint-investigation skill"},
+    )
+    with urllib.request.urlopen(req, timeout=120) as resp:  # noqa: S310
+        tmp = zip_path.with_suffix(".zip.tmp")
+        with open(tmp, "wb") as fh:
+            while True:
+                chunk = resp.read(1 << 16)
+                if not chunk:
+                    break
+                fh.write(chunk)
+    tmp.replace(zip_path)
+    return zip_path
+
+
+def _open_csv(zf: zipfile.ZipFile, name_pattern: str):
+    """Open the first CSV matching name_pattern (case-insensitive substring)."""
+    for info in zf.infolist():
+        if name_pattern.lower() in info.filename.lower() and info.filename.lower().endswith(".csv"):
+            return zf.open(info), info.filename
+    return None, None
+
+
+def _match(needle_norm: str, hay: str) -> bool:
+    return needle_norm in (hay or "").upper()
+
+
+def _normalize_query(s: str) -> str:
+    s = s.upper()
+    s = re.sub(r"[^\w\s]", " ", s)
+    s = re.sub(r"\s+", " ", s).strip()
+    return s
+
+
+def fetch(
+    entity: str | None,
+    officer: str | None,
+    jurisdiction: str | None,
+    out_path: str,
+    cache_dir: Path,
+    force_refresh: bool = False,
+    limit: int = 500,
+) -> int:
+    zip_path = _download(cache_dir, force=force_refresh)
+    rows: list[dict[str, str]] = []
+    needles: list[tuple[str, str]] = []  # (kind, normalized needle)
+    if entity:
+        needles.append(("Entity", _normalize_query(entity)))
+    if officer:
+        needles.append(("Officer", _normalize_query(officer)))
+    jur_norm = _normalize_query(jurisdiction) if jurisdiction else None
+
+    targets = [
+        ("Entity", "nodes-entities"),
+        ("Officer", "nodes-officers"),
+        ("Intermediary", "nodes-intermediaries"),
+    ]
+
+    with zipfile.ZipFile(zip_path) as zf:
+        for node_type, csv_substring in targets:
+            relevant_needles = [n for (k, n) in needles if k in {node_type, "Entity", "Officer"}] or []
+            # Only scan a CSV if we have a needle that could plausibly match it,
+            # or if we have ONLY a jurisdiction filter.
+            applicable_needles = [n for (k, n) in needles if k == node_type]
+            if needles and not applicable_needles and not jur_norm:
+                continue
+            stream, fname = _open_csv(zf, csv_substring)
+            if not stream:
+                continue
+            with stream:
+                text = io.TextIOWrapper(stream, encoding="utf-8", errors="replace")
+                reader = csv.DictReader(text)
+                for row in reader:
+                    name = (row.get("name") or "").strip()
+                    if not name:
+                        continue
+                    name_u = name.upper()
+                    matched = False
+                    for n in applicable_needles or relevant_needles:
+                        if _match(n, name_u):
+                            matched = True
+                            break
+                    if not needles:
+                        matched = True  # jurisdiction-only sweep
+                    if not matched:
+                        continue
+                    jur = (row.get("jurisdiction_description") or row.get("country_codes") or "").strip()
+                    if jur_norm and jur_norm not in jur.upper() and jur_norm not in (row.get("countries") or "").upper():
+                        continue
+                    node_id = (row.get("node_id") or "").strip()
+                    rows.append(
+                        {
+                            "node_id": node_id,
+                            "name": name,
+                            "node_type": node_type,
+                            "country_codes": row.get("country_codes", "") or "",
+                            "countries": row.get("countries", "") or "",
+                            "jurisdiction": jur,
+                            "incorporation_date": row.get("incorporation_date", "") or "",
+                            "inactivation_date": row.get("inactivation_date", "") or "",
+                            "source": row.get("sourceID", "") or row.get("source", "") or "",
+                            "entity_url": (
+                                f"https://offshoreleaks.icij.org/nodes/{node_id}" if node_id else ""
+                            ),
+                            "connections": "",
+                        }
+                    )
+                    if len(rows) >= limit:
+                        break
+            if len(rows) >= limit:
+                break
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    if not rows:
+        bits = []
+        if entity:
+            bits.append(f"entity={entity!r}")
+        if officer:
+            bits.append(f"officer={officer!r}")
+        if jurisdiction:
+            bits.append(f"jurisdiction={jurisdiction!r}")
+        print(
+            f"ICIJ: 0 matches for {', '.join(bits)}. "
+            "The bulk database covers offshore leaks (Panama, Paradise, Pandora, "
+            "Bahamas, Offshore Leaks). Most private US individuals are NOT in it.",
+            file=sys.stderr,
+        )
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--entity", help="Search by entity name (substring, case-insensitive)")
+    p.add_argument("--officer", help="Search by officer / individual name (substring, case-insensitive)")
+    p.add_argument("--jurisdiction", help="Filter results by jurisdiction substring")
+    p.add_argument("--limit", type=int, default=500)
+    p.add_argument("--out", required=True)
+    p.add_argument(
+        "--cache-dir",
+        type=Path,
+        default=None,
+        help="Override cache directory (default: $HERMES_OSINT_CACHE/icij or ~/.cache/hermes-osint/icij)",
+    )
+    p.add_argument(
+        "--force-refresh",
+        action="store_true",
+        help="Re-download the bulk ZIP even if a recent cached copy exists.",
+    )
+    a = p.parse_args()
+    if not (a.entity or a.officer or a.jurisdiction):
+        p.error("must supply at least one of --entity / --officer / --jurisdiction")
+    n = fetch(
+        entity=a.entity,
+        officer=a.officer,
+        jurisdiction=a.jurisdiction,
+        out_path=a.out,
+        cache_dir=a.cache_dir or _cache_dir(),
+        force_refresh=a.force_refresh,
+        limit=a.limit,
+    )
+    print(f"Wrote {n} ICIJ Offshore Leaks rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.mindforge/skills/osint-investigation/scripts/fetch_nyc_acris.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Search NYC property records via ACRIS (Automated City Register Information System).
+
+Uses the city's Socrata-backed open data API. No auth required for read access.
+
+Datasets:
+  bnx9-e6tj — Real Property Master (one row per recorded document)
+  636b-3b5g — Real Property Parties (names — grantor, grantee, etc.)
+  8h5j-fqxa — Real Property Legal (lot / property identifiers)
+  uqqa-hym2 — Real Property References
+
+The Parties dataset has the names. We search by name and optionally join to
+Master to get the doc type and date.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+import urllib.parse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _http import get_json  # noqa: E402
+
+PARTIES_URL = "https://data.cityofnewyork.us/resource/636b-3b5g.json"
+MASTER_URL = "https://data.cityofnewyork.us/resource/bnx9-e6tj.json"
+
+PARTY_TYPE = {
+    "1": "grantor (seller / mortgagor / debtor)",
+    "2": "grantee (buyer / mortgagee / creditor)",
+    "3": "other party",
+}
+
+BOROUGH = {
+    "1": "Manhattan",
+    "2": "Bronx",
+    "3": "Brooklyn",
+    "4": "Queens",
+    "5": "Staten Island",
+}
+
+COLUMNS = [
+    "document_id",
+    "name",
+    "party_type",
+    "party_role",
+    "address_1",
+    "address_2",
+    "city",
+    "state",
+    "zip",
+    "country",
+    "doc_type",
+    "doc_date",
+    "recorded_date",
+    "borough",
+    "amount",
+    "filing_url",
+]
+
+
+def _filing_url(document_id: str) -> str:
+    if not document_id:
+        return ""
+    return (
+        f"https://a836-acris.nyc.gov/DS/DocumentSearch/DocumentImageView?doc_id={document_id}"
+    )
+
+
+def fetch(
+    name: str | None,
+    address: str | None,
+    party_type: str | None,
+    limit: int,
+    out_path: str,
+    enrich: bool = True,
+) -> int:
+    if not (name or address):
+        raise SystemExit("must supply --name or --address")
+
+    where_clauses: list[str] = []
+    if name:
+        safe = name.upper().replace("'", "''")
+        where_clauses.append(f"upper(name) like '%{safe}%'")
+    if address:
+        safe_addr = address.upper().replace("'", "''")
+        where_clauses.append(f"upper(address_1) like '%{safe_addr}%'")
+    if party_type and party_type in {"1", "2", "3"}:
+        where_clauses.append(f"party_type='{party_type}'")
+
+    params = {
+        "$where": " AND ".join(where_clauses),
+        "$limit": str(limit),
+    }
+    url = f"{PARTIES_URL}?{urllib.parse.urlencode(params)}"
+    parties = get_json(url)
+    if not isinstance(parties, list):
+        raise SystemExit(f"Unexpected ACRIS response: {parties!r}")
+
+    # Enrich with master record (doc_type, dates, borough, amount).
+    doc_ids: list[str] = sorted({
+        d for d in (p.get("document_id") for p in parties) if d
+    })
+    masters: dict[str, dict] = {}
+    if enrich and doc_ids:
+        # Batch up to 100 doc_ids per request (Socrata IN-list is fine for this).
+        for i in range(0, len(doc_ids), 100):
+            chunk = doc_ids[i : i + 100]
+            id_list = ",".join(f"'{d}'" for d in chunk)
+            master_params = {
+                "$where": f"document_id in ({id_list})",
+                "$limit": "100",
+            }
+            url = f"{MASTER_URL}?{urllib.parse.urlencode(master_params)}"
+            try:
+                rows = get_json(url)
+            except Exception as e:  # noqa: BLE001
+                print(f"ACRIS master lookup failed for chunk: {e}", file=sys.stderr)
+                continue
+            if isinstance(rows, list):
+                for r in rows:
+                    did = r.get("document_id", "")
+                    if did:
+                        masters[did] = r
+
+    out_rows: list[dict[str, str]] = []
+    for p in parties:
+        did = p.get("document_id", "") or ""
+        m = masters.get(did, {})
+        out_rows.append(
+            {
+                "document_id": did,
+                "name": p.get("name", "") or "",
+                "party_type": p.get("party_type", "") or "",
+                "party_role": PARTY_TYPE.get(p.get("party_type", ""), ""),
+                "address_1": p.get("address_1", "") or "",
+                "address_2": p.get("address_2", "") or "",
+                "city": p.get("city", "") or "",
+                "state": p.get("state", "") or "",
+                "zip": p.get("zip", "") or "",
+                "country": p.get("country", "") or "",
+                "doc_type": m.get("doc_type", "") or "",
+                "doc_date": (m.get("document_date", "") or "")[:10],
+                "recorded_date": (m.get("recorded_datetime", "") or "")[:10],
+                "borough": BOROUGH.get(m.get("recorded_borough", ""), m.get("recorded_borough", "")),
+                "amount": m.get("document_amt", "") or "",
+                "filing_url": _filing_url(did),
+            }
+        )
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(out_rows)
+
+    if not out_rows:
+        filters = []
+        if name:
+            filters.append(f"name={name!r}")
+        if address:
+            filters.append(f"address={address!r}")
+        print(
+            f"NYC ACRIS: 0 records for {', '.join(filters)}. "
+            "ACRIS covers ONLY NYC (5 boroughs). For property records elsewhere, "
+            "search the relevant county recorder directly.",
+            file=sys.stderr,
+        )
+    return len(out_rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--name", help="Party name substring (case-insensitive)")
+    p.add_argument("--address", help="Address line 1 substring")
+    p.add_argument(
+        "--party-type",
+        choices=["1", "2", "3"],
+        help="Filter party type: 1=grantor (seller/mortgagor), 2=grantee (buyer/mortgagee), 3=other",
+    )
+    p.add_argument("--limit", type=int, default=200)
+    p.add_argument(
+        "--no-enrich",
+        action="store_true",
+        help="Skip the master-document lookup that adds doc_type/date/amount",
+    )
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    n = fetch(
+        name=a.name,
+        address=a.address,
+        party_type=a.party_type,
+        limit=a.limit,
+        out_path=a.out,
+        enrich=not a.no_enrich,
+    )
+    print(f"Wrote {n} NYC ACRIS rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())