mindforge-cc 11.8.2 → 11.9.0

package/.agent/skills/osint-investigation/scripts/fetch_usaspending.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Fetch federal contracts/awards from USAspending.gov API v2.
+
+No auth required. POST to /api/v2/search/spending_by_award/ with filters.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+ENDPOINT = "https://api.usaspending.gov/api/v2/search/spending_by_award/"
+COLUMNS = [
+    "award_id",
+    "recipient_name",
+    "recipient_uei",
+    "recipient_duns",
+    "recipient_parent_name",
+    "recipient_state",
+    "awarding_agency",
+    "awarding_sub_agency",
+    "award_type",
+    "award_amount",
+    "award_date",
+    "period_of_performance_start",
+    "period_of_performance_end",
+    "naics_code",
+    "psc_code",
+    "competition_extent",
+    "description",
+]
+
+# USAspending result column "code" → human label mapping for output.
+_FIELDS = [
+    "Award ID",
+    "Recipient Name",
+    "Recipient UEI",
+    "Recipient DUNS Number",
+    "Recipient Parent Name",
+    "Recipient State Code",
+    "Awarding Agency",
+    "Awarding Sub Agency",
+    "Award Type",
+    "Award Amount",
+    "Start Date",
+    "End Date",
+    "NAICS Code",
+    "PSC Code",
+    "Type of Set Aside",
+    "Description",
+]
+
+
+def _post(body: dict) -> dict:
+    req = urllib.request.Request(
+        ENDPOINT,
+        data=json.dumps(body).encode("utf-8"),
+        headers={"Content-Type": "application/json", "User-Agent": "hermes-agent osint-investigation"},
+        method="POST",
+    )
+    with urllib.request.urlopen(req, timeout=60) as resp:
+        return json.loads(resp.read().decode("utf-8"))
+
+
+def fetch(
+    recipient: str | None,
+    agency: str | None,
+    fy: int,
+    sole_source_only: bool,
+    out_path: str,
+    page_size: int = 100,
+    max_pages: int = 20,
+) -> int:
+    filters: dict = {
+        "time_period": [{"start_date": f"{fy - 1}-10-01", "end_date": f"{fy}-09-30"}],
+        # Contracts only by default; adjust award_type_codes for grants/loans.
+        "award_type_codes": ["A", "B", "C", "D"],
+    }
+    if recipient:
+        filters["recipient_search_text"] = [recipient]
+    if agency:
+        filters["agencies"] = [{"type": "awarding", "tier": "toptier", "name": agency}]
+
+    rows: list[dict[str, str]] = []
+    page = 1
+    while page <= max_pages:
+        body = {
+            "filters": filters,
+            "fields": _FIELDS,
+            "page": page,
+            "limit": page_size,
+            "sort": "Award Amount",
+            "order": "desc",
+        }
+        try:
+            payload = _post(body)
+        except Exception as e:  # noqa: BLE001
+            print(f"USAspending error on page {page}: {e}", file=sys.stderr)
+            break
+        results = payload.get("results", [])
+        if not results:
+            break
+        for r in results:
+            set_aside = r.get("Type of Set Aside", "") or ""
+            if sole_source_only and "sole" not in set_aside.lower():
+                continue
+            rows.append(
+                {
+                    "award_id": r.get("Award ID", "") or "",
+                    "recipient_name": r.get("Recipient Name", "") or "",
+                    "recipient_uei": r.get("Recipient UEI", "") or "",
+                    "recipient_duns": r.get("Recipient DUNS Number", "") or "",
+                    "recipient_parent_name": r.get("Recipient Parent Name", "") or "",
+                    "recipient_state": r.get("Recipient State Code", "") or "",
+                    "awarding_agency": r.get("Awarding Agency", "") or "",
+                    "awarding_sub_agency": r.get("Awarding Sub Agency", "") or "",
+                    "award_type": r.get("Award Type", "") or "",
+                    "award_amount": str(r.get("Award Amount", "") or ""),
+                    "award_date": r.get("Start Date", "") or "",
+                    "period_of_performance_start": r.get("Start Date", "") or "",
+                    "period_of_performance_end": r.get("End Date", "") or "",
+                    "naics_code": str(r.get("NAICS Code", "") or ""),
+                    "psc_code": str(r.get("PSC Code", "") or ""),
+                    "competition_extent": set_aside,
+                    "description": r.get("Description", "") or "",
+                }
+            )
+        meta = payload.get("page_metadata", {})
+        if not meta.get("hasNext"):
+            break
+        page += 1
+        time.sleep(0.5)
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--recipient", help="Recipient name search")
+    p.add_argument("--agency", help="Awarding agency (top-tier)")
+    p.add_argument("--fy", type=int, default=2024, help="Federal fiscal year")
+    p.add_argument("--sole-source-only", action="store_true")
+    p.add_argument("--max-pages", type=int, default=20)
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    if not (a.recipient or a.agency):
+        p.error("must supply at least one of --recipient / --agency")
+    n = fetch(
+        recipient=a.recipient,
+        agency=a.agency,
+        fy=a.fy,
+        sole_source_only=a.sole_source_only,
+        out_path=a.out,
+        max_pages=a.max_pages,
+    )
+    print(f"Wrote {n} USAspending rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.agent/skills/osint-investigation/scripts/fetch_wayback.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+"""Search the Internet Archive Wayback Machine via the CDX server.
+
+The CDX API indexes ~900B+ archived web pages. Anonymous read access,
+no auth required. Useful for finding deleted / changed pages by URL,
+domain, or substring match.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import sys
+import urllib.parse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _http import get_json  # noqa: E402
+
+BASE = "https://web.archive.org/cdx/search/cdx"
+
+COLUMNS = [
+    "url",
+    "timestamp",
+    "wayback_url",
+    "mimetype",
+    "status",
+    "digest",
+    "length",
+]
+
+
+def fetch(
+    url_or_host: str,
+    match_type: str,
+    from_date: str | None,
+    to_date: str | None,
+    status: str | None,
+    mime: str | None,
+    collapse: str | None,
+    limit: int,
+    out_path: str,
+) -> int:
+    params: dict[str, str] = {
+        "url": url_or_host,
+        "matchType": match_type,
+        "output": "json",
+        "limit": str(limit),
+    }
+    if from_date:
+        params["from"] = from_date.replace("-", "")
+    if to_date:
+        params["to"] = to_date.replace("-", "")
+    if status:
+        params["filter"] = f"statuscode:{status}"
+    if mime:
+        params.setdefault("filter", "")
+        # Multiple filters: CDX accepts repeated filter params via urlencode list
+        params["filter"] = f"mimetype:{mime}"
+    if collapse:
+        params["collapse"] = collapse
+
+    url = f"{BASE}?{urllib.parse.urlencode(params)}"
+    try:
+        payload = get_json(url)
+    except Exception as e:  # noqa: BLE001
+        print(f"Wayback CDX error: {e}", file=sys.stderr)
+        payload = []
+
+    rows: list[dict[str, str]] = []
+    if isinstance(payload, list) and len(payload) > 1:
+        header = payload[0]
+        idx = {h: i for i, h in enumerate(header)}
+        for entry in payload[1:]:
+            ts = entry[idx["timestamp"]] if "timestamp" in idx else ""
+            orig = entry[idx["original"]] if "original" in idx else ""
+            rows.append(
+                {
+                    "url": orig,
+                    "timestamp": ts,
+                    "wayback_url": f"https://web.archive.org/web/{ts}/{orig}" if ts and orig else "",
+                    "mimetype": entry[idx["mimetype"]] if "mimetype" in idx else "",
+                    "status": entry[idx["statuscode"]] if "statuscode" in idx else "",
+                    "digest": entry[idx["digest"]] if "digest" in idx else "",
+                    "length": entry[idx["length"]] if "length" in idx else "",
+                }
+            )
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    if not rows:
+        print(
+            f"Wayback Machine: 0 captures for {url_or_host!r} matchType={match_type}.",
+            file=sys.stderr,
+        )
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--url", required=True, help="URL or host to look up in the archive")
+    p.add_argument(
+        "--match",
+        default="exact",
+        choices=["exact", "prefix", "host", "domain"],
+        help=(
+            "exact: this URL only. "
+            "prefix: this URL's path-prefix. "
+            "host: any URL on this host. "
+            "domain: any URL on this domain or subdomains."
+        ),
+    )
+    p.add_argument("--from-date", help="Earliest capture YYYY-MM-DD")
+    p.add_argument("--to-date", help="Latest capture YYYY-MM-DD")
+    p.add_argument("--status", help="HTTP status filter (e.g. 200)")
+    p.add_argument("--mime", help="MIME type filter (e.g. text/html)")
+    p.add_argument(
+        "--collapse",
+        help="Collapse adjacent identical entries (e.g. 'digest' for unique-content captures)",
+    )
+    p.add_argument("--limit", type=int, default=200)
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    n = fetch(
+        url_or_host=a.url,
+        match_type=a.match,
+        from_date=a.from_date,
+        to_date=a.to_date,
+        status=a.status,
+        mime=a.mime,
+        collapse=a.collapse,
+        limit=a.limit,
+        out_path=a.out,
+    )
+    print(f"Wrote {n} Wayback capture rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/.agent/skills/osint-investigation/scripts/fetch_wikipedia.py

@@ -0,0 +1,266 @@
+#!/usr/bin/env python3
+"""Search Wikipedia + Wikidata for an entity (person, company, place, concept).
+
+Two free APIs:
+  - Wikipedia OpenSearch + REST summary endpoint for narrative bio
+  - Wikidata SPARQL endpoint for structured facts (birth, employer, awards, etc.)
+
+Both are anonymous-access. Useful for resolving who-is-this-entity questions
+and surfacing cross-references that other sources can join against.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import re
+import sys
+import urllib.parse
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+from _http import get_json  # noqa: E402
+
+WP_OPENSEARCH = "https://en.wikipedia.org/w/api.php"
+WP_SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"
+WD_ACTION = "https://www.wikidata.org/w/api.php"
+
+COLUMNS = [
+    "source",
+    "label",
+    "description",
+    "qid",
+    "wikipedia_title",
+    "wikipedia_url",
+    "wikidata_url",
+    "instance_of",
+    "country",
+    "occupation",
+    "employer",
+    "date_of_birth",
+    "place_of_birth",
+    "summary",
+]
+
+
+def _wp_search(query: str, limit: int) -> list[dict]:
+    params = {
+        "action": "opensearch",
+        "search": query,
+        "limit": str(min(limit, 20)),
+        "format": "json",
+    }
+    url = f"{WP_OPENSEARCH}?{urllib.parse.urlencode(params)}"
+    data = get_json(url)
+    if not isinstance(data, list) or len(data) < 4:
+        return []
+    titles, descs, urls = data[1], data[2], data[3]
+    out = []
+    for i, title in enumerate(titles):
+        out.append(
+            {
+                "title": title,
+                "description": descs[i] if i < len(descs) else "",
+                "url": urls[i] if i < len(urls) else "",
+            }
+        )
+    return out
+
+
+def _wp_summary(title: str) -> dict:
+    """Pull the REST summary for a title — short bio, image, type."""
+    url = f"{WP_SUMMARY}{urllib.parse.quote(title.replace(' ', '_'))}"
+    try:
+        return get_json(url)  # type: ignore[return-value]
+    except Exception as e:  # noqa: BLE001
+        print(f"Wikipedia summary lookup for {title!r} failed: {e}", file=sys.stderr)
+        return {}
+
+
+def _wd_lookup_by_qid(qid: str) -> dict:
+    """Pull common facts for a QID via Wikidata's Action API (no SPARQL).
+
+    The Action API is far more lenient on rate-limits than the SPARQL Query
+    Service. We get claims as QIDs and then resolve labels in one batch call.
+    """
+    # Properties of interest. The Action API returns claims as QIDs or
+    # typed literals, so the slot mapping is local-only.
+    interesting = {
+        "P31": "instance_of",
+        "P17": "country",          # for orgs / places
+        "P27": "country",          # for individuals (country of citizenship)
+        "P106": "occupation",
+        "P108": "employer",
+        "P569": "date_of_birth",
+        "P19": "place_of_birth",
+    }
+    params = {
+        "action": "wbgetentities",
+        "ids": qid,
+        "props": "claims",
+        "format": "json",
+    }
+    url = f"{WD_ACTION}?{urllib.parse.urlencode(params)}"
+    try:
+        data = get_json(url)
+    except Exception as e:  # noqa: BLE001
+        print(f"Wikidata wbgetentities for {qid} failed: {e}", file=sys.stderr)
+        return {}
+    if not isinstance(data, dict):
+        return {}
+    claims = (data.get("entities", {}).get(qid, {}) or {}).get("claims", {}) or {}
+
+    # Collect raw values (QIDs or literals) and remember which slot each
+    # came from. Date literals come back as ISO strings; QIDs need a label
+    # resolution pass.
+    qid_to_slots: dict[str, list[str]] = {}
+    facts: dict[str, list[str]] = {}
+    for prop_id, slot in interesting.items():
+        for claim in claims.get(prop_id, []) or []:
+            v = (claim.get("mainsnak", {}) or {}).get("datavalue", {}) or {}
+            vtype = v.get("type")
+            value = v.get("value")
+            if vtype == "wikibase-entityid" and isinstance(value, dict):
+                vqid = value.get("id", "")
+                if vqid:
+                    qid_to_slots.setdefault(vqid, [])
+                    if slot not in qid_to_slots[vqid]:
+                        qid_to_slots[vqid].append(slot)
+            elif vtype == "time" and isinstance(value, dict):
+                raw = value.get("time", "") or ""
+                # +1955-10-28T00:00:00Z → 1955-10-28
+                m = re.search(r"[+-]?(\d{4})-(\d{2})-(\d{2})", raw)
+                if m:
+                    facts.setdefault(slot, []).append(
+                        f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
+                    )
+            elif vtype == "string":
+                facts.setdefault(slot, []).append(str(value))
+
+    # Resolve labels for all referenced QIDs in one batch (up to 50 at a time).
+    qids = list(qid_to_slots)
+    for i in range(0, len(qids), 50):
+        batch = qids[i : i + 50]
+        params = {
+            "action": "wbgetentities",
+            "ids": "|".join(batch),
+            "props": "labels",
+            "languages": "en",
+            "format": "json",
+        }
+        url = f"{WD_ACTION}?{urllib.parse.urlencode(params)}"
+        try:
+            data = get_json(url)
+        except Exception as e:  # noqa: BLE001
+            print(f"Wikidata label batch failed: {e}", file=sys.stderr)
+            continue
+        if not isinstance(data, dict):
+            continue
+        ents = data.get("entities", {}) or {}
+        for vqid, ent in ents.items():
+            label = (ent.get("labels", {}).get("en", {}) or {}).get("value", "") or vqid
+            for slot in qid_to_slots.get(vqid, []):
+                facts.setdefault(slot, []).append(label)
+
+    # Deduplicate per slot, preserving order.
+    deduped: dict[str, list[str]] = {}
+    for slot, vals in facts.items():
+        seen = set()
+        out = []
+        for v in vals:
+            if v in seen:
+                continue
+            seen.add(v)
+            out.append(v)
+        deduped[slot] = out
+    return deduped
+
+
+def _wd_qid_for_title(title: str) -> str:
+    """Get the Wikidata QID associated with a Wikipedia article title."""
+    params = {
+        "action": "query",
+        "format": "json",
+        "prop": "pageprops",
+        "ppprop": "wikibase_item",
+        "titles": title,
+        "redirects": 1,
+    }
+    url = f"{WP_OPENSEARCH}?{urllib.parse.urlencode(params)}"
+    try:
+        data = get_json(url)
+    except Exception:  # noqa: BLE001
+        return ""
+    if not isinstance(data, dict):
+        return ""
+    pages = data.get("query", {}).get("pages", {}) or {}
+    for page in pages.values():
+        qid = (page.get("pageprops") or {}).get("wikibase_item", "")
+        if qid:
+            return qid
+    return ""
+
+
+def fetch(query: str, limit: int, no_wikidata: bool, out_path: str) -> int:
+    hits = _wp_search(query, limit)
+    rows: list[dict[str, str]] = []
+    for hit in hits[:limit]:
+        title = hit.get("title", "")
+        if not title:
+            continue
+        summary = _wp_summary(title)
+        qid = _wd_qid_for_title(title) if not no_wikidata else ""
+        facts: dict = {}
+        if qid:
+            facts = _wd_lookup_by_qid(qid)
+        rows.append(
+            {
+                "source": "wikipedia+wikidata" if qid else "wikipedia",
+                "label": title,
+                "description": (summary.get("description") or hit.get("description") or "").strip(),
+                "qid": qid,
+                "wikipedia_title": title,
+                "wikipedia_url": hit.get("url", ""),
+                "wikidata_url": f"https://www.wikidata.org/wiki/{qid}" if qid else "",
+                "instance_of": "; ".join(facts.get("instance_of", [])),
+                "country": "; ".join(facts.get("country", [])),
+                "occupation": "; ".join(facts.get("occupation", [])),
+                "employer": "; ".join(facts.get("employer", [])),
+                "date_of_birth": "; ".join(facts.get("date_of_birth", []))[:10] if facts.get("date_of_birth") else "",
+                "place_of_birth": "; ".join(facts.get("place_of_birth", [])),
+                "summary": (summary.get("extract") or "").replace("\n", " ")[:1000],
+            }
+        )
+
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=COLUMNS)
+        w.writeheader()
+        w.writerows(rows)
+    if not rows:
+        print(
+            f"Wikipedia: 0 articles for query={query!r}. "
+            "Private individuals not notable enough for a Wikipedia article "
+            "won't appear here (the bar is real).",
+            file=sys.stderr,
+        )
+    return len(rows)
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("--query", required=True, help="Entity name (person, company, place, concept)")
+    p.add_argument("--limit", type=int, default=5)
+    p.add_argument(
+        "--no-wikidata",
+        action="store_true",
+        help="Skip the Wikidata SPARQL enrichment (faster, less detail)",
+    )
+    p.add_argument("--out", required=True)
+    a = p.parse_args()
+    n = fetch(query=a.query, limit=a.limit, no_wikidata=a.no_wikidata, out_path=a.out)
+    print(f"Wrote {n} Wikipedia/Wikidata rows to {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())