@smilintux/skcapstone 0.9.0 → 0.12.5

package/scripts/wargov-ufo-capture/cdp_probe.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Probe war.gov/UFO/ via Lumina Chrome CDP (port 9222).
+
+Steps:
+  1. Open a new tab on war.gov/UFO/
+  2. Wait for Vue mount to load (CSV must be reachable)
+  3. Pull the CSV via in-page fetch
+  4. Inspect inline scripts for any release_2 link patterns
+  5. Save raw CSV + script index to ~/clawd/tmp/wargov-capture/probe-out/
+
+Output:
+  probe-out/uap-csv.csv          fresh CSV from the site
+  probe-out/file-index.json      inline-script link probe
+  probe-out/page-meta.json       URL/title/page render check
+"""
+from __future__ import annotations
+
+import json
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+import websocket  # websocket-client
+
+CDP_HTTP = "http://127.0.0.1:9222"
+TARGET = "https://www.war.gov/UFO/"
+OUT_DIR = Path("/home/cbrd21/clawd/tmp/wargov-capture/probe-out")
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def cdp_get(path: str) -> dict | list:
+    with urllib.request.urlopen(f"{CDP_HTTP}{path}") as r:
+        return json.loads(r.read())
+
+
+def open_tab(url: str) -> dict:
+    # Newer Chrome only accepts PUT on /json/new
+    req = urllib.request.Request(f"{CDP_HTTP}/json/new?{url}", method="PUT")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        return json.loads(r.read())
+
+
+def close_tab(target_id: str) -> None:
+    try:
+        with urllib.request.urlopen(f"{CDP_HTTP}/json/close/{target_id}", timeout=5):
+            pass
+    except Exception:
+        pass
+
+
+class CDP:
+    def __init__(self, ws_url: str):
+        self.ws = websocket.create_connection(ws_url, timeout=60)
+        self.mid = 0
+
+    def call(self, method: str, params: dict | None = None, timeout: float = 30.0) -> dict:
+        self.mid += 1
+        msg_id = self.mid
+        self.ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
+        self.ws.settimeout(timeout)
+        while True:
+            raw = self.ws.recv()
+            msg = json.loads(raw)
+            if msg.get("id") == msg_id:
+                if "error" in msg:
+                    raise RuntimeError(f"{method}: {msg['error']}")
+                return msg.get("result", {})
+
+    def wait_event(self, name: str, timeout: float = 30.0) -> dict:
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            self.ws.settimeout(max(0.1, deadline - time.time()))
+            try:
+                raw = self.ws.recv()
+            except websocket.WebSocketTimeoutException:
+                continue
+            msg = json.loads(raw)
+            if msg.get("method") == name:
+                return msg.get("params", {})
+        raise TimeoutError(f"event {name} did not fire within {timeout}s")
+
+    def close(self) -> None:
+        try:
+            self.ws.close()
+        except Exception:
+            pass
+
+
+def main() -> int:
+    print(f"[probe] opening tab → {TARGET}", flush=True)
+    tab = open_tab(TARGET)
+    target_id = tab["id"]
+    ws_url = tab["webSocketDebuggerUrl"]
+    print(f"[probe] tab id={target_id}", flush=True)
+
+    cdp = CDP(ws_url)
+    try:
+        cdp.call("Page.enable")
+        cdp.call("Runtime.enable")
+        cdp.call("Network.enable", {"maxPostDataSize": 0})
+        cdp.call("Page.navigate", {"url": TARGET})
+        try:
+            cdp.wait_event("Page.loadEventFired", timeout=30.0)
+        except TimeoutError:
+            print("[probe] Page.loadEventFired timeout — proceeding anyway", flush=True)
+
+        # Give the Vue mount a chance to render the CSV view
+        time.sleep(5.0)
+
+        # Page meta
+        meta_js = (
+            "({"
+            "  url: location.href,"
+            "  title: document.title,"
+            "  hasMainContent: !!document.querySelector('main'),"
+            "  scriptInlineCount: document.querySelectorAll('script:not([src])').length,"
+            "  ufoMentions: (document.body.innerText.match(/UAP|UFO|PURSUE/g) || []).length,"
+            "  releaseDateGuesses: Array.from(new Set((document.body.innerText.match(/\\b\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}\\b/g) || []))),"
+            "  release2HrefCount: document.querySelectorAll('a[href*=\"release_2\"]').length,"
+            "  release2InHtml: (document.documentElement.outerHTML.match(/release_2/gi) || []).length"
+            "})"
+        )
+        meta = cdp.call("Runtime.evaluate", {"expression": meta_js, "returnByValue": True})
+        meta_val = meta.get("result", {}).get("value", {})
+        (OUT_DIR / "page-meta.json").write_text(json.dumps(meta_val, indent=2))
+        print(f"[probe] page-meta: {json.dumps(meta_val)}", flush=True)
+
+        # Pull the CSV via in-page fetch
+        csv_js = (
+            "(async () => {"
+            "  const u = '/Portals/1/Interactive/2026/UFO/uap-csv.csv';"
+            "  const r = await fetch(u, {credentials: 'include', cache: 'no-store'});"
+            "  return {status: r.status, len: (await r.clone().text()).length, text: await r.text()};"
+            "})()"
+        )
+        csv_res = cdp.call("Runtime.evaluate", {
+            "expression": csv_js,
+            "awaitPromise": True,
+            "returnByValue": True,
+        }, timeout=60)
+        csv_val = csv_res.get("result", {}).get("value", {})
+        if isinstance(csv_val, dict) and csv_val.get("status") == 200:
+            (OUT_DIR / "uap-csv.csv").write_text(csv_val["text"])
+            print(f"[probe] CSV pulled, {csv_val['len']} bytes", flush=True)
+        else:
+            print(f"[probe] CSV fetch failed: {csv_val}", flush=True)
+            (OUT_DIR / "uap-csv-error.json").write_text(json.dumps(csv_val, indent=2, default=str))
+
+        # Inspect inline scripts for release_2 hints
+        scripts_js = (
+            "(() => {"
+            "  const out = [];"
+            "  document.querySelectorAll('script:not([src])').forEach((s, i) => {"
+            "    const t = s.textContent || '';"
+            "    out.push({idx: i, len: t.length, hasRelease2: /release_2/i.test(t), hasFetch: /fetch\\(/.test(t), hasCsv: /\\.csv/.test(t), preview: t.slice(0, 400)});"
+            "  });"
+            "  return out;"
+            "})()"
+        )
+        scripts_res = cdp.call("Runtime.evaluate", {"expression": scripts_js, "returnByValue": True})
+        scripts_val = scripts_res.get("result", {}).get("value", [])
+        (OUT_DIR / "inline-scripts.json").write_text(json.dumps(scripts_val, indent=2))
+        print(f"[probe] inline scripts: {len(scripts_val)} ({sum(1 for s in scripts_val if s.get('hasRelease2'))} mention release_2)", flush=True)
+
+        # Probe for press release link
+        pr_js = (
+            "(() => {"
+            "  const links = Array.from(document.querySelectorAll('a[href]')).map(a => a.href);"
+            "  const press = links.filter(h => /News\\/Releases/i.test(h));"
+            "  const medialink = links.filter(h => /medialink\\/ufo/i.test(h));"
+            "  return {pressCount: press.length, press: press.slice(0, 20), medialinkCount: medialink.length, medialinkSample: medialink.slice(0, 20)};"
+            "})()"
+        )
+        pr_res = cdp.call("Runtime.evaluate", {"expression": pr_js, "returnByValue": True})
+        pr_val = pr_res.get("result", {}).get("value", {})
+        (OUT_DIR / "link-probe.json").write_text(json.dumps(pr_val, indent=2))
+        print(f"[probe] link probe: press={pr_val.get('pressCount')} medialink={pr_val.get('medialinkCount')}", flush=True)
+
+        print("[probe] DONE", flush=True)
+        return 0
+    finally:
+        cdp.close()
+        close_tab(target_id)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/wargov-ufo-capture/cdp_splc_pressrelease.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Discover + capture the DOJ SPLC press release by NAVIGATING the search page
+(so JS renders the result list) then reading the rendered DOM. Falls back to
+scraping any /opa/pr/ or /news/ links the rendered page exposes.
+"""
+from __future__ import annotations
+import json, re, sys, time, urllib.request
+from pathlib import Path
+import websocket
+
+CDP_HTTP = "http://127.0.0.1:9222"
+SEARCH_URL = "https://www.justice.gov/news?search_api_fulltext=Southern%20Poverty%20Law%20Center"
+OUT = Path("/home/cbrd21/clawd/skills/substance-lens/captures/splc-doj-2026-06-03")
+
+
+def open_tab(url):
+    req = urllib.request.Request(f"{CDP_HTTP}/json/new?{url}", method="PUT")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        return json.loads(r.read())
+
+def close_tab(tid):
+    try:
+        urllib.request.urlopen(f"{CDP_HTTP}/json/close/{tid}", timeout=5)
+    except Exception:
+        pass
+
+class CDP:
+    def __init__(self, ws): self.ws=websocket.create_connection(ws,timeout=120); self.mid=0
+    def call(self, m, p=None, t=60.0):
+        self.mid+=1; i=self.mid
+        self.ws.send(json.dumps({"id":i,"method":m,"params":p or {}})); self.ws.settimeout(t)
+        while True:
+            msg=json.loads(self.ws.recv())
+            if msg.get("id")==i:
+                if "error" in msg: raise RuntimeError(f"{m}: {msg['error']}")
+                return msg.get("result",{})
+    def wait(self, name, t=30.0):
+        end=time.time()+t
+        while time.time()<end:
+            self.ws.settimeout(max(0.1,end-time.time()))
+            try: msg=json.loads(self.ws.recv())
+            except websocket.WebSocketTimeoutException: continue
+            if msg.get("method")==name: return msg.get("params",{})
+        return {}
+    def close(self):
+        try: self.ws.close()
+        except Exception: pass
+
+def jseval(cdp, expr, t=60):
+    r=cdp.call("Runtime.evaluate",{"expression":expr,"awaitPromise":True,"returnByValue":True},t)
+    return r.get("result",{}).get("value")
+
+def fetch_text(cdp,url):
+    expr=(f"(async()=>{{try{{const r=await fetch({json.dumps(url)},{{credentials:'include',cache:'no-store'}});"
+          f"return {{status:r.status,text:await r.text()}};}}catch(e){{return{{status:-1,text:String(e)}};}}}})()")
+    v=jseval(cdp,expr,180) or {}
+    return v.get("status",0), v.get("text","")
+
+def main():
+    tab=open_tab(SEARCH_URL); tid=tab["id"]; cdp=CDP(tab["webSocketDebuggerUrl"])
+    try:
+        cdp.call("Page.enable"); cdp.call("Runtime.enable")
+        cdp.call("Page.navigate",{"url":SEARCH_URL})
+        cdp.wait("Page.loadEventFired",30.0)
+        time.sleep(6.0)  # let result JS render
+        links=jseval(cdp,
+            "JSON.stringify(Array.from(document.querySelectorAll('a[href]'))"
+            ".map(a=>({h:a.getAttribute('href'),t:(a.innerText||'').trim()}))"
+            ".filter(x=>x.h&&(x.h.includes('/opa/pr/')||x.h.includes('/usao-mdal/pr/')||/southern.poverty|law.center|splc/i.test(x.t))))")
+        cands=json.loads(links) if links else []
+        print(f"[pr] rendered candidates: {len(cands)}", flush=True)
+        for c in cands[:15]: print("   ", c["h"], "::", c["t"][:70], flush=True)
+        # pick best
+        pr=None
+        for c in cands:
+            if re.search(r"southern.poverty|law.center|splc|wire.fraud", (c["h"]+c["t"]).lower()):
+                pr=c["h"]; break
+        if not pr and cands: pr=cands[0]["h"]
+        if pr and pr.startswith("/"): pr="https://www.justice.gov"+pr
+        manifest={"search_url":SEARCH_URL,"rendered_candidates":cands[:15],"chosen":pr}
+        if pr:
+            print(f"[pr] fetching -> {pr}", flush=True)
+            st,html=fetch_text(cdp,pr)
+            print(f"[pr] status={st} len={len(html)}", flush=True)
+            if st==200 and html:
+                (OUT/"doj-press-release.html").write_text(html)
+                txt=jseval(cdp,
+                    f"(async()=>{{const r=await fetch({json.dumps(pr)},{{credentials:'include'}});"
+                    f"const h=await r.text();const d=new DOMParser().parseFromString(h,'text/html');"
+                    f"const a=d.querySelector('.field--name-body')||d.querySelector('article')||d.querySelector('main')||d.body;"
+                    f"return a?a.innerText:'';}})()",60) or ""
+                if txt: (OUT/"doj-press-release.txt").write_text(txt); print(f"[pr] {len(txt)} chars text", flush=True)
+                manifest["status"]=st; manifest["txt_chars"]=len(txt)
+        (OUT/"press-release-discovery.json").write_text(json.dumps(manifest,indent=2))
+        print("[pr] done", flush=True)
+        return 0
+    finally:
+        cdp.close(); close_tab(tid)
+
+if __name__=="__main__":
+    sys.exit(main())

package/scripts/wargov-ufo-capture/parse_csv.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Parse the new uap-data.csv and split Release 01 vs Release 02 records.
+
+The CSV has multi-line quoted fields (newlines inside Title and Description Blurb),
+so we use Python's csv module rather than naive line counting.
+"""
+from __future__ import annotations
+
+import csv
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+
+CSV_PATH = Path("/home/cbrd21/nextcloud/cbrd21-share/reference/war-gov-UFO-PURSUE-2026/docs/release-02/uap-data.csv")
+OUT_DIR = Path("/home/cbrd21/clawd/tmp/wargov-capture/probe-out")
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+with CSV_PATH.open(newline="", encoding="utf-8") as f:
+    reader = csv.DictReader(f)
+    rows = [r for r in reader]
+
+print(f"Total records: {len(rows)}")
+
+date_counter = Counter()
+for r in rows:
+    date_counter[(r.get("Release Date") or "").strip()] += 1
+print("Release dates:")
+for d, c in sorted(date_counter.items(), key=lambda x: -x[1]):
+    print(f"  {d!r:15} → {c}")
+
+# Filter for Release 02
+release2 = [r for r in rows if (r.get("Release Date") or "").strip() == "5/22/26"]
+print(f"\nRelease 02 records: {len(release2)}")
+
+# Bucket by type
+type_counter = Counter()
+agency_counter = Counter()
+for r in release2:
+    type_counter[(r.get("Type") or "").strip()] += 1
+    agency_counter[(r.get("Agency") or "").strip()] += 1
+print("Types:")
+for t, c in type_counter.most_common():
+    print(f"  {t!r:15} → {c}")
+print("Agencies:")
+for a, c in agency_counter.most_common():
+    print(f"  {a!r:15} → {c}")
+
+# Extract download links
+links = []
+for r in release2:
+    pdf_link = (r.get("PDF | Image Link") or "").strip()
+    modal = (r.get("Modal Image") or "").strip()
+    dvids = (r.get("DVIDS Video ID") or "").strip()
+    title = (r.get("Title") or "").strip().replace("\n", " ").replace("\r", "")
+    rtype = (r.get("Type") or "").strip()
+    agency = (r.get("Agency") or "").strip()
+    incident_date = (r.get("Incident Date") or "").strip()
+    incident_loc = (r.get("Incident Location") or "").strip()
+    links.append({
+        "title": title,
+        "type": rtype,
+        "agency": agency,
+        "incident_date": incident_date,
+        "incident_location": incident_loc,
+        "pdf_link": pdf_link,
+        "modal_image": modal,
+        "dvids_id": dvids,
+    })
+
+# Save full inventory
+(OUT_DIR / "release-02-records.json").write_text(json.dumps(links, indent=2))
+print(f"\nSaved inventory: {OUT_DIR / 'release-02-records.json'}")
+
+# Unique direct-fetchable URLs
+urls = set()
+for L in links:
+    if L["pdf_link"]:
+        urls.add(L["pdf_link"])
+    if L["modal_image"]:
+        urls.add(L["modal_image"])
+urls_list = sorted(urls)
+print(f"\nUnique direct URLs: {len(urls_list)}")
+for u in urls_list[:15]:
+    print(f"  {u}")
+if len(urls_list) > 15:
+    print(f"  ... and {len(urls_list) - 15} more")
+
+(OUT_DIR / "release-02-urls.json").write_text(json.dumps(urls_list, indent=2))
+
+# DVIDS-only records (videos hosted exclusively on DVIDS)
+dvids_only = [L for L in links if L["dvids_id"] and not L["pdf_link"]]
+print(f"\nDVIDS-only video records: {len(dvids_only)}")
+for L in dvids_only[:10]:
+    print(f"  DVIDS {L['dvids_id']}: {L['title'][:80]}")
+(OUT_DIR / "release-02-dvids.json").write_text(json.dumps(dvids_only, indent=2))

package/scripts/wargov-ufo-capture/pull_dvids.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Pull DVIDS media for Release 02 in parallel (up to 4 at a time).
+# Reads inventory from probe-out/release-02-records.json.
+
+set -u
+OUT_DIR="$HOME/nextcloud/cbrd21-share/reference/war-gov-UFO-PURSUE-2026/release-02"
+INVENTORY="$HOME/clawd/tmp/wargov-capture/probe-out/release-02-records.json"
+LOG="$HOME/clawd/tmp/wargov-capture/probe-out/dvids-pull.log"
+mkdir -p "$OUT_DIR"
+: > "$LOG"
+
+pull_one() {
+  local dvids_id="$1"
+  local title_slug="$2"
+  local kind="$3"  # video or audio
+  local out_path="$OUT_DIR/dvids-${kind}-${dvids_id}-${title_slug}"
+
+  # Pick the right URL prefix
+  local page_url
+  if [[ "$kind" == "audio" ]]; then
+    page_url="https://www.dvidshub.net/audio/${dvids_id}"
+  else
+    page_url="https://www.dvidshub.net/video/${dvids_id}"
+  fi
+
+  local page_html
+  page_html=$(curl -sSL --max-time 60 "$page_url" 2>/dev/null) || {
+    echo "[FAIL fetch page] dvids=$dvids_id" | tee -a "$LOG"
+    return 1
+  }
+
+  # Extract mp4 (video) or mp3 (audio) CDN URL
+  local media_url ext
+  if [[ "$kind" == "audio" ]]; then
+    media_url=$(echo "$page_html" | grep -oE 'https?://[^"]+\.mp3[^"]*' | head -1)
+    ext="mp3"
+    # DVIDS audio sometimes is .m4a or hosted via a different path; fallback to grepping for asset URL
+    if [[ -z "$media_url" ]]; then
+      media_url=$(echo "$page_html" | grep -oE 'https?://[^"]+\.m4a[^"]*' | head -1)
+      ext="m4a"
+    fi
+    if [[ -z "$media_url" ]]; then
+      media_url=$(echo "$page_html" | grep -oE 'https?://d34w7g4gy10iej\.cloudfront\.net/[^"]+' | head -1)
+      ext="${media_url##*.}"
+    fi
+  else
+    media_url=$(echo "$page_html" | grep -oE 'https?://[^"]+\.mp4[^"]*' | head -1)
+    ext="mp4"
+  fi
+
+  if [[ -z "$media_url" ]]; then
+    echo "[FAIL no-media-url] dvids=$dvids_id kind=$kind" | tee -a "$LOG"
+    # Save the page HTML for inspection
+    echo "$page_html" > "${out_path}.html"
+    return 1
+  fi
+
+  local final_path="${out_path}.${ext}"
+  if [[ -f "$final_path" ]] && [[ -s "$final_path" ]]; then
+    echo "[SKIP already-have] $final_path" | tee -a "$LOG"
+    return 0
+  fi
+
+  curl -sSL --max-time 600 -o "$final_path" "$media_url" 2>/dev/null
+  local size
+  size=$(stat -c '%s' "$final_path" 2>/dev/null || echo 0)
+  if [[ "$size" -lt 1024 ]]; then
+    echo "[FAIL download too-small=$size] dvids=$dvids_id url=$media_url" | tee -a "$LOG"
+    return 1
+  fi
+  echo "[OK] dvids=$dvids_id kind=$kind size=$((size/1024))KB → $(basename "$final_path")" | tee -a "$LOG"
+}
+
+# Generate worker commands from the inventory using python
+python3 - <<'PY' > /tmp/wargov-dvids-jobs.txt
+import json, re
+with open("/home/cbrd21/clawd/tmp/wargov-capture/probe-out/release-02-records.json") as f:
+    records = json.load(f)
+for r in records:
+    if not r["dvids_id"]:
+        continue
+    kind = "audio" if r["type"] == "AUD" else "video"
+    # Slugify title: strip quotes, collapse non-alnum to dashes, limit length
+    title = r["title"]
+    slug = re.sub(r'[^a-zA-Z0-9]+', '-', title).strip('-').lower()[:60]
+    # Use the DOW-UAP-PR id from the title if available (more durable)
+    m = re.match(r'([A-Z]+-UAP-(?:PR|D)[0-9]+[a-z]?)', title)
+    if m:
+        slug = m.group(1).lower() + "-" + slug[:30]
+    print(f"{r['dvids_id']}\t{slug}\t{kind}")
+PY
+
+JOB_COUNT=$(wc -l < /tmp/wargov-dvids-jobs.txt)
+echo "[plan] $JOB_COUNT DVIDS jobs queued" | tee -a "$LOG"
+
+export -f pull_one
+export OUT_DIR LOG
+
+# Run with xargs -P 4 (parallelism 4)
+cat /tmp/wargov-dvids-jobs.txt | while IFS=$'\t' read -r id slug kind; do
+  echo "$id $slug $kind"
+done | xargs -L 1 -P 4 -I {} bash -c 'set -- {}; pull_one "$1" "$2" "$3"'
+
+OK_COUNT=$(grep -c '^\[OK\]' "$LOG" || true)
+FAIL_COUNT=$(grep -cE '^\[FAIL' "$LOG" || true)
+SKIP_COUNT=$(grep -c '^\[SKIP' "$LOG" || true)
+echo "[done] OK=$OK_COUNT FAIL=$FAIL_COUNT SKIP=$SKIP_COUNT" | tee -a "$LOG"