@smilintux/skcapstone 0.10.0 → 0.12.5

package/scripts/wargov-ufo-capture/cdp_finish.py

@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""CDP run 2: re-extract press release text, pull thumbnails, grab FBI Vault Part 15.
+
+Three sub-tasks:
+  A. Re-render the press release in a Chrome tab and pull the article-body innerText.
+  B. Page-context-fetch the 6 thumbnail JPGs for Release 02.
+  C. Navigate to FBI Vault and pull Part 15 of 16 from the 62-HQ-83894 series.
+"""
+from __future__ import annotations
+
+import base64
+import json
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+import websocket
+
+CDP_HTTP = "http://127.0.0.1:9222"
+
+PRESS_URL = "https://www.war.gov/News/Releases/Release/Article/4499305/department-of-war-publishes-second-release-of-unidentified-anomalous-phenomena/"
+THUMB_BASE = "https://www.war.gov/medialink/ufo/052226/release_02/thumbnails"
+THUMB_NAMES = [
+    "CIA-UAP-D001_Intelligence_Information_Report_USSR_1973",
+    "DOE-UAP-D001_PANTEX_Image",
+    "DOE-UAP-D002_JamesTuck_Correspondence",
+    "DOE-UAP-D003_Pajarito_Astronomers",
+    "DOW-UAP-D017_General_Correspondence_Of_Sandia",
+    "ODNI-UAP-D001_USPER_Narrative_Senior_USIC",
+]
+
+FBI_VAULT_BASE = "https://vault.fbi.gov"
+# FBI Vault organizes the 62-HQ-83894 UFO file as "Unidentified Flying Objects (UFO)" — Part X of Y
+# Known canonical layout has Parts 1-16. Tweet referenced Part 15.
+FBI_PART_PAGE = "https://vault.fbi.gov/UFO/UFO%20Part%2015%20of%2016/view"
+FBI_PART_PDF_GUESS = "https://vault.fbi.gov/UFO/UFO%20Part%2015%20of%2016/at_download/file"
+
+BASE = Path("/home/cbrd21/nextcloud/cbrd21-share/reference/war-gov-UFO-PURSUE-2026")
+DOC_DIR = BASE / "docs" / "release-02"
+THUMB_DIR = DOC_DIR / "thumbnails"
+THUMB_DIR.mkdir(parents=True, exist_ok=True)
+
+FBI_DIR = Path("/home/cbrd21/nextcloud/cbrd21-share/reference/fbi-vault-ufo-62-HQ-83894")
+FBI_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def open_tab(url: str) -> dict:
+    req = urllib.request.Request(f"{CDP_HTTP}/json/new?{url}", method="PUT")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        return json.loads(r.read())
+
+
+def close_tab(target_id: str) -> None:
+    try:
+        with urllib.request.urlopen(f"{CDP_HTTP}/json/close/{target_id}", timeout=5):
+            pass
+    except Exception:
+        pass
+
+
+class CDP:
+    def __init__(self, ws_url: str):
+        self.ws = websocket.create_connection(ws_url, timeout=120)
+        self.mid = 0
+
+    def call(self, method: str, params: dict | None = None, timeout: float = 60.0) -> dict:
+        self.mid += 1
+        msg_id = self.mid
+        self.ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
+        self.ws.settimeout(timeout)
+        while True:
+            raw = self.ws.recv()
+            msg = json.loads(raw)
+            if msg.get("id") == msg_id:
+                if "error" in msg:
+                    raise RuntimeError(f"{method}: {msg['error']}")
+                return msg.get("result", {})
+
+    def wait_event(self, name: str, timeout: float = 30.0) -> dict:
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            self.ws.settimeout(max(0.1, deadline - time.time()))
+            try:
+                raw = self.ws.recv()
+            except websocket.WebSocketTimeoutException:
+                continue
+            msg = json.loads(raw)
+            if msg.get("method") == name:
+                return msg.get("params", {})
+        raise TimeoutError(f"event {name} did not fire within {timeout}s")
+
+    def close(self) -> None:
+        try:
+            self.ws.close()
+        except Exception:
+            pass
+
+
+def fetch_binary_in_page(cdp: CDP, url: str) -> tuple[int, bytes | None]:
+    """Fetch a binary resource in page context and return as bytes."""
+    expr = (
+        f"(async () => {{"
+        f"  const r = await fetch({json.dumps(url)}, {{credentials: 'include', cache: 'no-store'}});"
+        f"  if (!r.ok) return {{status: r.status, b64: null}};"
+        f"  const buf = await r.arrayBuffer();"
+        f"  const bytes = new Uint8Array(buf);"
+        f"  let bin = '';"
+        f"  for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]);"
+        f"  return {{status: r.status, b64: btoa(bin), bytes: bytes.length}};"
+        f"}})()"
+    )
+    res = cdp.call("Runtime.evaluate", {
+        "expression": expr,
+        "awaitPromise": True,
+        "returnByValue": True,
+    }, timeout=300)
+    val = res.get("result", {}).get("value", {}) or {}
+    status = val.get("status", 0)
+    b64 = val.get("b64")
+    if status == 200 and b64:
+        return status, base64.b64decode(b64)
+    return status, None
+
+
+def fetch_text_in_page(cdp: CDP, url: str) -> tuple[int, str]:
+    expr = (
+        f"(async () => {{"
+        f"  const r = await fetch({json.dumps(url)}, {{credentials: 'include', cache: 'no-store'}});"
+        f"  return {{status: r.status, text: await r.text()}};"
+        f"}})()"
+    )
+    res = cdp.call("Runtime.evaluate", {
+        "expression": expr,
+        "awaitPromise": True,
+        "returnByValue": True,
+    }, timeout=120)
+    val = res.get("result", {}).get("value", {}) or {}
+    return val.get("status", 0), val.get("text", "")
+
+
+def task_a_press_release(cdp: CDP) -> None:
+    """Navigate to press release, extract innerText from main article."""
+    print(f"[A] navigating → press release", flush=True)
+    cdp.call("Page.navigate", {"url": PRESS_URL})
+    try:
+        cdp.wait_event("Page.loadEventFired", timeout=30.0)
+    except TimeoutError:
+        pass
+    time.sleep(3.0)
+
+    # Try multiple candidate selectors; press releases on DoW use various article wrappers
+    extract_js = (
+        "(() => {"
+        "  const candidates = ["
+        "    document.querySelector('.body-text'),"
+        "    document.querySelector('.article-body'),"
+        "    document.querySelector('.article-content'),"
+        "    document.querySelector('.press-release'),"
+        "    document.querySelector('main article'),"
+        "    document.querySelector('main .content'),"
+        "    document.querySelector('main'),"
+        "    document.querySelector('article'),"
+        "  ];"
+        "  for (const el of candidates) {"
+        "    if (el && el.innerText && el.innerText.length > 500) {"
+        "      return {selector: el.tagName + (el.className ? '.' + el.className.split(' ').join('.') : ''), text: el.innerText, len: el.innerText.length};"
+        "    }"
+        "  }"
+        "  // Last resort: full body innerText"
+        "  return {selector: 'body', text: document.body.innerText, len: document.body.innerText.length};"
+        "})()"
+    )
+    res = cdp.call("Runtime.evaluate", {"expression": extract_js, "returnByValue": True})
+    val = res.get("result", {}).get("value", {}) or {}
+    text = val.get("text", "")
+    print(f"[A] selector={val.get('selector')!r} len={val.get('len')}", flush=True)
+    if text:
+        (DOC_DIR / "press-release-2026-05-22.txt").write_text(text)
+        print(f"[A] wrote press-release-2026-05-22.txt ({len(text)} chars)", flush=True)
+
+
+def task_b_thumbnails(cdp: CDP) -> None:
+    """Page-context-fetch all 6 PDF thumbnails."""
+    print(f"[B] pulling {len(THUMB_NAMES)} thumbnails via in-page fetch", flush=True)
+    # Make sure we're on a war.gov tab so credentials/Akamai cookies apply
+    cdp.call("Page.navigate", {"url": "https://www.war.gov/UFO/"})
+    try:
+        cdp.wait_event("Page.loadEventFired", timeout=30.0)
+    except TimeoutError:
+        pass
+    time.sleep(2.0)
+    for name in THUMB_NAMES:
+        url = f"{THUMB_BASE}/{name}.jpg"
+        status, content = fetch_binary_in_page(cdp, url)
+        out_path = THUMB_DIR / f"{name}.jpg"
+        if status == 200 and content:
+            out_path.write_bytes(content)
+            print(f"[B]   OK {name}.jpg {len(content)} bytes", flush=True)
+        else:
+            print(f"[B]   FAIL {name}.jpg status={status}", flush=True)
+
+
+def task_c_fbi_vault_part_15(cdp: CDP) -> None:
+    """Try to fetch FBI Vault UFO Part 15 of 16."""
+    print(f"[C] navigating → FBI Vault Part 15 page", flush=True)
+    cdp.call("Page.navigate", {"url": FBI_PART_PAGE})
+    try:
+        cdp.wait_event("Page.loadEventFired", timeout=30.0)
+    except TimeoutError:
+        pass
+    time.sleep(3.0)
+
+    # Try to find the PDF link on the page (Plone reading-room standard pattern)
+    link_js = (
+        "(() => {"
+        "  const links = Array.from(document.querySelectorAll('a[href]')).map(a => a.href);"
+        "  const pdfish = links.filter(h => /\\.pdf(\\?|$)|at_download\\/file/i.test(h));"
+        "  return {title: document.title, total: links.length, pdfish: pdfish.slice(0, 10)};"
+        "})()"
+    )
+    res = cdp.call("Runtime.evaluate", {"expression": link_js, "returnByValue": True})
+    link_val = res.get("result", {}).get("value", {}) or {}
+    print(f"[C] page info: {json.dumps(link_val)}", flush=True)
+
+    pdf_url = None
+    for h in link_val.get("pdfish", []):
+        if "at_download/file" in h or h.lower().endswith(".pdf"):
+            pdf_url = h
+            break
+    if not pdf_url:
+        pdf_url = FBI_PART_PDF_GUESS
+        print(f"[C] using guess URL → {pdf_url}", flush=True)
+
+    print(f"[C] page-context fetch → {pdf_url}", flush=True)
+    status, content = fetch_binary_in_page(cdp, pdf_url)
+    if status == 200 and content:
+        out_path = FBI_DIR / "UFO-Part-15-of-16.pdf"
+        out_path.write_bytes(content)
+        print(f"[C] OK {out_path.name} {len(content)/1e6:.1f} MB", flush=True)
+    else:
+        # Maybe the page itself IS the PDF (some Vault items)
+        print(f"[C] direct fetch failed status={status}; trying alternate URLs", flush=True)
+        # Save the page HTML for inspection
+        html_status, html_text = fetch_text_in_page(cdp, FBI_PART_PAGE)
+        (FBI_DIR / "part-15-page.html").write_text(html_text or "")
+        print(f"[C] saved page HTML for inspection ({len(html_text)} chars)", flush=True)
+
+
+def main() -> int:
+    tab = open_tab("about:blank")
+    target_id = tab["id"]
+    ws_url = tab["webSocketDebuggerUrl"]
+    cdp = CDP(ws_url)
+    try:
+        cdp.call("Page.enable")
+        cdp.call("Runtime.enable")
+        cdp.call("Network.enable", {"maxPostDataSize": 0})
+
+        task_a_press_release(cdp)
+        task_b_thumbnails(cdp)
+        task_c_fbi_vault_part_15(cdp)
+
+        return 0
+    finally:
+        cdp.close()
+        close_tab(target_id)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/wargov-ufo-capture/cdp_probe.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Probe war.gov/UFO/ via Lumina Chrome CDP (port 9222).
+
+Steps:
+  1. Open a new tab on war.gov/UFO/
+  2. Wait for Vue mount to load (CSV must be reachable)
+  3. Pull the CSV via in-page fetch
+  4. Inspect inline scripts for any release_2 link patterns
+  5. Save raw CSV + script index to ~/clawd/tmp/wargov-capture/probe-out/
+
+Output:
+  probe-out/uap-csv.csv          fresh CSV from the site
+  probe-out/file-index.json      inline-script link probe
+  probe-out/page-meta.json       URL/title/page render check
+"""
+from __future__ import annotations
+
+import json
+import sys
+import time
+import urllib.request
+from pathlib import Path
+
+import websocket  # websocket-client
+
+CDP_HTTP = "http://127.0.0.1:9222"
+TARGET = "https://www.war.gov/UFO/"
+OUT_DIR = Path("/home/cbrd21/clawd/tmp/wargov-capture/probe-out")
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def cdp_get(path: str) -> dict | list:
+    with urllib.request.urlopen(f"{CDP_HTTP}{path}") as r:
+        return json.loads(r.read())
+
+
+def open_tab(url: str) -> dict:
+    # Newer Chrome only accepts PUT on /json/new
+    req = urllib.request.Request(f"{CDP_HTTP}/json/new?{url}", method="PUT")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        return json.loads(r.read())
+
+
+def close_tab(target_id: str) -> None:
+    try:
+        with urllib.request.urlopen(f"{CDP_HTTP}/json/close/{target_id}", timeout=5):
+            pass
+    except Exception:
+        pass
+
+
+class CDP:
+    def __init__(self, ws_url: str):
+        self.ws = websocket.create_connection(ws_url, timeout=60)
+        self.mid = 0
+
+    def call(self, method: str, params: dict | None = None, timeout: float = 30.0) -> dict:
+        self.mid += 1
+        msg_id = self.mid
+        self.ws.send(json.dumps({"id": msg_id, "method": method, "params": params or {}}))
+        self.ws.settimeout(timeout)
+        while True:
+            raw = self.ws.recv()
+            msg = json.loads(raw)
+            if msg.get("id") == msg_id:
+                if "error" in msg:
+                    raise RuntimeError(f"{method}: {msg['error']}")
+                return msg.get("result", {})
+
+    def wait_event(self, name: str, timeout: float = 30.0) -> dict:
+        deadline = time.time() + timeout
+        while time.time() < deadline:
+            self.ws.settimeout(max(0.1, deadline - time.time()))
+            try:
+                raw = self.ws.recv()
+            except websocket.WebSocketTimeoutException:
+                continue
+            msg = json.loads(raw)
+            if msg.get("method") == name:
+                return msg.get("params", {})
+        raise TimeoutError(f"event {name} did not fire within {timeout}s")
+
+    def close(self) -> None:
+        try:
+            self.ws.close()
+        except Exception:
+            pass
+
+
+def main() -> int:
+    print(f"[probe] opening tab → {TARGET}", flush=True)
+    tab = open_tab(TARGET)
+    target_id = tab["id"]
+    ws_url = tab["webSocketDebuggerUrl"]
+    print(f"[probe] tab id={target_id}", flush=True)
+
+    cdp = CDP(ws_url)
+    try:
+        cdp.call("Page.enable")
+        cdp.call("Runtime.enable")
+        cdp.call("Network.enable", {"maxPostDataSize": 0})
+        cdp.call("Page.navigate", {"url": TARGET})
+        try:
+            cdp.wait_event("Page.loadEventFired", timeout=30.0)
+        except TimeoutError:
+            print("[probe] Page.loadEventFired timeout — proceeding anyway", flush=True)
+
+        # Give the Vue mount a chance to render the CSV view
+        time.sleep(5.0)
+
+        # Page meta
+        meta_js = (
+            "({"
+            "  url: location.href,"
+            "  title: document.title,"
+            "  hasMainContent: !!document.querySelector('main'),"
+            "  scriptInlineCount: document.querySelectorAll('script:not([src])').length,"
+            "  ufoMentions: (document.body.innerText.match(/UAP|UFO|PURSUE/g) || []).length,"
+            "  releaseDateGuesses: Array.from(new Set((document.body.innerText.match(/\\b\\d{1,2}\\/\\d{1,2}\\/\\d{2,4}\\b/g) || []))),"
+            "  release2HrefCount: document.querySelectorAll('a[href*=\"release_2\"]').length,"
+            "  release2InHtml: (document.documentElement.outerHTML.match(/release_2/gi) || []).length"
+            "})"
+        )
+        meta = cdp.call("Runtime.evaluate", {"expression": meta_js, "returnByValue": True})
+        meta_val = meta.get("result", {}).get("value", {})
+        (OUT_DIR / "page-meta.json").write_text(json.dumps(meta_val, indent=2))
+        print(f"[probe] page-meta: {json.dumps(meta_val)}", flush=True)
+
+        # Pull the CSV via in-page fetch
+        csv_js = (
+            "(async () => {"
+            "  const u = '/Portals/1/Interactive/2026/UFO/uap-csv.csv';"
+            "  const r = await fetch(u, {credentials: 'include', cache: 'no-store'});"
+            "  return {status: r.status, len: (await r.clone().text()).length, text: await r.text()};"
+            "})()"
+        )
+        csv_res = cdp.call("Runtime.evaluate", {
+            "expression": csv_js,
+            "awaitPromise": True,
+            "returnByValue": True,
+        }, timeout=60)
+        csv_val = csv_res.get("result", {}).get("value", {})
+        if isinstance(csv_val, dict) and csv_val.get("status") == 200:
+            (OUT_DIR / "uap-csv.csv").write_text(csv_val["text"])
+            print(f"[probe] CSV pulled, {csv_val['len']} bytes", flush=True)
+        else:
+            print(f"[probe] CSV fetch failed: {csv_val}", flush=True)
+            (OUT_DIR / "uap-csv-error.json").write_text(json.dumps(csv_val, indent=2, default=str))
+
+        # Inspect inline scripts for release_2 hints
+        scripts_js = (
+            "(() => {"
+            "  const out = [];"
+            "  document.querySelectorAll('script:not([src])').forEach((s, i) => {"
+            "    const t = s.textContent || '';"
+            "    out.push({idx: i, len: t.length, hasRelease2: /release_2/i.test(t), hasFetch: /fetch\\(/.test(t), hasCsv: /\\.csv/.test(t), preview: t.slice(0, 400)});"
+            "  });"
+            "  return out;"
+            "})()"
+        )
+        scripts_res = cdp.call("Runtime.evaluate", {"expression": scripts_js, "returnByValue": True})
+        scripts_val = scripts_res.get("result", {}).get("value", [])
+        (OUT_DIR / "inline-scripts.json").write_text(json.dumps(scripts_val, indent=2))
+        print(f"[probe] inline scripts: {len(scripts_val)} ({sum(1 for s in scripts_val if s.get('hasRelease2'))} mention release_2)", flush=True)
+
+        # Probe for press release link
+        pr_js = (
+            "(() => {"
+            "  const links = Array.from(document.querySelectorAll('a[href]')).map(a => a.href);"
+            "  const press = links.filter(h => /News\\/Releases/i.test(h));"
+            "  const medialink = links.filter(h => /medialink\\/ufo/i.test(h));"
+            "  return {pressCount: press.length, press: press.slice(0, 20), medialinkCount: medialink.length, medialinkSample: medialink.slice(0, 20)};"
+            "})()"
+        )
+        pr_res = cdp.call("Runtime.evaluate", {"expression": pr_js, "returnByValue": True})
+        pr_val = pr_res.get("result", {}).get("value", {})
+        (OUT_DIR / "link-probe.json").write_text(json.dumps(pr_val, indent=2))
+        print(f"[probe] link probe: press={pr_val.get('pressCount')} medialink={pr_val.get('medialinkCount')}", flush=True)
+
+        print("[probe] DONE", flush=True)
+        return 0
+    finally:
+        cdp.close()
+        close_tab(target_id)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/wargov-ufo-capture/cdp_splc_pressrelease.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""Discover + capture the DOJ SPLC press release by NAVIGATING the search page
+(so JS renders the result list) then reading the rendered DOM. Falls back to
+scraping any /opa/pr/ or /news/ links the rendered page exposes.
+"""
+from __future__ import annotations
+import json, re, sys, time, urllib.request
+from pathlib import Path
+import websocket
+
+CDP_HTTP = "http://127.0.0.1:9222"
+SEARCH_URL = "https://www.justice.gov/news?search_api_fulltext=Southern%20Poverty%20Law%20Center"
+OUT = Path("/home/cbrd21/clawd/skills/substance-lens/captures/splc-doj-2026-06-03")
+
+
+def open_tab(url):
+    req = urllib.request.Request(f"{CDP_HTTP}/json/new?{url}", method="PUT")
+    with urllib.request.urlopen(req, timeout=10) as r:
+        return json.loads(r.read())
+
+def close_tab(tid):
+    try:
+        urllib.request.urlopen(f"{CDP_HTTP}/json/close/{tid}", timeout=5)
+    except Exception:
+        pass
+
+class CDP:
+    def __init__(self, ws): self.ws=websocket.create_connection(ws,timeout=120); self.mid=0
+    def call(self, m, p=None, t=60.0):
+        self.mid+=1; i=self.mid
+        self.ws.send(json.dumps({"id":i,"method":m,"params":p or {}})); self.ws.settimeout(t)
+        while True:
+            msg=json.loads(self.ws.recv())
+            if msg.get("id")==i:
+                if "error" in msg: raise RuntimeError(f"{m}: {msg['error']}")
+                return msg.get("result",{})
+    def wait(self, name, t=30.0):
+        end=time.time()+t
+        while time.time()<end:
+            self.ws.settimeout(max(0.1,end-time.time()))
+            try: msg=json.loads(self.ws.recv())
+            except websocket.WebSocketTimeoutException: continue
+            if msg.get("method")==name: return msg.get("params",{})
+        return {}
+    def close(self):
+        try: self.ws.close()
+        except Exception: pass
+
+def jseval(cdp, expr, t=60):
+    r=cdp.call("Runtime.evaluate",{"expression":expr,"awaitPromise":True,"returnByValue":True},t)
+    return r.get("result",{}).get("value")
+
+def fetch_text(cdp,url):
+    expr=(f"(async()=>{{try{{const r=await fetch({json.dumps(url)},{{credentials:'include',cache:'no-store'}});"
+          f"return {{status:r.status,text:await r.text()}};}}catch(e){{return{{status:-1,text:String(e)}};}}}})()")
+    v=jseval(cdp,expr,180) or {}
+    return v.get("status",0), v.get("text","")
+
+def main():
+    tab=open_tab(SEARCH_URL); tid=tab["id"]; cdp=CDP(tab["webSocketDebuggerUrl"])
+    try:
+        cdp.call("Page.enable"); cdp.call("Runtime.enable")
+        cdp.call("Page.navigate",{"url":SEARCH_URL})
+        cdp.wait("Page.loadEventFired",30.0)
+        time.sleep(6.0)  # let result JS render
+        links=jseval(cdp,
+            "JSON.stringify(Array.from(document.querySelectorAll('a[href]'))"
+            ".map(a=>({h:a.getAttribute('href'),t:(a.innerText||'').trim()}))"
+            ".filter(x=>x.h&&(x.h.includes('/opa/pr/')||x.h.includes('/usao-mdal/pr/')||/southern.poverty|law.center|splc/i.test(x.t))))")
+        cands=json.loads(links) if links else []
+        print(f"[pr] rendered candidates: {len(cands)}", flush=True)
+        for c in cands[:15]: print("   ", c["h"], "::", c["t"][:70], flush=True)
+        # pick best
+        pr=None
+        for c in cands:
+            if re.search(r"southern.poverty|law.center|splc|wire.fraud", (c["h"]+c["t"]).lower()):
+                pr=c["h"]; break
+        if not pr and cands: pr=cands[0]["h"]
+        if pr and pr.startswith("/"): pr="https://www.justice.gov"+pr
+        manifest={"search_url":SEARCH_URL,"rendered_candidates":cands[:15],"chosen":pr}
+        if pr:
+            print(f"[pr] fetching -> {pr}", flush=True)
+            st,html=fetch_text(cdp,pr)
+            print(f"[pr] status={st} len={len(html)}", flush=True)
+            if st==200 and html:
+                (OUT/"doj-press-release.html").write_text(html)
+                txt=jseval(cdp,
+                    f"(async()=>{{const r=await fetch({json.dumps(pr)},{{credentials:'include'}});"
+                    f"const h=await r.text();const d=new DOMParser().parseFromString(h,'text/html');"
+                    f"const a=d.querySelector('.field--name-body')||d.querySelector('article')||d.querySelector('main')||d.body;"
+                    f"return a?a.innerText:'';}})()",60) or ""
+                if txt: (OUT/"doj-press-release.txt").write_text(txt); print(f"[pr] {len(txt)} chars text", flush=True)
+                manifest["status"]=st; manifest["txt_chars"]=len(txt)
+        (OUT/"press-release-discovery.json").write_text(json.dumps(manifest,indent=2))
+        print("[pr] done", flush=True)
+        return 0
+    finally:
+        cdp.close(); close_tab(tid)
+
+if __name__=="__main__":
+    sys.exit(main())

package/scripts/wargov-ufo-capture/parse_csv.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Parse the new uap-data.csv and split Release 01 vs Release 02 records.
+
+The CSV has multi-line quoted fields (newlines inside Title and Description Blurb),
+so we use Python's csv module rather than naive line counting.
+"""
+from __future__ import annotations
+
+import csv
+import json
+from collections import Counter, defaultdict
+from pathlib import Path
+
+CSV_PATH = Path("/home/cbrd21/nextcloud/cbrd21-share/reference/war-gov-UFO-PURSUE-2026/docs/release-02/uap-data.csv")
+OUT_DIR = Path("/home/cbrd21/clawd/tmp/wargov-capture/probe-out")
+OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+with CSV_PATH.open(newline="", encoding="utf-8") as f:
+    reader = csv.DictReader(f)
+    rows = [r for r in reader]
+
+print(f"Total records: {len(rows)}")
+
+date_counter = Counter()
+for r in rows:
+    date_counter[(r.get("Release Date") or "").strip()] += 1
+print("Release dates:")
+for d, c in sorted(date_counter.items(), key=lambda x: -x[1]):
+    print(f"  {d!r:15} → {c}")
+
+# Filter for Release 02
+release2 = [r for r in rows if (r.get("Release Date") or "").strip() == "5/22/26"]
+print(f"\nRelease 02 records: {len(release2)}")
+
+# Bucket by type
+type_counter = Counter()
+agency_counter = Counter()
+for r in release2:
+    type_counter[(r.get("Type") or "").strip()] += 1
+    agency_counter[(r.get("Agency") or "").strip()] += 1
+print("Types:")
+for t, c in type_counter.most_common():
+    print(f"  {t!r:15} → {c}")
+print("Agencies:")
+for a, c in agency_counter.most_common():
+    print(f"  {a!r:15} → {c}")
+
+# Extract download links
+links = []
+for r in release2:
+    pdf_link = (r.get("PDF | Image Link") or "").strip()
+    modal = (r.get("Modal Image") or "").strip()
+    dvids = (r.get("DVIDS Video ID") or "").strip()
+    title = (r.get("Title") or "").strip().replace("\n", " ").replace("\r", "")
+    rtype = (r.get("Type") or "").strip()
+    agency = (r.get("Agency") or "").strip()
+    incident_date = (r.get("Incident Date") or "").strip()
+    incident_loc = (r.get("Incident Location") or "").strip()
+    links.append({
+        "title": title,
+        "type": rtype,
+        "agency": agency,
+        "incident_date": incident_date,
+        "incident_location": incident_loc,
+        "pdf_link": pdf_link,
+        "modal_image": modal,
+        "dvids_id": dvids,
+    })
+
+# Save full inventory
+(OUT_DIR / "release-02-records.json").write_text(json.dumps(links, indent=2))
+print(f"\nSaved inventory: {OUT_DIR / 'release-02-records.json'}")
+
+# Unique direct-fetchable URLs
+urls = set()
+for L in links:
+    if L["pdf_link"]:
+        urls.add(L["pdf_link"])
+    if L["modal_image"]:
+        urls.add(L["modal_image"])
+urls_list = sorted(urls)
+print(f"\nUnique direct URLs: {len(urls_list)}")
+for u in urls_list[:15]:
+    print(f"  {u}")
+if len(urls_list) > 15:
+    print(f"  ... and {len(urls_list) - 15} more")
+
+(OUT_DIR / "release-02-urls.json").write_text(json.dumps(urls_list, indent=2))
+
+# DVIDS-only records (videos hosted exclusively on DVIDS)
+dvids_only = [L for L in links if L["dvids_id"] and not L["pdf_link"]]
+print(f"\nDVIDS-only video records: {len(dvids_only)}")
+for L in dvids_only[:10]:
+    print(f"  DVIDS {L['dvids_id']}: {L['title'][:80]}")
+(OUT_DIR / "release-02-dvids.json").write_text(json.dumps(dvids_only, indent=2))