n-seo 0.1.0

package/ingest/analyze_metadata.py

@@ -0,0 +1,158 @@
+#!/usr/bin/env python3
+"""Page-level metadata audit: does each ranking page's title/description earn
+its clicks?
+
+For every significant page (by impressions, host-filtered), fetches the LIVE
+title + meta description and joins them with the page's top queries from the
+90-day Search Console pull. Flags: query language missing from the title,
+missing/short/title-duplicating descriptions, bad title lengths, and CTR
+below position expectations (positions 1-15).
+
+Writes data/metadata-audit.json — consumed by the dashboard's action queue.
+Stdlib + curl. Runs in the daily batch when modules.metadataAudit is on.
+"""
+
+import html
+import json
+import re
+import sys
+from datetime import date
+
+import seo_config
+from http_util import fetch_text
+
+EXPECTED_CTR = {1: .28, 2: .15, 3: .10, 4: .07, 5: .05, 6: .04,
+                7: .035, 8: .03, 9: .026, 10: .022,
+                11: .018, 12: .016, 13: .014, 14: .012, 15: .011}
+
+# A ranged request comes back 206 from a server that honours the Range header
+# and 200 from one that ignores it. Both mean the page serves; treating only
+# 200 as healthy would report every page on such a site as dead.
+SERVING = (200, 206)
+
+MIN_PAGE_IMPS = 15  # 90d window
+TOP_PAGES_PER_SITE = 15
+STOP = set("a an the and or of to in on for with vs what is how why your our "
+           "you we i it its this that de la".split())
+
+
+def fetch_head(url):
+    """(status, title, description) for a live page.
+
+    The status matters: a page that still ranks but no longer serves must not
+    be audited against whatever its 404 page happens to contain. The quote in
+    the description pattern is back-referenced so an apostrophe inside the
+    text cannot end the match early, and entities are decoded rather than
+    blanked, because both mistakes invent findings that are not there.
+    """
+    status, h = fetch_text(url, byte_range="0-40000")
+    title = re.search(r"<title[^>]*>(.*?)</title>", h, re.S | re.I)
+    desc = re.search(r'<meta[^>]+name=["\']description["\'][^>]+content=(["\'])(.*?)\1', h, re.S | re.I) \
+        or re.search(r'<meta[^>]+content=(["\'])(.*?)\1[^>]+name=["\']description["\']', h, re.S | re.I)
+    return (status,
+            html.unescape(title.group(1)).strip() if title else "",
+            html.unescape(desc.group(2)).strip() if desc else "")
+
+
+def tokens(text):
+    return {w for w in re.findall(r"[a-z0-9']+", text.lower()) if w not in STOP and len(w) > 1}
+
+
+def main():
+    qp_cache = {}
+    audit = {"generated": date.today().isoformat(), "window": "90d", "sites": {}}
+
+    for site in seo_config.sites():
+        gsc_dir = seo_config.gsc_dir_for(site)
+        if not gsc_dir:
+            continue
+        recent = gsc_dir / "query_page_90d.json"
+        gsc_file = recent if recent.exists() else gsc_dir / "query_page.json"
+        if not gsc_file.exists():
+            continue
+        host = site["gscHost"]
+        cache_key = str(gsc_file)
+        if cache_key not in qp_cache:
+            qp_cache[cache_key] = json.loads(gsc_file.read_text())["rows"]
+        rows = [r for r in qp_cache[cache_key]
+                if r["keys"][1].split("/")[2] == host]
+
+        by_page = {}
+        for r in rows:
+            p = by_page.setdefault(r["keys"][1], {"imps": 0, "clicks": 0, "queries": []})
+            p["imps"] += r["impressions"]
+            p["clicks"] += r["clicks"]
+            p["queries"].append({"q": r["keys"][0], "imps": r["impressions"],
+                                 "clicks": r["clicks"], "pos": round(r["position"], 1),
+                                 "ctr": r["ctr"]})
+        pages = sorted(((u, d) for u, d in by_page.items() if d["imps"] >= MIN_PAGE_IMPS),
+                       key=lambda t: -t[1]["imps"])[:TOP_PAGES_PER_SITE]
+
+        findings = []
+        for url, d in pages:
+            status, title, desc = fetch_head(url)
+            d["queries"].sort(key=lambda q: -q["imps"])
+            top_q = d["queries"][:5]
+
+            if status not in SERVING:
+                # It still earns impressions, so it is worth reporting — but a
+                # title rewrite is the wrong move and would burn one of the
+                # ~8 metadata changes a week on a page that does not serve.
+                findings.append({
+                    "page": url, "title": "", "description": "",
+                    "imps": round(d["imps"]), "clicks": round(d["clicks"]),
+                    "issues": [f"page does not serve (HTTP {status or 'no response'}) "
+                               f"but still ranks — fix, redirect or retire it"],
+                    "top_queries": top_q, "missed_clicks_window": 0,
+                })
+                continue
+
+            t_tokens = tokens(title)
+            issues, missed = [], 0.0
+
+            uncovered = []
+            for q in top_q:
+                q_tokens = tokens(q["q"])
+                if q_tokens and len(q_tokens & t_tokens) / len(q_tokens) < 0.5:
+                    uncovered.append(q["q"])
+            if uncovered and sum(q["imps"] for q in top_q) >= MIN_PAGE_IMPS:
+                issues.append(f"title misses ranking-query language: {', '.join(uncovered[:3])}")
+
+            for q in d["queries"]:
+                exp = EXPECTED_CTR.get(round(q["pos"]))
+                if exp and q["imps"] >= 20 and q["ctr"] < exp * 0.5:
+                    missed += q["imps"] * (exp - q["ctr"])
+            if missed >= 5:
+                issues.append(f"CTR below position expectation (~{missed:.0f} clicks missed in 90d)")
+
+            if not desc:
+                issues.append("meta description missing")
+            elif len(desc) < 60:
+                issues.append(f"meta description too short ({len(desc)} chars)")
+            elif title and desc.lower().startswith(title.lower()[:40]):
+                issues.append("meta description duplicates the title")
+            if title and len(title) > 65:
+                issues.append(f"title long ({len(title)} chars — SERP truncates ~60)")
+            if not title:
+                issues.append("no <title> found")
+
+            if issues:
+                findings.append({
+                    "page": url, "title": title[:120], "description": desc[:180],
+                    "imps": round(d["imps"]), "clicks": round(d["clicks"]),
+                    "issues": issues, "top_queries": top_q,
+                    "missed_clicks_window": round(missed),
+                })
+        findings.sort(key=lambda f: -(f["missed_clicks_window"] + f["imps"] / 50))
+        # keyed by the site's canonical host, which is how the dashboard looks it up
+        audit["sites"][site["host"]] = findings
+        print(f"{site['host']:28s} {len(pages)} pages audited, {len(findings)} with findings")
+
+    seo_config.DATA.mkdir(parents=True, exist_ok=True)
+    (seo_config.DATA / "metadata-audit.json").write_text(json.dumps(audit, indent=1))
+    print("saved data/metadata-audit.json")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ingest/analyze_trends.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Trend analysis for marketer-grade recommendations.
+
+Pulls supplemental windowed data (GSC recent-vs-prior query windows, GA4
+monthly AI-referral series) and computes:
+  - branded vs non-branded share (clicks + impressions), using each site's
+    `brand` regex from the config (no regex -> everything counts as generic)
+  - rising / falling queries (trailing 84d vs prior 84d)
+  - monthly clicks/impressions trajectory (from the existing dates.json)
+  - AI-referral sessions by month
+Writes data/trends-<date>.json and prints a summary.
+"""
+
+import json
+import re
+import sys
+from datetime import date, timedelta
+from urllib.parse import quote
+
+import google_auth
+import seo_config
+from http_util import post_json
+
+AI_RE = re.compile(r"chatgpt|chat\.openai|openai\.com|perplexity|claude\.ai|copilot|gemini\.google"
+                   r"|edgeservices|you\.com|poe\.com|phind|kagi|mistral|deepseek", re.I)
+
+
+PAGE = 25000
+
+
+def gsc_queries(tok, site, start, end):
+    """Every query in the window, paged — a single request caps out and the
+    truncation would quietly skew the branded/generic split."""
+    url = f"https://searchconsole.googleapis.com/webmasters/v3/sites/{quote(site, safe='')}/searchAnalytics/query"
+    rows, start_row = {}, 0
+    while True:
+        r = post_json(url, {"startDate": start, "endDate": end, "dimensions": ["query"],
+                            "rowLimit": PAGE, "startRow": start_row, "dataState": "final"},
+                      tok, label=f"trends {site}")
+        batch = r.get("rows", [])
+        rows.update({row["keys"][0]: row for row in batch})
+        if len(batch) < PAGE:
+            return rows
+        start_row += PAGE
+
+
+def main():
+    out = {"generated": date.today().isoformat(), "sites": {}}
+    gsc_props = seo_config.gsc_properties(include_extra=False)
+    brands = {p: re.compile(rx, re.I) for p, rx in seo_config.brand_patterns().items()}
+    ga4_props = seo_config.ga4_properties()
+
+    end = date.today() - timedelta(days=3)
+    mid = end - timedelta(days=84)
+    start = mid - timedelta(days=84)
+    # The windows must not share their boundary day, or it is counted on both
+    # sides of every rising/falling comparison.
+    prior_end = mid - timedelta(days=1)
+
+    if gsc_props:
+        gsc_tok = google_auth.access_token(google_auth.WEBMASTERS_RO)
+    for site, slug in gsc_props.items():
+        recent = gsc_queries(gsc_tok, site, mid.isoformat(), end.isoformat())
+        prior = gsc_queries(gsc_tok, site, start.isoformat(), prior_end.isoformat())
+        brand = brands.get(site)
+        is_brand = (lambda q: bool(brand.search(q))) if brand else (lambda q: False)
+
+        def split(rows):
+            return {
+                "branded_clicks": sum(r["clicks"] for q, r in rows.items() if is_brand(q)),
+                "generic_clicks": sum(r["clicks"] for q, r in rows.items() if not is_brand(q)),
+                "branded_imps": sum(r["impressions"] for q, r in rows.items() if is_brand(q)),
+                "generic_imps": sum(r["impressions"] for q, r in rows.items() if not is_brand(q)),
+            }
+
+        movers = []
+        for q in set(recent) | set(prior):
+            ri = recent.get(q, {}).get("impressions", 0)
+            pi = prior.get(q, {}).get("impressions", 0)
+            if max(ri, pi) >= 30:
+                movers.append({"query": q, "recent_imps": ri, "prior_imps": pi,
+                               "delta": ri - pi,
+                               "recent_pos": round(recent.get(q, {}).get("position", 0), 1),
+                               "recent_clicks": recent.get(q, {}).get("clicks", 0)})
+        movers.sort(key=lambda m: -abs(m["delta"]))
+
+        entry = {
+            "recent_split": split(recent), "prior_split": split(prior),
+            "rising": [m for m in movers if m["delta"] > 0][:20],
+            "falling": [m for m in movers if m["delta"] < 0][:15],
+        }
+        # monthly trajectory from the existing 16-month dates.json
+        p = seo_config.DATA / "gsc" / slug / "dates.json"
+        if p.exists():
+            monthly = {}
+            for r in json.loads(p.read_text())["rows"]:
+                m = r["keys"][0][:7]
+                cur = monthly.setdefault(m, {"clicks": 0, "imps": 0})
+                cur["clicks"] += r["clicks"]
+                cur["imps"] += r["impressions"]
+            entry["monthly"] = monthly
+        out["sites"][site] = entry
+
+    # GA4 AI referrals by month
+    if ga4_props:
+        ga_tok = google_auth.access_token(google_auth.ANALYTICS_RO)
+    for host, prop in ga4_props.items():
+        r = post_json(f"https://analyticsdata.googleapis.com/v1beta/properties/{prop}:runReport", {
+            "dateRanges": [{"startDate": "365daysAgo", "endDate": "yesterday"}],
+            "dimensions": [{"name": "yearMonth"}, {"name": "sessionSource"}],
+            "metrics": [{"name": "sessions"}], "limit": PAGE}, ga_tok, label=f"ga4 monthly {host}")
+        ai_by_month, total_by_month = {}, {}
+        for row in r.get("rows", []):
+            ym, src = row["dimensionValues"][0]["value"], row["dimensionValues"][1]["value"]
+            n = float(row["metricValues"][0]["value"])
+            total_by_month[ym] = total_by_month.get(ym, 0) + n
+            if AI_RE.search(src):
+                ai_by_month[ym] = ai_by_month.get(ym, 0) + n
+        out.setdefault("ai_referrals", {})[host] = {
+            "ai": dict(sorted(ai_by_month.items())),
+            "total": dict(sorted(total_by_month.items()))}
+
+    seo_config.DATA.mkdir(parents=True, exist_ok=True)
+    dest = seo_config.DATA / f"trends-{date.today().isoformat()}.json"
+    dest.write_text(json.dumps(out, indent=1))
+    print(f"saved {dest}\n")
+
+    for site, d in out["sites"].items():
+        rs, ps = d["recent_split"], d["prior_split"]
+        print(f"== {site}")
+        print(f"  clicks 84d: branded {rs['branded_clicks']:.0f} vs generic {rs['generic_clicks']:.0f}"
+              f" (prior: {ps['branded_clicks']:.0f}/{ps['generic_clicks']:.0f})")
+        print(f"  imps 84d: branded {rs['branded_imps']:.0f} vs generic {rs['generic_imps']:.0f}")
+        print("  RISING:", "; ".join(f"{m['query']} ({m['prior_imps']}->{m['recent_imps']}, pos {m['recent_pos']})" for m in d["rising"][:8]))
+        print("  FALLING:", "; ".join(f"{m['query']} ({m['prior_imps']}->{m['recent_imps']})" for m in d["falling"][:5]))
+    if out.get("ai_referrals"):
+        print("\n== AI referrals by month (sessions)")
+        for site, d in out["ai_referrals"].items():
+            series = ", ".join(f"{k[-2:]}:{v:.0f}" for k, v in list(d["ai"].items())[-6:])
+            print(f"  {site:28s} {series}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ingest/google_auth.py

@@ -0,0 +1,238 @@
+"""Access tokens for the Google APIs (Search Console, GA4), four ways.
+
+  service-account-key (default, recommended)
+      A service-account JSON key on disk (config google.serviceAccountKey,
+      or $GOOGLE_APPLICATION_CREDENTIALS). We build the OAuth JWT ourselves
+      and sign it with the `openssl` CLI, so there is no gcloud and no pip
+      dependency. Add the service account's email to your Search Console
+      property (Full user) and your GA4 property (Viewer) — that is all the
+      access it gets.
+
+  gcloud-impersonate
+      `gcloud auth print-access-token --impersonate-service-account=<sa>`.
+      For people who already run gcloud and would rather grant themselves
+      Token Creator on the SA than keep a key file.
+
+  gcloud-user
+      `gcloud auth print-access-token` with a user login. Only works if that
+      login already carries the webmasters/analytics scopes, which Google's
+      default gcloud client does not — kept for completeness, not
+      recommended.
+
+  metadata (for GCE / Cloud Run / GKE — no key file at all)
+      The runtime service account, taken from the metadata server. Its token
+      is `cloud-platform` scoped and Search Console rejects that, so the
+      account mints a correctly scoped token for itself through IAM
+      Credentials. That self-impersonation needs
+      `roles/iam.serviceAccountTokenCreator` on itself; the error says so if
+      it is missing. Prefer this over shipping a key file into a container.
+
+Scopes are requested per call, minimal: webmasters.readonly for pulls and
+inspection, analytics.readonly for GA4.
+"""
+import base64
+import json
+import os
+import subprocess
+import tempfile
+import time
+
+import seo_config
+from http_util import curl_json
+
+WEBMASTERS_RO = "https://www.googleapis.com/auth/webmasters.readonly"
+WEBMASTERS = "https://www.googleapis.com/auth/webmasters"
+ANALYTICS_RO = "https://www.googleapis.com/auth/analytics.readonly"
+
+METADATA_ROOT = "http://metadata.google.internal/computeMetadata/v1"
+IAM_CREDENTIALS = "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts"
+# Off GCP the hostname does not resolve, so the probe fails on curl's DNS
+# error rather than waiting; the short timeouts bound the case where some
+# network resolves it to something that then hangs.
+_MD_ARGS = ["--connect-timeout", "1", "-H", "Metadata-Flavor: Google"]
+
+_cache: dict[str, tuple[str, float]] = {}
+
+
+def _b64url(b: bytes) -> str:
+    return base64.urlsafe_b64encode(b).rstrip(b"=").decode()
+
+
+def key_path() -> str | None:
+    """The service-account key to sign with.
+
+    The configured path wins, but the example config ships a default one, so
+    a key that is not there must not shadow a working
+    GOOGLE_APPLICATION_CREDENTIALS — otherwise the documented env var can
+    never take effect on a fresh install.
+    """
+    g = seo_config.load()["google"]
+    configured = g.get("serviceAccountKey")
+    env = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS")
+    if configured:
+        p = str(seo_config.expand(configured))
+        if os.path.exists(p) or not env:
+            return p
+    return str(seo_config.expand(env)) if env else None
+
+
+def _sa_key_token(scope: str) -> str:
+    kp = key_path()
+    if not kp or not os.path.exists(kp):
+        raise RuntimeError(
+            "google.auth is service-account-key but no key file was found at "
+            f"{kp or '(unset)'} — see docs/SETUP-GOOGLE.md")
+    key = json.loads(open(kp).read())
+    # Backdate slightly: Google rejects a JWT issued in its future, so a
+    # machine whose clock runs a few seconds fast otherwise fails to
+    # authenticate at all, with an error that names nothing useful.
+    iat = int(time.time()) - 60
+    header = _b64url(json.dumps({"alg": "RS256", "typ": "JWT"}).encode())
+    claims = _b64url(json.dumps({
+        "iss": key["client_email"], "scope": scope, "aud": key["token_uri"],
+        "iat": iat, "exp": iat + 3600,
+    }).encode())
+    signing_input = f"{header}.{claims}".encode()
+    # openssl needs the private key in a file; keep it 0600 and short-lived.
+    fd, tmp = tempfile.mkstemp(prefix="n-seo-", suffix=".pem")
+    try:
+        os.fchmod(fd, 0o600)
+        with os.fdopen(fd, "w") as f:
+            f.write(key["private_key"])
+        sig = subprocess.run(["openssl", "dgst", "-sha256", "-sign", tmp],
+                             input=signing_input, capture_output=True, check=True).stdout
+    finally:
+        try:
+            os.unlink(tmp)
+        except OSError:
+            pass
+    assertion = signing_input.decode() + "." + _b64url(sig)
+    resp = curl_json([
+        "-X", "POST", "-d", f"grant_type=urn:ietf:params:oauth:grant-type:jwt-bearer&assertion={assertion}",
+        key["token_uri"]], label="token")
+    if "access_token" not in resp:
+        hint = ""
+        if resp.get("error") == "invalid_grant":
+            hint = (" — invalid_grant usually means this machine's clock is off, "
+                    "or the key has been disabled or deleted")
+        raise RuntimeError(f"token exchange failed{hint}: {json.dumps(resp)[:300]}")
+    return resp["access_token"]
+
+
+def _metadata_json(path: str, *, timeout: int = 2, attempts: int = 1):
+    return curl_json([*_MD_ARGS, f"{METADATA_ROOT}{path}"],
+                     timeout=timeout, attempts=attempts, label=f"metadata {path}")
+
+
+def metadata_service_account() -> str | None:
+    """The runtime service account's email, or None when not on GCP."""
+    try:
+        sa = _metadata_json("/instance/service-accounts/default/?recursive=true")
+    except (RuntimeError, OSError):
+        return None
+    return sa.get("email") if isinstance(sa, dict) else None
+
+
+def metadata_available() -> bool:
+    """True on GCE, Cloud Run and GKE. Safe (and quick) to call anywhere."""
+    return metadata_service_account() is not None
+
+
+def _metadata_token(scope: str) -> str:
+    """Runtime service account -> a token carrying `scope`.
+
+    Two steps, because the metadata server only issues `cloud-platform`
+    tokens and the Search Console API checks for its own scope: take the
+    metadata token, then ask IAM Credentials for a scoped one for the same
+    account.
+    """
+    g = seo_config.load()["google"]
+    email = g.get("impersonate") or metadata_service_account()
+    if not email:
+        raise RuntimeError(
+            "google.auth is metadata, but the metadata server did not answer. "
+            "That mode only works on GCE, Cloud Run or GKE. Off GCP, use "
+            "service-account-key; see docs/SETUP-GOOGLE.md")
+    md = _metadata_json("/instance/service-accounts/default/token", timeout=5, attempts=2)
+    base = md.get("access_token") if isinstance(md, dict) else None
+    if not base:
+        raise RuntimeError(f"metadata server returned no access_token: {json.dumps(md)[:200]}")
+    resp = curl_json([
+        "-X", "POST",
+        "-H", f"Authorization: Bearer {base}",
+        "-H", "Content-Type: application/json",
+        "-d", json.dumps({"scope": [scope], "lifetime": "3600s"}),
+        f"{IAM_CREDENTIALS}/{email}:generateAccessToken",
+    ], label="generateAccessToken")
+    if isinstance(resp, dict) and resp.get("accessToken"):
+        return resp["accessToken"]
+    err = resp.get("error") if isinstance(resp, dict) else None
+    code = int(err.get("code", 0)) if isinstance(err, dict) else 0
+    if code in (401, 403):
+        raise RuntimeError(
+            f"{email} is not allowed to mint tokens for itself. Grant it Token "
+            f"Creator on itself:\n"
+            f"  gcloud iam service-accounts add-iam-policy-binding {email} \\\n"
+            f"    --member=serviceAccount:{email} \\\n"
+            f"    --role=roles/iam.serviceAccountTokenCreator\n"
+            f"({json.dumps(resp)[:200]})")
+    raise RuntimeError(f"generateAccessToken failed: {json.dumps(resp)[:300]}")
+
+
+def _gcloud_token(scope: str, impersonate: str | None) -> str:
+    cmd = ["gcloud", "auth", "print-access-token"]
+    if impersonate:
+        cmd += [f"--impersonate-service-account={impersonate}", f"--scopes={scope}"]
+    p = subprocess.run(cmd, capture_output=True, text=True)
+    if p.returncode != 0:
+        raise RuntimeError(f"gcloud token failed: {p.stderr.strip()[:200]}")
+    return p.stdout.strip()
+
+
+def access_token(scope: str = WEBMASTERS_RO) -> str:
+    """A bearer token for `scope`, cached in-process for ~50 minutes."""
+    hit = _cache.get(scope)
+    if hit and hit[1] > time.time():
+        return hit[0]
+    g = seo_config.load()["google"]
+    mode = g.get("auth", "service-account-key")
+    if mode == "service-account-key":
+        tok = _sa_key_token(scope)
+    elif mode == "gcloud-impersonate":
+        if not g.get("impersonate"):
+            raise RuntimeError("google.auth is gcloud-impersonate but google.impersonate is empty")
+        tok = _gcloud_token(scope, g["impersonate"])
+    elif mode == "gcloud-user":
+        tok = _gcloud_token(scope, None)
+    elif mode == "metadata":
+        tok = _metadata_token(scope)
+    else:
+        raise RuntimeError(f"unknown google.auth mode {mode!r}")
+    _cache[scope] = (tok, time.time() + 50 * 60)
+    return tok
+
+
+def service_account_email() -> str | None:
+    """Who to add in the Search Console / GA4 consoles."""
+    g = seo_config.load()["google"]
+    if g.get("auth") == "gcloud-impersonate":
+        return g.get("impersonate") or None
+    if g.get("auth") == "metadata":
+        return g.get("impersonate") or metadata_service_account()
+    kp = key_path()
+    if kp and os.path.exists(kp):
+        try:
+            return json.loads(open(kp).read()).get("client_email")
+        except (OSError, json.JSONDecodeError):
+            return None
+    return None
+
+
+if __name__ == "__main__":
+    # `python3 ingest/google_auth.py` — mint one token and list GSC properties.
+    tok = access_token()
+    props = curl_json(["-H", f"Authorization: Bearer {tok}",
+                       "https://searchconsole.googleapis.com/webmasters/v3/sites"], label="sites")
+    print("token OK;", service_account_email() or "(user login)")
+    for e in props.get("siteEntry", []):
+        print(f"  {e.get('siteUrl'):45s} {e.get('permissionLevel')}")

package/ingest/http_util.py

@@ -0,0 +1,87 @@
+"""HTTP via curl, with retry.
+
+Why curl and not urllib: macOS's system Python ships without a CA bundle, so
+urllib fails TLS verification out of the box, and the whole point of this
+pipeline is "clone it and run it" with no pip installs. curl is present on
+every macOS and nearly every Linux box.
+
+Why retry: the largest pulls (16 months of query x page) occasionally time
+out or come back 5xx. One blip used to fail the whole step, and the daily run
+would then analyze yesterday's files and write a log entry that looked fine.
+"""
+import json
+import subprocess
+import sys
+import time
+
+# curl exits worth another try: 6/7 resolve+connect, 28 timeout,
+# 35 TLS handshake, 52 empty reply, 56 recv error.
+RETRYABLE_EXITS = {6, 7, 28, 35, 52, 56}
+TIMEOUT = 180
+ATTEMPTS = 4
+
+
+def curl_json(args, *, timeout=TIMEOUT, attempts=ATTEMPTS, label=""):
+    """Run curl with `args`, parse JSON, retry transient failures.
+
+    Raises RuntimeError once the attempts are spent, so the calling step
+    fails loudly rather than returning partial data.
+    """
+    last = None
+    for attempt in range(1, attempts + 1):
+        p = subprocess.run(["curl", "-s", "--max-time", str(timeout), *args],
+                           capture_output=True, text=True)
+        if p.returncode == 0:
+            try:
+                data = json.loads(p.stdout)
+            except json.JSONDecodeError as exc:
+                last = f"unparseable response ({exc})"
+            else:
+                err = data.get("error") if isinstance(data, dict) else None
+                # 5xx and 429 are the server's problem and worth retrying;
+                # 4xx means the request is wrong and retrying won't fix it.
+                code = int(err.get("code", 0)) if isinstance(err, dict) else 0
+                if code >= 500 or code == 429:
+                    last = f"HTTP {code} {str(err.get('message', ''))[:80]}"
+                else:
+                    return data
+        elif p.returncode in RETRYABLE_EXITS:
+            last = f"curl exit {p.returncode}"
+        else:
+            raise RuntimeError(f"curl exit {p.returncode}: {p.stderr.strip()[:200]}")
+        if attempt < attempts:
+            delay = 2 ** attempt  # 2s, 4s, 8s
+            print(f"  retry {attempt}/{attempts - 1} {label}: {last}; sleeping {delay}s",
+                  file=sys.stderr)
+            time.sleep(delay)
+    raise RuntimeError(f"{label or 'request'} failed after {attempts} attempts: {last}")
+
+
+def get_json(url, token=None, *, label=""):
+    args = []
+    if token:
+        args += ["-H", f"Authorization: Bearer {token}"]
+    return curl_json([*args, url], label=label or url[-60:])
+
+
+def post_json(url, body, token=None, *, label=""):
+    args = ["-X", "POST", "-H", "Content-Type: application/json"]
+    if token:
+        args += ["-H", f"Authorization: Bearer {token}"]
+    return curl_json([*args, "-d", json.dumps(body), url], label=label or url[-60:])
+
+
+def fetch_text(url, *, timeout=15, ua="Mozilla/5.0 (compatible; n-seo/0.1)", follow=True,
+               byte_range=None):
+    """Plain page fetch → (status, body). status is None when curl itself failed."""
+    cmd = ["curl", "-s", "-A", ua, "--max-time", str(timeout), "-w", "\n%{http_code}"]
+    if follow:
+        cmd.append("-L")
+    if byte_range:
+        cmd += ["-r", byte_range]
+    try:
+        p = subprocess.run([*cmd, url], capture_output=True, text=True, timeout=timeout + 5)
+    except subprocess.TimeoutExpired:
+        return None, ""
+    body, _, code = p.stdout.rpartition("\n")
+    return (int(code) if code.isdigit() else None), body