n-seo 0.1.0

package/ingest/pull_ga4.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Pull GA4 data for every configured property into data/ga4/<host>/.
+
+Per site (numeric `ga4Property` in the config), trailing 90 days:
+  daily.json    — sessions/users by date
+  sources.json  — sessions by sessionSource/sessionMedium (AI-referral detection
+                  happens downstream from this file)
+  landing.json  — sessions/engagement by landing page
+  funnel.json   — only for the `conversions.site`: your key events by date,
+                  optionally split by a custom dimension (e.g. which app or
+                  page the signup came from)
+
+Auth: analytics.readonly scope via ingest/google_auth.py. Stdlib + curl.
+"""
+
+import json
+import sys
+from datetime import date
+
+import google_auth
+import seo_config
+from http_util import post_json
+
+API = "https://analyticsdata.googleapis.com/v1beta"
+RANGE = [{"startDate": "90daysAgo", "endDate": "yesterday"}]
+
+
+def reports(conversions, host):
+    base = {
+        "daily": {
+            "dateRanges": RANGE,
+            "dimensions": [{"name": "date"}],
+            "metrics": [{"name": "sessions"}, {"name": "totalUsers"}],
+            "limit": 100,
+        },
+        "sources": {
+            "dateRanges": RANGE,
+            "dimensions": [{"name": "sessionSource"}, {"name": "sessionMedium"}],
+            "metrics": [{"name": "sessions"}, {"name": "totalUsers"}],
+            "orderBys": [{"metric": {"metricName": "sessions"}, "desc": True}],
+            "limit": 200,
+        },
+        "landing": {
+            "dateRanges": RANGE,
+            "dimensions": [{"name": "landingPage"}],
+            "metrics": [{"name": "sessions"}, {"name": "engagementRate"}],
+            "orderBys": [{"metric": {"metricName": "sessions"}, "desc": True}],
+            "limit": 100,
+        },
+    }
+    if conversions and conversions.get("site") == host and conversions.get("events"):
+        dims = [{"name": "date"}, {"name": "eventName"}]
+        if conversions.get("sourceDimension"):
+            dims.append({"name": conversions["sourceDimension"]})
+        base["funnel"] = {
+            "dateRanges": RANGE,
+            "dimensions": dims,
+            "metrics": [{"name": "eventCount"}],
+            "dimensionFilter": {"filter": {"fieldName": "eventName", "inListFilter": {
+                "values": list(conversions["events"])}}},
+            "limit": 5000,
+        }
+    return base
+
+
+def run_report(token, prop, body):
+    return post_json(f"{API}/properties/{prop}:runReport", body, token, label=f"ga4 {prop}")
+
+
+def main():
+    props = seo_config.ga4_properties()
+    if not props:
+        print("no site has a ga4Property configured — nothing to pull")
+        return 0
+    conversions = seo_config.load()["conversions"]
+    token = google_auth.access_token(google_auth.ANALYTICS_RO)
+    out_root = seo_config.DATA / "ga4"
+
+    had_error = False
+    for host, prop in props.items():
+        site_dir = out_root / host
+        site_dir.mkdir(parents=True, exist_ok=True)
+        for name, body in reports(conversions, host).items():
+            resp = run_report(token, prop, body)
+            # A custom dimension only works once registered in GA4 Admin
+            # (Custom definitions). Fall back to date+event until then.
+            if name == "funnel" and resp.get("error") and len(body["dimensions"]) == 3:
+                fallback = dict(body, dimensions=body["dimensions"][:2])
+                resp = run_report(token, prop, fallback)
+            if resp.get("error"):
+                # Keep the previous report rather than replacing it with an
+                # empty one; the step fails and the run reports it.
+                print(f"{host:28s} {name:8s} ERROR (kept previous): "
+                      f"{resp['error'].get('message', '')[:60]}")
+                had_error = True
+                continue
+            (site_dir / f"{name}.json").write_text(json.dumps(
+                {"site": host, "property": f"properties/{prop}",
+                 "pulled": date.today().isoformat(), **resp}))
+            print(f"{host:28s} {name:8s} {resp.get('rowCount', 0)} rows")
+
+    print(f"\nSaved under {out_root}")
+    return 1 if had_error else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ingest/pull_gsc.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Pull Search Console data for every configured property into data/gsc/.
+
+Datasets per property, two windows each:
+  queries.json      — by query        (what people search)
+  pages.json        — by page         (what lands)
+  query_page.json   — by query+page   (striking-distance analysis)
+  dates.json        — by date         (trend; full window only)
+and *_90d.json twins for the trailing 90 days — the decision window. The
+16-month window (Search Console's maximum) is for totals and history.
+
+Properties come from n-seo.config.json (ingest/seo_config.py); a domain
+property covers its subdomains, so several sites can share one pull.
+Stdlib only; HTTP via curl (see http_util.py).
+"""
+
+import json
+import sys
+from datetime import date, timedelta
+from urllib.parse import quote
+
+import google_auth
+import seo_config
+from http_util import get_json, post_json
+
+API = "https://searchconsole.googleapis.com/webmasters/v3/sites"
+
+DATASETS = {
+    "queries": ["query"],
+    "pages": ["page"],
+    "query_page": ["query", "page"],
+    "dates": ["date"],
+}
+
+
+def pull(token, site_url, dimensions, start, end):
+    rows, start_row = [], 0
+    url = f"{API}/{quote(site_url, safe='')}/searchAnalytics/query"
+    while True:
+        body = {
+            "startDate": start, "endDate": end,
+            "dimensions": dimensions,
+            "rowLimit": 25000, "startRow": start_row,
+            "dataState": "final",
+        }
+        resp = post_json(url, body, token, label=site_url[:60])
+        if "error" in resp:
+            return {"error": resp["error"], "rows": rows}
+        batch = resp.get("rows", [])
+        rows.extend(batch)
+        if len(batch) < 25000:
+            break
+        start_row += 25000
+    return {"rows": rows}
+
+
+def main():
+    props = seo_config.gsc_properties()
+    if not props:
+        print("no site has a gscProperty configured — nothing to pull")
+        return 0
+    end = (date.today() - timedelta(days=3)).isoformat()      # GSC finalizes ~3 days behind
+    start_full = (date.today() - timedelta(days=488)).isoformat()   # ~16 months (GSC max)
+    start_recent = (date.today() - timedelta(days=93)).isoformat()  # trailing 90d
+    token = google_auth.access_token(google_auth.WEBMASTERS_RO)
+
+    available = {
+        e["siteUrl"]
+        for e in get_json(API, token, label="sites list").get("siteEntry", [])
+        if "siteUrl" in e
+    }
+    who = google_auth.service_account_email() or "your Google account"
+
+    out_root = seo_config.DATA / "gsc"
+    windows = [("", start_full), ("_90d", start_recent)]
+    had_error = False
+    for site_url, slug in props.items():
+        if site_url not in available:
+            print(f"{slug}: {site_url} is not accessible — verify the property in "
+                  f"Search Console and add {who} as a user; skipping")
+            continue
+        site_dir = out_root / slug
+        site_dir.mkdir(parents=True, exist_ok=True)
+        for suffix, start in windows:
+            for name, dims in DATASETS.items():
+                if suffix and name == "dates":
+                    continue  # the daily series is windowable from the full pull
+                result = pull(token, site_url, dims, start, end)
+                if "error" in result:
+                    # Writing zero rows here would read as a traffic collapse
+                    # on the dashboard tomorrow. Keep yesterday's snapshot and
+                    # let the step fail so daily.py records and notifies it.
+                    print(f"{slug:28s} {name}{suffix:5s} ERROR (kept previous): "
+                          f"{result['error'].get('message', '?')[:70]}")
+                    had_error = True
+                    continue
+                payload = {
+                    "site": site_url, "dimensions": dims,
+                    "startDate": start, "endDate": end,
+                    "rowCount": len(result["rows"]),
+                    "rows": result["rows"],
+                }
+                (site_dir / f"{name}{suffix}.json").write_text(json.dumps(payload))
+                print(f"{slug:28s} {name}{suffix:5s} {len(result['rows'])} rows")
+
+    print(f"\nWindows: {start_full} and {start_recent} -> {end}\nSaved under {out_root}")
+    return 1 if had_error else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ingest/pull_index_status.py

@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""Ask Search Console whether each sitemap URL is actually in Google's index.
+
+The rest of the pipeline measures pages that already rank. Nothing measures
+the step before that: a page can sit in the sitemap for months in
+"Discovered - currently not indexed" — known to Google, never fetched — and
+look identical to a page nobody searches for, because both show zero
+impressions. The URL Inspection API is the only place that distinction is
+visible, and it decides where the manual "Request Indexing" quota (the
+Search Console UI, roughly 10/day) is worth spending.
+
+Quota: 2,000 inspections/day and 600/minute per property, so a full sweep of
+a modest sitemap is affordable daily. Read-only scope. Stdlib + curl.
+"""
+
+import json
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from datetime import datetime, timezone
+from urllib.parse import quote
+
+import google_auth
+import seo_config
+from http_util import fetch_text, get_json, post_json
+
+OUT = seo_config.DATA / "index-status.json"
+INSPECT = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect"
+SITES_API = "https://searchconsole.googleapis.com/webmasters/v3/sites"
+
+# The URL Inspection quota is per property (2,000/day), and several hosts can
+# share one domain property, so the budget has to be per property too.
+MAX_PER_PROPERTY = 1500
+WORKERS = 4
+
+# Anything other than this needs a human to look at it.
+INDEXED = "Submitted and indexed"
+
+
+def sitemap_urls(host: str) -> list[str]:
+    """The site's own sitemap is the list of pages it claims should rank.
+    Sitemap indexes are followed one level."""
+    status, raw = fetch_text(f"https://{host}/sitemap.xml", timeout=60)
+    if status != 200:
+        return []
+
+    def locs(xml):
+        out, rest = [], xml
+        while "<loc>" in rest:
+            _, _, rest = rest.partition("<loc>")
+            loc, _, rest = rest.partition("</loc>")
+            loc = loc.strip()
+            if loc.startswith("http"):
+                out.append(loc)
+        return out
+
+    urls = locs(raw)
+    if "<sitemapindex" in raw:
+        pages = []
+        for child in urls[:20]:
+            st, body = fetch_text(child, timeout=60)
+            if st == 200:
+                pages.extend(locs(body))
+        urls = pages
+    return urls
+
+
+def inspect(token: str, site_url: str, page_url: str) -> dict:
+    resp = post_json(INSPECT, {"inspectionUrl": page_url, "siteUrl": site_url},
+                     token, label=page_url[-60:])
+    if "error" in resp:
+        return {"url": page_url, "coverage": "API error",
+                "detail": str(resp["error"].get("message", ""))[:120]}
+    r = resp.get("inspectionResult", {}).get("indexStatusResult", {})
+    canonical = r.get("googleCanonical")
+    return {
+        "url": page_url,
+        "coverage": r.get("coverageState", "unknown"),
+        "lastCrawl": r.get("lastCrawlTime"),
+        "verdict": r.get("verdict"),
+        "robots": r.get("robotsTxtState"),
+        # A Google canonical pointing elsewhere is why a page can be crawled
+        # and still never appear — worth surfacing, not just the coverage line.
+        "canonicalMismatch": bool(canonical and canonical != page_url),
+        "googleCanonical": canonical,
+    }
+
+
+def accessible_properties(token: str) -> set[str]:
+    """Properties this account can actually inspect. An unverified or
+    unshared property answers every inspection with a 403, which would fill
+    the report with identical error rows and bury the real findings."""
+    resp = get_json(SITES_API, token, label="sites list")
+    return {e["siteUrl"] for e in resp.get("siteEntry", []) if "siteUrl" in e}
+
+
+def sitemap_state(token: str, site_url: str) -> dict:
+    """What Search Console has done with the submitted sitemap.
+
+    The UI shows a bare "Couldn't fetch" for a sitemap Google has simply not
+    read yet, which reads as a failure and is not one — the API distinguishes
+    them: isPending with errors 0 means queued, a real problem shows up as a
+    non-zero error count. Recording both keeps the difference visible."""
+    resp = get_json(f"{SITES_API}/{quote(site_url, safe='')}/sitemaps", token,
+                    label=f"sitemaps {site_url}")
+    maps = resp.get("sitemap", []) if isinstance(resp, dict) else []
+    return {
+        "submitted": len(maps),
+        "entries": [
+            {
+                "path": m.get("path"),
+                "lastSubmitted": m.get("lastSubmitted"),
+                "lastDownloaded": m.get("lastDownloaded"),
+                "pending": bool(m.get("isPending")),
+                "errors": int(m.get("errors", 0) or 0),
+                "warnings": int(m.get("warnings", 0) or 0),
+            }
+            for m in maps
+        ],
+    }
+
+
+def main() -> int:
+    hosts = seo_config.index_hosts()
+    if not hosts:
+        print("no site has a gscProperty configured — nothing to inspect")
+        return 0
+    token = google_auth.access_token(google_auth.WEBMASTERS_RO)
+    available = accessible_properties(token)
+    who = google_auth.service_account_email() or "your Google account"
+    out = {"generated": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+           "sites": {}}
+    total_problems = 0
+    budget: dict[str, int] = {}
+
+    for host, site_url in hosts.items():
+        if site_url not in available:
+            print(f"{host}: {site_url} not accessible — verify it in Search Console "
+                  f"and add {who}; skipping")
+            continue
+        left = budget.get(site_url, MAX_PER_PROPERTY)
+        if left <= 0:
+            print(f"{host}: today's inspection budget for {site_url} is spent; skipping")
+            continue
+        urls = sitemap_urls(host)
+        if not urls:
+            print(f"{host}: no sitemap URLs, skipping")
+            continue
+        if len(urls) > left:
+            print(f"{host}: {len(urls)} sitemap URLs, inspecting {left} "
+                  f"(shared quota for {site_url})")
+            urls = urls[:left]
+        budget[site_url] = left - len(urls)
+        with ThreadPoolExecutor(max_workers=WORKERS) as pool:
+            rows = list(pool.map(lambda u: inspect(token, site_url, u), urls))
+
+        problems = [r for r in rows if r["coverage"] != INDEXED]
+        never_crawled = [r for r in problems if not r.get("lastCrawl")]
+        out["sites"][host] = {
+            "property": site_url,
+            "sitemap": sitemap_state(token, site_url),
+            "checked": len(rows),
+            "indexed": len(rows) - len(problems),
+            "neverCrawled": len(never_crawled),
+            # Only the problems are stored: the point is the short list a
+            # human acts on; hundreds of healthy rows would bury it.
+            "problems": sorted(problems, key=lambda r: (bool(r.get("lastCrawl")), r["url"])),
+        }
+        total_problems += len(problems)
+        print(f"{host}: {len(rows)} checked, {len(problems)} not indexed "
+              f"({len(never_crawled)} never crawled)")
+
+    OUT.parent.mkdir(parents=True, exist_ok=True)
+    OUT.write_text(json.dumps(out, indent=2))
+    print(f"saved {OUT} — {total_problems} URLs needing attention")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ingest/pull_timeseries.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Daily time series for the Trends view.
+
+GSC: date x page, trailing 180 days, per property -> data/timeseries/gsc-<slug>.json
+GA4: date x pagePath, 180 days, per site  -> data/timeseries/ga4-<host>.json
+GA4: date x source/medium, 180 days, per site -> data/timeseries/ga4-sources-<host>.json
+
+The source series is what makes "where is traffic coming from, over time"
+answerable. data/ga4/<host>/sources.json is one 90-day aggregate, so it can
+say AI assistants sent 216 sessions but not whether that is growing.
+
+Runs in the daily batch. Stdlib + curl.
+"""
+
+import json
+import sys
+from datetime import date, timedelta
+from urllib.parse import quote
+
+import google_auth
+import seo_config
+from http_util import post_json
+
+OUT = seo_config.DATA / "timeseries"
+
+# Both APIs cap a single response; page until a batch comes back short.
+PAGE = 25000
+
+
+def ga4_series(tok, prop, dims, label):
+    """Every row of a GA4 date-series report, paged.
+
+    Returns (rows, ok). On an API error the caller keeps the previous file
+    rather than replacing a full series with a partial or empty one.
+    """
+    rows, offset = [], 0
+    while True:
+        r = post_json(
+            f"https://analyticsdata.googleapis.com/v1beta/properties/{prop}:runReport",
+            {"dateRanges": [{"startDate": "180daysAgo", "endDate": "yesterday"}],
+             "dimensions": [{"name": d} for d in dims],
+             "metrics": [{"name": "sessions"}],
+             "limit": PAGE, "offset": offset},
+            tok, label=label)
+        if r.get("error"):
+            print(f"ga4  {label:28s} ERROR (kept previous): "
+                  f"{r['error'].get('message', '')[:60]}")
+            return [], False
+        batch = r.get("rows", [])
+        rows.extend(batch)
+        if len(batch) < PAGE:
+            return rows, True
+        offset += PAGE
+
+
+def main():
+    OUT.mkdir(parents=True, exist_ok=True)
+    had_error = False
+    end = (date.today() - timedelta(days=3)).isoformat()
+    start = (date.today() - timedelta(days=183)).isoformat()
+
+    gsc_props = seo_config.gsc_properties()
+    if gsc_props:
+        tok = google_auth.access_token(google_auth.WEBMASTERS_RO)
+        for site, slug in gsc_props.items():
+            url = (f"https://searchconsole.googleapis.com/webmasters/v3/sites/"
+                   f"{quote(site, safe='')}/searchAnalytics/query")
+            rows, start_row, failed = [], 0, False
+            while True:
+                r = post_json(url, {
+                    "startDate": start, "endDate": end,
+                    "dimensions": ["date", "page"],
+                    "rowLimit": PAGE, "startRow": start_row, "dataState": "final"},
+                    tok, label=f"timeseries {slug}")
+                if "error" in r:
+                    # Keep the previous series rather than replacing a full
+                    # one with a partial or empty write.
+                    print(f"gsc  {slug:28s} ERROR (kept previous): "
+                          f"{r['error'].get('message', '')[:70]}")
+                    had_error = failed = True
+                    break
+                batch = r.get("rows", [])
+                rows.extend(batch)
+                if len(batch) < PAGE:
+                    break
+                start_row += PAGE
+            if failed:
+                continue
+            (OUT / f"gsc-{slug}.json").write_text(json.dumps(
+                {"site": site, "startDate": start, "endDate": end, "rows": rows}))
+            print(f"gsc  {slug:28s} {len(rows)} date x page rows")
+
+    ga4_props = seo_config.ga4_properties()
+    if ga4_props:
+        tok = google_auth.access_token(google_auth.ANALYTICS_RO)
+        for host, prop in ga4_props.items():
+            # 180 days x a few hundred pages passes 25,000 rows on a modest
+            # site, and GA4 truncates silently — ga4_series pages for us.
+            batch, ok = ga4_series(tok, prop, ["date", "pagePath"], f"pages {host}")
+            if not ok:
+                had_error = True
+            else:
+                rows = [{"date": row["dimensionValues"][0]["value"],
+                         "page": row["dimensionValues"][1]["value"],
+                         "sessions": float(row["metricValues"][0]["value"])}
+                        for row in batch]
+                (OUT / f"ga4-{host}.json").write_text(json.dumps({"site": host, "rows": rows}))
+                print(f"ga4  {host:28s} {len(rows)} date x page rows")
+
+            batch, ok = ga4_series(tok, prop, ["date", "sessionSource", "sessionMedium"],
+                                   f"sources {host}")
+            if not ok:
+                had_error = True
+            else:
+                rows = [{"date": row["dimensionValues"][0]["value"],
+                         "source": row["dimensionValues"][1]["value"],
+                         "medium": row["dimensionValues"][2]["value"],
+                         "sessions": float(row["metricValues"][0]["value"])}
+                        for row in batch]
+                (OUT / f"ga4-sources-{host}.json").write_text(
+                    json.dumps({"site": host, "rows": rows}))
+                print(f"ga4  {host:28s} {len(rows)} date x source rows")
+
+    if not gsc_props and not ga4_props:
+        print("no GSC or GA4 properties configured — nothing to pull")
+    return 1 if had_error else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())