n-seo 0.1.0

package/ingest/seo_config.py

@@ -0,0 +1,213 @@
+"""The one config file, as seen from Python.
+
+n-seo.config.json (or $N_SEO_CONFIG) is shared with the dashboard
+(src/config.ts). Every script imports this instead of carrying its own site
+list, so a site added to the config is picked up by every pull, probe, audit
+and export. Stdlib only.
+"""
+
+import json
+import os
+import subprocess
+from pathlib import Path
+
+# The engine checkout: code, engine docs, public assets.
+ROOT = Path(__file__).resolve().parent.parent
+# The instance: one user's config, queue, content and data. Defaults to the
+# engine checkout ("in-place" mode); N_SEO_INSTANCE separates them so that
+# upgrading the engine is a git pull that never touches your files.
+INSTANCE = Path(os.environ["N_SEO_INSTANCE"]).resolve() if os.environ.get("N_SEO_INSTANCE") else ROOT
+DATA = INSTANCE / "data"
+CONFIG_PATH = Path(os.environ.get("N_SEO_CONFIG") or INSTANCE / "n-seo.config.json")
+EXAMPLE_PATH = ROOT / "n-seo.config.example.json"
+
+MODULE_KEYS = [
+    "indexStatus", "metadataAudit", "opportunityScan", "llm", "hackerNews",
+    "reddit", "indexNow", "staticExport", "publish", "gitAutoCommit",
+    "notifications",
+]
+
+# Per-module defaults, so a half-written block cannot make a step guess.
+# Mirrors MODULE_DEFAULTS in src/config.ts.
+MODULE_DEFAULTS = {
+    "staticExport": {"signOutUrl": "", "signOutLabel": "Sign out"},
+    "publish": {"target": "gcs", "destination": "", "command": "",
+                "delete": False, "dryRun": False, "env": {}},
+}
+
+_cache = None
+
+
+def using_example() -> bool:
+    return not CONFIG_PATH.exists()
+
+
+def load(force: bool = False) -> dict:
+    """The normalized config. Falls back to the example file so a fresh
+    checkout can run `ops/demo_data.py` and open the dashboard before any
+    Google setup."""
+    global _cache
+    if _cache is not None and not force:
+        return _cache
+    path = CONFIG_PATH if CONFIG_PATH.exists() else EXAMPLE_PATH
+    raw = json.loads(path.read_text())
+    modules = {k: {"enabled": False, **MODULE_DEFAULTS.get(k, {})} for k in MODULE_KEYS}
+    for k, v in (raw.get("modules") or {}).items():
+        modules[k] = {"enabled": False, **MODULE_DEFAULTS.get(k, {}), **(v or {})}
+    sites = []
+    for s in raw.get("sites") or []:
+        if not s.get("host"):
+            continue
+        sites.append({
+            **s,
+            "label": s.get("label") or s["host"],
+            "gscHost": s.get("gscHost") or s["host"],
+            "gscProperty": s.get("gscProperty") or None,
+            "ga4Property": str(s.get("ga4Property") or "") or None,
+            "brand": s.get("brand") or None,
+        })
+    conv = raw.get("conversions") or {}
+
+    def str_list(v):
+        return [x for x in (v or []) if isinstance(x, str) and x.strip()] if isinstance(v, list) else []
+
+    raw_hooks = raw.get("hooks") or {}
+    hooks = {
+        "beforeRun": str_list(raw_hooks.get("beforeRun")),
+        "afterRun": str_list(raw_hooks.get("afterRun")),
+        "afterStep": {k: str_list(v) for k, v in (raw_hooks.get("afterStep") or {}).items()},
+    }
+    _cache = {
+        "name": raw.get("name") or "n-seo",
+        "port": int(os.environ.get("SEO_PORT") or raw.get("port") or 4600),
+        "google": {**(raw.get("google") or {}),
+                   "auth": (raw.get("google") or {}).get("auth") or "service-account-key"},
+        "sites": sites,
+        "watchPages": raw.get("watchPages") or [],
+        "conversions": conv if conv.get("site") else None,
+        "participation": raw.get("participation") or {},
+        "modules": modules,
+        "gscExtraProperties": str_list(raw.get("gscExtraProperties")),
+        "hooks": hooks,
+    }
+    return _cache
+
+
+def engine_info() -> dict:
+    """Mirrors src/config.ts engineInfo(): what engine is running, for which instance."""
+    try:
+        version = json.loads((ROOT / "package.json").read_text()).get("version", "0.0.0")
+    except (OSError, json.JSONDecodeError):
+        version = "0.0.0"
+    try:
+        p = subprocess.run(["git", "-C", str(ROOT), "rev-parse", "--short", "HEAD"],
+                           capture_output=True, text=True)
+        commit = p.stdout.strip() or None if p.returncode == 0 else None
+    except OSError:
+        commit = None
+    return {"version": version, "commit": commit, "root": str(ROOT), "instance": str(INSTANCE),
+            "mode": "in-place" if INSTANCE == ROOT else "instance"}
+
+
+def hooks() -> dict:
+    return load()["hooks"]
+
+
+def sites() -> list[dict]:
+    return load()["sites"]
+
+
+def hosts() -> list[str]:
+    return [s["host"] for s in sites()]
+
+
+def module(name: str) -> dict:
+    return load()["modules"].get(name, {"enabled": False})
+
+
+def enabled(name: str) -> bool:
+    return bool(module(name).get("enabled"))
+
+
+def port() -> int:
+    return load()["port"]
+
+
+def dashboard_base() -> str:
+    return f"http://localhost:{port()}"
+
+
+def gsc_slug(prop: str) -> str:
+    """data/gsc/<slug>/ — must match src/config.ts gscSlug()."""
+    s = prop
+    if s.startswith("sc-domain:"):
+        s = s[len("sc-domain:"):]
+    for prefix in ("https://", "http://"):
+        if s.startswith(prefix):
+            s = s[len(prefix):]
+    return s.rstrip("/").replace("/", "_")
+
+
+def gsc_data_slug(prop: str) -> str:
+    """The data/gsc/ directory for a property. A url-prefix property
+    ("https://example.com/") would slug to the same name as the domain
+    property ("sc-domain:example.com"), so it gets a "-urlprefix" suffix.
+    Must match src/config.ts gscDataSlug()."""
+    s = gsc_slug(prop)
+    return s + "-urlprefix" if prop.lower().startswith(("http://", "https://")) else s
+
+
+def gsc_properties(include_extra: bool = True) -> dict[str, str]:
+    """{GSC property -> data/gsc subdirectory}, deduplicated. A domain
+    property covers its subdomains, so several hosts can share one.
+    gscExtraProperties (pulled for their data, never shown as sites) are
+    appended unless include_extra is False."""
+    out = {}
+    for s in sites():
+        if s["gscProperty"]:
+            out[s["gscProperty"]] = gsc_data_slug(s["gscProperty"])
+    if include_extra:
+        for prop in load()["gscExtraProperties"]:
+            out.setdefault(prop, gsc_data_slug(prop))
+    return out
+
+
+def gsc_dir_for(site: dict) -> Path | None:
+    return DATA / "gsc" / gsc_data_slug(site["gscProperty"]) if site.get("gscProperty") else None
+
+
+def ga4_properties() -> dict[str, str]:
+    """{host -> numeric GA4 property id}"""
+    return {s["host"]: s["ga4Property"] for s in sites() if s.get("ga4Property")}
+
+
+def index_hosts() -> dict[str, str]:
+    """{crawlable host -> GSC property to inspect its URLs against}"""
+    return {s["gscHost"]: s["gscProperty"] for s in sites() if s.get("gscProperty")}
+
+
+def brand_patterns() -> dict[str, str]:
+    """{GSC property -> brand regex}; hosts sharing a property are OR-ed."""
+    out: dict[str, list[str]] = {}
+    for s in sites():
+        if s.get("gscProperty") and s.get("brand"):
+            out.setdefault(s["gscProperty"], []).append(s["brand"])
+    return {p: "|".join(f"(?:{b})" for b in bs) for p, bs in out.items()}
+
+
+def expand(path_str: str) -> Path:
+    return Path(os.path.expanduser(os.path.expandvars(path_str))).resolve()
+
+
+def env(key: str, default: str = "") -> str:
+    """Read one key from the environment, then from the instance's .env (KEY=value lines)."""
+    if os.environ.get(key):
+        return os.environ[key]
+    try:
+        for line in (INSTANCE / ".env").read_text().splitlines():
+            line = line.strip()
+            if line.startswith(f"{key}="):
+                return line.split("=", 1)[1].strip().strip('"').strip("'")
+    except OSError:
+        pass
+    return default

package/n-seo.config.example.json

@@ -0,0 +1,110 @@
+{
+  "name": "My sites",
+  "port": 4600,
+  "google": {
+    "auth": "service-account-key",
+    "serviceAccountKey": "~/.config/n-seo/service-account.json",
+    "impersonate": ""
+  },
+  "sites": [
+    {
+      "host": "example.com",
+      "label": "example",
+      "gscProperty": "sc-domain:example.com",
+      "gscHost": "example.com",
+      "ga4Property": "",
+      "brand": "example",
+      "repo": "../example-site",
+      "hosting": "Netlify"
+    },
+    {
+      "host": "docs.example.com",
+      "label": "docs",
+      "gscProperty": "sc-domain:example.com",
+      "gscHost": "docs.example.com",
+      "ga4Property": "",
+      "brand": "example"
+    }
+  ],
+  "watchPages": [
+    "https://example.com/",
+    "https://example.com/pricing"
+  ],
+  "gscExtraProperties": [],
+  "conversions": {
+    "site": "",
+    "events": [
+      "sign_up",
+      "newsletter_signup"
+    ],
+    "sourceDimension": "customEvent:source_app"
+  },
+  "participation": {
+    "expertise": "Describe who you are and what you genuinely know first-hand. This is the only context the briefing generator gets, and it is what keeps briefings honest: they point at threads where YOUR experience applies."
+  },
+  "modules": {
+    "indexStatus": {
+      "enabled": true
+    },
+    "metadataAudit": {
+      "enabled": true
+    },
+    "opportunityScan": {
+      "enabled": true
+    },
+    "llm": {
+      "enabled": false,
+      "command": "claude -p --model sonnet",
+      "fastCommand": "claude -p --model haiku"
+    },
+    "hackerNews": {
+      "enabled": false,
+      "user": "",
+      "topics": [
+        [
+          "your niche keyword",
+          "why you can speak to this"
+        ]
+      ]
+    },
+    "reddit": {
+      "enabled": false,
+      "user": "",
+      "topics": [
+        [
+          "subreddit",
+          "search query",
+          "why you can speak to this"
+        ]
+      ]
+    },
+    "indexNow": {
+      "enabled": false,
+      "keyFile": "indexnow.key"
+    },
+    "staticExport": {
+      "enabled": false,
+      "signOutUrl": "",
+      "signOutLabel": "Sign out"
+    },
+    "publish": {
+      "enabled": false,
+      "target": "gcs",
+      "destination": "gs://your-bucket",
+      "delete": false,
+      "dryRun": false,
+      "env": {}
+    },
+    "gitAutoCommit": {
+      "enabled": false
+    },
+    "notifications": {
+      "enabled": false
+    }
+  },
+  "hooks": {
+    "beforeRun": [],
+    "afterStep": {},
+    "afterRun": []
+  }
+}

package/ops/daily.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""The daily run: refresh every snapshot, diff, digest, export.
+
+Portable replacement for a shell script — schedule it with launchd, cron or
+systemd (see docs/SCHEDULING.md), or run it by hand any time:
+
+  python3 ops/daily.py                 # everything the config enables
+  python3 ops/daily.py --list          # show the steps and which are on
+  python3 ops/daily.py --only probe,gsc
+  python3 ops/daily.py --skip index-status --no-network-wait
+  python3 ops/daily.py --skip hooks       # steps only, no config hooks
+
+Hooks (config `hooks`: beforeRun / afterStep.<name> / afterRun) are shell
+commands run in the instance directory around the steps; a failing hook is
+recorded as a failure but never aborts the run.
+
+Each step is a subprocess; its output is teed to the console and to
+data/daily-ops.log. A failed step is retried once after waiting for the
+network — on a laptop, a run that fires on wake usually fails only because
+Wi-Fi is not up yet. Results land in data/last-run.json for the dashboard.
+"""
+
+import argparse
+import json
+import os
+import platform
+import subprocess
+import sys
+import time
+from datetime import date, datetime, timezone
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
+import seo_config  # noqa: E402
+
+ROOT = seo_config.ROOT            # the engine: scripts run from here
+INSTANCE = seo_config.INSTANCE    # the instance: data, log, git commits, hooks run here
+LOG = seo_config.DATA / "daily-ops.log"
+
+# (name, script, module gate or None)
+STEPS = [
+    ("probe",            "probes/site_probe.py",          None),
+    ("gsc",              "ingest/pull_gsc.py",            None),
+    ("ga4",              "ingest/pull_ga4.py",            None),
+    ("timeseries",       "ingest/pull_timeseries.py",     None),
+    ("metadata-audit",   "ingest/analyze_metadata.py",    "metadataAudit"),
+    ("index-status",     "ingest/pull_index_status.py",   "indexStatus"),
+    ("opportunity-scan", "ops/opportunity_scan.py",       "opportunityScan"),
+    ("daily-diff",       "ops/daily_diff.py",             None),
+    ("hn-digest",        "ops/hn_digest.py",              "hackerNews"),
+    ("reddit-digest",    "ops/reddit_digest.py",          "reddit"),
+    ("static-export",    "ops/export_static.py",          "staticExport"),
+    ("publish",          "ops/publish.py",                "publish"),
+]
+
+
+def log(line: str):
+    print(line, flush=True)
+    LOG.parent.mkdir(parents=True, exist_ok=True)
+    with LOG.open("a") as f:
+        f.write(line.rstrip("\n") + "\n")
+
+
+def wait_for_network(max_wait=300):
+    waited = 0
+    while waited < max_wait:
+        p = subprocess.run(["curl", "-sf", "--max-time", "5", "-o", "/dev/null",
+                            "https://www.google.com/generate_204"], capture_output=True)
+        if p.returncode == 0:
+            if waited:
+                log(f"network: waited {waited}s for connectivity")
+            return True
+        time.sleep(10)
+        waited += 10
+    log(f"network: still down after {max_wait}s — continuing; network steps will fail")
+    return False
+
+
+def _hook_env(step: str | None = None) -> dict:
+    env = dict(os.environ)
+    env["N_SEO_ROOT"] = str(ROOT)
+    env["N_SEO_INSTANCE"] = str(INSTANCE)
+    if step:
+        env["N_SEO_STEP"] = step
+    return env
+
+
+def run_step(script: str) -> bool:
+    # Engine scripts run with cwd=ROOT; they find the instance through the
+    # inherited N_SEO_INSTANCE (seo_config resolves it), never through cwd.
+    p = subprocess.Popen([sys.executable, str(ROOT / script)], cwd=ROOT, env=_hook_env(),
+                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    for line in p.stdout:
+        log("  " + line.rstrip("\n"))
+    return p.wait() == 0
+
+
+def run_hook(cmd: str, step: str | None = None) -> bool:
+    """One `hooks` command: a shell string run in the instance directory with
+    N_SEO_ROOT / N_SEO_INSTANCE / N_SEO_STEP set. Output is teed like a step."""
+    p = subprocess.Popen(cmd, shell=True, cwd=INSTANCE, env=_hook_env(step),
+                         stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    for line in p.stdout:
+        log("  " + line.rstrip("\n"))
+    return p.wait() == 0
+
+
+def run_hooks(kind: str, cmds: list, results: list, failures: list, step: str | None = None):
+    """Run a hook list, recording each as hook:<kind>:<i>. A failing hook is a
+    recorded failure, never an abort — the run's own steps still matter more."""
+    for i, cmd in enumerate(cmds):
+        name = f"hook:{kind}:{i}"
+        t0 = time.time()
+        log(f"--- {name}  $ {cmd}")
+        ok = run_hook(cmd, step)
+        if not ok:
+            failures.append(name)
+            log(f"{name} FAILED")
+        results.append({"name": name, "ok": ok, "seconds": round(time.time() - t0, 1)})
+
+
+def notify(text: str):
+    if not seo_config.enabled("notifications") or platform.system() != "Darwin":
+        return
+    subprocess.run(["osascript", "-e",
+                    f'display notification "{text}" with title "n-seo daily run"'],
+                   capture_output=True)
+
+
+def git_autocommit():
+    if not seo_config.enabled("gitAutoCommit"):
+        return
+    # The instance is what has history worth committing; the engine is code.
+    paths = [p for p in ("docs/daily-log.md", "docs/reports", "site") if (INSTANCE / p).exists()]
+    if not paths:
+        return
+    subprocess.run(["git", "add", *paths], cwd=INSTANCE, capture_output=True)
+    staged = subprocess.run(["git", "diff", "--cached", "--quiet"], cwd=INSTANCE).returncode != 0
+    if not staged:
+        return
+    c = subprocess.run(["git", "commit", "-q", "-m", f"Daily refresh {date.today().isoformat()}"],
+                       cwd=INSTANCE, capture_output=True, text=True)
+    if c.returncode != 0:
+        log(f"git: commit FAILED — {(c.stderr or c.stdout).strip()[:200]}")
+        return
+    log("git: committed daily refresh")
+    remotes = subprocess.run(["git", "remote"], cwd=INSTANCE, capture_output=True, text=True).stdout.split()
+    if remotes:
+        p = subprocess.run(["git", "push", "-q"], cwd=INSTANCE, capture_output=True, text=True)
+        log("git: pushed" if p.returncode == 0 else f"git: push FAILED ({p.stderr.strip()[:120]})")
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--list", action="store_true", help="list steps and exit")
+    ap.add_argument("--only", help="comma-separated step names to run")
+    ap.add_argument("--skip", help="comma-separated step names to skip")
+    ap.add_argument("--no-network-wait", action="store_true")
+    args = ap.parse_args()
+
+    enabled_steps = [(n, s) for n, s, gate in STEPS if gate is None or seo_config.enabled(gate)]
+    hooks = seo_config.hooks()
+    if args.list:
+        e = seo_config.engine_info()
+        print(f"  engine {e['version']} ({e['mode']}) — instance {e['instance']}")
+        for c in hooks["beforeRun"]:
+            print(f"  on   hook:before          $ {c}")
+        for n, s, gate in STEPS:
+            on = gate is None or seo_config.enabled(gate)
+            print(f"  {'on ' if on else 'off'}  {n:18s} {s}" + (f"  (modules.{gate})" if gate else ""))
+            for c in hooks["afterStep"].get(n, []):
+                print(f"  {'on ' if on else 'off'}  hook:{n:13s}  $ {c}")
+        for c in hooks["afterRun"]:
+            print(f"  on   hook:after           $ {c}")
+        return 0
+
+    only = set(args.only.split(",")) if args.only else None
+    skip = set(args.skip.split(",")) if args.skip else set()
+    run_hooks_flag = "hooks" not in skip
+    skip.discard("hooks")
+    known = {n for n, _, _ in STEPS}
+    for bad in (only or set()) | skip:
+        if bad not in known:
+            print(f"unknown step {bad!r}; known: {', '.join(known)}")
+            return 2
+    todo = [(n, s) for n, s in enabled_steps if (not only or n in only) and n not in skip]
+    if only and run_hooks_flag and (hooks["beforeRun"] or hooks["afterRun"]):
+        # Hooks are run-level, not step-level: --only narrows the steps but
+        # beforeRun/afterRun still fire, which surprises anyone using --only
+        # as a quick smoke test.
+        print("note: beforeRun/afterRun hooks still run under --only "
+              "(pass --skip hooks to suppress them)", file=sys.stderr)
+    enabled_names = {n for n, _ in enabled_steps}
+    for name in sorted((only or set()) - enabled_names):
+        if name in dict(STEPS):
+            print(f"note: {name} is disabled by its module in the config — skipped", file=sys.stderr)
+
+    log(f"=== daily run {datetime.now():%Y-%m-%d %H:%M} ({len(todo)} steps) ===")
+    if seo_config.using_example():
+        log("note: no n-seo.config.json — running on the example config")
+    if not args.no_network_wait:
+        wait_for_network()
+
+    results, failures = [], []
+    if run_hooks_flag:
+        run_hooks("before", hooks["beforeRun"], results, failures)
+    for name, script in todo:
+        t0 = time.time()
+        log(f"--- {name}")
+        ok = run_step(script)
+        if not ok:
+            log(f"{name} failed, waiting for network and retrying once")
+            # --no-network-wait means "do not sit here"; without this the
+            # retry path waits the full timeout for every failing step.
+            if not args.no_network_wait:
+                wait_for_network()
+            ok = run_step(script)
+            if ok:
+                log(f"{name} recovered on retry")
+        if not ok:
+            failures.append(name)
+            log(f"{name} FAILED")
+        results.append({"name": name, "ok": ok, "seconds": round(time.time() - t0, 1)})
+        if run_hooks_flag:
+            run_hooks(name, hooks["afterStep"].get(name, []), results, failures, step=name)
+
+    def write_last_run():
+        seo_config.DATA.mkdir(parents=True, exist_ok=True)
+        (seo_config.DATA / "last-run.json").write_text(json.dumps({
+            "ts": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%MZ"),
+            "failures": "; ".join(failures),
+            "steps": results,
+        }, indent=1))
+
+    write_last_run()
+    if run_hooks_flag and hooks["afterRun"]:
+        # afterRun sees a complete last-run.json (a mirror sync wants it), and
+        # is then recorded in it too.
+        run_hooks("after", hooks["afterRun"], results, failures)
+        write_last_run()
+
+    if failures:
+        notify(f"{', '.join(failures)} failed — see the Logs page")
+    git_autocommit()
+    log(f"=== done ({len(failures)} failures) ===")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())