n-seo 0.1.0

package/ops/reddit_digest.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+"""Daily Reddit comment-opportunity digest (modules.reddit).
+
+Mirrors ops/hn_digest.py for the subreddits where your audience lives.
+Reddit blocks unauthenticated JSON from most IPs, so this needs a free
+"script" app (https://www.reddit.com/prefs/apps) with REDDIT_CLIENT_ID /
+REDDIT_CLIENT_SECRET in .env. Briefings only — the comments are yours; the
+prompt forbids generating participation text.
+
+Config: modules.reddit.topics = [[subreddit, query, why-you], ...],
+modules.reddit.user = your username (marks threads you already joined).
+"""
+
+import json
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from urllib.parse import quote
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
+import seo_config  # noqa: E402
+import llm  # noqa: E402
+
+UA = "n-seo/0.1 (participation research)"
+MAX_AGE_DAYS = 7
+MAX_COMMENTS = 80  # not already saturated; early threads are prime real estate
+MAX_PICKS = 6
+
+_TOKEN = None
+
+
+def oauth_token():
+    """App-only OAuth token; '' when no credentials are configured."""
+    global _TOKEN
+    if _TOKEN is not None:
+        return _TOKEN
+    cid, secret = seo_config.env("REDDIT_CLIENT_ID"), seo_config.env("REDDIT_CLIENT_SECRET")
+    if not cid or not secret:
+        _TOKEN = ""
+        return ""
+    p = subprocess.run(
+        ["curl", "-s", "--max-time", "20", "-A", UA, "-u", f"{cid}:{secret}",
+         "-d", "grant_type=client_credentials",
+         "https://www.reddit.com/api/v1/access_token"],
+        capture_output=True, text=True)
+    try:
+        _TOKEN = json.loads(p.stdout).get("access_token", "")
+    except json.JSONDecodeError:
+        _TOKEN = ""
+    return _TOKEN
+
+
+def get_json(url):
+    token = oauth_token()
+    if token:
+        url = url.replace("https://www.reddit.com/", "https://oauth.reddit.com/")
+        cmd = ["curl", "-s", "--max-time", "20", "-A", UA,
+               "-H", f"Authorization: Bearer {token}", url]
+    else:
+        cmd = ["curl", "-s", "--max-time", "20", "-A", UA, url]
+    p = subprocess.run(cmd, capture_output=True, text=True)
+    try:
+        return json.loads(p.stdout)
+    except json.JSONDecodeError:
+        return {}
+
+
+def search(sub, query):
+    url = (f"https://www.reddit.com/r/{sub}/search.json?q={quote(query)}"
+           f"&restrict_sr=on&sort=new&t=week&limit=8&raw_json=1")
+    children = (get_json(url).get("data") or {}).get("children") or []
+    return [c.get("data", {}) for c in children if c.get("kind") == "t3"]
+
+
+def my_commented_links(user):
+    if not user:
+        return set()
+    url = f"https://www.reddit.com/user/{user}/comments.json?limit=100&raw_json=1"
+    children = (get_json(url).get("data") or {}).get("children") or []
+    return {c.get("data", {}).get("link_id", "").replace("t3_", "") for c in children}
+
+
+def briefing(pick, expertise):
+    body = (pick.get("selftext") or "")[:3000]
+    prompt = (
+        f"About the reader (first-hand expertise): {expertise}\n\n"
+        f"Reddit thread in r/{pick['sub']}: {pick['title']}\n"
+        f"Post body:\n{body or '(link post or empty body)'}\n\n"
+        "Write a briefing for the reader in EXACTLY this format, one line each:\n"
+        "GIST: <one sentence — what the post is asking or saying>\n"
+        "ANGLE: <1-2 sentences — where the reader's genuine first-hand experience connects, and "
+        "whether linking one of their own resources would be welcome in this sub or read as "
+        "self-promo. If their expertise does not genuinely connect, say 'weak fit — skip "
+        "unless personally interested.'>\n"
+        "Do NOT write any comment text or suggested wording — briefing only."
+    )
+    out = llm.infer(prompt, fast=True) or ""
+    return out if "GIST:" in out else ""
+
+
+def main():
+    m = seo_config.module("reddit")
+    if not m.get("enabled"):
+        print("reddit module disabled (modules.reddit.enabled) — skipping")
+        return 0
+    topics = [t for t in (m.get("topics") or []) if isinstance(t, list) and len(t) >= 2 and t[0]]
+    if not topics:
+        print("reddit: no topics configured — add [subreddit, query, why] rows in Settings")
+        return 0
+    expertise = (seo_config.load()["participation"] or {}).get("expertise", "").strip()
+    user = (m.get("user") or "").strip()
+
+    now = datetime.now(timezone.utc).timestamp()
+    picks, seen = [], set()
+    for sub, query, *rest in topics:
+        why = rest[0] if rest else ""
+        for d in search(sub, query):
+            pid = d.get("id")
+            if not pid or pid in seen:
+                continue
+            age_days = (now - (d.get("created_utc") or now)) / 86400
+            if age_days > MAX_AGE_DAYS or (d.get("num_comments") or 0) >= MAX_COMMENTS:
+                continue
+            seen.add(pid)
+            picks.append({
+                "id": pid,
+                "title": d.get("title", ""),
+                "url": f"https://www.reddit.com{d.get('permalink', '')}",
+                "sub": d.get("subreddit", sub),
+                "comments": d.get("num_comments", 0),
+                "score": d.get("score", 0),
+                "age_days": round(age_days, 1),
+                "why": why,
+                "selftext": d.get("selftext", ""),
+            })
+    # freshest, least-saturated, still-alive threads first
+    picks.sort(key=lambda p: (-(p["score"] + 2 * p["comments"]) / (1 + p["age_days"])))
+    picks = picks[:MAX_PICKS]
+
+    commented = my_commented_links(user)
+    can_brief = llm.available(fast=True) and bool(expertise)
+    if picks and not can_brief:
+        print("briefings skipped: " + ("participation.expertise is empty" if not expertise
+                                       else "llm module off"))
+    for p in picks:
+        p["commented"] = p["id"] in commented
+        p["briefing"] = briefing(p, expertise) if (can_brief and not p["commented"]) else ""
+        p.pop("selftext", None)
+
+    seo_config.DATA.mkdir(parents=True, exist_ok=True)
+    (seo_config.DATA / "reddit-digest.json").write_text(json.dumps(
+        {"generated": datetime.now(timezone.utc).isoformat(timespec="minutes"),
+         "user": user or None, "auth": bool(oauth_token()), "picks": picks}, indent=1))
+    for p in picks:
+        print(f"- r/{p['sub']}: {p['title']} ({p['comments']}c/{p['score']}pts, {p['age_days']}d) {p['url']}")
+    if not picks:
+        if not oauth_token():
+            print("no picks — Reddit blocks unauthenticated JSON from most IPs. One-time fix: "
+                  "create a 'script' app at https://www.reddit.com/prefs/apps and add "
+                  "REDDIT_CLIENT_ID / REDDIT_CLIENT_SECRET to .env")
+        else:
+            print("no matching active threads this week")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/ops/templates/n-seo-daily.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=n-seo daily run (pull, probe, audit, log)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=__REPO__
+ExecStart=/usr/bin/python3 ops/daily.py
+StandardOutput=append:__REPO__/data/daily-systemd.log
+StandardError=append:__REPO__/data/daily-systemd.log

package/ops/templates/n-seo-daily.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run n-seo daily at 07:00
+
+[Timer]
+OnCalendar=*-*-* 07:00:00
+# Run a missed slot at the next boot instead of skipping the day.
+Persistent=true
+RandomizedDelaySec=120
+
+[Install]
+WantedBy=timers.target

package/ops/templates/n-seo-dashboard.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=n-seo dashboard
+After=network.target
+
+[Service]
+WorkingDirectory=__REPO__
+# npx resolves the tsx pinned in package.json; adjust the path if node is not on PATH for user units.
+ExecStart=/usr/bin/env npx tsx src/server.tsx
+Restart=always
+RestartSec=3
+StandardOutput=append:__REPO__/data/dashboard.log
+StandardError=append:__REPO__/data/dashboard.log
+
+[Install]
+WantedBy=default.target

package/ops/templates/n-seo.cron

@@ -0,0 +1,3 @@
+# n-seo daily run at 07:00 local. Paste into `crontab -e`, fix the path.
+# Cron does not run missed jobs; a machine that is off at 07:00 skips that day.
+0 7 * * * cd /path/to/n-seo && /usr/bin/python3 ops/daily.py >> data/daily-cron.log 2>&1

package/ops/templates/n-seo.daily.plist

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>n-seo.daily</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/sh</string>
+    <string>-c</string>
+    <string>export PATH="__NODE_BIN__:/usr/local/bin:/usr/bin:/bin"; export HOME="__HOME__"; export USER="${USER:-$(id -un)}"; export TMPDIR="${TMPDIR:-/tmp}"; cd "__REPO__" &amp;&amp; exec python3 ops/daily.py</string>
+  </array>
+  <!-- 07:00 local. A slot missed while asleep fires on wake; the run then
+       waits for the network before it starts. -->
+  <key>StartCalendarInterval</key>
+  <dict>
+    <key>Hour</key>
+    <integer>7</integer>
+    <key>Minute</key>
+    <integer>0</integer>
+  </dict>
+  <key>RunAtLoad</key>
+  <false/>
+  <key>StandardOutPath</key>
+  <string>__REPO__/data/daily-launchd.log</string>
+  <key>StandardErrorPath</key>
+  <string>__REPO__/data/daily-launchd.log</string>
+</dict>
+</plist>

package/ops/templates/n-seo.dashboard.plist

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>n-seo.dashboard</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/sh</string>
+    <string>-c</string>
+    <string>export PATH="__NODE_BIN__:/usr/local/bin:/usr/bin:/bin"; export HOME="__HOME__"; cd "__REPO__" &amp;&amp; exec npx tsx src/server.tsx</string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+  <key>KeepAlive</key>
+  <true/>
+  <key>StandardOutPath</key>
+  <string>__REPO__/data/dashboard.log</string>
+  <key>StandardErrorPath</key>
+  <string>__REPO__/data/dashboard.log</string>
+</dict>
+</plist>

package/package.json

@@ -0,0 +1,77 @@
+{
+  "name": "n-seo",
+  "version": "0.1.0",
+  "description": "Local-first SEO / AEO / GEO control plane: pulls Search Console + GA4, probes your sites, and turns the data into a ranked action queue you can act on without an agency.",
+  "license": "MIT",
+  "author": "En Dash Consulting (https://endash.us)",
+  "homepage": "https://en-dash-consulting.github.io/n-seo/",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/en-dash-consulting/n-seo.git"
+  },
+  "bugs": {
+    "url": "https://github.com/en-dash-consulting/n-seo/issues"
+  },
+  "keywords": [
+    "seo",
+    "aeo",
+    "geo",
+    "search-console",
+    "google-analytics",
+    "ga4",
+    "llms-txt",
+    "ai-crawlers",
+    "mcp",
+    "dashboard",
+    "local-first",
+    "cli"
+  ],
+  "type": "module",
+  "bin": {
+    "n-seo": "bin/n-seo.mjs"
+  },
+  "files": [
+    "bin/",
+    "src/",
+    "tsconfig.json",
+    "ingest/",
+    "ops/",
+    "probes/",
+    "public/",
+    "n-seo.config.example.json",
+    ".env.example",
+    "docs/",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "dev": "tsx watch src/server.tsx",
+    "start": "tsx src/server.tsx",
+    "typecheck": "tsc --noEmit",
+    "mcp": "tsx src/mcp-stdio.ts",
+    "mcp:smoke": "node ops/mcp-smoke-stdio.mjs",
+    "doctor": "python3 ops/doctor.py",
+    "demo": "python3 ops/demo_data.py",
+    "daily": "python3 ops/daily.py",
+    "test": "tsx --test tests/ts/*.test.ts",
+    "test:py": "python3 -m unittest discover -s tests/py -v",
+    "check": "npm run typecheck && npm test && npm run test:py",
+    "export": "python3 ops/export_static.py",
+    "docker:build": "docker build -t n-seo:latest .",
+    "docker:up": "docker compose -f docker/compose.yml up -d"
+  },
+  "dependencies": {
+    "@hono/node-server": "^1.13.7",
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "hono": "^4.6.14",
+    "tsx": "^4.19.2",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "@types/node": "^26.4.1",
+    "typescript": "^7.0.2"
+  }
+}

package/probes/site_probe.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""No-auth SEO health probe for every configured site.
+
+Checks each site's robots.txt (incl. AI-crawler rules), sitemap.xml, llms.txt,
+llms-full.txt, homepage HTML (title, meta description, canonical, OG, JSON-LD,
+server-rendered text) and 404 behaviour. Prints a summary and writes a
+timestamped JSON snapshot to data/probes/ so regressions can be diffed
+day over day — deploys are where SEO quietly dies.
+
+Stdlib only — no pip installs needed.
+"""
+
+import json
+import re
+import sys
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
+import seo_config  # noqa: E402
+from http_util import fetch_text  # noqa: E402
+
+# gscHost rather than host so a site served on www is probed where it lives.
+SITES = [f"https://{s['gscHost']}" for s in seo_config.sites()]
+
+AI_CRAWLERS = [
+    "GPTBot", "ClaudeBot", "Claude-Web", "PerplexityBot",
+    "Google-Extended", "CCBot", "Bytespider", "OAI-SearchBot",
+]
+
+UA = "Mozilla/5.0 (compatible; n-seo-probe/0.1)"
+
+
+def fetch(url, timeout=15):
+    return fetch_text(url, timeout=timeout, ua=UA)
+
+
+class MetaParser(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.title = ""
+        self._in_title = False
+        self.meta_description = None
+        self.canonical = None
+        self.og = {}
+        self.jsonld_types = []
+        self._in_jsonld = False
+        self.h1s = []
+        self._in_h1 = False
+        self.lang = None
+
+    def handle_starttag(self, tag, attrs):
+        a = dict(attrs)
+        if tag == "html":
+            self.lang = a.get("lang")
+        elif tag == "title":
+            self._in_title = True
+        elif tag == "h1":
+            self._in_h1 = True
+            self.h1s.append("")
+        elif tag == "meta":
+            if a.get("name") == "description":
+                self.meta_description = a.get("content")
+            prop = a.get("property", "")
+            if prop.startswith("og:"):
+                self.og[prop] = a.get("content")
+        elif tag == "link" and a.get("rel") == "canonical":
+            self.canonical = a.get("href")
+        elif tag == "script" and a.get("type") == "application/ld+json":
+            self._in_jsonld = True
+
+    def handle_endtag(self, tag):
+        if tag == "title":
+            self._in_title = False
+        elif tag == "h1":
+            self._in_h1 = False
+        elif tag == "script":
+            self._in_jsonld = False
+
+    def handle_data(self, data):
+        if self._in_title:
+            self.title += data
+        if self._in_h1 and self.h1s:
+            self.h1s[-1] += data
+        if self._in_jsonld:
+            try:
+                d = json.loads(data)
+                items = d if isinstance(d, list) else [d]
+                for item in items:
+                    t = item.get("@type")
+                    if t:
+                        self.jsonld_types.append(t)
+            except (json.JSONDecodeError, AttributeError):
+                pass
+
+
+def visible_text_bytes(html):
+    """Rough server-rendering signal: bytes of visible text once tags and
+    scripts are stripped. A JS shell scores near zero — and AI crawlers
+    don't run JS."""
+    stripped = re.sub(r"<(script|style|noscript)[^>]*>.*?</\1>", " ", html,
+                      flags=re.DOTALL | re.IGNORECASE)
+    stripped = re.sub(r"<[^>]+>", " ", stripped)
+    return len(re.sub(r"\s+", " ", stripped).strip())
+
+
+def probe_site(base):
+    result = {"site": base}
+
+    status, body = fetch(base + "/robots.txt")
+    robots = {"status": status, "exists": status == 200}
+    if status == 200:
+        robots["sitemap_declared"] = "sitemap:" in body.lower()
+        blocked = []
+        for bot in AI_CRAWLERS:
+            m = re.search(rf"user-agent:\s*{re.escape(bot)}\s*\n(?:(?!user-agent:)[^\n]*\n)*?\s*disallow:\s*/\s*$",
+                          body, re.IGNORECASE | re.MULTILINE)
+            if m:
+                blocked.append(bot)
+        robots["ai_crawlers_blocked"] = blocked
+    result["robots"] = robots
+
+    status, body = fetch(base + "/sitemap.xml")
+    sitemap = {"status": status,
+               "exists": bool(status == 200 and ("<urlset" in body or "<sitemapindex" in body))}
+    if status == 200:
+        sitemap["url_count"] = body.count("<loc>")
+        lastmods = re.findall(r"<lastmod>([^<]+)</lastmod>", body)
+        sitemap["newest_lastmod"] = max(lastmods) if lastmods else None
+    result["sitemap"] = sitemap
+
+    for f in ("llms.txt", "llms-full.txt"):
+        status, body = fetch(f"{base}/{f}")
+        ok = status == 200 and not body.lstrip().lower().startswith("<!doctype")
+        result[f] = {"status": status, "exists": ok, "bytes": len(body) if ok else 0}
+
+    status, body = fetch(base + "/")
+    home = {"status": status}
+    if status == 200:
+        p = MetaParser()
+        try:
+            p.feed(body)
+        except Exception:
+            pass
+        home.update({
+            "title": p.title.strip()[:120],
+            "meta_description": (p.meta_description or "")[:200] or None,
+            "canonical": p.canonical,
+            "og_tags": len(p.og),
+            "jsonld_types": p.jsonld_types,
+            "h1_count": len(p.h1s),
+            "lang": p.lang,
+            "visible_text_bytes": visible_text_bytes(body),
+        })
+    result["homepage"] = home
+
+    status, _ = fetch(base + "/definitely-not-a-real-page-9x7q")
+    result["soft_404"] = {"status": status, "real_404": status == 404}
+
+    return result
+
+
+def main():
+    if not SITES:
+        print("no sites configured — nothing to probe")
+        return 0
+    snapshot = {
+        "probed_at": datetime.now(timezone.utc).isoformat(),
+        "sites": [probe_site(s) for s in SITES],
+    }
+
+    out_dir = seo_config.DATA / "probes"
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out = out_dir / f"probe-{datetime.now(timezone.utc):%Y%m%d-%H%M%S}.json"
+    out.write_text(json.dumps(snapshot, indent=2))
+
+    for s in snapshot["sites"]:
+        h = s["homepage"]
+        print(f"\n{s['site']}")
+        print(f"  robots.txt: {'OK' if s['robots']['exists'] else 'MISSING'}"
+              + (f" (blocks AI: {s['robots']['ai_crawlers_blocked']})"
+                 if s['robots'].get('ai_crawlers_blocked') else ""))
+        print(f"  sitemap.xml: {'OK, %s URLs' % s['sitemap'].get('url_count') if s['sitemap'].get('exists') else 'MISSING'}")
+        print(f"  llms.txt: {'OK' if s['llms.txt']['exists'] else 'missing'}"
+              f" | llms-full.txt: {'OK' if s['llms-full.txt']['exists'] else 'missing'}")
+        if h.get("status") == 200:
+            print(f"  title: {h.get('title') or 'MISSING'}")
+            print(f"  meta description: {'OK' if h.get('meta_description') else 'MISSING'}"
+                  f" | canonical: {'OK' if h.get('canonical') else 'MISSING'}"
+                  f" | JSON-LD: {h.get('jsonld_types') or 'none'}")
+            print(f"  visible text: {h.get('visible_text_bytes')} bytes"
+                  f" ({'likely JS shell' if h.get('visible_text_bytes', 0) < 500 else 'server-rendered content'})")
+        print(f"  404 handling: {'real 404' if s['soft_404']['real_404'] else 'SOFT 404 (status %s)' % s['soft_404']['status']}")
+
+    print(f"\nSnapshot written: {out}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/public/favicon.svg

@@ -0,0 +1,6 @@
+<svg viewBox="0 0 300 300" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M300 0H0V300H300V0Z" fill="#00E5B9"/>
+<path d="M300 239.902H32.0074V255.489H300V239.902Z" fill="#00FFCE"/>
+<path d="M55.7684 202.351L118.59 139.53L160.47 181.41L244.232 97.6489" stroke="#001769" stroke-width="27.4428" stroke-linecap="round" stroke-linejoin="round"/>
+<path d="M181.411 97.6489H244.232V160.47" stroke="#001769" stroke-width="27.4428" stroke-linecap="round" stroke-linejoin="round"/>
+</svg>