n-seo 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/.env.example +13 -0
  2. package/LICENSE +21 -0
  3. package/README.md +184 -0
  4. package/bin/n-seo.mjs +310 -0
  5. package/docs/ADDING-A-SITE.md +82 -0
  6. package/docs/ARCHITECTURE.md +213 -0
  7. package/docs/DEPLOY.md +300 -0
  8. package/docs/FAQ.md +93 -0
  9. package/docs/INSTANCE.md +365 -0
  10. package/docs/MCP.md +104 -0
  11. package/docs/OPERATING-RULES.md +106 -0
  12. package/docs/PLAYBOOK.md +122 -0
  13. package/docs/PRD.md +249 -0
  14. package/docs/RELEASING.md +189 -0
  15. package/docs/SCHEDULING.md +104 -0
  16. package/docs/SETUP-GOOGLE.md +215 -0
  17. package/docs/examples/campaign.json +59 -0
  18. package/docs/examples/draft.md +43 -0
  19. package/docs/screenshots/overview.png +0 -0
  20. package/ingest/__pycache__/analyze_ga4.cpython-313.pyc +0 -0
  21. package/ingest/__pycache__/analyze_gsc.cpython-313.pyc +0 -0
  22. package/ingest/__pycache__/analyze_metadata.cpython-313.pyc +0 -0
  23. package/ingest/__pycache__/analyze_trends.cpython-313.pyc +0 -0
  24. package/ingest/__pycache__/google_auth.cpython-313.pyc +0 -0
  25. package/ingest/__pycache__/http_util.cpython-313.pyc +0 -0
  26. package/ingest/__pycache__/pull_ga4.cpython-313.pyc +0 -0
  27. package/ingest/__pycache__/pull_gsc.cpython-313.pyc +0 -0
  28. package/ingest/__pycache__/pull_index_status.cpython-313.pyc +0 -0
  29. package/ingest/__pycache__/pull_timeseries.cpython-313.pyc +0 -0
  30. package/ingest/__pycache__/seo_config.cpython-313.pyc +0 -0
  31. package/ingest/analyze_ga4.py +79 -0
  32. package/ingest/analyze_gsc.py +136 -0
  33. package/ingest/analyze_metadata.py +158 -0
  34. package/ingest/analyze_trends.py +145 -0
  35. package/ingest/google_auth.py +238 -0
  36. package/ingest/http_util.py +87 -0
  37. package/ingest/pull_ga4.py +107 -0
  38. package/ingest/pull_gsc.py +111 -0
  39. package/ingest/pull_index_status.py +179 -0
  40. package/ingest/pull_timeseries.py +130 -0
  41. package/ingest/seo_config.py +213 -0
  42. package/n-seo.config.example.json +110 -0
  43. package/ops/__pycache__/daily.cpython-313.pyc +0 -0
  44. package/ops/__pycache__/daily_diff.cpython-313.pyc +0 -0
  45. package/ops/__pycache__/demo_data.cpython-313.pyc +0 -0
  46. package/ops/__pycache__/doctor.cpython-313.pyc +0 -0
  47. package/ops/__pycache__/export_static.cpython-313.pyc +0 -0
  48. package/ops/__pycache__/hn_digest.cpython-313.pyc +0 -0
  49. package/ops/__pycache__/indexnow.cpython-313.pyc +0 -0
  50. package/ops/__pycache__/llm.cpython-313.pyc +0 -0
  51. package/ops/__pycache__/opportunity_scan.cpython-313.pyc +0 -0
  52. package/ops/__pycache__/publish.cpython-313.pyc +0 -0
  53. package/ops/__pycache__/reddit_digest.cpython-313.pyc +0 -0
  54. package/ops/daily.py +250 -0
  55. package/ops/daily_diff.py +151 -0
  56. package/ops/demo_data.py +529 -0
  57. package/ops/doctor.py +266 -0
  58. package/ops/export_static.py +125 -0
  59. package/ops/hn_digest.py +169 -0
  60. package/ops/indexnow.py +107 -0
  61. package/ops/install-launchd.sh +76 -0
  62. package/ops/llm.py +139 -0
  63. package/ops/mcp-smoke-stdio.mjs +61 -0
  64. package/ops/opportunity_scan.py +185 -0
  65. package/ops/publish.py +158 -0
  66. package/ops/reddit_digest.py +168 -0
  67. package/ops/templates/n-seo-daily.service +11 -0
  68. package/ops/templates/n-seo-daily.timer +11 -0
  69. package/ops/templates/n-seo-dashboard.service +15 -0
  70. package/ops/templates/n-seo.cron +3 -0
  71. package/ops/templates/n-seo.daily.plist +29 -0
  72. package/ops/templates/n-seo.dashboard.plist +22 -0
  73. package/package.json +77 -0
  74. package/probes/__pycache__/site_probe.cpython-313.pyc +0 -0
  75. package/probes/site_probe.py +201 -0
  76. package/public/favicon.svg +6 -0
  77. package/public/styles.css +632 -0
  78. package/src/actions.ts +255 -0
  79. package/src/backlog.ts +197 -0
  80. package/src/config.ts +220 -0
  81. package/src/data.ts +895 -0
  82. package/src/insights.ts +22 -0
  83. package/src/mcp-stdio.ts +21 -0
  84. package/src/mcp.ts +490 -0
  85. package/src/server.tsx +260 -0
  86. package/src/settings.tsx +329 -0
  87. package/src/views.tsx +1487 -0
  88. package/tsconfig.json +15 -0
@@ -0,0 +1,76 @@
1
+ #!/usr/bin/env bash
2
+ # Install (or remove) the two macOS LaunchAgents for n-seo:
3
+ # n-seo.dashboard — the dashboard, kept alive, starts at login
4
+ # n-seo.daily — ops/daily.py at 07:00 local (fires on wake if missed)
5
+ #
6
+ # Usage: ops/install-launchd.sh install / reinstall
7
+ # ops/install-launchd.sh --uninstall
8
+ #
9
+ # Idempotent: re-running replaces the plists and reloads the jobs.
10
+ set -euo pipefail
11
+
12
+ REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
13
+ TEMPLATES="$REPO/ops/templates"
14
+ AGENTS="$HOME/Library/LaunchAgents"
15
+ LABELS=(n-seo.dashboard n-seo.daily)
16
+ DOMAIN="gui/$(id -u)"
17
+
18
+ if [[ "$(uname)" != "Darwin" ]]; then
19
+ echo "This installer is for macOS launchd. On Linux see docs/SCHEDULING.md (cron / systemd)." >&2
20
+ exit 1
21
+ fi
22
+
23
+ unload() {
24
+ local label="$1" plist="$AGENTS/$1.plist"
25
+ if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then
26
+ launchctl bootout "$DOMAIN/$label" 2>/dev/null \
27
+ || launchctl unload -w "$plist" 2>/dev/null || true
28
+ echo "unloaded $label"
29
+ fi
30
+ }
31
+
32
+ if [[ "${1:-}" == "--uninstall" ]]; then
33
+ for label in "${LABELS[@]}"; do
34
+ unload "$label"
35
+ if [[ -f "$AGENTS/$label.plist" ]]; then
36
+ rm -f "$AGENTS/$label.plist"
37
+ echo "removed $AGENTS/$label.plist"
38
+ fi
39
+ done
40
+ echo "done — n-seo LaunchAgents removed"
41
+ exit 0
42
+ fi
43
+
44
+ NODE="$(command -v node || true)"
45
+ if [[ -z "$NODE" ]]; then
46
+ echo "node not found on PATH — install Node 20+ first" >&2
47
+ exit 1
48
+ fi
49
+ NODE_BIN="$(dirname "$NODE")"
50
+
51
+ mkdir -p "$AGENTS" "$REPO/data"
52
+
53
+ for label in "${LABELS[@]}"; do
54
+ src="$TEMPLATES/$label.plist"
55
+ dst="$AGENTS/$label.plist"
56
+ [[ -f "$src" ]] || { echo "missing template $src" >&2; exit 1; }
57
+ unload "$label"
58
+ sed -e "s|__REPO__|$REPO|g" \
59
+ -e "s|__NODE_BIN__|$NODE_BIN|g" \
60
+ -e "s|__HOME__|$HOME|g" "$src" > "$dst"
61
+ plutil -lint "$dst" >/dev/null
62
+ if launchctl bootstrap "$DOMAIN" "$dst" 2>/dev/null; then
63
+ echo "loaded $label (bootstrap)"
64
+ else
65
+ launchctl load -w "$dst"
66
+ echo "loaded $label (load -w)"
67
+ fi
68
+ done
69
+
70
+ echo
71
+ echo "installed:"
72
+ echo " $AGENTS/n-seo.dashboard.plist → dashboard, log: $REPO/data/dashboard.log"
73
+ echo " $AGENTS/n-seo.daily.plist → ops/daily.py at 07:00, log: $REPO/data/daily-launchd.log"
74
+ echo
75
+ echo "restart the dashboard after code changes:"
76
+ echo " launchctl kickstart -k $DOMAIN/n-seo.dashboard"
package/ops/llm.py ADDED
@@ -0,0 +1,139 @@
1
+ """Optional LLM inference, behind modules.llm. Two ways to reach a model.
2
+
3
+ command (default)
4
+ Any CLI that reads a prompt on stdin and prints a reply on stdout —
5
+ `claude -p --model claude-sonnet-5`, `llm -m gpt-4o`,
6
+ `ollama run llama3`, or a shell script.
7
+
8
+ http
9
+ An HTTP endpoint, for machines with no CLI signed in — which is every
10
+ container and every server you did not log into by hand. Configure:
11
+
12
+ "llm": {
13
+ "enabled": true,
14
+ "http": {
15
+ "provider": "anthropic", // or "openai"
16
+ "model": "claude-sonnet-5",
17
+ "fastModel": "claude-haiku-4-5-20251001",
18
+ "apiKeyEnv": "ANTHROPIC_API_KEY", // read from the env or .env
19
+ "baseUrl": "" // optional, for a gateway
20
+ }
21
+ }
22
+
23
+ `provider: "openai"` speaks the OpenAI chat-completions shape, so it
24
+ also covers the many gateways and local servers that emulate it.
25
+
26
+ `http` wins when it is configured and its key resolves; otherwise the CLI
27
+ path runs. Nothing that comes back is applied automatically: callers turn
28
+ replies into briefings or proposals a human accepts or ignores.
29
+ """
30
+ import json
31
+ import shlex
32
+ import shutil
33
+ import subprocess
34
+ import sys
35
+ from pathlib import Path
36
+
37
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
38
+ import seo_config # noqa: E402
39
+ from http_util import curl_json # noqa: E402
40
+
41
+ ANTHROPIC_VERSION = "2023-06-01"
42
+ MAX_TOKENS = 4096
43
+
44
+
45
+ def _enabled() -> dict | None:
46
+ m = seo_config.module("llm")
47
+ return m if m.get("enabled") else None
48
+
49
+
50
+ def http_config(fast: bool = False) -> dict | None:
51
+ """The resolved HTTP config, or None when it is absent or has no key."""
52
+ m = _enabled()
53
+ if not m:
54
+ return None
55
+ h = m.get("http")
56
+ if not isinstance(h, dict):
57
+ return None
58
+ provider = str(h.get("provider") or "").lower()
59
+ model = (h.get("fastModel") if fast else None) or h.get("model")
60
+ if provider not in ("anthropic", "openai") or not model:
61
+ return None
62
+ key = seo_config.env(str(h.get("apiKeyEnv") or ""))
63
+ if not key:
64
+ return None
65
+ return {"provider": provider, "model": str(model),
66
+ "baseUrl": str(h.get("baseUrl") or "").rstrip("/"), "key": key}
67
+
68
+
69
+ def command(fast: bool = False) -> list[str] | None:
70
+ m = _enabled()
71
+ if not m:
72
+ return None
73
+ cmd = (m.get("fastCommand") if fast else None) or m.get("command") or ""
74
+ parts = shlex.split(str(cmd))
75
+ if not parts or not shutil.which(parts[0]):
76
+ return None
77
+ return parts
78
+
79
+
80
+ def available(fast: bool = False) -> bool:
81
+ return http_config(fast) is not None or command(fast) is not None
82
+
83
+
84
+ def _http_infer(cfg: dict, prompt: str, fast: bool) -> str | None:
85
+ timeout = 120 if fast else 300
86
+ if cfg["provider"] == "anthropic":
87
+ url = (cfg["baseUrl"] or "https://api.anthropic.com") + "/v1/messages"
88
+ args = ["-X", "POST",
89
+ "-H", f"x-api-key: {cfg['key']}",
90
+ "-H", f"anthropic-version: {ANTHROPIC_VERSION}",
91
+ "-H", "Content-Type: application/json",
92
+ "-d", json.dumps({"model": cfg["model"], "max_tokens": MAX_TOKENS,
93
+ "messages": [{"role": "user", "content": prompt}]}),
94
+ url]
95
+ else:
96
+ url = (cfg["baseUrl"] or "https://api.openai.com") + "/v1/chat/completions"
97
+ args = ["-X", "POST",
98
+ "-H", f"Authorization: Bearer {cfg['key']}",
99
+ "-H", "Content-Type: application/json",
100
+ "-d", json.dumps({"model": cfg["model"],
101
+ "messages": [{"role": "user", "content": prompt}]}),
102
+ url]
103
+ try:
104
+ resp = curl_json(args, timeout=timeout, attempts=2, label=f"llm {cfg['provider']}")
105
+ except (RuntimeError, OSError) as exc:
106
+ print(f" llm: {exc}", file=sys.stderr)
107
+ return None
108
+ try:
109
+ if cfg["provider"] == "anthropic":
110
+ out = "".join(b.get("text", "") for b in resp["content"] if b.get("type") == "text")
111
+ else:
112
+ out = resp["choices"][0]["message"]["content"]
113
+ except (KeyError, IndexError, TypeError):
114
+ detail = resp.get("error") if isinstance(resp, dict) else resp
115
+ print(f" llm: unexpected response: {json.dumps(detail)[:200]}", file=sys.stderr)
116
+ return None
117
+ return (out or "").strip() or None
118
+
119
+
120
+ def infer(prompt: str, fast: bool = False) -> str | None:
121
+ """A reply, or None when the module is off, nothing is reachable, or the
122
+ call fails or times out. Never raises: every caller degrades instead."""
123
+ http = http_config(fast)
124
+ if http:
125
+ return _http_infer(http, prompt, fast)
126
+ parts = command(fast)
127
+ if not parts:
128
+ return None
129
+ try:
130
+ p = subprocess.run(parts, input=prompt, capture_output=True, text=True,
131
+ timeout=120 if fast else 300)
132
+ except (subprocess.TimeoutExpired, OSError) as exc:
133
+ print(f" llm: {exc}", file=sys.stderr)
134
+ return None
135
+ if p.returncode != 0:
136
+ print(f" llm: exit {p.returncode}: {p.stderr.strip()[:200]}", file=sys.stderr)
137
+ return None
138
+ out = p.stdout.strip()
139
+ return out or None
@@ -0,0 +1,61 @@
1
+ // End-to-end check of the stdio MCP server against whatever is in data/.
2
+ // Run from the repo root: `npm run mcp:smoke`
3
+ import { Client } from "@modelcontextprotocol/sdk/client/index.js";
4
+ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
5
+
6
+ const transport = new StdioClientTransport({
7
+ command: "npx",
8
+ args: ["tsx", "src/mcp-stdio.ts"],
9
+ cwd: process.cwd(),
10
+ env: process.env,
11
+ });
12
+ const client = new Client({ name: "smoke-test", version: "1.0.0" });
13
+ await client.connect(transport);
14
+
15
+ const { tools } = await client.listTools();
16
+ console.log(`TOOLS (${tools.length}):`);
17
+ for (const t of tools) console.log(` ${t.name.padEnd(24)} ${(t.description ?? "").slice(0, 62)}…`);
18
+
19
+ const { resources } = await client.listResources();
20
+ console.log(`\nRESOURCES (${resources.length}): ${resources.map((r) => r.uri).join(", ")}`);
21
+
22
+ const call = async (name, args = {}) => {
23
+ const r = await client.callTool({ name, arguments: args });
24
+ const text = r.content?.[0]?.text ?? "";
25
+ return { isError: !!r.isError, text };
26
+ };
27
+
28
+ console.log("\n--- settings ---");
29
+ let r = await call("settings");
30
+ const settings = JSON.parse(r.text);
31
+ console.log(` name=${settings.name} sites=${settings.sites.map((s) => s.host).join(",") || "(none)"} modules on: ${Object.entries(settings.modules).filter(([, v]) => v).map(([k]) => k).join(", ") || "(none)"}`);
32
+ const firstHost = settings.sites[0]?.host;
33
+
34
+ console.log("\n--- list_actions {status:active, limit:3} ---");
35
+ r = await call("list_actions", { status: "active", limit: 3 });
36
+ const acts = JSON.parse(r.text);
37
+ console.log(` matched=${acts.matched} returned=${acts.returned}`);
38
+ acts.actions.forEach((a) => console.log(` #${a.rank} +${a.impact} ${a.effort} ${a.host} — ${a.title.slice(0, 54)}`));
39
+
40
+ console.log("\n--- ops_status ---");
41
+ r = await call("ops_status");
42
+ const ops = JSON.parse(r.text);
43
+ console.log(` lastRun=${ops.lastRun?.ts ?? "none"} failures="${ops.lastRun?.failures ?? ""}" probe=${ops.probe ? `${ops.probe.healthy}/${ops.probe.total} healthy` : "none"}`);
44
+ (ops.probe?.sites ?? []).filter((s) => s.findings.length).forEach((s) => console.log(` ${s.host}: ${s.findings.join("; ")}`));
45
+
46
+ if (firstHost) {
47
+ console.log(`\n--- top_queries ${firstHost} 90d limit 3 ---`);
48
+ r = await call("top_queries", { host: firstHost, window: "90d", limit: 3 });
49
+ JSON.parse(r.text).queries.forEach((q) => console.log(` "${q.query}" clicks=${q.clicks} imps=${q.impressions} pos=${(q.position ?? 0).toFixed(1)}`));
50
+ }
51
+
52
+ console.log("\n--- error path: unknown host ---");
53
+ r = await call("site_report", { host: "nope.example.invalid" });
54
+ console.log(` isError=${r.isError} :: ${r.text.slice(0, 70)}`);
55
+
56
+ console.log("\n--- conversions_status ---");
57
+ r = await call("conversions_status");
58
+ console.log(" " + r.text.replace(/\s+/g, " ").slice(0, 130));
59
+
60
+ await client.close();
61
+ console.log("\nstdio transport OK");
@@ -0,0 +1,185 @@
1
+ #!/usr/bin/env python3
2
+ """Automated opportunity discovery.
3
+
4
+ Closes the loop that otherwise needs a human analysis session:
5
+ 1. Refreshes the trend analysis (rising/falling queries, 84d windows).
6
+ 2. SCRIPTED detection: rising queries not covered by any existing queue
7
+ action become candidates.
8
+ 3. INFERENCE (optional, modules.llm): candidates + queue summary + watching
9
+ items go to the model, which returns strictly-JSON *proposals* (new
10
+ queue cards) and *verdicts* on watching items (succeeded / failed /
11
+ keep-watching).
12
+ 4. Output -> data/opportunity-proposals.json, rendered on the Actions page
13
+ as PROPOSED cards. Nothing self-modifies the curated queue — you accept
14
+ proposals into config/backlog.json (the dashboard has a button).
15
+
16
+ Inference runs when there are new candidates, and always on Mondays (weekly
17
+ review of watching items). Without the LLM module it still writes candidates.
18
+ """
19
+
20
+ import json
21
+ import re
22
+ import subprocess
23
+ import sys
24
+ from datetime import date
25
+ from pathlib import Path
26
+
27
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
28
+ import seo_config # noqa: E402
29
+ import llm # noqa: E402
30
+
31
+ ROOT = seo_config.ROOT
32
+ DATA = seo_config.DATA
33
+ RISE_MIN_IMPS = 30 # a riser must reach this many imps in the recent 84d
34
+ RISE_MIN_RATIO = 2.5 # and have grown at least this much vs prior 84d
35
+
36
+
37
+ def latest_trends():
38
+ files = sorted(DATA.glob("trends-*.json"))
39
+ return json.loads(files[-1].read_text()) if files else None
40
+
41
+
42
+ def queue():
43
+ """The live queue from the dashboard (data-derived + curated). Falls back
44
+ to the curated backlog alone when the dashboard is not running."""
45
+ p = subprocess.run(["curl", "-sf", "--max-time", "30", seo_config.dashboard_base() + "/api/actions"],
46
+ capture_output=True, text=True)
47
+ try:
48
+ acts = json.loads(p.stdout)
49
+ if isinstance(acts, list):
50
+ return acts
51
+ except json.JSONDecodeError:
52
+ pass
53
+ try:
54
+ return json.loads((seo_config.INSTANCE / "config" / "backlog.json").read_text()).get("actions", [])
55
+ except (OSError, json.JSONDecodeError):
56
+ return []
57
+
58
+
59
+ def portfolio_description():
60
+ parts = []
61
+ for s in seo_config.sites():
62
+ desc = s["host"]
63
+ if s.get("label") and s["label"] != s["host"]:
64
+ desc += f" ({s['label']})"
65
+ parts.append(desc)
66
+ return ", ".join(parts)
67
+
68
+
69
+ def _host_resolver(prop):
70
+ """query -> the configured host a riser belongs to.
71
+
72
+ A domain property can cover several configured sites, and a url-prefix
73
+ property's slug is not a host at all, so the trend file's key cannot be
74
+ used directly. One site on the property: that site. Several: the host of
75
+ the page with the most impressions for that query in the 90-day
76
+ query x page pull, falling back to the first configured site.
77
+ """
78
+ owners = [s for s in seo_config.sites() if s.get("gscProperty") == prop]
79
+ if not owners:
80
+ return lambda q: seo_config.gsc_slug(prop) # a bare host is the best we have
81
+ if len(owners) == 1:
82
+ return lambda q: owners[0]["host"]
83
+ by_gsc_host = {s["gscHost"]: s["host"] for s in owners}
84
+ best = {}
85
+ qp = seo_config.DATA / "gsc" / seo_config.gsc_data_slug(prop) / "query_page_90d.json"
86
+ try:
87
+ for r in json.loads(qp.read_text()).get("rows", []):
88
+ q, page = r["keys"][0], r["keys"][1]
89
+ h = page.split("/")[2] if page.count("/") >= 2 else ""
90
+ if h in by_gsc_host and r["impressions"] > best.get(q, (0, ""))[0]:
91
+ best[q] = (r["impressions"], by_gsc_host[h])
92
+ except (OSError, json.JSONDecodeError, KeyError, IndexError):
93
+ pass
94
+ return lambda q: best.get(q, (0, owners[0]["host"]))[1]
95
+
96
+
97
+ def main():
98
+ # 1. refresh trends (also keeps the /insights tables current)
99
+ if seo_config.gsc_properties() or seo_config.ga4_properties():
100
+ r = subprocess.run([sys.executable, str(ROOT / "ingest" / "analyze_trends.py")],
101
+ capture_output=True, text=True)
102
+ if r.returncode != 0:
103
+ last = (r.stderr or r.stdout).strip().splitlines()
104
+ print("trend refresh failed:", last[-1][:200] if last else "(no output)",
105
+ "— using the latest trends file on disk")
106
+
107
+ trends = latest_trends()
108
+ if not trends:
109
+ print("no trends file — scan aborted (run ingest/analyze_trends.py once data exists)")
110
+ return 1
111
+ actions = queue()
112
+
113
+ queue_text = " || ".join(
114
+ f"{a.get('title','')} :: {a.get('why','')} :: {' '.join(a.get('spec', []))}" for a in actions
115
+ ).lower()
116
+ active = [a for a in actions if not a.get("watching")]
117
+ watching = [a for a in actions if a.get("watching")]
118
+
119
+ # 2. scripted candidate detection: uncovered risers
120
+ candidates = []
121
+ for site, d in trends.get("sites", {}).items():
122
+ host_for = _host_resolver(site)
123
+ for m in d.get("rising", []):
124
+ if m["recent_imps"] < RISE_MIN_IMPS:
125
+ continue
126
+ if m["prior_imps"] and m["recent_imps"] / max(1, m["prior_imps"]) < RISE_MIN_RATIO:
127
+ continue
128
+ # Substring matching dropped genuine risers: a one-word query
129
+ # ("pricing", "canvas") appears inside some unrelated card's prose
130
+ # and the riser is written off as already covered.
131
+ if re.search(rf"\b{re.escape(m['query'].lower())}\b", queue_text):
132
+ continue # already covered by an action
133
+ candidates.append({"host": host_for(m["query"]), **m})
134
+ candidates.sort(key=lambda c: -c["recent_imps"])
135
+ candidates = candidates[:12]
136
+
137
+ run_inference = (bool(candidates) or date.today().weekday() == 0) and llm.available()
138
+ out = {"generated": date.today().isoformat(), "candidates": candidates,
139
+ "proposals": [], "verdicts": [], "inference_ran": False}
140
+
141
+ if run_inference:
142
+ prompt = (
143
+ "You are the SEO strategist for a small portfolio of websites: "
144
+ + portfolio_description() + ". "
145
+ "Growth levers available: new/updated pages, titles/meta, internal links, llms.txt/AI-crawler surface, "
146
+ "distribution (communities, newsletters, directories). No paid ads.\n\n"
147
+ "UNCOVERED RISING QUERIES (84d vs prior 84d, none matched by existing queue actions):\n"
148
+ + json.dumps(candidates, indent=1)
149
+ + "\n\nEXISTING ACTIVE QUEUE (titles only — do NOT duplicate):\n"
150
+ + json.dumps([a.get("title") for a in active], indent=1)
151
+ + "\n\nWATCHING ITEMS (shipped work + status note; judge each against its note):\n"
152
+ + json.dumps([{"title": a.get("title"), "note": a.get("watching"), "why": a.get("why", "")} for a in watching], indent=1)
153
+ + "\n\nReturn STRICT JSON only, no prose, matching exactly:\n"
154
+ '{"proposals":[{"host":"...","title":"...","kind":"...","why":"... (cite the query numbers)",'
155
+ '"how":"...","spec":["..."],"impact":<int clicks/mo estimate>,"effort":"S|M|L","tag":"content|striking|metadata|distribution|hygiene"}],'
156
+ '"verdicts":[{"title":"<exact watching title>","verdict":"succeeded|failed|keep-watching","evidence":"..."}]}\n'
157
+ "Rules: 0-4 proposals, only where the rising-query evidence genuinely supports a concrete move; "
158
+ "each proposal must name real pages/URLs in spec; impact is an ordering estimate, not a forecast — be conservative; "
159
+ "verdicts only where the data above lets you judge — otherwise keep-watching with a one-line reason."
160
+ )
161
+ raw = llm.infer(prompt) or ""
162
+ m = re.search(r"\{.*\}", raw, re.S)
163
+ if m:
164
+ try:
165
+ parsed = json.loads(m.group(0))
166
+ out["proposals"] = [x for x in parsed.get("proposals", [])
167
+ if x.get("title") and x.get("host") and x.get("spec")]
168
+ out["verdicts"] = parsed.get("verdicts", [])
169
+ out["inference_ran"] = True
170
+ except json.JSONDecodeError:
171
+ print("inference returned unparseable JSON — kept candidates only")
172
+ else:
173
+ print("inference produced no JSON — kept candidates only")
174
+ elif candidates and not llm.available():
175
+ print("llm module off — candidates recorded without proposals")
176
+
177
+ DATA.mkdir(parents=True, exist_ok=True)
178
+ (DATA / "opportunity-proposals.json").write_text(json.dumps(out, indent=1))
179
+ print(f"candidates: {len(candidates)} | proposals: {len(out['proposals'])} | "
180
+ f"verdicts: {len(out['verdicts'])} | inference: {out['inference_ran']}")
181
+ return 0
182
+
183
+
184
+ if __name__ == "__main__":
185
+ sys.exit(main())
package/ops/publish.py ADDED
@@ -0,0 +1,158 @@
1
+ #!/usr/bin/env python3
2
+ """Publish the exported mirror somewhere (modules.publish).
3
+
4
+ `ops/export_static.py` builds a read-only copy of the dashboard into
5
+ `site/`. Getting that copy to wherever people read it — a bucket, an object
6
+ store, a box over ssh — is the other half, and it used to mean writing an
7
+ `afterRun` hook by hand. This makes it a normal pipeline step, so it is
8
+ logged, retried once and recorded in `last-run.json` like everything else.
9
+
10
+ python3 ops/publish.py # publish site/ to modules.publish.destination
11
+
12
+ Targets:
13
+
14
+ gcs gcloud storage rsync site <dest> --recursive
15
+ s3 aws s3 sync site <dest>
16
+ rsync rsync -a site/ <dest>
17
+ command run modules.publish.command — the escape hatch for anything else
18
+
19
+ `delete` adds each tool's "remove what is no longer here" flag, which is what
20
+ makes the mirror match the export rather than accumulate stale pages. Set
21
+ `dryRun` to print the command without running it: that is how you rehearse a
22
+ cutover against a bucket that is already serving something.
23
+
24
+ `env` is merged into the child's environment only — a deployment that keeps
25
+ its cloud credentials in an isolated config directory points at it there,
26
+ without exporting it for the whole daily run. Values are never printed.
27
+ """
28
+
29
+ import os
30
+ import shlex
31
+ import shutil
32
+ import subprocess
33
+ import sys
34
+ from pathlib import Path
35
+
36
+ sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
37
+ import seo_config # noqa: E402
38
+
39
+ TARGETS = ("gcs", "s3", "rsync", "command")
40
+
41
+ # target -> the binary that must be on PATH ("command" runs a shell string)
42
+ BINARY = {"gcs": "gcloud", "s3": "aws", "rsync": "rsync"}
43
+
44
+ DEST_EXAMPLE = {
45
+ "gcs": "gs://your-bucket",
46
+ "s3": "s3://your-bucket",
47
+ "rsync": "user@host:/srv/n-seo-mirror",
48
+ }
49
+
50
+
51
+ def site_dir() -> Path:
52
+ return seo_config.INSTANCE / "site"
53
+
54
+
55
+ def build_command(mod: dict, site: Path):
56
+ """(command, shell) for the configured target.
57
+
58
+ `command` is an argv list for the three built-in targets and a shell
59
+ string for `command`; `shell` says which. Raises ValueError with a
60
+ message meant for the operator, not a stack trace.
61
+ """
62
+ target = (mod.get("target") or "").strip()
63
+ if target not in TARGETS:
64
+ raise ValueError(
65
+ f"unknown publish target {target!r} — use one of: {', '.join(TARGETS)}")
66
+
67
+ if target == "command":
68
+ cmd = (mod.get("command") or "").strip()
69
+ if not cmd:
70
+ raise ValueError(
71
+ "modules.publish.command is empty — target 'command' needs the "
72
+ "shell command that publishes site/")
73
+ return cmd, True
74
+
75
+ dest = (mod.get("destination") or "").strip()
76
+ if not dest:
77
+ raise ValueError(
78
+ f"modules.publish.destination is empty — set it to something like "
79
+ f"{DEST_EXAMPLE[target]} for target {target!r}")
80
+
81
+ delete = bool(mod.get("delete"))
82
+ if target == "gcs":
83
+ argv = ["gcloud", "storage", "rsync", str(site), dest, "--recursive"]
84
+ if delete:
85
+ argv.append("--delete-unmatched-destination-objects")
86
+ elif target == "s3":
87
+ argv = ["aws", "s3", "sync", str(site), dest]
88
+ if delete:
89
+ argv.append("--delete")
90
+ else: # rsync — the trailing slash copies the contents, not the directory
91
+ argv = ["rsync", "-a", str(site) + "/", dest]
92
+ if delete:
93
+ argv.append("--delete")
94
+ return argv, False
95
+
96
+
97
+ def child_env(mod: dict) -> dict:
98
+ """The daily run's environment plus modules.publish.env.
99
+
100
+ `~` and $VARS in the values are expanded so a credentials path can be
101
+ written the way a person would type it.
102
+ """
103
+ env = dict(os.environ)
104
+ for k, v in (mod.get("env") or {}).items():
105
+ if isinstance(k, str) and k:
106
+ env[k] = os.path.expanduser(os.path.expandvars(str(v)))
107
+ return env
108
+
109
+
110
+ def main() -> int:
111
+ if not seo_config.enabled("publish"):
112
+ print("modules.publish is off — nothing to publish")
113
+ return 0
114
+ mod = seo_config.module("publish")
115
+
116
+ site = site_dir()
117
+ try:
118
+ cmd, shell = build_command(mod, site)
119
+ except ValueError as exc:
120
+ print(f"publish: {exc}", file=sys.stderr)
121
+ return 1
122
+
123
+ if not site.is_dir():
124
+ print(f"publish: nothing to publish — {site} does not exist. Enable "
125
+ f"modules.staticExport so the daily run builds it first.",
126
+ file=sys.stderr)
127
+ return 1
128
+
129
+ target = mod["target"]
130
+ binary = BINARY.get(target)
131
+ if binary and shutil.which(binary) is None:
132
+ print(f"publish: {binary!r} is not on PATH — target {target!r} needs it "
133
+ f"(install it, or use target 'command')", file=sys.stderr)
134
+ return 1
135
+
136
+ printable = cmd if shell else " ".join(shlex.quote(a) for a in cmd)
137
+ # Key names only: a publish env is where credentials paths and tokens live.
138
+ env_keys = sorted(k for k in (mod.get("env") or {}) if isinstance(k, str) and k)
139
+ if env_keys:
140
+ print(f"publish: env {', '.join(env_keys)}")
141
+
142
+ if mod.get("dryRun"):
143
+ print(f"publish: DRY RUN — would run:\n {printable}\n"
144
+ f"publish: set modules.publish.dryRun to false to publish for real")
145
+ return 0
146
+
147
+ print(f"publish: {printable}")
148
+ p = subprocess.run(cmd, shell=shell, cwd=seo_config.INSTANCE, env=child_env(mod))
149
+ if p.returncode != 0:
150
+ print(f"publish: FAILED (exit {p.returncode}) — {target} → "
151
+ f"{mod.get('destination') or 'command'}", file=sys.stderr)
152
+ return 1
153
+ print(f"publish: {site.name}/ → {mod.get('destination') or 'command'}")
154
+ return 0
155
+
156
+
157
+ if __name__ == "__main__":
158
+ sys.exit(main())