n-seo 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +13 -0
- package/LICENSE +21 -0
- package/README.md +184 -0
- package/bin/n-seo.mjs +310 -0
- package/docs/ADDING-A-SITE.md +82 -0
- package/docs/ARCHITECTURE.md +213 -0
- package/docs/DEPLOY.md +300 -0
- package/docs/FAQ.md +93 -0
- package/docs/INSTANCE.md +365 -0
- package/docs/MCP.md +104 -0
- package/docs/OPERATING-RULES.md +106 -0
- package/docs/PLAYBOOK.md +122 -0
- package/docs/PRD.md +249 -0
- package/docs/RELEASING.md +189 -0
- package/docs/SCHEDULING.md +104 -0
- package/docs/SETUP-GOOGLE.md +215 -0
- package/docs/examples/campaign.json +59 -0
- package/docs/examples/draft.md +43 -0
- package/docs/screenshots/overview.png +0 -0
- package/ingest/__pycache__/analyze_ga4.cpython-313.pyc +0 -0
- package/ingest/__pycache__/analyze_gsc.cpython-313.pyc +0 -0
- package/ingest/__pycache__/analyze_metadata.cpython-313.pyc +0 -0
- package/ingest/__pycache__/analyze_trends.cpython-313.pyc +0 -0
- package/ingest/__pycache__/google_auth.cpython-313.pyc +0 -0
- package/ingest/__pycache__/http_util.cpython-313.pyc +0 -0
- package/ingest/__pycache__/pull_ga4.cpython-313.pyc +0 -0
- package/ingest/__pycache__/pull_gsc.cpython-313.pyc +0 -0
- package/ingest/__pycache__/pull_index_status.cpython-313.pyc +0 -0
- package/ingest/__pycache__/pull_timeseries.cpython-313.pyc +0 -0
- package/ingest/__pycache__/seo_config.cpython-313.pyc +0 -0
- package/ingest/analyze_ga4.py +79 -0
- package/ingest/analyze_gsc.py +136 -0
- package/ingest/analyze_metadata.py +158 -0
- package/ingest/analyze_trends.py +145 -0
- package/ingest/google_auth.py +238 -0
- package/ingest/http_util.py +87 -0
- package/ingest/pull_ga4.py +107 -0
- package/ingest/pull_gsc.py +111 -0
- package/ingest/pull_index_status.py +179 -0
- package/ingest/pull_timeseries.py +130 -0
- package/ingest/seo_config.py +213 -0
- package/n-seo.config.example.json +110 -0
- package/ops/__pycache__/daily.cpython-313.pyc +0 -0
- package/ops/__pycache__/daily_diff.cpython-313.pyc +0 -0
- package/ops/__pycache__/demo_data.cpython-313.pyc +0 -0
- package/ops/__pycache__/doctor.cpython-313.pyc +0 -0
- package/ops/__pycache__/export_static.cpython-313.pyc +0 -0
- package/ops/__pycache__/hn_digest.cpython-313.pyc +0 -0
- package/ops/__pycache__/indexnow.cpython-313.pyc +0 -0
- package/ops/__pycache__/llm.cpython-313.pyc +0 -0
- package/ops/__pycache__/opportunity_scan.cpython-313.pyc +0 -0
- package/ops/__pycache__/publish.cpython-313.pyc +0 -0
- package/ops/__pycache__/reddit_digest.cpython-313.pyc +0 -0
- package/ops/daily.py +250 -0
- package/ops/daily_diff.py +151 -0
- package/ops/demo_data.py +529 -0
- package/ops/doctor.py +266 -0
- package/ops/export_static.py +125 -0
- package/ops/hn_digest.py +169 -0
- package/ops/indexnow.py +107 -0
- package/ops/install-launchd.sh +76 -0
- package/ops/llm.py +139 -0
- package/ops/mcp-smoke-stdio.mjs +61 -0
- package/ops/opportunity_scan.py +185 -0
- package/ops/publish.py +158 -0
- package/ops/reddit_digest.py +168 -0
- package/ops/templates/n-seo-daily.service +11 -0
- package/ops/templates/n-seo-daily.timer +11 -0
- package/ops/templates/n-seo-dashboard.service +15 -0
- package/ops/templates/n-seo.cron +3 -0
- package/ops/templates/n-seo.daily.plist +29 -0
- package/ops/templates/n-seo.dashboard.plist +22 -0
- package/package.json +77 -0
- package/probes/__pycache__/site_probe.cpython-313.pyc +0 -0
- package/probes/site_probe.py +201 -0
- package/public/favicon.svg +6 -0
- package/public/styles.css +632 -0
- package/src/actions.ts +255 -0
- package/src/backlog.ts +197 -0
- package/src/config.ts +220 -0
- package/src/data.ts +895 -0
- package/src/insights.ts +22 -0
- package/src/mcp-stdio.ts +21 -0
- package/src/mcp.ts +490 -0
- package/src/server.tsx +260 -0
- package/src/settings.tsx +329 -0
- package/src/views.tsx +1487 -0
- package/tsconfig.json +15 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# Install (or remove) the two macOS LaunchAgents for n-seo:
|
|
3
|
+
# n-seo.dashboard — the dashboard, kept alive, starts at login
|
|
4
|
+
# n-seo.daily — ops/daily.py at 07:00 local (fires on wake if missed)
|
|
5
|
+
#
|
|
6
|
+
# Usage: ops/install-launchd.sh install / reinstall
|
|
7
|
+
# ops/install-launchd.sh --uninstall
|
|
8
|
+
#
|
|
9
|
+
# Idempotent: re-running replaces the plists and reloads the jobs.
|
|
10
|
+
set -euo pipefail
|
|
11
|
+
|
|
12
|
+
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
13
|
+
TEMPLATES="$REPO/ops/templates"
|
|
14
|
+
AGENTS="$HOME/Library/LaunchAgents"
|
|
15
|
+
LABELS=(n-seo.dashboard n-seo.daily)
|
|
16
|
+
DOMAIN="gui/$(id -u)"
|
|
17
|
+
|
|
18
|
+
if [[ "$(uname)" != "Darwin" ]]; then
|
|
19
|
+
echo "This installer is for macOS launchd. On Linux see docs/SCHEDULING.md (cron / systemd)." >&2
|
|
20
|
+
exit 1
|
|
21
|
+
fi
|
|
22
|
+
|
|
23
|
+
unload() {
|
|
24
|
+
local label="$1" plist="$AGENTS/$1.plist"
|
|
25
|
+
if launchctl print "$DOMAIN/$label" >/dev/null 2>&1; then
|
|
26
|
+
launchctl bootout "$DOMAIN/$label" 2>/dev/null \
|
|
27
|
+
|| launchctl unload -w "$plist" 2>/dev/null || true
|
|
28
|
+
echo "unloaded $label"
|
|
29
|
+
fi
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
if [[ "${1:-}" == "--uninstall" ]]; then
|
|
33
|
+
for label in "${LABELS[@]}"; do
|
|
34
|
+
unload "$label"
|
|
35
|
+
if [[ -f "$AGENTS/$label.plist" ]]; then
|
|
36
|
+
rm -f "$AGENTS/$label.plist"
|
|
37
|
+
echo "removed $AGENTS/$label.plist"
|
|
38
|
+
fi
|
|
39
|
+
done
|
|
40
|
+
echo "done — n-seo LaunchAgents removed"
|
|
41
|
+
exit 0
|
|
42
|
+
fi
|
|
43
|
+
|
|
44
|
+
NODE="$(command -v node || true)"
|
|
45
|
+
if [[ -z "$NODE" ]]; then
|
|
46
|
+
echo "node not found on PATH — install Node 20+ first" >&2
|
|
47
|
+
exit 1
|
|
48
|
+
fi
|
|
49
|
+
NODE_BIN="$(dirname "$NODE")"
|
|
50
|
+
|
|
51
|
+
mkdir -p "$AGENTS" "$REPO/data"
|
|
52
|
+
|
|
53
|
+
for label in "${LABELS[@]}"; do
|
|
54
|
+
src="$TEMPLATES/$label.plist"
|
|
55
|
+
dst="$AGENTS/$label.plist"
|
|
56
|
+
[[ -f "$src" ]] || { echo "missing template $src" >&2; exit 1; }
|
|
57
|
+
unload "$label"
|
|
58
|
+
sed -e "s|__REPO__|$REPO|g" \
|
|
59
|
+
-e "s|__NODE_BIN__|$NODE_BIN|g" \
|
|
60
|
+
-e "s|__HOME__|$HOME|g" "$src" > "$dst"
|
|
61
|
+
plutil -lint "$dst" >/dev/null
|
|
62
|
+
if launchctl bootstrap "$DOMAIN" "$dst" 2>/dev/null; then
|
|
63
|
+
echo "loaded $label (bootstrap)"
|
|
64
|
+
else
|
|
65
|
+
launchctl load -w "$dst"
|
|
66
|
+
echo "loaded $label (load -w)"
|
|
67
|
+
fi
|
|
68
|
+
done
|
|
69
|
+
|
|
70
|
+
echo
|
|
71
|
+
echo "installed:"
|
|
72
|
+
echo " $AGENTS/n-seo.dashboard.plist → dashboard, log: $REPO/data/dashboard.log"
|
|
73
|
+
echo " $AGENTS/n-seo.daily.plist → ops/daily.py at 07:00, log: $REPO/data/daily-launchd.log"
|
|
74
|
+
echo
|
|
75
|
+
echo "restart the dashboard after code changes:"
|
|
76
|
+
echo " launchctl kickstart -k $DOMAIN/n-seo.dashboard"
|
package/ops/llm.py
ADDED
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Optional LLM inference, behind modules.llm. Two ways to reach a model.
|
|
2
|
+
|
|
3
|
+
command (default)
|
|
4
|
+
Any CLI that reads a prompt on stdin and prints a reply on stdout —
|
|
5
|
+
`claude -p --model claude-sonnet-5`, `llm -m gpt-4o`,
|
|
6
|
+
`ollama run llama3`, or a shell script.
|
|
7
|
+
|
|
8
|
+
http
|
|
9
|
+
An HTTP endpoint, for machines with no CLI signed in — which is every
|
|
10
|
+
container and every server you did not log into by hand. Configure:
|
|
11
|
+
|
|
12
|
+
"llm": {
|
|
13
|
+
"enabled": true,
|
|
14
|
+
"http": {
|
|
15
|
+
"provider": "anthropic", // or "openai"
|
|
16
|
+
"model": "claude-sonnet-5",
|
|
17
|
+
"fastModel": "claude-haiku-4-5-20251001",
|
|
18
|
+
"apiKeyEnv": "ANTHROPIC_API_KEY", // read from the env or .env
|
|
19
|
+
"baseUrl": "" // optional, for a gateway
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
`provider: "openai"` speaks the OpenAI chat-completions shape, so it
|
|
24
|
+
also covers the many gateways and local servers that emulate it.
|
|
25
|
+
|
|
26
|
+
`http` wins when it is configured and its key resolves; otherwise the CLI
|
|
27
|
+
path runs. Nothing that comes back is applied automatically: callers turn
|
|
28
|
+
replies into briefings or proposals a human accepts or ignores.
|
|
29
|
+
"""
|
|
30
|
+
import json
|
|
31
|
+
import shlex
|
|
32
|
+
import shutil
|
|
33
|
+
import subprocess
|
|
34
|
+
import sys
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
|
|
38
|
+
import seo_config # noqa: E402
|
|
39
|
+
from http_util import curl_json # noqa: E402
|
|
40
|
+
|
|
41
|
+
ANTHROPIC_VERSION = "2023-06-01"
|
|
42
|
+
MAX_TOKENS = 4096
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _enabled() -> dict | None:
|
|
46
|
+
m = seo_config.module("llm")
|
|
47
|
+
return m if m.get("enabled") else None
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def http_config(fast: bool = False) -> dict | None:
|
|
51
|
+
"""The resolved HTTP config, or None when it is absent or has no key."""
|
|
52
|
+
m = _enabled()
|
|
53
|
+
if not m:
|
|
54
|
+
return None
|
|
55
|
+
h = m.get("http")
|
|
56
|
+
if not isinstance(h, dict):
|
|
57
|
+
return None
|
|
58
|
+
provider = str(h.get("provider") or "").lower()
|
|
59
|
+
model = (h.get("fastModel") if fast else None) or h.get("model")
|
|
60
|
+
if provider not in ("anthropic", "openai") or not model:
|
|
61
|
+
return None
|
|
62
|
+
key = seo_config.env(str(h.get("apiKeyEnv") or ""))
|
|
63
|
+
if not key:
|
|
64
|
+
return None
|
|
65
|
+
return {"provider": provider, "model": str(model),
|
|
66
|
+
"baseUrl": str(h.get("baseUrl") or "").rstrip("/"), "key": key}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def command(fast: bool = False) -> list[str] | None:
|
|
70
|
+
m = _enabled()
|
|
71
|
+
if not m:
|
|
72
|
+
return None
|
|
73
|
+
cmd = (m.get("fastCommand") if fast else None) or m.get("command") or ""
|
|
74
|
+
parts = shlex.split(str(cmd))
|
|
75
|
+
if not parts or not shutil.which(parts[0]):
|
|
76
|
+
return None
|
|
77
|
+
return parts
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def available(fast: bool = False) -> bool:
|
|
81
|
+
return http_config(fast) is not None or command(fast) is not None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _http_infer(cfg: dict, prompt: str, fast: bool) -> str | None:
|
|
85
|
+
timeout = 120 if fast else 300
|
|
86
|
+
if cfg["provider"] == "anthropic":
|
|
87
|
+
url = (cfg["baseUrl"] or "https://api.anthropic.com") + "/v1/messages"
|
|
88
|
+
args = ["-X", "POST",
|
|
89
|
+
"-H", f"x-api-key: {cfg['key']}",
|
|
90
|
+
"-H", f"anthropic-version: {ANTHROPIC_VERSION}",
|
|
91
|
+
"-H", "Content-Type: application/json",
|
|
92
|
+
"-d", json.dumps({"model": cfg["model"], "max_tokens": MAX_TOKENS,
|
|
93
|
+
"messages": [{"role": "user", "content": prompt}]}),
|
|
94
|
+
url]
|
|
95
|
+
else:
|
|
96
|
+
url = (cfg["baseUrl"] or "https://api.openai.com") + "/v1/chat/completions"
|
|
97
|
+
args = ["-X", "POST",
|
|
98
|
+
"-H", f"Authorization: Bearer {cfg['key']}",
|
|
99
|
+
"-H", "Content-Type: application/json",
|
|
100
|
+
"-d", json.dumps({"model": cfg["model"],
|
|
101
|
+
"messages": [{"role": "user", "content": prompt}]}),
|
|
102
|
+
url]
|
|
103
|
+
try:
|
|
104
|
+
resp = curl_json(args, timeout=timeout, attempts=2, label=f"llm {cfg['provider']}")
|
|
105
|
+
except (RuntimeError, OSError) as exc:
|
|
106
|
+
print(f" llm: {exc}", file=sys.stderr)
|
|
107
|
+
return None
|
|
108
|
+
try:
|
|
109
|
+
if cfg["provider"] == "anthropic":
|
|
110
|
+
out = "".join(b.get("text", "") for b in resp["content"] if b.get("type") == "text")
|
|
111
|
+
else:
|
|
112
|
+
out = resp["choices"][0]["message"]["content"]
|
|
113
|
+
except (KeyError, IndexError, TypeError):
|
|
114
|
+
detail = resp.get("error") if isinstance(resp, dict) else resp
|
|
115
|
+
print(f" llm: unexpected response: {json.dumps(detail)[:200]}", file=sys.stderr)
|
|
116
|
+
return None
|
|
117
|
+
return (out or "").strip() or None
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def infer(prompt: str, fast: bool = False) -> str | None:
|
|
121
|
+
"""A reply, or None when the module is off, nothing is reachable, or the
|
|
122
|
+
call fails or times out. Never raises: every caller degrades instead."""
|
|
123
|
+
http = http_config(fast)
|
|
124
|
+
if http:
|
|
125
|
+
return _http_infer(http, prompt, fast)
|
|
126
|
+
parts = command(fast)
|
|
127
|
+
if not parts:
|
|
128
|
+
return None
|
|
129
|
+
try:
|
|
130
|
+
p = subprocess.run(parts, input=prompt, capture_output=True, text=True,
|
|
131
|
+
timeout=120 if fast else 300)
|
|
132
|
+
except (subprocess.TimeoutExpired, OSError) as exc:
|
|
133
|
+
print(f" llm: {exc}", file=sys.stderr)
|
|
134
|
+
return None
|
|
135
|
+
if p.returncode != 0:
|
|
136
|
+
print(f" llm: exit {p.returncode}: {p.stderr.strip()[:200]}", file=sys.stderr)
|
|
137
|
+
return None
|
|
138
|
+
out = p.stdout.strip()
|
|
139
|
+
return out or None
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// End-to-end check of the stdio MCP server against whatever is in data/.
|
|
2
|
+
// Run from the repo root: `npm run mcp:smoke`
|
|
3
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
4
|
+
import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
|
|
5
|
+
|
|
6
|
+
const transport = new StdioClientTransport({
|
|
7
|
+
command: "npx",
|
|
8
|
+
args: ["tsx", "src/mcp-stdio.ts"],
|
|
9
|
+
cwd: process.cwd(),
|
|
10
|
+
env: process.env,
|
|
11
|
+
});
|
|
12
|
+
const client = new Client({ name: "smoke-test", version: "1.0.0" });
|
|
13
|
+
await client.connect(transport);
|
|
14
|
+
|
|
15
|
+
const { tools } = await client.listTools();
|
|
16
|
+
console.log(`TOOLS (${tools.length}):`);
|
|
17
|
+
for (const t of tools) console.log(` ${t.name.padEnd(24)} ${(t.description ?? "").slice(0, 62)}…`);
|
|
18
|
+
|
|
19
|
+
const { resources } = await client.listResources();
|
|
20
|
+
console.log(`\nRESOURCES (${resources.length}): ${resources.map((r) => r.uri).join(", ")}`);
|
|
21
|
+
|
|
22
|
+
const call = async (name, args = {}) => {
|
|
23
|
+
const r = await client.callTool({ name, arguments: args });
|
|
24
|
+
const text = r.content?.[0]?.text ?? "";
|
|
25
|
+
return { isError: !!r.isError, text };
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
console.log("\n--- settings ---");
|
|
29
|
+
let r = await call("settings");
|
|
30
|
+
const settings = JSON.parse(r.text);
|
|
31
|
+
console.log(` name=${settings.name} sites=${settings.sites.map((s) => s.host).join(",") || "(none)"} modules on: ${Object.entries(settings.modules).filter(([, v]) => v).map(([k]) => k).join(", ") || "(none)"}`);
|
|
32
|
+
const firstHost = settings.sites[0]?.host;
|
|
33
|
+
|
|
34
|
+
console.log("\n--- list_actions {status:active, limit:3} ---");
|
|
35
|
+
r = await call("list_actions", { status: "active", limit: 3 });
|
|
36
|
+
const acts = JSON.parse(r.text);
|
|
37
|
+
console.log(` matched=${acts.matched} returned=${acts.returned}`);
|
|
38
|
+
acts.actions.forEach((a) => console.log(` #${a.rank} +${a.impact} ${a.effort} ${a.host} — ${a.title.slice(0, 54)}`));
|
|
39
|
+
|
|
40
|
+
console.log("\n--- ops_status ---");
|
|
41
|
+
r = await call("ops_status");
|
|
42
|
+
const ops = JSON.parse(r.text);
|
|
43
|
+
console.log(` lastRun=${ops.lastRun?.ts ?? "none"} failures="${ops.lastRun?.failures ?? ""}" probe=${ops.probe ? `${ops.probe.healthy}/${ops.probe.total} healthy` : "none"}`);
|
|
44
|
+
(ops.probe?.sites ?? []).filter((s) => s.findings.length).forEach((s) => console.log(` ${s.host}: ${s.findings.join("; ")}`));
|
|
45
|
+
|
|
46
|
+
if (firstHost) {
|
|
47
|
+
console.log(`\n--- top_queries ${firstHost} 90d limit 3 ---`);
|
|
48
|
+
r = await call("top_queries", { host: firstHost, window: "90d", limit: 3 });
|
|
49
|
+
JSON.parse(r.text).queries.forEach((q) => console.log(` "${q.query}" clicks=${q.clicks} imps=${q.impressions} pos=${(q.position ?? 0).toFixed(1)}`));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
console.log("\n--- error path: unknown host ---");
|
|
53
|
+
r = await call("site_report", { host: "nope.example.invalid" });
|
|
54
|
+
console.log(` isError=${r.isError} :: ${r.text.slice(0, 70)}`);
|
|
55
|
+
|
|
56
|
+
console.log("\n--- conversions_status ---");
|
|
57
|
+
r = await call("conversions_status");
|
|
58
|
+
console.log(" " + r.text.replace(/\s+/g, " ").slice(0, 130));
|
|
59
|
+
|
|
60
|
+
await client.close();
|
|
61
|
+
console.log("\nstdio transport OK");
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Automated opportunity discovery.
|
|
3
|
+
|
|
4
|
+
Closes the loop that otherwise needs a human analysis session:
|
|
5
|
+
1. Refreshes the trend analysis (rising/falling queries, 84d windows).
|
|
6
|
+
2. SCRIPTED detection: rising queries not covered by any existing queue
|
|
7
|
+
action become candidates.
|
|
8
|
+
3. INFERENCE (optional, modules.llm): candidates + queue summary + watching
|
|
9
|
+
items go to the model, which returns strictly-JSON *proposals* (new
|
|
10
|
+
queue cards) and *verdicts* on watching items (succeeded / failed /
|
|
11
|
+
keep-watching).
|
|
12
|
+
4. Output -> data/opportunity-proposals.json, rendered on the Actions page
|
|
13
|
+
as PROPOSED cards. Nothing self-modifies the curated queue — you accept
|
|
14
|
+
proposals into config/backlog.json (the dashboard has a button).
|
|
15
|
+
|
|
16
|
+
Inference runs when there are new candidates, and always on Mondays (weekly
|
|
17
|
+
review of watching items). Without the LLM module it still writes candidates.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import re
|
|
22
|
+
import subprocess
|
|
23
|
+
import sys
|
|
24
|
+
from datetime import date
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
|
|
27
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
|
|
28
|
+
import seo_config # noqa: E402
|
|
29
|
+
import llm # noqa: E402
|
|
30
|
+
|
|
31
|
+
ROOT = seo_config.ROOT
|
|
32
|
+
DATA = seo_config.DATA
|
|
33
|
+
RISE_MIN_IMPS = 30 # a riser must reach this many imps in the recent 84d
|
|
34
|
+
RISE_MIN_RATIO = 2.5 # and have grown at least this much vs prior 84d
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def latest_trends():
|
|
38
|
+
files = sorted(DATA.glob("trends-*.json"))
|
|
39
|
+
return json.loads(files[-1].read_text()) if files else None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def queue():
|
|
43
|
+
"""The live queue from the dashboard (data-derived + curated). Falls back
|
|
44
|
+
to the curated backlog alone when the dashboard is not running."""
|
|
45
|
+
p = subprocess.run(["curl", "-sf", "--max-time", "30", seo_config.dashboard_base() + "/api/actions"],
|
|
46
|
+
capture_output=True, text=True)
|
|
47
|
+
try:
|
|
48
|
+
acts = json.loads(p.stdout)
|
|
49
|
+
if isinstance(acts, list):
|
|
50
|
+
return acts
|
|
51
|
+
except json.JSONDecodeError:
|
|
52
|
+
pass
|
|
53
|
+
try:
|
|
54
|
+
return json.loads((seo_config.INSTANCE / "config" / "backlog.json").read_text()).get("actions", [])
|
|
55
|
+
except (OSError, json.JSONDecodeError):
|
|
56
|
+
return []
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def portfolio_description():
|
|
60
|
+
parts = []
|
|
61
|
+
for s in seo_config.sites():
|
|
62
|
+
desc = s["host"]
|
|
63
|
+
if s.get("label") and s["label"] != s["host"]:
|
|
64
|
+
desc += f" ({s['label']})"
|
|
65
|
+
parts.append(desc)
|
|
66
|
+
return ", ".join(parts)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _host_resolver(prop):
|
|
70
|
+
"""query -> the configured host a riser belongs to.
|
|
71
|
+
|
|
72
|
+
A domain property can cover several configured sites, and a url-prefix
|
|
73
|
+
property's slug is not a host at all, so the trend file's key cannot be
|
|
74
|
+
used directly. One site on the property: that site. Several: the host of
|
|
75
|
+
the page with the most impressions for that query in the 90-day
|
|
76
|
+
query x page pull, falling back to the first configured site.
|
|
77
|
+
"""
|
|
78
|
+
owners = [s for s in seo_config.sites() if s.get("gscProperty") == prop]
|
|
79
|
+
if not owners:
|
|
80
|
+
return lambda q: seo_config.gsc_slug(prop) # a bare host is the best we have
|
|
81
|
+
if len(owners) == 1:
|
|
82
|
+
return lambda q: owners[0]["host"]
|
|
83
|
+
by_gsc_host = {s["gscHost"]: s["host"] for s in owners}
|
|
84
|
+
best = {}
|
|
85
|
+
qp = seo_config.DATA / "gsc" / seo_config.gsc_data_slug(prop) / "query_page_90d.json"
|
|
86
|
+
try:
|
|
87
|
+
for r in json.loads(qp.read_text()).get("rows", []):
|
|
88
|
+
q, page = r["keys"][0], r["keys"][1]
|
|
89
|
+
h = page.split("/")[2] if page.count("/") >= 2 else ""
|
|
90
|
+
if h in by_gsc_host and r["impressions"] > best.get(q, (0, ""))[0]:
|
|
91
|
+
best[q] = (r["impressions"], by_gsc_host[h])
|
|
92
|
+
except (OSError, json.JSONDecodeError, KeyError, IndexError):
|
|
93
|
+
pass
|
|
94
|
+
return lambda q: best.get(q, (0, owners[0]["host"]))[1]
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def main():
|
|
98
|
+
# 1. refresh trends (also keeps the /insights tables current)
|
|
99
|
+
if seo_config.gsc_properties() or seo_config.ga4_properties():
|
|
100
|
+
r = subprocess.run([sys.executable, str(ROOT / "ingest" / "analyze_trends.py")],
|
|
101
|
+
capture_output=True, text=True)
|
|
102
|
+
if r.returncode != 0:
|
|
103
|
+
last = (r.stderr or r.stdout).strip().splitlines()
|
|
104
|
+
print("trend refresh failed:", last[-1][:200] if last else "(no output)",
|
|
105
|
+
"— using the latest trends file on disk")
|
|
106
|
+
|
|
107
|
+
trends = latest_trends()
|
|
108
|
+
if not trends:
|
|
109
|
+
print("no trends file — scan aborted (run ingest/analyze_trends.py once data exists)")
|
|
110
|
+
return 1
|
|
111
|
+
actions = queue()
|
|
112
|
+
|
|
113
|
+
queue_text = " || ".join(
|
|
114
|
+
f"{a.get('title','')} :: {a.get('why','')} :: {' '.join(a.get('spec', []))}" for a in actions
|
|
115
|
+
).lower()
|
|
116
|
+
active = [a for a in actions if not a.get("watching")]
|
|
117
|
+
watching = [a for a in actions if a.get("watching")]
|
|
118
|
+
|
|
119
|
+
# 2. scripted candidate detection: uncovered risers
|
|
120
|
+
candidates = []
|
|
121
|
+
for site, d in trends.get("sites", {}).items():
|
|
122
|
+
host_for = _host_resolver(site)
|
|
123
|
+
for m in d.get("rising", []):
|
|
124
|
+
if m["recent_imps"] < RISE_MIN_IMPS:
|
|
125
|
+
continue
|
|
126
|
+
if m["prior_imps"] and m["recent_imps"] / max(1, m["prior_imps"]) < RISE_MIN_RATIO:
|
|
127
|
+
continue
|
|
128
|
+
# Substring matching dropped genuine risers: a one-word query
|
|
129
|
+
# ("pricing", "canvas") appears inside some unrelated card's prose
|
|
130
|
+
# and the riser is written off as already covered.
|
|
131
|
+
if re.search(rf"\b{re.escape(m['query'].lower())}\b", queue_text):
|
|
132
|
+
continue # already covered by an action
|
|
133
|
+
candidates.append({"host": host_for(m["query"]), **m})
|
|
134
|
+
candidates.sort(key=lambda c: -c["recent_imps"])
|
|
135
|
+
candidates = candidates[:12]
|
|
136
|
+
|
|
137
|
+
run_inference = (bool(candidates) or date.today().weekday() == 0) and llm.available()
|
|
138
|
+
out = {"generated": date.today().isoformat(), "candidates": candidates,
|
|
139
|
+
"proposals": [], "verdicts": [], "inference_ran": False}
|
|
140
|
+
|
|
141
|
+
if run_inference:
|
|
142
|
+
prompt = (
|
|
143
|
+
"You are the SEO strategist for a small portfolio of websites: "
|
|
144
|
+
+ portfolio_description() + ". "
|
|
145
|
+
"Growth levers available: new/updated pages, titles/meta, internal links, llms.txt/AI-crawler surface, "
|
|
146
|
+
"distribution (communities, newsletters, directories). No paid ads.\n\n"
|
|
147
|
+
"UNCOVERED RISING QUERIES (84d vs prior 84d, none matched by existing queue actions):\n"
|
|
148
|
+
+ json.dumps(candidates, indent=1)
|
|
149
|
+
+ "\n\nEXISTING ACTIVE QUEUE (titles only — do NOT duplicate):\n"
|
|
150
|
+
+ json.dumps([a.get("title") for a in active], indent=1)
|
|
151
|
+
+ "\n\nWATCHING ITEMS (shipped work + status note; judge each against its note):\n"
|
|
152
|
+
+ json.dumps([{"title": a.get("title"), "note": a.get("watching"), "why": a.get("why", "")} for a in watching], indent=1)
|
|
153
|
+
+ "\n\nReturn STRICT JSON only, no prose, matching exactly:\n"
|
|
154
|
+
'{"proposals":[{"host":"...","title":"...","kind":"...","why":"... (cite the query numbers)",'
|
|
155
|
+
'"how":"...","spec":["..."],"impact":<int clicks/mo estimate>,"effort":"S|M|L","tag":"content|striking|metadata|distribution|hygiene"}],'
|
|
156
|
+
'"verdicts":[{"title":"<exact watching title>","verdict":"succeeded|failed|keep-watching","evidence":"..."}]}\n'
|
|
157
|
+
"Rules: 0-4 proposals, only where the rising-query evidence genuinely supports a concrete move; "
|
|
158
|
+
"each proposal must name real pages/URLs in spec; impact is an ordering estimate, not a forecast — be conservative; "
|
|
159
|
+
"verdicts only where the data above lets you judge — otherwise keep-watching with a one-line reason."
|
|
160
|
+
)
|
|
161
|
+
raw = llm.infer(prompt) or ""
|
|
162
|
+
m = re.search(r"\{.*\}", raw, re.S)
|
|
163
|
+
if m:
|
|
164
|
+
try:
|
|
165
|
+
parsed = json.loads(m.group(0))
|
|
166
|
+
out["proposals"] = [x for x in parsed.get("proposals", [])
|
|
167
|
+
if x.get("title") and x.get("host") and x.get("spec")]
|
|
168
|
+
out["verdicts"] = parsed.get("verdicts", [])
|
|
169
|
+
out["inference_ran"] = True
|
|
170
|
+
except json.JSONDecodeError:
|
|
171
|
+
print("inference returned unparseable JSON — kept candidates only")
|
|
172
|
+
else:
|
|
173
|
+
print("inference produced no JSON — kept candidates only")
|
|
174
|
+
elif candidates and not llm.available():
|
|
175
|
+
print("llm module off — candidates recorded without proposals")
|
|
176
|
+
|
|
177
|
+
DATA.mkdir(parents=True, exist_ok=True)
|
|
178
|
+
(DATA / "opportunity-proposals.json").write_text(json.dumps(out, indent=1))
|
|
179
|
+
print(f"candidates: {len(candidates)} | proposals: {len(out['proposals'])} | "
|
|
180
|
+
f"verdicts: {len(out['verdicts'])} | inference: {out['inference_ran']}")
|
|
181
|
+
return 0
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
if __name__ == "__main__":
|
|
185
|
+
sys.exit(main())
|
package/ops/publish.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Publish the exported mirror somewhere (modules.publish).
|
|
3
|
+
|
|
4
|
+
`ops/export_static.py` builds a read-only copy of the dashboard into
|
|
5
|
+
`site/`. Getting that copy to wherever people read it — a bucket, an object
|
|
6
|
+
store, a box over ssh — is the other half, and it used to mean writing an
|
|
7
|
+
`afterRun` hook by hand. This makes it a normal pipeline step, so it is
|
|
8
|
+
logged, retried once and recorded in `last-run.json` like everything else.
|
|
9
|
+
|
|
10
|
+
python3 ops/publish.py # publish site/ to modules.publish.destination
|
|
11
|
+
|
|
12
|
+
Targets:
|
|
13
|
+
|
|
14
|
+
gcs gcloud storage rsync site <dest> --recursive
|
|
15
|
+
s3 aws s3 sync site <dest>
|
|
16
|
+
rsync rsync -a site/ <dest>
|
|
17
|
+
command run modules.publish.command — the escape hatch for anything else
|
|
18
|
+
|
|
19
|
+
`delete` adds each tool's "remove what is no longer here" flag, which is what
|
|
20
|
+
makes the mirror match the export rather than accumulate stale pages. Set
|
|
21
|
+
`dryRun` to print the command without running it: that is how you rehearse a
|
|
22
|
+
cutover against a bucket that is already serving something.
|
|
23
|
+
|
|
24
|
+
`env` is merged into the child's environment only — a deployment that keeps
|
|
25
|
+
its cloud credentials in an isolated config directory points at it there,
|
|
26
|
+
without exporting it for the whole daily run. Values are never printed.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
import os
|
|
30
|
+
import shlex
|
|
31
|
+
import shutil
|
|
32
|
+
import subprocess
|
|
33
|
+
import sys
|
|
34
|
+
from pathlib import Path
|
|
35
|
+
|
|
36
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "ingest"))
|
|
37
|
+
import seo_config # noqa: E402
|
|
38
|
+
|
|
39
|
+
TARGETS = ("gcs", "s3", "rsync", "command")
|
|
40
|
+
|
|
41
|
+
# target -> the binary that must be on PATH ("command" runs a shell string)
|
|
42
|
+
BINARY = {"gcs": "gcloud", "s3": "aws", "rsync": "rsync"}
|
|
43
|
+
|
|
44
|
+
DEST_EXAMPLE = {
|
|
45
|
+
"gcs": "gs://your-bucket",
|
|
46
|
+
"s3": "s3://your-bucket",
|
|
47
|
+
"rsync": "user@host:/srv/n-seo-mirror",
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def site_dir() -> Path:
|
|
52
|
+
return seo_config.INSTANCE / "site"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def build_command(mod: dict, site: Path):
|
|
56
|
+
"""(command, shell) for the configured target.
|
|
57
|
+
|
|
58
|
+
`command` is an argv list for the three built-in targets and a shell
|
|
59
|
+
string for `command`; `shell` says which. Raises ValueError with a
|
|
60
|
+
message meant for the operator, not a stack trace.
|
|
61
|
+
"""
|
|
62
|
+
target = (mod.get("target") or "").strip()
|
|
63
|
+
if target not in TARGETS:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f"unknown publish target {target!r} — use one of: {', '.join(TARGETS)}")
|
|
66
|
+
|
|
67
|
+
if target == "command":
|
|
68
|
+
cmd = (mod.get("command") or "").strip()
|
|
69
|
+
if not cmd:
|
|
70
|
+
raise ValueError(
|
|
71
|
+
"modules.publish.command is empty — target 'command' needs the "
|
|
72
|
+
"shell command that publishes site/")
|
|
73
|
+
return cmd, True
|
|
74
|
+
|
|
75
|
+
dest = (mod.get("destination") or "").strip()
|
|
76
|
+
if not dest:
|
|
77
|
+
raise ValueError(
|
|
78
|
+
f"modules.publish.destination is empty — set it to something like "
|
|
79
|
+
f"{DEST_EXAMPLE[target]} for target {target!r}")
|
|
80
|
+
|
|
81
|
+
delete = bool(mod.get("delete"))
|
|
82
|
+
if target == "gcs":
|
|
83
|
+
argv = ["gcloud", "storage", "rsync", str(site), dest, "--recursive"]
|
|
84
|
+
if delete:
|
|
85
|
+
argv.append("--delete-unmatched-destination-objects")
|
|
86
|
+
elif target == "s3":
|
|
87
|
+
argv = ["aws", "s3", "sync", str(site), dest]
|
|
88
|
+
if delete:
|
|
89
|
+
argv.append("--delete")
|
|
90
|
+
else: # rsync — the trailing slash copies the contents, not the directory
|
|
91
|
+
argv = ["rsync", "-a", str(site) + "/", dest]
|
|
92
|
+
if delete:
|
|
93
|
+
argv.append("--delete")
|
|
94
|
+
return argv, False
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def child_env(mod: dict) -> dict:
|
|
98
|
+
"""The daily run's environment plus modules.publish.env.
|
|
99
|
+
|
|
100
|
+
`~` and $VARS in the values are expanded so a credentials path can be
|
|
101
|
+
written the way a person would type it.
|
|
102
|
+
"""
|
|
103
|
+
env = dict(os.environ)
|
|
104
|
+
for k, v in (mod.get("env") or {}).items():
|
|
105
|
+
if isinstance(k, str) and k:
|
|
106
|
+
env[k] = os.path.expanduser(os.path.expandvars(str(v)))
|
|
107
|
+
return env
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def main() -> int:
|
|
111
|
+
if not seo_config.enabled("publish"):
|
|
112
|
+
print("modules.publish is off — nothing to publish")
|
|
113
|
+
return 0
|
|
114
|
+
mod = seo_config.module("publish")
|
|
115
|
+
|
|
116
|
+
site = site_dir()
|
|
117
|
+
try:
|
|
118
|
+
cmd, shell = build_command(mod, site)
|
|
119
|
+
except ValueError as exc:
|
|
120
|
+
print(f"publish: {exc}", file=sys.stderr)
|
|
121
|
+
return 1
|
|
122
|
+
|
|
123
|
+
if not site.is_dir():
|
|
124
|
+
print(f"publish: nothing to publish — {site} does not exist. Enable "
|
|
125
|
+
f"modules.staticExport so the daily run builds it first.",
|
|
126
|
+
file=sys.stderr)
|
|
127
|
+
return 1
|
|
128
|
+
|
|
129
|
+
target = mod["target"]
|
|
130
|
+
binary = BINARY.get(target)
|
|
131
|
+
if binary and shutil.which(binary) is None:
|
|
132
|
+
print(f"publish: {binary!r} is not on PATH — target {target!r} needs it "
|
|
133
|
+
f"(install it, or use target 'command')", file=sys.stderr)
|
|
134
|
+
return 1
|
|
135
|
+
|
|
136
|
+
printable = cmd if shell else " ".join(shlex.quote(a) for a in cmd)
|
|
137
|
+
# Key names only: a publish env is where credentials paths and tokens live.
|
|
138
|
+
env_keys = sorted(k for k in (mod.get("env") or {}) if isinstance(k, str) and k)
|
|
139
|
+
if env_keys:
|
|
140
|
+
print(f"publish: env {', '.join(env_keys)}")
|
|
141
|
+
|
|
142
|
+
if mod.get("dryRun"):
|
|
143
|
+
print(f"publish: DRY RUN — would run:\n {printable}\n"
|
|
144
|
+
f"publish: set modules.publish.dryRun to false to publish for real")
|
|
145
|
+
return 0
|
|
146
|
+
|
|
147
|
+
print(f"publish: {printable}")
|
|
148
|
+
p = subprocess.run(cmd, shell=shell, cwd=seo_config.INSTANCE, env=child_env(mod))
|
|
149
|
+
if p.returncode != 0:
|
|
150
|
+
print(f"publish: FAILED (exit {p.returncode}) — {target} → "
|
|
151
|
+
f"{mod.get('destination') or 'command'}", file=sys.stderr)
|
|
152
|
+
return 1
|
|
153
|
+
print(f"publish: {site.name}/ → {mod.get('destination') or 'command'}")
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
if __name__ == "__main__":
|
|
158
|
+
sys.exit(main())
|