@houtini/seo-audit-console 0.2.3 → 0.4.3

package/dist/server.js

@@ -8,6 +8,23 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { z } from 'zod';
 import { getDashboardData } from './core/dashboardData.js';
+import { startDashboardServer, stopDashboardServer, dashboardServerUrl, listLocalProperties } from './core/webServer.js';
+import { computeSerpFootprint, persistSerpFootprint } from './core/serpFootprint.js';
+import { computeMarketSizing, persistMarketSizing } from './core/marketSizing.js';
+import { FirecrawlClient } from './core/FirecrawlClient.js';
+import { SupadataClient } from './core/SupadataClient.js';
+import { fetchCompetitorContent } from './core/reconResearch.js';
+import { fetchOwnPage } from './core/reconFetch.js';
+import { parseSerpForRecon, reconVerdict } from './core/serpRecon.js';
+import { selectReconTargets, deterministicTodos, persistReconPage, insertTodos } from './audit/recon.js';
+/** A clickable browser-dashboard link appended to tool outputs — the user should always
+ * know the full interactive report is one click away (or one serve_dashboard call away). */
+function browserLink(siteUrl) {
+    const base = dashboardServerUrl();
+    if (base)
+        return `\n\nBrowser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
+    return `\n\nTip: run serve_dashboard to open the full interactive dashboard in your browser.`;
+}
 import { runAudit, runSingleCheck, listChecks } from './audit/engine.js';
 import { buildAuditMarkdown } from './audit/report.js';
 import { diffLatest, buildDriftMarkdown } from './audit/drift.js';
@@ -34,6 +51,8 @@ import { Backlinks } from './core/Backlinks.js';
 import { WikidataClient } from './core/WikidataClient.js';
 import { Entities } from './core/Entities.js';
 import { JobManager } from './core/JobManager.js';
+import { gscFreshness } from './core/gscFreshness.js';
+import { clusterKeywordList } from './audit/keywordList.js';
 const SERVER_NAME = 'seo-audit-console';
 const SERVER_VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;
 // Where per-property crawl/audit DBs live. Resolution order (computed once per run):
@@ -91,6 +110,18 @@ ARCHETYPE CHAINS (compose along these lines):
 2. Authority → waste: iPR / backlinks flowing into non-200, redirected, or orphaned URLs → recover the equity with 301s or internal links (fix_finding generates them).
 3. Competitor → gap: competitor keyword footprints (ranked_keywords / topic_gaps) minus our GSC + crawled-page footprint → topics to cover, each tied to the nearest existing page.
 
+COST DISCIPLINE (behave like a strategist who knows the margins):
+- Free and instant, use liberally: everything on synced data — query_data, run_audit, query_audit, suggest_pages, list_templates, detect_changes, get_dashboard, serve_dashboard, export_report.
+- Paid but CHEAP and 20-day cached (Labs/Keywords, ~$0.01–0.13 a call): keyword_volume, search_intent, ranked_keywords, serp_features, domain_visibility, top_pages, competitors_domain, page_intersection, topic_gaps. Top-down pulls only — ONE ranked_keywords call answers "what does this domain rank for"; NEVER loop keywords through SERP endpoints to reconstruct what a Labs call returns.
+- Paid per-keyword (SERP): related_terms and AI-Overview CITATION checks. On-demand for a handful of clicked/explicit keywords, never a list.
+- Separate subscription: pull_backlinks (DataForSEO Backlinks — a 40204 error means it isn't activated).
+
+AGENCY MACRO-WORKFLOWS (the engagement arc — each stage feeds the next):
+1. Baseline: refresh_property → run_audit → serve_dashboard (share the URL).
+2. Market: serp_features (feature exposure) + domain_visibility + competitors_domain → topic_gaps vs the named rivals.
+3. Content plan: suggest_pages (demand you already have) + topic_gaps (demand rivals own) → draft_content briefs.
+4. Fix cycle: fix_finding per top finding → re-crawl → detect_changes to prove the fix landed.
+
 Before planning ANY complex multi-source question, call composition_cookbook — it returns the full data-surface map and worked recipes using these exact tool and table names.`;
 const COOKBOOK_TEXT = `# Composition cookbook — the data surface and how to join it
 
@@ -134,6 +165,15 @@ Raw access: query_audit runs any single check with full evidence; every table ab
 - **Crawl-to-first-impression latency:** url_inspection.last_crawl_time vs the first date a page appears in search_analytics — how fast does Google turn a crawl into impressions, per template? Slow templates have an indexing-pipeline problem.
 - **Crawled-as vs response times:** url_inspection.crawled_as (mobile/desktop agent) × pages.response_time_ms — slow responses specifically on the agent Google uses against you.
 
+## Agency engagement recipes (the deliverable arc)
+
+- **Week-one baseline:** refresh_property → run_audit → serve_dashboard. Share the dashboard URL; the ranked findings ARE the technical workstream.
+- **Market read:** serp_features (feature/AIO exposure, volume-weighted) + domain_visibility for the client and each named rival (one cached call each) → who is structurally winning, and how much of the market SERP features already absorb.
+- **Content plan:** suggest_pages (demand you already earn impressions for) + topic_gaps (demand rivals own that you don't) → draft_content for the winners. Every proposal traces to real impressions or a rival's real footprint - no invented "keyword ideas".
+- **Fix-and-prove cycle:** fix_finding on the top finding → ship → start_crawl → detect_changes shows the fix landed → re-run run_audit and watch the finding drop off. That screenshot is the client update.
+- **Content recon (why a page is losing):** recon_targets picks the worst declining/striking pages, fetches our live page + the Google SERP (organic rank + AI-Overview citations + video), and classifies WHY — the sharpest class is "we rank but the AI Overview won't quote us" = a data-accuracy/freshness/markup problem. Then research the competitor set it returns (firecrawl for pages, supadata for the ranking videos), write the gaps back with save_recon_todo, and track the fixes with recon_todos (which can re-measure whether you moved from uncited→cited). Pass location to match where your impressions come from — organic rank is location-sensitive.
+- **Cost rule of thumb:** an entire competitive read (visibility + footprint + gaps for 4 domains) is a handful of cached Labs calls - under a dollar. If a plan involves looping SERP calls over a keyword list, it is the wrong plan; a Labs endpoint already has that answer top-down.
+
 Plan the join first (url_key / query / domain), state the grain of each side, then run the fewest paid calls that answer it.`;
 // The check catalogue rendered as markdown — single source of truth is listChecks();
 // shared by the seo-audit://checks-reference resource (and buildable for any category subset).
@@ -213,6 +253,12 @@ export function createServer() {
         : null;
     const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
     const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
+    // Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
+    const firecrawlKey = process.env.FIRECRAWL_API_KEY;
+    const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
+    // Supadata (transcribes the ranking videos for content recon) — optional; degrades gracefully.
+    const supadataKey = process.env.SUPADATA_API_KEY;
+    const supadata = supadataKey ? new SupadataClient(supadataKey, path.join(dataDir(), 'supadata-cache.db')) : null;
     const entities = new Entities(new WikidataClient(path.join(dataDir(), 'wikidata-cache.db')), dataDir());
     const refresh = new Refresh(sync, crawler, inspector, rankTracker);
     const requireGsc = (v) => {
@@ -336,7 +382,7 @@ export function createServer() {
     }, async ({ siteUrl, scope, categories, includeJudgement }) => {
         const result = runAudit(dataDir(), siteUrl, { scope, categories, includeJudgement });
         return {
-            content: [{ type: 'text', text: buildAuditMarkdown(result, siteUrl) }],
+            content: [{ type: 'text', text: buildAuditMarkdown(result, siteUrl) + browserLink(siteUrl) }],
             structuredContent: result,
         };
     });
@@ -740,7 +786,7 @@ export function createServer() {
     // ── DataForSEO (cached 20 days, single-worker) ──────────────────────────
     server.registerTool('keyword_volume', {
         title: 'Keyword search volume (DataForSEO)',
-        description: 'True monthly search volume + CPC + competition for keywords (DataForSEO KEYWORDS_DATA). Served from a 20-day cache; live calls are serialised. Default location: United States (2840).',
+        description: '[Paid: Keywords API, cheap, cached 20d | Use for: demand sizing] True monthly search volume + CPC + competition for keywords (DataForSEO KEYWORDS_DATA). Served from a 20-day cache; live calls are serialised. Default location: United States (2840).',
         inputSchema: {
             keywords: z.array(z.string()).min(1).max(700),
             location: z.union([z.string(), z.number()]).optional(),
@@ -759,7 +805,7 @@ export function createServer() {
     });
     server.registerTool('related_terms', {
         title: 'Related terms (People Also Ask + related searches)',
-        description: 'People Also Ask questions and related searches for a keyword (DataForSEO SERP). Powers click-through "related terms" on the keyword charts. SERP call — cached 20 days.',
+        description: '[Paid: SERP call PER KEYWORD - only for clicked/explicit keywords, never lists | Use for: the SERP context of a single keyword] People Also Ask questions and related searches for a keyword (DataForSEO SERP). Powers click-through "related terms" on the keyword charts. SERP call — cached 20 days.',
         inputSchema: { keyword: z.string(), location: z.union([z.string(), z.number()]).optional(), languageCode: z.string().optional() },
     }, async ({ keyword, location, languageCode }) => {
         const client = requireDfs(dfs);
@@ -788,9 +834,312 @@ export function createServer() {
             db.close();
         }
     });
+    // content_opportunities — the content marketer's report: everything the stored data
+    // says about what to WRITE, REFRESH and REWRITE, in one free composition.
+    server.registerTool('content_opportunities', {
+        title: 'Content opportunity report (write / refresh / rewrite)',
+        description: 'The content marketer\'s report, composed entirely from stored data - NO paid calls. Four sections: WRITE NEXT (new pages proposed from queries you already earn impressions for but have no winning page - suggest_pages), REFRESH NOW (pages that lost 20%+ of their clicks vs the prior period - content decay), REWRITE SNIPPETS (page-1 rankings earning far below expected CTR - title/meta rewrites, the fastest wins), and STRENGTHEN (keyword clusters where you rank 4-20 - one push from the money positions). Every line traces to real Search Console data. Chain into draft_content for a brief, or keyword_volume to size a cluster against the market.',
+        inputSchema: { siteUrl: z.string(), limit: z.number().int().min(3).max(30).optional().describe('Rows per section (default 10)') },
+    }, async ({ siteUrl, limit }) => {
+        const n = limit ?? 10;
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        let proposals;
+        let weak;
+        try {
+            proposals = suggestPages(db.db, { maxProposals: n }).proposals;
+            const fresh = gscFreshness(db.db);
+            weak = clusterKeywordList(db.db, undefined, { maxDate: fresh.effectiveMax }).clusters
+                .filter(c => c.verdict === 'weak' && c.bestPosition != null && c.bestPosition >= 4).slice(0, n);
+        }
+        finally {
+            db.close();
+        }
+        const dash = getDashboardData(dataDir(), siteUrl);
+        const decay = (dash.contentDecay ?? []).slice(0, n);
+        const snippets = (dash.quickWins ?? []).filter(q => q.type === 'snippet').sort((a, b) => b.potential - a.potential).slice(0, n);
+        const sect = [];
+        sect.push(`## 1. Write next - demand you already have, no winning page\n` + (proposals.length
+            ? `| Proposed page (head term) | Impressions/mo | Best pos today | Queries |\n|---|---|---|---|\n` +
+                proposals.map(p => `| ${p.headTerm.replace(/\|/g, '\\|')} | ${fmtNum(p.totalImpressions)} | ${p.bestPosition} | ${p.queries.length} |`).join('\n')
+            : '_No unserved-demand gaps found._'));
+        sect.push(`## 2. Refresh now - pages losing clicks\n` + (decay.length
+            ? `| Page | Clicks were | Now | Drop | Clicks lost |\n|---|---|---|---|---|\n` +
+                decay.map(d => `| ${d.urlKey.replace(/\|/g, '\\|')} | ${fmtNum(d.prevClicks)} | ${fmtNum(d.clicks)} | ${d.dropPct}% | ${fmtNum(d.lost)} |`).join('\n')
+            : '_No significant decay - nothing lost 20%+ of its clicks._'));
+        sect.push(`## 3. Rewrite snippets - ranking well, under-clicked\n` + (snippets.length
+            ? `| Query | Position | CTR | Expected | Clicks recoverable/mo |\n|---|---|---|---|---|\n` +
+                snippets.map(q => `| ${q.query.replace(/\|/g, '\\|')} | ${q.position} | ${(q.ctr * 100).toFixed(1)}% | ${(q.expectedCtr * 100).toFixed(1)}% | ${fmtNum(Math.round(q.potential))} |`).join('\n')
+            : '_No page-1 CTR gaps found._'));
+        sect.push(`## 4. Strengthen - clusters one push from the money positions\n` + (weak.length
+            ? `| Cluster | Best pos | Impressions 90d | Keywords | Ranking URL |\n|---|---|---|---|---|\n` +
+                weak.map(c => `| ${c.head.replace(/\|/g, '\\|')} | ${c.bestPosition} | ${fmtNum(c.impressions)} | ${c.keywords.length} | ${c.url ?? '-'} |`).join('\n')
+            : '_No weak clusters - your rankings are polarised own/absent._'));
+        const md = `# Content opportunity report - ${siteUrl}\n\nEverything below is from your own Search Console + crawl data (no paid calls, no invented keywords).\n\n` +
+            sect.join('\n\n') +
+            `\n\nNext moves: draft_content for a brief on any row · keyword_volume to size a cluster against the market · topic_gaps to add what competitors own.` +
+            browserLink(siteUrl);
+        return {
+            content: [{ type: 'text', text: md }],
+            structuredContent: {
+                siteUrl,
+                writeNext: proposals, refreshNow: decay, rewriteSnippets: snippets,
+                strengthen: weak.map(c => ({ head: c.head, bestPosition: c.bestPosition, impressions: c.impressions, url: c.url, keywords: c.keywords.length })),
+            },
+        };
+    });
+    // ── Content recon (recon_targets) — the data-intensive "why are we losing, what to do" mission ──
+    server.registerTool('recon_targets', {
+        title: 'Content recon: why a page is losing, and what to do about it',
+        description: '[Paid: DataForSEO SERP per page (~$0.004 each), bounded to the batch | Use for: the deep "why are we behind and what to add" recon] For each of your worst declining / striking-distance pages (auto-selected by impressions x decline, position 3-15; or pass explicit urls), this fetches OUR live page with the crawler, pulls the live Google SERP (DataForSEO SERP-advanced), and classifies WHY we are behind using the organic-rank x AI-Overview-citation matrix: defend-and-deepen (cited + strong), accuracy-or-freshness (rank but the AIO will not quote us - the sharpest, most actionable class), consolidate-weak-page, or competitive-gap. It writes a per-page classification plus deterministic to-dos (schema gaps, freshness, cannibalisation, video format) into a trackable ledger, and returns the competitor set (organic-above + AI-Overview references + ranking videos) for the research session to diff. Set scrapeCompetitors:true to also pull the top competitors as content, routed by source: YouTube/video → Supadata transcript (SUPADATA_API_KEY), Reddit → its .json, other pages → Firecrawl (FIRECRAWL_API_KEY) with a free HTTP fallback. Cloudflare-challenge sites (e.g. PCMag) still can\'t be fetched from a server and come back as a per-URL error — the SERP still tells you they rank; if one matters, ask the user to paste its copy or supply a text file and diff that in. Transcribe the videos (usually what wins these SERPs) and use the reachable pages. Then write findings back with save_recon_todo; track with recon_todos.',
+        inputSchema: {
+            siteUrl: z.string(),
+            limit: z.number().int().min(1).max(15).optional().describe('Pages per batch (default 5)'),
+            minImpressions: z.number().int().min(1).optional(),
+            location: z.union([z.string(), z.number()]).optional(),
+            urls: z.array(z.string()).optional().describe('Analyse these exact pages instead of auto-selecting'),
+            scrapeCompetitors: z.boolean().optional().describe('Also fetch the top competitors (video→transcript, pages→markdown/HTML)'),
+            crawlAs: z.enum(['browser', 'googlebot']).optional().describe('UA for the free HTTP fetch: browser (default, mimics a visit from Google — gets Reddit + mid-tier) or googlebot'),
+        },
+    }, async ({ siteUrl, limit, minImpressions, location, urls, scrapeCompetitors, crawlAs }) => {
+        const client = requireDfs(dfs);
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            const fresh = gscFreshness(db.db);
+            if (!fresh.effectiveMax)
+                return { content: [{ type: 'text', text: `No synced GSC data for ${siteUrl} — run refresh_property first.` }], structuredContent: { error: 'empty', siteUrl } };
+            const today = new Date().toISOString().slice(0, 10);
+            const ownDomain = dfsHost(siteUrl);
+            const hostForm = hostFormForProperty(siteUrl) ?? 'asis';
+            let targets;
+            if (urls?.length) {
+                targets = [];
+                for (const u of urls) {
+                    const key = urlKey(u, { hostForm });
+                    const row = db.db.prepare(`SELECT query, SUM(impressions) imp, SUM(clicks) clk, SUM(position*impressions)*1.0/NULLIF(SUM(impressions),0) pos
+              FROM search_analytics WHERE page_key=? AND query IS NOT NULL AND date > date(?, '-28 days') AND date <= ? GROUP BY query ORDER BY imp DESC LIMIT 1`)
+                        .get(key, fresh.effectiveMax, fresh.effectiveMax);
+                    if (row?.query)
+                        targets.push({ urlKey: key, query: row.query, impressions: row.imp, clicks: row.clk, position: Math.round(row.pos * 10) / 10, priorPosition: null, slipped: false, competingUrls: 0 });
+                }
+            }
+            else {
+                targets = selectReconTargets(db.db, { ...(limit != null ? { limit } : {}), ...(minImpressions != null ? { minImpressions } : {}), maxDate: fresh.effectiveMax });
+            }
+            if (!targets.length)
+                return { content: [{ type: 'text', text: `No recon targets found (declining/striking-distance pages, position 3-15, ${minImpressions ?? 300}+ impressions). Try a lower minImpressions or pass explicit urls.` }], structuredContent: { siteUrl, targets: [] } };
+            const results = [];
+            let cost = 0;
+            for (const t of targets) {
+                let own = null;
+                try {
+                    own = await fetchOwnPage(t.urlKey, hostForm === 'asis' ? 'asis' : hostForm);
+                }
+                catch { /* page unreachable — classify on SERP alone */ }
+                const serpResp = await client.serpOrganic(t.query, location, 'en', 20);
+                cost += serpResp.cost;
+                const serp = parseSerpForRecon(serpResp, ownDomain);
+                const verdict = reconVerdict(serp);
+                const baseline = { organicRank: serp.ourOrganicRank, aioCitesUs: serp.aioCitesUs, gscPosition: t.position, gscImpressions: t.impressions, at: today };
+                persistReconPage(db.db, {
+                    urlKey: t.urlKey, query: t.query, verdict: verdict.verdict, verdictNote: verdict.note,
+                    organicRank: serp.ourOrganicRank, gscPosition: t.position, gscImpressions: t.impressions,
+                    aioPresent: serp.aioPresent, aioCitesUs: serp.aioCitesUs, videoPresent: serp.videoPresent,
+                    hasProductSchema: own?.hasProductOrReview ?? false, schemaTypes: own?.jsonLdTypes ?? [],
+                    competitors: { organicAbove: serp.organicAbove, aioReferences: serp.aioReferences, videoItems: serp.videoItems },
+                });
+                const todos = own ? deterministicTodos(t, own, serp, verdict, today) : [];
+                const inserted = todos.length ? insertTodos(db.db, t.urlKey, t.query, todos, baseline, 'auto') : 0;
+                let competitorContent = undefined;
+                if (scrapeCompetitors && (firecrawl || supadata)) {
+                    // Build a routed candidate set: organic-above + a couple of AI-Overview references +
+                    // one ranking video, deduped, our own domain removed. Each URL routes by domain
+                    // (YouTube→supadata, Reddit→.json, else→firecrawl). Bounded to keep the payload sane.
+                    const seen = new Set();
+                    const candidates = [];
+                    const add = (u) => { if (u && !seen.has(u) && dfsHost(u) !== ownDomain) {
+                        seen.add(u);
+                        candidates.push(u);
+                    } };
+                    serp.organicAbove.forEach(o => add(o.url));
+                    serp.aioReferences.slice(0, 4).forEach(r => add(r.url));
+                    serp.videoItems.slice(0, 1).forEach(v => add(v.url));
+                    competitorContent = [];
+                    for (const url of candidates.slice(0, 3)) {
+                        competitorContent.push(await fetchCompetitorContent(url, { firecrawl, supadata }, { maxChars: 4000, ua: crawlAs }));
+                    }
+                }
+                results.push({
+                    urlKey: t.urlKey, query: t.query, impressions: t.impressions, gscPosition: t.position, priorPosition: t.priorPosition, slipped: t.slipped,
+                    organicRank: serp.ourOrganicRank, aioPresent: serp.aioPresent, aioCitesUs: serp.aioCitesUs, videoPresent: serp.videoPresent,
+                    verdict: verdict.verdict, verdictNote: verdict.note,
+                    ownHeadings: own?.headings.map(h => h.heading) ?? null, schemaTypes: own?.jsonLdTypes ?? null, hasProductSchema: own?.hasProductOrReview ?? null, dateModified: own?.dateModified ?? null,
+                    todos, todosInserted: inserted,
+                    competitors: { organicAbove: serp.organicAbove, aioReferences: serp.aioReferences, videoItems: serp.videoItems },
+                    ...(competitorContent ? { competitorContent } : {}),
+                });
+            }
+            const md = `# Content recon — ${siteUrl}\n\n${results.length} page(s), ${results.reduce((s, r) => s + r.todosInserted, 0)} to-dos saved. DataForSEO SERP cost $${cost.toFixed(4)}.\n\n` +
+                results.map(r => {
+                    const flags = [r.aioPresent ? (r.aioCitesUs ? 'AIO: cited' : 'AIO: NOT cited') : 'no AIO', r.videoPresent ? 'video pack' : null].filter(Boolean).join(' · ');
+                    return `## ${r.urlKey}\n"${r.query}" — organic #${r.organicRank ?? '?'} (GSC avg ${r.gscPosition}${r.slipped ? `, slipped from ${r.priorPosition}` : ''}), ${r.impressions} impr. ${flags}\n**${r.verdict}** — ${r.verdictNote}\n` +
+                        (r.todos.length ? '\nTo do:\n' + r.todos.map((t) => `- [${t.type}] ${t.action}`).join('\n') : '');
+                }).join('\n\n') +
+                (() => {
+                    const errs = results.flatMap(r => (r.competitorContent ?? []).filter((c) => c.error));
+                    if (!errs.length)
+                        return '';
+                    const needKey = errs.filter(c => /not set/i.test(c.error));
+                    const blocked = errs.filter(c => !/not set/i.test(c.error));
+                    let note = '';
+                    if (blocked.length)
+                        note += `\n\n${blocked.length} competitor(s) couldn't be fetched (bot-protection like Cloudflare, a block, or a fetch error): ${blocked.slice(0, 5).map(c => c.url).join(', ')}. If one matters, paste its copy here or drop a text file and I'll diff it in.`;
+                    if (needKey.length)
+                        note += `\n\n${needKey.length} competitor(s) skipped for a missing API key: ${[...new Set(needKey.map(c => c.error))].join('; ')}.`;
+                    return note;
+                })() +
+                `\n\nNext: research the competitors (transcribe the videos, read the reachable pages), write gaps back with save_recon_todo, and track with recon_todos.` + browserLink(siteUrl);
+            return { content: [{ type: 'text', text: md }], structuredContent: { siteUrl, cost, targets: results } };
+        }
+        finally {
+            db.close();
+        }
+    });
+    // save_recon_todo — the research session writes its content-gap / originality findings back
+    // into the ledger against a page (source: 'research').
+    server.registerTool('save_recon_todo', {
+        title: 'Save content-recon to-dos (research writeback)',
+        description: 'Write content-recon findings back into the trackable ledger for a page — the gaps and originality the research session found by diffing competitors (firecrawl) and videos (supadata) against our content. Each to-do is an action with a type (content-gap / originality / schema / freshness / format), rationale and evidence. Snapshots the page baseline so the fix\'s effect on rank/AIO-citation is measurable later. Run recon_targets first (it classifies the page and seeds the deterministic to-dos); this adds the judgement ones. Track everything with recon_todos.',
+        inputSchema: {
+            siteUrl: z.string(),
+            urlKey: z.string().describe('The page (any URL form — normalised to its key)'),
+            todos: z.array(z.object({
+                action: z.string(),
+                type: z.string().optional(),
+                rationale: z.string().optional(),
+                evidence: z.record(z.any()).optional(),
+                priority: z.number().optional(),
+            })).min(1),
+        },
+    }, async ({ siteUrl, urlKey: rawUrl, todos }) => {
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            const key = urlKey(rawUrl, { hostForm: hostFormForProperty(siteUrl) ?? 'asis' });
+            const page = db.db.prepare(`SELECT query, organic_rank, aio_cites_us, gsc_position FROM recon_page WHERE url_key=?`).get(key);
+            const baseline = page ? { organicRank: page.organic_rank, aioCitesUs: !!page.aio_cites_us, gscPosition: page.gsc_position, at: new Date().toISOString().slice(0, 10) } : {};
+            const drafts = todos.map(t => ({ action: t.action, type: t.type ?? 'content-gap', rationale: t.rationale ?? '', evidence: t.evidence ?? {}, priority: t.priority ?? 0 }));
+            const n = insertTodos(db.db, key, page?.query ?? '', drafts, baseline, 'research');
+            return { content: [{ type: 'text', text: `Saved ${n} recon to-do(s) for ${key}${n < drafts.length ? ` (${drafts.length - n} already open)` : ''}. Track with recon_todos.` }], structuredContent: { urlKey: key, inserted: n } };
+        }
+        finally {
+            db.close();
+        }
+    });
+    // recon_todos — list, track and annotate the ledger. No id → list (optionally filtered);
+    // id → update status and/or append a dated annotation, and optionally re-measure the outcome.
+    server.registerTool('recon_todos', {
+        title: 'List, track and annotate content-recon to-dos',
+        description: 'The content-recon to-do board. With no id: list to-dos (optionally filter by page or status), grouped by page with each page\'s verdict — the pick-a-page-to-work-on surface, and the hand-off to content-machine. With id: update one to-do — set status (open → researching → drafted → shipped → dismissed) and/or append a dated annotation note (your own observations, the history). On status:shipped with remeasure:true it re-fetches the SERP and records the outcome, so you can see whether the fix moved you from AIO-uncited to cited, or up the organic ranks.',
+        inputSchema: {
+            siteUrl: z.string(),
+            urlKey: z.string().optional().describe('Filter the list to one page'),
+            status: z.enum(['open', 'researching', 'drafted', 'shipped', 'dismissed']).optional().describe('Filter the list, or the new status when id is set'),
+            id: z.number().int().optional().describe('Update this to-do'),
+            note: z.string().optional().describe('Append a dated annotation to this to-do'),
+            remeasure: z.boolean().optional().describe('On status:shipped, re-fetch the SERP and record the outcome (paid)'),
+            location: z.union([z.string(), z.number()]).optional(),
+        },
+    }, async ({ siteUrl, urlKey: rawUrl, status, id, note, remeasure, location }) => {
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            if (id != null) {
+                const row = db.db.prepare(`SELECT * FROM recon_todo WHERE id=?`).get(id);
+                if (!row)
+                    throw new Error(`No recon to-do #${id}.`);
+                let notes = row.notes ?? '';
+                if (note)
+                    notes = (notes ? notes + '\n' : '') + `[${new Date().toISOString().slice(0, 10)}] ${note}`;
+                let outcome = row.outcome;
+                let outcomeNote = '';
+                if (status === 'shipped' && remeasure && dfs) {
+                    const serpResp = await dfs.serpOrganic(row.query, location, 'en', 20);
+                    const serp = parseSerpForRecon(serpResp, dfsHost(siteUrl));
+                    outcome = JSON.stringify({ organicRank: serp.ourOrganicRank, aioCitesUs: serp.aioCitesUs, at: new Date().toISOString().slice(0, 10) });
+                    const base = row.baseline ? JSON.parse(row.baseline) : {};
+                    outcomeNote = ` Outcome: organic ${base.organicRank ?? '?'}→${serp.ourOrganicRank ?? '?'}, AIO cited ${base.aioCitesUs ? 'yes' : 'no'}→${serp.aioCitesUs ? 'yes' : 'no'}.`;
+                }
+                db.db.prepare(`UPDATE recon_todo SET status=COALESCE(?,status), notes=?, outcome=COALESCE(?,outcome), updated_at=datetime('now') WHERE id=?`)
+                    .run(status ?? null, notes, outcome ?? null, id);
+                return { content: [{ type: 'text', text: `Updated to-do #${id}${status ? ` → ${status}` : ''}${note ? ' (note added)' : ''}.${outcomeNote}` }], structuredContent: { id, status: status ?? row.status, outcome: outcome ? JSON.parse(outcome) : null } };
+            }
+            const key = rawUrl ? urlKey(rawUrl, { hostForm: hostFormForProperty(siteUrl) ?? 'asis' }) : null;
+            const where = [];
+            const args = [];
+            if (key) {
+                where.push('url_key=?');
+                args.push(key);
+            }
+            if (status) {
+                where.push('status=?');
+                args.push(status);
+            }
+            const rows = db.db.prepare(`SELECT id, url_key, query, action, type, rationale, priority, status, source, notes, baseline, outcome
+          FROM recon_todo ${where.length ? 'WHERE ' + where.join(' AND ') : ''} ORDER BY url_key, priority DESC`).all(...args);
+            if (!rows.length)
+                return { content: [{ type: 'text', text: `No recon to-dos${key ? ` for ${key}` : ''}${status ? ` with status ${status}` : ''}. Run recon_targets to generate some.` }], structuredContent: { todos: [] } };
+            const byPage = new Map();
+            for (const r of rows)
+                (byPage.get(r.url_key) ?? byPage.set(r.url_key, []).get(r.url_key)).push(r);
+            const verdictStmt = db.db.prepare(`SELECT verdict, verdict_note FROM recon_page WHERE url_key=?`);
+            const md = `# Content-recon to-dos — ${rows.length} item(s)${status ? `, status ${status}` : ''}\n\n` +
+                [...byPage.entries()].map(([url, items]) => {
+                    const v = verdictStmt.get(url);
+                    return `## ${url}${v ? `\n_${v.verdict}_ — ${v.verdict_note}` : ''}\n\n| # | Status | Type | Action |\n|---|---|---|---|\n` +
+                        items.map(i => `| ${i.id} | ${i.status} | ${i.type ?? ''} | ${String(i.action).replace(/\|/g, '\\|')} |`).join('\n') +
+                        (items.some(i => i.notes) ? '\n\nNotes:\n' + items.filter(i => i.notes).map(i => `- #${i.id}: ${String(i.notes).replace(/\n/g, ' / ')}`).join('\n') : '');
+                }).join('\n\n') +
+                browserLink(siteUrl);
+            return { content: [{ type: 'text', text: md }], structuredContent: { todos: rows } };
+        }
+        finally {
+            db.close();
+        }
+    });
+    // keyword_list — demand-first clustering ("list mode"): a keyword list becomes topics
+    // with own/weak/absent verdicts, clustered by the URL Google already answers them with.
+    server.registerTool('keyword_list', {
+        title: 'Cluster a keyword list into topics with own/weak/absent verdicts',
+        description: 'Demand-first keyword clustering from stored data - NO paid calls. Give it a keyword list (or omit keywords to use your top 500 GSC queries) and it clusters them by the page Google ALREADY answers them with (two keywords that rank via the same URL belong together - the strongest clustering signal, free from your own GSC data), then groups non-ranking keywords lexically. Each cluster gets a deterministic verdict: OWN (best position <=3), WEAK (4-20), ABSENT (no ranking page), plus the ranking URL, summed 90-day impressions and clicks. The keyword-research workhorse: paste a client keyword list, get the topic map and where you stand. Chain with keyword_volume for market volumes on the interesting clusters, or draft_content for the absent ones.',
+        inputSchema: {
+            siteUrl: z.string(),
+            keywords: z.array(z.string()).max(2000).optional().describe('The keyword list. Omit to derive from your top 500 GSC queries by impressions'),
+            limit: z.number().int().min(1).max(200).optional().describe('Clusters to show in the table (default 40)'),
+        },
+    }, async ({ siteUrl, keywords, limit }) => {
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            const fresh = gscFreshness(db.db);
+            const r = clusterKeywordList(db.db, keywords, { maxDate: fresh.effectiveMax });
+            const show = r.clusters.slice(0, limit ?? 40);
+            const counts = { own: 0, weak: 0, absent: 0 };
+            for (const c of r.clusters)
+                counts[c.verdict]++;
+            const rowsMd = show.map(c => `| ${c.head.replace(/\|/g, '\\|')} | ${c.verdict.toUpperCase()} | ${c.bestPosition ?? '-'} | ${fmtNum(c.impressions)} | ${c.keywords.length} | ${c.url ? c.url.replace(/\|/g, '\\|') : '-'} |`);
+            const md = `**Keyword list clustered for ${siteUrl}** - ${r.totalKeywords} keywords (${r.derived ? 'derived from your GSC demand' : 'your list'}, ${r.inGsc} with GSC data) → ${r.clusters.length} clusters: ${counts.own} own · ${counts.weak} weak · ${counts.absent} absent\n\n` +
+                `| Cluster (head term) | Verdict | Best pos | Impressions 90d | Keywords | Ranking URL |\n|---|---|---|---|---|---|\n` +
+                rowsMd.join('\n') +
+                `\n\nVerdicts are deterministic from your GSC positions. ABSENT clusters are content opportunities - chain into keyword_volume (market volume) or draft_content (a brief).`;
+            return {
+                content: [{ type: 'text', text: md }],
+                structuredContent: { ...r, clusters: r.clusters.slice(0, 200) },
+            };
+        }
+        finally {
+            db.close();
+        }
+    });
     server.registerTool('search_intent', {
         title: 'Search intent classification (DataForSEO Labs)',
-        description: 'Classify the search intent (informational / navigational / commercial / transactional) of keywords — primary label + probability + secondary intents. Use it to spot intent mismatch: e.g. a transactional product page ranking for an informational query (a common cause of high impressions / low CTR). Pass siteUrl to persist the intents so run_audit can surface intent-vs-pagetype-mismatch. Labs call, cached 20 days, up to 1000 keywords. Language-only (no location).',
+        description: '[Paid: Labs, cheap, cached 20d, up to 1000 kw/call | Use for: intent mapping before content planning] Classify the search intent (informational / navigational / commercial / transactional) of keywords — primary label + probability + secondary intents. Use it to spot intent mismatch: e.g. a transactional product page ranking for an informational query (a common cause of high impressions / low CTR). Pass siteUrl to persist the intents so run_audit can surface intent-vs-pagetype-mismatch. Labs call, cached 20 days, up to 1000 keywords. Language-only (no location).',
         inputSchema: { keywords: z.array(z.string()).min(1).max(1000), languageCode: z.string().optional(), siteUrl: z.string().optional() },
     }, async ({ keywords, languageCode, siteUrl }) => {
         const client = requireDfs(dfs);
@@ -824,7 +1173,7 @@ export function createServer() {
     });
     server.registerTool('page_lighthouse', {
         title: 'Page Lighthouse — lab Core Web Vitals (DataForSEO On-Page)',
-        description: 'Run a live Lighthouse audit for ONE url: lab Core Web Vitals (LCP, CLS, TBT, FCP, Speed Index), category scores (performance/SEO/best-practices/accessibility), and the top time-saving opportunities. Complements GSC/CrUX field data (aggregate + delayed) with on-demand lab data. Pass siteUrl to persist the CWV so run_audit can surface high-yield-cwv-fail. Paid On-Page call (~2000 credits), slow (~20–120s), cached 20 days.',
+        description: '[Paid: On-Page, ONE URL per call | Use for: CWV evidence on a page that earns clicks] Run a live Lighthouse audit for ONE url: lab Core Web Vitals (LCP, CLS, TBT, FCP, Speed Index), category scores (performance/SEO/best-practices/accessibility), and the top time-saving opportunities. Complements GSC/CrUX field data (aggregate + delayed) with on-demand lab data. Pass siteUrl to persist the CWV so run_audit can surface high-yield-cwv-fail. Paid On-Page call (~2000 credits), slow (~20–120s), cached 20 days.',
         inputSchema: { url: z.string().url(), forMobile: z.boolean().optional(), siteUrl: z.string().optional() },
     }, async ({ url, forMobile, siteUrl }) => {
         const client = requireDfs(dfs);
@@ -878,7 +1227,7 @@ export function createServer() {
     });
     server.registerTool('competitors_domain', {
         title: 'Competitor domains (DataForSEO Labs)',
-        description: 'Discover the domains competing with a target for the same organic keywords (ranked by keyword overlap), with intersection counts and organic traffic estimates. The seed list for content-gap analysis (feed these into page_intersection). Labs call, cached 20 days. Pass location + language for the right market.',
+        description: '[Paid: Labs, cheap, cached 20d | Use for: naming the real competitor set - the opening move of competitive work] Discover the domains competing with a target for the same organic keywords (ranked by keyword overlap), with intersection counts and organic traffic estimates. The seed list for content-gap analysis (feed these into page_intersection). Labs call, cached 20 days. Pass location + language for the right market.',
         inputSchema: { target: z.string(), location: z.union([z.string(), z.number()]).optional(), languageCode: z.string().optional(), limit: z.number().int().min(1).max(100).optional() },
     }, async ({ target, location, languageCode, limit }) => {
         const client = requireDfs(dfs);
@@ -900,7 +1249,7 @@ export function createServer() {
     });
     server.registerTool('page_intersection', {
         title: 'Content gap — page intersection (DataForSEO Labs)',
-        description: 'Find keywords that competitor pages rank for but your page does NOT (the content gap). Pass competitor URLs as `competitorUrls` (max 20; wildcards like https://site.com/blog/* allowed) and your own URL(s) in `excludePages` (max 10) to subtract. Returns gap keywords sorted by search volume, with each competitor’s rank. Labs call, cached 20 days. Pass location + language.',
+        description: '[Paid: Labs, cheap, cached 20d | Use for: page-level content gap vs named rival URLs] Find keywords that competitor pages rank for but your page does NOT (the content gap). Pass competitor URLs as `competitorUrls` (max 20; wildcards like https://site.com/blog/* allowed) and your own URL(s) in `excludePages` (max 10) to subtract. Returns gap keywords sorted by search volume, with each competitor’s rank. Labs call, cached 20 days. Pass location + language.',
         inputSchema: {
             competitorUrls: z.array(z.string()).min(1).max(20),
             excludePages: z.array(z.string()).max(10).optional(),
@@ -952,7 +1301,7 @@ export function createServer() {
     };
     server.registerTool('domain_visibility', {
         title: 'Domain visibility over time (DataForSEO Labs)',
-        description: 'Monthly organic visibility for ANY domain or subdomain — no Search Console access needed: ranking-keyword totals, position distribution (1–3 / 4–10 / 11–20 / 21–100) and estimated traffic value (ETV) per month, plus a trend verdict. The Semrush-style "organic overview / visibility over time" for you or a competitor. Labs call, cached 20 days. Pass location (name or code) for the right market. Grain: month × domain. Joins: domain → Labs/backlinks tools; period → rank_history.',
+        description: '[Paid: Labs, cheap, cached 20d | Use for: visibility-over-time trend, any domain, no GSC needed] Monthly organic visibility for ANY domain or subdomain — no Search Console access needed: ranking-keyword totals, position distribution (1–3 / 4–10 / 11–20 / 21–100) and estimated traffic value (ETV) per month, plus a trend verdict. The Semrush-style "organic overview / visibility over time" for you or a competitor. Labs call, cached 20 days. Pass location (name or code) for the right market. Grain: month × domain. Joins: domain → Labs/backlinks tools; period → rank_history.',
         inputSchema: {
             target: z.string().describe('Domain or subdomain, no scheme (e.g. example.com or blog.example.com)'),
             location: z.union([z.string(), z.number()]).optional(),
@@ -1008,7 +1357,7 @@ export function createServer() {
     });
     server.registerTool('top_pages', {
         title: 'Top ranking pages on a domain (DataForSEO Labs)',
-        description: 'The top organic pages of ANY domain or subdomain, ranked by estimated traffic value (ETV): page, ranking-keyword count, ETV and top-3 / top-10 keyword counts. The Semrush-style "top pages" view — works on competitors, no crawl or GSC needed. Labs call, cached 20 days. Pass location for the right market. Grain: page × domain. Joins: domain → Labs tools; URL → pages.url_key on your own property.',
+        description: '[Paid: Labs, cheap, cached 20d | Use for: where rival traffic actually lands] The top organic pages of ANY domain or subdomain, ranked by estimated traffic value (ETV): page, ranking-keyword count, ETV and top-3 / top-10 keyword counts. The Semrush-style "top pages" view — works on competitors, no crawl or GSC needed. Labs call, cached 20 days. Pass location for the right market. Grain: page × domain. Joins: domain → Labs tools; URL → pages.url_key on your own property.',
         inputSchema: {
             target: z.string().describe('Domain or subdomain, no scheme'),
             location: z.union([z.string(), z.number()]).optional(),
@@ -1055,7 +1404,7 @@ export function createServer() {
     });
     server.registerTool('ranked_keywords', {
         title: 'Ranked keywords for a domain / URL / folder (DataForSEO Labs)',
-        description: 'Every keyword a target ranks for in Google organic — scope it to a whole domain, a subdomain, ONE page (scope:url with the full URL), or a subfolder (scope:folder + folder:"/blog/"). Returns keyword, position, search volume, ETV, the ranking URL, plus keyword difficulty, search intent and SERP features where the response carries them. Set aioOnly:true to list only keywords whose SERP shows a Google AI Overview (your AIO exposure list; per-keyword CITATION checking uses the SERP tools — see the cookbook). The Semrush-style "keywords a page or site ranks for" view — works on any site. Labs call, cached 20 days. Pass location for the right market. Grain: keyword × target. Joins: keyword → GSC search_analytics.query; URL → pages.url_key.',
+        description: '[Paid: Labs, cheap, cached 20d - ONE call replaces any keyword loop | Use for: full keyword footprint of a domain/page/folder] Every keyword a target ranks for in Google organic — scope it to a whole domain, a subdomain, ONE page (scope:url with the full URL), or a subfolder (scope:folder + folder:"/blog/"). Returns keyword, position, search volume, ETV, the ranking URL, plus keyword difficulty, search intent and SERP features where the response carries them. Set aioOnly:true to list only keywords whose SERP shows a Google AI Overview (your AIO exposure list; per-keyword CITATION checking uses the SERP tools — see the cookbook). The Semrush-style "keywords a page or site ranks for" view — works on any site. Labs call, cached 20 days. Pass location for the right market. Grain: keyword × target. Joins: keyword → GSC search_analytics.query; URL → pages.url_key.',
         inputSchema: {
             target: z.string().describe('Domain, subdomain, or full URL (full URL required for scope:url)'),
             scope: z.enum(['domain', 'subdomain', 'url', 'folder']).optional(),
@@ -1151,9 +1500,112 @@ export function createServer() {
             structuredContent: { target: dfsTarget, scope: mode, aioOnly: aioOnly ?? false, totalCount, rowsTotal: kws.length, keywords: kws.slice(0, 100), cached: r.cached, cost: r.cost },
         };
     });
+    // serp_features — the SERP-feature footprint: "how much of my market do AI Overviews,
+    // snippets and other features sit on, and do I already rank page 1 there?" One cached
+    // Labs pull, volume-weighted; persisted so the dashboard can chart it.
+    server.registerTool('serp_features', {
+        title: 'SERP-feature footprint (AI Overviews, snippets, PAA)',
+        description: '[Paid: Labs, ONE cached call | Use for: AI-Overview / zero-click exposure with real numbers] How much of your keyword universe carries each SERP feature - AI Overviews, featured snippets, People Also Ask, shopping, video - weighted by search volume, and how much of that volume you already rank page 1 for. ONE DataForSEO Labs ranked_keywords pull (top-volume sample, cached 20 days, never per-keyword SERP loops). Answers "how exposed are we to AI Overviews / zero-click?" with real numbers. Deterministic: feature PRESENCE (from the Labs index). Judgement proxy: page-1 rank stands in for feature ownership - true ownership needs per-keyword SERP calls (see ranked_keywords aioOnly + the cookbook). Persists to the property DB so the dashboard charts it.',
+        inputSchema: {
+            siteUrl: z.string().describe('The GSC property - results persist to its database'),
+            target: z.string().optional().describe('Override the analysed domain (defaults to the property host)'),
+            location: z.union([z.string(), z.number()]).optional(),
+            languageCode: z.string().optional(),
+            limit: z.number().int().min(100).max(1000).optional().describe('Keyword sample size (default 1000, ordered by volume)'),
+        },
+    }, async ({ siteUrl, target, location, languageCode, limit }) => {
+        const client = requireDfs(dfs);
+        const domain = dfsHost(target ?? siteUrl);
+        const r = await client.rankedKeywords(domain, location, languageCode ?? 'en', limit ?? 1000, 'keyword_data.keyword_info.search_volume,desc');
+        const items = r.tasks[0]?.result?.[0]?.items ?? [];
+        const sample = items.map((it) => {
+            const kd = it.keyword_data ?? {};
+            const serp = it.ranked_serp_element?.serp_item ?? {};
+            return {
+                keyword: kd.keyword,
+                position: Number(serp.rank_absolute) || null,
+                searchVolume: kd.keyword_info?.search_volume ?? null,
+                serpFeatures: Array.isArray(kd.serp_info?.serp_item_types) ? kd.serp_info.serp_item_types : null,
+            };
+        }).filter(s => s.keyword);
+        const fp = computeSerpFootprint(domain, typeof location === 'string' ? location : location != null ? String(location) : null, sample);
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            persistSerpFootprint(db.db, fp);
+        }
+        finally {
+            db.close();
+        }
+        const lines = fp.features.map(f => `| ${f.label} | ${f.volumeSharePct}% | ${fmtNum(f.volume)} | ${f.keywords} | ${f.page1SharePct}% |`);
+        const md = `**SERP-feature footprint for ${domain}** - sample: top ${fp.sampleKeywords} keywords by volume (${fmtNum(fp.sampleVolume)} searches/mo)\n\n` +
+            `| Feature | % of sample volume | Volume on feature SERPs | Keywords | You rank page 1 (share of that volume) |\n|---|---|---|---|---|\n` +
+            lines.join('\n') +
+            `\n\nPresence is deterministic (Labs index); "page 1" is an ownership PROXY - per-keyword citation/ownership checks are the SERP tools. ${r.cached ? 'Cached.' : `Live ($${r.cost.toFixed(4)}).`} Persisted for the dashboard.` +
+            browserLink(siteUrl);
+        return {
+            content: [{ type: 'text', text: md }],
+            structuredContent: fp,
+        };
+    });
+    // market_sizing — Market Sizing and Prioritisation: the organic market read that opens
+    // an engagement. Top-down Labs pulls only (client + <=4 rivals), cached, ~$0.65 worst case.
+    server.registerTool('market_sizing', {
+        title: 'Market Sizing and Prioritisation (share of voice vs competitors)',
+        description: '[Paid: Labs, one cached call per domain (<=5) | Use for: sizing the organic market and who owns it] Build the organic market map: your domain plus up to 4 named competitors, ONE cached Labs ranked_keywords pull each, unioned into a keyword universe. Returns total monthly demand (deduplicated search volume), each domain\'s share of voice (ETV share) overall and per topic cluster, and the leader per cluster - the "here is the market, here is who owns it, here is where to attack" table that opens an engagement. Deterministic: the competitor set and ranked keywords (Labs index). Judgement: ETV is DataForSEO\'s CTR-curve traffic estimate - the SoV percentages inherit that. Persists to the property DB for the dashboard chart. Get the competitor set from competitors_domain first if unsure.',
+        inputSchema: {
+            siteUrl: z.string(),
+            competitors: z.array(z.string()).min(1).max(4).describe('Competitor domains, e.g. ["rival.com", "other.co.uk"]'),
+            location: z.union([z.string(), z.number()]).optional(),
+            languageCode: z.string().optional(),
+            limitPerDomain: z.number().int().min(100).max(1000).optional().describe('Keywords sampled per domain (default 1000, by volume)'),
+        },
+    }, async ({ siteUrl, competitors, location, languageCode, limitPerDomain }) => {
+        const client = requireDfs(dfs);
+        const own = dfsHost(siteUrl);
+        const domains = [own, ...competitors.map(c => dfsHost(c)).filter(c => c && c !== own)];
+        const inputs = [];
+        let cost = 0;
+        let cachedAll = true;
+        for (const domain of domains) { // sequential - the client serialises anyway
+            const r = await client.rankedKeywords(domain, location, languageCode ?? 'en', limitPerDomain ?? 1000, 'keyword_data.keyword_info.search_volume,desc');
+            cost += r.cost;
+            cachedAll = cachedAll && r.cached;
+            const items = r.tasks[0]?.result?.[0]?.items ?? [];
+            inputs.push({
+                domain,
+                keywords: items.map((it) => ({
+                    keyword: it.keyword_data?.keyword,
+                    volume: it.keyword_data?.keyword_info?.search_volume ?? null,
+                    position: Number(it.ranked_serp_element?.serp_item?.rank_absolute) || null,
+                    etv: it.ranked_serp_element?.serp_item?.etv != null ? Number(it.ranked_serp_element.serp_item.etv) : null,
+                })).filter((k) => k.keyword),
+            });
+        }
+        const m = computeMarketSizing(inputs, typeof location === 'string' ? location : location != null ? String(location) : null);
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            persistMarketSizing(db.db, m);
+        }
+        finally {
+            db.close();
+        }
+        const sovLine = m.domains.map(d => `${d === own ? '**' + d + '**' : d} ${m.sovByDomain[d]}%`).join(' · ');
+        const clusterRows = m.clusters.map(c => `| ${c.head.replace(/\|/g, '\\|')} | ${fmtNum(c.volume)} | ${c.keywords} | ${c.leader === own ? '**you**' : (c.leader ?? '-')} | ${m.domains.map(d => `${c.sov[d]}%`).join(' / ')} |`);
+        const md = `**Market Sizing and Prioritisation** - ${m.domains.join(' vs ')}\n\n` +
+            `Universe: ${fmtNum(m.universeKeywords)} unique keywords, **${fmtNum(m.universeVolume)} searches/mo** total demand.\n` +
+            `Share of voice (ETV share): ${sovLine}\n\n` +
+            `| Topic cluster | Volume/mo | Keywords | Leader | SoV (${m.domains.join(' / ')}) |\n|---|---|---|---|---|\n` +
+            clusterRows.join('\n') +
+            `\n\nD: competitor set + ranked keywords (Labs index). N: ETV is a CTR-curve estimate - SoV inherits it. ${cachedAll ? 'All cached.' : `Live cost $${cost.toFixed(4)}.`} Persisted for the dashboard.` +
+            browserLink(siteUrl);
+        return {
+            content: [{ type: 'text', text: md }],
+            structuredContent: m,
+        };
+    });
     server.registerTool('topic_gaps', {
         title: 'Topic gaps — what to cover to be expert in your space (Labs + GSC)',
-        description: 'What related topics should this site cover to be seen as expert in its space? Pulls competitor keyword footprints (DataForSEO Labs ranked_keywords, ≤4 bounded + 20-day-cached calls), subtracts everything YOU already surface for (every GSC query with impressions) or already have a page about (title/H1/slug near-match), clusters the surviving gap keywords lexically, and scores each topic by summed search volume × how many competitors rank there. Each topic names the owning competitor with an example URL, your nearest existing page to build from, and (when resolve_entities has run) whether it sits beside entities you already cover. Pass competitors explicitly (max 3) or let it derive the top 2 from ranking overlap. Needs synced GSC data + DataForSEO credentials.',
+        description: '[Paid: Labs, <=4 cached calls | Use for: topic-level gap plan vs competitors] What related topics should this site cover to be seen as expert in its space? Pulls competitor keyword footprints (DataForSEO Labs ranked_keywords, ≤4 bounded + 20-day-cached calls), subtracts everything YOU already surface for (every GSC query with impressions) or already have a page about (title/H1/slug near-match), clusters the surviving gap keywords lexically, and scores each topic by summed search volume × how many competitors rank there. Each topic names the owning competitor with an example URL, your nearest existing page to build from, and (when resolve_entities has run) whether it sits beside entities you already cover. Pass competitors explicitly (max 3) or let it derive the top 2 from ranking overlap. Needs synced GSC data + DataForSEO credentials.',
         inputSchema: {
             siteUrl: z.string(),
             competitors: z.array(z.string()).max(3).optional().describe('Competitor domains (max 3, e.g. ["rival.com"]). Omitted → derived via competitors_domain'),
@@ -1286,7 +1738,7 @@ export function createServer() {
         }
         const c = data.summary?.current;
         const summary = `Dashboard opened for ${siteUrl} — ${c?.clicks ?? 0} clicks / ${c?.impressions ?? 0} impressions (last 28d)` +
-            `${data.findings ? `, ${data.findings.total} audit findings` : ''}. Interactive charts + findings render in the widget.`;
+            `${data.findings ? `, ${data.findings.total} audit findings` : ''}. Interactive charts + findings render in the widget.` + browserLink(siteUrl);
         return { content: [{ type: 'text', text: summary }], structuredContent: { siteUrl } };
     });
     // App-only data tool: the dashboard widget calls this via app.callServerTool to fetch its
@@ -1316,21 +1768,73 @@ export function createServer() {
         const tpl = readFileSync(path.join(__dirname, 'src', 'ui', 'dashboard.html'), 'utf8');
         const json = JSON.stringify(data).replace(/</g, '\\u003c'); // prevent </script> breakout
         const inject = `<script>window.__DASH_FIXTURE__=${json};window.__DASH_THEME__=${JSON.stringify(theme ?? 'light')};</script>`;
-        const html = tpl.replace(/<head([^>]*)>/i, `<head$1>${inject}`);
+        // Replacement FUNCTION, not string — crawl data containing $& / $' would otherwise
+        // be interpreted as String.replace substitution patterns and corrupt the report.
+        const html = tpl.replace(/<head([^>]*)>/i, (_m, attrs) => `<head${attrs}>${inject}`);
         const dir = path.join(dataDir(), 'reports');
         mkdirSync(dir, { recursive: true });
         const file = path.join(dir, `${sanitizeProperty(siteUrl)}-dashboard.html`);
         writeFileSync(file, html);
         return {
-            content: [{ type: 'text', text: `Report saved: ${file}\nOpen it in any browser for the full interactive dashboard (${data.findings?.total ?? 0} findings). Shareable — send it to a client as-is.` }],
+            content: [{ type: 'text', text: `Report saved: ${file}\nOpen it in any browser for the full interactive dashboard (${data.findings?.total ?? 0} findings). Shareable — send it to a client as-is.` + browserLink(siteUrl) }],
             structuredContent: { path: file, siteUrl, findings: data.findings?.total ?? 0, bytes: html.length },
         };
     });
+    // serve_dashboard — the local webserver delivery surface: the full dashboard in a real
+    // browser tab (live data, property switcher, native downloads), no MCP-App host needed.
+    server.registerTool('serve_dashboard', {
+        title: 'Serve the dashboard on a local webserver',
+        description: 'Start a localhost-only webserver and return a URL that opens the full interactive dashboard in your browser — live data straight from the local database (always current, unlike export_report snapshots), a property switcher, working CSV downloads, and no host widget limits. The server stays up while the MCP server runs; call again with stop=true to shut it down. Localhost only — nothing is exposed to the network.',
+        inputSchema: { siteUrl: z.string().optional(), port: z.number().int().min(1024).max(65535).optional(), stop: z.boolean().optional() },
+    }, async ({ siteUrl, port, stop }) => {
+        if (stop) {
+            const was = dashboardServerUrl();
+            const stopped = await stopDashboardServer();
+            return {
+                content: [{ type: 'text', text: stopped ? `Dashboard server stopped (was ${was}).` : 'No dashboard server running.' }],
+                structuredContent: { stopped },
+            };
+        }
+        const { url } = await startDashboardServer({
+            dataDir,
+            uiHtml: () => readFileSync(path.join(__dirname, 'src', 'ui', 'dashboard.html'), 'utf8'),
+            call: {
+                get_dashboard_data: async (a) => {
+                    const want = String(a.siteUrl ?? '');
+                    // Only serve properties that actually exist locally — getDashboardData would
+                    // otherwise CREATE an empty DB file for any bogus siteUrl posted at the API.
+                    if (!listLocalProperties(dataDir()).some(p => p.siteUrl === want))
+                        throw new Error(`unknown property ${want}`);
+                    return getDashboardData(dataDir(), want);
+                },
+                related_terms: async (a) => {
+                    const r = await requireDfs(dfs).relatedTerms(String(a.keyword ?? ''), a.location, a.languageCode);
+                    return r;
+                },
+                keyword_volume: async (a) => {
+                    const r = await requireDfs(dfs).searchVolume(a.keywords ?? [], a.location, a.languageCode);
+                    const items = (r.tasks[0]?.result ?? []).map((k) => ({
+                        keyword: k.keyword, searchVolume: k.search_volume, cpc: k.cpc, competition: k.competition,
+                    }));
+                    return { keywords: items, cached: r.cached, cost: r.cost };
+                },
+            },
+            ...(port != null ? { port } : {}),
+        });
+        const open = siteUrl ? `${url}/dashboard?siteUrl=${encodeURIComponent(siteUrl)}` : `${url}/`;
+        const portNote = port != null && !url.endsWith(`:${port}`)
+            ? ` (already running on its original port — requested port ${port} ignored; stop=true first to move it)`
+            : '';
+        return {
+            content: [{ type: 'text', text: `Dashboard server running — open ${open} in your browser${portNote}. Live data, all charts, CSV downloads, property switcher. Stays up while this MCP server runs; serve_dashboard stop=true to stop it.` }],
+            structuredContent: { url: open, base: url },
+        };
+    });
     // pull_backlinks — on-demand backlink profile (DataForSEO, paid + 20-day cached). Powers
     // backlinks-to-404 (the big quick win), top-linked pages, and true-orphan detection.
     server.registerTool('pull_backlinks', {
         title: 'Pull backlink profile (DataForSEO)',
-        description: 'Fetch the property’s backlink profile (overall summary — total backlinks, referring domains, Domain Rank, broken backlinks/pages, nofollow share — plus per-page backlink/referring-domain counts) into page_backlinks, and resolve each backlinked page’s live HTTP status so run_audit can flag external backlinks pointing to dead (4xx/5xx) pages. Paid DataForSEO call, 20-day cached, on-demand only. Async job — poll check_sync_status. Grain: one row per backlinked URL. Joins: url_key → pages/GSC; domain → Labs tools.',
+        description: '[Paid: Backlinks subscription (separate - 40204 = not activated), cached 20d | Use for: authority + dead-backlink recovery] Fetch the property’s backlink profile (overall summary — total backlinks, referring domains, Domain Rank, broken backlinks/pages, nofollow share — plus per-page backlink/referring-domain counts) into page_backlinks, and resolve each backlinked page’s live HTTP status so run_audit can flag external backlinks pointing to dead (4xx/5xx) pages. Paid DataForSEO call, 20-day cached, on-demand only. Async job — poll check_sync_status. Grain: one row per backlinked URL. Joins: url_key → pages/GSC; domain → Labs tools.',
         inputSchema: { siteUrl: z.string(), limit: z.number().int().min(1).max(1000).optional(), statusLimit: z.number().int().min(0).max(1000).optional() },
     }, async ({ siteUrl, limit, statusLimit }) => {
         const bl = requireDfs(backlinks);