@houtini/seo-audit-console 0.3.0 → 0.4.6

package/dist/server.js

@@ -11,6 +11,12 @@ import { getDashboardData } from './core/dashboardData.js';
 import { startDashboardServer, stopDashboardServer, dashboardServerUrl, listLocalProperties } from './core/webServer.js';
 import { computeSerpFootprint, persistSerpFootprint } from './core/serpFootprint.js';
 import { computeMarketSizing, persistMarketSizing } from './core/marketSizing.js';
+import { FirecrawlClient } from './core/FirecrawlClient.js';
+import { SupadataClient } from './core/SupadataClient.js';
+import { fetchCompetitorContent, routeFor } from './core/reconResearch.js';
+import { fetchOwnPage } from './core/reconFetch.js';
+import { parseSerpForRecon, reconVerdict } from './core/serpRecon.js';
+import { selectReconTargets, deterministicTodos, persistReconPage, insertTodos, pageState, crawlRealityOverride, clusterCannibalisation, opportunityBasis } from './audit/recon.js';
 /** A clickable browser-dashboard link appended to tool outputs — the user should always
  * know the full interactive report is one click away (or one serve_dashboard call away). */
 function browserLink(siteUrl) {
@@ -165,6 +171,7 @@ Raw access: query_audit runs any single check with full evidence; every table ab
 - **Market read:** serp_features (feature/AIO exposure, volume-weighted) + domain_visibility for the client and each named rival (one cached call each) → who is structurally winning, and how much of the market SERP features already absorb.
 - **Content plan:** suggest_pages (demand you already earn impressions for) + topic_gaps (demand rivals own that you don't) → draft_content for the winners. Every proposal traces to real impressions or a rival's real footprint - no invented "keyword ideas".
 - **Fix-and-prove cycle:** fix_finding on the top finding → ship → start_crawl → detect_changes shows the fix landed → re-run run_audit and watch the finding drop off. That screenshot is the client update.
+- **Content recon (why a page is losing):** recon_targets picks the worst declining/striking pages, fetches our live page + the Google SERP (organic rank + AI-Overview citations + video), and classifies WHY — the sharpest class is "we rank but the AI Overview won't quote us" = a data-accuracy/freshness/markup problem. Then research the competitor set it returns (firecrawl for pages, supadata for the ranking videos), write the gaps back with save_recon_todo, and track the fixes with recon_todos (which can re-measure whether you moved from uncited→cited). Pass location to match where your impressions come from — organic rank is location-sensitive.
 - **Cost rule of thumb:** an entire competitive read (visibility + footprint + gaps for 4 domains) is a handful of cached Labs calls - under a dollar. If a plan involves looping SERP calls over a keyword list, it is the wrong plan; a Labs endpoint already has that answer top-down.
 
 Plan the join first (url_key / query / domain), state the grain of each side, then run the fewest paid calls that answer it.`;
@@ -246,6 +253,12 @@ export function createServer() {
         : null;
     const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
     const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
+    // Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
+    const firecrawlKey = process.env.FIRECRAWL_API_KEY;
+    const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
+    // Supadata (transcribes the ranking videos for content recon) — optional; degrades gracefully.
+    const supadataKey = process.env.SUPADATA_API_KEY;
+    const supadata = supadataKey ? new SupadataClient(supadataKey, path.join(dataDir(), 'supadata-cache.db')) : null;
     const entities = new Entities(new WikidataClient(path.join(dataDir(), 'wikidata-cache.db')), dataDir());
     const refresh = new Refresh(sync, crawler, inspector, rankTracker);
     const requireGsc = (v) => {
@@ -440,14 +453,21 @@ export function createServer() {
     });
     server.registerTool('score_passages', {
         title: 'Score passage relevance (AI-search readiness)',
-        description: 'Run a local cross-encoder reranker (ms-marco-MiniLM-L-6-v2, downloaded once to a cache, no Python) over each ranking page\'s heading chunks against its top Search Console query, and persist the single best-passage relevance score. It SCORES relevance (a classifier — it cannot hallucinate). Powers the `weak-passage-answer` check — the "RAG snippetability" test: does any passage confidently answer the query, the way AI/passage search re-ranks? On-demand + ML-heavy (~0.6s/page), bounded to pages that already rank. `limit` caps pages (highest-impression first); `minImpressions` sets the floor (default 50). First run downloads ~25MB. Re-run after a re-crawl.',
-        inputSchema: { siteUrl: z.string(), limit: z.number().int().min(1).max(2000).optional(), minImpressions: z.number().int().min(0).optional() },
-    }, async ({ siteUrl, limit, minImpressions }) => {
+        description: 'Run a local cross-encoder reranker (ms-marco-MiniLM-L-6-v2, downloaded once to a cache, no Python) over each ranking page\'s heading chunks against its top Search Console query, and persist the single best-passage relevance score. It SCORES relevance (a classifier — it cannot hallucinate). Powers the `weak-passage-answer` check — the "RAG snippetability" test: does any passage confidently answer the query, the way AI/passage search re-ranks? Returns the score for EVERY page it scored (not just the failures) with the flag threshold, the band convention and the run\'s own distribution, so a passing page\'s score is readable without a second query_data hop. On-demand + ML-heavy (~0.6s/page), bounded to pages that already rank. `limit` caps pages (highest-impression first); `minImpressions` sets the floor (default 50). First run downloads ~25MB. Re-run after a re-crawl.',
+        inputSchema: { siteUrl: z.string(), limit: z.number().int().min(1).max(2000).optional(), minImpressions: z.number().int().min(0).optional(), urls: z.array(z.string()).optional().describe('Only report these pages (they are still scored site-wide; this filters the returned list)') },
+    }, async ({ siteUrl, limit, minImpressions, urls }) => {
         const r = await scoreSitePassages(dataDir(), siteUrl, { limit, minImpressions });
-        const lines = r.weakest.slice(0, 15).map(w => `  ${w.score}  ${w.url.replace(/^https?:\/\/[^/]+/, '')} · "${w.query}"`).join('\n');
+        const keys = urls?.length ? new Set(urls.map(u => urlKey(u, { hostForm: hostFormForProperty(siteUrl) ?? 'asis' }))) : null;
+        const shown = keys ? r.pages.filter(p => keys.has(p.url)) : r.pages;
+        const rows = shown.slice(0, 25).map(p => `  ${String(p.score).padStart(7)}  ${p.band.padEnd(8)} ${p.url.replace(/^https?:\/\/[^/]+/, '')} · "${p.query}" (${p.impressions} impr)`).join('\n');
+        const d = r.distribution;
+        const text = `Scored ${r.scored} ranking pages; ${r.flagged} below the weak threshold of ${r.threshold} — run_audit surfaces those as weak-passage-answer.\n\n` +
+            `Scale: ${r.scale}\n` +
+            (d ? `This run: min ${d.min} · p25 ${d.p25} · median ${d.median} · p75 ${d.p75} · max ${d.max}\n` : '') +
+            (rows ? `\nScores (${shown.length === r.pagesTotal ? `all ${r.pagesTotal}` : `${shown.length} of ${r.pagesTotal}`}, showing up to 25):\n${rows}` : '');
         return {
-            content: [{ type: 'text', text: `Scored ${r.scored} ranking pages; ${r.flagged} have no strongly-relevant passage (score < 3) — run_audit surfaces them as weak-passage-answer.${lines ? `\n\nWeakest:\n${lines}` : ''}` }],
-            structuredContent: r,
+            content: [{ type: 'text', text }],
+            structuredContent: { ...r, ...(keys ? { pages: shown, pagesReturned: shown.length } : {}) },
         };
     });
     server.registerTool('draft_content', {
@@ -874,6 +894,314 @@ export function createServer() {
             },
         };
     });
+    // ── Content recon (recon_targets) — the data-intensive "why are we losing, what to do" mission ──
+    server.registerTool('recon_targets', {
+        title: 'Content recon: why a page is losing, and what to do about it',
+        description: '[Paid: DataForSEO SERP per page (~$0.004 each, plus a small refundable surcharge for loading async AI Overviews), bounded to the batch | Use for: the deep "why are we behind and what to add" recon] For each of your worst declining / striking-distance pages (auto-selected by impressions x decline, position 3-15; or pass explicit urls), this fetches OUR live page with the crawler, pulls the live Google SERP (DataForSEO SERP-advanced, depth 20 organic), and classifies WHY we are behind using the organic-rank x AI-Overview-citation matrix: defend-and-deepen (cited + strong), accuracy-or-freshness (rank but the AIO will not quote us - the sharpest, most actionable class), consolidate-weak-page, competitive-gap, or page-cannot-rank (the crawl says the URL is noindex/canonicalised away, so its GSC history is legacy and the SERP read belongs to another page). Honesty rules baked in: every GSC figure carries its 28-day window; organicRank:null means "absent from the top 20 ORGANIC results", never a position; and an AI Overview whose citations could not be resolved reports aioCitesUs:null (UNKNOWN) rather than "not cited". To-dos are prioritised by OPPORTUNITY (impressions x the CTR gap between where you rank and a realistic target, damped by the verdict), not by raw impressions, and cannibalisation is counted across the whole query cluster. Set scrapeCompetitors:true to also pull the top competitors as content, routed by host: YouTube/video → Supadata transcript (SUPADATA_API_KEY), Reddit → its .json, other pages → Firecrawl (FIRECRAWL_API_KEY) with a free HTTP fallback; competitorLimit (default 5) caps how many are fetched and everything above the cap is listed as skipped. Cloudflare-challenge sites (e.g. PCMag) still can\'t be fetched from a server and come back as a per-URL error — the SERP still tells you they rank; if one matters, ask the user to paste its copy or supply a text file and diff that in. Transcribe the videos (usually what wins these SERPs) and use the reachable pages. Then write findings back with save_recon_todo; track with recon_todos. **Async job:** returns a jobId immediately - poll check_sync_status; the finished job carries the per-page verdicts, to-dos and a summary. (Each page is persisted to the ledger as it completes, so recon_todos shows results even mid-run.)',
+        inputSchema: {
+            siteUrl: z.string(),
+            limit: z.number().int().min(1).max(50).optional().describe('Pages per batch (default 5; async so large batches are fine)'),
+            minImpressions: z.number().int().min(1).optional(),
+            location: z.union([z.string(), z.number()]).optional(),
+            urls: z.array(z.string()).optional().describe('Analyse these exact pages instead of auto-selecting'),
+            scrapeCompetitors: z.boolean().optional().describe('Also fetch the top competitors (video→transcript, pages→markdown/HTML)'),
+            competitorLimit: z.number().int().min(1).max(12).optional().describe('How many competitors to fetch per page when scrapeCompetitors is on (default 5). Whatever is not fetched is listed as skipped, never dropped silently.'),
+            crawlAs: z.enum(['browser', 'googlebot']).optional().describe('UA for the free HTTP fetch: browser (default, mimics a visit from Google — gets Reddit + mid-tier) or googlebot'),
+        },
+    }, async ({ siteUrl, limit, minImpressions, location, urls, scrapeCompetitors, competitorLimit, crawlAs }) => {
+        const client = requireDfs(dfs); // fail fast if no DataForSEO creds, before starting the job
+        const count = urls?.length ?? limit ?? 5;
+        // Async job: N live page-fetches + N serialised SERP calls exceed the ~60s MCP ceiling
+        // past a handful of pages, so return a jobId and poll (like refresh_property).
+        const jobId = jobs.start('recon', async (update, signal) => {
+            const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+            try {
+                const fresh = gscFreshness(db.db);
+                if (!fresh.effectiveMax)
+                    return { siteUrl, empty: true, note: 'No synced GSC data — run refresh_property first.' };
+                const today = new Date().toISOString().slice(0, 10);
+                const ownDomain = dfsHost(siteUrl);
+                const hostForm = hostFormForProperty(siteUrl) ?? 'asis';
+                const SERP_DEPTH = 20;
+                // Every GSC number below is this window and only this window. Undeclared, a 28-day figure
+                // read against an all-time baseline looks exactly like a decline.
+                const windowStart = db.db.prepare(`SELECT date(?, '-27 days') d`).get(fresh.effectiveMax);
+                const gscWindow = { start: windowStart.d, end: fresh.effectiveMax, days: 28, metric: 'Search Console, last 28 days of synced data (not all-time)' };
+                let targets;
+                if (urls?.length) {
+                    targets = [];
+                    for (const u of urls) {
+                        const key = urlKey(u, { hostForm });
+                        const row = db.db.prepare(`SELECT query, SUM(impressions) imp, SUM(clicks) clk, SUM(position*impressions)*1.0/NULLIF(SUM(impressions),0) pos
+                FROM search_analytics WHERE page_key=? AND query IS NOT NULL AND date > date(?, '-28 days') AND date <= ? GROUP BY query ORDER BY imp DESC LIMIT 1`)
+                            .get(key, fresh.effectiveMax, fresh.effectiveMax);
+                        if (row?.query)
+                            targets.push({ urlKey: key, query: row.query, impressions: row.imp, clicks: row.clk, position: Math.round(row.pos * 10) / 10, priorPosition: null, slipped: false, competingUrls: 0 });
+                    }
+                }
+                else {
+                    targets = selectReconTargets(db.db, { ...(limit != null ? { limit } : {}), ...(minImpressions != null ? { minImpressions } : {}), maxDate: fresh.effectiveMax });
+                }
+                if (!targets.length)
+                    return { siteUrl, targets: [], note: `No recon targets (declining/striking pages, position 3-15, ${minImpressions ?? 300}+ impressions). Lower minImpressions or pass urls.` };
+                const results = [];
+                let cost = 0;
+                for (let i = 0; i < targets.length; i++) {
+                    if (signal.aborted)
+                        break;
+                    const t = targets[i];
+                    update({ phase: 'recon', done: i, total: targets.length, current: t.urlKey });
+                    let own = null;
+                    try {
+                        own = await fetchOwnPage(t.urlKey, hostForm === 'asis' ? 'asis' : hostForm);
+                    }
+                    catch { /* page unreachable — classify on SERP alone */ }
+                    const serpResp = await client.serpOrganic(t.query, location, 'en', SERP_DEPTH, { loadAsyncAiOverview: true });
+                    cost += serpResp.cost;
+                    const serp = parseSerpForRecon(serpResp, ownDomain, SERP_DEPTH);
+                    // The crawl row overrides the SERP/GSC read: GSC keeps reporting impressions for URLs
+                    // that have since been canonicalised away or set noindex.
+                    const state = pageState(db.db, t.urlKey);
+                    const serpVerdict = reconVerdict(serp);
+                    const verdict = crawlRealityOverride(serpVerdict, state) ?? serpVerdict;
+                    const cluster = clusterCannibalisation(db.db, t.query, fresh.effectiveMax);
+                    const basis = opportunityBasis(t.impressions, serp.ourOrganicRank, t.position, verdict.verdict);
+                    const baseline = { organicRank: serp.ourOrganicRank, serpDepth: SERP_DEPTH, aioCitesUs: serp.aioCitesUs, gscPosition: t.position, gscImpressions: t.impressions, window: gscWindow, at: today };
+                    persistReconPage(db.db, {
+                        urlKey: t.urlKey, query: t.query, verdict: verdict.verdict, verdictNote: verdict.note,
+                        organicRank: serp.ourOrganicRank, gscPosition: t.position, gscImpressions: t.impressions,
+                        aioPresent: serp.aioPresent, aioCitesUs: serp.aioCitesUs, videoPresent: serp.videoPresent,
+                        hasProductSchema: own?.hasProductOrReview ?? false, schemaTypes: own?.jsonLdTypes ?? [],
+                        competitors: { organicAbove: serp.organicAbove, aioReferences: serp.aioReferences, videoItems: serp.videoItems },
+                    });
+                    const todos = own ? deterministicTodos(t, own, serp, verdict, today, { state, cluster }) : [];
+                    const inserted = todos.length ? insertTodos(db.db, t.urlKey, t.query, todos, baseline, 'auto') : 0;
+                    let competitorContent = undefined;
+                    let competitorFetch = undefined;
+                    if (scrapeCompetitors && (firecrawl || supadata)) {
+                        // Build a routed candidate set: organic-above + AI-Overview references + one ranking
+                        // video, deduped, our own domain removed. Each URL routes by HOST (YouTube→supadata,
+                        // Reddit→.json, else→firecrawl). What we don't fetch is REPORTED, not silently dropped
+                        // — an undeclared cap of 3 is what made this look like it returned nothing.
+                        const seen = new Set();
+                        const candidates = [];
+                        const add = (url, from) => {
+                            if (url && !seen.has(url) && dfsHost(url) !== ownDomain) {
+                                seen.add(url);
+                                candidates.push({ url, from, route: routeFor(url) });
+                            }
+                        };
+                        serp.organicAbove.forEach(o => add(o.url, `organic #${o.rank}`));
+                        serp.aioReferences.slice(0, 6).forEach(r => add(r.url, 'aio-reference'));
+                        serp.videoItems.slice(0, 2).forEach(v => add(v.url, 'video-pack'));
+                        const lim = competitorLimit ?? 5;
+                        // Organic-above first (the editorial pages actually worth diffing), but keep one slot
+                        // for a ranking video — video is usually what wins these SERPs.
+                        const video = candidates.find(c => c.route === 'video');
+                        const rest = candidates.filter(c => c !== video);
+                        const reserve = video && lim >= 3 ? 1 : 0;
+                        const chosen = [...rest.slice(0, lim - reserve), ...(reserve && video ? [video] : [])];
+                        const skipped = candidates.filter(c => !chosen.includes(c));
+                        competitorContent = [];
+                        for (const c of chosen) {
+                            competitorContent.push(await fetchCompetitorContent(c.url, { firecrawl, supadata }, { maxChars: 4000, ua: crawlAs }));
+                        }
+                        competitorFetch = {
+                            limit: lim,
+                            identified: candidates.length,
+                            attempted: chosen.length,
+                            fetched: competitorContent.filter((c) => !c.error && c.content).length,
+                            failed: competitorContent.filter((c) => c.error).length,
+                            empty: competitorContent.filter((c) => !c.error && !c.content).length,
+                            routes: chosen.map(c => ({ url: c.url, from: c.from, route: c.route })),
+                            skipped: skipped.map(c => ({ url: c.url, from: c.from, route: c.route })),
+                            note: skipped.length ? `${skipped.length} identified competitor(s) not fetched (competitorLimit ${lim}) — raise competitorLimit or fetch them directly.` : undefined,
+                        };
+                    }
+                    results.push({
+                        urlKey: t.urlKey, query: t.query, window: gscWindow,
+                        impressions: t.impressions, gscPosition: t.position, priorPosition: t.priorPosition, slipped: t.slipped,
+                        organicRank: serp.ourOrganicRank, serpDepth: SERP_DEPTH,
+                        organicRankNote: serp.ourOrganicRank == null ? `absent from the top ${SERP_DEPTH} organic results — position beyond ${SERP_DEPTH} was not measured` : undefined,
+                        aioPresent: serp.aioPresent, aioCitesUs: serp.aioCitesUs, aioResolved: serp.aioResolved, aioAsync: serp.aioAsync,
+                        aioNote: serp.aioCitesUs == null && serp.aioPresent ? 'AI Overview present but citations unresolved — citation status UNKNOWN, not "not cited"' : undefined,
+                        videoPresent: serp.videoPresent,
+                        verdict: verdict.verdict, verdictNote: verdict.note,
+                        ...(verdict.verdict === 'page-cannot-rank' ? { serpVerdict: serpVerdict.verdict, serpVerdictNote: serpVerdict.note } : {}),
+                        pageState: state, priorityBasis: basis, cannibalisationCluster: cluster,
+                        ownHeadings: own?.headings.map(h => h.heading) ?? null, schemaTypes: own?.jsonLdTypes ?? null, hasProductSchema: own?.hasProductOrReview ?? null, dateModified: own?.dateModified ?? null,
+                        todos, todosInserted: inserted,
+                        competitors: { organicAbove: serp.organicAbove, aioReferences: serp.aioReferences, videoItems: serp.videoItems },
+                        ...(competitorContent ? { competitorContent, competitorFetch } : {}),
+                    });
+                }
+                update({ phase: 'done', done: results.length, total: targets.length });
+                const md = `# Content recon — ${siteUrl}\n\n${results.length} page(s), ${results.reduce((s, r) => s + r.todosInserted, 0)} to-dos saved. DataForSEO SERP cost $${cost.toFixed(4)}.\n` +
+                    `All Search Console figures below are **${gscWindow.start} to ${gscWindow.end}** (28 days) — not all-time; SERP ranks are live at depth ${SERP_DEPTH} organic.\n\n` +
+                    results.map(r => {
+                        const aio = !r.aioPresent ? 'no AIO'
+                            : r.aioCitesUs == null ? 'AIO: citation UNKNOWN (unresolved)'
+                                : r.aioCitesUs ? 'AIO: cited' : `AIO: NOT cited (${r.competitors.aioReferences.length} refs checked)`;
+                        const flags = [aio, r.videoPresent ? 'video pack' : null].filter(Boolean).join(' · ');
+                        const rank = r.organicRank != null ? `organic #${r.organicRank}` : `not in the top ${SERP_DEPTH} organic`;
+                        const cann = r.cannibalisationCluster?.cannibalising ? `\nCluster overlap: ${r.cannibalisationCluster.urls.length} of your URLs across "${r.cannibalisationCluster.core.join(' ')}", colliding on ${r.cannibalisationCluster.collidingQueries} of ${r.cannibalisationCluster.clusterQueries} queries.` : '';
+                        const cf = r.competitorFetch
+                            ? `\nCompetitors: ${r.competitorFetch.fetched}/${r.competitorFetch.attempted} fetched of ${r.competitorFetch.identified} identified (limit ${r.competitorFetch.limit})${r.competitorFetch.skipped.length ? `; skipped ${r.competitorFetch.skipped.map((s) => s.url).join(', ')}` : ''}.`
+                            : '';
+                        return `## ${r.urlKey}\n"${r.query}" — ${rank} (GSC avg ${r.gscPosition}${r.slipped ? `, slipped from ${r.priorPosition}` : ''}), ${r.impressions} impr. ${flags}\n` +
+                            `**${r.verdict}** — ${r.verdictNote}\nOpportunity: ~${r.priorityBasis.opportunityClicks} clicks/28d if it moved from ${r.priorityBasis.position} (${r.priorityBasis.positionSource}) to ${r.priorityBasis.targetPosition}.${cann}${cf}\n` +
+                            (r.todos.length ? '\nTo do:\n' + r.todos.map((t) => `- [${t.type}] ${t.action}`).join('\n') : '');
+                    }).join('\n\n') +
+                    (() => {
+                        const errs = results.flatMap(r => (r.competitorContent ?? []).filter((c) => c.error));
+                        if (!errs.length)
+                            return '';
+                        const needKey = errs.filter(c => /not set/i.test(c.error));
+                        const blocked = errs.filter(c => !/not set/i.test(c.error));
+                        let note = '';
+                        if (blocked.length)
+                            note += `\n\n${blocked.length} competitor(s) couldn't be fetched (bot-protection like Cloudflare, a block, or a fetch error): ${blocked.slice(0, 5).map(c => `${c.url} [${c.via}: ${c.error}]`).join('; ')}. If one matters, paste its copy here or drop a text file and I'll diff it in.`;
+                        if (needKey.length)
+                            note += `\n\n${needKey.length} competitor(s) skipped for a missing API key: ${[...new Set(needKey.map(c => c.error))].join('; ')}.`;
+                        return note;
+                    })() +
+                    `\n\nNext: research the competitors (transcribe the videos, read the reachable pages), write gaps back with save_recon_todo, and track with recon_todos.` + browserLink(siteUrl);
+                // The poll (check_sync_status) injects this result into the model's context, which the
+                // host caps (~25k tokens). The full per-page detail (heading lists, competitor URL arrays,
+                // full to-do objects, cannibalisation clusters) is already persisted to the ledger and read
+                // back via recon_todos, so the job result returns the readable `summary` plus a SLIM per-page
+                // view — verdict, ranks, opportunity — and keeps only competitorContent (the scraped research
+                // payload, which lives nowhere else). This keeps a 10-page poll well under the cap.
+                const slimTargets = results.map(r => ({
+                    urlKey: r.urlKey, query: r.query, window: r.window,
+                    impressions: r.impressions, gscPosition: r.gscPosition, priorPosition: r.priorPosition, slipped: r.slipped,
+                    organicRank: r.organicRank, serpDepth: r.serpDepth, organicRankNote: r.organicRankNote,
+                    aioPresent: r.aioPresent, aioCitesUs: r.aioCitesUs, aioResolved: r.aioResolved, aioNote: r.aioNote, videoPresent: r.videoPresent,
+                    verdict: r.verdict, verdictNote: r.verdictNote,
+                    opportunityClicks: r.priorityBasis?.opportunityClicks, positionSource: r.priorityBasis?.positionSource, targetPosition: r.priorityBasis?.targetPosition,
+                    hasProductSchema: r.hasProductSchema, dateModified: r.dateModified, todosInserted: r.todosInserted,
+                    ...(r.competitorContent ? { competitorContent: r.competitorContent, competitorFetch: r.competitorFetch } : {}),
+                }));
+                return { siteUrl, cost, window: gscWindow, serpDepth: SERP_DEPTH, targets: slimTargets, summary: md };
+            }
+            finally {
+                db.close();
+            }
+        });
+        return {
+            content: [{ type: 'text', text: `Content recon started for ${count} page(s) — job ${jobId}. Each page is a live page-fetch + a serialised SERP call (a few seconds each), so poll check_sync_status with jobId "${jobId}"; the finished job's result carries the per-page verdicts, to-dos and a summary.` }],
+            structuredContent: { jobId, status: 'running', siteUrl },
+        };
+    });
+    // save_recon_todo — the research session writes its content-gap / originality findings back
+    // into the ledger against a page (source: 'research').
+    server.registerTool('save_recon_todo', {
+        title: 'Save content-recon to-dos (research writeback)',
+        description: 'Write content-recon findings back into the trackable ledger for a page — the gaps and originality the research session found by diffing competitors (firecrawl) and videos (supadata) against our content. Each to-do is an action with a type (content-gap / originality / schema / freshness / format), rationale and evidence. Snapshots the page baseline so the fix\'s effect on rank/AIO-citation is measurable later. Run recon_targets first (it classifies the page and seeds the deterministic to-dos); this adds the judgement ones. Track everything with recon_todos.',
+        inputSchema: {
+            siteUrl: z.string(),
+            urlKey: z.string().describe('The page (any URL form — normalised to its key)'),
+            todos: z.array(z.object({
+                action: z.string(),
+                type: z.string().optional(),
+                rationale: z.string().optional(),
+                evidence: z.record(z.any()).optional(),
+                priority: z.number().optional(),
+            })).min(1),
+        },
+    }, async ({ siteUrl, urlKey: rawUrl, todos }) => {
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            const key = urlKey(rawUrl, { hostForm: hostFormForProperty(siteUrl) ?? 'asis' });
+            const page = db.db.prepare(`SELECT query, organic_rank, aio_cites_us, gsc_position, gsc_impressions, verdict FROM recon_page WHERE url_key=?`).get(key);
+            // aio_cites_us is NULL when the AI Overview never resolved — keep that as unknown rather
+            // than coercing it to false, or the outcome diff reads "no → yes" off a missing fetch.
+            const baseline = page ? { organicRank: page.organic_rank, aioCitesUs: page.aio_cites_us == null ? null : !!page.aio_cites_us, gscPosition: page.gsc_position, at: new Date().toISOString().slice(0, 10) } : {};
+            // Default priority to the PAGE's opportunity so research to-dos sort on the same scale as
+            // the deterministic ones (a default of 0 buried every judgement finding at the bottom).
+            const basis = page ? opportunityBasis(page.gsc_impressions ?? 0, page.organic_rank, page.gsc_position ?? 20, page.verdict) : null;
+            const fallbackPriority = basis ? Math.max(1, Math.round(basis.opportunityClicks * basis.verdictFactor)) : 0;
+            const drafts = todos.map(t => ({ action: t.action, type: t.type ?? 'content-gap', rationale: t.rationale ?? '', evidence: t.evidence ?? {}, priority: t.priority ?? fallbackPriority }));
+            const n = insertTodos(db.db, key, page?.query ?? '', drafts, baseline, 'research');
+            return { content: [{ type: 'text', text: `Saved ${n} recon to-do(s) for ${key}${n < drafts.length ? ` (${drafts.length - n} already open)` : ''}. Track with recon_todos.` }], structuredContent: { urlKey: key, inserted: n } };
+        }
+        finally {
+            db.close();
+        }
+    });
+    // recon_todos — list, track and annotate the ledger. No id → list (optionally filtered);
+    // id → update status and/or append a dated annotation, and optionally re-measure the outcome.
+    server.registerTool('recon_todos', {
+        title: 'List, track and annotate content-recon to-dos',
+        description: 'The content-recon to-do board. With no id: list to-dos (optionally filter by page or status), grouped by page with each page\'s verdict — the pick-a-page-to-work-on surface, and the hand-off to content-machine. With id: update one to-do — set status (open → researching → drafted → shipped → dismissed) and/or append a dated annotation note (your own observations, the history). On status:shipped with remeasure:true it re-fetches the SERP and records the outcome, so you can see whether the fix moved you from AIO-uncited to cited, or up the organic ranks.',
+        inputSchema: {
+            siteUrl: z.string(),
+            urlKey: z.string().optional().describe('Filter the list to one page'),
+            status: z.enum(['open', 'researching', 'drafted', 'shipped', 'dismissed']).optional().describe('Filter the list, or the new status when id is set'),
+            id: z.number().int().optional().describe('Update this to-do'),
+            note: z.string().optional().describe('Append a dated annotation to this to-do'),
+            remeasure: z.boolean().optional().describe('On status:shipped, re-fetch the SERP and record the outcome (paid)'),
+            location: z.union([z.string(), z.number()]).optional(),
+        },
+    }, async ({ siteUrl, urlKey: rawUrl, status, id, note, remeasure, location }) => {
+        const db = new AuditDatabase(dbPathFor(dataDir(), siteUrl));
+        try {
+            if (id != null) {
+                const row = db.db.prepare(`SELECT * FROM recon_todo WHERE id=?`).get(id);
+                if (!row)
+                    throw new Error(`No recon to-do #${id}.`);
+                let notes = row.notes ?? '';
+                if (note)
+                    notes = (notes ? notes + '\n' : '') + `[${new Date().toISOString().slice(0, 10)}] ${note}`;
+                let outcome = row.outcome;
+                let outcomeNote = '';
+                if (status === 'shipped' && remeasure && dfs) {
+                    // Same depth + async-AIO handling as recon_targets, so before/after are comparable.
+                    const serpResp = await dfs.serpOrganic(row.query, location, 'en', 20, { loadAsyncAiOverview: true });
+                    const serp = parseSerpForRecon(serpResp, dfsHost(siteUrl), 20);
+                    const cited = (v) => (v == null ? 'unknown' : v ? 'yes' : 'no');
+                    outcome = JSON.stringify({ organicRank: serp.ourOrganicRank, serpDepth: 20, aioCitesUs: serp.aioCitesUs, aioResolved: serp.aioResolved, at: new Date().toISOString().slice(0, 10) });
+                    const base = row.baseline ? JSON.parse(row.baseline) : {};
+                    const rankStr = (r) => (r == null ? 'not in top 20' : `#${r}`);
+                    outcomeNote = ` Outcome: organic ${rankStr(base.organicRank)}→${rankStr(serp.ourOrganicRank)}, AIO cited ${cited(base.aioCitesUs)}→${cited(serp.aioCitesUs)}.`;
+                }
+                db.db.prepare(`UPDATE recon_todo SET status=COALESCE(?,status), notes=?, outcome=COALESCE(?,outcome), updated_at=datetime('now') WHERE id=?`)
+                    .run(status ?? null, notes, outcome ?? null, id);
+                return { content: [{ type: 'text', text: `Updated to-do #${id}${status ? ` → ${status}` : ''}${note ? ' (note added)' : ''}.${outcomeNote}` }], structuredContent: { id, status: status ?? row.status, outcome: outcome ? JSON.parse(outcome) : null } };
+            }
+            const key = rawUrl ? urlKey(rawUrl, { hostForm: hostFormForProperty(siteUrl) ?? 'asis' }) : null;
+            const where = [];
+            const args = [];
+            if (key) {
+                where.push('url_key=?');
+                args.push(key);
+            }
+            if (status) {
+                where.push('status=?');
+                args.push(status);
+            }
+            const rows = db.db.prepare(`SELECT id, url_key, query, action, type, rationale, priority, status, source, notes, baseline, outcome
+          FROM recon_todo ${where.length ? 'WHERE ' + where.join(' AND ') : ''} ORDER BY url_key, priority DESC`).all(...args);
+            if (!rows.length)
+                return { content: [{ type: 'text', text: `No recon to-dos${key ? ` for ${key}` : ''}${status ? ` with status ${status}` : ''}. Run recon_targets to generate some.` }], structuredContent: { todos: [] } };
+            const byPage = new Map();
+            for (const r of rows)
+                (byPage.get(r.url_key) ?? byPage.set(r.url_key, []).get(r.url_key)).push(r);
+            const verdictStmt = db.db.prepare(`SELECT verdict, verdict_note FROM recon_page WHERE url_key=?`);
+            const md = `# Content-recon to-dos — ${rows.length} item(s)${status ? `, status ${status}` : ''}\n\n` +
+                [...byPage.entries()].map(([url, items]) => {
+                    const v = verdictStmt.get(url);
+                    return `## ${url}${v ? `\n_${v.verdict}_ — ${v.verdict_note}` : ''}\n\n| # | Status | Type | Action |\n|---|---|---|---|\n` +
+                        items.map(i => `| ${i.id} | ${i.status} | ${i.type ?? ''} | ${String(i.action).replace(/\|/g, '\\|')} |`).join('\n') +
+                        (items.some(i => i.notes) ? '\n\nNotes:\n' + items.filter(i => i.notes).map(i => `- #${i.id}: ${String(i.notes).replace(/\n/g, ' / ')}`).join('\n') : '');
+                }).join('\n\n') +
+                browserLink(siteUrl);
+            return { content: [{ type: 'text', text: md }], structuredContent: { todos: rows } };
+        }
+        finally {
+            db.close();
+        }
+    });
     // keyword_list — demand-first clustering ("list mode"): a keyword list becomes topics
     // with own/weak/absent verdicts, clustered by the URL Google already answers them with.
     server.registerTool('keyword_list', {