@houtini/seo-audit-console 0.6.0 → 0.8.0

package/dist/server.js

@@ -8,7 +8,8 @@ import path from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { z } from 'zod';
 import { getDashboardData } from './core/dashboardData.js';
-import { startDashboardServer, stopDashboardServer, dashboardServerUrl, listLocalProperties } from './core/webServer.js';
+import { startDashboardServer, stopDashboardServer, dashboardServerUrl } from './core/webServer.js';
+import { buildWebCallHandlers } from './core/webHandlers.js';
 import { computeSerpFootprint, persistSerpFootprint } from './core/serpFootprint.js';
 import { computeMarketSizing, persistMarketSizing } from './core/marketSizing.js';
 import { FirecrawlClient } from './core/FirecrawlClient.js';
@@ -22,8 +23,11 @@ import { selectReconTargets, deterministicTodos, persistReconPage, insertTodos,
 function browserLink(siteUrl) {
     const base = dashboardServerUrl();
     if (base)
-        return `\n\nBrowser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
-    return `\n\nTip: run serve_dashboard to open the full interactive dashboard in your browser.`;
+        return `\n\n📊 Browser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
+    // No live server: point at the dashboard surfaces that always work. get_dashboard renders the
+    // interactive dashboard in chat (works through the Docker gateway where a served port would not);
+    // serve_dashboard opens a browser tab locally; export_report writes a shareable HTML file.
+    return `\n\n📊 See it in the dashboard — run get_dashboard${siteUrl ? ` for ${siteUrl}` : ''} (interactive, in chat), serve_dashboard (browser tab), or export_report (shareable HTML).`;
 }
 import { runAudit, runSingleCheck, listChecks } from './audit/engine.js';
 import { buildAuditMarkdown } from './audit/report.js';
@@ -50,6 +54,7 @@ import { RankTracker } from './core/RankTracker.js';
 import { Backlinks } from './core/Backlinks.js';
 import { LinkIntersect } from './core/LinkIntersect.js';
 import { MajesticClient } from './core/MajesticClient.js';
+import { fetchGoogleNews } from './core/googleNews.js';
 import { WikidataClient } from './core/WikidataClient.js';
 import { Entities } from './core/Entities.js';
 import { JobManager } from './core/JobManager.js';
@@ -253,17 +258,18 @@ export function createServer() {
     const crawler = new Crawler(dataDir()); // no GSC credentials required
     const dfsUser = process.env.DATAFORSEO_USERNAME;
     const dfsPass = process.env.DATAFORSEO_PASSWORD;
-    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 20;
+    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 7;
     const dfs = dfsUser && dfsPass
         ? new DataForSeoClient(dfsUser, dfsPass, path.join(dataDir(), 'dataforseo-cache.db'), dfsCacheDays)
         : null;
     const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
-    const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
     const linkIntersect = dfs ? new LinkIntersect(dfs, dataDir()) : null;
-    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect enrichment tier.
+    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect + trapped-authority tier.
     const majesticKey = process.env.MAJESTIC_API_KEY;
-    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 20;
+    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 30;
     const majestic = majesticKey ? new MajesticClient(majesticKey, path.join(dataDir(), 'majestic-cache.db'), majesticCacheDays) : null;
+    // Backlinks after Majestic, so pull_backlinks can enrich per-URL pages with Trust Flow.
+    const backlinks = dfs ? new Backlinks(dfs, dataDir(), majestic) : null;
     // Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
     const firecrawlKey = process.env.FIRECRAWL_API_KEY;
     const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
@@ -282,6 +288,32 @@ export function createServer() {
             throw new Error('DATAFORSEO_USERNAME / DATAFORSEO_PASSWORD not set — required for DataForSEO.');
         return v;
     };
+    // Which optional integrations have a key set — drives the dashboard's key-gated tabs
+    // (Links, Content research) and their affiliate-linked upsell states. The data layer has
+    // no env access, so it's injected here onto every dashboard payload surface.
+    const apiKeysStatus = () => ({ dataforseo: !!dfs, majestic: !!majestic, firecrawl: !!firecrawl, supadata: !!supadata });
+    // The dashboard webserver's options — one definition shared by serve_dashboard AND the
+    // auto-start at the end of refresh_property / run_audit, so a populated property always has
+    // a live browser link (browserLink() surfaces dashboardServerUrl() once this is running).
+    const buildWebOpts = (port) => ({
+        dataDir,
+        uiHtml: () => readFileSync(path.join(__dirname, 'src', 'ui', 'dashboard.html'), 'utf8'),
+        call: buildWebCallHandlers({ dataDir, dfs, apiKeys: apiKeysStatus }),
+        ...(port != null ? { port } : {}),
+    });
+    // Idempotent auto-start used by refresh_property / run_audit. Best-effort: a served port that
+    // can't bind (e.g. Docker without a published port) must never fail the populate/audit — the
+    // in-chat get_dashboard and export_report still work, and browserLink() falls back to those.
+    // Opt-out with SAC_AUTOSERVE=0 for headless/CI/sandboxed runs where binding a localhost port
+    // is unwanted (default: on, since the whole point is to hand the user a link).
+    const autoServeDashboard = async () => {
+        if (/^(0|false|no|off)$/i.test(process.env.SAC_AUTOSERVE ?? ''))
+            return;
+        try {
+            await startDashboardServer(buildWebOpts());
+        }
+        catch { /* dashboard is a convenience, not a dependency */ }
+    };
     // ── Introspection (no data / creds required) ────────────────────────────
     server.registerTool('seo_audit_help', {
         title: 'Help — what this audit can do',
@@ -405,9 +437,10 @@ export function createServer() {
         },
     }, async ({ siteUrl, scope, categories, includeJudgement }) => {
         const result = runAudit(dataDir(), siteUrl, { scope, categories, includeJudgement });
+        await autoServeDashboard(); // spin up the browser dashboard so browserLink() hands over a live URL
         return {
             content: [{ type: 'text', text: buildAuditMarkdown(result, siteUrl) + browserLink(siteUrl) }],
-            structuredContent: result,
+            structuredContent: { ...result, dashboardUrl: dashboardServerUrl() },
         };
     });
     server.registerTool('query_audit', {
@@ -721,7 +754,14 @@ export function createServer() {
         },
         _meta: { ui: { resourceUri: SYNC_PROGRESS_URI } },
     }, async ({ siteUrl, gsc: doGsc, crawl, inspect, ranks, segments, full, location, startDate, endDate, maxPages, inspectLimit }) => {
-        const jobId = jobs.start('refresh', (update, signal) => refresh.run(siteUrl, { gsc: doGsc, crawl, inspect, ranks, segments, full, location, startDate, endDate, maxPages, inspectLimit }, update, signal));
+        const jobId = jobs.start('refresh', async (update, signal) => {
+            const r = await refresh.run(siteUrl, { gsc: doGsc, crawl, inspect, ranks, segments, full, location, startDate, endDate, maxPages, inspectLimit }, update, signal);
+            // Property is now populated — spin up the browser dashboard and hand back its URL so the
+            // finished job (polled via check_sync_status) carries a one-click link, no extra tool call.
+            await autoServeDashboard();
+            const dashboardUrl = dashboardServerUrl();
+            return dashboardUrl ? { ...r, dashboardUrl, dashboard: `${dashboardUrl}/dashboard?siteUrl=${encodeURIComponent(siteUrl)}` } : r;
+        });
         return {
             content: [{ type: 'text', text: `Refresh started for ${siteUrl} (job ${jobId}). Poll check_sync_status.` }],
             structuredContent: { jobId, status: 'running', siteUrl },
@@ -865,21 +905,61 @@ export function createServer() {
         };
     });
     server.registerTool('news_discovery', {
-        title: 'Google News discovery (DataForSEO SERP)',
-        description: '[Paid: SERP call PER KEYWORD, cached 20d | Use for: what has been PUBLISHED on a topic — freshness, "what changed since {date}", competitor coverage] Recent news articles ranking for a keyword (DataForSEO Google News SERP): title, source, snippet, publish timestamp and URL, in rank order (flattens both news results and top-stories). Feeds the "what\'s new / what changed" research step. SERP scope, 20-day cache. Default location: United States (2840).',
+        title: 'News discovery (Google News RSS + DataForSEO)',
+        description: '[Google News is FREE (no key); DataForSEO adds a paid, richer Google News SERP | Use for: what has been PUBLISHED on a topic — freshness, "what changed since {date}", competitor coverage] Recent news for a keyword from Google News (the free RSS search feed — Google\'s old News API is retired) and/or the DataForSEO Google News SERP. Returns title, source, publish time, URL. Default source "both" merges + dedupes; pass source:"google" for a free keyless lookup, "dataforseo" for the paid SERP only. Default location: United States.',
         inputSchema: {
             keyword: z.string(),
             location: z.union([z.string(), z.number()]).optional(),
             languageCode: z.string().optional(),
             depth: z.number().int().min(1).max(200).optional().describe('How many news results (default 20)'),
+            source: z.enum(['google', 'dataforseo', 'both']).optional().describe('google = free Google News RSS; dataforseo = paid Google News SERP; both (default) merges + dedupes'),
         },
-    }, async ({ keyword, location, languageCode, depth }) => {
-        const client = requireDfs(dfs);
-        const r = await client.serpNews(keyword, location, languageCode, depth ?? 20);
-        const top = r.articles.slice(0, 15).map(a => `• ${a.title ?? '(untitled)'}${a.source ? ` — ${a.source}` : ''}${a.timestamp ? `, ${a.timestamp}` : ''}\n  ${a.url ?? ''}`).join('\n');
+    }, async ({ keyword, location, languageCode, depth, source }) => {
+        const src = source ?? 'both';
+        const n = depth ?? 20;
+        const seen = new Set();
+        const articles = [];
+        const add = (a) => { const k = String(a.url || a.title || '').toLowerCase(); if (k && !seen.has(k)) {
+            seen.add(k);
+            articles.push(a);
+        } };
+        let cost = 0, cached = true, googleCount = 0, dfsCount = 0, googleError = null;
+        // Free Google News RSS.
+        if (src === 'google' || src === 'both') {
+            try {
+                const g = await fetchGoogleNews(keyword, { limit: n });
+                googleCount = g.articles.length;
+                for (const a of g.articles)
+                    add({ title: a.title, source: a.source, timestamp: a.timestamp, url: a.url, via: 'google-news' });
+            }
+            catch (e) {
+                googleError = e instanceof Error ? e.message : 'failed';
+            }
+        }
+        // Paid DataForSEO Google News SERP (only when explicitly asked, or 'both' AND a key is set).
+        if (src === 'dataforseo' || (src === 'both' && !!dfs)) {
+            const r = await requireDfs(dfs).serpNews(keyword, location, languageCode, n);
+            cost += r.cost;
+            cached = cached && r.cached;
+            dfsCount = r.articles.length;
+            for (const a of r.articles)
+                add({ ...a, via: 'dataforseo' });
+        }
+        // Top sources: which publishers are covering this topic, ranked by article count — the
+        // "who is talking about X" read. Powers "trending news + sources for X" in one call.
+        const sourceCount = new Map();
+        for (const a of articles) {
+            const s = String(a.source ?? '').trim();
+            if (s)
+                sourceCount.set(s, (sourceCount.get(s) ?? 0) + 1);
+        }
+        const topSources = [...sourceCount.entries()].sort((x, y) => y[1] - x[1]).slice(0, 12).map(([source, count]) => ({ source, count }));
+        const top = articles.slice(0, 15).map(a => `• ${a.title ?? '(untitled)'}${a.source ? ` — ${a.source}` : ''}${a.timestamp ? `, ${a.timestamp}` : ''}\n  ${a.url ?? ''}`).join('\n');
+        const srcNote = [googleCount ? `${googleCount} Google News` : '', dfsCount ? `${dfsCount} DataForSEO` : ''].filter(Boolean).join(' + ') || 'no results';
+        const sourcesLine = topSources.length ? `\n\nTop sources: ${topSources.map(s => `${s.source} (${s.count})`).join(', ')}` : '';
         return {
-            content: [{ type: 'text', text: `${r.articles.length} news results for "${keyword}"${r.cached ? ' (cached)' : ` (live, $${r.cost.toFixed(4)})`}${r.articles.length ? `:\n${top}` : ''}` }],
-            structuredContent: { keyword, articles: r.articles, cached: r.cached, cost: r.cost },
+            content: [{ type: 'text', text: `${articles.length} news results for "${keyword}" (${srcNote}${cost ? `, $${cost.toFixed(4)}` : ', free'})${googleError ? ` [Google News error: ${googleError}]` : ''}${sourcesLine}${articles.length ? `\n\n${top}` : ''}\n\nCompose with topic_trend for direction (rising/falling); schedule this call daily/hourly for a topic radar.` }],
+            structuredContent: { keyword, articles, topSources, cached, cost, sources: { googleNews: googleCount, dataforseo: dfsCount }, googleError },
         };
     });
     server.registerTool('topic_trend', {
@@ -1937,7 +2017,7 @@ export function createServer() {
         inputSchema: { siteUrl: z.string() },
         _meta: { ui: { visibility: ['app'] } },
     }, async ({ siteUrl }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         return { content: [{ type: 'text', text: 'ok' }], structuredContent: data };
     });
     // export_report — the dependable deliverable: a self-contained interactive dashboard
@@ -1948,7 +2028,7 @@ export function createServer() {
         description: 'Write a self-contained, interactive dashboard HTML for a property (all data + charts inlined) to the reports folder, and return the file path. Open it in any browser or send it to a client — no server, no MCP-App host support needed. Run refresh_property (+ run_audit for findings) first.',
         inputSchema: { siteUrl: z.string(), theme: z.enum(['light', 'dark']).optional() },
     }, async ({ siteUrl, theme }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         if (data.empty) {
             return { content: [{ type: 'text', text: `No synced data for ${siteUrl} — run refresh_property first.` }], structuredContent: { error: 'empty', siteUrl } };
         }
@@ -1982,32 +2062,7 @@ export function createServer() {
                 structuredContent: { stopped },
             };
         }
-        const { url } = await startDashboardServer({
-            dataDir,
-            uiHtml: () => readFileSync(path.join(__dirname, 'src', 'ui', 'dashboard.html'), 'utf8'),
-            call: {
-                get_dashboard_data: async (a) => {
-                    const want = String(a.siteUrl ?? '');
-                    // Only serve properties that actually exist locally — getDashboardData would
-                    // otherwise CREATE an empty DB file for any bogus siteUrl posted at the API.
-                    if (!listLocalProperties(dataDir()).some(p => p.siteUrl === want))
-                        throw new Error(`unknown property ${want}`);
-                    return getDashboardData(dataDir(), want);
-                },
-                related_terms: async (a) => {
-                    const r = await requireDfs(dfs).relatedTerms(String(a.keyword ?? ''), a.location, a.languageCode);
-                    return r;
-                },
-                keyword_volume: async (a) => {
-                    const r = await requireDfs(dfs).searchVolume(a.keywords ?? [], a.location, a.languageCode);
-                    const items = (r.tasks[0]?.result ?? []).map((k) => ({
-                        keyword: k.keyword, searchVolume: k.search_volume, cpc: k.cpc, competition: k.competition,
-                    }));
-                    return { keywords: items, cached: r.cached, cost: r.cost };
-                },
-            },
-            ...(port != null ? { port } : {}),
-        });
+        const { url } = await startDashboardServer(buildWebOpts(port));
         const open = siteUrl ? `${url}/dashboard?siteUrl=${encodeURIComponent(siteUrl)}` : `${url}/`;
         const portNote = port != null && !url.endsWith(`:${port}`)
             ? ` (already running on its original port — requested port ${port} ignored; stop=true first to move it)`