@houtini/seo-audit-console 0.5.1 → 0.7.8

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AA6EpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AAiLD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA6hE9E"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAiFpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AAkLD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAiqE9E"}

package/dist/server.js

@@ -22,8 +22,11 @@ import { selectReconTargets, deterministicTodos, persistReconPage, insertTodos,
 function browserLink(siteUrl) {
     const base = dashboardServerUrl();
     if (base)
-        return `\n\nBrowser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
-    return `\n\nTip: run serve_dashboard to open the full interactive dashboard in your browser.`;
+        return `\n\n📊 Browser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
+    // No live server: point at the dashboard surfaces that always work. get_dashboard renders the
+    // interactive dashboard in chat (works through the Docker gateway where a served port would not);
+    // serve_dashboard opens a browser tab locally; export_report writes a shareable HTML file.
+    return `\n\n📊 See it in the dashboard — run get_dashboard${siteUrl ? ` for ${siteUrl}` : ''} (interactive, in chat), serve_dashboard (browser tab), or export_report (shareable HTML).`;
 }
 import { runAudit, runSingleCheck, listChecks } from './audit/engine.js';
 import { buildAuditMarkdown } from './audit/report.js';
@@ -50,6 +53,7 @@ import { RankTracker } from './core/RankTracker.js';
 import { Backlinks } from './core/Backlinks.js';
 import { LinkIntersect } from './core/LinkIntersect.js';
 import { MajesticClient } from './core/MajesticClient.js';
+import { fetchGoogleNews } from './core/googleNews.js';
 import { WikidataClient } from './core/WikidataClient.js';
 import { Entities } from './core/Entities.js';
 import { JobManager } from './core/JobManager.js';
@@ -114,8 +118,9 @@ ARCHETYPE CHAINS (compose along these lines):
 
 COST DISCIPLINE (behave like a strategist who knows the margins):
 - Free and instant, use liberally: everything on synced data — query_data, run_audit, query_audit, suggest_pages, list_templates, detect_changes, get_dashboard, serve_dashboard, export_report.
-- Paid but CHEAP and 20-day cached (Labs/Keywords, ~$0.01–0.13 a call): keyword_volume, search_intent, ranked_keywords, serp_features, domain_visibility, top_pages, competitors_domain, page_intersection, topic_gaps. Top-down pulls only — ONE ranked_keywords call answers "what does this domain rank for"; NEVER loop keywords through SERP endpoints to reconstruct what a Labs call returns.
-- Paid per-keyword (SERP): related_terms and AI-Overview CITATION checks. On-demand for a handful of clicked/explicit keywords, never a list.
+- Paid but CHEAP and 20-day cached (Labs/Keywords, ~$0.01–0.13 a call): keyword_volume, topic_trend, search_intent, ranked_keywords, serp_features, domain_visibility, top_pages, competitors_domain, page_intersection, topic_gaps. Top-down pulls only — ONE ranked_keywords call answers "what does this domain rank for"; NEVER loop keywords through SERP endpoints to reconstruct what a Labs call returns.
+- Paid per-keyword (SERP): related_terms, youtube_discovery (ranking videos for a topic — pair with a transcript tool), news_discovery (recent coverage / freshness), and AI-Overview CITATION checks. On-demand for a handful of clicked/explicit keywords, never a list.
+- Content research (what to write / what changed): topic_trend (is a topic rising/seasonal) → youtube_discovery + news_discovery (what the winning videos/articles cover) → draft_content.
 - Separate subscription: pull_backlinks (DataForSEO Backlinks — a 40204 error means it isn't activated).
 
 AGENCY MACRO-WORKFLOWS (the engagement arc — each stage feeds the next):
@@ -252,17 +257,18 @@ export function createServer() {
     const crawler = new Crawler(dataDir()); // no GSC credentials required
     const dfsUser = process.env.DATAFORSEO_USERNAME;
     const dfsPass = process.env.DATAFORSEO_PASSWORD;
-    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 20;
+    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 7;
     const dfs = dfsUser && dfsPass
         ? new DataForSeoClient(dfsUser, dfsPass, path.join(dataDir(), 'dataforseo-cache.db'), dfsCacheDays)
         : null;
     const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
-    const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
     const linkIntersect = dfs ? new LinkIntersect(dfs, dataDir()) : null;
-    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect enrichment tier.
+    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect + trapped-authority tier.
     const majesticKey = process.env.MAJESTIC_API_KEY;
-    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 20;
+    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 30;
     const majestic = majesticKey ? new MajesticClient(majesticKey, path.join(dataDir(), 'majestic-cache.db'), majesticCacheDays) : null;
+    // Backlinks after Majestic, so pull_backlinks can enrich per-URL pages with Trust Flow.
+    const backlinks = dfs ? new Backlinks(dfs, dataDir(), majestic) : null;
     // Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
     const firecrawlKey = process.env.FIRECRAWL_API_KEY;
     const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
@@ -281,6 +287,10 @@ export function createServer() {
             throw new Error('DATAFORSEO_USERNAME / DATAFORSEO_PASSWORD not set — required for DataForSEO.');
         return v;
     };
+    // Which optional integrations have a key set — drives the dashboard's key-gated tabs
+    // (Links, Content research) and their affiliate-linked upsell states. The data layer has
+    // no env access, so it's injected here onto every dashboard payload surface.
+    const apiKeysStatus = () => ({ dataforseo: !!dfs, majestic: !!majestic, firecrawl: !!firecrawl, supadata: !!supadata });
     // ── Introspection (no data / creds required) ────────────────────────────
     server.registerTool('seo_audit_help', {
         title: 'Help — what this audit can do',
@@ -845,6 +855,111 @@ export function createServer() {
             structuredContent: r,
         };
     });
+    server.registerTool('youtube_discovery', {
+        title: 'YouTube video discovery (DataForSEO SERP)',
+        description: '[Paid: SERP call PER KEYWORD, cached 20d | Use for: finding the videos that rank for a topic — content research, competitor video scan, freshness] The ranking YouTube videos for a keyword (DataForSEO YouTube organic SERP). Returns title, URL, channel, view count, publish date, duration and shorts/live flags, in rank order. Pair with a transcript tool (Supadata) to read what the winning videos actually say. SERP scope, 20-day cache. Default location: United States (2840).',
+        inputSchema: {
+            keyword: z.string(),
+            location: z.union([z.string(), z.number()]).optional(),
+            languageCode: z.string().optional(),
+            blockDepth: z.number().int().min(1).max(200).optional().describe('How many ranking videos to collect (default 20)'),
+        },
+    }, async ({ keyword, location, languageCode, blockDepth }) => {
+        const client = requireDfs(dfs);
+        const r = await client.serpYoutube(keyword, location, languageCode, blockDepth ?? 20);
+        const top = r.videos.slice(0, 15).map(v => `• ${v.title ?? '(untitled)'}${v.channel ? ` — ${v.channel}` : ''}${v.views != null ? `, ${v.views.toLocaleString()} views` : ''}${v.published ? `, ${v.published}` : ''}\n  ${v.url ?? ''}`).join('\n');
+        return {
+            content: [{ type: 'text', text: `${r.videos.length} YouTube videos for "${keyword}"${r.cached ? ' (cached)' : ` (live, $${r.cost.toFixed(4)})`}${r.videos.length ? `:\n${top}` : ''}` }],
+            structuredContent: { keyword, videos: r.videos, cached: r.cached, cost: r.cost },
+        };
+    });
+    server.registerTool('news_discovery', {
+        title: 'News discovery (Google News RSS + DataForSEO)',
+        description: '[Google News is FREE (no key); DataForSEO adds a paid, richer Google News SERP | Use for: what has been PUBLISHED on a topic — freshness, "what changed since {date}", competitor coverage] Recent news for a keyword from Google News (the free RSS search feed — Google\'s old News API is retired) and/or the DataForSEO Google News SERP. Returns title, source, publish time, URL. Default source "both" merges + dedupes; pass source:"google" for a free keyless lookup, "dataforseo" for the paid SERP only. Default location: United States.',
+        inputSchema: {
+            keyword: z.string(),
+            location: z.union([z.string(), z.number()]).optional(),
+            languageCode: z.string().optional(),
+            depth: z.number().int().min(1).max(200).optional().describe('How many news results (default 20)'),
+            source: z.enum(['google', 'dataforseo', 'both']).optional().describe('google = free Google News RSS; dataforseo = paid Google News SERP; both (default) merges + dedupes'),
+        },
+    }, async ({ keyword, location, languageCode, depth, source }) => {
+        const src = source ?? 'both';
+        const n = depth ?? 20;
+        const seen = new Set();
+        const articles = [];
+        const add = (a) => { const k = String(a.url || a.title || '').toLowerCase(); if (k && !seen.has(k)) {
+            seen.add(k);
+            articles.push(a);
+        } };
+        let cost = 0, cached = true, googleCount = 0, dfsCount = 0, googleError = null;
+        // Free Google News RSS.
+        if (src === 'google' || src === 'both') {
+            try {
+                const g = await fetchGoogleNews(keyword, { limit: n });
+                googleCount = g.articles.length;
+                for (const a of g.articles)
+                    add({ title: a.title, source: a.source, timestamp: a.timestamp, url: a.url, via: 'google-news' });
+            }
+            catch (e) {
+                googleError = e instanceof Error ? e.message : 'failed';
+            }
+        }
+        // Paid DataForSEO Google News SERP (only when explicitly asked, or 'both' AND a key is set).
+        if (src === 'dataforseo' || (src === 'both' && !!dfs)) {
+            const r = await requireDfs(dfs).serpNews(keyword, location, languageCode, n);
+            cost += r.cost;
+            cached = cached && r.cached;
+            dfsCount = r.articles.length;
+            for (const a of r.articles)
+                add({ ...a, via: 'dataforseo' });
+        }
+        // Top sources: which publishers are covering this topic, ranked by article count — the
+        // "who is talking about X" read. Powers "trending news + sources for X" in one call.
+        const sourceCount = new Map();
+        for (const a of articles) {
+            const s = String(a.source ?? '').trim();
+            if (s)
+                sourceCount.set(s, (sourceCount.get(s) ?? 0) + 1);
+        }
+        const topSources = [...sourceCount.entries()].sort((x, y) => y[1] - x[1]).slice(0, 12).map(([source, count]) => ({ source, count }));
+        const top = articles.slice(0, 15).map(a => `• ${a.title ?? '(untitled)'}${a.source ? ` — ${a.source}` : ''}${a.timestamp ? `, ${a.timestamp}` : ''}\n  ${a.url ?? ''}`).join('\n');
+        const srcNote = [googleCount ? `${googleCount} Google News` : '', dfsCount ? `${dfsCount} DataForSEO` : ''].filter(Boolean).join(' + ') || 'no results';
+        const sourcesLine = topSources.length ? `\n\nTop sources: ${topSources.map(s => `${s.source} (${s.count})`).join(', ')}` : '';
+        return {
+            content: [{ type: 'text', text: `${articles.length} news results for "${keyword}" (${srcNote}${cost ? `, $${cost.toFixed(4)}` : ', free'})${googleError ? ` [Google News error: ${googleError}]` : ''}${sourcesLine}${articles.length ? `\n\n${top}` : ''}\n\nCompose with topic_trend for direction (rising/falling); schedule this call daily/hourly for a topic radar.` }],
+            structuredContent: { keyword, articles, topSources, cached, cost, sources: { googleNews: googleCount, dataforseo: dfsCount }, googleError },
+        };
+    });
+    server.registerTool('topic_trend', {
+        title: 'Google Trends interest over time (DataForSEO)',
+        description: '[Paid: Keywords API, cheap, cached 20d | Use for: seasonality + "is this rising or fading" — should we update now, when to publish] Google Trends relative interest (0–100) over time for up to 5 keywords (DataForSEO KEYWORDS_DATA / google_trends). Returns a per-keyword time series plus a rising/falling/flat read, so you can see direction and seasonality. timeRange: past_7_days | past_30_days | past_90_days | past_12_months (default) | past_5_years | 2004_present. type: web (default) | news | youtube | images. Default location: United States.',
+        inputSchema: {
+            keywords: z.array(z.string()).min(1).max(5),
+            location: z.union([z.string(), z.number()]).optional(),
+            languageCode: z.string().optional(),
+            timeRange: z.enum(['past_7_days', 'past_30_days', 'past_90_days', 'past_12_months', 'past_5_years', '2004_present']).optional(),
+            type: z.enum(['web', 'news', 'youtube', 'images', 'froogle']).optional(),
+        },
+    }, async ({ keywords, location, languageCode, timeRange, type }) => {
+        const client = requireDfs(dfs);
+        const r = await client.googleTrends(keywords, location, languageCode, { ...(timeRange ? { timeRange } : {}), ...(type ? { type } : {}) });
+        const summarise = (k) => {
+            const vals = r.series.map(s => s.values[k]).filter((v) => typeof v === 'number');
+            if (!vals.length)
+                return `${k}: no data`;
+            const win = Math.max(1, Math.floor(vals.length / 4));
+            const avg = (xs) => xs.reduce((a, b) => a + b, 0) / xs.length;
+            const first = avg(vals.slice(0, win)), last = avg(vals.slice(-win));
+            const dir = last > first * 1.1 ? 'rising' : last < first * 0.9 ? 'falling' : 'flat';
+            return `${k}: ${dir} (${Math.round(first)}→${Math.round(last)}, peak ${Math.max(...vals)})`;
+        };
+        const txt = r.keywords.map(summarise).join('\n');
+        return {
+            content: [{ type: 'text', text: `Trend, ${r.series.length} points${r.cached ? ' (cached)' : ` (live, $${r.cost.toFixed(4)})`}:\n${txt}` }],
+            structuredContent: { keywords: r.keywords, series: r.series, cached: r.cached, cost: r.cost },
+        };
+    });
     server.registerTool('suggest_pages', {
         title: 'Suggest new pages from real demand (gaps)',
         description: 'Propose NEW pages grounded in real Search Console demand: queries you already get impressions for but rank 11+ (no winning page), after subtracting demand you already satisfy (you rank ≤10, or a page already covers the query, or one URL owns it). Survivors are clustered into one proposed page per intent and scored by impressions × intent. Returns evidence + the nearest existing page to link the new page from. Computable from synced GSC + crawl — no paid calls (run search_intent siteUrl:<property> first to weight by intent).',
@@ -1871,7 +1986,7 @@ export function createServer() {
         inputSchema: { siteUrl: z.string() },
         _meta: { ui: { visibility: ['app'] } },
     }, async ({ siteUrl }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         return { content: [{ type: 'text', text: 'ok' }], structuredContent: data };
     });
     // export_report — the dependable deliverable: a self-contained interactive dashboard
@@ -1882,7 +1997,7 @@ export function createServer() {
         description: 'Write a self-contained, interactive dashboard HTML for a property (all data + charts inlined) to the reports folder, and return the file path. Open it in any browser or send it to a client — no server, no MCP-App host support needed. Run refresh_property (+ run_audit for findings) first.',
         inputSchema: { siteUrl: z.string(), theme: z.enum(['light', 'dark']).optional() },
     }, async ({ siteUrl, theme }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         if (data.empty) {
             return { content: [{ type: 'text', text: `No synced data for ${siteUrl} — run refresh_property first.` }], structuredContent: { error: 'empty', siteUrl } };
         }
@@ -1926,12 +2041,46 @@ export function createServer() {
                     // otherwise CREATE an empty DB file for any bogus siteUrl posted at the API.
                     if (!listLocalProperties(dataDir()).some(p => p.siteUrl === want))
                         throw new Error(`unknown property ${want}`);
-                    return getDashboardData(dataDir(), want);
+                    return { ...getDashboardData(dataDir(), want), apiKeys: apiKeysStatus() };
                 },
                 related_terms: async (a) => {
                     const r = await requireDfs(dfs).relatedTerms(String(a.keyword ?? ''), a.location, a.languageCode);
                     return r;
                 },
+                // Content research (on-demand) — News (free Google News + paid DFS) / Videos / Trends.
+                news_discovery: async (a) => {
+                    const kw = String(a.keyword ?? '');
+                    const seen = new Set();
+                    const articles = [];
+                    const add = (x) => { const k = String(x.url || x.title || '').toLowerCase(); if (k && !seen.has(k)) {
+                        seen.add(k);
+                        articles.push(x);
+                    } };
+                    try {
+                        const g = await fetchGoogleNews(kw, { limit: 20 });
+                        for (const x of g.articles)
+                            add({ ...x, via: 'google-news' });
+                    }
+                    catch { /* free source optional */ }
+                    if (dfs) {
+                        try {
+                            const r = await dfs.serpNews(kw, a.location, a.languageCode);
+                            for (const x of r.articles)
+                                add({ ...x, via: 'dataforseo' });
+                        }
+                        catch { /* paid optional */ }
+                    }
+                    return { articles };
+                },
+                youtube_discovery: async (a) => {
+                    const r = await requireDfs(dfs).serpYoutube(String(a.keyword ?? ''), a.location, a.languageCode, a.blockDepth);
+                    return r;
+                },
+                topic_trend: async (a) => {
+                    const kws = Array.isArray(a.keywords) ? a.keywords : [String(a.keyword ?? '')].filter(Boolean);
+                    const r = await requireDfs(dfs).googleTrends(kws, a.location, a.languageCode, {});
+                    return r;
+                },
                 keyword_volume: async (a) => {
                     const r = await requireDfs(dfs).searchVolume(a.keywords ?? [], a.location, a.languageCode);
                     const items = (r.tasks[0]?.result ?? []).map((k) => ({