@houtini/seo-audit-console 0.6.0 → 0.7.8

package/dist/core/googleNews.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Google News — the free, keyless source for news_discovery. Google's old News API was
+ * retired long ago; the live endpoint is the Google News RSS *search* feed, which returns
+ * recent articles for a query with title, source, link and publish time. No credentials,
+ * no cost — so news works even without a DataForSEO key (DataForSEO's Google News SERP
+ * stays the richer paid option; this is the always-available fallback / second source).
+ */
+export interface GoogleNewsArticle {
+    title: string;
+    url: string;
+    source: string | null;
+    timestamp: string | null;
+}
+/**
+ * Fetch recent Google News results for a keyword via the RSS search feed.
+ * `hl` = interface language, `gl` = country; `ceid` is derived (country:lang).
+ */
+export declare function fetchGoogleNews(keyword: string, opts?: {
+    hl?: string;
+    gl?: string;
+    limit?: number;
+}): Promise<{
+    articles: GoogleNewsArticle[];
+    source: 'google-news-rss';
+}>;
+//# sourceMappingURL=googleNews.d.ts.map

package/dist/core/googleNews.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"googleNews.d.ts","sourceRoot":"","sources":["../../src/core/googleNews.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAED;;;GAGG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,GAAE;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CAAO,GACtD,OAAO,CAAC;IAAE,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAAC,MAAM,EAAE,iBAAiB,CAAA;CAAE,CAAC,CAsCvE"}

package/dist/core/googleNews.js

@@ -0,0 +1,49 @@
+import * as cheerio from 'cheerio';
+/**
+ * Fetch recent Google News results for a keyword via the RSS search feed.
+ * `hl` = interface language, `gl` = country; `ceid` is derived (country:lang).
+ */
+export async function fetchGoogleNews(keyword, opts = {}) {
+    const hl = opts.hl ?? 'en-US';
+    const gl = opts.gl ?? 'US';
+    const ceid = `${gl}:${hl.split('-')[0]}`;
+    const url = `https://news.google.com/rss/search?q=${encodeURIComponent(keyword)}&hl=${encodeURIComponent(hl)}&gl=${encodeURIComponent(gl)}&ceid=${encodeURIComponent(ceid)}`;
+    const res = await fetch(url, {
+        headers: { 'user-agent': 'Mozilla/5.0 (compatible; seo-audit-console/1.0; +https://github.com/houtini-ai/seo-audit)' },
+        signal: AbortSignal.timeout(15000),
+    });
+    if (!res.ok)
+        throw new Error(`Google News RSS ${res.status}`);
+    const xml = await res.text();
+    // RSS is well-formed XML — parse with cheerio in xmlMode (already a dependency).
+    const $ = cheerio.load(xml, { xmlMode: true });
+    const limit = Math.min(opts.limit ?? 20, 100);
+    const articles = [];
+    $('item').each((_i, el) => {
+        if (articles.length >= limit)
+            return;
+        const $el = $(el);
+        const title = $el.find('title').first().text().trim();
+        if (!title)
+            return;
+        // <source url="...">Publisher</source>. Google News titles are usually "Headline - Publisher";
+        // prefer the explicit <source> element, fall back to the trailing " - Publisher".
+        let source = $el.find('source').first().text().trim() || null;
+        let headline = title;
+        if (!source) {
+            const m = title.match(/^(.*)\s+-\s+([^-]+)$/);
+            if (m) {
+                headline = m[1].trim();
+                source = m[2].trim();
+            }
+        }
+        articles.push({
+            title: headline,
+            url: $el.find('link').first().text().trim(),
+            source,
+            timestamp: $el.find('pubDate').first().text().trim() || null,
+        });
+    });
+    return { articles, source: 'google-news-rss' };
+}
+//# sourceMappingURL=googleNews.js.map

package/dist/core/googleNews.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"googleNews.js","sourceRoot":"","sources":["../../src/core/googleNews.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAgBnC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,OAAe,EACf,OAAqD,EAAE;IAEvD,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,OAAO,CAAC;IAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC;IAC3B,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzC,MAAM,GAAG,GAAG,wCAAwC,kBAAkB,CAAC,OAAO,CAAC,OAAO,kBAAkB,CAAC,EAAE,CAAC,OAAO,kBAAkB,CAAC,EAAE,CAAC,SAAS,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC;IAE7K,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC3B,OAAO,EAAE,EAAE,YAAY,EAAE,2FAA2F,EAAE;QACtH,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC;KACnC,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAE7B,iFAAiF;IACjF,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QACxB,IAAI,QAAQ,CAAC,MAAM,IAAI,KAAK;YAAE,OAAO;QACrC,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,CAAC,KAAK;YAAE,OAAO;QACnB,+FAA+F;QAC/F,kFAAkF;QAClF,IAAI,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC;QAC9D,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;YAC9C,IAAI,CAAC,EAAE,CAAC;gBAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAAC,CAAC;QAC1D,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC;YACZ,KAAK,EAAE,QAAQ;YACf,GAAG,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;YAC3C,MAAM;YACN,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,IAAI;SAC7D,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;AACjD,CAAC"}

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AA6EpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AAkLD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA8mE9E"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAiFpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AAkLD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAiqE9E"}

package/dist/server.js

@@ -22,8 +22,11 @@ import { selectReconTargets, deterministicTodos, persistReconPage, insertTodos,
 function browserLink(siteUrl) {
     const base = dashboardServerUrl();
     if (base)
-        return `\n\nBrowser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
-    return `\n\nTip: run serve_dashboard to open the full interactive dashboard in your browser.`;
+        return `\n\n📊 Browser dashboard: ${base}/dashboard${siteUrl ? `?siteUrl=${encodeURIComponent(siteUrl)}` : ''}`;
+    // No live server: point at the dashboard surfaces that always work. get_dashboard renders the
+    // interactive dashboard in chat (works through the Docker gateway where a served port would not);
+    // serve_dashboard opens a browser tab locally; export_report writes a shareable HTML file.
+    return `\n\n📊 See it in the dashboard — run get_dashboard${siteUrl ? ` for ${siteUrl}` : ''} (interactive, in chat), serve_dashboard (browser tab), or export_report (shareable HTML).`;
 }
 import { runAudit, runSingleCheck, listChecks } from './audit/engine.js';
 import { buildAuditMarkdown } from './audit/report.js';
@@ -50,6 +53,7 @@ import { RankTracker } from './core/RankTracker.js';
 import { Backlinks } from './core/Backlinks.js';
 import { LinkIntersect } from './core/LinkIntersect.js';
 import { MajesticClient } from './core/MajesticClient.js';
+import { fetchGoogleNews } from './core/googleNews.js';
 import { WikidataClient } from './core/WikidataClient.js';
 import { Entities } from './core/Entities.js';
 import { JobManager } from './core/JobManager.js';
@@ -253,17 +257,18 @@ export function createServer() {
     const crawler = new Crawler(dataDir()); // no GSC credentials required
     const dfsUser = process.env.DATAFORSEO_USERNAME;
     const dfsPass = process.env.DATAFORSEO_PASSWORD;
-    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 20;
+    const dfsCacheDays = Number(process.env.DATAFORSEO_CACHE_DAYS) || 7;
     const dfs = dfsUser && dfsPass
         ? new DataForSeoClient(dfsUser, dfsPass, path.join(dataDir(), 'dataforseo-cache.db'), dfsCacheDays)
         : null;
     const rankTracker = dfs ? new RankTracker(dfs, dataDir()) : null;
-    const backlinks = dfs ? new Backlinks(dfs, dataDir()) : null;
     const linkIntersect = dfs ? new LinkIntersect(dfs, dataDir()) : null;
-    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect enrichment tier.
+    // Majestic (Trust Flow / Topical Trust Flow) — optional link_intersect + trapped-authority tier.
     const majesticKey = process.env.MAJESTIC_API_KEY;
-    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 20;
+    const majesticCacheDays = Number(process.env.MAJESTIC_CACHE_DAYS) || 30;
     const majestic = majesticKey ? new MajesticClient(majesticKey, path.join(dataDir(), 'majestic-cache.db'), majesticCacheDays) : null;
+    // Backlinks after Majestic, so pull_backlinks can enrich per-URL pages with Trust Flow.
+    const backlinks = dfs ? new Backlinks(dfs, dataDir(), majestic) : null;
     // Firecrawl (competitor-page scraping for content recon) — optional; degrades gracefully.
     const firecrawlKey = process.env.FIRECRAWL_API_KEY;
     const firecrawl = firecrawlKey ? new FirecrawlClient(firecrawlKey, path.join(dataDir(), 'firecrawl-cache.db')) : null;
@@ -282,6 +287,10 @@ export function createServer() {
             throw new Error('DATAFORSEO_USERNAME / DATAFORSEO_PASSWORD not set — required for DataForSEO.');
         return v;
     };
+    // Which optional integrations have a key set — drives the dashboard's key-gated tabs
+    // (Links, Content research) and their affiliate-linked upsell states. The data layer has
+    // no env access, so it's injected here onto every dashboard payload surface.
+    const apiKeysStatus = () => ({ dataforseo: !!dfs, majestic: !!majestic, firecrawl: !!firecrawl, supadata: !!supadata });
     // ── Introspection (no data / creds required) ────────────────────────────
     server.registerTool('seo_audit_help', {
         title: 'Help — what this audit can do',
@@ -865,21 +874,61 @@ export function createServer() {
         };
     });
     server.registerTool('news_discovery', {
-        title: 'Google News discovery (DataForSEO SERP)',
-        description: '[Paid: SERP call PER KEYWORD, cached 20d | Use for: what has been PUBLISHED on a topic — freshness, "what changed since {date}", competitor coverage] Recent news articles ranking for a keyword (DataForSEO Google News SERP): title, source, snippet, publish timestamp and URL, in rank order (flattens both news results and top-stories). Feeds the "what\'s new / what changed" research step. SERP scope, 20-day cache. Default location: United States (2840).',
+        title: 'News discovery (Google News RSS + DataForSEO)',
+        description: '[Google News is FREE (no key); DataForSEO adds a paid, richer Google News SERP | Use for: what has been PUBLISHED on a topic — freshness, "what changed since {date}", competitor coverage] Recent news for a keyword from Google News (the free RSS search feed — Google\'s old News API is retired) and/or the DataForSEO Google News SERP. Returns title, source, publish time, URL. Default source "both" merges + dedupes; pass source:"google" for a free keyless lookup, "dataforseo" for the paid SERP only. Default location: United States.',
         inputSchema: {
             keyword: z.string(),
             location: z.union([z.string(), z.number()]).optional(),
             languageCode: z.string().optional(),
             depth: z.number().int().min(1).max(200).optional().describe('How many news results (default 20)'),
+            source: z.enum(['google', 'dataforseo', 'both']).optional().describe('google = free Google News RSS; dataforseo = paid Google News SERP; both (default) merges + dedupes'),
         },
-    }, async ({ keyword, location, languageCode, depth }) => {
-        const client = requireDfs(dfs);
-        const r = await client.serpNews(keyword, location, languageCode, depth ?? 20);
-        const top = r.articles.slice(0, 15).map(a => `• ${a.title ?? '(untitled)'}${a.source ? ` — ${a.source}` : ''}${a.timestamp ? `, ${a.timestamp}` : ''}\n  ${a.url ?? ''}`).join('\n');
+    }, async ({ keyword, location, languageCode, depth, source }) => {
+        const src = source ?? 'both';
+        const n = depth ?? 20;
+        const seen = new Set();
+        const articles = [];
+        const add = (a) => { const k = String(a.url || a.title || '').toLowerCase(); if (k && !seen.has(k)) {
+            seen.add(k);
+            articles.push(a);
+        } };
+        let cost = 0, cached = true, googleCount = 0, dfsCount = 0, googleError = null;
+        // Free Google News RSS.
+        if (src === 'google' || src === 'both') {
+            try {
+                const g = await fetchGoogleNews(keyword, { limit: n });
+                googleCount = g.articles.length;
+                for (const a of g.articles)
+                    add({ title: a.title, source: a.source, timestamp: a.timestamp, url: a.url, via: 'google-news' });
+            }
+            catch (e) {
+                googleError = e instanceof Error ? e.message : 'failed';
+            }
+        }
+        // Paid DataForSEO Google News SERP (only when explicitly asked, or 'both' AND a key is set).
+        if (src === 'dataforseo' || (src === 'both' && !!dfs)) {
+            const r = await requireDfs(dfs).serpNews(keyword, location, languageCode, n);
+            cost += r.cost;
+            cached = cached && r.cached;
+            dfsCount = r.articles.length;
+            for (const a of r.articles)
+                add({ ...a, via: 'dataforseo' });
+        }
+        // Top sources: which publishers are covering this topic, ranked by article count — the
+        // "who is talking about X" read. Powers "trending news + sources for X" in one call.
+        const sourceCount = new Map();
+        for (const a of articles) {
+            const s = String(a.source ?? '').trim();
+            if (s)
+                sourceCount.set(s, (sourceCount.get(s) ?? 0) + 1);
+        }
+        const topSources = [...sourceCount.entries()].sort((x, y) => y[1] - x[1]).slice(0, 12).map(([source, count]) => ({ source, count }));
+        const top = articles.slice(0, 15).map(a => `• ${a.title ?? '(untitled)'}${a.source ? ` — ${a.source}` : ''}${a.timestamp ? `, ${a.timestamp}` : ''}\n  ${a.url ?? ''}`).join('\n');
+        const srcNote = [googleCount ? `${googleCount} Google News` : '', dfsCount ? `${dfsCount} DataForSEO` : ''].filter(Boolean).join(' + ') || 'no results';
+        const sourcesLine = topSources.length ? `\n\nTop sources: ${topSources.map(s => `${s.source} (${s.count})`).join(', ')}` : '';
         return {
-            content: [{ type: 'text', text: `${r.articles.length} news results for "${keyword}"${r.cached ? ' (cached)' : ` (live, $${r.cost.toFixed(4)})`}${r.articles.length ? `:\n${top}` : ''}` }],
-            structuredContent: { keyword, articles: r.articles, cached: r.cached, cost: r.cost },
+            content: [{ type: 'text', text: `${articles.length} news results for "${keyword}" (${srcNote}${cost ? `, $${cost.toFixed(4)}` : ', free'})${googleError ? ` [Google News error: ${googleError}]` : ''}${sourcesLine}${articles.length ? `\n\n${top}` : ''}\n\nCompose with topic_trend for direction (rising/falling); schedule this call daily/hourly for a topic radar.` }],
+            structuredContent: { keyword, articles, topSources, cached, cost, sources: { googleNews: googleCount, dataforseo: dfsCount }, googleError },
         };
     });
     server.registerTool('topic_trend', {
@@ -1937,7 +1986,7 @@ export function createServer() {
         inputSchema: { siteUrl: z.string() },
         _meta: { ui: { visibility: ['app'] } },
     }, async ({ siteUrl }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         return { content: [{ type: 'text', text: 'ok' }], structuredContent: data };
     });
     // export_report — the dependable deliverable: a self-contained interactive dashboard
@@ -1948,7 +1997,7 @@ export function createServer() {
         description: 'Write a self-contained, interactive dashboard HTML for a property (all data + charts inlined) to the reports folder, and return the file path. Open it in any browser or send it to a client — no server, no MCP-App host support needed. Run refresh_property (+ run_audit for findings) first.',
         inputSchema: { siteUrl: z.string(), theme: z.enum(['light', 'dark']).optional() },
     }, async ({ siteUrl, theme }) => {
-        const data = getDashboardData(dataDir(), siteUrl);
+        const data = { ...getDashboardData(dataDir(), siteUrl), apiKeys: apiKeysStatus() };
         if (data.empty) {
             return { content: [{ type: 'text', text: `No synced data for ${siteUrl} — run refresh_property first.` }], structuredContent: { error: 'empty', siteUrl } };
         }
@@ -1992,12 +2041,46 @@ export function createServer() {
                     // otherwise CREATE an empty DB file for any bogus siteUrl posted at the API.
                     if (!listLocalProperties(dataDir()).some(p => p.siteUrl === want))
                         throw new Error(`unknown property ${want}`);
-                    return getDashboardData(dataDir(), want);
+                    return { ...getDashboardData(dataDir(), want), apiKeys: apiKeysStatus() };
                 },
                 related_terms: async (a) => {
                     const r = await requireDfs(dfs).relatedTerms(String(a.keyword ?? ''), a.location, a.languageCode);
                     return r;
                 },
+                // Content research (on-demand) — News (free Google News + paid DFS) / Videos / Trends.
+                news_discovery: async (a) => {
+                    const kw = String(a.keyword ?? '');
+                    const seen = new Set();
+                    const articles = [];
+                    const add = (x) => { const k = String(x.url || x.title || '').toLowerCase(); if (k && !seen.has(k)) {
+                        seen.add(k);
+                        articles.push(x);
+                    } };
+                    try {
+                        const g = await fetchGoogleNews(kw, { limit: 20 });
+                        for (const x of g.articles)
+                            add({ ...x, via: 'google-news' });
+                    }
+                    catch { /* free source optional */ }
+                    if (dfs) {
+                        try {
+                            const r = await dfs.serpNews(kw, a.location, a.languageCode);
+                            for (const x of r.articles)
+                                add({ ...x, via: 'dataforseo' });
+                        }
+                        catch { /* paid optional */ }
+                    }
+                    return { articles };
+                },
+                youtube_discovery: async (a) => {
+                    const r = await requireDfs(dfs).serpYoutube(String(a.keyword ?? ''), a.location, a.languageCode, a.blockDepth);
+                    return r;
+                },
+                topic_trend: async (a) => {
+                    const kws = Array.isArray(a.keywords) ? a.keywords : [String(a.keyword ?? '')].filter(Boolean);
+                    const r = await requireDfs(dfs).googleTrends(kws, a.location, a.languageCode, {});
+                    return r;
+                },
                 keyword_volume: async (a) => {
                     const r = await requireDfs(dfs).searchVolume(a.keywords ?? [], a.location, a.languageCode);
                     const items = (r.tasks[0]?.result ?? []).map((k) => ({