@houtini/seo-audit-console 0.3.0 → 0.4.6

package/dist/core/reconResearch.js

@@ -0,0 +1,122 @@
+import { Agent, fetch as ufetch } from 'undici';
+import { hostOf } from './serpRecon.js';
+import { extractPage } from './extract.js';
+const httpAgent = new Agent({ allowH2: true, connections: 6, keepAliveTimeout: 30_000 });
+const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
+const GOOGLEBOT_UA = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/131.0.0.0 Safari/537.36';
+function browserHeaders(mode) {
+    if (mode === 'googlebot') {
+        return { 'user-agent': GOOGLEBOT_UA, 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-encoding': 'gzip, deflate, br', 'from': 'googlebot(at)googlebot.com' };
+    }
+    return {
+        'user-agent': CHROME_UA,
+        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        'accept-language': 'en-GB,en-US;q=0.9,en;q=0.8',
+        'accept-encoding': 'gzip, deflate, br',
+        'referer': 'https://www.google.com/',
+        'sec-ch-ua': '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+        'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"',
+        'sec-fetch-dest': 'document', 'sec-fetch-mode': 'navigate', 'sec-fetch-site': 'cross-site', 'sec-fetch-user': '?1',
+        'upgrade-insecure-requests': '1',
+    };
+}
+/** Hosts whose content only Supadata can read (transcripts) and hosts we read via Reddit's own JSON. */
+const VIDEO_HOSTS = ['youtube.com', 'youtu.be', 'tiktok.com', 'twitter.com', 'x.com', 'vimeo.com', 'dailymotion.com', 'instagram.com', 'facebook.com'];
+const REDDIT_HOSTS = ['reddit.com', 'redd.it'];
+/** Exact host or subdomain match — NEVER a substring test. A substring `/x\.com/` classified
+ * pimax.com as video and sent every Shopify product page to the transcriber, which 400'd
+ * (build lesson, 2026-08-07: that one regex is why scrapeCompetitors returned almost nothing). */
+export function hostIsOneOf(host, domains) {
+    const h = hostOf(host);
+    return domains.some(d => h === d || h.endsWith(`.${d}`));
+}
+const isVideo = (host) => hostIsOneOf(host, VIDEO_HOSTS);
+const isReddit = (host) => hostIsOneOf(host, REDDIT_HOSTS);
+/** Which fetcher a URL will be routed to — exposed so recon can report routing per URL
+ * instead of leaving a silent misroute to look like an empty competitor set. */
+export function routeFor(url) {
+    const host = hostOf(url);
+    return isVideo(host) ? 'video' : isReddit(host) ? 'reddit' : 'page';
+}
+/** Reddit serves clean JSON at url.json — title + selftext + top comments, no scraper needed.
+ * Needs the browser header profile (UA + Google referer) — a bot UA gets a 403. */
+export async function fetchRedditJson(url, maxChars = 4000, ua = 'browser') {
+    const jsonUrl = url.split('?')[0].replace(/\/$/, '') + '.json';
+    const res = await ufetch(jsonUrl, { headers: browserHeaders(ua), dispatcher: httpAgent, signal: AbortSignal.timeout(20000) });
+    if (!res.ok)
+        throw new Error(`Reddit .json ${res.status}`);
+    const data = await res.json();
+    const post = data?.[0]?.data?.children?.[0]?.data ?? {};
+    const comments = data?.[1]?.data?.children ?? [];
+    const parts = [];
+    if (post.title)
+        parts.push(`# ${post.title}`);
+    if (post.selftext)
+        parts.push(post.selftext);
+    const top = comments
+        .map(c => c?.data)
+        .filter(d => d && d.body && d.body !== '[deleted]')
+        .sort((a, b) => (b.score ?? 0) - (a.score ?? 0))
+        .slice(0, 12)
+        .map(d => `- (${d.score ?? 0}) ${d.body}`);
+    if (top.length)
+        parts.push('## Top comments', ...top);
+    return { title: post.title ?? null, content: parts.join('\n\n').slice(0, maxChars) };
+}
+/** Free HTTP fallback for pages Firecrawl refuses (its blocklist covers major publishers).
+ * Fetches with a browser-like UA and reuses our own extractor for clean main-content text. */
+export async function fetchPlainPage(url, maxChars = 4000, ua = 'browser') {
+    const res = await ufetch(url, { headers: browserHeaders(ua), dispatcher: httpAgent, signal: AbortSignal.timeout(20000) });
+    if (!res.ok)
+        throw new Error(`plain fetch ${res.status}`);
+    const html = await res.text();
+    const host = new URL(url).hostname.replace(/^www\./, '');
+    const ex = extractPage(html, url, host, { hostForm: 'asis' }, null);
+    const text = (ex.bodyChunks ?? []).map(c => `${c.heading} ${c.text}`).join('\n').trim();
+    return { title: ex.title, content: text.slice(0, maxChars) };
+}
+export async function fetchCompetitorContent(url, clients, opts = {}) {
+    const max = opts.maxChars ?? 4000;
+    const ua = opts.ua ?? 'browser';
+    const kind = routeFor(url);
+    let firecrawlError = null;
+    try {
+        if (kind === 'video') {
+            if (!clients.supadata)
+                return { url, kind, via: 'none', title: null, content: '', cached: false, error: 'SUPADATA_API_KEY not set — cannot transcribe video' };
+            const t = await clients.supadata.transcript(url, { maxChars: Math.max(max, 5000) });
+            return { url, kind, via: 'supadata', title: null, content: t.content, cached: t.cached };
+        }
+        if (kind === 'reddit') {
+            // .json first (cleaner structured data); HTML fallback. Both need the browser profile.
+            try {
+                const r = await fetchRedditJson(url, max, ua);
+                return { url, kind, via: 'reddit-json', title: r.title, content: r.content, cached: false };
+            }
+            catch {
+                const r = await fetchPlainPage(url, max, ua);
+                return { url, kind, via: 'plain-fetch', title: r.title, content: r.content, cached: false };
+            }
+        }
+        // Pages: Firecrawl first (cleaner, handles JS); fall back to the free browser-profile fetch
+        // for sites Firecrawl refuses ("we do not support this site") or when no key is set.
+        if (clients.firecrawl) {
+            try {
+                const s = await clients.firecrawl.scrape(url, { maxChars: max, proxy: 'auto' });
+                return { url, kind, via: 'firecrawl', title: s.title, content: s.markdown, cached: s.cached };
+            }
+            catch (e) {
+                firecrawlError = e instanceof Error ? e.message : String(e); /* fall through to the browser-profile fetch */
+            }
+        }
+        const p = await fetchPlainPage(url, max, ua);
+        return { url, kind, via: 'plain-fetch', title: p.title, content: p.content, cached: false };
+    }
+    catch (e) {
+        // Report BOTH failures when Firecrawl was tried first — "plain fetch 403" alone hides the
+        // fact that the paid fetcher refused the site for a different reason.
+        const msg = e instanceof Error ? e.message : String(e);
+        return { url, kind, via: 'none', title: null, content: '', cached: false, error: firecrawlError ? `${msg} (firecrawl first: ${firecrawlError})` : msg };
+    }
+}
+//# sourceMappingURL=reconResearch.js.map

package/dist/core/reconResearch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reconResearch.js","sourceRoot":"","sources":["../../src/core/reconResearch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,KAAK,IAAI,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhD,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,MAAM,SAAS,GAAG,IAAI,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,gBAAgB,EAAE,MAAM,EAAE,CAAC,CAAC;AACzF,MAAM,SAAS,GAAG,iHAAiH,CAAC;AACpI,MAAM,YAAY,GAAG,+IAA+I,CAAC;AAQrK,SAAS,cAAc,CAAC,IAAY;IAClC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;QACzB,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,iEAAiE,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IACnM,CAAC;IACD,OAAO;QACL,YAAY,EAAE,SAAS;QACvB,QAAQ,EAAE,kGAAkG;QAC5G,iBAAiB,EAAE,4BAA4B;QAC/C,iBAAiB,EAAE,mBAAmB;QACtC,SAAS,EAAE,yBAAyB;QACpC,WAAW,EAAE,mEAAmE;QAChF,kBAAkB,EAAE,IAAI,EAAE,oBAAoB,EAAE,WAAW;QAC3D,gBAAgB,EAAE,UAAU,EAAE,gBAAgB,EAAE,UAAU,EAAE,gBAAgB,EAAE,YAAY,EAAE,gBAAgB,EAAE,IAAI;QAClH,2BAA2B,EAAE,GAAG;KACjC,CAAC;AACJ,CAAC;AAmBD,wGAAwG;AACxG,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,iBAAiB,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC;AACvJ,MAAM,YAAY,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;AAE/C;;kGAEkG;AAClG,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,OAAiB;IACzD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;IACvB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;AAC1E,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAW,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;AAE5E;gFACgF;AAChF,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACzB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AACtE,CAAC;AAED;mFACmF;AACnF,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,GAAW,EAAE,QAAQ,GAAG,IAAI,EAAE,KAAa,SAAS;IACxF,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC;IAC/D,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9H,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAQ,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC;IACxD,MAAM,QAAQ,GAAU,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,IAAI,EAAE,CAAC;IACxD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,KAAK;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IAC9C,IAAI,IAAI,CAAC,QAAQ;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC7C,MAAM,GAAG,GAAG,QAAQ;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC;SACjB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC;SAClD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;SAC/C,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,IAAI,GAAG,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,EAAE,GAAG,GAAG,CAAC,CAAC;IACtD,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;AACvF,CAAC;AAED;8FAC8F;AAC9F,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,QAAQ,GAAG,IAAI,EAAE,KAAa,SAAS;IACvF,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,cAAc,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC1H,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,CAAC;AAC/D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,GAAW,EACX,OAA+E,EAC/E,OAA2C,EAAE;IAE7C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IAClC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,IAAI,SAAS,CAAC;IAChC,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC3B,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,CAAC;QACH,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,oDAAoD,EAAE,CAAC;YAC/J,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QAC3F,CAAC;QACD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACtB,uFAAuF;YACvF,IAAI,CAAC;gBAAC,MAAM,CAAC,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAAC,CAAC;YACnJ,MAAM,CAAC;gBAAC,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YAAC,CAAC;QACtJ,CAAC;QACD,4FAA4F;QAC5F,qFAAqF;QACrF,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;gBAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YAAC,CAAC;YACvL,OAAO,CAAC,EAAE,CAAC;gBAAC,cAAc,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,+CAA+C;YAAC,CAAC;QAC5H,CAAC;QACD,MAAM,CAAC,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7C,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;IAC9F,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,0FAA0F;QAC1F,sEAAsE;QACtE,MAAM,GAAG,GAAG,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACvD,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,GAAG,sBAAsB,cAAc,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC1J,CAAC;AACH,CAAC"}

package/dist/core/serpRecon.d.ts

@@ -0,0 +1,52 @@
+import type { DfsResponse } from './DataForSeoClient.js';
+/**
+ * Parse a DataForSEO SERP-advanced response into the fields content recon needs:
+ * where WE actually rank in organic, whether the AI Overview cites us and who it does
+ * cite, and how much of the SERP is video/product (format signals). This is the single
+ * fetch that powers the organic-rank x AIO-citation verdict matrix.
+ */
+export interface SerpReconResult {
+    itemTypes: string[];
+    ourOrganicRank: number | null;
+    serpDepth: number;
+    aioPresent: boolean;
+    aioCitesUs: boolean | null;
+    aioAsync: boolean;
+    aioResolved: boolean;
+    aioReferences: {
+        rank: number | null;
+        domain: string;
+        url: string;
+        title: string;
+        nested: boolean;
+    }[];
+    videoPresent: boolean;
+    organicAbove: {
+        rank: number;
+        domain: string;
+        url: string;
+        title: string;
+    }[];
+    videoItems: {
+        url: string;
+        title: string;
+        source: string;
+    }[];
+}
+/** Registrable-ish host: lowercase, strip scheme/path/www. Good enough to match a domain field. */
+export declare function hostOf(s: string | undefined | null): string;
+export declare function parseSerpForRecon(resp: DfsResponse, ownDomain: string, serpDepth?: number): SerpReconResult;
+/**
+ * The deterministic verdict from organic rank x AIO citation (Richard's rule):
+ * strong organic but NOT in the AIO = a data-accuracy/freshness problem — Google ranks
+ * us but won't quote us.
+ */
+export type ReconVerdict = 'defend-and-deepen' | 'accuracy-or-freshness' | 'consolidate-weak-page' | 'competitive-gap' | 'page-cannot-rank';
+/** How to say "we didn't find you" without implying a position we never measured. Rank 11 and
+ * rank 80 both come back as null, so the honest phrasing has to carry the depth we fetched. */
+export declare const rankPhrase: (r: SerpReconResult) => string;
+export declare function reconVerdict(r: SerpReconResult): {
+    verdict: ReconVerdict;
+    note: string;
+};
+//# sourceMappingURL=serpRecon.d.ts.map

package/dist/core/serpRecon.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serpRecon.d.ts","sourceRoot":"","sources":["../../src/core/serpRecon.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEzD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,CAAC;IAClB,WAAW,EAAE,OAAO,CAAC;IACrB,aAAa,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,CAAA;KAAE,EAAE,CAAC;IACtG,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC7E,UAAU,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC9D;AAED,mGAAmG;AACnG,wBAAgB,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,MAAM,CAG3D;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,SAAK,GAAG,eAAe,CA2DvG;AAED;;;;GAIG;AACH,MAAM,MAAM,YAAY,GAAG,mBAAmB,GAAG,uBAAuB,GAAG,uBAAuB,GAAG,iBAAiB,GAAG,kBAAkB,CAAC;AAE5I;+FAC+F;AAC/F,eAAO,MAAM,UAAU,GAAI,GAAG,eAAe,KAAG,MACwH,CAAC;AAEzK,wBAAgB,YAAY,CAAC,CAAC,EAAE,eAAe,GAAG;IAAE,OAAO,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CA2BxF"}

package/dist/core/serpRecon.js

@@ -0,0 +1,99 @@
+/** Registrable-ish host: lowercase, strip scheme/path/www. Good enough to match a domain field. */
+export function hostOf(s) {
+    if (!s)
+        return '';
+    return String(s).replace(/^https?:\/\//i, '').replace(/\/.*$/, '').replace(/^www\./i, '').toLowerCase().trim();
+}
+export function parseSerpForRecon(resp, ownDomain, serpDepth = 20) {
+    const own = hostOf(ownDomain);
+    const result = resp.tasks?.[0]?.result?.[0] ?? {};
+    const items = result.items ?? [];
+    const itemTypes = result.item_types ?? [...new Set(items.map(i => i.type))];
+    let ourOrganicRank = null;
+    const organic = [];
+    let aioPresent = false, aioAsync = false, videoPresent = false;
+    const aioReferences = [];
+    const videoItems = [];
+    for (const it of items) {
+        const t = it.type;
+        if (t === 'organic') {
+            const domain = hostOf(it.domain || it.url);
+            const rank = Number(it.rank_group) || organic.length + 1; // rank_group = position among ORGANIC results
+            organic.push({ rank, domain, url: it.url ?? '', title: it.title ?? '' });
+            if (own && domain === own && ourOrganicRank == null)
+                ourOrganicRank = rank;
+        }
+        else if (t === 'ai_overview') {
+            aioPresent = true;
+            aioAsync = !!it.asynchronous_ai_overview;
+            // Citations live in TWO places and the nested set is NOT a subset of the top-level one
+            // (verified against cached responses: "ai website builder" had 1 top-level reference and
+            // 3 further domains only present on the ai_overview_element chunks). Read both, dedupe.
+            const seenRef = new Set();
+            let i = 0;
+            const collect = (refs, nested) => {
+                for (const ref of refs ?? []) {
+                    const url = ref?.url ?? '';
+                    if (!url || seenRef.has(url))
+                        continue;
+                    seenRef.add(url);
+                    aioReferences.push({ rank: Number(ref.rank_absolute) || ++i, domain: hostOf(ref.domain || url), url, title: ref.title ?? '', nested });
+                }
+            };
+            collect(it.references, false);
+            for (const el of it.items ?? [])
+                collect(el?.references, true);
+        }
+        else if (t === 'video' || t === 'short_videos') {
+            videoPresent = true;
+            for (const v of it.items ?? []) {
+                if (v?.url)
+                    videoItems.push({ url: v.url, title: v.title ?? '', source: hostOf(v.source || v.domain || v.url) });
+            }
+        }
+    }
+    // Citation is only knowable when the AIO actually resolved. An async AIO with no references
+    // is UNKNOWN, not "not cited" — 5 of 11 cached AIO SERPs came back that way, and reading them
+    // as "not cited" manufactured accuracy-or-freshness verdicts out of a missing fetch.
+    const aioResolved = aioPresent && aioReferences.length > 0;
+    const aioCitesUs = !aioPresent ? false
+        : aioResolved ? aioReferences.some(r => own && r.domain === own)
+            : null;
+    // Organic ranked above us (the ones actually beating us); if we're absent, the whole top block.
+    const organicAbove = ourOrganicRank == null
+        ? organic.slice(0, 8)
+        : organic.filter(o => o.rank < ourOrganicRank);
+    return { itemTypes, ourOrganicRank, serpDepth, aioPresent, aioCitesUs, aioAsync, aioResolved, aioReferences, videoPresent, organicAbove, videoItems };
+}
+/** How to say "we didn't find you" without implying a position we never measured. Rank 11 and
+ * rank 80 both come back as null, so the honest phrasing has to carry the depth we fetched. */
+export const rankPhrase = (r) => r.ourOrganicRank != null ? `#${r.ourOrganicRank}` : `absent from the top ${r.serpDepth} organic results (true position unknown, and unmeasured beyond ${r.serpDepth})`;
+export function reconVerdict(r) {
+    const strongOrganic = r.ourOrganicRank != null && r.ourOrganicRank <= 5;
+    const shape = r.videoPresent ? ' The SERP carries a video pack, so some click loss is format, not content.' : '';
+    const pos = rankPhrase(r);
+    if (!r.aioPresent) {
+        return strongOrganic
+            ? { verdict: 'defend-and-deepen', note: `No AI Overview on this SERP; you rank organic ${pos}. Defend and deepen.${shape}` }
+            : { verdict: 'competitive-gap', note: `No AI Overview; you rank organic ${pos}. Close the competitive gap.${shape}` };
+    }
+    // AIO present but unresolved (async, no references pulled): citation is UNKNOWN. Classify on
+    // organic strength alone and say so — never bill an unfetched AIO as "does not cite you".
+    if (r.aioCitesUs == null) {
+        const why = `An AI Overview is present but its citations were not resolved${r.aioAsync ? ' (Google loads it asynchronously)' : ''}, so whether it cites you is UNKNOWN — re-run with the async AI Overview loaded before treating this as an accuracy problem.`;
+        return strongOrganic
+            ? { verdict: 'defend-and-deepen', note: `You rank organic ${pos}. ${why}${shape}` }
+            : { verdict: 'competitive-gap', note: `You rank organic ${pos}. ${why}${shape}` };
+    }
+    if (strongOrganic && r.aioCitesUs) {
+        return { verdict: 'defend-and-deepen', note: `You rank organic ${pos} and the AI Overview already cites you. Deepen to become the primary source.${shape}` };
+    }
+    if (strongOrganic && !r.aioCitesUs) {
+        return { verdict: 'accuracy-or-freshness', note: `You rank organic ${pos} but the AI Overview does NOT cite you (checked ${r.aioReferences.length} references) — a data-accuracy or freshness signal. Make the facts current, correct and marked up so they are liftable.${shape}` };
+    }
+    if (!strongOrganic && r.aioCitesUs) {
+        return { verdict: 'consolidate-weak-page', note: `The AI Overview cites you but your organic rank is weak (${pos}) — you have a quotable nugget on an under-powered page; consolidate and strengthen it.${shape}` };
+    }
+    return { verdict: 'competitive-gap', note: `Not cited in the AI Overview (checked ${r.aioReferences.length} references) and organic rank is ${pos} — full coverage gap; run the deep competitor diff.${shape}` };
+}
+//# sourceMappingURL=serpRecon.js.map

package/dist/core/serpRecon.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"serpRecon.js","sourceRoot":"","sources":["../../src/core/serpRecon.ts"],"names":[],"mappings":"AAsBA,mGAAmG;AACnG,MAAM,UAAU,MAAM,CAAC,CAA4B;IACjD,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;AACjH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,IAAiB,EAAE,SAAiB,EAAE,SAAS,GAAG,EAAE;IACpF,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,MAAM,KAAK,GAAU,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IACxC,MAAM,SAAS,GAAa,MAAM,CAAC,UAAU,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAEtF,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,MAAM,OAAO,GAAmE,EAAE,CAAC;IACnF,IAAI,UAAU,GAAG,KAAK,EAAE,QAAQ,GAAG,KAAK,EAAE,YAAY,GAAG,KAAK,CAAC;IAC/D,MAAM,aAAa,GAAqC,EAAE,CAAC;IAC3D,MAAM,UAAU,GAAkC,EAAE,CAAC;IAErD,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;QAClB,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,MAAM,MAAM,GAAG,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAE,8CAA8C;YACzG,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,CAAC;YACzE,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,cAAc,IAAI,IAAI;gBAAE,cAAc,GAAG,IAAI,CAAC;QAC7E,CAAC;aAAM,IAAI,CAAC,KAAK,aAAa,EAAE,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC;YAClB,QAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,wBAAwB,CAAC;YACzC,uFAAuF;YACvF,yFAAyF;YACzF,wFAAwF;YACxF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;YAClC,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,MAAM,OAAO,GAAG,CAAC,IAAW,EAAE,MAAe,EAAQ,EAAE;gBACrD,KAAK,MAAM,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,CAAC;oBAC7B,MAAM,GAAG,GAAG,GAAG,EAAE,GAAG,IAAI,EAAE,CAAC;oBAC3B,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;wBAAE,SAAS;oBACvC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACjB,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;gBACzI,CAAC;YACH,CAAC,CAAC;YACF,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC9B,KAAK,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE;gBAAE,OAAO,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;QACjE,CAAC;aAAM,IAAI,CAAC,KAAK,OAAO,IAAI,CAAC,KAAK,cAAc,EAAE,CAAC;YACjD,YAAY,GAAG,IAAI,CAAC;YACpB,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;gBAC/B,IAAI,CAAC,EAAE,GAAG;oBAAE,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACnH,CAAC;QACH,CAAC;IACH,CAAC;IAED,4FAA4F;IAC5F,8FAA8F;IAC9F,qFAAqF;IACrF,MAAM,WAAW,GAAG,UAAU,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3D,MAAM,UAAU,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK;QACpC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,CAAC;YAChE,CAAC,CAAC,IAAI,CAAC;IAET,gGAAgG;IAChG,MAAM,YAAY,GAAG,cAAc,IAAI,IAAI;QACzC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACrB,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,cAAe,CAAC,CAAC;IAElD,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC;AACxJ,CAAC;AASD;+FAC+F;AAC/F,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAkB,EAAU,EAAE,CACvD,CAAC,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,SAAS,kEAAkE,CAAC,CAAC,SAAS,GAAG,CAAC;AAEzK,MAAM,UAAU,YAAY,CAAC,CAAkB;IAC7C,MAAM,aAAa,GAAG,CAAC,CAAC,cAAc,IAAI,IAAI,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,4EAA4E,CAAC,CAAC,CAAC,EAAE,CAAC;IACjH,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;QAClB,OAAO,aAAa;YAClB,CAAC,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,iDAAiD,GAAG,uBAAuB,KAAK,EAAE,EAAE;YAC5H,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,oCAAoC,GAAG,+BAA+B,KAAK,EAAE,EAAE,CAAC;IAC1H,CAAC;IACD,6FAA6F;IAC7F,0FAA0F;IAC1F,IAAI,CAAC,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,gEAAgE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,mCAAmC,CAAC,CAAC,CAAC,EAAE,8HAA8H,CAAC;QAChQ,OAAO,aAAa;YAClB,CAAC,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,oBAAoB,GAAG,KAAK,GAAG,GAAG,KAAK,EAAE,EAAE;YACnF,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,oBAAoB,GAAG,KAAK,GAAG,GAAG,KAAK,EAAE,EAAE,CAAC;IACtF,CAAC;IACD,IAAI,aAAa,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,oBAAoB,GAAG,+EAA+E,KAAK,EAAE,EAAE,CAAC;IAC/J,CAAC;IACD,IAAI,aAAa,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;QACnC,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,IAAI,EAAE,oBAAoB,GAAG,mDAAmD,CAAC,CAAC,aAAa,CAAC,MAAM,0HAA0H,KAAK,EAAE,EAAE,CAAC;IACvR,CAAC;IACD,IAAI,CAAC,aAAa,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;QACnC,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,IAAI,EAAE,4DAA4D,GAAG,0FAA0F,KAAK,EAAE,EAAE,CAAC;IACtN,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,iBAAiB,EAAE,IAAI,EAAE,yCAAyC,CAAC,CAAC,aAAa,CAAC,MAAM,oCAAoC,GAAG,sDAAsD,KAAK,EAAE,EAAE,CAAC;AACnN,CAAC"}

package/dist/server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAqEpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AA6KD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAmnD9E"}
+{"version":3,"file":"server.d.ts","sourceRoot":"","sources":["../src/server.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AA2EpE,wBAAgB,OAAO,IAAI,MAAM,CAGhC;AA8KD,wBAAgB,YAAY,IAAI;IAAE,MAAM,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CA26D9E"}