@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.0

package/dist/core/static.js

@@ -0,0 +1,184 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { extractPage } from './html/extract.js';
+import { DEFAULT_USER_AGENT } from './fetch.js';
+import { buildRobots } from './robots.js';
+import { parseSitemap } from './sitemap.js';
+import { normalizePath, normalizeUrl } from './url.js';
+const SKIP_DIRS = new Set(['node_modules', '.git', '.next', '.nuxt', '.svelte-kit', '.cache', '.vercel', '.output/server']);
+export async function scanDirectory(dir, options) {
+    const started = performance.now();
+    const root = path.resolve(dir);
+    const stat = await fs.stat(root).catch(() => null);
+    if (!stat?.isDirectory())
+        throw new Error(`Not a directory: ${root}`);
+    const base = new URL(options.baseUrl.endsWith('/') ? options.baseUrl : `${options.baseUrl}/`);
+    const files = (await walk(root)).sort();
+    const htmlFiles = files.filter((f) => /\.html?$/i.test(f));
+    const hitLimit = htmlFiles.length > options.maxPages;
+    const selected = htmlFiles.slice(0, options.maxPages);
+    const toUrlPath = (file) => `/${path.relative(root, file).split(path.sep).map(encodeURIComponent).join('/')}`;
+    const fileByPathKey = new Map();
+    for (const f of htmlFiles)
+        fileByPathKey.set(normalizePath(toUrlPath(f)), f);
+    const records = [];
+    const linkChecks = new Map();
+    const linkSources = new Map();
+    const recordByKey = new Map();
+    for (const file of selected) {
+        const html = await fs.readFile(file, 'utf8');
+        const urlPath = toUrlPath(file).replace(/\/index\.html?$/i, '/');
+        const url = new URL(urlPath, base).href;
+        const page = extractPage(html, { url, siteHost: base.hostname, includeSubdomains: options.includeSubdomains });
+        const record = { url, requestedUrl: url, page, depth: -1, file: path.relative(root, file) };
+        records.push(record);
+        recordByKey.set(normalizePath(new URL(url).pathname), record);
+    }
+    for (const record of records) {
+        for (const link of record.page.links) {
+            if (!link.internal || !link.url)
+                continue;
+            const target = normalizeUrl(link.url) ?? link.url;
+            if (!linkSources.has(target))
+                linkSources.set(target, new Set());
+            linkSources.get(target).add(record.url);
+            if (!linkChecks.has(target)) {
+                const key = normalizePath(new URL(target).pathname);
+                const exists = fileByPathKey.has(key) || (await fileExists(root, new URL(target).pathname));
+                linkChecks.set(target, { url: target, status: exists ? 200 : 404, finalUrl: target, redirectChain: [], error: exists ? undefined : 'No matching file in output directory' });
+            }
+        }
+    }
+    /* Depth via BFS from the root index (if present). */
+    const start = recordByKey.get('/') ?? records[0];
+    if (start) {
+        start.depth = 0;
+        const queue = [start];
+        while (queue.length) {
+            const current = queue.shift();
+            for (const link of current.page.links) {
+                if (!link.internal || !link.url)
+                    continue;
+                const target = recordByKey.get(normalizePath(new URL(link.url).pathname));
+                if (target && target.depth === -1) {
+                    target.depth = current.depth + 1;
+                    queue.push(target);
+                }
+            }
+        }
+    }
+    const linkedKeys = new Set([...linkSources.keys()].map((u) => normalizePath(new URL(u).pathname)));
+    const unlinkedPages = records.filter((r) => r !== start && !linkedKeys.has(normalizePath(new URL(r.url).pathname))).map((r) => r.file ?? r.url);
+    /* robots.txt and sitemap.xml from the output directory. */
+    const robotsText = await fs.readFile(path.join(root, 'robots.txt'), 'utf8').catch(() => null);
+    const robots = buildRobots(new URL('/robots.txt', base).href, robotsText === null ? 404 : 200, undefined, robotsText, options.userAgent ?? DEFAULT_USER_AGENT);
+    const sitemaps = { sources: [], urls: [], urlCount: 0, errors: [] };
+    const sitemapCandidates = [...new Set([...robots.info.sitemaps, new URL('/sitemap.xml', base).href])];
+    const seen = new Set();
+    const urls = new Set();
+    while (sitemapCandidates.length && seen.size < 25) {
+        const candidate = sitemapCandidates.shift();
+        if (seen.has(candidate))
+            continue;
+        seen.add(candidate);
+        let pathname;
+        try {
+            pathname = decodeURIComponent(new URL(candidate, base).pathname);
+        }
+        catch {
+            sitemaps.errors.push(`${candidate}: invalid URL`);
+            continue;
+        }
+        const file = path.join(root, pathname);
+        const xml = await fs.readFile(file, 'utf8').catch(() => null);
+        if (xml === null) {
+            if (robots.info.sitemaps.includes(candidate))
+                sitemaps.errors.push(`${candidate}: declared in robots.txt but ${pathname} is not in the output directory`);
+            continue;
+        }
+        const parsed = parseSitemap(xml);
+        if (parsed.kind === 'unknown') {
+            sitemaps.errors.push(`${pathname}: could not parse as a sitemap`);
+            continue;
+        }
+        sitemaps.sources.push(candidate);
+        if (parsed.kind === 'sitemapindex')
+            sitemapCandidates.push(...parsed.locs);
+        else
+            parsed.locs.forEach((l) => urls.add(l));
+    }
+    sitemaps.urls = [...urls];
+    sitemaps.urlCount = urls.size;
+    for (const u of sitemaps.urls) {
+        let key;
+        try {
+            key = normalizePath(new URL(u).pathname);
+        }
+        catch {
+            continue;
+        }
+        const exists = fileByPathKey.has(key);
+        const synthetic = new URL(key === '/' ? '/' : key, base).href;
+        if (!linkChecks.has(synthetic))
+            linkChecks.set(synthetic, { url: u, status: exists ? 200 : 404, finalUrl: u, redirectChain: [], error: exists ? undefined : 'No matching file in output directory' });
+    }
+    return {
+        startUrl: base.href,
+        records,
+        linkChecks,
+        linkSources,
+        robots: robots.info,
+        robotsAllows: robots.info.found ? robots.allows : null,
+        blockedByRobots: robots.info.found ? records.filter((r) => !robots.allows(r.url)).map((r) => r.url) : [],
+        sitemaps,
+        probes: {},
+        stats: {
+            pagesCrawled: records.length,
+            pagesDiscovered: htmlFiles.length,
+            linksChecked: linkChecks.size,
+            maxPages: options.maxPages,
+            maxDepth: Infinity,
+            hitPageLimit: hitLimit,
+            durationMs: Math.round(performance.now() - started),
+            respectedRobots: false,
+        },
+        unlinkedPages,
+    };
+}
+async function walk(dir) {
+    const out = [];
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    for (const entry of entries) {
+        if (entry.name.startsWith('.') && entry.name !== '.well-known')
+            continue;
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) {
+            if (SKIP_DIRS.has(entry.name))
+                continue;
+            out.push(...(await walk(full)));
+        }
+        else if (entry.isFile())
+            out.push(full);
+    }
+    return out;
+}
+async function fileExists(root, pathname) {
+    let decoded = pathname;
+    try {
+        decoded = decodeURIComponent(pathname);
+    }
+    catch {
+        /* keep */
+    }
+    const candidates = [decoded, `${decoded}.html`, `${decoded}.htm`, path.posix.join(decoded, 'index.html'), path.posix.join(decoded, 'index.htm')];
+    for (const c of candidates) {
+        const full = path.join(root, c);
+        if (!full.startsWith(root))
+            continue;
+        const stat = await fs.stat(full).catch(() => null);
+        if (stat?.isFile())
+            return true;
+    }
+    return false;
+}
+//# sourceMappingURL=static.js.map

package/dist/core/static.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"static.js","sourceRoot":"","sources":["../../src/core/static.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAI5C,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAUvD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC,CAAC;AAE5H,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,GAAW,EAAE,OAAyB;IACxE,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAC/B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI,EAAE,WAAW,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,EAAE,CAAC,CAAC;IACtE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,OAAO,GAAG,CAAC,CAAC;IAE9F,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IACrD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAEtD,MAAM,SAAS,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;IAC9H,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,SAAS;QAAE,aAAa,CAAC,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE7E,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAuB,CAAC;IACnD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAElD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;QACxC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;QAC/G,MAAM,MAAM,GAAe,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACxG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACrB,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACrC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG;gBAAE,SAAS;YAC1C,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;YACjE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACpD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC5F,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,sCAAsC,EAAE,CAAC,CAAC;YAC/K,CAAC;QACH,CAAC;IACH,CAAC;IAED,qDAAqD;IACrD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;QAChB,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC;QACtB,OAAO,KAAK,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,EAAG,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,GAAG;oBAAE,SAAS;gBAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAC1E,IAAI,MAAM,IAAI,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;oBAClC,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC;oBACjC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnG,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAEhJ,2DAA2D;IAC3D,MAAM,UAAU,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;IAE/J,MAAM,QAAQ,GAAgB,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IACjF,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACtG,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,iBAAiB,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,GAAG,EAAE,EAAE,CAAC;QAClD,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,EAAG,CAAC;QAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,SAAS;QAClC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACpB,IAAI,QAAgB,CAAC;QACrB,IAAI,CAAC;YACH,QAAQ,GAAG,kBAAkB,CAAC,IAAI,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnE,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,eAAe,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QAC9D,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;YACjB,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,gCAAgC,QAAQ,iCAAiC,CAAC,CAAC;YAC1J,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC9B,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,gCAAgC,CAAC,CAAC;YAClE,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACjC,IAAI,MAAM,CAAC,IAAI,KAAK,cAAc;YAAE,iBAAiB,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;;YACtE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IAC1B,QAAQ,CAAC,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IAC9B,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC9B,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC;QAC9D,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,sCAAsC,EAAE,CAAC,CAAC;IACxM,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,IAAI;QACnB,OAAO;QACP,UAAU;QACV,WAAW;QACX,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;QACtD,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE;QACxG,QAAQ;QACR,MAAM,EAAE,EAAE;QACV,KAAK,EAAE;YACL,YAAY,EAAE,OAAO,CAAC,MAAM;YAC5B,eAAe,EAAE,SAAS,CAAC,MAAM;YACjC,YAAY,EAAE,UAAU,CAAC,IAAI;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,QAAQ,EAAE,QAAQ;YAClB,YAAY,EAAE,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;YACnD,eAAe,EAAE,KAAK;SACvB;QACD,aAAa;KACd,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,GAAW;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa;YAAE,SAAS;QACzE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YACxC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,IAAY,EAAE,QAAgB;IACtD,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,IAAI,CAAC;QACH,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,UAAU;IACZ,CAAC;IACD,MAAM,UAAU,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,OAAO,EAAE,GAAG,OAAO,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC;IACjJ,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QACrC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;QACnD,IAAI,IAAI,EAAE,MAAM,EAAE;YAAE,OAAO,IAAI,CAAC;IAClC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/core/summary.d.ts

@@ -0,0 +1,2 @@
+import type { ExtractedPage, PageSummary, ResponseInfo } from './types.js';
+export declare function buildSummary(page: ExtractedPage, response?: ResponseInfo): PageSummary;

package/dist/core/summary.js

@@ -0,0 +1,28 @@
+import { robotsDirectives } from './rules/indexing.js';
+export function buildSummary(page, response) {
+    const robots = robotsDirectives(page, response);
+    return {
+        title: page.title,
+        titleLength: page.title?.length ?? 0,
+        metaDescription: page.metaDescription,
+        metaDescriptionLength: page.metaDescription?.length ?? 0,
+        canonical: page.canonical,
+        lang: page.lang,
+        robots: robots.directives,
+        indexable: !robots.noindex && (response?.status ?? 200) < 400,
+        h1: page.h1s,
+        headings: page.headings.slice(0, 40),
+        wordCount: page.wordCount,
+        images: { total: page.images.length, missingAlt: page.images.filter((i) => i.alt === null).length },
+        links: {
+            internal: page.links.filter((l) => l.internal).length,
+            external: page.links.filter((l) => l.url && !l.internal).length,
+            nofollow: page.links.filter((l) => l.nofollow).length,
+        },
+        openGraph: page.og,
+        twitterCard: page.twitter['card'] ?? null,
+        structuredDataTypes: [...new Set(page.structuredData.flatMap((i) => i.types))],
+        hreflangCount: page.hreflang.length,
+    };
+}
+//# sourceMappingURL=summary.js.map

package/dist/core/summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary.js","sourceRoot":"","sources":["../../src/core/summary.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEvD,MAAM,UAAU,YAAY,CAAC,IAAmB,EAAE,QAAuB;IACvE,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAChD,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,WAAW,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,CAAC;QACpC,eAAe,EAAE,IAAI,CAAC,eAAe;QACrC,qBAAqB,EAAE,IAAI,CAAC,eAAe,EAAE,MAAM,IAAI,CAAC;QACxD,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,MAAM,EAAE,MAAM,CAAC,UAAU;QACzB,SAAS,EAAE,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG;QAC7D,EAAE,EAAE,IAAI,CAAC,GAAG;QACZ,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;QACpC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE;QACnG,KAAK,EAAE;YACL,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;YACrD,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;YAC/D,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM;SACtD;QACD,SAAS,EAAE,IAAI,CAAC,EAAE;QAClB,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI;QACzC,mBAAmB,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,aAAa,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;KACpC,CAAC;AACJ,CAAC"}

package/dist/core/types.d.ts

@@ -0,0 +1,296 @@
+/**
+ * Shared types for the SEO audit core. These shapes are the public contract
+ * of the CLI JSON output and the MCP tool results (`schemaVersion: 1`).
+ */
+export type Severity = 'error' | 'warning' | 'info';
+export declare const SEVERITY_RANK: Record<Severity, number>;
+export type Category = 'title' | 'meta' | 'headings' | 'canonical' | 'indexing' | 'document' | 'social' | 'images' | 'links' | 'international' | 'structured-data' | 'content' | 'keyword' | 'response' | 'rendering' | 'performance' | 'site' | 'tool';
+export type Requirement = 'render' | 'lighthouse' | 'crawl' | 'keyword';
+/** Static description of a rule, exposed through `seo_list_rules` / `agentic-seo-check rules`. */
+export interface RuleMeta {
+    id: string;
+    category: Category;
+    /** Typical severity. Individual findings may escalate or downgrade. */
+    severity: Severity;
+    description: string;
+    recommendation: string;
+    docs?: string;
+    /** Only runs when this capability/mode is active. */
+    requires?: Requirement;
+}
+/** One actionable problem. Designed to be read by an agent that will edit code. */
+export interface Finding {
+    rule: string;
+    severity: Severity;
+    category: Category;
+    /** What is wrong, stated as a fact about this page/site. */
+    message: string;
+    /** What to change. Concrete and framework-agnostic. */
+    recommendation: string;
+    /** Page URL or file path the finding belongs to (absent for site-wide findings that list pages in evidence). */
+    url?: string;
+    /** CSS selector or location hint for where the problem lives in the HTML. */
+    selector?: string;
+    /** Current values, counts, offending items (capped lists). */
+    evidence?: Record<string, unknown>;
+    docs?: string;
+}
+export interface SeverityCounts {
+    error: number;
+    warning: number;
+    info: number;
+}
+export interface HeadingInfo {
+    level: number;
+    text: string;
+}
+export interface ImageInfo {
+    src: string | null;
+    alt: string | null;
+    width: string | null;
+    height: string | null;
+    loading: string | null;
+}
+export interface LinkInfo {
+    href: string;
+    /** Absolute resolved URL, or null if it could not be resolved (mailto:, javascript:, malformed). */
+    url: string | null;
+    text: string;
+    rel: string | null;
+    internal: boolean;
+    nofollow: boolean;
+    /** True when the link has visible text, an aria-label, a title, or an image with alt text. */
+    hasAccessibleName: boolean;
+}
+export interface HreflangInfo {
+    lang: string;
+    href: string;
+    url: string | null;
+}
+export interface StructuredDataItem {
+    source: 'json-ld' | 'microdata';
+    types: string[];
+    context: string | null;
+    data: Record<string, unknown>;
+}
+export interface StructuredDataError {
+    source: 'json-ld';
+    message: string;
+    snippet: string;
+}
+/** Everything the extractor pulls out of one HTML document. Rules read only this. */
+export interface ExtractedPage {
+    /** URL used to resolve relative links (the final URL after redirects, or the base URL for files/inline HTML). */
+    url: string;
+    hasDoctype: boolean;
+    lang: string | null;
+    charset: string | null;
+    viewport: string | null;
+    titles: string[];
+    title: string | null;
+    metaDescriptions: string[];
+    metaDescription: string | null;
+    /** Content of <meta name="robots"> and <meta name="googlebot">, lower-cased. */
+    metaRobots: string[];
+    canonicals: string[];
+    canonical: string | null;
+    headings: HeadingInfo[];
+    h1s: string[];
+    og: Record<string, string>;
+    twitter: Record<string, string>;
+    images: ImageInfo[];
+    links: LinkInfo[];
+    hreflang: HreflangInfo[];
+    structuredData: StructuredDataItem[];
+    structuredDataErrors: StructuredDataError[];
+    /** Visible text of <body> with scripts/styles removed and whitespace collapsed. */
+    text: string;
+    wordCount: number;
+    htmlBytes: number;
+    hasBaseHref: boolean;
+}
+export interface RedirectHop {
+    url: string;
+    status: number;
+}
+export interface ResponseInfo {
+    status: number | null;
+    finalUrl: string;
+    redirectChain: RedirectHop[];
+    timeMs: number | null;
+    contentType: string | null;
+    /** Lower-cased header names. Only SEO-relevant headers are kept. */
+    headers: Record<string, string>;
+    error?: string;
+}
+export interface PageSummary {
+    title: string | null;
+    titleLength: number;
+    metaDescription: string | null;
+    metaDescriptionLength: number;
+    canonical: string | null;
+    lang: string | null;
+    robots: string[];
+    indexable: boolean;
+    h1: string[];
+    headings: HeadingInfo[];
+    wordCount: number;
+    images: {
+        total: number;
+        missingAlt: number;
+    };
+    links: {
+        internal: number;
+        external: number;
+        nofollow: number;
+    };
+    openGraph: Record<string, string>;
+    twitterCard: string | null;
+    structuredDataTypes: string[];
+    hreflangCount: number;
+}
+export interface LighthouseMetrics {
+    performanceScore: number | null;
+    seoScore: number | null;
+    agenticBrowsingScore: number | null;
+    lcpMs: number | null;
+    cls: number | null;
+    tbtMs: number | null;
+    fcpMs: number | null;
+    speedIndexMs: number | null;
+    ttfbMs: number | null;
+    inpMs: number | null;
+}
+export interface LighthouseOpportunity {
+    id: string;
+    title: string;
+    description: string;
+    score: number | null;
+    savingsMs: number | null;
+    savingsBytes: number | null;
+}
+export interface LighthouseAuditSummary {
+    id: string;
+    title: string;
+    description: string;
+    score: number | null;
+    displayValue: string | null;
+}
+export interface PerformanceResult {
+    tool: 'lighthouse';
+    lighthouseVersion: string;
+    formFactor: 'mobile' | 'desktop';
+    requestedUrl: string;
+    finalUrl: string;
+    metrics: LighthouseMetrics;
+    opportunities: LighthouseOpportunity[];
+    failedSeoAudits: LighthouseAuditSummary[];
+    agenticBrowsingAudits: LighthouseAuditSummary[];
+}
+export interface RenderComparison {
+    tool: 'playwright';
+    renderedUrl: string;
+    consoleErrors: string[];
+    raw: RenderSnapshot;
+    rendered: RenderSnapshot;
+}
+export interface RenderSnapshot {
+    title: string | null;
+    metaDescription: string | null;
+    canonical: string | null;
+    robots: string[];
+    h1Count: number;
+    wordCount: number;
+    internalLinks: number;
+    structuredDataCount: number;
+}
+export type PageSource = 'url' | 'file' | 'html';
+export interface PageAudit {
+    schemaVersion: 1;
+    kind: 'page';
+    /** What the caller asked for (URL, path, or "inline-html"). */
+    target: string;
+    /** Final URL after redirects, or the file path. */
+    url: string;
+    source: PageSource;
+    auditedAt: string;
+    response?: ResponseInfo;
+    summary: PageSummary;
+    findings: Finding[];
+    counts: SeverityCounts;
+    /** 0-100. Indicative only: 100 minus weighted deductions per finding. */
+    score: number;
+    performance?: PerformanceResult;
+    rendering?: RenderComparison;
+    /** Crawl depth from the start URL (site audits only). */
+    depth?: number;
+}
+export interface RobotsTxtInfo {
+    url: string;
+    status: number | null;
+    found: boolean;
+    sitemaps: string[];
+    /** Raw text, capped. */
+    text: string | null;
+    error?: string;
+}
+export interface SitemapInfo {
+    /** Sitemap (and sitemap index) URLs that were fetched. */
+    sources: string[];
+    /** Page URLs listed across all sitemaps (capped). */
+    urls: string[];
+    urlCount: number;
+    errors: string[];
+}
+export interface CrawlStats {
+    pagesCrawled: number;
+    pagesDiscovered: number;
+    linksChecked: number;
+    maxPages: number;
+    maxDepth: number;
+    hitPageLimit: boolean;
+    durationMs: number;
+    respectedRobots: boolean;
+}
+export interface IssueRollup {
+    rule: string;
+    severity: Severity;
+    category: Category;
+    pages: number;
+    message: string;
+    recommendation: string;
+    /** Up to 10 example URLs. */
+    examples: string[];
+}
+export interface SiteAudit {
+    schemaVersion: 1;
+    kind: 'site';
+    target: string;
+    source: 'crawl' | 'directory';
+    auditedAt: string;
+    crawl: CrawlStats;
+    robotsTxt: RobotsTxtInfo | null;
+    sitemaps: SitemapInfo;
+    pages: PageAudit[];
+    /** Findings that span pages (duplicates, broken links, sitemap mismatches, ...). */
+    siteFindings: Finding[];
+    /** Page-level findings rolled up by rule, most severe and most frequent first. */
+    issues: IssueRollup[];
+    counts: SeverityCounts;
+    score: number;
+}
+export type Audit = PageAudit | SiteAudit;
+export interface Thresholds {
+    titleMinChars: number;
+    titleMaxChars: number;
+    descriptionMinChars: number;
+    descriptionMaxChars: number;
+    h1MaxChars: number;
+    thinContentWords: number;
+    slowResponseMs: number;
+    deepPageDepth: number;
+    maxKeywordDensityPercent: number;
+}
+export declare const DEFAULT_THRESHOLDS: Thresholds;
+export declare function emptyCounts(): SeverityCounts;
+export declare function countFindings(findings: Finding[]): SeverityCounts;

package/dist/core/types.js

@@ -0,0 +1,26 @@
+/**
+ * Shared types for the SEO audit core. These shapes are the public contract
+ * of the CLI JSON output and the MCP tool results (`schemaVersion: 1`).
+ */
+export const SEVERITY_RANK = { error: 3, warning: 2, info: 1 };
+export const DEFAULT_THRESHOLDS = {
+    titleMinChars: 30,
+    titleMaxChars: 60,
+    descriptionMinChars: 70,
+    descriptionMaxChars: 160,
+    h1MaxChars: 70,
+    thinContentWords: 300,
+    slowResponseMs: 2000,
+    deepPageDepth: 3,
+    maxKeywordDensityPercent: 3,
+};
+export function emptyCounts() {
+    return { error: 0, warning: 0, info: 0 };
+}
+export function countFindings(findings) {
+    const counts = emptyCounts();
+    for (const f of findings)
+        counts[f.severity]++;
+    return counts;
+}
+//# sourceMappingURL=types.js.map

package/dist/core/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/core/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,CAAC,MAAM,aAAa,GAA6B,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AA0UzF,MAAM,CAAC,MAAM,kBAAkB,GAAe;IAC5C,aAAa,EAAE,EAAE;IACjB,aAAa,EAAE,EAAE;IACjB,mBAAmB,EAAE,EAAE;IACvB,mBAAmB,EAAE,GAAG;IACxB,UAAU,EAAE,EAAE;IACd,gBAAgB,EAAE,GAAG;IACrB,cAAc,EAAE,IAAI;IACpB,aAAa,EAAE,CAAC;IAChB,wBAAwB,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,UAAU,WAAW;IACzB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,QAAmB;IAC/C,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC/C,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/core/url.d.ts

@@ -0,0 +1,20 @@
+export declare function isHttpUrl(value: string): boolean;
+/** Resolve `href` against `base`, returning null for non-http(s) or unparsable values. */
+export declare function resolveUrl(href: string, base: string): string | null;
+/**
+ * Canonical form used for visited-sets and comparisons: lower-cased scheme/host,
+ * default ports dropped, fragment removed, tracking parameters removed.
+ * Path and remaining query are kept verbatim (trailing slashes matter to servers).
+ */
+export declare function normalizeUrl(input: string): string | null;
+export declare function hostOf(url: string): string | null;
+export declare function stripWww(host: string): string;
+/** Same site: identical host ignoring `www.`; optionally any subdomain of the start host. */
+export declare function isSameSite(candidate: string, startHost: string, includeSubdomains?: boolean): boolean;
+/** Heuristic: does the path look like it points at an HTML document (not an asset)? */
+export declare function looksLikeHtml(url: string): boolean;
+/** Compare two URLs by path only (used for static-directory audits where the host is synthetic). */
+export declare function samePath(a: string, b: string): boolean;
+/** `/dir/index.html` -> `/dir/`, `/page.html` -> `/page`, trailing slash dropped except root. */
+export declare function normalizePath(pathname: string): string;
+export declare function truncate(value: string, max?: number): string;