@rivi-software/agentic-seo-check 0.0.0-stage → 0.1.0

package/dist/core/audit.js

@@ -0,0 +1,337 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import { crawlSite } from './crawl.js';
+import { DEFAULT_USER_AGENT, fetchPage, toResponseInfo } from './fetch.js';
+import { extractPage } from './html/extract.js';
+import { runLighthouse, LighthouseUnavailableError } from './lighthouse.js';
+import { performanceFindings } from './performance-rules.js';
+import { compareRender } from './render-rules.js';
+import { renderPage, RenderUnavailableError } from './render.js';
+import { runPageRules, sortFindings } from './rules/index.js';
+import { pageScore, siteScore } from './score.js';
+import { runSiteRules } from './site-rules.js';
+import { scanDirectory } from './static.js';
+import { buildSummary } from './summary.js';
+import { countFindings, DEFAULT_THRESHOLDS, } from './types.js';
+import { isHttpUrl } from './url.js';
+export const DEFAULT_CRAWL = { maxPages: 50, maxDepth: 3, concurrency: 5, delayMs: 0, maxLinkChecks: 200 };
+/** Audit an HTML string. `url` is used to resolve links and for display. */
+export async function auditHtml(html, input = {}, options = {}) {
+    const url = input.url ?? options.baseUrl ?? 'http://localhost/';
+    const page = extractPage(html, { url, includeSubdomains: options.includeSubdomains });
+    const source = input.source ?? 'html';
+    const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds ?? {}) };
+    let findings = runPageRules({ page, source, response: input.response, keyword: options.keyword, thresholds, depth: input.depth }, { disable: new Set(options.disableRules ?? []) });
+    if (options.includeInfo === false)
+        findings = findings.filter((f) => f.severity !== 'info');
+    const audit = {
+        schemaVersion: 1,
+        kind: 'page',
+        target: input.target ?? url,
+        url: input.response?.finalUrl ?? url,
+        source,
+        auditedAt: new Date().toISOString(),
+        summary: buildSummary(page, input.response),
+        findings,
+        counts: countFindings(findings),
+        score: 0,
+    };
+    if (input.response)
+        audit.response = input.response;
+    if (input.depth !== undefined)
+        audit.depth = input.depth;
+    audit.score = pageScore(audit.counts);
+    return audit;
+}
+/** Fetch and audit one URL. Optionally render and/or run Lighthouse. */
+export async function auditUrl(url, options = {}) {
+    const res = await fetchPage(url, options.fetch ?? {});
+    const response = toResponseInfo(res);
+    const audit = await auditHtml(res.body ?? '', { url: res.finalUrl, response, source: 'url', target: url }, options);
+    if (res.error) {
+        audit.findings.unshift({
+            rule: 'response.fetch-failed',
+            severity: 'error',
+            category: 'response',
+            message: `Could not fetch ${url}: ${res.error}`,
+            recommendation: 'Check that the URL is reachable from this machine (DNS, TLS, firewall, dev server running) and that it responds within the timeout.',
+            url,
+            evidence: { error: res.error, redirectChain: res.redirectChain },
+        });
+        audit.counts = countFindings(audit.findings);
+        audit.score = pageScore(audit.counts);
+        return audit;
+    }
+    await enrichPage(audit, options);
+    return audit;
+}
+/** Audit a single .html file from disk. */
+export async function auditFile(filePath, options = {}) {
+    const html = await fs.readFile(filePath, 'utf8');
+    const base = options.baseUrl ?? 'http://localhost/';
+    const url = new URL(`/${path.basename(filePath)}`, base.endsWith('/') ? base : `${base}/`).href;
+    return auditHtml(html, { url, source: 'file', target: filePath }, options);
+}
+/** Crawl a site and audit every page plus site-level checks. */
+export async function auditSite(startUrl, options = {}) {
+    const fetchOpts = { ...(options.fetch ?? {}) };
+    fetchOpts.userAgent ??= DEFAULT_USER_AGENT;
+    const internal = {
+        maxPages: options.maxPages ?? DEFAULT_CRAWL.maxPages,
+        maxDepth: options.maxDepth ?? DEFAULT_CRAWL.maxDepth,
+        concurrency: options.concurrency ?? DEFAULT_CRAWL.concurrency,
+        delayMs: options.delayMs ?? DEFAULT_CRAWL.delayMs,
+        respectRobots: options.respectRobots ?? true,
+        includeSubdomains: options.includeSubdomains ?? false,
+        checkExternalLinks: options.checkExternalLinks ?? false,
+        maxLinkChecks: options.maxLinkChecks ?? DEFAULT_CRAWL.maxLinkChecks,
+        includePatterns: (options.includePatterns ?? []).map((p) => new RegExp(p)),
+        excludePatterns: (options.excludePatterns ?? []).map((p) => new RegExp(p)),
+        fetch: fetchOpts,
+        onProgress: options.onProgress,
+    };
+    const result = await crawlSite(startUrl, internal);
+    const site = await buildSiteAudit(result, 'crawl', startUrl, options);
+    // Render/Lighthouse are expensive: apply them to the start page only.
+    const startPage = site.pages.find((p) => p.depth === 0) ?? site.pages[0];
+    if (startPage && (options.render || options.lighthouse)) {
+        await enrichPage(startPage, options);
+        recomputeSite(site);
+    }
+    return shapeSite(site, options);
+}
+/** Audit a directory of built HTML files (dist/, out/, build/). */
+export async function auditDirectory(dir, options = {}) {
+    const result = await scanDirectory(dir, {
+        baseUrl: options.baseUrl ?? 'http://localhost/',
+        includeSubdomains: options.includeSubdomains ?? false,
+        maxPages: options.maxPages ?? 500,
+        userAgent: options.fetch?.userAgent,
+    });
+    const site = await buildSiteAudit(result, 'directory', dir, options);
+    return shapeSite(site, options);
+}
+/**
+ * Auto-detect the target type: http(s) URL, directory, .html file, or "-" for stdin.
+ * `crawl` forces a site crawl for URLs.
+ */
+export async function auditTarget(target, options = {}) {
+    if (target === '-' || options.stdinHtml !== undefined) {
+        const html = options.stdinHtml ?? (await readStdin());
+        return auditHtml(html, { url: options.baseUrl, source: 'html', target: 'stdin' }, options);
+    }
+    if (isHttpUrl(target))
+        return options.crawl ? auditSite(target, options) : auditUrl(target, options);
+    if (/^localhost(:\d+)?(\/|$)/i.test(target) || /^[\w.-]+\.[a-z]{2,}(\/|$)/i.test(target)) {
+        const url = `http${/^localhost/i.test(target) ? '' : 's'}://${target}`;
+        return options.crawl ? auditSite(url, options) : auditUrl(url, options);
+    }
+    const stat = await fs.stat(target).catch(() => null);
+    if (!stat)
+        throw new Error(`Target "${target}" is not a URL, file or directory.`);
+    if (stat.isDirectory())
+        return auditDirectory(target, options);
+    return auditFile(target, options);
+}
+async function enrichPage(audit, options) {
+    const url = audit.response?.finalUrl ?? audit.url;
+    const extra = [];
+    if (options.render) {
+        options.onProgress?.({ phase: 'render', message: `Rendering ${url} with Playwright` });
+        try {
+            const rendered = await renderPage(url, { timeoutMs: Math.max(options.fetch?.timeoutMs ?? 0, 30_000), userAgent: options.fetch?.userAgent, headers: options.fetch?.headers });
+            const rawPage = extractPage(await rawHtmlFor(audit, options), { url, includeSubdomains: options.includeSubdomains });
+            const renderedPage = extractPage(rendered.html, { url: rendered.finalUrl, siteHost: new URL(url).hostname, includeSubdomains: options.includeSubdomains });
+            const { comparison, findings } = compareRender(rawPage, renderedPage, rendered.finalUrl, rendered.consoleErrors, audit.response);
+            audit.rendering = comparison;
+            extra.push(...findings);
+        }
+        catch (err) {
+            extra.push(toolFinding('tool.render-unavailable', err instanceof RenderUnavailableError ? err.message : `Rendering failed: ${err.message}`, url));
+        }
+    }
+    if (options.lighthouse) {
+        options.onProgress?.({ phase: 'lighthouse', message: `Running Lighthouse (${options.formFactor ?? 'mobile'}) on ${url}` });
+        try {
+            const perf = await runLighthouse(url, { formFactor: options.formFactor ?? 'mobile', userAgent: options.fetch?.userAgent, headers: options.fetch?.headers });
+            audit.performance = perf;
+            extra.push(...performanceFindings(perf));
+        }
+        catch (err) {
+            extra.push(toolFinding('tool.lighthouse-unavailable', err instanceof LighthouseUnavailableError ? err.message : `Lighthouse failed: ${err.message}`, url));
+        }
+    }
+    if (extra.length) {
+        const filtered = options.includeInfo === false ? extra.filter((f) => f.severity !== 'info') : extra;
+        audit.findings = sortFindings([...audit.findings, ...filtered]);
+        audit.counts = countFindings(audit.findings);
+        audit.score = pageScore(audit.counts);
+    }
+}
+// The raw HTML is not kept on the audit (too large); re-fetch it for the render comparison.
+async function rawHtmlFor(audit, options) {
+    const res = await fetchPage(audit.response?.finalUrl ?? audit.url, options.fetch ?? {});
+    return res.body ?? '';
+}
+function toolFinding(rule, message, url) {
+    return {
+        rule,
+        severity: 'info',
+        category: 'tool',
+        message,
+        recommendation: 'Install the optional dependency (see message) or run without this option. The rest of the audit is complete.',
+        url,
+    };
+}
+async function buildSiteAudit(result, source, target, options) {
+    options.onProgress?.({ phase: 'rules', message: `Evaluating rules for ${result.records.length} pages` });
+    const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds ?? {}) };
+    const pages = [];
+    for (const record of result.records) {
+        const audit = await auditRecord(record, source === 'directory' ? 'file' : 'url', options);
+        pages.push(audit);
+    }
+    let siteFindings = runSiteRules({
+        startUrl: result.startUrl,
+        source,
+        pages: result.records,
+        linkChecks: result.linkChecks,
+        linkSources: result.linkSources,
+        externalChecks: result.externalChecks,
+        externalSources: result.externalSources,
+        robots: result.robots,
+        robotsAllows: result.robotsAllows,
+        blockedByRobots: result.blockedByRobots,
+        sitemaps: result.sitemaps,
+        probes: result.probes,
+        crawl: result.stats,
+        thresholds,
+        pathOnly: source === 'directory',
+        unlinkedPages: result.unlinkedPages,
+    });
+    const disabled = new Set(options.disableRules ?? []);
+    siteFindings = sortFindings(siteFindings.filter((f) => !disabled.has(f.rule)));
+    if (options.includeInfo === false)
+        siteFindings = siteFindings.filter((f) => f.severity !== 'info');
+    const site = {
+        schemaVersion: 1,
+        kind: 'site',
+        target,
+        source,
+        auditedAt: new Date().toISOString(),
+        crawl: result.stats,
+        robotsTxt: result.robots,
+        sitemaps: { ...result.sitemaps, urls: result.sitemaps.urls.slice(0, 200) },
+        pages: pages.sort((a, b) => (a.depth ?? 0) - (b.depth ?? 0) || a.score - b.score),
+        siteFindings,
+        issues: [],
+        counts: { error: 0, warning: 0, info: 0 },
+        score: 0,
+    };
+    recomputeSite(site);
+    return site;
+}
+async function auditRecord(record, source, options) {
+    const thresholds = { ...DEFAULT_THRESHOLDS, ...(options.thresholds ?? {}) };
+    let findings = runPageRules({ page: record.page, source, response: record.response, keyword: options.keyword, thresholds, depth: record.depth }, { disable: new Set(options.disableRules ?? []) });
+    if (options.includeInfo === false)
+        findings = findings.filter((f) => f.severity !== 'info');
+    const audit = {
+        schemaVersion: 1,
+        kind: 'page',
+        target: record.file ?? record.requestedUrl,
+        url: record.file ?? record.url,
+        source,
+        auditedAt: new Date().toISOString(),
+        summary: buildSummary(record.page, record.response),
+        findings,
+        counts: countFindings(findings),
+        score: 0,
+        depth: record.depth,
+    };
+    if (record.response)
+        audit.response = record.response;
+    audit.score = pageScore(audit.counts);
+    return audit;
+}
+export function recomputeSite(site) {
+    const counts = countFindings(site.siteFindings);
+    for (const p of site.pages) {
+        counts.error += p.counts.error;
+        counts.warning += p.counts.warning;
+        counts.info += p.counts.info;
+    }
+    site.counts = counts;
+    site.issues = rollupIssues(site.pages);
+    site.score = siteScore(site.pages, site.siteFindings);
+}
+export function rollupIssues(pages) {
+    const map = new Map();
+    for (const page of pages) {
+        const seenHere = new Set();
+        for (const f of page.findings) {
+            if (seenHere.has(f.rule))
+                continue;
+            seenHere.add(f.rule);
+            let entry = map.get(f.rule);
+            if (!entry) {
+                entry = { rule: f.rule, severity: f.severity, category: f.category, pages: 0, message: f.message, recommendation: f.recommendation, examples: [], urls: new Set() };
+                map.set(f.rule, entry);
+            }
+            entry.pages++;
+            entry.urls.add(page.url);
+        }
+    }
+    const rank = { error: 0, warning: 1, info: 2 };
+    return [...map.values()]
+        .sort((a, b) => rank[a.severity] - rank[b.severity] || b.pages - a.pages)
+        .map(({ urls, ...rest }) => ({ ...rest, examples: [...urls].slice(0, 10) }));
+}
+function shapeSite(site, options) {
+    if (options.pageDetail === 'summary') {
+        site.pages = site.pages.map((p) => ({ ...p, findings: [], summary: { ...p.summary, headings: [], openGraph: {} } }));
+    }
+    return site;
+}
+async function readStdin() {
+    const chunks = [];
+    for await (const chunk of process.stdin)
+        chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+    return Buffer.concat(chunks).toString('utf8');
+}
+/** Drop bulky, low-value fields to save tokens when the result is consumed by an LLM. */
+export function compactAudit(audit) {
+    const trimFinding = (f) => {
+        if (!f.evidence)
+            return f;
+        const evidence = {};
+        for (const [k, v] of Object.entries(f.evidence))
+            evidence[k] = Array.isArray(v) ? v.slice(0, 3) : v;
+        return { ...f, evidence };
+    };
+    const trimPage = (p) => {
+        const { performance, rendering, ...rest } = p;
+        const out = {
+            ...rest,
+            summary: { ...p.summary, headings: p.summary.headings.slice(0, 10), openGraph: {} },
+            findings: p.findings.map(trimFinding),
+        };
+        if (p.response)
+            out.response = { ...p.response, headers: {} };
+        if (performance)
+            out.performance = { ...performance, opportunities: performance.opportunities.slice(0, 5), agenticBrowsingAudits: [], failedSeoAudits: performance.failedSeoAudits.slice(0, 5) };
+        if (rendering)
+            out.rendering = { ...rendering, consoleErrors: rendering.consoleErrors.slice(0, 3) };
+        return out;
+    };
+    if (audit.kind === 'page')
+        return trimPage(audit);
+    return {
+        ...audit,
+        robotsTxt: audit.robotsTxt ? { ...audit.robotsTxt, text: null } : null,
+        sitemaps: { ...audit.sitemaps, urls: audit.sitemaps.urls.slice(0, 20) },
+        pages: audit.pages.map(trimPage),
+        siteFindings: audit.siteFindings.map(trimFinding),
+    };
+}
+//# sourceMappingURL=audit.js.map

package/dist/core/audit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../src/core/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAmF,MAAM,YAAY,CAAC;AACxH,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,aAAa,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,YAAY,EAAmB,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EACL,aAAa,EACb,kBAAkB,GASnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AA8CrC,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,aAAa,EAAE,GAAG,EAAW,CAAC;AAEpH,4EAA4E;AAC5E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY,EAAE,QAAwB,EAAE,EAAE,UAAwB,EAAE;IAClG,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IAChE,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC;IACtC,MAAM,UAAU,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;IAC5E,IAAI,QAAQ,GAAG,YAAY,CACzB,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,EACpG,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CACjD,CAAC;IACF,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAc;QACvB,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,MAAM;QACZ,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,GAAG;QAC3B,GAAG,EAAE,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,GAAG;QACpC,MAAM;QACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC;QAC3C,QAAQ;QACR,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE,CAAC;KACT,CAAC;IACF,IAAI,KAAK,CAAC,QAAQ;QAAE,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;IACpD,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;IACzD,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,GAAW,EAAE,UAAwB,EAAE;IACpE,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,OAAO,CAAC,CAAC;IACpH,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QACd,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;YACrB,IAAI,EAAE,uBAAuB;YAC7B,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,UAAU;YACpB,OAAO,EAAE,mBAAmB,GAAG,KAAK,GAAG,CAAC,KAAK,EAAE;YAC/C,cAAc,EAAE,qIAAqI;YACrJ,GAAG;YACH,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,EAAE;SACjE,CAAC,CAAC;QACH,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACtC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,2CAA2C;AAC3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAwB,EAAE;IAC1E,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,IAAI,mBAAmB,CAAC;IACpD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IAChG,OAAO,SAAS,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;AAC7E,CAAC;AAED,gEAAgE;AAChE,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAwB,EAAE;IAC1E,MAAM,SAAS,GAAiB,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC;IAC7D,SAAS,CAAC,SAAS,KAAK,kBAAkB,CAAC;IAC3C,MAAM,QAAQ,GAAyB;QACrC,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;QACpD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,aAAa,CAAC,WAAW;QAC7D,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,aAAa,CAAC,OAAO;QACjD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,IAAI;QAC5C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,IAAI,KAAK;QACvD,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,aAAa,CAAC,aAAa;QACnE,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,eAAe,EAAE,CAAC,OAAO,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;QAC1E,KAAK,EAAE,SAAS;QAChB,UAAU,EAAE,OAAO,CAAC,UAAU;KAC/B,CAAC;IACF,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,sEAAsE;IACtE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACzE,IAAI,SAAS,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACxD,MAAM,UAAU,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACrC,aAAa,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED,mEAAmE;AACnE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,UAAwB,EAAE;IAC1E,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,mBAAmB;QAC/C,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,IAAI,KAAK;QACrD,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS;KACpC,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IACrE,OAAO,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAClC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAc,EAAE,UAAkE,EAAE;IACpH,IAAI,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QACtD,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,IAAI,CAAC,MAAM,SAAS,EAAE,CAAC,CAAC;QACtD,OAAO,SAAS,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,CAAC;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrG,IAAI,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,4BAA4B,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACzF,MAAM,GAAG,GAAG,OAAO,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,MAAM,EAAE,CAAC;QACvE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAC1E,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,oCAAoC,CAAC,CAAC;IAClF,IAAI,IAAI,CAAC,WAAW,EAAE;QAAE,OAAO,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,KAAgB,EAAE,OAAqB;IAC/D,MAAM,GAAG,GAAG,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,GAAG,CAAC;IAClD,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,GAAG,kBAAkB,EAAE,CAAC,CAAC;QACvF,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,IAAI,CAAC,EAAE,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC7K,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;YACrH,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,iBAAiB,EAAE,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;YAC3J,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,aAAa,CAAC,OAAO,EAAE,YAAY,EAAE,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,aAAa,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjI,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,yBAAyB,EAAE,GAAG,YAAY,sBAAsB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,qBAAsB,GAAa,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/J,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,uBAAuB,OAAO,CAAC,UAAU,IAAI,QAAQ,QAAQ,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3H,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;YAC5J,KAAK,CAAC,WAAW,GAAG,IAAI,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,6BAA6B,EAAE,GAAG,YAAY,0BAA0B,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAuB,GAAa,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QACxK,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACpG,KAAK,CAAC,QAAQ,GAAG,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,QAAQ,CAAC,CAAC,CAAC;QAChE,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC7C,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,4FAA4F;AAC5F,KAAK,UAAU,UAAU,CAAC,KAAgB,EAAE,OAAqB;IAC/D,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACxF,OAAO,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,WAAW,CAAC,IAAY,EAAE,OAAe,EAAE,GAAW;IAC7D,OAAO;QACL,IAAI;QACJ,QAAQ,EAAE,MAAM;QAChB,QAAQ,EAAE,MAAM;QAChB,OAAO;QACP,cAAc,EAAE,8GAA8G;QAC9H,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAmB,EAAE,MAA6B,EAAE,MAAc,EAAE,OAAqB;IACrH,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,wBAAwB,MAAM,CAAC,OAAO,CAAC,MAAM,QAAQ,EAAE,CAAC,CAAC;IACzG,MAAM,UAAU,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;IAC5E,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC1F,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,YAAY,GAAG,YAAY,CAAC;QAC9B,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,OAAO;QACrB,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,eAAe,EAAE,MAAM,CAAC,eAAe;QACvC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU;QACV,QAAQ,EAAE,MAAM,KAAK,WAAW;QAChC,aAAa,EAAE,MAAM,CAAC,aAAa;KACpC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;IACrD,YAAY,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK;QAAE,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAEpG,MAAM,IAAI,GAAc;QACtB,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,MAAM;QACZ,MAAM;QACN,MAAM;QACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;QAC1E,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;QACjF,YAAY;QACZ,MAAM,EAAE,EAAE;QACV,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;QACzC,KAAK,EAAE,CAAC;KACT,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,MAAkB,EAAE,MAAkB,EAAE,OAAqB;IACtF,MAAM,UAAU,GAAG,EAAE,GAAG,kBAAkB,EAAE,GAAG,CAAC,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,CAAC;IAC5E,IAAI,QAAQ,GAAG,YAAY,CACzB,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,EACnH,EAAE,OAAO,EAAE,IAAI,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,EAAE,CACjD,CAAC;IACF,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK;QAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAc;QACvB,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,MAAM;QACZ,MAAM,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,YAAY;QAC1C,GAAG,EAAE,MAAM,CAAC,IAAI,IAAI,MAAM,CAAC,GAAG;QAC9B,MAAM;QACN,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,OAAO,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,QAAQ,CAAC;QACnD,QAAQ;QACR,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC;QAC/B,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC;IACF,IAAI,MAAM,CAAC,QAAQ;QAAE,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACtD,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAe;IAC3C,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAChD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;QAC3B,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QAC/B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC;QACnC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;IAC/B,CAAC;IACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACrB,IAAI,CAAC,MAAM,GAAG,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAAkB;IAC7C,MAAM,GAAG,GAAG,IAAI,GAAG,EAA+C,CAAC;IACnE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;QACnC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9B,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;gBAAE,SAAS;YACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrB,IAAI,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC5B,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,KAAK,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;gBACpK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;YACD,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAW,CAAC;IACxD,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;SACrB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;SACxE,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AACjF,CAAC;AAED,SAAS,SAAS,CAAC,IAAe,EAAE,OAAqB;IACvD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QACrC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IACvH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,SAAS;IACtB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1G,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,YAAY,CAAkB,KAAQ;IACpD,MAAM,WAAW,GAAG,CAAC,CAAU,EAAW,EAAE;QAC1C,IAAI,CAAC,CAAC,CAAC,QAAQ;YAAE,OAAO,CAAC,CAAC;QAC1B,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;YAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpG,OAAO,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC;IACF,MAAM,QAAQ,GAAG,CAAC,CAAY,EAAa,EAAE;QAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,IAAI,EAAE,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAc;YACrB,GAAG,IAAI;YACP,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE;YACnF,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,WAAW,CAAC;SACtC,CAAC;QACF,IAAI,CAAC,CAAC,QAAQ;YAAE,GAAG,CAAC,QAAQ,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QAC9D,IAAI,WAAW;YAAE,GAAG,CAAC,WAAW,GAAG,EAAE,GAAG,WAAW,EAAE,aAAa,EAAE,WAAW,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,qBAAqB,EAAE,EAAE,EAAE,eAAe,EAAE,WAAW,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjM,IAAI,SAAS;YAAE,GAAG,CAAC,SAAS,GAAG,EAAE,GAAG,SAAS,EAAE,aAAa,EAAE,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACpG,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,QAAQ,CAAC,KAAK,CAAM,CAAC;IACvD,OAAO;QACL,GAAG,KAAK;QACR,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;QACtE,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;QACvE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC;QAChC,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC;KAC7C,CAAC;AACT,CAAC"}

package/dist/core/concurrency.d.ts

@@ -0,0 +1,4 @@
+/** Minimal promise pool. */
+export declare function createPool(concurrency: number): <T>(task: () => Promise<T>) => Promise<T>;
+export declare function mapConcurrent<T, R>(items: T[], concurrency: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]>;
+export declare function sleep(ms: number): Promise<void>;

package/dist/core/concurrency.js

@@ -0,0 +1,29 @@
+/** Minimal promise pool. */
+export function createPool(concurrency) {
+    let active = 0;
+    const queue = [];
+    const next = () => {
+        active--;
+        queue.shift()?.();
+    };
+    return function run(task) {
+        return new Promise((resolve, reject) => {
+            const start = () => {
+                active++;
+                task().then(resolve, reject).finally(next);
+            };
+            if (active < Math.max(1, concurrency))
+                start();
+            else
+                queue.push(start);
+        });
+    };
+}
+export async function mapConcurrent(items, concurrency, fn) {
+    const run = createPool(concurrency);
+    return Promise.all(items.map((item, i) => run(() => fn(item, i))));
+}
+export function sleep(ms) {
+    return new Promise((r) => setTimeout(r, ms));
+}
+//# sourceMappingURL=concurrency.js.map

package/dist/core/concurrency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../src/core/concurrency.ts"],"names":[],"mappings":"AAAA,4BAA4B;AAC5B,MAAM,UAAU,UAAU,CAAC,WAAmB;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,KAAK,GAAmB,EAAE,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,MAAM,EAAE,CAAC;QACT,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;IACpB,CAAC,CAAC;IACF,OAAO,SAAS,GAAG,CAAI,IAAsB;QAC3C,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,MAAM,KAAK,GAAG,GAAG,EAAE;gBACjB,MAAM,EAAE,CAAC;gBACT,IAAI,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7C,CAAC,CAAC;YACF,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;gBAAE,KAAK,EAAE,CAAC;;gBAC1C,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAO,KAAU,EAAE,WAAmB,EAAE,EAA0C;IACnH,MAAM,GAAG,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IACpC,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;AAC/C,CAAC"}

package/dist/core/crawl.d.ts

@@ -0,0 +1,40 @@
+import { type FetchOptions } from './fetch.js';
+import type { LinkCheck, PageRecord, ProbeResults } from './site-rules.js';
+import type { CrawlStats, RobotsTxtInfo, SitemapInfo } from './types.js';
+export interface ProgressEvent {
+    phase: 'robots' | 'sitemap' | 'crawl' | 'links' | 'probes' | 'rules' | 'render' | 'lighthouse';
+    message: string;
+    crawled?: number;
+    discovered?: number;
+    url?: string;
+}
+export interface CrawlOptions {
+    maxPages: number;
+    maxDepth: number;
+    concurrency: number;
+    delayMs: number;
+    respectRobots: boolean;
+    includeSubdomains: boolean;
+    checkExternalLinks: boolean;
+    maxLinkChecks: number;
+    includePatterns: RegExp[];
+    excludePatterns: RegExp[];
+    fetch: FetchOptions;
+    onProgress?: (event: ProgressEvent) => void;
+}
+export interface CrawlResult {
+    startUrl: string;
+    records: PageRecord[];
+    linkChecks: Map<string, LinkCheck>;
+    linkSources: Map<string, Set<string>>;
+    externalChecks?: Map<string, LinkCheck>;
+    externalSources?: Map<string, Set<string>>;
+    robots: RobotsTxtInfo | null;
+    robotsAllows: ((url: string) => boolean) | null;
+    blockedByRobots: string[];
+    sitemaps: SitemapInfo;
+    probes: ProbeResults;
+    stats: CrawlStats;
+    unlinkedPages?: string[];
+}
+export declare function crawlSite(startInput: string, options: CrawlOptions): Promise<CrawlResult>;

package/dist/core/crawl.js

@@ -0,0 +1,195 @@
+import { extractPage } from './html/extract.js';
+import { DEFAULT_USER_AGENT, fetchPage, isHtmlResponse, probeUrl, toResponseInfo } from './fetch.js';
+import { loadRobotsTxt } from './robots.js';
+import { loadSitemaps } from './sitemap.js';
+import { isLocalHost } from './rules/canonical.js';
+import { hostOf, isSameSite, looksLikeHtml, normalizeUrl, stripWww } from './url.js';
+import { mapConcurrent, sleep } from './concurrency.js';
+export async function crawlSite(startInput, options) {
+    const started = performance.now();
+    const progress = options.onProgress ?? (() => undefined);
+    const startUrl = normalizeUrl(startInput) ?? startInput;
+    const origin = new URL(startUrl).origin;
+    const startHost = hostOf(startUrl) ?? '';
+    const userAgent = options.fetch.userAgent ?? DEFAULT_USER_AGENT;
+    progress({ phase: 'robots', message: `Fetching ${origin}/robots.txt` });
+    const robots = await loadRobotsTxt(origin, userAgent, options.fetch);
+    const allows = options.respectRobots ? robots.allows : () => true;
+    progress({ phase: 'sitemap', message: 'Fetching sitemaps' });
+    const sitemapCandidates = [...new Set([...robots.info.sitemaps, new URL('/sitemap.xml', origin).href])];
+    const sitemaps = await loadSitemaps(sitemapCandidates, options.fetch);
+    const visited = new Set([startUrl]);
+    const discovered = new Set([startUrl]);
+    const linkChecks = new Map();
+    const linkSources = new Map();
+    const externalSources = new Map();
+    const records = [];
+    const blocked = [];
+    let limitSkips = 0;
+    const accepts = (url) => {
+        if (options.includePatterns.length && !options.includePatterns.some((re) => re.test(url)))
+            return false;
+        if (options.excludePatterns.some((re) => re.test(url)))
+            return false;
+        return true;
+    };
+    const queue = [{ url: startUrl, depth: 0 }];
+    const processItem = async (item) => {
+        if (!allows(item.url)) {
+            blocked.push(item.url);
+            return;
+        }
+        const res = await fetchPage(item.url, options.fetch);
+        const requestedKey = item.url;
+        const finalKey = normalizeUrl(res.finalUrl) ?? res.finalUrl;
+        linkChecks.set(requestedKey, { url: item.url, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
+        if (finalKey !== requestedKey) {
+            if (!linkChecks.has(finalKey))
+                linkChecks.set(finalKey, { url: res.finalUrl, status: res.status, finalUrl: res.finalUrl, redirectChain: [], error: res.error });
+            if (visited.has(finalKey) && records.some((r) => r.url === finalKey))
+                return;
+            visited.add(finalKey);
+            if (!isSameSite(finalKey, startHost, options.includeSubdomains))
+                return; // redirected off-site
+        }
+        progress({ phase: 'crawl', message: `${res.status ?? 'ERR'} ${res.finalUrl}`, crawled: records.length + 1, discovered: discovered.size, url: res.finalUrl });
+        if (res.error || typeof res.body !== 'string' || !isHtmlResponse(res)) {
+            if (res.status !== null && res.status < 400 && !res.error && typeof res.body === 'string' && !isHtmlResponse(res))
+                return; // non-HTML asset
+            // Record error pages so page-level findings (response.error-status) appear for the start URL.
+            if (item.depth === 0 || res.status === null) {
+                const page = extractPage(res.body ?? '', { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
+                records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
+            }
+            return;
+        }
+        const page = extractPage(res.body, { url: res.finalUrl, siteHost: startHost, includeSubdomains: options.includeSubdomains });
+        records.push({ url: finalKey, requestedUrl: item.url, page, response: toResponseInfo(res), depth: item.depth });
+        for (const link of page.links) {
+            if (!link.url)
+                continue;
+            const target = normalizeUrl(link.url);
+            if (!target)
+                continue;
+            if (!link.internal) {
+                if (!externalSources.has(target))
+                    externalSources.set(target, new Set());
+                externalSources.get(target).add(finalKey);
+                continue;
+            }
+            if (!linkSources.has(target))
+                linkSources.set(target, new Set());
+            linkSources.get(target).add(finalKey);
+            if (!looksLikeHtml(target) || !accepts(target))
+                continue;
+            if (discovered.has(target))
+                continue;
+            discovered.add(target);
+            if (item.depth + 1 > options.maxDepth)
+                continue;
+            if (visited.size >= options.maxPages) {
+                limitSkips++;
+                continue;
+            }
+            visited.add(target);
+            queue.push({ url: target, depth: item.depth + 1 });
+        }
+    };
+    await runQueue(queue, options.concurrency, options.delayMs, processItem);
+    /* Check internal link targets that were discovered but not crawled (limits, depth, non-HTML). */
+    progress({ phase: 'links', message: 'Checking internal link targets' });
+    const toCheck = [...linkSources.keys()].filter((u) => !linkChecks.has(u) && allows(u)).slice(0, options.maxLinkChecks);
+    const sitemapExtra = sitemaps.urls
+        .map((u) => normalizeUrl(u) ?? u)
+        .filter((u) => isSameSite(u, startHost, options.includeSubdomains) && !linkChecks.has(u) && !toCheck.includes(u) && allows(u))
+        .slice(0, Math.max(0, options.maxLinkChecks - toCheck.length));
+    for (const u of [...linkSources.keys()].filter((u) => !allows(u)))
+        if (!blocked.includes(u))
+            blocked.push(u);
+    await mapConcurrent([...toCheck, ...sitemapExtra], options.concurrency, async (u) => {
+        const res = await probeUrl(u, options.fetch);
+        linkChecks.set(u, { url: u, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
+        if (options.delayMs)
+            await sleep(options.delayMs);
+    });
+    let externalChecks;
+    if (options.checkExternalLinks) {
+        progress({ phase: 'links', message: `Checking ${Math.min(externalSources.size, options.maxLinkChecks)} external links` });
+        externalChecks = new Map();
+        const targets = [...externalSources.keys()].slice(0, options.maxLinkChecks);
+        await mapConcurrent(targets, Math.min(options.concurrency, 4), async (u) => {
+            const res = await probeUrl(u, { ...options.fetch, timeoutMs: Math.min(options.fetch.timeoutMs ?? 15_000, 10_000) });
+            externalChecks.set(u, { url: u, status: res.status, finalUrl: res.finalUrl, redirectChain: res.redirectChain, error: res.error });
+        });
+    }
+    progress({ phase: 'probes', message: 'Probing soft-404 and host redirects' });
+    const probes = await runProbes(startUrl, options.fetch);
+    const stats = {
+        pagesCrawled: records.length,
+        pagesDiscovered: discovered.size,
+        linksChecked: linkChecks.size + (externalChecks?.size ?? 0),
+        maxPages: options.maxPages,
+        maxDepth: options.maxDepth,
+        hitPageLimit: limitSkips > 0,
+        durationMs: Math.round(performance.now() - started),
+        respectedRobots: options.respectRobots,
+    };
+    return {
+        startUrl,
+        records,
+        linkChecks,
+        linkSources,
+        externalChecks,
+        externalSources: options.checkExternalLinks ? externalSources : undefined,
+        robots: robots.info,
+        robotsAllows: robots.info.found ? robots.allows : null,
+        blockedByRobots: blocked,
+        sitemaps,
+        probes,
+        stats,
+    };
+}
+async function runQueue(queue, concurrency, delayMs, work) {
+    let active = 0;
+    await new Promise((resolve) => {
+        const pump = () => {
+            while (active < Math.max(1, concurrency) && queue.length) {
+                const item = queue.shift();
+                active++;
+                work(item)
+                    .catch(() => undefined)
+                    .finally(() => {
+                    active--;
+                    if (delayMs > 0)
+                        setTimeout(pump, delayMs);
+                    else
+                        pump();
+                });
+            }
+            if (active === 0 && queue.length === 0)
+                resolve();
+        };
+        pump();
+    });
+}
+async function runProbes(startUrl, fetchOpts) {
+    const probes = {};
+    const url = new URL(startUrl);
+    const soft404Url = new URL(`/agentic-seo-probe-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`, url.origin).href;
+    const soft = await probeUrl(soft404Url, fetchOpts);
+    probes.soft404 = soft.error ? null : { url: soft404Url, status: soft.status };
+    if (isLocalHost(startUrl))
+        return probes;
+    if (url.protocol === 'https:') {
+        const httpUrl = `http://${url.host}/`;
+        const res = await probeUrl(httpUrl, { ...fetchOpts, timeoutMs: 10_000 });
+        probes.httpRedirect = res.error ? null : { url: httpUrl, status: res.status, finalUrl: res.finalUrl, redirectsToHttps: res.finalUrl.startsWith('https://') };
+    }
+    const host = url.hostname;
+    const variantHost = host.startsWith('www.') ? stripWww(host) : `www.${host}`;
+    const variantUrl = `${url.protocol}//${variantHost}/`;
+    const res = await probeUrl(variantUrl, { ...fetchOpts, timeoutMs: 10_000 });
+    probes.hostVariant = res.error ? null : { url: variantUrl, status: res.status, finalUrl: res.finalUrl, redirectsToMain: (hostOf(res.finalUrl) ?? '') === host };
+    return probes;
+}
+//# sourceMappingURL=crawl.js.map