aeorank 1.3.0 → 1.4.0

package/dist/index.d.cts

@@ -35,7 +35,7 @@ interface PitchMetric {
     value: string;
     significance: string;
 }
-type PageCategory$1 = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'content';
+type PageCategory$1 = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'faq' | 'content';
 interface PageIssue$1 {
     check: string;
     label: string;
@@ -84,6 +84,12 @@ interface AuditOptions {
     noMultiPage?: boolean;
     /** Fetch timeout in ms (default: 15000) */
     timeout?: number;
+    /** Enable full-site BFS crawl (default: false) */
+    fullCrawl?: boolean;
+    /** Max pages for full crawl (default: 200) */
+    maxPages?: number;
+    /** Parallel fetch concurrency for full crawl (default: 5) */
+    concurrency?: number;
 }
 interface AuditResult extends AuditData {
     /** True if headless browser was used for SPA rendering */
@@ -111,7 +117,7 @@ interface CriterionResult {
     findings: AuditFinding[];
     fix_priority: Priority;
 }
-type PageCategory = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'content';
+type PageCategory = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'faq' | 'content';
 interface FetchResult {
     text: string;
     status: number;
@@ -134,6 +140,13 @@ interface SiteData {
     parkedReason: string | null;
     /** Sampled blog/content pages from sitemap (up to 5) */
     blogSample?: FetchResult[];
+    /** Full-crawl statistics (set when --full-crawl is used) */
+    crawlStats?: {
+        discovered: number;
+        fetched: number;
+        skipped: number;
+        elapsed: number;
+    };
 }
 interface RawDataSummary {
     domain: string;
@@ -203,6 +216,9 @@ interface RawDataSummary {
     has_visible_date: boolean;
     has_schema_date_in_ld: boolean;
     date_modified_recency_days: number | null;
+    crawl_discovered: number;
+    crawl_fetched: number;
+    crawl_skipped: number;
 }
 /**
  * Fetches all site data in parallel with HTTPS/HTTP fallback.
@@ -289,6 +305,51 @@ interface MultiPageOptions {
  */
 declare function fetchMultiPageData(siteData: SiteData, options?: MultiPageOptions): Promise<number>;
 
+/**
+ * Full-site crawler for deep AEO audits.
+ * BFS crawl that discovers all internal pages up to a configurable limit.
+ */
+
+interface CrawlOptions {
+    /** Maximum pages to fetch (default 200) */
+    maxPages?: number;
+    /** Per-page fetch timeout in ms (default 10000) */
+    timeoutMs?: number;
+    /** Parallel fetches (default 5) */
+    concurrency?: number;
+    /** Honor robots.txt Disallow rules (default true) */
+    respectRobots?: boolean;
+    /** Include asset files — skipped by default */
+    includeAssets?: boolean;
+}
+interface CrawlResult {
+    pages: FetchResult[];
+    discoveredUrls: string[];
+    fetchedUrls: string[];
+    skippedUrls: string[];
+    elapsed: number;
+}
+/**
+ * Extract all page URLs from sitemap XML (handles sitemapindex with sub-sitemaps).
+ * Filters to same domain only, skips resource files.
+ */
+declare function extractAllUrlsFromSitemap(sitemapText: string, domain: string, timeoutMs?: number): Promise<string[]>;
+/**
+ * Extract ALL internal links from HTML (not just nav).
+ * Returns deduplicated full URLs for the same domain.
+ */
+declare function extractInternalLinks(html: string, domain: string): string[];
+/**
+ * Infer PageCategory from URL path patterns.
+ */
+declare function inferCategory(url: string): PageCategory;
+/**
+ * BFS crawl of a site, discovering all internal pages up to maxPages.
+ * Seeds from sitemap URLs + homepage internal links.
+ * Skips URLs already in siteData.blogSample and homepage.
+ */
+declare function crawlFullSite(siteData: SiteData, options?: CrawlOptions): Promise<CrawlResult>;
+
 /**
  * SPA detection and headless Chromium rendering for pre-crawl.
  *
@@ -395,4 +456,4 @@ interface ComparisonResult {
  */
 declare function compare(domainA: string, domainB: string, options?: AuditOptions): Promise<ComparisonResult>;
 
-export { type AuditData, type AuditFinding, type AuditOptions, type AuditResult, type AuditStatus, CRITERION_LABELS, type ComparisonResult, type CriterionComparison, type CriterionDetail, type CriterionResult, type Deliverable, type DetailedFinding, type FetchResult, type FindingSeverity, type FindingType, type HeadlessOptions, type ImpactLevel, type PageCategory$1 as PageCategory, type PageIssue$1 as PageIssue, type PageReview$1 as PageReview, type ParkedDomainResult, type PitchMetric, type Priority, type RawDataSummary, type RenderingMethod, type ScoreCardItem, type Severity, type SiteData, type Status, analyzeAllPages, analyzePage, audit, auditSiteFromData, buildDetailedFindings, buildScorecard, calculateOverallScore, classifyRendering, compare, detectParkedDomain, extractContentPagesFromSitemap, extractNavLinks, extractRawDataSummary, fetchMultiPageData, fetchWithHeadless, generateBottomLine, generateComparisonHtmlReport, generateHtmlReport, generateOpportunities, generatePitchNumbers, generateVerdict, isSpaShell, prefetchSiteData, scoreToStatus };
+export { type AuditData, type AuditFinding, type AuditOptions, type AuditResult, type AuditStatus, CRITERION_LABELS, type ComparisonResult, type CrawlOptions, type CrawlResult, type CriterionComparison, type CriterionDetail, type CriterionResult, type Deliverable, type DetailedFinding, type FetchResult, type FindingSeverity, type FindingType, type HeadlessOptions, type ImpactLevel, type PageCategory$1 as PageCategory, type PageIssue$1 as PageIssue, type PageReview$1 as PageReview, type ParkedDomainResult, type PitchMetric, type Priority, type RawDataSummary, type RenderingMethod, type ScoreCardItem, type Severity, type SiteData, type Status, analyzeAllPages, analyzePage, audit, auditSiteFromData, buildDetailedFindings, buildScorecard, calculateOverallScore, classifyRendering, compare, crawlFullSite, detectParkedDomain, extractAllUrlsFromSitemap, extractContentPagesFromSitemap, extractInternalLinks, extractNavLinks, extractRawDataSummary, fetchMultiPageData, fetchWithHeadless, generateBottomLine, generateComparisonHtmlReport, generateHtmlReport, generateOpportunities, generatePitchNumbers, generateVerdict, inferCategory, isSpaShell, prefetchSiteData, scoreToStatus };

package/dist/index.d.ts

@@ -35,7 +35,7 @@ interface PitchMetric {
     value: string;
     significance: string;
 }
-type PageCategory$1 = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'content';
+type PageCategory$1 = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'faq' | 'content';
 interface PageIssue$1 {
     check: string;
     label: string;
@@ -84,6 +84,12 @@ interface AuditOptions {
     noMultiPage?: boolean;
     /** Fetch timeout in ms (default: 15000) */
     timeout?: number;
+    /** Enable full-site BFS crawl (default: false) */
+    fullCrawl?: boolean;
+    /** Max pages for full crawl (default: 200) */
+    maxPages?: number;
+    /** Parallel fetch concurrency for full crawl (default: 5) */
+    concurrency?: number;
 }
 interface AuditResult extends AuditData {
     /** True if headless browser was used for SPA rendering */
@@ -111,7 +117,7 @@ interface CriterionResult {
     findings: AuditFinding[];
     fix_priority: Priority;
 }
-type PageCategory = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'content';
+type PageCategory = 'homepage' | 'blog' | 'about' | 'pricing' | 'services' | 'contact' | 'team' | 'resources' | 'docs' | 'cases' | 'faq' | 'content';
 interface FetchResult {
     text: string;
     status: number;
@@ -134,6 +140,13 @@ interface SiteData {
     parkedReason: string | null;
     /** Sampled blog/content pages from sitemap (up to 5) */
     blogSample?: FetchResult[];
+    /** Full-crawl statistics (set when --full-crawl is used) */
+    crawlStats?: {
+        discovered: number;
+        fetched: number;
+        skipped: number;
+        elapsed: number;
+    };
 }
 interface RawDataSummary {
     domain: string;
@@ -203,6 +216,9 @@ interface RawDataSummary {
     has_visible_date: boolean;
     has_schema_date_in_ld: boolean;
     date_modified_recency_days: number | null;
+    crawl_discovered: number;
+    crawl_fetched: number;
+    crawl_skipped: number;
 }
 /**
  * Fetches all site data in parallel with HTTPS/HTTP fallback.
@@ -289,6 +305,51 @@ interface MultiPageOptions {
  */
 declare function fetchMultiPageData(siteData: SiteData, options?: MultiPageOptions): Promise<number>;
 
+/**
+ * Full-site crawler for deep AEO audits.
+ * BFS crawl that discovers all internal pages up to a configurable limit.
+ */
+
+interface CrawlOptions {
+    /** Maximum pages to fetch (default 200) */
+    maxPages?: number;
+    /** Per-page fetch timeout in ms (default 10000) */
+    timeoutMs?: number;
+    /** Parallel fetches (default 5) */
+    concurrency?: number;
+    /** Honor robots.txt Disallow rules (default true) */
+    respectRobots?: boolean;
+    /** Include asset files — skipped by default */
+    includeAssets?: boolean;
+}
+interface CrawlResult {
+    pages: FetchResult[];
+    discoveredUrls: string[];
+    fetchedUrls: string[];
+    skippedUrls: string[];
+    elapsed: number;
+}
+/**
+ * Extract all page URLs from sitemap XML (handles sitemapindex with sub-sitemaps).
+ * Filters to same domain only, skips resource files.
+ */
+declare function extractAllUrlsFromSitemap(sitemapText: string, domain: string, timeoutMs?: number): Promise<string[]>;
+/**
+ * Extract ALL internal links from HTML (not just nav).
+ * Returns deduplicated full URLs for the same domain.
+ */
+declare function extractInternalLinks(html: string, domain: string): string[];
+/**
+ * Infer PageCategory from URL path patterns.
+ */
+declare function inferCategory(url: string): PageCategory;
+/**
+ * BFS crawl of a site, discovering all internal pages up to maxPages.
+ * Seeds from sitemap URLs + homepage internal links.
+ * Skips URLs already in siteData.blogSample and homepage.
+ */
+declare function crawlFullSite(siteData: SiteData, options?: CrawlOptions): Promise<CrawlResult>;
+
 /**
  * SPA detection and headless Chromium rendering for pre-crawl.
  *
@@ -395,4 +456,4 @@ interface ComparisonResult {
  */
 declare function compare(domainA: string, domainB: string, options?: AuditOptions): Promise<ComparisonResult>;
 
-export { type AuditData, type AuditFinding, type AuditOptions, type AuditResult, type AuditStatus, CRITERION_LABELS, type ComparisonResult, type CriterionComparison, type CriterionDetail, type CriterionResult, type Deliverable, type DetailedFinding, type FetchResult, type FindingSeverity, type FindingType, type HeadlessOptions, type ImpactLevel, type PageCategory$1 as PageCategory, type PageIssue$1 as PageIssue, type PageReview$1 as PageReview, type ParkedDomainResult, type PitchMetric, type Priority, type RawDataSummary, type RenderingMethod, type ScoreCardItem, type Severity, type SiteData, type Status, analyzeAllPages, analyzePage, audit, auditSiteFromData, buildDetailedFindings, buildScorecard, calculateOverallScore, classifyRendering, compare, detectParkedDomain, extractContentPagesFromSitemap, extractNavLinks, extractRawDataSummary, fetchMultiPageData, fetchWithHeadless, generateBottomLine, generateComparisonHtmlReport, generateHtmlReport, generateOpportunities, generatePitchNumbers, generateVerdict, isSpaShell, prefetchSiteData, scoreToStatus };
+export { type AuditData, type AuditFinding, type AuditOptions, type AuditResult, type AuditStatus, CRITERION_LABELS, type ComparisonResult, type CrawlOptions, type CrawlResult, type CriterionComparison, type CriterionDetail, type CriterionResult, type Deliverable, type DetailedFinding, type FetchResult, type FindingSeverity, type FindingType, type HeadlessOptions, type ImpactLevel, type PageCategory$1 as PageCategory, type PageIssue$1 as PageIssue, type PageReview$1 as PageReview, type ParkedDomainResult, type PitchMetric, type Priority, type RawDataSummary, type RenderingMethod, type ScoreCardItem, type Severity, type SiteData, type Status, analyzeAllPages, analyzePage, audit, auditSiteFromData, buildDetailedFindings, buildScorecard, calculateOverallScore, classifyRendering, compare, crawlFullSite, detectParkedDomain, extractAllUrlsFromSitemap, extractContentPagesFromSitemap, extractInternalLinks, extractNavLinks, extractRawDataSummary, fetchMultiPageData, fetchWithHeadless, generateBottomLine, generateComparisonHtmlReport, generateHtmlReport, generateOpportunities, generatePitchNumbers, generateVerdict, inferCategory, isSpaShell, prefetchSiteData, scoreToStatus };

package/dist/index.js

@@ -1,3 +1,10 @@
+import {
+  crawlFullSite,
+  extractAllUrlsFromSitemap,
+  extractInternalLinks,
+  inferCategory
+} from "./chunk-3IJISYWT.js";
+
 // src/parked-domain.ts
 var PARKING_PATHS = ["/lander", "/parking", "/park", "/sedoparking"];
 var PARKING_SERVICE_DOMAINS = [
@@ -1753,7 +1760,11 @@ function extractRawDataSummary(data) {
       const d = new Date(m[1]);
       if (isNaN(d.getTime())) return null;
       return Math.floor((Date.now() - d.getTime()) / (1e3 * 60 * 60 * 24));
-    })()
+    })(),
+    // Full-crawl stats
+    crawl_discovered: data.crawlStats?.discovered ?? 0,
+    crawl_fetched: data.crawlStats?.fetched ?? 0,
+    crawl_skipped: data.crawlStats?.skipped ?? 0
   };
 }
 function auditSiteFromData(data) {
@@ -2749,7 +2760,20 @@ async function audit(domain, options) {
       }
     }
   }
-  if (!options?.noMultiPage) {
+  if (options?.fullCrawl) {
+    const { crawlFullSite: crawlFullSite2 } = await import("./full-site-crawler-VFARFR2C.js");
+    const crawlResult = await crawlFullSite2(siteData, {
+      maxPages: options.maxPages ?? 200,
+      concurrency: options.concurrency ?? 5
+    });
+    siteData.blogSample = crawlResult.pages;
+    siteData.crawlStats = {
+      discovered: crawlResult.discoveredUrls.length,
+      fetched: crawlResult.fetchedUrls.length,
+      skipped: crawlResult.skippedUrls.length,
+      elapsed: crawlResult.elapsed
+    };
+  } else if (!options?.noMultiPage) {
     await fetchMultiPageData(siteData);
   }
   const results = auditSiteFromData(siteData);
@@ -3115,8 +3139,11 @@ export {
   calculateOverallScore,
   classifyRendering,
   compare,
+  crawlFullSite,
   detectParkedDomain,
+  extractAllUrlsFromSitemap,
   extractContentPagesFromSitemap,
+  extractInternalLinks,
   extractNavLinks,
   extractRawDataSummary,
   fetchMultiPageData,
@@ -3127,6 +3154,7 @@ export {
   generateOpportunities,
   generatePitchNumbers,
   generateVerdict,
+  inferCategory,
   isSpaShell,
   prefetchSiteData,
   scoreToStatus