@forkpoint/agent-lighthouse-core 3.0.0 → 3.1.0

package/dist/index.d.mts

@@ -1,3 +1,4 @@
+import { Dispatcher } from 'undici';
 import { CheerioAPI } from 'cheerio';
 import { z } from 'zod';
 
@@ -61,7 +62,44 @@ interface FetchResult {
  * Use before fetching any URL extracted from scanned site content.
  */
 declare function isSafeUrl(url: string): Promise<boolean>;
-declare function createFetcher(): {
+interface FetcherOptions {
+    /**
+     * undici dispatcher for every request this fetcher makes.
+     *
+     * Omitted, requests go through a shared `new Agent()` whose per-origin
+     * connection count is unlimited — right for a site owner scanning their own
+     * site, who wants the scan over quickly. A caller scanning origins that did
+     * not invite it passes a bounded agent (`new Agent({ connections: 2 })`) so
+     * one scan cannot open 28 sockets at once against a stranger's WAF.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this fetcher may have in flight at once.
+     *
+     * Omitted, there is no ceiling and every request is issued as it arrives.
+     *
+     * A bounded dispatcher alone is not enough for a caller that wants one, and
+     * the difference is what the clocks measure. `Agent({ connections: 2 })`
+     * accepts all 26 root-file requests the scan fires in one `Promise.all` and
+     * queues 24 of them inside undici — but the per-request deadline and the
+     * `ttfbMs` clock both start when `fetch()` is called, so an origin averaging
+     * 800 ms per file would have its tail requests time out on the scanner's own
+     * queue and be reported as unreachable. Waiting here instead holds a request
+     * outside both clocks until it can actually be issued, so what they measure
+     * is the origin.
+     */
+    maxConcurrent?: number;
+}
+/**
+ * A dispatcher that opens at most `connections` sockets per origin.
+ *
+ * The scanner's own default has no such ceiling, and for a site owner scanning
+ * their own site that is the right trade. A caller scanning origins that did
+ * not invite it wants the ceiling, and this saves it from taking a direct
+ * dependency on undici to express one line of politeness.
+ */
+declare function boundedDispatcher(connections: number): Dispatcher;
+declare function createFetcher(fetcherOptions?: FetcherOptions): {
     fetch: (options: FetchOptions) => Promise<FetchResult>;
 };
 
@@ -593,9 +631,12 @@ interface PlanOptions {
     /**
      * Skip audits whose `requires` the scan did not obtain.
      *
-     * Off by default while the gate is calibrated: turning it on changes what
-     * every blocked or client-rendered scan reports, and that lands as one
-     * deliberate change rather than as a side effect of this one.
+     * The orchestrator turns this on for every scan
+     * (`enforceEvidenceGate ?? true`), so an audit's `requires` decides what a
+     * blocked or client-rendered scan reports. It defaults to off in this option
+     * bag only so a caller planning audits directly gets the unfiltered set;
+     * passing `false` is the measurement escape hatch for comparing a gated run
+     * against an ungated one.
      */
     enforceEvidence?: boolean;
 }
@@ -720,6 +761,39 @@ interface ScanOptions {
      * useful when measuring what the gate removes.
      */
     enforceEvidenceGate?: boolean;
+    /**
+     * undici dispatcher for every request this scan makes.
+     *
+     * Omitted, the scan uses the fetcher's shared agent, which opens as many
+     * connections per origin as the scan asks for. A caller scanning origins that
+     * did not invite it — the nightly corpus job — passes a bounded agent so the
+     * ~28 root-file requests the scan issues at once do not arrive at a stranger's
+     * WAF as a burst.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this scan may have in flight at once.
+     *
+     * Omitted, the scan issues every request as it reaches the fetcher. A caller
+     * scanning origins that did not invite it sets this alongside `dispatcher`:
+     * the dispatcher bounds the sockets, and this bounds what is issued, so the
+     * requests the scan holds back wait outside the per-request deadline and
+     * outside `ttfbMs` instead of inside undici's queue. Without it a bounded
+     * scan reports its own queueing as the origin being slow or unreachable.
+     */
+    maxConcurrent?: number;
+    /**
+     * A `robots.txt` response the caller already has, used instead of fetching
+     * it again.
+     *
+     * For a caller that must read `robots.txt` before it decides to scan at all.
+     * The nightly corpus job is one: it asks permission first, and without this
+     * every site it visits gets two requests for the one file its owner watches.
+     * It must be the response to `<baseUrl>/robots.txt` fetched with this
+     * scanner's own user agent — anything else and the scan judges a file it was
+     * not served.
+     */
+    robotsTxt?: FetchResult;
 }
 declare function runScan(url: string, options?: ScanOptions): Promise<ScanReport>;
 
@@ -1516,4 +1590,4 @@ type AcpLinkType = (typeof ACP_LINK_TYPES)[number];
  */
 declare function resolvePolicyLinks(ctx: CheckContext): Map<AcpLinkType, string>;
 
-export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type EvidenceKey, EvidenceKeySchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanEvidence, type ScanOptions, type ScanReport, type ScanValidity, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_NO_EVIDENCE, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allEvidenceMet, allJsonLdNodes, buildCategoryResult, buildScanEvidence, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getRenderedText, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };
+export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type EvidenceKey, EvidenceKeySchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FetcherOptions, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanEvidence, type ScanOptions, type ScanReport, type ScanValidity, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_NO_EVIDENCE, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allEvidenceMet, allJsonLdNodes, boundedDispatcher, buildCategoryResult, buildScanEvidence, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getRenderedText, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+import { Dispatcher } from 'undici';
 import { CheerioAPI } from 'cheerio';
 import { z } from 'zod';
 
@@ -61,7 +62,44 @@ interface FetchResult {
  * Use before fetching any URL extracted from scanned site content.
  */
 declare function isSafeUrl(url: string): Promise<boolean>;
-declare function createFetcher(): {
+interface FetcherOptions {
+    /**
+     * undici dispatcher for every request this fetcher makes.
+     *
+     * Omitted, requests go through a shared `new Agent()` whose per-origin
+     * connection count is unlimited — right for a site owner scanning their own
+     * site, who wants the scan over quickly. A caller scanning origins that did
+     * not invite it passes a bounded agent (`new Agent({ connections: 2 })`) so
+     * one scan cannot open 28 sockets at once against a stranger's WAF.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this fetcher may have in flight at once.
+     *
+     * Omitted, there is no ceiling and every request is issued as it arrives.
+     *
+     * A bounded dispatcher alone is not enough for a caller that wants one, and
+     * the difference is what the clocks measure. `Agent({ connections: 2 })`
+     * accepts all 26 root-file requests the scan fires in one `Promise.all` and
+     * queues 24 of them inside undici — but the per-request deadline and the
+     * `ttfbMs` clock both start when `fetch()` is called, so an origin averaging
+     * 800 ms per file would have its tail requests time out on the scanner's own
+     * queue and be reported as unreachable. Waiting here instead holds a request
+     * outside both clocks until it can actually be issued, so what they measure
+     * is the origin.
+     */
+    maxConcurrent?: number;
+}
+/**
+ * A dispatcher that opens at most `connections` sockets per origin.
+ *
+ * The scanner's own default has no such ceiling, and for a site owner scanning
+ * their own site that is the right trade. A caller scanning origins that did
+ * not invite it wants the ceiling, and this saves it from taking a direct
+ * dependency on undici to express one line of politeness.
+ */
+declare function boundedDispatcher(connections: number): Dispatcher;
+declare function createFetcher(fetcherOptions?: FetcherOptions): {
     fetch: (options: FetchOptions) => Promise<FetchResult>;
 };
 
@@ -593,9 +631,12 @@ interface PlanOptions {
     /**
      * Skip audits whose `requires` the scan did not obtain.
      *
-     * Off by default while the gate is calibrated: turning it on changes what
-     * every blocked or client-rendered scan reports, and that lands as one
-     * deliberate change rather than as a side effect of this one.
+     * The orchestrator turns this on for every scan
+     * (`enforceEvidenceGate ?? true`), so an audit's `requires` decides what a
+     * blocked or client-rendered scan reports. It defaults to off in this option
+     * bag only so a caller planning audits directly gets the unfiltered set;
+     * passing `false` is the measurement escape hatch for comparing a gated run
+     * against an ungated one.
      */
     enforceEvidence?: boolean;
 }
@@ -720,6 +761,39 @@ interface ScanOptions {
      * useful when measuring what the gate removes.
      */
     enforceEvidenceGate?: boolean;
+    /**
+     * undici dispatcher for every request this scan makes.
+     *
+     * Omitted, the scan uses the fetcher's shared agent, which opens as many
+     * connections per origin as the scan asks for. A caller scanning origins that
+     * did not invite it — the nightly corpus job — passes a bounded agent so the
+     * ~28 root-file requests the scan issues at once do not arrive at a stranger's
+     * WAF as a burst.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this scan may have in flight at once.
+     *
+     * Omitted, the scan issues every request as it reaches the fetcher. A caller
+     * scanning origins that did not invite it sets this alongside `dispatcher`:
+     * the dispatcher bounds the sockets, and this bounds what is issued, so the
+     * requests the scan holds back wait outside the per-request deadline and
+     * outside `ttfbMs` instead of inside undici's queue. Without it a bounded
+     * scan reports its own queueing as the origin being slow or unreachable.
+     */
+    maxConcurrent?: number;
+    /**
+     * A `robots.txt` response the caller already has, used instead of fetching
+     * it again.
+     *
+     * For a caller that must read `robots.txt` before it decides to scan at all.
+     * The nightly corpus job is one: it asks permission first, and without this
+     * every site it visits gets two requests for the one file its owner watches.
+     * It must be the response to `<baseUrl>/robots.txt` fetched with this
+     * scanner's own user agent — anything else and the scan judges a file it was
+     * not served.
+     */
+    robotsTxt?: FetchResult;
 }
 declare function runScan(url: string, options?: ScanOptions): Promise<ScanReport>;
 
@@ -1516,4 +1590,4 @@ type AcpLinkType = (typeof ACP_LINK_TYPES)[number];
  */
 declare function resolvePolicyLinks(ctx: CheckContext): Map<AcpLinkType, string>;
 
-export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type EvidenceKey, EvidenceKeySchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanEvidence, type ScanOptions, type ScanReport, type ScanValidity, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_NO_EVIDENCE, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allEvidenceMet, allJsonLdNodes, buildCategoryResult, buildScanEvidence, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getRenderedText, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };
+export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type EvidenceKey, EvidenceKeySchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FetcherOptions, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanEvidence, type ScanOptions, type ScanReport, type ScanValidity, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_NO_EVIDENCE, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allEvidenceMet, allJsonLdNodes, boundedDispatcher, buildCategoryResult, buildScanEvidence, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getRenderedText, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };