@forkpoint/agent-lighthouse-core 2.0.0 → 3.1.0

package/dist/index.d.ts

@@ -1,3 +1,4 @@
+import { Dispatcher } from 'undici';
 import { CheerioAPI } from 'cheerio';
 import { z } from 'zod';
 
@@ -41,6 +42,19 @@ interface FetchResult {
     contentLength: number;
     /** Raw response bytes. Present only when the request asked for `binary`. */
     bytes?: Uint8Array;
+    /**
+     * Every redirect hop the fetcher walked, in order. Absent when the caller
+     * disabled redirects or the response was not a redirect.
+     *
+     * `finalUrl` alone cannot say whether a host change was permanent. A scan
+     * has to tell a domain migration (301/308) from a temporary hop to somebody
+     * else's domain, so the status of each hop is kept.
+     */
+    redirectChain?: Array<{
+        status: number;
+        from: string;
+        to: string;
+    }>;
     error?: string;
 }
 /**
@@ -48,7 +62,44 @@ interface FetchResult {
  * Use before fetching any URL extracted from scanned site content.
  */
 declare function isSafeUrl(url: string): Promise<boolean>;
-declare function createFetcher(): {
+interface FetcherOptions {
+    /**
+     * undici dispatcher for every request this fetcher makes.
+     *
+     * Omitted, requests go through a shared `new Agent()` whose per-origin
+     * connection count is unlimited — right for a site owner scanning their own
+     * site, who wants the scan over quickly. A caller scanning origins that did
+     * not invite it passes a bounded agent (`new Agent({ connections: 2 })`) so
+     * one scan cannot open 28 sockets at once against a stranger's WAF.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this fetcher may have in flight at once.
+     *
+     * Omitted, there is no ceiling and every request is issued as it arrives.
+     *
+     * A bounded dispatcher alone is not enough for a caller that wants one, and
+     * the difference is what the clocks measure. `Agent({ connections: 2 })`
+     * accepts all 26 root-file requests the scan fires in one `Promise.all` and
+     * queues 24 of them inside undici — but the per-request deadline and the
+     * `ttfbMs` clock both start when `fetch()` is called, so an origin averaging
+     * 800 ms per file would have its tail requests time out on the scanner's own
+     * queue and be reported as unreachable. Waiting here instead holds a request
+     * outside both clocks until it can actually be issued, so what they measure
+     * is the origin.
+     */
+    maxConcurrent?: number;
+}
+/**
+ * A dispatcher that opens at most `connections` sockets per origin.
+ *
+ * The scanner's own default has no such ceiling, and for a site owner scanning
+ * their own site that is the right trade. A caller scanning origins that did
+ * not invite it wants the ceiling, and this saves it from taking a direct
+ * dependency on undici to express one line of politeness.
+ */
+declare function boundedDispatcher(connections: number): Dispatcher;
+declare function createFetcher(fetcherOptions?: FetcherOptions): {
     fetch: (options: FetchOptions) => Promise<FetchResult>;
 };
 
@@ -154,7 +205,25 @@ interface AuditMeta {
     tier?: AuditTier;
     /** Repo-relative path to the audit's evidence dossier. */
     dossier?: string;
+    /**
+     * Which classes of scan evidence this audit needs to say anything true.
+     *
+     * Declared per audit rather than inferred at runtime, and checked against
+     * what the source actually reads by `scripts/check-requires.mjs`. An audit
+     * that reads the sampled pages — directly or through a page-fed gatherer —
+     * needs all four keys; one that reads only root files needs the origin to
+     * have answered. The deliberate disagreements are the audits whose subject
+     * *is* the missing evidence, and they are listed in that script.
+     */
+    requires?: EvidenceKey[];
 }
+/**
+ * A class of evidence a scan either obtained or did not.
+ *
+ * Declared here rather than in `scan-evidence.ts` so `AuditMeta` can name it
+ * without the audit layer importing the scan layer.
+ */
+type EvidenceKey = 'origin-reachable' | 'unblocked-fetches' | 'rendered-body' | 'sample-adequate';
 interface AuditResult {
     status: CheckStatus;
     score: number;
@@ -231,12 +300,31 @@ interface CategoryResult {
     failCount: number;
 }
 type ScoreTier = 'agent-ready' | 'partially-ready' | 'needs-work' | 'not-ready';
+/**
+ * Whether a verdict about this site can mean anything, and what is missing.
+ *
+ * A scan that fetched nothing still produced a number before this existed:
+ * `ridge.com` scored 43 and `westontable.com` 37 on scans that read no page,
+ * against a real store's 51. The numbers overlap, so a reader could not
+ * separate "mediocre site" from "we never saw this site". When `judgeable` is
+ * false the report carries no score at all — not a zero, not a low number.
+ */
+interface ScanValidity {
+    judgeable: boolean;
+    evidence: Record<EvidenceKey, boolean>;
+    reasons: Partial<Record<EvidenceKey, string>>;
+    /** Why the score was suppressed, when it was. */
+    unscoredReason?: string;
+}
 interface ScanReport {
     scanId: string;
     url: string;
     domain: string;
-    overallScore: number;
-    scoreTier: ScoreTier;
+    /** Null when the scan obtained too little evidence to judge the site. */
+    overallScore: number | null;
+    /** Null exactly when `overallScore` is. */
+    scoreTier: ScoreTier | null;
+    scanValidity?: ScanValidity;
     summary?: string;
     categories: CategoryResult[];
     topPasses: CheckResult[];
@@ -269,6 +357,39 @@ interface ReadinessVitals {
     technical: number;
 }
 
+/**
+ * What a scan actually obtained, decided once, before any audit runs.
+ *
+ * An audit that runs on evidence the scan never got reports a verdict about
+ * the scanner, not about the site: "no structured data" when the fetch was
+ * refused, "0 words" when the origin never answered. This module names the
+ * classes of evidence a scan can be missing so those audits can be skipped
+ * with the reason attached instead of answering blind.
+ *
+ * Pure: no network, no IO. It reads what the orchestrator already has.
+ */
+
+interface ScanEvidence {
+    met: Record<EvidenceKey, boolean>;
+    /** One sentence per unmet key, shown in the `na` stub and the trace. */
+    reasons: Partial<Record<EvidenceKey, string>>;
+    /** Page URL to "the served HTML carried text a non-JS consumer can read". */
+    renderedByPage: Record<string, boolean>;
+    usablePageTypes: Set<PageType>;
+    /** False when no verdict about this site can mean anything. */
+    judgeable: boolean;
+}
+interface ScanEvidenceInput {
+    requestedUrl: string;
+    homepageResult: FetchResult;
+    pages: PageContext[];
+    rootFiles: Record<string, FetchResult>;
+    wafProtection: WafProtection | null;
+}
+declare function buildScanEvidence(input: ScanEvidenceInput): ScanEvidence;
+/** All requirements met. For test harnesses not exercising the gate. */
+declare function allEvidenceMet(): ScanEvidence;
+
 interface A11yNodeFinding {
     target: string;
     summary: string;
@@ -316,6 +437,14 @@ interface CheckContext {
     baseUrl: string;
     fetch: (options: FetchOptions) => Promise<FetchResult>;
     wafProtection?: WafProtection;
+    /**
+     * What the scan actually obtained, decided once before any audit ran.
+     *
+     * Required, not optional. An optional field fails open, and a caller that
+     * forgets is exactly the silent-nothing verdict this exists to remove. Test
+     * harnesses that do not exercise the gate pass `allEvidenceMet()`.
+     */
+    evidence: ScanEvidence;
 }
 type CheckFn = (ctx: CheckContext) => CheckResult | Promise<CheckResult>;
 
@@ -441,9 +570,10 @@ interface AuditTrace {
     /**
      * `ran` — the audit executed and returned.
      * `skipped` — no scanned page matched its `applicablePageTypes`.
+     * `gated` — the scan never obtained evidence the audit needs.
      * `error` — it threw, or its result was rejected by the schema.
      */
-    outcome: 'ran' | 'skipped' | 'error';
+    outcome: 'ran' | 'skipped' | 'gated' | 'error';
     status: CheckStatus;
     score: number;
     weight: number;
@@ -457,7 +587,7 @@ interface AuditTrace {
     /** The structured evidence behind the verdict, as the report carries it. */
     details?: CheckResult['details'];
 }
-/** Which of the three outcomes a finished check represents. */
+/** Which outcome a finished check represents. */
 declare function outcomeOf(check: CheckResult): AuditTrace['outcome'];
 /** Build a trace record from a finished check. */
 declare function traceFromCheck(check: CheckResult, durationMs: number): AuditTrace;
@@ -496,12 +626,26 @@ interface AuditPlan {
     }>;
     skipped: CheckResult[];
 }
+/** How `planAudits` should treat audits the scan cannot feed. */
+interface PlanOptions {
+    /**
+     * Skip audits whose `requires` the scan did not obtain.
+     *
+     * The orchestrator turns this on for every scan
+     * (`enforceEvidenceGate ?? true`), so an audit's `requires` decides what a
+     * blocked or client-rendered scan reports. It defaults to off in this option
+     * bag only so a caller planning audits directly gets the unfiltered set;
+     * passing `false` is the measurement escape hatch for comparing a gated run
+     * against an ungated one.
+     */
+    enforceEvidence?: boolean;
+}
 /**
  * Split a scan config into the audits that will actually execute and the
- * page-type-skipped `na` stubs. Exported so the orchestrator can size the
+ * `na` stubs for those it will not. Exported so the orchestrator can size the
  * audits progress phase before running it.
  */
-declare function planAudits(ctx: CheckContext, config: ScanConfig): AuditPlan;
+declare function planAudits(ctx: CheckContext, config: ScanConfig, options?: PlanOptions): AuditPlan;
 /**
  * Execute all audits registered in the config against the scan context.
  * Returns check results grouped into categories with weighted scores.
@@ -545,7 +689,8 @@ type ScanEvent = {
 } | {
     type: 'scan:done';
     durationMs: number;
-    score: number;
+    /** Null when the scan obtained too little evidence to judge the site. */
+    score: number | null;
     fraction: number;
     elapsedMs: number;
 };
@@ -583,7 +728,7 @@ declare class ProgressTracker {
     /** A failed unit still counts as settled work so the phase can complete. */
     unitFail(label: string, error: string): void;
     phaseDone(): void;
-    scanDone(score: number): void;
+    scanDone(score: number | null): void;
 }
 
 interface ScanOptions {
@@ -607,6 +752,48 @@ interface ScanOptions {
      * evidence it was drawn from; see {@link AuditTrace}.
      */
     onAuditTrace?: AuditTraceHandler;
+    /**
+     * Skip audits the scan could not feed, instead of running them blind.
+     *
+     * On by default. An audit whose required evidence the scan never obtained
+     * reports `na` tagged `skipped:no-evidence`, with the reason attached, and
+     * is never constructed. Set it to `false` to run every audit regardless —
+     * useful when measuring what the gate removes.
+     */
+    enforceEvidenceGate?: boolean;
+    /**
+     * undici dispatcher for every request this scan makes.
+     *
+     * Omitted, the scan uses the fetcher's shared agent, which opens as many
+     * connections per origin as the scan asks for. A caller scanning origins that
+     * did not invite it — the nightly corpus job — passes a bounded agent so the
+     * ~28 root-file requests the scan issues at once do not arrive at a stranger's
+     * WAF as a burst.
+     */
+    dispatcher?: Dispatcher;
+    /**
+     * How many requests this scan may have in flight at once.
+     *
+     * Omitted, the scan issues every request as it reaches the fetcher. A caller
+     * scanning origins that did not invite it sets this alongside `dispatcher`:
+     * the dispatcher bounds the sockets, and this bounds what is issued, so the
+     * requests the scan holds back wait outside the per-request deadline and
+     * outside `ttfbMs` instead of inside undici's queue. Without it a bounded
+     * scan reports its own queueing as the origin being slow or unreachable.
+     */
+    maxConcurrent?: number;
+    /**
+     * A `robots.txt` response the caller already has, used instead of fetching
+     * it again.
+     *
+     * For a caller that must read `robots.txt` before it decides to scan at all.
+     * The nightly corpus job is one: it asks permission first, and without this
+     * every site it visits gets two requests for the one file its owner watches.
+     * It must be the response to `<baseUrl>/robots.txt` fetched with this
+     * scanner's own user agent — anything else and the scan judges a file it was
+     * not served.
+     */
+    robotsTxt?: FetchResult;
 }
 declare function runScan(url: string, options?: ScanOptions): Promise<ScanReport>;
 
@@ -910,7 +1097,28 @@ declare function extractHeadings($: CheerioAPI): Array<{
     level: number;
     text: string;
 }>;
+/**
+ * The text of the page's main content region.
+ *
+ * Answers what the main content says, so page chrome — navigation, headers,
+ * footers, cookie banners — stays out of it. Among several <main> elements the
+ * one holding the most text wins: themes ship empty or fragmented <main>
+ * wrappers, and the first one is often a short stub. Falls back to the whole
+ * <body> only when no <main> holds any text.
+ *
+ * For shell detection — did the server render any readable text at all — use
+ * `getRenderedText` instead.
+ */
 declare function getMainContentText($: CheerioAPI): string;
+/**
+ * The text of the served HTML body, minus script/style/noscript/template.
+ *
+ * Answers whether the server rendered any readable text at all, so page chrome
+ * counts: a shell that ships only a nav and a footer is exactly what this has
+ * to see. Use `getMainContentText` when the question is what the main content
+ * says rather than whether anything was served.
+ */
+declare function getRenderedText($: CheerioAPI): string;
 declare function getWordCount($: CheerioAPI): number;
 declare function extractImages($: CheerioAPI): Array<{
     src: string;
@@ -963,6 +1171,8 @@ declare const MAX_CONCURRENT_REQUESTS = 10;
 declare const SCANNER_USER_AGENT = "AgentLighthouse/1.0 (+https://github.com/ForkPoint/agent-lighthouse)";
 declare const TAG_SKIPPED_PAGE_TYPE = "skipped:page-type";
 declare const TAG_SCAN_ERROR = "scan-error";
+/** An audit the scan could not feed: it needs evidence this scan never got. */
+declare const TAG_SKIPPED_NO_EVIDENCE = "skipped:no-evidence";
 /** Display names for the 8 v2 categories. */
 declare const CATEGORY_NAMES: Record<string, string>;
 declare const READINESS_WEIGHTS: {
@@ -1113,6 +1323,7 @@ declare const EvidenceGradeSchema: z.ZodEnum<["A", "B", "C", "D"]>;
 declare const AuditTierSchema: z.ZodEnum<["scored", "informative", "experimental"]>;
 /** v2 audit identity: `category/slug`, stable across releases (spec §6). */
 declare const AUDIT_ID_PATTERN: RegExp;
+declare const EvidenceKeySchema: z.ZodEnum<["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"]>;
 declare const AuditMetaSchema: z.ZodObject<{
     id: z.ZodString;
     category: z.ZodString;
@@ -1158,6 +1369,7 @@ declare const AuditMetaSchema: z.ZodObject<{
     evidenceGrade: z.ZodEnum<["A", "B", "C", "D"]>;
     tier: z.ZodEnum<["scored", "informative", "experimental"]>;
     dossier: z.ZodString;
+    requires: z.ZodOptional<z.ZodArray<z.ZodEnum<["origin-reachable", "unblocked-fetches", "rendered-body", "sample-adequate"]>, "many">>;
 }, "strip", z.ZodTypeAny, {
     category: string;
     title: string;
@@ -1183,6 +1395,7 @@ declare const AuditMetaSchema: z.ZodObject<{
         link: string;
         notice: string;
     } | undefined;
+    requires?: ("origin-reachable" | "unblocked-fetches" | "rendered-body" | "sample-adequate")[] | undefined;
 }, {
     category: string;
     title: string;
@@ -1208,6 +1421,7 @@ declare const AuditMetaSchema: z.ZodObject<{
         link: string;
         notice: string;
     } | undefined;
+    requires?: ("origin-reachable" | "unblocked-fetches" | "rendered-body" | "sample-adequate")[] | undefined;
 }>;
 declare const CheckResultSchema: z.ZodObject<{
     id: z.ZodString;
@@ -1376,4 +1590,4 @@ type AcpLinkType = (typeof ACP_LINK_TYPES)[number];
  */
 declare function resolvePolicyLinks(ctx: CheckContext): Map<AcpLinkType, string>;
 
-export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanOptions, type ScanReport, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allJsonLdNodes, buildCategoryResult, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };
+export { ACP_LINK_TYPES, AI_CRAWLER_UAS, AUDIT_ID_PATTERN, type AcpLinkType, type AgentLighthouseConfig, Audit, type AuditGuidance, AuditGuidanceSchema, type AuditMeta, AuditMetaSchema, type AuditPlan, type AuditProgressEvent, type AuditRegistration, type AuditResult, AuditResultSchema, type AuditRunResult, type AuditTier, AuditTierSchema, type AuditTrace, type AuditTraceHandler, BASELINE_UA, type BlockClass, CATEGORY_IDS, CATEGORY_MASS, CATEGORY_NAMES, type CategoryConfig, type CategoryResult, type CheckContext, type CheckFn, type CheckPriority, CheckPrioritySchema, type CheckRecommendation, type CheckResult, CheckResultSchema, type CheckStatus, CheckStatusSchema, type CssRule, DEFAULT_SCAN_LIMIT, type DeprecationNotice, DeprecationNoticeSchema, type EvidenceGrade, EvidenceGradeSchema, type EvidenceKey, EvidenceKeySchema, type ExpectedKind, type FetchClass, type FetchOptions, type FetchResult, type FetcherOptions, type FieldStatus, type FixEffort, FixEffortSchema, MAX_CONCURRENT_REQUESTS, MAX_PAGES_PER_SCAN, MAX_RESPONSE_BODY_BYTES, PAGE_TYPE_LABELS, PHASE_WEIGHTS, PRESETS, type PageContext, type PageCss, type PageJudgement, type PageOverride, type PageType, type PhaseId, type PresetName, type PresetOptions, type ProductFieldVerification, ProgressTracker, READINESS_WEIGHTS, REQUEST_TIMEOUT_MS, type ReadinessVitals, type RobotsFile, type RobotsGroup, type RobotsRule, SCANNER_USER_AGENT, SCAN_TIMEOUT_MS, SCORE_TIER_LABELS, type ScanConfig, type ScanEvent, type ScanEvidence, type ScanOptions, type ScanReport, type ScanValidity, type ScoreDisplayMode, ScoreDisplayModeSchema, type ScoreTier, type SitemapEntry, type SitemapTree, TAG_SCAN_ERROR, TAG_SKIPPED_NO_EVIDENCE, TAG_SKIPPED_PAGE_TYPE, type UaProbe, type WafProtection, allEvidenceMet, allJsonLdNodes, boundedDispatcher, buildCategoryResult, buildScanEvidence, calculateCategoryScore, calculateOverallScore, classifyFetch, classifyResponse, collectPageCss, collectSitemapEntries, createFetcher, defaultConfig, defineConfig, detectPageType, detectWafProtection, evidenceUrl, extractDomain, extractForms, extractHeadLinks, extractHeadings, extractImages, extractInternalLinks, extractJsonLd, extractMarkdownLinks, extractMetaTags, extractNavLinks, extractProductFieldVerification, extractScripts, extractStylesheetUrls, filterConfig, flattenJsonLd, formatTrace, getMainContentText, getPreset, getRenderedText, getScoreTier, getTierColor, getTierLabel, getWordCount, groupsForBot, hasNamedGroup, isBlanketBlocked, isInformative, isPathAllowed, isPrivateIp, isRealFile, isSafeUrl, isW3CDateTime, joinUrl, judgePages, loadConfigFile, logger, matchesUserAgent, normalizeNewlines, normalizeUrl, outcomeOf, pagesOfType, parseCssRules, parseHtml, parseRobots, parseRobotsFile, planAudits, probeUaParity, resolvePolicyLinks, runAudits, runScan, sampleEntries, stripBom, topLevelJsonLd, traceFromCheck, weightForGrade };