lighthouse 13.4.1-dev.20260824 → 13.4.1-dev.20260831

package/core/gather/fetcher.js

@@ -10,7 +10,7 @@ import * as LH from '../../types/lh.js';
  * ignoring normal browser constraints such as CORS.
  */
 
-/** @typedef {{content: string|null, status: number|null}} FetchResponse */
+/** @typedef {{content: string|null, status: number|null, headers?: Record<string, string>|null}} FetchResponse */
 
 class Fetcher {
   /**
@@ -37,6 +37,26 @@ class Fetcher {
     return this._fetchResourceOverProtocol(url, options);
   }
 
+  /**
+   * @param {Headers|Record<string, string|number|boolean>|null|undefined} rawHeaders
+   * @return {Record<string, string>|null}
+   */
+  _normalizeHeaders(rawHeaders) {
+    if (!rawHeaders) return null;
+
+    /** @type {Record<string, string>} */
+    const headers = {};
+    const entries = typeof /** @type {Headers} */ (rawHeaders).entries === 'function' ?
+    /** @type {Headers} */ (rawHeaders).entries() :
+      Object.entries(rawHeaders);
+
+    for (const [key, value] of entries) {
+      headers[key.toLowerCase()] = String(value);
+    }
+
+    return headers;
+  }
+
   /**
    * @param {string} url
    * @return {Promise<FetchResponse>}
@@ -49,9 +69,12 @@ class Fetcher {
       content = await response.text();
     } catch {}
 
+    const headers = this._normalizeHeaders(response.headers);
+
     return {
       content,
       status: response.status,
+      headers,
     };
   }
 
@@ -82,7 +105,7 @@ class Fetcher {
 
   /**
    * @param {string} url
-   * @return {Promise<{stream: LH.Crdp.IO.StreamHandle|null, status: number|null}>}
+   * @return {Promise<{stream: LH.Crdp.IO.StreamHandle|null, status: number|null, headers?: Record<string, string>|null}>}
    */
   async _loadNetworkResource(url) {
     const frameTreeResponse = await this.session.sendCommand('Page.getFrameTree');
@@ -98,6 +121,7 @@ class Fetcher {
     return {
       stream: networkResponse.resource.success ? (networkResponse.resource.stream || null) : null,
       status: networkResponse.resource.httpStatusCode || null,
+      headers: networkResponse.resource.headers || null,
     };
   }
 
@@ -110,12 +134,14 @@ class Fetcher {
     const startTime = Date.now();
     const response = await this._wrapWithTimeout(this._loadNetworkResource(url), options.timeout);
 
+    const headers = this._normalizeHeaders(response.headers);
+
     const isOk = response.status && response.status >= 200 && response.status <= 299;
-    if (!response.stream || !isOk) return {status: response.status, content: null};
+    if (!response.stream || !isOk) return {status: response.status, content: null, headers};
 
     const timeout = options.timeout - (Date.now() - startTime);
     const content = await this._readIOStream(response.stream, {timeout});
-    return {status: response.status, content};
+    return {status: response.status, content, headers};
   }
 
   /**

package/core/gather/gatherers/accessibility.js

@@ -13,7 +13,6 @@ import {pageFunctions} from '../../lib/page-functions.js';
 /**
  * @return {Promise<LH.Artifacts.Accessibility>}
  */
-/* c8 ignore start */
 async function runA11yChecks() {
   /** @type {typeof import('axe-core/axe')} */
   // @ts-expect-error - axe defined by axeLibSource
@@ -183,7 +182,6 @@ function createAxeRuleResultArtifact(result) {
     description: result.description,
   };
 }
-/* c8 ignore stop */
 
 class Accessibility extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */

package/core/gather/gatherers/agentic/ard.d.ts

@@ -0,0 +1,30 @@
+export default AgentResourceDiscovery;
+declare class AgentResourceDiscovery extends BaseGatherer {
+    /**
+     * @param {LH.Artifacts['RobotsTxt']|null|undefined} robotsTxt
+     * @param {string} finalDisplayedUrl
+     * @return {string|null}
+     */
+    static getRobotsTxtAgentmap(robotsTxt: LH.Artifacts["RobotsTxt"] | null | undefined, finalDisplayedUrl: string): string | null;
+    /**
+     * @param {LH.Gatherer.Context} context
+     * @param {string} finalDisplayedUrl
+     * @return {Promise<string|null>}
+     */
+    static getHtmlLinkFromDom(context: LH.Gatherer.Context, finalDisplayedUrl: string): Promise<string | null>;
+    /**
+     * @param {LH.Gatherer.Context} context
+     * @param {string} finalDisplayedUrl
+     * @return {Promise<string|null>}
+     */
+    static getHttpHeaderLink(context: LH.Gatherer.Context, finalDisplayedUrl: string): Promise<string | null>;
+    /** @type {LH.Gatherer.GathererMeta<'RobotsTxt'>} */
+    meta: LH.Gatherer.GathererMeta<"RobotsTxt">;
+    /**
+     * @param {LH.Gatherer.Context<'RobotsTxt'>} context
+     * @return {Promise<LH.Artifacts['AgentResourceDiscovery']>}
+     */
+    getArtifact(context: LH.Gatherer.Context<"RobotsTxt">): Promise<LH.Artifacts["AgentResourceDiscovery"]>;
+}
+import BaseGatherer from '../../base-gatherer.js';
+//# sourceMappingURL=ard.d.ts.map

package/core/gather/gatherers/agentic/ard.js

@@ -0,0 +1,125 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import log from 'lighthouse-logger';
+import LinkHeader from 'http-link-header';
+
+import BaseGatherer from '../../base-gatherer.js';
+import RobotsTxt from '../seo/robots-txt.js';
+import {pageFunctions} from '../../../lib/page-functions.js';
+function getAiCatalogLinkInDOM() {
+  const link = document.querySelector('link[rel~="ai-catalog" i]');
+  return link instanceof HTMLLinkElement ? link.href : null;
+}
+
+class AgentResourceDiscovery extends BaseGatherer {
+  /** @type {LH.Gatherer.GathererMeta<'RobotsTxt'>} */
+  meta = {
+    supportedModes: ['snapshot', 'navigation'],
+    dependencies: {RobotsTxt: RobotsTxt.symbol},
+  };
+
+  /**
+   * @param {LH.Artifacts['RobotsTxt']|null|undefined} robotsTxt
+   * @param {string} finalDisplayedUrl
+   * @return {string|null}
+   */
+  static getRobotsTxtAgentmap(robotsTxt, finalDisplayedUrl) {
+    if (!robotsTxt?.content) return null;
+    const match = robotsTxt.content.match(/^\s*Agentmap:\s*(\S+)/im);
+    if (!match) return null;
+    try {
+      return new URL(match[1], finalDisplayedUrl).href;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.Context} context
+   * @param {string} finalDisplayedUrl
+   * @return {Promise<string|null>}
+   */
+  static async getHtmlLinkFromDom(context, finalDisplayedUrl) {
+    try {
+      const href = await context.driver.executionContext.evaluate(getAiCatalogLinkInDOM, {
+        args: [],
+        useIsolation: true,
+        deps: [pageFunctions.getNodeDetails],
+      });
+      if (!href) return null;
+      return new URL(href, finalDisplayedUrl).href;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.Context} context
+   * @param {string} finalDisplayedUrl
+   * @return {Promise<string|null>}
+   */
+  static async getHttpHeaderLink(context, finalDisplayedUrl) {
+    // Only check in navigation mode
+    if (context.gatherMode !== 'navigation') return null;
+
+    try {
+      const mainResourceResponse = await context.driver.fetcher.fetchResource(finalDisplayedUrl);
+      const linkHeader = mainResourceResponse.headers?.['link'];
+      if (!linkHeader) return null;
+
+      const parsed = LinkHeader.parse(linkHeader);
+      const aiCatalogRef = parsed.get('rel', 'ai-catalog')[0];
+      if (!aiCatalogRef?.uri) return null;
+
+      return new URL(aiCatalogRef.uri, finalDisplayedUrl).href;
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * @param {LH.Gatherer.Context<'RobotsTxt'>} context
+   * @return {Promise<LH.Artifacts['AgentResourceDiscovery']>}
+   */
+  async getArtifact(context) {
+    const {finalDisplayedUrl} = context.baseArtifacts.URL;
+    const robotsTxt = context.dependencies.RobotsTxt;
+    const robotsTxtAgentmap = AgentResourceDiscovery.getRobotsTxtAgentmap(
+      robotsTxt, finalDisplayedUrl);
+    const [htmlLink, httpHeaderLink] = await Promise.all([
+      AgentResourceDiscovery.getHtmlLinkFromDom(context, finalDisplayedUrl),
+      AgentResourceDiscovery.getHttpHeaderLink(context, finalDisplayedUrl),
+    ]);
+    const wellKnown = new URL('/.well-known/ai-catalog.json', finalDisplayedUrl).href;
+    const catalogUrl = robotsTxtAgentmap || htmlLink || httpHeaderLink || wellKnown;
+
+    /** @type {string|undefined} */
+    let errorMessage;
+    const fetchResult = await context.driver.fetcher.fetchResource(catalogUrl)
+      .catch(err => {
+        log.error('AgentResourceDiscovery', err);
+        errorMessage = err.message;
+        return {status: null, content: null, headers: null};
+      });
+
+    return {
+      status: fetchResult.status,
+      content: fetchResult.content,
+      headers: fetchResult.headers || null,
+      catalogUrl,
+      discoverySignals: {
+        robotsTxtAgentmap,
+        htmlLink,
+        httpHeaderLink,
+        wellKnown,
+      },
+      errorMessage,
+    };
+  }
+}
+
+export default AgentResourceDiscovery;

package/core/gather/gatherers/anchor-elements.js

@@ -19,7 +19,6 @@ import {resolveDevtoolsNodePathToObjectId} from '../driver/dom.js';
  *
  * @return {LH.Artifacts['AnchorElements']}
  */
-/* c8 ignore start */
 function collectAnchorElements() {
   /** @param {string} url */
   const resolveURLOrEmpty = url => {
@@ -114,7 +113,6 @@ function collectAnchorElements() {
     };
   });
 }
-/* c8 ignore stop */
 
 /**
  * @param {LH.Gatherer.ProtocolSession} session

package/core/gather/gatherers/full-page-screenshot.js

@@ -31,8 +31,6 @@ function kebabCaseToCamelCase(str) {
   );
 }
 
-/* c8 ignore start */
-
 function getObservedDeviceMetrics() {
   // Convert the Web API's kebab case (landscape-primary) to camel case (landscapePrimary).
   const screenOrientationType = kebabCaseToCamelCase(window.screen.orientation.type);
@@ -53,8 +51,6 @@ function waitForDoubleRaf() {
   });
 }
 
-/* c8 ignore stop */
-
 class FullPageScreenshot extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */
   meta = {

package/core/gather/gatherers/iframe-elements.js

@@ -14,7 +14,6 @@ import {pageFunctions} from '../../lib/page-functions.js';
 /**
  * @return {LH.Artifacts['IFrameElements']}
  */
-/* c8 ignore start */
 function collectIFrameElements() {
   const realBoundingClientRect = window.__HTMLElementBoundingClientRect ||
     window.HTMLElement.prototype.getBoundingClientRect;
@@ -35,7 +34,6 @@ function collectIFrameElements() {
     };
   });
 }
-/* c8 ignore stop */
 
 class IFrameElements extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */

package/core/gather/gatherers/image-elements.js

@@ -17,7 +17,6 @@ import {pageFunctions} from '../../lib/page-functions.js';
 /* global getElementsInDocument, getNodeDetails */
 
 /** @param {Element} element */
-/* c8 ignore start */
 function getClientRect(element) {
   const clientRect = element.getBoundingClientRect();
   return {
@@ -28,7 +27,6 @@ function getClientRect(element) {
     right: clientRect.right,
   };
 }
-/* c8 ignore stop */
 
 /**
  * If an image is within `picture`, the `picture` element's css position
@@ -36,7 +34,6 @@ function getClientRect(element) {
  * @param {Element} element
  * @param {CSSStyleDeclaration} computedStyle
  */
-/* c8 ignore start */
 function getPosition(element, computedStyle) {
   if (element.parentElement && element.parentElement.tagName === 'PICTURE') {
     const parentStyle = window.getComputedStyle(element.parentElement);
@@ -44,13 +41,11 @@ function getPosition(element, computedStyle) {
   }
   return computedStyle.getPropertyValue('position');
 }
-/* c8 ignore stop */
 
 /**
  * @param {Array<Element>} allElements
  * @return {Array<LH.Artifacts.ImageElement>}
  */
-/* c8 ignore start */
 function getHTMLImages(allElements) {
   const allImageElements = /** @type {Array<HTMLImageElement>} */ (allElements.filter(element => {
     return element.localName === 'img';
@@ -89,13 +84,11 @@ function getHTMLImages(allElements) {
     };
   });
 }
-/* c8 ignore stop */
 
 /**
  * @param {Array<Element>} allElements
  * @return {Array<LH.Artifacts.ImageElement>}
  */
-/* c8 ignore start */
 function getCSSImages(allElements) {
   // Chrome normalizes background image style from getComputedStyle to be an absolute URL in quotes.
   // Only match basic background-image: url("http://host/image.jpeg") declarations
@@ -138,23 +131,19 @@ function getCSSImages(allElements) {
 
   return images;
 }
-/* c8 ignore stop */
 
 /** @return {Array<LH.Artifacts.ImageElement>} */
-/* c8 ignore start */
 function collectImageElementInfo() {
   /** @type {Array<Element>} */
   // @ts-expect-error - added by getElementsInDocumentFnString
   const allElements = getElementsInDocument();
   return getHTMLImages(allElements).concat(getCSSImages(allElements));
 }
-/* c8 ignore stop */
 
 /**
  * @param {string} url
  * @return {Promise<{naturalWidth: number, naturalHeight: number}>}
  */
-/* c8 ignore start */
 function determineNaturalSize(url) {
   return new Promise((resolve, reject) => {
     const img = new Image();
@@ -169,7 +158,6 @@ function determineNaturalSize(url) {
     img.src = url;
   });
 }
-/* c8 ignore stop */
 
 /**
  * @param {Partial<Pick<LH.Crdp.CSS.CSSStyle, 'cssProperties'>>|undefined} rule

package/core/gather/gatherers/inputs.js

@@ -14,7 +14,6 @@ import {pageFunctions} from '../../lib/page-functions.js';
 /**
  * @return {LH.Artifacts['Inputs']}
  */
-/* c8 ignore start */
 function collectElements() {
   /** @type {LH.Artifacts.InputElement[]} */
   const inputArtifacts = [];
@@ -93,7 +92,6 @@ function collectElements() {
     labels: [...labelElToArtifact.values()],
   };
 }
-/* c8 ignore stop */
 
 class Inputs extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */

package/core/gather/gatherers/link-elements.js

@@ -59,7 +59,6 @@ function getCrossoriginFromHeader(value) {
 /**
  * @return {LH.Artifacts['LinkElements']}
  */
-/* c8 ignore start */
 function getLinkElementsInDOM() {
   /** @type {Array<HTMLOrSVGElement>} */
   // @ts-expect-error - getElementsInDocument put into scope via stringification
@@ -91,7 +90,6 @@ function getLinkElementsInDOM() {
 
   return linkElements;
 }
-/* c8 ignore stop */
 
 class LinkElements extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta<'DevtoolsLog'>} */

package/core/gather/gatherers/meta-elements.js

@@ -8,8 +8,6 @@ import BaseGatherer from '../base-gatherer.js';
 import {pageFunctions} from '../../lib/page-functions.js';
 
 /* globals getElementsInDocument getNodeDetails */
-
-/* c8 ignore start */
 function collectMetaElements() {
   const functions = /** @type {typeof pageFunctions} */({
     // @ts-expect-error - getElementsInDocument put into scope via stringification
@@ -36,7 +34,6 @@ function collectMetaElements() {
     };
   });
 }
-/* c8 ignore stop */
 
 class MetaElements extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */

package/core/gather/gatherers/seo/robots-txt.d.ts

@@ -1,5 +1,6 @@
 export default RobotsTxt;
 declare class RobotsTxt extends BaseGatherer {
+    static symbol: symbol;
     /**
      * @param {LH.Gatherer.Context} passContext
      * @return {Promise<LH.Artifacts['RobotsTxt']>}

package/core/gather/gatherers/seo/robots-txt.js

@@ -9,8 +9,11 @@ import log from 'lighthouse-logger';
 import BaseGatherer from '../../base-gatherer.js';
 
 class RobotsTxt extends BaseGatherer {
+  static symbol = Symbol('RobotsTxt');
+
   /** @type {LH.Gatherer.GathererMeta} */
   meta = {
+    symbol: RobotsTxt.symbol,
     supportedModes: ['snapshot', 'navigation'],
   };
 

package/core/gather/gatherers/stacks.js

@@ -46,7 +46,6 @@ const libDetectorSource = fs.readFileSync(
 /**
  * Obtains a list of detected JS libraries and their versions.
  */
-/* c8 ignore start */
 async function detectLibraries() {
   /** @type {JSLibrary[]} */
   const libraries = [];
@@ -81,7 +80,6 @@ async function detectLibraries() {
 
   return libraries;
 }
-/* c8 ignore stop */
 
 
 /** @implements {LH.Gatherer.GathererInstance} */

package/core/gather/gatherers/trace-elements.js

@@ -30,7 +30,6 @@ const MAX_LAYOUT_SHIFTS = 15;
 /**
  * @param {Element | ShadowRoot | Text} node
  */
-/* c8 ignore start */
 function getNodeDetailsData(node) {
   /** @type {Element | ShadowRoot | Text | null} */
   let elem = node.nodeType === document.ELEMENT_NODE ? node : node.parentElement;
@@ -45,7 +44,6 @@ function getNodeDetailsData(node) {
   }
   return traceElement;
 }
-/* c8 ignore stop */
 
 class TraceElements extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta<'Trace'|'SourceMaps'>} */

package/core/gather/gatherers/viewport-dimensions.js

@@ -9,7 +9,6 @@ import BaseGatherer from '../base-gatherer.js';
 /**
  * @return {LH.Artifacts.ViewportDimensions}
  */
-/* c8 ignore start */
 function getViewportDimensions() {
   // window.innerWidth to get the scrollable size of the window (irrespective of zoom)
   // window.outerWidth to get the size of the visible area
@@ -22,7 +21,6 @@ function getViewportDimensions() {
     devicePixelRatio: window.devicePixelRatio,
   };
 }
-/* c8 ignore stop */
 
 class ViewportDimensions extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta} */

package/core/gather/session.js

@@ -123,6 +123,7 @@ class ProtocolSession extends CrdpEventEmitter {
       }));
     });
 
+    // @ts-ignore-error CdpSession.send expects a method key of ProtocolMapping.Commands, but we use LH.CrdpCommands
     const resultPromise = this._cdpSession.send(method, ...params, {
       // Add 50ms to the Puppeteer timeout to ensure the Lighthouse timeout finishes first.
       timeout: timeoutMs + PPTR_BUFFER,