lighthouse 13.5.0-dev.20261007 → 13.5.0-dev.20261009

package/core/gather/gatherers/agentic/ard.js

@@ -4,17 +4,112 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+/**
+ * @fileoverview Gatherer that discovers and fetches a site's Agentic Resource Discovery (ARD)
+ * manifest. It follows the discovery mechanisms in ARD §5.1 and also consults the predecessor
+ * `ai-catalog` names so that sites that have not yet migrated are still audited.
+ * @see https://agenticresourcediscovery.org/spec/
+ */
+
 import log from 'lighthouse-logger';
 import LinkHeader from 'http-link-header';
 
 import BaseGatherer from '../../base-gatherer.js';
 import RobotsTxt from '../seo/robots-txt.js';
-import {pageFunctions} from '../../../lib/page-functions.js';
-function getAiCatalogLinkInDOM() {
-  const link = document.querySelector('link[rel~="ai-catalog" i]');
-  return link instanceof HTMLLinkElement ? link.href : null;
+
+/**
+ * Link relations for ARD entry sources. `ai-catalog` is the predecessor relation, which
+ * consumers MAY consult but are not required to (ARD §5.1).
+ */
+const REL_ARD = 'ard';
+const REL_LEGACY = 'ai-catalog';
+
+const WELL_KNOWN_PATH = '/.well-known/ard.json';
+const LEGACY_WELL_KNOWN_PATH = '/.well-known/ai-catalog.json';
+
+const HTTP_OK = 200;
+
+/** @typedef {{ard: string|null, legacy: string|null}} ArdLinks */
+
+/** @typedef {LH.Artifacts['AgentResourceDiscovery']['discoverySignals']} DiscoverySignals */
+
+/**
+ * @typedef FetchResult
+ * @property {number|null} status
+ * @property {string|null} content
+ * @property {Record<string, string>|null} headers
+ * @property {string} [errorMessage]
+ */
+
+/**
+ * A location to try when resolving the manifest. `advertised` locations are ones the site
+ * points at (robots.txt, link tags, Link headers); the well-known paths are probed by
+ * Lighthouse, so a failure there is not the site's fault and is not reported.
+ * @typedef ManifestCandidate
+ * @property {LH.Artifacts.ArdDiscoverySource} source
+ * @property {string|null} url
+ * @property {boolean} advertised
+ */
+
+/** @type {ReadonlyArray<LH.Artifacts.ArdDiscoverySource>} */
+const SOURCE_PRIORITY = [
+  'robotsTxtAgentmap',
+  'htmlLink',
+  'httpHeaderLink',
+  'wellKnown',
+  'legacyHtmlLink',
+  'legacyHttpHeaderLink',
+  'legacyWellKnown',
+];
+
+/** @type {Set<LH.Artifacts.ArdDiscoverySource>} */
+const PROBED_SOURCES = new Set(['wellKnown', 'legacyWellKnown']);
+
+/**
+ * @param {DiscoverySignals} discoverySignals
+ * @return {ManifestCandidate[]}
+ */
+function getManifestCandidates(discoverySignals) {
+  return SOURCE_PRIORITY.map(source => ({
+    source,
+    url: discoverySignals[source],
+    advertised: !PROBED_SOURCES.has(source),
+  }));
+}
+
+/**
+ * Resolves `href` against `baseUrl`, returning null for empty or unparseable values.
+ * @param {string|null|undefined} href
+ * @param {string} baseUrl
+ * @return {string|null}
+ */
+function resolveUrl(href, baseUrl) {
+  if (!href) return null;
+  try {
+    return new URL(href, baseUrl).href;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Runs in the page. Returns the `href` of the first `<link>` for each relation, or null.
+ * @param {string} relArd
+ * @param {string} relLegacy
+ * @return {ArdLinks}
+ */
+function getArdLinksInDOM(relArd, relLegacy) {
+  /** @param {string} rel */
+  const getHref = rel => {
+    const link = document.querySelector(`link[rel~="${rel}" i]`);
+    return link instanceof HTMLLinkElement ? link.href : null;
+  };
+  return {ard: getHref(relArd), legacy: getHref(relLegacy)};
 }
 
+/** @type {ArdLinks} */
+const NO_LINKS = {ard: null, legacy: null};
+
 class AgentResourceDiscovery extends BaseGatherer {
   /** @type {LH.Gatherer.GathererMeta<'RobotsTxt'>} */
   meta = {
@@ -23,6 +118,7 @@ class AgentResourceDiscovery extends BaseGatherer {
   };
 
   /**
+   * Returns the absolute URL from the first `Agentmap:` directive in robots.txt, if any.
    * @param {LH.Artifacts['RobotsTxt']|null|undefined} robotsTxt
    * @param {string} finalDisplayedUrl
    * @return {string|null}
@@ -31,93 +127,159 @@ class AgentResourceDiscovery extends BaseGatherer {
     if (!robotsTxt?.content) return null;
     const match = robotsTxt.content.match(/^\s*Agentmap:\s*(\S+)/im);
     if (!match) return null;
-    try {
-      return new URL(match[1], finalDisplayedUrl).href;
-    } catch {
-      return null;
-    }
+    return resolveUrl(match[1], finalDisplayedUrl);
   }
 
   /**
+   * Finds `<link rel="ard">` and `<link rel="ai-catalog">` in the page.
    * @param {LH.Gatherer.Context} context
    * @param {string} finalDisplayedUrl
-   * @return {Promise<string|null>}
+   * @return {Promise<ArdLinks>}
    */
-  static async getHtmlLinkFromDom(context, finalDisplayedUrl) {
+  static async getHtmlLinksFromDom(context, finalDisplayedUrl) {
     try {
-      const href = await context.driver.executionContext.evaluate(getAiCatalogLinkInDOM, {
-        args: [],
+      const links = await context.driver.executionContext.evaluate(getArdLinksInDOM, {
+        args: [REL_ARD, REL_LEGACY],
         useIsolation: true,
-        deps: [pageFunctions.getNodeDetails],
       });
-      if (!href) return null;
-      return new URL(href, finalDisplayedUrl).href;
-    } catch {
-      return null;
+      return {
+        ard: resolveUrl(links?.ard, finalDisplayedUrl),
+        legacy: resolveUrl(links?.legacy, finalDisplayedUrl),
+      };
+    } catch (err) {
+      log.verbose('AgentResourceDiscovery', `Could not read link tags: ${err.message}`);
+      return NO_LINKS;
     }
   }
 
   /**
+   * Finds `rel="ard"` and `rel="ai-catalog"` in the main document's HTTP `Link` header.
    * @param {LH.Gatherer.Context} context
    * @param {string} finalDisplayedUrl
-   * @return {Promise<string|null>}
+   * @return {Promise<ArdLinks>}
    */
-  static async getHttpHeaderLink(context, finalDisplayedUrl) {
-    // Only check in navigation mode
-    if (context.gatherMode !== 'navigation') return null;
+  static async getHttpHeaderLinks(context, finalDisplayedUrl) {
+    if (context.gatherMode !== 'navigation') return NO_LINKS;
 
     try {
       const mainResourceResponse = await context.driver.fetcher.fetchResource(finalDisplayedUrl);
-      const linkHeader = mainResourceResponse.headers?.['link'];
-      if (!linkHeader) return null;
+      const linkHeader = mainResourceResponse.headers?.link;
+      if (!linkHeader) return NO_LINKS;
+
+      const parsed = LinkHeader.parse(linkHeader.replace(/\n/g, ','));
+      /** @param {string} rel */
+      const getUri = rel =>
+        resolveUrl(parsed.refs.find(ref => ref.rel?.toLowerCase() === rel)?.uri, finalDisplayedUrl);
+      return {ard: getUri(REL_ARD), legacy: getUri(REL_LEGACY)};
+    } catch (err) {
+      log.verbose('AgentResourceDiscovery', `Could not read Link header: ${err.message}`);
+      return NO_LINKS;
+    }
+  }
+
+  /**
+   * Fetches a candidate manifest URL. Never throws; network errors are returned as a
+   * null status with an `errorMessage`.
+   * @param {LH.Gatherer.Context} context
+   * @param {string} url
+   * @return {Promise<FetchResult>}
+   */
+  static async fetchManifest(context, url) {
+    try {
+      const {status, content, headers} = await context.driver.fetcher.fetchResource(url);
+      return {status, content, headers: headers || null};
+    } catch (err) {
+      log.error('AgentResourceDiscovery', err);
+      return {status: null, content: null, headers: null, errorMessage: err.message};
+    }
+  }
 
-      const parsed = LinkHeader.parse(linkHeader);
-      const aiCatalogRef = parsed.get('rel', 'ai-catalog')[0];
-      if (!aiCatalogRef?.uri) return null;
+  /**
+   * Tries each candidate in order and returns the first that loads. Advertised candidates
+   * that fail are recorded in `failedSources` and skipped. If nothing loads, the result
+   * describes the first advertised location that failed, or else the `/.well-known/ard.json`
+   * path consumers are required to fetch.
+   * @param {LH.Gatherer.Context} context
+   * @param {ManifestCandidate[]} candidates
+   * @return {Promise<{
+   *   discoverySource: LH.Artifacts.ArdDiscoverySource,
+   *   catalogUrl: string,
+   *   fetchResult: FetchResult,
+   *   failedSources: LH.Artifacts['AgentResourceDiscovery']['failedSources'],
+   * }>}
+   */
+  static async resolveManifest(context, candidates) {
+    /** @type {Array<{source: LH.Artifacts.ArdDiscoverySource, url: string, advertised: boolean, fetchResult: FetchResult}>} */
+    const attempts = [];
+    const failedSources = () => attempts
+      .filter(attempt => attempt.advertised)
+      .map(({source, url, fetchResult}) => ({
+        source,
+        url,
+        status: fetchResult.status,
+        errorMessage: fetchResult.errorMessage,
+      }));
+
+    for (const {source, url, advertised} of candidates) {
+      if (!url || attempts.some(attempt => attempt.url === url)) continue;
 
-      return new URL(aiCatalogRef.uri, finalDisplayedUrl).href;
-    } catch {
-      return null;
+      const fetchResult = await AgentResourceDiscovery.fetchManifest(context, url);
+      if (fetchResult.status === HTTP_OK && fetchResult.content) {
+        return {discoverySource: source, catalogUrl: url, fetchResult,
+          failedSources: failedSources()};
+      }
+      attempts.push({source, url, advertised, fetchResult});
     }
+
+    const fallback = attempts.find(attempt => attempt.advertised) ||
+      attempts.find(attempt => attempt.source === 'wellKnown');
+    if (!fallback) throw new Error('ARD discovery attempted no candidates');
+
+    return {
+      discoverySource: fallback.source,
+      catalogUrl: fallback.url,
+      fetchResult: fallback.fetchResult,
+      failedSources: failedSources(),
+    };
   }
 
   /**
+   * Resolves the manifest following ARD §5.1: explicit ARD signals first, then the
+   * `/.well-known/ard.json` path consumers MUST fetch, and only then the predecessor
+   * `ai-catalog` relation and well-known path, which consumers MAY consult.
    * @param {LH.Gatherer.Context<'RobotsTxt'>} context
    * @return {Promise<LH.Artifacts['AgentResourceDiscovery']>}
    */
   async getArtifact(context) {
     const {finalDisplayedUrl} = context.baseArtifacts.URL;
-    const robotsTxt = context.dependencies.RobotsTxt;
     const robotsTxtAgentmap = AgentResourceDiscovery.getRobotsTxtAgentmap(
-      robotsTxt, finalDisplayedUrl);
-    const [htmlLink, httpHeaderLink] = await Promise.all([
-      AgentResourceDiscovery.getHtmlLinkFromDom(context, finalDisplayedUrl),
-      AgentResourceDiscovery.getHttpHeaderLink(context, finalDisplayedUrl),
+      context.dependencies.RobotsTxt, finalDisplayedUrl);
+    const [htmlLinks, httpHeaderLinks] = await Promise.all([
+      AgentResourceDiscovery.getHtmlLinksFromDom(context, finalDisplayedUrl),
+      AgentResourceDiscovery.getHttpHeaderLinks(context, finalDisplayedUrl),
     ]);
-    const wellKnown = new URL('/.well-known/ai-catalog.json', finalDisplayedUrl).href;
-    const catalogUrl = robotsTxtAgentmap || htmlLink || httpHeaderLink || wellKnown;
-
-    /** @type {string|undefined} */
-    let errorMessage;
-    const fetchResult = await context.driver.fetcher.fetchResource(catalogUrl)
-      .catch(err => {
-        log.error('AgentResourceDiscovery', err);
-        errorMessage = err.message;
-        return {status: null, content: null, headers: null};
-      });
+
+    /** @type {DiscoverySignals} */
+    const discoverySignals = {
+      robotsTxtAgentmap,
+      htmlLink: htmlLinks.ard,
+      httpHeaderLink: httpHeaderLinks.ard,
+      legacyHtmlLink: htmlLinks.legacy,
+      legacyHttpHeaderLink: httpHeaderLinks.legacy,
+      wellKnown: new URL(WELL_KNOWN_PATH, finalDisplayedUrl).href,
+      legacyWellKnown: new URL(LEGACY_WELL_KNOWN_PATH, finalDisplayedUrl).href,
+    };
+
+    const candidates = getManifestCandidates(discoverySignals);
+    const {discoverySource, catalogUrl, fetchResult, failedSources} =
+      await AgentResourceDiscovery.resolveManifest(context, candidates);
 
     return {
-      status: fetchResult.status,
-      content: fetchResult.content,
-      headers: fetchResult.headers || null,
+      ...fetchResult,
       catalogUrl,
-      discoverySignals: {
-        robotsTxtAgentmap,
-        htmlLink,
-        httpHeaderLink,
-        wellKnown,
-      },
-      errorMessage,
+      discoverySource,
+      discoverySignals,
+      failedSources,
     };
   }
 }

package/core/lib/trace-engine.d.ts

@@ -16,5 +16,6 @@ export const TraceProcessor: typeof TraceEngine.Processor.TraceProcessor;
 export const TraceHandlers: typeof TraceEngine.Handlers.ModelHandlers;
 export const Insights: typeof TraceEngine.Insights;
 export const Helpers: typeof TraceEngine.Helpers;
+export const Types: typeof TraceEngine.Types;
 import * as TraceEngine from '@paulirish/trace_engine';
 //# sourceMappingURL=trace-engine.d.ts.map

package/core/lib/trace-engine.js

@@ -12,10 +12,12 @@ const TraceProcessor = TraceEngine.Processor.TraceProcessor;
 const TraceHandlers = TraceEngine.Handlers.ModelHandlers;
 const Insights = TraceEngine.Insights;
 const Helpers = TraceEngine.Helpers;
+const Types = TraceEngine.Types;
 
 export {
   TraceProcessor,
   TraceHandlers,
   Insights,
   Helpers,
+  Types,
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "lighthouse",
   "type": "module",
-  "version": "13.5.0-dev.20261007",
+  "version": "13.5.0-dev.20261009",
   "description": "Automated auditing, performance metrics, and best practices for the web.",
   "main": "./core/index.js",
   "bin": {
@@ -38,6 +38,7 @@
     "clean": "rm -r dist proto/scripts/*.json proto/scripts/*_pb2.* proto/scripts/*_pb.* proto/scripts/__pycache__ proto/scripts/*.pyc *.report.html *.report.dom.html *.report.json *.devtoolslog.json *.trace.json shared/localization/locales/*.ctc.json || true",
     "clean-types": "git clean -xfq '*.d.ts' '*.d.cts' -e 'node_modules/' -e 'dist/' -e '.tmp/' -e '**/types/'",
     "lint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe . || eslint .",
+    "zizmor": "uvx zizmor@1.25.2 --gh-token=\"$(gh auth token)\" .github/workflows",
     "smoke": "node -r source-map-support/register cli/test/smokehouse/frontends/smokehouse-bin.js",
     "debug": "node --inspect-brk ./cli/index.js",
     "start": "yarn build-report --standalone && node ./cli/index.js",
@@ -53,14 +54,13 @@
     "test-proto": "yarn compile-proto && yarn build-proto-roundtrip",
     "unit-core": "yarn mocha core",
     "unit-cli": "yarn mocha --testMatch cli/**/*-test.js",
-    "unit-lantern-trace": "INTERNAL_LANTERN_USE_TRACE=1 yarn mocha core/test/computed/metrics core/test/audits",
+    "unit-lantern-trace": "INTERNAL_LANTERN_USE_TRACE=1 yarn mocha core/test/computed/metrics core/test/computed/page-dependency-graph-test.js core/test/computed/tbt-impact-tasks-test.js core/test/audits",
     "unit-report": "yarn mocha --testMatch report/**/*-test.js",
     "unit-treemap": "yarn mocha --testMatch treemap/**/*-test.js",
     "unit-viewer": "yarn mocha --testMatch viewer/**/*-test.js",
     "unit-flow": "bash flow-report/test/run-flow-report-tests.sh",
     "unit": "yarn unit-flow && yarn mocha && yarn unit-lantern-trace",
     "unit:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit",
-    "unit-lantern-trace:ci": "NODE_OPTIONS=--max-old-space-size=8192 npm run unit-lantern-trace",
     "core-unit": "yarn unit-core",
     "cli-unit": "yarn unit-cli",
     "viewer-unit": "yarn unit-viewer",
@@ -71,7 +71,6 @@
     "deploy-treemap": "yarn build-treemap --deploy",
     "deploy-viewer": "yarn build-viewer --deploy",
     "vercel-build": "yarn build-sample-reports && yarn build-viewer && yarn build-treemap",
-    "dogfood-lhci": "./core/scripts/dogfood-lhci.sh",
     "timing-trace": "node core/scripts/generate-timing-trace.js",
     "changelog": "conventional-changelog --config ./build/changelog-generator/index.cjs --infile changelog.md --same-file",
     "type-check": "tsc --build ./tsconfig-all.json",
@@ -111,7 +110,7 @@
     "@jest/fake-timers": "^29.7.0",
     "@testing-library/preact": "^3.1.1",
     "@testing-library/preact-hooks": "^1.1.0",
-    "@types/archiver": "^2.1.2",
+    "@types/archiver": "^8.0.0",
     "@types/chrome": "^0.1.42",
     "@types/configstore": "^6.0.2",
     "@types/cpy": "^5.1.0",
@@ -138,7 +137,7 @@
     "@typescript-eslint/parser": "^8.60.1",
     "acorn": "^8.5.0",
     "angular": "^1.7.4",
-    "archiver": "^7.0.1",
+    "archiver": "^8.0.0",
     "builtin-modules": "^3.3.0",
     "chalk": "^2.4.1",
     "chrome-devtools-frontend": "1.0.1705227",
@@ -183,7 +182,7 @@
     "webtreemap-cdt": "^3.2.1"
   },
   "dependencies": {
-    "@paulirish/trace_engine": "0.0.65",
+    "@paulirish/trace_engine": "0.0.66",
     "@sentry/node": "^10.0.0",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",