ax-audit 3.6.0 → 4.0.0

package/dist/constants.d.ts

@@ -1,29 +1,133 @@
-import type { Grade, SecurityHeader } from './types.js';
+import type { CheckCategory, Grade, SecurityHeader } from './types.js';
 export declare const VERSION: string;
 export declare const USER_AGENT: string;
 /**
- * Known AI / LLM crawlers grouped by primary purpose. The audit recommends explicit
- * `User-agent:` rules for every entry so site operators can see exactly which agents
- * have access. Buckets:
+ * What an AI client does with a page, and therefore what a site loses by
+ * blocking it. This distinction is the single most important thing an audit can
+ * tell an operator: blocking a training crawler is a policy choice, while
+ * blocking a search crawler removes the site from that assistant's answers.
  *
- * - `training`   — bots that scrape content for model training corpora
- * - `search`     — bots that fetch on behalf of a live answer engine / search UI
- * - `fetching`   — generic on-demand fetchers (browsing, summarization, automation)
+ * - `training`   — builds model training corpora.
+ * - `search`     — builds the index a live assistant cites from.
+ * - `user-fetch` — retrieves one URL because a human asked for it, right now.
+ * - `agent`      — an autonomous browser acting on a user's behalf.
  */
-export declare const AI_CRAWLERS: Record<string, string[]>;
+export type CrawlerPurpose = 'training' | 'search' | 'user-fetch' | 'agent';
+export interface CrawlerInfo {
+    vendor: string;
+    purpose: CrawlerPurpose;
+    /**
+     * Whether the vendor documents that this client obeys robots.txt.
+     * `partial` covers clients documented as "may not apply" — chiefly
+     * user-triggered fetchers, which most vendors exempt.
+     */
+    honorsRobots: boolean | 'partial';
+    /** One sentence on what blocking this client costs. Surfaced directly in findings. */
+    impact: string;
+    /** Vendor documentation for the token. */
+    docUrl: string;
+    /** Published IP-range list, where the vendor offers one for reverse verification. */
+    ipListUrl?: string;
+    /** Vendor signs requests with Web Bot Auth (RFC 9421 HTTP Message Signatures). */
+    signsRequests?: boolean;
+    /**
+     * A robots.txt control token that never appears in a request. `Google-Extended`
+     * and `Applebot-Extended` govern how an already-crawled page may be used; no
+     * user agent carries these strings, so probing a site with them tests nothing.
+     */
+    tokenOnly?: boolean;
+    /** Extra context: renames, documentation changes, unusual behaviour. */
+    note?: string;
+}
+/**
+ * Per-token metadata for the crawlers worth explaining. Tokens in
+ * `AI_CRAWLERS` without an entry here are recognised for matching but carry no
+ * tailored advice — the long tail of data brokers and regional assistants.
+ *
+ * Every entry was verified against vendor documentation on 2026-09-04.
+ */
+export declare const CRAWLER_META: Record<string, CrawlerInfo>;
+/**
+ * Known AI clients grouped by what they do with a page. Matching is
+ * case-insensitive, per RFC 9309 §2.2.1, so the casing here is cosmetic and
+ * follows each vendor's own documentation.
+ *
+ * Verified against vendor documentation on 2026-09-04. Tokens that turned out
+ * never to have existed (`Gemini`, `GeminiBot`, `DeepSeek-AI`) or to belong to
+ * discontinued products (`NeevaBot`, `GoogleAgent-Mariner`) moved to
+ * `LEGACY_AI_CRAWLERS`.
+ */
+export declare const AI_CRAWLERS: Record<CrawlerPurpose, string[]>;
+/**
+ * Tokens recognised for matching but never recommended: renamed, retired, or
+ * never real. A site that lists these is not wrong, but the rules are inert, so
+ * the audit says so rather than counting them as coverage.
+ */
+export declare const LEGACY_AI_CRAWLERS: Record<string, string>;
 export declare const ALL_AI_CRAWLERS: string[];
 /**
- * The "must-configure" subset — these are the agents a typical operator should grant
- * (or knowingly deny) explicit access to. ax-audit grades the robots.txt section heavily
- * on coverage of this short list, while still rewarding broader explicit rules.
+ * The crawlers that matter most as of September 2026, by request share
+ * (Cloudflare Radar) and by what a site loses when each is blocked. Used for
+ * reporting, access probes, and remediation advice.
+ *
+ * Composition: the six highest-volume clients (Googlebot's AI use is governed
+ * by Google-Extended, Applebot's by Applebot-Extended, hence the opt-out tokens
+ * standing in for them), the broadest training corpus (CCBot), the three search
+ * bots whose absence costs citations, and the one user-triggered fetcher with
+ * material volume.
  */
 export declare const CORE_AI_CRAWLERS: string[];
 /**
- * Default weight per check (sum: 100). Individual `CheckMeta.weight` overrides this map.
- * Keep weights aligned with real-world impact — discovery + content-rendering are the
- * highest-leverage signals for AI agents.
+ * Core crawlers that actually issue HTTP requests.
+ *
+ * `Google-Extended` and `Applebot-Extended` are robots.txt control tokens: they
+ * govern how an already-crawled page may be used, and no request ever carries
+ * them as a user agent. Probing a site with those strings tests nothing, so
+ * access checks use this list instead of the full core set.
+ */
+export declare const PROBEABLE_CORE_CRAWLERS: string[];
+/** Look up a crawler's metadata case-insensitively. */
+export declare function crawlerInfo(token: string): CrawlerInfo | undefined;
+/** Purpose bucket a token belongs to, or `undefined` when it is not a known AI client. */
+export declare function crawlerPurpose(token: string): CrawlerPurpose | undefined;
+/** Explanation for a retired or fictional token, or `undefined` when the token is current. */
+export declare function legacyCrawlerNote(token: string): string | undefined;
+/**
+ * Weight per check, summing to 100. A check's own `meta.weight` overrides this
+ * map.
+ *
+ * The 4.0 distribution follows the evidence about what actually stops an agent,
+ * rather than what is easiest to check:
+ *
+ * - **Content (33)** leads, because the failure that breaks the most agents is
+ *   a page with nothing in the HTML. Most crawlers do not run JavaScript, and a
+ *   site whose content only appears after hydration is invisible to them no
+ *   matter how many discovery files it publishes. `agent-operability` joins it
+ *   at 7: browser agents read the accessibility tree, and a page of unnamed
+ *   controls cannot be operated at all.
+ * - **Access (24)** is second, because a WAF rule or a `nosnippet` directive
+ *   silently undoes everything else. These are also the failures operators are
+ *   least likely to know about.
+ * - **Discovery (21)** carries robots.txt at 9 and llms.txt at 5. llms.txt was
+ *   the highest-weighted check in 3.x at 11; it is demoted because the evidence
+ *   is that most published files are never fetched by an AI search crawler, and
+ *   the vendors that do read it are coding agents. It matters, but not twice as
+ *   much as whether the page has content.
+ * - **Protocols (13)** are all conditional: a site without the surface reports
+ *   N/A and the weight leaves the denominator, so a blog is never marked down
+ *   for lacking an API description.
+ * - **Policy (9)** covers declared usage rights and security contact.
+ *
+ * Checks on draft specifications (`ai-catalog`, `webmcp`, `commerce-discovery`)
+ * stay at 0. Scoring a site against a specification that may be renamed next
+ * quarter would make the number less trustworthy, not more.
  */
 export declare const CHECK_WEIGHTS: Record<string, number>;
+/**
+ * Report grouping per check. A check's own `meta.category` takes precedence.
+ * Mirrors the `CHECK_WEIGHTS` pattern: data lives here, overrides live on the check.
+ */
+export declare const CHECK_CATEGORIES: Record<string, CheckCategory>;
 export declare const GRADES: Grade[];
 /**
  * Content Signals Policy vocabulary (https://contentsignals.org, CC0): machine-readable
@@ -31,6 +135,28 @@ export declare const GRADES: Grade[];
  * Absence of a signal is neutral — it neither grants nor restricts.
  */
 export declare const CONTENT_SIGNALS: string[];
+/**
+ * Values of the optional fourth Content Signals field `use`, introduced by
+ * Cloudflare on 2026-07-01 and now emitted by its managed robots.txt:
+ * `immediate` (interact, store nothing), `reference` (index, excerpt, link
+ * back — the stated default) and `full` (summarize and reproduce).
+ * https://blog.cloudflare.com/content-independence-day-ai-options/
+ */
+export declare const CONTENT_SIGNAL_USE_VALUES: string[];
+/**
+ * IETF AIPREF vocabulary (draft-ietf-aipref-vocab-07, 2026-08-19). Two
+ * categories only: `train-ai` (modifying model parameters) and `search`
+ * (retrieval that links back to the original). Values are `y` / `n`; an absent
+ * token means "unknown", never "allowed".
+ *
+ * The drafts are pre-working-group-last-call and carry the "DO NOT REFLECT
+ * CONSENSUS" boilerplate, so ax-audit reports these directives without ever
+ * requiring them. Note the token order is inverted against Content Signals
+ * (`train-ai` here, `ai-train` there).
+ * https://datatracker.ietf.org/wg/aipref/documents/
+ */
+export declare const AIPREF_TOKENS: string[];
+export declare const AIPREF_VALUES: string[];
 /**
  * Really Simple Licensing 1.0 (https://rslstandard.org/rsl): machine-readable licensing
  * terms for content, discovered via robots.txt `License:`, HTTP `Link: rel="license"`,
@@ -42,7 +168,62 @@ export declare const RSL_PERMIT_TYPES: string[];
 export declare const RSL_USAGE_TOKENS: string[];
 export declare const RSL_USER_TOKENS: string[];
 export declare const RSL_PAYMENT_TYPES: string[];
+/**
+ * A2A Agent Card (https://a2a-protocol.org).
+ *
+ * Two generations are in the wild. A2A 1.0 (2026-03-12) replaced the top-level
+ * `url`, `protocolVersion`, `preferredTransport` and `additionalInterfaces`
+ * fields with one `supportedInterfaces[]` array; 0.3-shaped cards are still the
+ * majority deployed. Required-field lists are taken from `specification/a2a.proto`
+ * (1.0) and `specification/json/a2a.json` at tag v0.3.0.
+ */
+export declare const AGENT_CARD_REQUIRED_V1: string[];
+export declare const AGENT_CARD_REQUIRED_V03: string[];
+/** Transport bindings a 1.0 interface may declare. */
+export declare const A2A_PROTOCOL_BINDINGS: string[];
+/** Media type registered for A2A payloads (spec §14.1.1). */
+export declare const A2A_MEDIA_TYPE = "application/a2a+json";
+/** @deprecated Superseded by AGENT_CARD_REQUIRED_V03 / _V1 in 3.7. Kept for one minor. */
 export declare const AGENT_JSON_REQUIRED_FIELDS: string[];
+/**
+ * Model Context Protocol.
+ *
+ * `/.well-known/mcp.json` was never part of the specification; ax-audit
+ * recommended it before the ecosystem settled. The actual discovery work is
+ * SEP-2127 (open draft) and the `experimental-ext-server-card` repository,
+ * which recommend a **server card** at `<streamable-http-url>/server-card` and
+ * an entry in `/.well-known/ai-catalog.json`. Cloudflare and Mintlify serve one
+ * at `/.well-known/mcp/server-card.json`.
+ *
+ * Server cards deliberately carry no `tools[]`: tool lists come from a live
+ * `tools/list` call, not from a static document that would immediately drift.
+ */
+export declare const MCP_SERVER_CARD_MEDIA_TYPE = "application/mcp-server-card+json";
+export declare const MCP_SERVER_CARD_REQUIRED: string[];
+export declare const MCP_REMOTE_TYPES: string[];
+/**
+ * Released MCP protocol versions, newest first. `2026-07-28` removed sessions
+ * and `initialize`, made `MCP-Protocol-Version`, `Mcp-Method` and `Mcp-Name`
+ * mandatory on every POST, and added the `server/discover` RPC.
+ * https://modelcontextprotocol.io/specification/versioning
+ */
+export declare const MCP_PROTOCOL_VERSIONS: string[];
+/** Versions old enough that a card advertising only these is worth flagging. */
+export declare const MCP_STALE_PROTOCOL_VERSIONS: string[];
+/** Media types an ai-catalog entry may declare, per the Agent Card WG draft. */
+export declare const AI_CATALOG_ENTRY_TYPES: Record<string, string>;
+/**
+ * Where API descriptions live, in order of authority.
+ *
+ * `/.well-known/openapi.json` is a folk convention: it is not IANA-registered,
+ * and the OpenAPI specification recommends the file name `openapi.json` /
+ * `openapi.yaml` without prescribing a location. It stays first because
+ * ax-audit recommended it before 3.7 and sites followed that advice, but the
+ * common real-world locations are probed too.
+ */
+export declare const API_DESCRIPTION_PATHS: string[];
+/** RFC 9264 linkset media type, required by RFC 9727 for the API catalog. */
+export declare const LINKSET_MEDIA_TYPE = "application/linkset+json";
 export declare const SECURITY_TXT_REQUIRED_FIELDS: string[];
 export declare const SECURITY_HEADERS: SecurityHeader[];
 //# sourceMappingURL=constants.d.ts.map

package/dist/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKxD,eAAO,MAAM,OAAO,EAAE,MAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,QAAqE,CAAC;AAE7F;;;;;;;;GAQG;AACH,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAuDhD,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,EAA8E,CAAC;AAEnH;;;;GAIG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,EASpC,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAqBhD,CAAC;AAEF,eAAO,MAAM,MAAM,EAAE,KAAK,EAKzB,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,MAAM,EAAuC,CAAC;AAE5E;;;;GAIG;AACH,eAAO,MAAM,aAAa,gCAAgC,CAAC;AAC3D,eAAO,MAAM,QAAQ,wBAAwB,CAAC;AAC9C,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAA6B,CAAC;AACnE,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAAoE,CAAC;AAC1G,eAAO,MAAM,eAAe,EAAE,MAAM,EAA4E,CAAC;AACjH,eAAO,MAAM,iBAAiB,EAAE,MAAM,EASrC,CAAC;AAEF,eAAO,MAAM,0BAA0B,EAAE,MAAM,EAA6C,CAAC;AAE7F,eAAO,MAAM,4BAA4B,EAAE,MAAM,EAA2B,CAAC;AAE7E,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAQ5C,CAAC"}
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,aAAa,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAKvE,eAAO,MAAM,OAAO,EAAE,MAAoB,CAAC;AAC3C,eAAO,MAAM,UAAU,QAAqE,CAAC;AAE7F;;;;;;;;;;GAUG;AACH,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,QAAQ,GAAG,YAAY,GAAG,OAAO,CAAC;AAE5E,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,cAAc,CAAC;IACxB;;;;OAIG;IACH,YAAY,EAAE,OAAO,GAAG,SAAS,CAAC;IAClC,sFAAsF;IACtF,MAAM,EAAE,MAAM,CAAC;IACf,0CAA0C;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,qFAAqF;IACrF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kFAAkF;IAClF,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAkRpD,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,MAAM,EAAE,CA2DxD,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBrD,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,EAKnC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAapC,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB,EAAE,MAAM,EAE3C,CAAC;AAEF,uDAAuD;AACvD,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,GAAG,SAAS,CAGlE;AAED,0FAA0F;AAC1F,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAMxE;AAED,8FAA8F;AAC9F,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGnE;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAsChD,CAAC;AAEF;;;GAGG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CA+B1D,CAAC;AAEF,eAAO,MAAM,MAAM,EAAE,KAAK,EAKzB,CAAC;AAEF;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAAE,MAAM,EAAuC,CAAC;AAE5E;;;;;;GAMG;AACH,eAAO,MAAM,yBAAyB,EAAE,MAAM,EAAuC,CAAC;AAEtF;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,aAAa,EAAE,MAAM,EAA2B,CAAC;AAC9D,eAAO,MAAM,aAAa,EAAE,MAAM,EAAe,CAAC;AAElD;;;;GAIG;AACH,eAAO,MAAM,aAAa,gCAAgC,CAAC;AAC3D,eAAO,MAAM,QAAQ,wBAAwB,CAAC;AAC9C,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAA6B,CAAC;AACnE,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAAoE,CAAC;AAC1G,eAAO,MAAM,eAAe,EAAE,MAAM,EAA4E,CAAC;AACjH,eAAO,MAAM,iBAAiB,EAAE,MAAM,EASrC,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,MAAM,EAS1C,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAM,EAU3C,CAAC;AAEF,sDAAsD;AACtD,eAAO,MAAM,qBAAqB,EAAE,MAAM,EAAqC,CAAC;AAEhF,6DAA6D;AAC7D,eAAO,MAAM,cAAc,yBAAyB,CAAC;AAErD,0FAA0F;AAC1F,eAAO,MAAM,0BAA0B,EAAE,MAAM,EAA6C,CAAC;AAE7F;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,0BAA0B,qCAAqC,CAAC;AAC7E,eAAO,MAAM,wBAAwB,EAAE,MAAM,EAAkD,CAAC;AAChG,eAAO,MAAM,gBAAgB,EAAE,MAAM,EAA+B,CAAC;AAErE;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,EAA2E,CAAC;AAEtH,gFAAgF;AAChF,eAAO,MAAM,2BAA2B,EAAE,MAAM,EAAiC,CAAC;AAElF,gFAAgF;AAChF,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAIzD,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,qBAAqB,EAAE,MAAM,EAWzC,CAAC;AAEF,6EAA6E;AAC7E,eAAO,MAAM,kBAAkB,6BAA6B,CAAC;AAE7D,eAAO,MAAM,4BAA4B,EAAE,MAAM,EAA2B,CAAC;AAE7E,eAAO,MAAM,gBAAgB,EAAE,cAAc,EAQ5C,CAAC"}