@traceten/ai-crawl 0.1.0

package/src/types.ts

@@ -0,0 +1,190 @@
+/**
+ * Public types for @traceten/ai-crawl.
+ *
+ * The package is a PRE-FILTER only: it decides "is this plausibly an AI
+ * crawler worth reporting" locally, and sends a minimal payload. Provider
+ * classification, category, verification and confidence are all decided
+ * server-side, so the crawler list can change without customers upgrading.
+ * The local token list is a cost filter, not the source of truth.
+ */
+
+/**
+ * The four-category taxonomy Traceten uses server-side. Used locally ONLY to
+ * apply the `disable*` opt-outs — the authoritative category is assigned
+ * server-side.
+ */
+export type CrawlerCategory = "answer_fetch" | "search_index" | "training" | "ai_crawler";
+
+/** Result of the local two-tier user-agent match. */
+export interface CrawlerMatch {
+  /** Exact agent token that matched (tier 1), or `null` for a tier-2 provider-alias match. */
+  readonly agent: string | null;
+  /** Coarse provider slug (e.g. `"openai"`). */
+  readonly provider: string;
+  /** Local category guess — used only for the `disable*` opt-outs. */
+  readonly category: CrawlerCategory;
+  /** `"exact"` = tier-1 agent token; `"provider"` = tier-2 coarse alias. */
+  readonly tier: "exact" | "provider";
+}
+
+/** User-supplied configuration. Validate with {@link import("./config.js").defineAiCrawlConfig}. */
+export interface AiCrawlConfig {
+  /**
+   * The site's public site key — the `ttid_…` value shown on the dashboard's
+   * install page (`data-site`). Case-sensitive. Sent as `site_id` on the
+   * wire. Required.
+   */
+  siteId: string;
+  /**
+   * Server-side auth token (`tt_bot_...`). REQUIRED — `POST /v1/ai-crawls`
+   * rejects unauthenticated reports. Unlike the browser snippet, a
+   * server-side package can hold a real secret; requiring it removes the
+   * cost-attack and data-poisoning surface. Missing/malformed tokens fail
+   * loudly at construction rather than silently sending requests that 401.
+   */
+  authToken: string;
+  /** Ingestion endpoint. Default: `https://ingest.traceten.com/v1/ai-crawls`. */
+  endpoint?: string;
+  /** HTTP methods to consider. Default `["GET", "HEAD"]`. Replaces the default. */
+  allowedMethods?: readonly string[];
+  /** Extra path prefixes to deny, ADDED to the built-in list (never replaces it). */
+  extraDenyPathPrefixes?: readonly string[];
+  /** Extra file extensions to deny (with or without leading dot), ADDED to the built-in list. */
+  extraDenyExtensions?: readonly string[];
+  /** Skip reporting `answer_fetch` crawlers (ChatGPT-User, Claude-User, …). */
+  disableAnswerFetch?: boolean;
+  /** Skip reporting `search_index` crawlers (OAI-SearchBot, PerplexityBot, …). */
+  disableSearchCrawlers?: boolean;
+  /** Skip reporting `training` crawlers (GPTBot, ClaudeBot, CCBot, …). */
+  disableTrainingCrawlers?: boolean;
+  /** Skip reporting `ai_crawler` (uncategorised) crawlers and tier-2 provider-alias matches. */
+  disableOtherCrawlers?: boolean;
+  /**
+   * Trust `x-forwarded-for` when resolving the crawler IP. Default `false`:
+   * XFF is attacker-controlled unless a proxy you operate strips/appends it,
+   * so it is never trusted blindly. Set `true` only when your app sits behind
+   * proxies you control, and set {@link proxyDepth} to how many of them
+   * append to XFF.
+   */
+  trustProxy?: boolean;
+  /**
+   * Trust the `cf-connecting-ip` header. Default `false`: on an origin NOT
+   * behind Cloudflare, any client can forge this header, and a forged
+   * vendor-range IP paired with a vendor UA is exactly the spoof the
+   * server-side verification exists to catch. The two Cloudflare adapters
+   * enable it automatically (the platform strips and rewrites the header
+   * there). Set it manually only when your origin genuinely sits behind
+   * Cloudflare (e.g. Express behind orange-cloud DNS).
+   */
+  trustCfConnectingIp?: boolean;
+  /**
+   * Number of trusted reverse proxies that append to `x-forwarded-for`.
+   * Default `1`. With N trusted proxies the crawler IP is the Nth entry from
+   * the right (rightmost-untrusted-aware selection). Only used when
+   * {@link trustProxy} is `true`.
+   */
+  proxyDepth?: number;
+  /**
+   * Origin (e.g. `"https://example.com"`) used to rebuild the reported URL
+   * when the runtime sees an internal one (containers / reverse proxies that
+   * construct `request.url` from an internal host).
+   */
+  publicOrigin?: string;
+  /**
+   * Called when a report does not reach the endpoint: a non-2xx response, or
+   * a network/timeout failure.
+   *
+   * Delivery stays silent by contract. This never throws into your request
+   * path, nothing is retried, and the response is not surfaced any other way.
+   * Without it a rejected report is indistinguishable from a delivered one, so
+   * a misconfigured token or blocked egress reads as "no crawlers visited"
+   * indefinitely. Wire it to your logger in staging at minimum.
+   *
+   * Your callback is itself wrapped in try/catch. Throwing from it cannot
+   * break the host response.
+   */
+  onError?: (error: AiCrawlDeliveryError) => void;
+  /** Injectable fetch, for tests. Defaults to the global `fetch`. */
+  fetch?: typeof fetch;
+}
+
+/** Why a report failed to land. Passed to {@link AiCrawlConfig.onError}. */
+export interface AiCrawlDeliveryError {
+  /**
+   * `"http"` when the endpoint answered with a non-2xx status.
+   * `"network"` when the request never completed (DNS, TLS, timeout, abort).
+   */
+  kind: "http" | "network";
+  /** Status when `kind` is `"http"`. A 401/403 means the token or the origin. */
+  status?: number;
+  /** The thrown value when `kind` is `"network"`. */
+  cause?: unknown;
+  /** The endpoint the report was addressed to. */
+  endpoint: string;
+}
+
+/** Config after validation + defaulting. Produced by `defineAiCrawlConfig`. */
+export interface ResolvedAiCrawlConfig {
+  readonly siteId: string;
+  readonly authToken: string;
+  readonly endpoint: string;
+  readonly allowedMethods: readonly string[];
+  readonly denyPathPrefixes: readonly string[];
+  readonly denyExtensions: readonly string[];
+  readonly disableAnswerFetch: boolean;
+  readonly disableSearchCrawlers: boolean;
+  readonly disableTrainingCrawlers: boolean;
+  readonly disableOtherCrawlers: boolean;
+  readonly trustProxy: boolean;
+  readonly trustCfConnectingIp: boolean;
+  readonly proxyDepth: number;
+  readonly publicOrigin: string | undefined;
+  readonly onError: ((error: AiCrawlDeliveryError) => void) | undefined;
+  readonly fetch: typeof fetch | undefined;
+  /** Brand so per-request paths can tell a validated config from a raw one. */
+  readonly __resolved: true;
+}
+
+/**
+ * Wire payload for `POST /v1/ai-crawls`. snake_case on the wire, per
+ * Traceten's wire-format convention.
+ *
+ * `ip` is the CRAWLER's IP as observed by the customer's server — the only
+ * place it is observable, since the crawler's TCP connection terminates
+ * there. Traceten's edge verifies it against vendor-published ranges while
+ * it is in scope; retention is verdict-dependent and decided server-side.
+ * The field is omitted entirely when no trustworthy value is derivable —
+ * never guessed.
+ *
+ * ⚠️ `ip` is a CLAIM made by customer-controlled software, not network
+ * proof. The receiving service must verify it (CIDR / rDNS) before treating
+ * it as crawler infrastructure, and must never let an unverified claimed IP
+ * mint a verified/raw-retained row.
+ */
+export interface AiCrawlWirePayload {
+  site_id: string;
+  /** Full request URL (absolute where derivable). */
+  url: string;
+  method: string;
+  /** Response status where the adapter can observe it; omitted otherwise. */
+  status?: number;
+  user_agent: string;
+  /** Crawler IP. Omitted rather than wrong. */
+  ip?: string;
+  /** Epoch milliseconds at observation time. */
+  ts: number;
+}
+
+/**
+ * Runtime-agnostic view of a request, produced by each adapter.
+ * Header names are looked up case-insensitively by the adapter.
+ */
+export interface RequestFacts {
+  method: string;
+  /** Absolute URL if the runtime provides one, else a path like `/docs/x`. */
+  url: string;
+  /** Case-insensitive header lookup; returns `null` when absent. */
+  header: (name: string) => string | null;
+  /** Socket remote address, where the runtime exposes one (Node servers). */
+  socketAddr?: string | undefined;
+}