@traceten/ai-crawl 0.1.0 → 1.0.0

package/CHANGELOG.md

@@ -5,7 +5,23 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/).
 
-## [0.1.0] - Unreleased
+## [1.0.0] - Unreleased
+
+First stable release. No functional change since 0.1.1 — the API has held
+steady in production use, so this marks it stable rather than adding to it.
+
+## [0.1.1] - 2026-09-05
+
+### Changed
+
+- Delivery timeout raised from 1500 ms to 5000 ms. Reports are scheduled off
+  the host response path in every adapter, so the old value delayed nothing and
+  bounded nothing useful. It did abort reports the ingest edge was still
+  answering: measured against a real deployment, roughly 15% of reports were
+  lost sequentially and 32% under concurrency, each one a client-side
+  `TimeoutError`. Those crawls are now delivered.
+
+## [0.1.0] - 2026-09-05
 
 Initial public release.
 

package/README.md

@@ -1,6 +1,7 @@
+<img src="https://traceten.com/logos/traceten-wordmark-black.png" alt="Traceten" width="320" />
+
 # @traceten/ai-crawl
 
-[![CI](https://github.com/traceten/ai-crawl/actions/workflows/ci.yml/badge.svg)](https://github.com/traceten/ai-crawl/actions/workflows/ci.yml)
 [![npm version](https://img.shields.io/npm/v/@traceten/ai-crawl.svg)](https://www.npmjs.com/package/@traceten/ai-crawl)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE)
 [![Node](https://img.shields.io/badge/node-%3E%3D18-brightgreen)](https://nodejs.org)
@@ -21,7 +22,8 @@ Provider, category (answer fetch, search index, training, other), verification a
 
 - Never blocks the response. Never throws per-request. Failure is silent.
 - Zero runtime dependencies.
-- 1500 ms delivery timeout, `keepalive: true`.
+- 5000 ms delivery timeout, `keepalive: true`. Delivery is scheduled off your
+  response path, so this bounds a background task and never delays a response.
 - Requires Node 18+ or an edge runtime with `fetch`.
 - Each adapter bundle is under 8 KB gzipped (CI-enforced).
 

package/dist/matcher.d.ts

@@ -1,9 +1,9 @@
 /**
  * Anchored two-tier user-agent matching.
  *
- * NEVER `includes()` on the whole string — that is the reference package's
- * defect (its page claims token matching; its code does substring matching,
- * so a browser UA containing `grok` or `copilot` false-positives). Follows
+ * NEVER `includes()` on the whole string. Substring matching false-positives
+ * on ordinary browser traffic: a UA containing `grok` or `copilot` anywhere
+ * would be reported as a crawler. Follows
  * the same anchoring discipline Traceten's server-side classifier uses,
  * adapted for crawler UAs where the token sits mid-string
  * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).

package/dist/matcher.js

@@ -1,9 +1,9 @@
 /**
  * Anchored two-tier user-agent matching.
  *
- * NEVER `includes()` on the whole string — that is the reference package's
- * defect (its page claims token matching; its code does substring matching,
- * so a browser UA containing `grok` or `copilot` false-positives). Follows
+ * NEVER `includes()` on the whole string. Substring matching false-positives
+ * on ordinary browser traffic: a UA containing `grok` or `copilot` anywhere
+ * would be reported as a crawler. Follows
  * the same anchoring discipline Traceten's server-side classifier uses,
  * adapted for crawler UAs where the token sits mid-string
  * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).

package/dist/report.d.ts

@@ -1,7 +1,7 @@
 /**
  * Payload construction + delivery.
  *
- * Contract: never throws, never blocks the response, 1500 ms
+ * Contract: never throws, never blocks the response, 5000 ms
  * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
  * silent — a broken analytics call must never surface on a customer's site.
  *
@@ -10,7 +10,25 @@
  * told from a delivered one, and a bad token reads as "no crawlers visited".
  */
 import type { AiCrawlWirePayload, ResolvedAiCrawlConfig } from "./types.js";
-export declare const REPORT_TIMEOUT_MS = 1500;
+/**
+ * Delivery budget for one report.
+ *
+ * Generous ON PURPOSE. Every adapter schedules delivery off the host's
+ * response path - `event.waitUntil()` on Next and both Cloudflare adapters,
+ * bare `void sendReport(...)` on Express and Hono - so this value never adds
+ * a millisecond to your response. It bounds a background task, not a request.
+ *
+ * It was 1500, chosen as though it were on the response path. It is not, and
+ * Traceten's edge grants itself time for a rate-limiter check before its 204.
+ * At 1500 the two were close enough that ordinary edge latency aborted the
+ * report client-side before the edge's own fail-open could answer, so the
+ * crawl was lost rather than degraded. Measured against a real deployment,
+ * 2026-09-05: ~15% of reports lost sequentially, ~32% under 8-way
+ * concurrency, every one a client-side TimeoutError.
+ *
+ * If you ever move delivery ONTO your response path, lower this.
+ */
+export declare const REPORT_TIMEOUT_MS = 5000;
 /** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
 export declare function buildReportUrl(cfg: ResolvedAiCrawlConfig, url: string): string;
 export interface ReportInput {

package/dist/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAEV,kBAAkB,EAClB,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC,qFAAqF;AACrF,wBAAgB,cAAc,CAAC,GAAG,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAa9E;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,CAAC,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,WAAW,GAAG,kBAAkB,CAe/F;AAcD,wBAAwB;AACxB,MAAM,WAAW,WAAW;IAC1B;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,qBAAqB,EAC1B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC,CAsCf"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAEV,kBAAkB,EAClB,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;;;;;;;;;GAiBG;AACH,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC,qFAAqF;AACrF,wBAAgB,cAAc,CAAC,GAAG,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAa9E;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,CAAC,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,WAAW,GAAG,kBAAkB,CAe/F;AAcD,wBAAwB;AACxB,MAAM,WAAW,WAAW;IAC1B;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,qBAAqB,EAC1B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC,CAsCf"}

package/dist/report.js

@@ -1,7 +1,7 @@
 /**
  * Payload construction + delivery.
  *
- * Contract: never throws, never blocks the response, 1500 ms
+ * Contract: never throws, never blocks the response, 5000 ms
  * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
  * silent — a broken analytics call must never surface on a customer's site.
  *
@@ -9,7 +9,25 @@
  * or a network failure to the caller. Without it a rejected report cannot be
  * told from a delivered one, and a bad token reads as "no crawlers visited".
  */
-export const REPORT_TIMEOUT_MS = 1500;
+/**
+ * Delivery budget for one report.
+ *
+ * Generous ON PURPOSE. Every adapter schedules delivery off the host's
+ * response path - `event.waitUntil()` on Next and both Cloudflare adapters,
+ * bare `void sendReport(...)` on Express and Hono - so this value never adds
+ * a millisecond to your response. It bounds a background task, not a request.
+ *
+ * It was 1500, chosen as though it were on the response path. It is not, and
+ * Traceten's edge grants itself time for a rate-limiter check before its 204.
+ * At 1500 the two were close enough that ordinary edge latency aborted the
+ * report client-side before the edge's own fail-open could answer, so the
+ * crawl was lost rather than degraded. Measured against a real deployment,
+ * 2026-09-05: ~15% of reports lost sequentially, ~32% under 8-way
+ * concurrency, every one a client-side TimeoutError.
+ *
+ * If you ever move delivery ONTO your response path, lower this.
+ */
+export const REPORT_TIMEOUT_MS = 5000;
 /** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
 export function buildReportUrl(cfg, url) {
     try {
@@ -44,7 +62,7 @@ export function buildPayload(cfg, input) {
     }
     return payload;
 }
-/** Abort signal with a 1500 ms timeout, where the runtime supports it. */
+/** Abort signal with the delivery timeout, where the runtime supports it. */
 function timeoutSignal() {
     try {
         if (typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function") {

package/dist/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAQH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC,qFAAqF;AACrF,MAAM,UAAU,cAAc,CAAC,GAA0B,EAAE,GAAW;IACpE,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,sCAAsC;YACtC,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC7F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAUD,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAAC,GAA0B,EAAE,KAAkB;IACzE,MAAM,OAAO,GAAuB;QAClC,OAAO,EAAE,GAAG,CAAC,MAAM;QACnB,GAAG,EAAE,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE;QAClC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QAC1C,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;KACf,CAAC;IACF,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,SAAS,aAAa;IACpB,IAAI,CAAC;QACH,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,WAAW,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACpF,OAAO,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAeD;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,GAA0B,EAC1B,OAA2B,EAC3B,OAAqB;IAErB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAEtD,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAgB;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,SAAS,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC;QACF,oEAAoE;QACpE,6CAA6C;QAC7C,IAAI,OAAO,EAAE,SAAS,KAAK,KAAK;YAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxD,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAE/C,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CACvC,CAAC,GAAG,EAAE,EAAE;YACN,uEAAuE;YACvE,uEAAuE;YACvE,iEAAiE;YACjE,uEAAuE;YACvE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;gBACzD,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;YACjB,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAA0B,EAAE,KAA2B;IAC1E,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;AACH,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAQH;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC,qFAAqF;AACrF,MAAM,UAAU,cAAc,CAAC,GAA0B,EAAE,GAAW;IACpE,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,sCAAsC;YACtC,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC7F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAUD,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAAC,GAA0B,EAAE,KAAkB;IACzE,MAAM,OAAO,GAAuB;QAClC,OAAO,EAAE,GAAG,CAAC,MAAM;QACnB,GAAG,EAAE,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE;QAClC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QAC1C,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;KACf,CAAC;IACF,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,6EAA6E;AAC7E,SAAS,aAAa;IACpB,IAAI,CAAC;QACH,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,WAAW,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACpF,OAAO,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAeD;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,GAA0B,EAC1B,OAA2B,EAC3B,OAAqB;IAErB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAEtD,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAgB;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,SAAS,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC;QACF,oEAAoE;QACpE,6CAA6C;QAC7C,IAAI,OAAO,EAAE,SAAS,KAAK,KAAK;YAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxD,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAE/C,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CACvC,CAAC,GAAG,EAAE,EAAE;YACN,uEAAuE;YACvE,uEAAuE;YACvE,iEAAiE;YACjE,uEAAuE;YACvE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;gBACzD,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;YACjB,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAA0B,EAAE,KAA2B;IAC1E,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;AACH,CAAC"}

package/package.json

@@ -1,16 +1,28 @@
 {
   "name": "@traceten/ai-crawl",
-  "version": "0.1.0",
+  "version": "1.0.0",
   "private": false,
   "description": "Traceten server-side AI crawler tracking — see the GPTBot/ClaudeBot/PerplexityBot crawls that browser JavaScript can never see. Five adapters, zero dependencies, never blocks a response.",
   "keywords": [
+    "traceten",
+    "traceten.com",
     "ai-crawler",
     "gptbot",
     "claudebot",
     "perplexitybot",
     "bot-detection",
     "analytics",
-    "traceten"
+    "ai-analytics",
+    "web-analytics",
+    "ai-traffic",
+    "ai-visibility",
+    "chatgpt",
+    "perplexity",
+    "llm-traffic",
+    "generative-engine-optimization",
+    "aeo",
+    "server-side-analytics",
+    "nodejs"
   ],
   "license": "MIT",
   "homepage": "https://docs.traceten.com/install/ai-crawler-tracking",

package/src/matcher.ts

@@ -1,9 +1,9 @@
 /**
  * Anchored two-tier user-agent matching.
  *
- * NEVER `includes()` on the whole string — that is the reference package's
- * defect (its page claims token matching; its code does substring matching,
- * so a browser UA containing `grok` or `copilot` false-positives). Follows
+ * NEVER `includes()` on the whole string. Substring matching false-positives
+ * on ordinary browser traffic: a UA containing `grok` or `copilot` anywhere
+ * would be reported as a crawler. Follows
  * the same anchoring discipline Traceten's server-side classifier uses,
  * adapted for crawler UAs where the token sits mid-string
  * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).

package/src/report.ts

@@ -1,7 +1,7 @@
 /**
  * Payload construction + delivery.
  *
- * Contract: never throws, never blocks the response, 1500 ms
+ * Contract: never throws, never blocks the response, 5000 ms
  * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
  * silent — a broken analytics call must never surface on a customer's site.
  *
@@ -16,7 +16,25 @@ import type {
   ResolvedAiCrawlConfig,
 } from "./types.js";
 
-export const REPORT_TIMEOUT_MS = 1500;
+/**
+ * Delivery budget for one report.
+ *
+ * Generous ON PURPOSE. Every adapter schedules delivery off the host's
+ * response path - `event.waitUntil()` on Next and both Cloudflare adapters,
+ * bare `void sendReport(...)` on Express and Hono - so this value never adds
+ * a millisecond to your response. It bounds a background task, not a request.
+ *
+ * It was 1500, chosen as though it were on the response path. It is not, and
+ * Traceten's edge grants itself time for a rate-limiter check before its 204.
+ * At 1500 the two were close enough that ordinary edge latency aborted the
+ * report client-side before the edge's own fail-open could answer, so the
+ * crawl was lost rather than degraded. Measured against a real deployment,
+ * 2026-09-05: ~15% of reports lost sequentially, ~32% under 8-way
+ * concurrency, every one a client-side TimeoutError.
+ *
+ * If you ever move delivery ONTO your response path, lower this.
+ */
+export const REPORT_TIMEOUT_MS = 5000;
 
 /** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
 export function buildReportUrl(cfg: ResolvedAiCrawlConfig, url: string): string {
@@ -60,7 +78,7 @@ export function buildPayload(cfg: ResolvedAiCrawlConfig, input: ReportInput): Ai
   return payload;
 }
 
-/** Abort signal with a 1500 ms timeout, where the runtime supports it. */
+/** Abort signal with the delivery timeout, where the runtime supports it. */
 function timeoutSignal(): AbortSignal | undefined {
   try {
     if (typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function") {