@traceten/ai-crawl 0.1.0

package/dist/adapters/express.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"express.js","sourceRoot":"","sources":["../../src/adapters/express.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAwB1C,SAAS,YAAY,CAAC,GAAoB;IACxC,OAAO,CAAC,IAAY,EAAiB,EAAE;QACrC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YAC9C,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAO,IAAI,CAAC;YACrC,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,wEAAwE;AACxE,SAAS,UAAU,CAAC,GAA0B,EAAE,GAAoB;IAClE,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC;IAC/C,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC,CAAC,0BAA0B;IACvE,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC;IACnE,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;IACpE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,IAAI,CAAC;IACnD,MAAM,KAAK,GAAG,GAAG,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;IACzD,OAAO,GAAG,KAAK,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC;AACrC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iCAAiC,CAC/C,MAA6C;IAE7C,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5E,OAAO,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,KAAK,GAAiB;gBAC1B,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,KAAK;gBAC3B,GAAG,EAAE,UAAU,CAAC,GAAG,EAAE,GAAG,CAAC;gBACzB,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC;gBACzB,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa;aACtC,CAAC;YAEF,mEAAmE;YACnE,+DAA+D;YAC/D,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAC5C,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;oBACpB,IAAI,CAAC;wBACH,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC;wBAC9B,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;4BACjE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;wBAC1B,CAAC;wBACD,KAAK,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;oBAChC,CAAC;oBAAC,MAAM,CAAC;wBACP,wBAAwB;oBAC1B,CAAC;gBACH,CAAC,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;QACD,IAAI,EAAE,CAAC;IACT,CAAC,CAAC;AACJ,CAAC"}

package/dist/adapters/hono.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Hono adapter + generic `Request`/`Response` helper.
+ *
+ * Hono (structural types — no dependency on hono):
+ * ```ts
+ * import { aiCrawlerTracking } from "@traceten/ai-crawl/hono";
+ * app.use("*", aiCrawlerTracking({ siteId: "…", authToken: "tt_bot_…" }));
+ * ```
+ *
+ * Any framework that exposes fetch-API `Request`/`Response`:
+ * ```ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerFetch } from "@traceten/ai-crawl/hono";
+ * const config = defineAiCrawlConfig({ … });
+ * // after producing the response:
+ * trackAICrawlerFetch(request, response, config, ctx?.waitUntil?.bind(ctx));
+ * ```
+ */
+import { type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+/**
+ * Generic helper for any `Request`/`Response` runtime. Never throws; pass a
+ * `waitUntil` when the runtime has one, otherwise delivery is
+ * fire-and-forget.
+ */
+export declare function trackAICrawlerFetch(request: FetchLikeRequest, response: {
+    status: number;
+} | undefined, config: ResolvedAiCrawlConfig, waitUntil?: ((promise: Promise<unknown>) => void) | undefined): void;
+/** Structural subset of Hono's `Context`. */
+export interface HonoLikeContext {
+    req: {
+        raw: FetchLikeRequest;
+    };
+    res: {
+        status: number;
+    };
+    /** Throws on runtimes without an execution context — accessed inside try. */
+    executionCtx?: {
+        waitUntil(promise: Promise<unknown>): void;
+    };
+}
+/**
+ * Hono middleware factory. THROWS at construction when the config is
+ * invalid; the middleware itself never throws its own errors (route errors
+ * propagate untouched).
+ */
+export declare function aiCrawlerTracking(config: AiCrawlConfig | ResolvedAiCrawlConfig): (c: HonoLikeContext, next: () => Promise<void>) => Promise<void>;
+//# sourceMappingURL=hono.d.ts.map

package/dist/adapters/hono.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"hono.d.ts","sourceRoot":"","sources":["../../src/adapters/hono.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAAqC,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,KAAK,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAExE;;;;GAIG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,gBAAgB,EACzB,QAAQ,EAAE;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,EACxC,MAAM,EAAE,qBAAqB,EAC7B,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,SAAS,GAC5D,IAAI,CAYN;AAED,6CAA6C;AAC7C,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE;QAAE,GAAG,EAAE,gBAAgB,CAAA;KAAE,CAAC;IAC/B,GAAG,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,6EAA6E;IAC7E,YAAY,CAAC,EAAE;QAAE,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAA;KAAE,CAAC;CAC/D;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,aAAa,GAAG,qBAAqB,GAC5C,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CA0BlE"}

package/dist/adapters/hono.js

@@ -0,0 +1,64 @@
+/**
+ * Hono adapter + generic `Request`/`Response` helper.
+ *
+ * Hono (structural types — no dependency on hono):
+ * ```ts
+ * import { aiCrawlerTracking } from "@traceten/ai-crawl/hono";
+ * app.use("*", aiCrawlerTracking({ siteId: "…", authToken: "tt_bot_…" }));
+ * ```
+ *
+ * Any framework that exposes fetch-API `Request`/`Response`:
+ * ```ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerFetch } from "@traceten/ai-crawl/hono";
+ * const config = defineAiCrawlConfig({ … });
+ * // after producing the response:
+ * trackAICrawlerFetch(request, response, config, ctx?.waitUntil?.bind(ctx));
+ * ```
+ */
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts } from "../track.js";
+/**
+ * Generic helper for any `Request`/`Response` runtime. Never throws; pass a
+ * `waitUntil` when the runtime has one, otherwise delivery is
+ * fire-and-forget.
+ */
+export function trackAICrawlerFetch(request, response, config, waitUntil) {
+    try {
+        if (!isResolvedConfig(config))
+            return; // raw config: refuse silently, never throw per-request
+        trackFacts(config, factsFromFetchRequest(request), response !== undefined && typeof response.status === "number" ? response.status : undefined, waitUntil);
+    }
+    catch {
+        /* silent by contract */
+    }
+}
+/**
+ * Hono middleware factory. THROWS at construction when the config is
+ * invalid; the middleware itself never throws its own errors (route errors
+ * propagate untouched).
+ */
+export function aiCrawlerTracking(config) {
+    const cfg = isResolvedConfig(config) ? config : defineAiCrawlConfig(config);
+    return async (c, next) => {
+        // Route errors propagate untouched — tracking must not change behaviour.
+        await next();
+        try {
+            let waitUntil;
+            try {
+                const ctx = c.executionCtx; // getter throws on Node's hono server
+                if (ctx !== undefined && typeof ctx.waitUntil === "function") {
+                    waitUntil = (p) => ctx.waitUntil(p);
+                }
+            }
+            catch {
+                waitUntil = undefined;
+            }
+            trackFacts(cfg, factsFromFetchRequest(c.req.raw), typeof c.res?.status === "number" ? c.res.status : undefined, waitUntil);
+        }
+        catch {
+            /* silent by contract */
+        }
+    };
+}
+//# sourceMappingURL=hono.js.map

package/dist/adapters/hono.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hono.js","sourceRoot":"","sources":["../../src/adapters/hono.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAyB,MAAM,aAAa,CAAC;AAGvF;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,OAAyB,EACzB,QAAwC,EACxC,MAA6B,EAC7B,SAA6D;IAE7D,IAAI,CAAC;QACH,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;YAAE,OAAO,CAAC,uDAAuD;QAC9F,UAAU,CACR,MAAM,EACN,qBAAqB,CAAC,OAAO,CAAC,EAC9B,QAAQ,KAAK,SAAS,IAAI,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAC3F,SAAS,CACV,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;AACH,CAAC;AAUD;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAA6C;IAE7C,MAAM,GAAG,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAE5E,OAAO,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE;QACvB,yEAAyE;QACzE,MAAM,IAAI,EAAE,CAAC;QACb,IAAI,CAAC;YACH,IAAI,SAAsD,CAAC;YAC3D,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,sCAAsC;gBAClE,IAAI,GAAG,KAAK,SAAS,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;oBAC7D,SAAS,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;gBACtC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS,GAAG,SAAS,CAAC;YACxB,CAAC;YACD,UAAU,CACR,GAAG,EACH,qBAAqB,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAChC,OAAO,CAAC,CAAC,GAAG,EAAE,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAC5D,SAAS,CACV,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,wBAAwB;QAC1B,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/adapters/next.d.ts

@@ -0,0 +1,41 @@
+/**
+ * Next.js adapter — call from `proxy.ts` (Next ≥15.5; formerly
+ * `middleware.ts`).
+ *
+ * ```ts
+ * // proxy.ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerRequest } from "@traceten/ai-crawl/next";
+ *
+ * const config = defineAiCrawlConfig({
+ *   siteId: process.env.TRACETEN_SITE_ID!,
+ *   authToken: process.env.TRACETEN_CRAWL_TOKEN!,
+ * });
+ *
+ * export function proxy(request: NextRequest, event: NextFetchEvent) {
+ *   trackAICrawlerRequest(request, event, config);
+ *   return NextResponse.next();
+ * }
+ * ```
+ *
+ * Validate the config at module scope with `defineAiCrawlConfig` so a
+ * missing token fails the deploy, not your visitors' requests. If a raw
+ * (unvalidated) config reaches this function it is validated lazily; an
+ * invalid one logs ONE loud error and no-ops — per-request code paths never
+ * throw.
+ *
+ * The middleware runs before the response exists, so no `status` is sent —
+ * the edge treats a crawl report without a status as "served".
+ */
+import { type FetchLikeRequest } from "../track.js";
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "../types.js";
+/** Structural subset of Next's `NextFetchEvent`. */
+export interface WaitUntilEvent {
+    waitUntil(promise: Promise<unknown>): void;
+}
+/**
+ * Report the request if it looks like an AI crawler. Never throws, never
+ * blocks — delivery is scheduled via `event.waitUntil` when available.
+ */
+export declare function trackAICrawlerRequest(request: FetchLikeRequest, event: WaitUntilEvent | undefined, config: AiCrawlConfig | ResolvedAiCrawlConfig): void;
+//# sourceMappingURL=next.d.ts.map

package/dist/adapters/next.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.d.ts","sourceRoot":"","sources":["../../src/adapters/next.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,EAAqC,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,KAAK,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAExE,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC7B,SAAS,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC;CAC5C;AA4BD;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,gBAAgB,EACzB,KAAK,EAAE,cAAc,GAAG,SAAS,EACjC,MAAM,EAAE,aAAa,GAAG,qBAAqB,GAC5C,IAAI,CAaN"}

package/dist/adapters/next.js

@@ -0,0 +1,70 @@
+/**
+ * Next.js adapter — call from `proxy.ts` (Next ≥15.5; formerly
+ * `middleware.ts`).
+ *
+ * ```ts
+ * // proxy.ts
+ * import { defineAiCrawlConfig } from "@traceten/ai-crawl";
+ * import { trackAICrawlerRequest } from "@traceten/ai-crawl/next";
+ *
+ * const config = defineAiCrawlConfig({
+ *   siteId: process.env.TRACETEN_SITE_ID!,
+ *   authToken: process.env.TRACETEN_CRAWL_TOKEN!,
+ * });
+ *
+ * export function proxy(request: NextRequest, event: NextFetchEvent) {
+ *   trackAICrawlerRequest(request, event, config);
+ *   return NextResponse.next();
+ * }
+ * ```
+ *
+ * Validate the config at module scope with `defineAiCrawlConfig` so a
+ * missing token fails the deploy, not your visitors' requests. If a raw
+ * (unvalidated) config reaches this function it is validated lazily; an
+ * invalid one logs ONE loud error and no-ops — per-request code paths never
+ * throw.
+ *
+ * The middleware runs before the response exists, so no `status` is sent —
+ * the edge treats a crawl report without a status as "served".
+ */
+import { defineAiCrawlConfig, isResolvedConfig } from "../config.js";
+import { factsFromFetchRequest, trackFacts } from "../track.js";
+const resolvedCache = new WeakMap();
+/** Resolve lazily, warning loudly ONCE per config object on invalid input. */
+function resolveLazily(config) {
+    if (isResolvedConfig(config))
+        return config;
+    if (resolvedCache.has(config))
+        return resolvedCache.get(config) ?? null;
+    let resolved = null;
+    try {
+        resolved = defineAiCrawlConfig(config);
+    }
+    catch (err) {
+        try {
+            // eslint-disable-next-line no-console
+            console.error("[@traceten/ai-crawl] invalid config — crawler tracking is DISABLED:", err instanceof Error ? err.message : err);
+        }
+        catch {
+            /* ignore */
+        }
+    }
+    resolvedCache.set(config, resolved);
+    return resolved;
+}
+/**
+ * Report the request if it looks like an AI crawler. Never throws, never
+ * blocks — delivery is scheduled via `event.waitUntil` when available.
+ */
+export function trackAICrawlerRequest(request, event, config) {
+    try {
+        const cfg = resolveLazily(config);
+        if (cfg === null)
+            return;
+        trackFacts(cfg, factsFromFetchRequest(request), undefined, event !== undefined ? (p) => event.waitUntil(p) : undefined);
+    }
+    catch {
+        /* silent by contract */
+    }
+}
+//# sourceMappingURL=next.js.map

package/dist/adapters/next.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"next.js","sourceRoot":"","sources":["../../src/adapters/next.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,mBAAmB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAyB,MAAM,aAAa,CAAC;AAQvF,MAAM,aAAa,GAAG,IAAI,OAAO,EAA+C,CAAC;AAEjF,8EAA8E;AAC9E,SAAS,aAAa,CACpB,MAA6C;IAE7C,IAAI,gBAAgB,CAAC,MAAM,CAAC;QAAE,OAAO,MAAM,CAAC;IAC5C,IAAI,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC;IACxE,IAAI,QAAQ,GAAiC,IAAI,CAAC;IAClD,IAAI,CAAC;QACH,QAAQ,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC;YACH,sCAAsC;YACtC,OAAO,CAAC,KAAK,CACX,qEAAqE,EACrE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACzC,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;IACD,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;IACpC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAAyB,EACzB,KAAiC,EACjC,MAA6C;IAE7C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QAClC,IAAI,GAAG,KAAK,IAAI;YAAE,OAAO;QACzB,UAAU,CACR,GAAG,EACH,qBAAqB,CAAC,OAAO,CAAC,EAC9B,SAAS,EACT,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAC5D,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;AACH,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Config validation + defaulting.
+ *
+ * This is the ONE place the package is allowed to throw: at construction,
+ * in the customer's module scope / server boot — never per-request. A
+ * missing `authToken` must fail loudly here rather than silently sending
+ * requests that the endpoint will 401.
+ */
+import type { AiCrawlConfig, ResolvedAiCrawlConfig } from "./types.js";
+export declare const DEFAULT_ENDPOINT = "https://ingest.traceten.com/v1/ai-crawls";
+export declare const DEFAULT_ALLOWED_METHODS: readonly string[];
+export declare const AUTH_TOKEN_PREFIX = "tt_bot_";
+/**
+ * Validate a config and apply defaults. THROWS on invalid input — call it at
+ * module scope (or in your middleware factory), so a misconfiguration fails
+ * your deploy, not your visitors' requests.
+ */
+export declare function defineAiCrawlConfig(config: AiCrawlConfig): ResolvedAiCrawlConfig;
+/** True when the object already passed through {@link defineAiCrawlConfig}. */
+export declare function isResolvedConfig(config: AiCrawlConfig | ResolvedAiCrawlConfig): config is ResolvedAiCrawlConfig;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEvE,eAAO,MAAM,gBAAgB,6CAA6C,CAAC;AAC3E,eAAO,MAAM,uBAAuB,EAAE,SAAS,MAAM,EAAoB,CAAC;AAC1E,eAAO,MAAM,iBAAiB,YAAY,CAAC;AAM3C;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,aAAa,GAAG,qBAAqB,CA8FhF;AAED,+EAA+E;AAC/E,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,aAAa,GAAG,qBAAqB,GAC5C,MAAM,IAAI,qBAAqB,CAEjC"}

package/dist/config.js

@@ -0,0 +1,99 @@
+/**
+ * Config validation + defaulting.
+ *
+ * This is the ONE place the package is allowed to throw: at construction,
+ * in the customer's module scope / server boot — never per-request. A
+ * missing `authToken` must fail loudly here rather than silently sending
+ * requests that the endpoint will 401.
+ */
+import { DEFAULT_DENY_EXTENSIONS, DEFAULT_DENY_PATH_PREFIXES } from "./filter.js";
+export const DEFAULT_ENDPOINT = "https://ingest.traceten.com/v1/ai-crawls";
+export const DEFAULT_ALLOWED_METHODS = ["GET", "HEAD"];
+export const AUTH_TOKEN_PREFIX = "tt_bot_";
+function fail(message) {
+    throw new Error(`[@traceten/ai-crawl] ${message}`);
+}
+/**
+ * Validate a config and apply defaults. THROWS on invalid input — call it at
+ * module scope (or in your middleware factory), so a misconfiguration fails
+ * your deploy, not your visitors' requests.
+ */
+export function defineAiCrawlConfig(config) {
+    if (typeof config !== "object" || config === null) {
+        fail("config must be an object");
+    }
+    if (typeof config.siteId !== "string" || config.siteId.trim() === "") {
+        fail("siteId is required — use your site key (the ttid_… data-site value on the install page)");
+    }
+    if (typeof config.authToken !== "string" || config.authToken.trim() === "") {
+        fail("authToken is required. POST /v1/ai-crawls rejects unauthenticated reports, " +
+            "so without it every report would silently 401. Create a crawl token " +
+            "(tt_bot_...) in the Traceten dashboard and keep it server-side.");
+    }
+    if (!config.authToken.startsWith(AUTH_TOKEN_PREFIX)) {
+        fail(`authToken must start with "${AUTH_TOKEN_PREFIX}". It looks like you passed ` +
+            "something else (perhaps the public snippet key?) — the crawl token is a " +
+            "server-side secret created in the dashboard.");
+    }
+    const proxyDepth = config.proxyDepth ?? 1;
+    if (!Number.isInteger(proxyDepth) || proxyDepth < 1) {
+        fail("proxyDepth must be a positive integer (number of trusted proxies appending to x-forwarded-for)");
+    }
+    if (config.endpoint !== undefined) {
+        // https only: the request carries the tt_bot_ Bearer token and crawler
+        // IPs. A typo'd http:// endpoint would leak both in cleartext, silently.
+        // Plain http is allowed solely for local development loopback.
+        const isHttps = /^https:\/\//.test(config.endpoint);
+        const isLocalHttp = /^http:\/\/(localhost|127\.0\.0\.1|\[::1\])(:\d+)?(\/|$)/.test(config.endpoint);
+        if (!isHttps && !isLocalHttp) {
+            fail("endpoint must be an https:// URL (http:// is allowed only for localhost). " +
+                "Reports carry your tt_bot_ token and crawler IPs — never send them in cleartext.");
+        }
+    }
+    if (config.publicOrigin !== undefined && !/^https?:\/\/[^/]+$/.test(config.publicOrigin)) {
+        fail('publicOrigin must be an origin with no path, e.g. "https://example.com"');
+    }
+    const allowedMethods = (config.allowedMethods ?? DEFAULT_ALLOWED_METHODS).map((m) => String(m).toUpperCase());
+    if (allowedMethods.length === 0) {
+        fail("allowedMethods must not be empty");
+    }
+    // Extras EXTEND the built-in lists; they never replace them.
+    const denyPathPrefixes = [
+        ...DEFAULT_DENY_PATH_PREFIXES,
+        // Trailing slashes are stripped so the segment-boundary match in
+        // filter.ts (`=== prefix || startsWith(prefix + "/")`) works either way.
+        ...(config.extraDenyPathPrefixes ?? []).map((p) => String(p).toLowerCase().replace(/\/+$/, "")),
+    ];
+    const denyExtensions = [
+        ...DEFAULT_DENY_EXTENSIONS,
+        ...(config.extraDenyExtensions ?? []).map((e) => {
+            const ext = String(e).toLowerCase();
+            return ext.startsWith(".") ? ext : `.${ext}`;
+        }),
+    ];
+    const resolved = {
+        siteId: config.siteId.trim(),
+        authToken: config.authToken.trim(),
+        endpoint: config.endpoint ?? DEFAULT_ENDPOINT,
+        allowedMethods,
+        denyPathPrefixes,
+        denyExtensions,
+        disableAnswerFetch: config.disableAnswerFetch === true,
+        disableSearchCrawlers: config.disableSearchCrawlers === true,
+        disableTrainingCrawlers: config.disableTrainingCrawlers === true,
+        disableOtherCrawlers: config.disableOtherCrawlers === true,
+        trustProxy: config.trustProxy === true,
+        trustCfConnectingIp: config.trustCfConnectingIp === true,
+        proxyDepth,
+        publicOrigin: config.publicOrigin,
+        onError: typeof config.onError === "function" ? config.onError : undefined,
+        fetch: config.fetch,
+        __resolved: true,
+    };
+    return resolved;
+}
+/** True when the object already passed through {@link defineAiCrawlConfig}. */
+export function isResolvedConfig(config) {
+    return config.__resolved === true;
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,aAAa,CAAC;AAGlF,MAAM,CAAC,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAC3E,MAAM,CAAC,MAAM,uBAAuB,GAAsB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,SAAS,CAAC;AAE3C,SAAS,IAAI,CAAC,OAAe;IAC3B,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,EAAE,CAAC,CAAC;AACrD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAqB;IACvD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACrE,IAAI,CACF,yFAAyF,CAC1F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QAC3E,IAAI,CACF,6EAA6E;YAC3E,sEAAsE;YACtE,iEAAiE,CACpE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACpD,IAAI,CACF,8BAA8B,iBAAiB,8BAA8B;YAC3E,0EAA0E;YAC1E,8CAA8C,CACjD,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpD,IAAI,CACF,gGAAgG,CACjG,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClC,uEAAuE;QACvE,yEAAyE;QACzE,+DAA+D;QAC/D,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,WAAW,GAAG,yDAAyD,CAAC,IAAI,CAChF,MAAM,CAAC,QAAQ,CAChB,CAAC;QACF,IAAI,CAAC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC;YAC7B,IAAI,CACF,4EAA4E;gBAC1E,kFAAkF,CACrF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QACzF,IAAI,CAAC,yEAAyE,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,cAAc,GAAG,CAAC,MAAM,CAAC,cAAc,IAAI,uBAAuB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAClF,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CACxB,CAAC;IACF,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC3C,CAAC;IAED,6DAA6D;IAC7D,MAAM,gBAAgB,GAAG;QACvB,GAAG,0BAA0B;QAC7B,iEAAiE;QACjE,yEAAyE;QACzE,GAAG,CAAC,MAAM,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;KAChG,CAAC;IACF,MAAM,cAAc,GAAG;QACrB,GAAG,uBAAuB;QAC1B,GAAG,CAAC,MAAM,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YAC9C,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACpC,OAAO,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC;QAC/C,CAAC,CAAC;KACH,CAAC;IAEF,MAAM,QAAQ,GAA0B;QACtC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,IAAI,EAAE;QAClC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,gBAAgB;QAC7C,cAAc;QACd,gBAAgB;QAChB,cAAc;QACd,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,KAAK,IAAI;QACtD,qBAAqB,EAAE,MAAM,CAAC,qBAAqB,KAAK,IAAI;QAC5D,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,KAAK,IAAI;QAChE,oBAAoB,EAAE,MAAM,CAAC,oBAAoB,KAAK,IAAI;QAC1D,UAAU,EAAE,MAAM,CAAC,UAAU,KAAK,IAAI;QACtC,mBAAmB,EAAE,MAAM,CAAC,mBAAmB,KAAK,IAAI;QACxD,UAAU;QACV,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,OAAO,EAAE,OAAO,MAAM,CAAC,OAAO,KAAK,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QAC1E,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,UAAU,EAAE,IAAI;KACjB,CAAC;IACF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,gBAAgB,CAC9B,MAA6C;IAE7C,OAAQ,MAAgC,CAAC,UAAU,KAAK,IAAI,CAAC;AAC/D,CAAC"}

package/dist/crawlers.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Local crawler token data — a deliberately-loose COST FILTER, not the source
+ * of truth. Traceten's server-side crawler registry decides provider,
+ * category, verification and confidence; this list only decides "worth
+ * reporting". Keep it small and keep every token anchored against current
+ * vendor documentation.
+ *
+ * Recent reconciliation notes, kept because they explain non-obvious entries:
+ *   - Deleted `GoogleAgent` (a hyphenation typo of the real `Google-Agent`,
+ *     which stays), `FacebookBot`, Microsoft `Copilot` and `AliyunBot` — no
+ *     vendor documents any of them.
+ *   - Added `Google-GeminiNotebook`, Google's 2026-07-16 rename of
+ *     `Google-NotebookLM`; the old token is honoured only until August 2026.
+ *     BOTH are listed on purpose — stale traffic still carries the old one.
+ *   - Added `MistralAI-Training`, documented on Mistral's crawler page.
+ *   - Recategorised `GoogleOther` (training → ai_crawler; Google states it
+ *     "doesn't affect any specific product") and `Bytespider` (training →
+ *     search_index; ByteDance documents only a Toutiao Search index crawler).
+ *
+ * DELIBERATE EXCLUSIONS: `Google-Extended` and `Applebot-Extended` are
+ * robots.txt control tokens, not crawlers — they never appear as a live
+ * user agent, so listing them here would be dead code that can never fire.
+ *
+ * ⚠️ THE EXCLUSION ONLY ACTUALLY HOLDS FOR `Google-Extended`. Nothing matches
+ * it, so it is never reported. `Applebot-Extended` IS reported, by two
+ * independent paths:
+ *   1. Apple's UA strings carry `+http://www.apple.com/go/applebot`, whose
+ *      `applebot` substring is a TIER-1 exact token match (anchored between
+ *      `/` and `)`), so the report lands as agent `Applebot`, training.
+ *   2. The bare string `Applebot-Extended` hits the tier-2 alias
+ *      `{ alias: "applebot" }` below, because prefix anchoring accepts a
+ *      trailing `-`. That yields provider `apple`, agent null, `ai_crawler`.
+ * Neither is a bug — attributing an Apple fetch to Apple is right, and this
+ * file is a cost filter, not the classifier. Do NOT write, in code comments
+ * or in customer copy, that these tokens are "never reported" — only
+ * `Google-Extended` qualifies.
+ *
+ * All tokens are stored lowercase; matching lowercases the UA once.
+ */
+import type { CrawlerCategory } from "./types.js";
+export interface AgentToken {
+    /** Lowercased agent token, matched with token-boundary anchoring. */
+    readonly token: string;
+    /** Canonical display name of the agent (original casing). */
+    readonly agent: string;
+    readonly provider: string;
+    readonly category: CrawlerCategory;
+}
+/** Tier 1 — exact agent tokens. First match wins. */
+export declare const AGENT_TOKENS: readonly AgentToken[];
+export interface ProviderAlias {
+    /** Lowercased alias, matched as a LEFT-anchored prefix at a token boundary. */
+    readonly alias: string;
+    readonly provider: string;
+}
+/**
+ * Tier 2 — coarse provider aliases, so a NEW agent from a known vendor
+ * (e.g. a hypothetical `ChatGPT-Reader/1.0`) is still captured without a
+ * package release. Left-boundary-anchored prefix match: the alias must start
+ * at the beginning of a token (start of string or after a non-token
+ * character), which is what defeats `not-really-GPTBot`-style spoofs while
+ * still catching `Claude-NewAgent/1.0`.
+ *
+ * Tier-2 matches are reported with category `ai_crawler` locally (the server
+ * assigns the real category) and are gated by `disableOtherCrawlers`.
+ */
+export declare const PROVIDER_ALIASES: readonly ProviderAlias[];
+//# sourceMappingURL=crawlers.d.ts.map

package/dist/crawlers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawlers.d.ts","sourceRoot":"","sources":["../src/crawlers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,UAAU;IACzB,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;CACpC;AAED,qDAAqD;AACrD,eAAO,MAAM,YAAY,EAAE,SAAS,UAAU,EAwK7C,CAAC;AAEF,MAAM,WAAW,aAAa;IAC5B,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED;;;;;;;;;;GAUG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,aAAa,EA6BpD,CAAC"}