@traceten/ai-crawl 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/LICENSE +21 -0
  3. package/README.md +260 -0
  4. package/dist/adapters/cloudflare-pages.d.ts +42 -0
  5. package/dist/adapters/cloudflare-pages.d.ts.map +1 -0
  6. package/dist/adapters/cloudflare-pages.js +46 -0
  7. package/dist/adapters/cloudflare-pages.js.map +1 -0
  8. package/dist/adapters/cloudflare-workers.d.ts +41 -0
  9. package/dist/adapters/cloudflare-workers.d.ts.map +1 -0
  10. package/dist/adapters/cloudflare-workers.js +49 -0
  11. package/dist/adapters/cloudflare-workers.js.map +1 -0
  12. package/dist/adapters/express.d.ts +49 -0
  13. package/dist/adapters/express.d.ts.map +1 -0
  14. package/dist/adapters/express.js +91 -0
  15. package/dist/adapters/express.js.map +1 -0
  16. package/dist/adapters/hono.d.ts +48 -0
  17. package/dist/adapters/hono.d.ts.map +1 -0
  18. package/dist/adapters/hono.js +64 -0
  19. package/dist/adapters/hono.js.map +1 -0
  20. package/dist/adapters/next.d.ts +41 -0
  21. package/dist/adapters/next.d.ts.map +1 -0
  22. package/dist/adapters/next.js +70 -0
  23. package/dist/adapters/next.js.map +1 -0
  24. package/dist/config.d.ts +21 -0
  25. package/dist/config.d.ts.map +1 -0
  26. package/dist/config.js +99 -0
  27. package/dist/config.js.map +1 -0
  28. package/dist/crawlers.d.ts +68 -0
  29. package/dist/crawlers.d.ts.map +1 -0
  30. package/dist/crawlers.js +248 -0
  31. package/dist/crawlers.js.map +1 -0
  32. package/dist/filter.d.ts +33 -0
  33. package/dist/filter.d.ts.map +1 -0
  34. package/dist/filter.js +169 -0
  35. package/dist/filter.js.map +1 -0
  36. package/dist/index.d.ts +21 -0
  37. package/dist/index.d.ts.map +1 -0
  38. package/dist/index.js +20 -0
  39. package/dist/index.js.map +1 -0
  40. package/dist/ip.d.ts +35 -0
  41. package/dist/ip.d.ts.map +1 -0
  42. package/dist/ip.js +109 -0
  43. package/dist/ip.js.map +1 -0
  44. package/dist/matcher.d.ts +44 -0
  45. package/dist/matcher.d.ts.map +1 -0
  46. package/dist/matcher.js +111 -0
  47. package/dist/matcher.js.map +1 -0
  48. package/dist/report.d.ts +43 -0
  49. package/dist/report.d.ts.map +1 -0
  50. package/dist/report.js +116 -0
  51. package/dist/report.js.map +1 -0
  52. package/dist/track.d.ts +30 -0
  53. package/dist/track.d.ts.map +1 -0
  54. package/dist/track.js +96 -0
  55. package/dist/track.js.map +1 -0
  56. package/dist/types.d.ts +184 -0
  57. package/dist/types.d.ts.map +1 -0
  58. package/dist/types.js +11 -0
  59. package/dist/types.js.map +1 -0
  60. package/package.json +87 -0
  61. package/src/adapters/cloudflare-pages.ts +64 -0
  62. package/src/adapters/cloudflare-workers.ts +70 -0
  63. package/src/adapters/express.ts +113 -0
  64. package/src/adapters/hono.ts +89 -0
  65. package/src/adapters/next.ts +87 -0
  66. package/src/config.ts +127 -0
  67. package/src/crawlers.ts +269 -0
  68. package/src/filter.ts +178 -0
  69. package/src/index.ts +46 -0
  70. package/src/ip.ts +112 -0
  71. package/src/matcher.ts +119 -0
  72. package/src/report.ts +149 -0
  73. package/src/track.ts +117 -0
  74. package/src/types.ts +190 -0
package/dist/ip.js ADDED
@@ -0,0 +1,109 @@
1
+ /**
2
+ * Crawler IP resolution.
3
+ *
4
+ * The crawler's TCP connection terminates at the CUSTOMER's server, so the
5
+ * crawler IP is only observable here — Traceten's edge sees the customer's
6
+ * origin IP, not the crawler's. This module
7
+ * derives the best-available value; the payload OMITS the field rather than
8
+ * send a wrong one.
9
+ *
10
+ * Resolution order:
11
+ * 1. `cf-connecting-ip` — ONLY when `trustCfConnectingIp` is set. The
12
+ * Cloudflare adapters set it automatically (the platform strips and
13
+ * rewrites the header there); anywhere else it is client-forgeable,
14
+ * and a forged vendor-range IP paired with a vendor UA is exactly the
15
+ * spoof the server-side verification exists to catch.
16
+ * 2. `x-forwarded-for` — ONLY when `trustProxy` is true, selecting the
17
+ * entry `proxyDepth` hops from the right (each trusted proxy appends
18
+ * one entry; anything further left is attacker-controllable)
19
+ * 3. Socket remote address, where the runtime exposes one
20
+ */
21
+ const IPV4_RE = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
22
+ // Loose IPv6 shape check — up to 8 hex groups separated by colons ("::"
23
+ // compression appears as empty groups). Deliberately permissive: this is a
24
+ // plausibility gate, not a validator; the edge re-validates.
25
+ const IPV6_RE = /^[0-9a-f]{0,4}(:[0-9a-f]{0,4}){1,7}$/i;
26
+ /** Normalise a candidate; returns `undefined` when it is not a plausible IP. */
27
+ export function normalizeIp(raw) {
28
+ if (!raw)
29
+ return undefined;
30
+ let value = raw.trim();
31
+ if (value === "")
32
+ return undefined;
33
+ // Bracketed IPv6, possibly with a port: [::1]:8080
34
+ if (value.startsWith("[")) {
35
+ const close = value.indexOf("]");
36
+ if (close === -1)
37
+ return undefined;
38
+ value = value.slice(1, close);
39
+ }
40
+ else {
41
+ // IPv4 with a port: 1.2.3.4:5678 (a bare IPv6 also contains ":", so only
42
+ // strip when what remains is a valid IPv4).
43
+ const colon = value.indexOf(":");
44
+ if (colon !== -1 && value.indexOf(":", colon + 1) === -1) {
45
+ const head = value.slice(0, colon);
46
+ if (IPV4_RE.test(head))
47
+ value = head;
48
+ }
49
+ }
50
+ // Node reports IPv4-mapped addresses as ::ffff:1.2.3.4.
51
+ if (value.toLowerCase().startsWith("::ffff:")) {
52
+ const tail = value.slice(7);
53
+ if (IPV4_RE.test(tail))
54
+ value = tail;
55
+ }
56
+ const v4 = IPV4_RE.exec(value);
57
+ if (v4 !== null) {
58
+ for (let i = 1; i <= 4; i++) {
59
+ if (Number(v4[i]) > 255)
60
+ return undefined;
61
+ }
62
+ return value;
63
+ }
64
+ if (value.includes(":") && value.length <= 45 && IPV6_RE.test(value)) {
65
+ return value.toLowerCase();
66
+ }
67
+ return undefined;
68
+ }
69
+ /**
70
+ * Select the client entry from an `x-forwarded-for` value given the number
71
+ * of trusted proxies that append to it. Returns `undefined` when the chain
72
+ * is shorter than `proxyDepth` (a wrong value is worse than none).
73
+ */
74
+ export function ipFromForwardedFor(headerValue, proxyDepth) {
75
+ const entries = headerValue
76
+ .split(",")
77
+ .map((e) => e.trim())
78
+ .filter((e) => e !== "");
79
+ if (entries.length === 0 || proxyDepth < 1 || entries.length < proxyDepth) {
80
+ return undefined;
81
+ }
82
+ return normalizeIp(entries[entries.length - proxyDepth]);
83
+ }
84
+ /**
85
+ * Resolve the crawler IP from a request. Returns `undefined` when no
86
+ * trustworthy value is derivable. Never throws.
87
+ */
88
+ export function resolveCrawlerIp(cfg, facts) {
89
+ try {
90
+ if (cfg.trustCfConnectingIp) {
91
+ const cf = normalizeIp(facts.header("cf-connecting-ip"));
92
+ if (cf !== undefined)
93
+ return cf;
94
+ }
95
+ if (cfg.trustProxy) {
96
+ const xff = facts.header("x-forwarded-for");
97
+ if (xff) {
98
+ const fromXff = ipFromForwardedFor(xff, cfg.proxyDepth);
99
+ if (fromXff !== undefined)
100
+ return fromXff;
101
+ }
102
+ }
103
+ return normalizeIp(facts.socketAddr);
104
+ }
105
+ catch {
106
+ return undefined;
107
+ }
108
+ }
109
+ //# sourceMappingURL=ip.js.map
package/dist/ip.js.map ADDED
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ip.js","sourceRoot":"","sources":["../src/ip.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,MAAM,OAAO,GAAG,8CAA8C,CAAC;AAC/D,wEAAwE;AACxE,2EAA2E;AAC3E,6DAA6D;AAC7D,MAAM,OAAO,GAAG,uCAAuC,CAAC;AAExD,gFAAgF;AAChF,MAAM,UAAU,WAAW,CAAC,GAA8B;IACxD,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,IAAI,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACvB,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,SAAS,CAAC;IAEnC,mDAAmD;IACnD,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,CAAC;YAAE,OAAO,SAAS,CAAC;QACnC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAChC,CAAC;SAAM,CAAC;QACN,yEAAyE;QACzE,4CAA4C;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACzD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;QACvC,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,KAAK,GAAG,IAAI,CAAC;IACvC,CAAC;IAED,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;QAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG;gBAAE,OAAO,SAAS,CAAC;QAC5C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,WAAmB,EAAE,UAAkB;IACxE,MAAM,OAAO,GAAG,WAAW;SACxB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;QAC1E,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,GAA0B,EAC1B,KAAmB;IAEnB,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,mBAAmB,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC;YACzD,IAAI,EAAE,KAAK,SAAS;gBAAE,OAAO,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC5C,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,OAAO,GAAG,kBAAkB,CAAC,GAAG,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;gBACxD,IAAI,OAAO,KAAK,SAAS;oBAAE,OAAO,OAAO,CAAC;YAC5C,CAAC;QACH,CAAC;QAED,OAAO,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Anchored two-tier user-agent matching.
3
+ *
4
+ * NEVER `includes()` on the whole string — that is the reference package's
5
+ * defect (its page claims token matching; its code does substring matching,
6
+ * so a browser UA containing `grok` or `copilot` false-positives). Follows
7
+ * the same anchoring discipline Traceten's server-side classifier uses,
8
+ * adapted for crawler UAs where the token sits mid-string
9
+ * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).
10
+ *
11
+ * Boundary rule: a token matches only when the character before it and (for
12
+ * tier 1) the character after it are NOT token characters `[a-z0-9._-]`.
13
+ * So `compatible; GPTBot/1.1` matches, and `not-really-GPTBot` does not
14
+ * (preceded by `-`), and `GPTBotnet` does not (followed by `n`).
15
+ */
16
+ import type { CrawlerMatch } from "./types.js";
17
+ /** Anchoring mode. */
18
+ export type AnchorMode = "token" | "prefix";
19
+ /**
20
+ * True when `needle` occurs in `haystack` (both lowercase) anchored at a
21
+ * token boundary.
22
+ *
23
+ * `"token"` (tier 1): boundary required on BOTH sides — `GPTBot/1.1`
24
+ * matches, `not-really-GPTBot` and `GPTBotnet` do not.
25
+ *
26
+ * `"prefix"` (tier 2): left boundary required; the right side must be a
27
+ * boundary OR a compound-token delimiter (`-`, `_`, `.`) — so `claude`
28
+ * catches a future `Claude-NewAgent/1.0` without matching `claudius`.
29
+ * A needle that itself ends in a delimiter (e.g. `oai-`, `meta-`) is a free
30
+ * prefix past that delimiter.
31
+ */
32
+ export declare function hasAnchoredToken(haystack: string, needle: string, mode: AnchorMode): boolean;
33
+ /**
34
+ * Match a raw User-Agent string against the local crawler list.
35
+ *
36
+ * Tier 1: exact agent tokens, boundary-anchored on both sides.
37
+ * Tier 2: coarse provider aliases, left-boundary-anchored prefixes, so a new
38
+ * agent from a known vendor is captured without a release. Tier-2 matches
39
+ * carry category `ai_crawler` locally; the server assigns the real one.
40
+ *
41
+ * Returns `null` for no match. Never throws.
42
+ */
43
+ export declare function matchCrawler(userAgent: string | null | undefined): CrawlerMatch | null;
44
+ //# sourceMappingURL=matcher.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matcher.d.ts","sourceRoot":"","sources":["../src/matcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAS/C,sBAAsB;AACtB,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,QAAQ,CAAC;AAE5C;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CA+B5F;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,YAAY,GAAG,IAAI,CAiCtF"}
@@ -0,0 +1,111 @@
1
+ /**
2
+ * Anchored two-tier user-agent matching.
3
+ *
4
+ * NEVER `includes()` on the whole string — that is the reference package's
5
+ * defect (its page claims token matching; its code does substring matching,
6
+ * so a browser UA containing `grok` or `copilot` false-positives). Follows
7
+ * the same anchoring discipline Traceten's server-side classifier uses,
8
+ * adapted for crawler UAs where the token sits mid-string
9
+ * (`Mozilla/5.0 … compatible; GPTBot/1.1; +https://openai.com/gptbot`).
10
+ *
11
+ * Boundary rule: a token matches only when the character before it and (for
12
+ * tier 1) the character after it are NOT token characters `[a-z0-9._-]`.
13
+ * So `compatible; GPTBot/1.1` matches, and `not-really-GPTBot` does not
14
+ * (preceded by `-`), and `GPTBotnet` does not (followed by `n`).
15
+ */
16
+ import { AGENT_TOKENS, PROVIDER_ALIASES } from "./crawlers.js";
17
+ /** Characters that can appear inside a UA product token. */
18
+ function isTokenChar(ch) {
19
+ return ((ch >= "a" && ch <= "z") || (ch >= "0" && ch <= "9") || ch === "." || ch === "_" || ch === "-");
20
+ }
21
+ /**
22
+ * True when `needle` occurs in `haystack` (both lowercase) anchored at a
23
+ * token boundary.
24
+ *
25
+ * `"token"` (tier 1): boundary required on BOTH sides — `GPTBot/1.1`
26
+ * matches, `not-really-GPTBot` and `GPTBotnet` do not.
27
+ *
28
+ * `"prefix"` (tier 2): left boundary required; the right side must be a
29
+ * boundary OR a compound-token delimiter (`-`, `_`, `.`) — so `claude`
30
+ * catches a future `Claude-NewAgent/1.0` without matching `claudius`.
31
+ * A needle that itself ends in a delimiter (e.g. `oai-`, `meta-`) is a free
32
+ * prefix past that delimiter.
33
+ */
34
+ export function hasAnchoredToken(haystack, needle, mode) {
35
+ const needleEndsWithDelimiter = needle.endsWith("-") || needle.endsWith("_") || needle.endsWith(".");
36
+ let idx = haystack.indexOf(needle);
37
+ while (idx !== -1) {
38
+ let leftOk = idx === 0 || !isTokenChar(haystack.charAt(idx - 1));
39
+ // Prefix (tier-2) matches skip occurrences that start a URL host —
40
+ // `SomeScraper/1.0 (+https://anthropic.com/policies)` is a scraper
41
+ // citing a vendor page, not a vendor agent. Tier-1 tokens keep matching
42
+ // anywhere: real crawler tokens never begin a hostname.
43
+ if (leftOk && mode === "prefix" && idx >= 3 && haystack.slice(idx - 3, idx) === "://") {
44
+ leftOk = false;
45
+ }
46
+ const rightIdx = idx + needle.length;
47
+ const nextCh = rightIdx >= haystack.length ? "" : haystack.charAt(rightIdx);
48
+ let rightOk;
49
+ if (mode === "token") {
50
+ rightOk = nextCh === "" || !isTokenChar(nextCh);
51
+ }
52
+ else {
53
+ rightOk =
54
+ needleEndsWithDelimiter ||
55
+ nextCh === "" ||
56
+ !isTokenChar(nextCh) ||
57
+ nextCh === "-" ||
58
+ nextCh === "_" ||
59
+ nextCh === ".";
60
+ }
61
+ if (leftOk && rightOk)
62
+ return true;
63
+ idx = haystack.indexOf(needle, idx + 1);
64
+ }
65
+ return false;
66
+ }
67
+ /**
68
+ * Match a raw User-Agent string against the local crawler list.
69
+ *
70
+ * Tier 1: exact agent tokens, boundary-anchored on both sides.
71
+ * Tier 2: coarse provider aliases, left-boundary-anchored prefixes, so a new
72
+ * agent from a known vendor is captured without a release. Tier-2 matches
73
+ * carry category `ai_crawler` locally; the server assigns the real one.
74
+ *
75
+ * Returns `null` for no match. Never throws.
76
+ */
77
+ export function matchCrawler(userAgent) {
78
+ if (!userAgent)
79
+ return null;
80
+ let ua;
81
+ try {
82
+ ua = userAgent.toLowerCase();
83
+ }
84
+ catch {
85
+ return null;
86
+ }
87
+ if (ua.length === 0 || ua.length > 4096)
88
+ return null;
89
+ for (const entry of AGENT_TOKENS) {
90
+ if (hasAnchoredToken(ua, entry.token, "token")) {
91
+ return {
92
+ agent: entry.agent,
93
+ provider: entry.provider,
94
+ category: entry.category,
95
+ tier: "exact",
96
+ };
97
+ }
98
+ }
99
+ for (const entry of PROVIDER_ALIASES) {
100
+ if (hasAnchoredToken(ua, entry.alias, "prefix")) {
101
+ return {
102
+ agent: null,
103
+ provider: entry.provider,
104
+ category: "ai_crawler",
105
+ tier: "provider",
106
+ };
107
+ }
108
+ }
109
+ return null;
110
+ }
111
+ //# sourceMappingURL=matcher.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"matcher.js","sourceRoot":"","sources":["../src/matcher.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAG/D,4DAA4D;AAC5D,SAAS,WAAW,CAAC,EAAU;IAC7B,OAAO,CACL,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,GAAG,CAC/F,CAAC;AACJ,CAAC;AAKD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,MAAc,EAAE,IAAgB;IACjF,MAAM,uBAAuB,GAC3B,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACvE,IAAI,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACnC,OAAO,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;QAClB,IAAI,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,mEAAmE;QACnE,mEAAmE;QACnE,wEAAwE;QACxE,wDAAwD;QACxD,IAAI,MAAM,IAAI,IAAI,KAAK,QAAQ,IAAI,GAAG,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,KAAK,EAAE,CAAC;YACtF,MAAM,GAAG,KAAK,CAAC;QACjB,CAAC;QACD,MAAM,QAAQ,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC;QACrC,MAAM,MAAM,GAAG,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC5E,IAAI,OAAgB,CAAC;QACrB,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;YACrB,OAAO,GAAG,MAAM,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO;gBACL,uBAAuB;oBACvB,MAAM,KAAK,EAAE;oBACb,CAAC,WAAW,CAAC,MAAM,CAAC;oBACpB,MAAM,KAAK,GAAG;oBACd,MAAM,KAAK,GAAG;oBACd,MAAM,KAAK,GAAG,CAAC;QACnB,CAAC;QACD,IAAI,MAAM,IAAI,OAAO;YAAE,OAAO,IAAI,CAAC;QACnC,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,YAAY,CAAC,SAAoC;IAC/D,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,IAAI,EAAU,CAAC;IACf,IAAI,CAAC;QACH,EAAE,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC;IAErD,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;YAC/C,OAAO;gBACL,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,IAAI,EAAE,OAAO;aACd,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,IAAI,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,EAAE,CAAC;YAChD,OAAO;gBACL,KAAK,EAAE,IAAI;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,YAAY;gBACtB,IAAI,EAAE,UAAU;aACjB,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Payload construction + delivery.
3
+ *
4
+ * Contract: never throws, never blocks the response, 1500 ms
5
+ * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
6
+ * silent — a broken analytics call must never surface on a customer's site.
7
+ *
8
+ * Silent is not the same as invisible: an opt-in `onError` reports a non-2xx
9
+ * or a network failure to the caller. Without it a rejected report cannot be
10
+ * told from a delivered one, and a bad token reads as "no crawlers visited".
11
+ */
12
+ import type { AiCrawlWirePayload, ResolvedAiCrawlConfig } from "./types.js";
13
+ export declare const REPORT_TIMEOUT_MS = 1500;
14
+ /** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
15
+ export declare function buildReportUrl(cfg: ResolvedAiCrawlConfig, url: string): string;
16
+ export interface ReportInput {
17
+ url: string;
18
+ method: string;
19
+ userAgent: string;
20
+ status?: number | undefined;
21
+ ip?: string | undefined;
22
+ }
23
+ /** Build the wire payload. Optional fields are OMITTED, never null/wrong. */
24
+ export declare function buildPayload(cfg: ResolvedAiCrawlConfig, input: ReportInput): AiCrawlWirePayload;
25
+ /** Delivery options. */
26
+ export interface SendOptions {
27
+ /**
28
+ * Set `keepalive: true` on the fetch. Default `true` for fire-and-forget
29
+ * paths (Express, bare fetch) where nothing else keeps delivery alive
30
+ * past the response. Callers that schedule via `waitUntil` pass `false`:
31
+ * `waitUntil` already extends the lifetime, and `keepalive` is not
32
+ * reliably supported on workerd — if the option ever threw there, every
33
+ * report from the Cloudflare adapters would be lost, silently.
34
+ */
35
+ keepalive?: boolean;
36
+ }
37
+ /**
38
+ * Send a report. Returns a promise that ALWAYS resolves — network errors,
39
+ * timeouts and non-2xx responses are swallowed. Suitable for `waitUntil`
40
+ * (which needs the promise) and for fire-and-forget (ignore it).
41
+ */
42
+ export declare function sendReport(cfg: ResolvedAiCrawlConfig, payload: AiCrawlWirePayload, options?: SendOptions): Promise<void>;
43
+ //# sourceMappingURL=report.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAEV,kBAAkB,EAClB,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB,eAAO,MAAM,iBAAiB,OAAO,CAAC;AAEtC,qFAAqF;AACrF,wBAAgB,cAAc,CAAC,GAAG,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,CAa9E;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,EAAE,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzB;AAED,6EAA6E;AAC7E,wBAAgB,YAAY,CAAC,GAAG,EAAE,qBAAqB,EAAE,KAAK,EAAE,WAAW,GAAG,kBAAkB,CAe/F;AAcD,wBAAwB;AACxB,MAAM,WAAW,WAAW;IAC1B;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,qBAAqB,EAC1B,OAAO,EAAE,kBAAkB,EAC3B,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,IAAI,CAAC,CAsCf"}
package/dist/report.js ADDED
@@ -0,0 +1,116 @@
1
+ /**
2
+ * Payload construction + delivery.
3
+ *
4
+ * Contract: never throws, never blocks the response, 1500 ms
5
+ * timeout, `keepalive: true`, snake_case on the wire. Delivery failures are
6
+ * silent — a broken analytics call must never surface on a customer's site.
7
+ *
8
+ * Silent is not the same as invisible: an opt-in `onError` reports a non-2xx
9
+ * or a network failure to the caller. Without it a rejected report cannot be
10
+ * told from a delivered one, and a bad token reads as "no crawlers visited".
11
+ */
12
+ export const REPORT_TIMEOUT_MS = 1500;
13
+ /** Rebuild the reported URL against `publicOrigin` when configured. Never throws. */
14
+ export function buildReportUrl(cfg, url) {
15
+ try {
16
+ if (cfg.publicOrigin === undefined)
17
+ return url;
18
+ const schemeIdx = url.indexOf("://");
19
+ if (schemeIdx === -1) {
20
+ // Bare path — just prefix the origin.
21
+ return cfg.publicOrigin + (url.startsWith("/") ? url : `/${url}`);
22
+ }
23
+ const pathStart = url.indexOf("/", schemeIdx + 3);
24
+ return pathStart === -1 ? cfg.publicOrigin + "/" : cfg.publicOrigin + url.slice(pathStart);
25
+ }
26
+ catch {
27
+ return url;
28
+ }
29
+ }
30
+ /** Build the wire payload. Optional fields are OMITTED, never null/wrong. */
31
+ export function buildPayload(cfg, input) {
32
+ const payload = {
33
+ site_id: cfg.siteId,
34
+ url: buildReportUrl(cfg, input.url),
35
+ method: input.method.toUpperCase(),
36
+ user_agent: input.userAgent.slice(0, 1024),
37
+ ts: Date.now(),
38
+ };
39
+ if (typeof input.status === "number" && input.status >= 100 && input.status <= 599) {
40
+ payload.status = input.status;
41
+ }
42
+ if (input.ip !== undefined) {
43
+ payload.ip = input.ip;
44
+ }
45
+ return payload;
46
+ }
47
+ /** Abort signal with a 1500 ms timeout, where the runtime supports it. */
48
+ function timeoutSignal() {
49
+ try {
50
+ if (typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function") {
51
+ return AbortSignal.timeout(REPORT_TIMEOUT_MS);
52
+ }
53
+ }
54
+ catch {
55
+ /* ignore */
56
+ }
57
+ return undefined;
58
+ }
59
+ /**
60
+ * Send a report. Returns a promise that ALWAYS resolves — network errors,
61
+ * timeouts and non-2xx responses are swallowed. Suitable for `waitUntil`
62
+ * (which needs the promise) and for fire-and-forget (ignore it).
63
+ */
64
+ export function sendReport(cfg, payload, options) {
65
+ try {
66
+ const fetchImpl = cfg.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined);
67
+ if (fetchImpl === undefined)
68
+ return Promise.resolve();
69
+ const signal = timeoutSignal();
70
+ const init = {
71
+ method: "POST",
72
+ headers: {
73
+ "content-type": "application/json",
74
+ authorization: `Bearer ${cfg.authToken}`,
75
+ },
76
+ body: JSON.stringify(payload),
77
+ };
78
+ // Omit the property entirely when disabled — safer than `keepalive:
79
+ // false` on runtimes that reject the option.
80
+ if (options?.keepalive !== false)
81
+ init.keepalive = true;
82
+ if (signal !== undefined)
83
+ init.signal = signal;
84
+ return fetchImpl(cfg.endpoint, init).then((res) => {
85
+ // A rejected report used to be indistinguishable from a delivered one:
86
+ // both branches discarded their argument, so a 401 from a bad token or
87
+ // a 403 from blocked egress read as success and the customer saw
88
+ // "no crawlers" forever. Still silent by default: `onError` is opt-in.
89
+ if (res !== undefined && res !== null && res.ok !== true) {
90
+ notifyError(cfg, { kind: "http", status: res.status, endpoint: cfg.endpoint });
91
+ }
92
+ return undefined;
93
+ }, (cause) => {
94
+ notifyError(cfg, { kind: "network", cause, endpoint: cfg.endpoint });
95
+ return undefined;
96
+ });
97
+ }
98
+ catch {
99
+ return Promise.resolve();
100
+ }
101
+ }
102
+ /**
103
+ * Hand a delivery failure to the customer's callback.
104
+ *
105
+ * Guarded: a throwing `onError` must not escape into the host's response path,
106
+ * which is the whole reason delivery is silent in the first place.
107
+ */
108
+ function notifyError(cfg, error) {
109
+ try {
110
+ cfg.onError?.(error);
111
+ }
112
+ catch {
113
+ // Deliberately empty — see above.
114
+ }
115
+ }
116
+ //# sourceMappingURL=report.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAQH,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAEtC,qFAAqF;AACrF,MAAM,UAAU,cAAc,CAAC,GAA0B,EAAE,GAAW;IACpE,IAAI,CAAC;QACH,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS;YAAE,OAAO,GAAG,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrC,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,sCAAsC;YACtC,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;QAClD,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAC7F,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,GAAG,CAAC;IACb,CAAC;AACH,CAAC;AAUD,6EAA6E;AAC7E,MAAM,UAAU,YAAY,CAAC,GAA0B,EAAE,KAAkB;IACzE,MAAM,OAAO,GAAuB;QAClC,OAAO,EAAE,GAAG,CAAC,MAAM;QACnB,GAAG,EAAE,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,CAAC;QACnC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE;QAClC,UAAU,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;QAC1C,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE;KACf,CAAC;IACF,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACnF,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,EAAE,KAAK,SAAS,EAAE,CAAC;QAC3B,OAAO,CAAC,EAAE,GAAG,KAAK,CAAC,EAAE,CAAC;IACxB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,0EAA0E;AAC1E,SAAS,aAAa;IACpB,IAAI,CAAC;QACH,IAAI,OAAO,WAAW,KAAK,WAAW,IAAI,OAAO,WAAW,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACpF,OAAO,WAAW,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,YAAY;IACd,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAeD;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,GAA0B,EAC1B,OAA2B,EAC3B,OAAqB;IAErB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,OAAO,KAAK,KAAK,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAClF,IAAI,SAAS,KAAK,SAAS;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAEtD,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAgB;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,GAAG,CAAC,SAAS,EAAE;aACzC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC;QACF,oEAAoE;QACpE,6CAA6C;QAC7C,IAAI,OAAO,EAAE,SAAS,KAAK,KAAK;YAAE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxD,IAAI,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAE/C,OAAO,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,IAAI,CACvC,CAAC,GAAG,EAAE,EAAE;YACN,uEAAuE;YACvE,uEAAuE;YACvE,iEAAiE;YACjE,uEAAuE;YACvE,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;gBACzD,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,SAAS,CAAC;QACnB,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;YACjB,WAAW,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrE,OAAO,SAAS,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,WAAW,CAAC,GAA0B,EAAE,KAA2B;IAC1E,IAAI,CAAC;QACH,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,kCAAkC;IACpC,CAAC;AACH,CAAC"}
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Shared orchestrator used by every adapter:
3
+ * pre-filter → anchored UA match → category opt-outs → IP resolution → send.
4
+ *
5
+ * `trackFacts` never throws and never blocks: it either schedules the send
6
+ * via the runtime's `waitUntil` or fires-and-forgets it.
7
+ */
8
+ import type { AiCrawlWirePayload, RequestFacts, ResolvedAiCrawlConfig } from "./types.js";
9
+ /** Minimal structural view of a fetch-API `Request` — avoids DOM/runtime type deps. */
10
+ export interface FetchLikeRequest {
11
+ readonly method: string;
12
+ readonly url: string;
13
+ readonly headers: {
14
+ get(name: string): string | null;
15
+ };
16
+ }
17
+ /** Build {@link RequestFacts} from a fetch-API `Request`. Never throws. */
18
+ export declare function factsFromFetchRequest(request: FetchLikeRequest): RequestFacts;
19
+ /**
20
+ * Evaluate a request. Returns the wire payload when it should be reported,
21
+ * `null` otherwise. Pure apart from `Date.now()`; never throws.
22
+ */
23
+ export declare function evaluateRequest(cfg: ResolvedAiCrawlConfig, facts: RequestFacts, status?: number | undefined): AiCrawlWirePayload | null;
24
+ /**
25
+ * Evaluate and deliver. Uses `waitUntil` where the runtime provides one so
26
+ * the report outlives the response without ever delaying it; otherwise
27
+ * fire-and-forget. Never throws.
28
+ */
29
+ export declare function trackFacts(cfg: ResolvedAiCrawlConfig, facts: RequestFacts, status?: number | undefined, waitUntil?: ((promise: Promise<unknown>) => void) | undefined): void;
30
+ //# sourceMappingURL=track.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"track.d.ts","sourceRoot":"","sources":["../src/track.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EACV,kBAAkB,EAElB,YAAY,EACZ,qBAAqB,EACtB,MAAM,YAAY,CAAC;AAEpB,uFAAuF;AACvF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,OAAO,EAAE;QAAE,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAA;KAAE,CAAC;CACxD;AAED,2EAA2E;AAC3E,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,gBAAgB,GAAG,YAAY,CAY7E;AAgBD;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,qBAAqB,EAC1B,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,GAC1B,kBAAkB,GAAG,IAAI,CAoB3B;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CACxB,GAAG,EAAE,qBAAqB,EAC1B,KAAK,EAAE,YAAY,EACnB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,EAC3B,SAAS,CAAC,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,SAAS,GAC5D,IAAI,CAqBN"}
package/dist/track.js ADDED
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Shared orchestrator used by every adapter:
3
+ * pre-filter → anchored UA match → category opt-outs → IP resolution → send.
4
+ *
5
+ * `trackFacts` never throws and never blocks: it either schedules the send
6
+ * via the runtime's `waitUntil` or fires-and-forgets it.
7
+ */
8
+ import { passesPreFilter } from "./filter.js";
9
+ import { resolveCrawlerIp } from "./ip.js";
10
+ import { matchCrawler } from "./matcher.js";
11
+ import { buildPayload, sendReport } from "./report.js";
12
+ /** Build {@link RequestFacts} from a fetch-API `Request`. Never throws. */
13
+ export function factsFromFetchRequest(request) {
14
+ return {
15
+ method: request.method,
16
+ url: request.url,
17
+ header: (name) => {
18
+ try {
19
+ return request.headers.get(name);
20
+ }
21
+ catch {
22
+ return null;
23
+ }
24
+ },
25
+ };
26
+ }
27
+ /** Is this category disabled by config? */
28
+ function categoryDisabled(cfg, category) {
29
+ switch (category) {
30
+ case "answer_fetch":
31
+ return cfg.disableAnswerFetch;
32
+ case "search_index":
33
+ return cfg.disableSearchCrawlers;
34
+ case "training":
35
+ return cfg.disableTrainingCrawlers;
36
+ case "ai_crawler":
37
+ return cfg.disableOtherCrawlers;
38
+ }
39
+ }
40
+ /**
41
+ * Evaluate a request. Returns the wire payload when it should be reported,
42
+ * `null` otherwise. Pure apart from `Date.now()`; never throws.
43
+ */
44
+ export function evaluateRequest(cfg, facts, status) {
45
+ try {
46
+ const secFetchDest = facts.header("sec-fetch-dest");
47
+ if (!passesPreFilter(cfg, facts.method, facts.url, secFetchDest))
48
+ return null;
49
+ const userAgent = facts.header("user-agent");
50
+ const match = matchCrawler(userAgent);
51
+ if (match === null)
52
+ return null;
53
+ if (categoryDisabled(cfg, match.category))
54
+ return null;
55
+ return buildPayload(cfg, {
56
+ url: facts.url,
57
+ method: facts.method,
58
+ userAgent: userAgent, // non-null: matchCrawler(null) === null
59
+ status,
60
+ ip: resolveCrawlerIp(cfg, facts),
61
+ });
62
+ }
63
+ catch {
64
+ return null;
65
+ }
66
+ }
67
+ /**
68
+ * Evaluate and deliver. Uses `waitUntil` where the runtime provides one so
69
+ * the report outlives the response without ever delaying it; otherwise
70
+ * fire-and-forget. Never throws.
71
+ */
72
+ export function trackFacts(cfg, facts, status, waitUntil) {
73
+ try {
74
+ const payload = evaluateRequest(cfg, facts, status);
75
+ if (payload === null)
76
+ return;
77
+ // With a waitUntil, delivery lifetime is already guaranteed — skip
78
+ // keepalive (unverified on workerd). Without one, keepalive is what
79
+ // keeps the request alive past the response.
80
+ const delivery = sendReport(cfg, payload, {
81
+ keepalive: typeof waitUntil !== "function",
82
+ });
83
+ if (typeof waitUntil === "function") {
84
+ try {
85
+ waitUntil(delivery);
86
+ }
87
+ catch {
88
+ /* runtime rejected the promise registration — delivery still runs */
89
+ }
90
+ }
91
+ }
92
+ catch {
93
+ /* silent by contract */
94
+ }
95
+ }
96
+ //# sourceMappingURL=track.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"track.js","sourceRoot":"","sources":["../src/track.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAevD,2EAA2E;AAC3E,MAAM,UAAU,qBAAqB,CAAC,OAAyB;IAC7D,OAAO;QACL,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,MAAM,EAAE,CAAC,IAAY,EAAiB,EAAE;YACtC,IAAI,CAAC;gBACH,OAAO,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,2CAA2C;AAC3C,SAAS,gBAAgB,CAAC,GAA0B,EAAE,QAAyB;IAC7E,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,cAAc;YACjB,OAAO,GAAG,CAAC,kBAAkB,CAAC;QAChC,KAAK,cAAc;YACjB,OAAO,GAAG,CAAC,qBAAqB,CAAC;QACnC,KAAK,UAAU;YACb,OAAO,GAAG,CAAC,uBAAuB,CAAC;QACrC,KAAK,YAAY;YACf,OAAO,GAAG,CAAC,oBAAoB,CAAC;IACpC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,GAA0B,EAC1B,KAAmB,EACnB,MAA2B;IAE3B,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC;QACpD,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,EAAE,YAAY,CAAC;YAAE,OAAO,IAAI,CAAC;QAE9E,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAC7C,MAAM,KAAK,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACtC,IAAI,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChC,IAAI,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvD,OAAO,YAAY,CAAC,GAAG,EAAE;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,SAAS,EAAE,SAAmB,EAAE,wCAAwC;YACxE,MAAM;YACN,EAAE,EAAE,gBAAgB,CAAC,GAAG,EAAE,KAAK,CAAC;SACjC,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,GAA0B,EAC1B,KAAmB,EACnB,MAA2B,EAC3B,SAA6D;IAE7D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC;QACpD,IAAI,OAAO,KAAK,IAAI;YAAE,OAAO;QAE7B,mEAAmE;QACnE,oEAAoE;QACpE,6CAA6C;QAC7C,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE;YACxC,SAAS,EAAE,OAAO,SAAS,KAAK,UAAU;SAC3C,CAAC,CAAC;QACH,IAAI,OAAO,SAAS,KAAK,UAAU,EAAE,CAAC;YACpC,IAAI,CAAC;gBACH,SAAS,CAAC,QAAQ,CAAC,CAAC;YACtB,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;YACvE,CAAC;QACH,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wBAAwB;IAC1B,CAAC;AACH,CAAC"}