@devtune/ai-traffic 0.12.1 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  "use strict";
2
2
  /**
3
3
  * ABOUTME: Implements the DevTune AI Traffic edge-push client.
4
- * Handles registry refresh, AI bot matching, batched ingest beacons, and route wrappers.
4
+ * Handles registry refresh, AI bot and AI referral matching, batched ingest beacons, and route wrappers.
5
5
  */
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
7
  exports.DevTuneAiTrafficClient = exports.DEFAULT_REGISTRY_URL = exports.DEFAULT_INGEST_ENDPOINT = void 0;
@@ -10,6 +10,7 @@ exports.createDevTuneAiTrafficMiddleware = createDevTuneAiTrafficMiddleware;
10
10
  exports.mightBeAiBotUserAgent = mightBeAiBotUserAgent;
11
11
  exports.withDevTuneAiTrafficRoute = withDevTuneAiTrafficRoute;
12
12
  const registry_js_1 = require("./registry.js");
13
+ const referral_js_1 = require("./referral.js");
13
14
  const client_ip_js_1 = require("./client-ip.js");
14
15
  const version_js_1 = require("./version.js");
15
16
  const web_bot_auth_js_1 = require("./web-bot-auth.js");
@@ -27,6 +28,9 @@ const MAX_EVENT_PATH_LENGTH = 2048;
27
28
  const MAX_EVENT_UA_LENGTH = 2048;
28
29
  const MAX_EVENT_URL_LENGTH = 4096;
29
30
  const MAX_EVENT_IP_LENGTH = 45;
31
+ const MAX_EVENT_REFERRER_LENGTH = 2048;
32
+ const MAX_CAMPAIGN_VALUE_LENGTH = 512;
33
+ const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
30
34
  const UNAUTHORIZED_STATUS = 401;
31
35
  const RATE_LIMITED_STATUS = 429;
32
36
  const UNAVAILABLE_STATUS = 503;
@@ -36,6 +40,7 @@ const RETRY_BASE_DELAY_MS = 250;
36
40
  const DEFAULT_MAX_RETRY_DELAY_MS = 5_000;
37
41
  const RETRY_DELAY_CEILING_MS = 60_000;
38
42
  const DEFAULT_MAX_QUEUED_EVENTS = 1_000;
43
+ const DEFAULT_MAX_QUEUED_REFERRAL_EVENTS = 200;
39
44
  const MAX_QUEUED_EVENTS_CEILING = 100_000;
40
45
  const AI_BOT_USER_AGENT_HINTS = [
41
46
  "-ai",
@@ -79,6 +84,8 @@ class DevTuneAiTrafficClient {
79
84
  maxRetryAttempts;
80
85
  maxRetryDelayMs;
81
86
  maxQueuedEvents;
87
+ maxQueuedReferralEvents;
88
+ captureReferrals;
82
89
  trustedProxy;
83
90
  projectId;
84
91
  keyCredential;
@@ -90,6 +97,7 @@ class DevTuneAiTrafficClient {
90
97
  flushPromise = null;
91
98
  delayedFlushPromise = null;
92
99
  queue = [];
100
+ referralQueue = [];
93
101
  constructor(options) {
94
102
  this.options = options;
95
103
  if (!options.ingestKey && !options.projectId) {
@@ -124,58 +132,57 @@ class DevTuneAiTrafficClient {
124
132
  this.maxRetryAttempts = clampInteger(options.maxRetryAttempts ?? DEFAULT_MAX_RETRY_ATTEMPTS, 0, RETRY_ATTEMPTS_CEILING);
125
133
  this.maxRetryDelayMs = clampInteger(options.maxRetryDelayMs ?? DEFAULT_MAX_RETRY_DELAY_MS, 0, RETRY_DELAY_CEILING_MS);
126
134
  this.maxQueuedEvents = clampInteger(options.maxQueuedEvents ?? DEFAULT_MAX_QUEUED_EVENTS, MAX_INGEST_BATCH_SIZE, MAX_QUEUED_EVENTS_CEILING);
135
+ this.maxQueuedReferralEvents = clampInteger(options.maxQueuedReferralEvents ?? DEFAULT_MAX_QUEUED_REFERRAL_EVENTS, MAX_INGEST_BATCH_SIZE, MAX_QUEUED_EVENTS_CEILING);
136
+ this.captureReferrals = options.captureReferrals ?? true;
127
137
  const fallbackEntries = (0, registry_js_1.compileAiBotRegistryEntries)(registry_js_1.FALLBACK_AI_BOT_REGISTRY_ENTRIES);
128
138
  const fallbackPrefilter = createRegistryPrefilter(fallbackEntries);
129
139
  this.registry = {
130
140
  entries: fallbackEntries,
131
141
  etag: null,
132
142
  fetchedAt: 0,
143
+ referrers: referral_js_1.FALLBACK_AI_REFERRAL_SOURCES,
133
144
  ...fallbackPrefilter,
134
145
  version: "bundled",
135
146
  };
136
147
  }
148
+ /**
149
+ * A cheap check that `trackRequest()` could record this request: true for
150
+ * an AI bot candidate, by user agent alone, or for an AI referral candidate
151
+ * (see `matchReferralPlatform`). Adapters that record after the response
152
+ * call it first, so other requests cost no response work. A request without
153
+ * a `url` is checked as a bot candidate only.
154
+ */
137
155
  mightTrackRequest(request) {
138
156
  const userAgent = request.headers.get("user-agent");
139
157
  if (!userAgent) {
140
158
  return false;
141
159
  }
142
- if (mightBeAiBotUserAgent(userAgent) || this.registry.requiresFullPrefilterScan) {
143
- return true;
144
- }
145
- const normalized = userAgent.toLowerCase();
146
- return this.registry.userAgentHints.some((hint) => normalized.includes(hint));
160
+ return this.mightBeAiBot(userAgent) || this.matchReferralPlatform(request) !== null;
147
161
  }
148
162
  /**
149
- * Records a matched AI bot request. Pass `receivedAt` when the request is
150
- * recorded after its response, taken from `signedArrivalTime()` as the
151
- * request arrived: a signed event is checked against the time it was signed
152
- * for. Other events keep the time they are recorded.
163
+ * Records a matched AI bot request or AI referral visit. A user agent in
164
+ * the bot registry makes it a bot request whatever its `Referer`. Pass
165
+ * `receivedAt` when the request is recorded after its response, taken from
166
+ * `signedArrivalTime()` as the request arrived: a signed bot event is
167
+ * checked against the time it was signed for. Other events keep the time
168
+ * they are recorded.
153
169
  */
154
170
  trackRequest(request, waitUntilContext, responseStatus, receivedAt) {
155
171
  this.refreshRegistryIfStale(waitUntilContext);
156
172
  this.captureVercelOidcToken(request);
157
- if (!this.mightTrackRequest(request) ||
158
- isPageInitiatedRequest(request) ||
159
- isFrameworkDataRequest(request)) {
160
- return null;
161
- }
162
173
  const userAgent = request.headers.get("user-agent");
163
- if (!userAgent) {
174
+ const source = userAgent ? this.matchSource(request, userAgent) : null;
175
+ if (!userAgent || !source) {
164
176
  return null;
165
177
  }
166
- const entry = (0, registry_js_1.matchAiBotRegistryEntry)(userAgent, this.registry.entries);
167
- if (!entry) {
168
- return null;
169
- }
170
- const event = this.createEvent(request, userAgent, responseStatus, receivedAt);
178
+ const event = source.kind === "bot"
179
+ ? this.createBotEvent(request, userAgent, responseStatus, receivedAt)
180
+ : this.createReferralEvent(request, userAgent, responseStatus);
171
181
  if (!event) {
172
182
  return null;
173
183
  }
174
- this.enqueue(event, waitUntilContext);
175
- return {
176
- entry,
177
- event,
178
- };
184
+ this.enqueue(source.kind, event, waitUntilContext);
185
+ return { ...source, event };
179
186
  }
180
187
  /**
181
188
  * The client's clock now for a request carrying Web Bot Auth signature
@@ -214,7 +221,8 @@ class DevTuneAiTrafficClient {
214
221
  if (!response.ok) {
215
222
  throw new Error(`Registry request failed with status ${response.status}`);
216
223
  }
217
- const payload = (0, registry_js_1.parseAiBotRegistryPayload)(await response.json());
224
+ const body = await response.json();
225
+ const payload = (0, registry_js_1.parseAiBotRegistryPayload)(body);
218
226
  if (!payload) {
219
227
  throw new Error("Registry response was not a valid DevTune registry");
220
228
  }
@@ -226,15 +234,38 @@ class DevTuneAiTrafficClient {
226
234
  entries,
227
235
  etag: response.headers.get("etag") ?? `"${payload.version}"`,
228
236
  fetchedAt: Date.now(),
237
+ // Missing or unusable referrers, as from a registry older than referral
238
+ // capture, keep the last working lists.
239
+ referrers: (0, referral_js_1.parseAiReferralSources)((0, registry_js_1.isRecord)(body) ? body.referrers : undefined) ??
240
+ this.registry.referrers,
229
241
  ...createRegistryPrefilter(entries),
230
242
  version: payload.version,
231
243
  };
232
244
  }
233
245
  async flush() {
246
+ return this.startDrain(true);
247
+ }
248
+ /**
249
+ * Starts a drain, or joins the one in flight. A drain that spends a retry
250
+ * budget leaves events queued that would otherwise wait for the next tracked
251
+ * request, so it schedules one follow-up drain, no sooner than a capped
252
+ * retry's wait. The follow-up schedules none: if it is throttled too, the
253
+ * rest waits for the next tracked request, as before 0.13.0. The timer is
254
+ * unref'd and not tied to `waitUntil`, so only a long-lived process runs it.
255
+ */
256
+ startDrain(followUpWhenThrottled) {
234
257
  if (this.flushPromise) {
235
258
  return this.flushPromise;
236
259
  }
237
- this.flushPromise = this.flushQueuedEvents().finally(() => {
260
+ this.flushPromise = this.flushQueuedEvents()
261
+ .then((spentRetries) => {
262
+ if (spentRetries && followUpWhenThrottled && this.queuedEventCount() > 0) {
263
+ const timer = setTimeout(() => scheduleBackground(this.startDrain(false)), Math.max(this.flushIntervalMs, this.maxRetryDelayMs));
264
+ // Retrying telemetry must never hold a Node process open.
265
+ timer.unref?.();
266
+ }
267
+ })
268
+ .finally(() => {
238
269
  this.flushPromise = null;
239
270
  });
240
271
  return this.flushPromise;
@@ -260,26 +291,133 @@ class DevTuneAiTrafficClient {
260
291
  });
261
292
  scheduleBackground(this.refreshPromise, this.waitUntilRegistryRefresh ? waitUntilContext : undefined);
262
293
  }
263
- createEvent(request, userAgent, responseStatus, receivedAt) {
264
- const url = parseRequestUrl(request.url);
265
- if (!url) {
294
+ mightBeAiBot(userAgent) {
295
+ if (mightBeAiBotUserAgent(userAgent) || this.registry.requiresFullPrefilterScan) {
296
+ return true;
297
+ }
298
+ const normalized = userAgent.toLowerCase();
299
+ return this.registry.userAgentHints.some((hint) => normalized.includes(hint));
300
+ }
301
+ /**
302
+ * A bot registry match that is not a page's own request, or failing that the
303
+ * AI platform a referral visit came from. A bot user agent wins: it is never
304
+ * recorded as a referral.
305
+ */
306
+ matchSource(request, userAgent) {
307
+ const entry = this.mightBeAiBot(userAgent)
308
+ ? (0, registry_js_1.matchAiBotRegistryEntry)(userAgent, this.registry.entries)
309
+ : null;
310
+ if (entry) {
311
+ return isPageInitiatedRequest(request) || isFrameworkDataRequest(request)
312
+ ? null
313
+ : { kind: "bot", entry };
314
+ }
315
+ const platform = this.matchReferralPlatform(request);
316
+ return platform ? { kind: "referral", platform } : null;
317
+ }
318
+ /**
319
+ * The AI platform a referral candidate came from, or null. A candidate is a
320
+ * `GET` from another site (`Sec-Fetch-Site` not `same-origin` or
321
+ * `same-site`) whose `Referer` host, or failing that `utm_source`, is in the
322
+ * registry, and that loads a top-level page: `Sec-Fetch-Dest` is `document`
323
+ * or absent, it is not a prefetch, and it is not a framework data request.
324
+ * Frames stay bot-only. Every request reaches this, so the method and
325
+ * fetch-site checks and substring tests run before any URL is parsed.
326
+ * Browsers send the `Referer` host lower-cased, as the registry lists it.
327
+ * Always null when `captureReferrals` is off.
328
+ */
329
+ matchReferralPlatform(request) {
330
+ const { headers, method, url } = request;
331
+ if (!this.captureReferrals ||
332
+ (method !== "GET" && method !== "get") ||
333
+ typeof url !== "string") {
334
+ return null;
335
+ }
336
+ const site = headers.get("sec-fetch-site");
337
+ if (site === "same-origin" || site === "same-site") {
338
+ return null;
339
+ }
340
+ const platform = this.matchReferrerOrUtmSource(headers.get("referer"), url);
341
+ const destination = headers.get("sec-fetch-dest");
342
+ if (!platform ||
343
+ (destination !== null && destination !== "document") ||
344
+ isPageInitiatedRequest(request) ||
345
+ isFrameworkDataRequest({ headers, url })) {
346
+ return null;
347
+ }
348
+ return platform;
349
+ }
350
+ /** The `Referer` host's platform, or else the `utm_source`'s, as DevTune's classifier orders them. */
351
+ matchReferrerOrUtmSource(referer, url) {
352
+ const { referrers } = this.registry;
353
+ if (referer && mentionsAnyHost(referer, referrers.hosts)) {
354
+ const referrer = parseHttpUrl(referer);
355
+ const platform = referrer ? (0, referral_js_1.matchAiReferralHost)(referrer.hostname, referrers.hosts) : null;
356
+ if (platform) {
357
+ return platform;
358
+ }
359
+ }
360
+ if (!url.includes("utm_source=")) {
361
+ return null;
362
+ }
363
+ const source = parseRequestUrl(url)?.searchParams.get("utm_source");
364
+ return source ? (0, referral_js_1.matchAiReferralUtmSource)(source, referrers) : null;
365
+ }
366
+ createBotEvent(request, userAgent, responseStatus, receivedAt) {
367
+ const page = this.createBaseFields(request, userAgent, responseStatus);
368
+ if (!page) {
266
369
  return null;
267
370
  }
268
- const pathname = truncateString(url.pathname || "/", MAX_EVENT_PATH_LENGTH);
269
- const status = this.resolveStatus(request, pathname, responseStatus);
270
371
  const ip = (0, client_ip_js_1.resolveClientIp)(request, this.trustedProxy);
271
372
  const webBotAuth = readWebBotAuthSafely(request);
272
373
  const ts = webBotAuth && receivedAt && Number.isFinite(receivedAt.getTime()) ? receivedAt : this.now();
273
374
  return {
274
- path: pathname,
275
- url: truncateString(`${url.origin}${pathname}`, MAX_EVENT_URL_LENGTH),
276
- ua: truncateString(userAgent, MAX_EVENT_UA_LENGTH),
277
- ...(status === null ? {} : { status }),
375
+ ...page.fields,
278
376
  ts: ts.toISOString(),
279
377
  ...(ip === null ? {} : { ip: truncateString(ip, MAX_EVENT_IP_LENGTH) }),
280
378
  ...(webBotAuth === null ? {} : { webBotAuth }),
281
379
  };
282
380
  }
381
+ /**
382
+ * A referral visit carries its `Referer` origin and campaign parameters,
383
+ * and never an address or signature material: there is no claimed crawler
384
+ * identity to verify. A visit answered with a redirect is not recorded:
385
+ * the browser takes the same `Referer` to the redirect's target, which is
386
+ * where DevTune counts it, so ingest drops the redirect's event.
387
+ */
388
+ createReferralEvent(request, userAgent, responseStatus) {
389
+ const page = this.createBaseFields(request, userAgent, responseStatus);
390
+ if (!page || (page.fields.status !== undefined && REDIRECT_STATUSES.has(page.fields.status))) {
391
+ return null;
392
+ }
393
+ const referrer = readReferrerOrigin(request.headers.get("referer"));
394
+ const campaign = readCampaign(page.url);
395
+ return {
396
+ ...page.fields,
397
+ ts: this.now().toISOString(),
398
+ ...(referrer === null ? {} : { referrer }),
399
+ ...(campaign === null ? {} : { campaign }),
400
+ };
401
+ }
402
+ /** The fields every event carries. The `url` is the origin and path, without the query. */
403
+ createBaseFields(request, userAgent, responseStatus) {
404
+ const url = parseRequestUrl(request.url);
405
+ if (!url) {
406
+ return null;
407
+ }
408
+ const pathname = truncateString(url.pathname || "/", MAX_EVENT_PATH_LENGTH);
409
+ const status = this.resolveStatus(request, pathname, responseStatus);
410
+ return {
411
+ url,
412
+ fields: {
413
+ path: pathname,
414
+ url: truncateString(`${url.origin}${pathname}`, MAX_EVENT_URL_LENGTH),
415
+ ua: truncateString(userAgent, MAX_EVENT_UA_LENGTH),
416
+ ...(status === null ? {} : { status }),
417
+ ...(isBehindCdn(request.headers) ? { cdn: true } : {}),
418
+ },
419
+ };
420
+ }
283
421
  resolveStatus(request, pathname, responseStatus) {
284
422
  if (responseStatus === null) {
285
423
  return null;
@@ -317,24 +455,20 @@ class DevTuneAiTrafficClient {
317
455
  return false;
318
456
  }
319
457
  /**
320
- * Adds an event to the queue and decides when to send.
458
+ * Adds an event to its queue and decides when to send.
321
459
  *
322
460
  * Backing off on a 429 means the queue can now drain slower than traffic
323
461
  * arrives, which it never could when every batch cost exactly one request.
324
462
  * The bound is what keeps that from growing without limit in a customer's
325
463
  * process: past `maxQueuedEvents` the oldest events are shed, because the
326
- * newest ones are the ones still worth measuring.
464
+ * newest ones are the ones still worth measuring. Referral visits have
465
+ * their own queue and bound, `maxQueuedReferralEvents`, so a burst of them
466
+ * never sheds a bot event.
327
467
  */
328
- enqueue(event, waitUntilContext) {
329
- this.queue.push(event);
330
- if (this.queue.length > this.maxQueuedEvents) {
331
- const shed = this.queue.splice(0, this.queue.length - this.maxQueuedEvents);
332
- this.sampleWarn("DevTune AI Traffic ingest queue overflowed", {
333
- droppedEvents: shed.length,
334
- maxQueuedEvents: this.maxQueuedEvents,
335
- });
336
- }
337
- if (this.queue.length >= this.batchSize) {
468
+ enqueue(kind, event, waitUntilContext) {
469
+ (kind === "bot" ? this.queue : this.referralQueue).push(event);
470
+ this.shedOverflow(kind);
471
+ if (this.queuedEventCount() >= this.batchSize) {
338
472
  scheduleBackground(this.flush(), waitUntilContext);
339
473
  return;
340
474
  }
@@ -349,8 +483,37 @@ class DevTuneAiTrafficClient {
349
483
  });
350
484
  scheduleBackground(this.delayedFlushPromise, waitUntilContext);
351
485
  }
486
+ /** Sheds the oldest events past a queue's bound, with a sampled warning. */
487
+ shedOverflow(kind) {
488
+ const bot = kind === "bot";
489
+ const queue = bot ? this.queue : this.referralQueue;
490
+ const maxEvents = bot ? this.maxQueuedEvents : this.maxQueuedReferralEvents;
491
+ if (queue.length <= maxEvents) {
492
+ return;
493
+ }
494
+ const droppedEvents = queue.splice(0, queue.length - maxEvents).length;
495
+ if (bot) {
496
+ this.sampleWarn("DevTune AI Traffic ingest queue overflowed", {
497
+ droppedEvents,
498
+ maxQueuedEvents: maxEvents,
499
+ });
500
+ }
501
+ else {
502
+ this.sampleWarn("DevTune AI Traffic referral queue overflowed", {
503
+ droppedEvents,
504
+ maxQueuedReferralEvents: maxEvents,
505
+ });
506
+ }
507
+ }
352
508
  /**
353
- * Drains the queue one wire batch at a time.
509
+ * Drains the queues one wire batch at a time, bots before referrals. A
510
+ * batch never mixes the two, and referrals never cost bots: each has its own
511
+ * retry budget per flush, a referral batch that spends its budget is dropped
512
+ * while bots keep draining, and a referral batch that gets a 429 while bots
513
+ * are waiting goes back to the front of its queue after the backoff, with
514
+ * its attempt count, so the bots are sent first. When the bots' budget is
515
+ * spent the drain ends there: the rate limit covers both kinds, so referrals
516
+ * would only meet the same 429, and the next drain starts with bots.
354
517
  *
355
518
  * A 429 is the one failure the ingest endpoint tells us is temporary, so the
356
519
  * batch is held out of the queue and redelivered after the server's
@@ -360,14 +523,14 @@ class DevTuneAiTrafficClient {
360
523
  * are not folded into the batch being retried, so they cannot be dropped on a
361
524
  * bound they never had a chance to spend.
362
525
  *
363
- * The retry budget belongs to the whole drain, not to each batch. A per-batch
526
+ * Each budget belongs to the whole drain, not to each batch. A per-batch
364
527
  * bound says nothing about how long a drain can run, because a throttled
365
528
  * endpoint rejects every batch in turn and traffic keeps arriving behind it.
366
- * One budget per pass caps the added latency of a flush at
367
- * `maxRetryAttempts * maxRetryDelayMs` no matter how deep the queue is. When
368
- * it is spent the current batch is dropped with the usual sampled warning and
369
- * the drain stops, leaving the rest queued for the next flush, which gets a
370
- * fresh budget.
529
+ * One budget per kind per pass caps the added latency of a flush at
530
+ * `2 * maxRetryAttempts * maxRetryDelayMs` no matter how deep the queues
531
+ * are. When a budget is spent the current batch is dropped with the usual
532
+ * sampled warning, leaving the rest queued for the next flush, which gets
533
+ * fresh budgets. Resolves true when a budget was spent.
371
534
  *
372
535
  * Every other failure keeps its previous behavior: one attempt, dropped, and
373
536
  * a sampled warning.
@@ -382,12 +545,26 @@ class DevTuneAiTrafficClient {
382
545
  * keys, is retried exactly like a 429.
383
546
  */
384
547
  async flushQueuedEvents() {
385
- let retryBudget = this.maxRetryAttempts;
548
+ const retryBudgets = { bot: this.maxRetryAttempts, referral: this.maxRetryAttempts };
549
+ let referralsStopped = false;
386
550
  let keyOnly = false;
387
551
  const failedExchanges = new Map();
388
- while (this.queue.length > 0) {
389
- const batch = this.queue.splice(0, MAX_INGEST_BATCH_SIZE);
390
- let attempts = 0;
552
+ // The last referral batch that stepped aside for bots. The queue bound
553
+ // sheds from the front, so its last event is the one that survives.
554
+ let requeued = null;
555
+ for (;;) {
556
+ const kind = this.queue.length > 0
557
+ ? "bot"
558
+ : !referralsStopped && this.referralQueue.length > 0
559
+ ? "referral"
560
+ : null;
561
+ if (kind === null) {
562
+ return referralsStopped;
563
+ }
564
+ const batch = (kind === "bot" ? this.queue : this.referralQueue).splice(0, MAX_INGEST_BATCH_SIZE);
565
+ let attempts = kind === "referral" && requeued?.last !== undefined && batch.includes(requeued.last)
566
+ ? requeued.attempts
567
+ : 0;
391
568
  for (;;) {
392
569
  // The key path stays synchronous up to each send, as it always was.
393
570
  const credential = !keyOnly && this.projectId && this.vercelOidcToken
@@ -398,10 +575,11 @@ class DevTuneAiTrafficClient {
398
575
  }
399
576
  if (credential === null || credential === "drop-queue") {
400
577
  if (credential === null) {
401
- this.warnMissingVercelOidcTokenOnce(batch.length + this.queue.length);
578
+ this.warnMissingVercelOidcTokenOnce(batch.length + this.queuedEventCount());
402
579
  }
403
580
  this.queue.splice(0);
404
- return;
581
+ this.referralQueue.splice(0);
582
+ return false;
405
583
  }
406
584
  let response;
407
585
  try {
@@ -436,17 +614,29 @@ class DevTuneAiTrafficClient {
436
614
  }
437
615
  break;
438
616
  }
439
- if (retryBudget === 0) {
617
+ if (retryBudgets[kind] === 0) {
440
618
  this.sampleWarn("DevTune AI Traffic ingest request failed", {
441
619
  attempts,
442
620
  droppedEvents: batch.length,
443
621
  status: response.status,
444
622
  });
445
- return;
623
+ if (kind === "bot") {
624
+ return true;
625
+ }
626
+ referralsStopped = true;
627
+ break;
446
628
  }
447
- const retryDelayMs = this.resolveRetryDelayMs(response.headers.get("retry-after"), this.maxRetryAttempts - retryBudget);
448
- retryBudget -= 1;
629
+ const retryDelayMs = this.resolveRetryDelayMs(response.headers.get("retry-after"), this.maxRetryAttempts - retryBudgets[kind]);
630
+ retryBudgets[kind] -= 1;
449
631
  await delay(retryDelayMs);
632
+ // The rate limit covers both kinds, so waiting bots go after the
633
+ // backoff rather than into the same 429.
634
+ if (kind === "referral" && this.queue.length > 0) {
635
+ this.referralQueue.unshift(...batch);
636
+ this.shedOverflow("referral");
637
+ requeued = { last: batch.at(-1), attempts };
638
+ break;
639
+ }
450
640
  }
451
641
  }
452
642
  }
@@ -497,7 +687,7 @@ class DevTuneAiTrafficClient {
497
687
  if (this.keyCredential) {
498
688
  return this.keyCredential;
499
689
  }
500
- this.sampleWarn("DevTune AI Traffic dropped events: the Vercel OIDC token expired before it could be used", { droppedEvents: batchEvents + this.queue.length });
690
+ this.sampleWarn("DevTune AI Traffic dropped events: the Vercel OIDC token expired before it could be used", { droppedEvents: batchEvents + this.queuedEventCount() });
501
691
  return "drop-queue";
502
692
  }
503
693
  const earlierFailure = failedExchanges.get(token.value);
@@ -506,7 +696,7 @@ class DevTuneAiTrafficClient {
506
696
  return this.keyCredential;
507
697
  }
508
698
  this.sampleWarn("DevTune AI Traffic could not exchange the Vercel OIDC token", {
509
- droppedEvents: batchEvents + this.queue.length,
699
+ droppedEvents: batchEvents + this.queuedEventCount(),
510
700
  ...earlierFailure,
511
701
  });
512
702
  return "drop-queue";
@@ -546,6 +736,9 @@ class DevTuneAiTrafficClient {
546
736
  this.warnedMissingVercelOidcToken = true;
547
737
  this.logger?.warn("DevTune AI Traffic dropped events: projectId sends only from a Vercel deployment with OIDC enabled, and no ingestKey is set", { droppedEvents });
548
738
  }
739
+ queuedEventCount() {
740
+ return this.queue.length + this.referralQueue.length;
741
+ }
549
742
  resolveRetryDelayMs(retryAfterHeader, retryAttempts) {
550
743
  const retryAfterMs = parseRetryAfterMs(retryAfterHeader, this.now());
551
744
  const backoffMs = RETRY_BASE_DELAY_MS * 2 ** retryAttempts;
@@ -601,6 +794,69 @@ function readWebBotAuthSafely(request) {
601
794
  return null;
602
795
  }
603
796
  }
797
+ /** A cheap substring test, so a `Referer` naming no registry host is never parsed. */
798
+ function mentionsAnyHost(referer, hosts) {
799
+ for (const { hostname } of hosts) {
800
+ if (referer.includes(hostname)) {
801
+ return true;
802
+ }
803
+ }
804
+ return false;
805
+ }
806
+ /** A URL, only when it parses and is `http:` or `https:`. */
807
+ function parseHttpUrl(value) {
808
+ const url = value ? parseRequestUrl(value) : null;
809
+ return url && (url.protocol === "http:" || url.protocol === "https:") ? url : null;
810
+ }
811
+ /** The `Referer` cut to its origin, as DevTune's snippet keeps it. */
812
+ function readReferrerOrigin(referer) {
813
+ const origin = parseHttpUrl(referer)?.origin;
814
+ return origin && origin.length <= MAX_EVENT_REFERRER_LENGTH ? origin : null;
815
+ }
816
+ /**
817
+ * The campaign parameters DevTune's referral and paid-click rules read, and
818
+ * nothing else from the query. A value DevTune would not accept is left out
819
+ * rather than truncated: an over-long medium is never a paid one.
820
+ */
821
+ function readCampaign(url) {
822
+ const utmSource = readCampaignValue(url, "utm_source");
823
+ const utmMedium = readCampaignValue(url, "utm_medium");
824
+ const oppref = url.searchParams.has("oppref");
825
+ if (utmSource === null && utmMedium === null && !oppref) {
826
+ return null;
827
+ }
828
+ return {
829
+ ...(utmSource === null ? {} : { utm_source: utmSource }),
830
+ ...(utmMedium === null ? {} : { utm_medium: utmMedium }),
831
+ ...(oppref ? { oppref: true } : {}),
832
+ };
833
+ }
834
+ function readCampaignValue(url, name) {
835
+ const value = url.searchParams.get(name);
836
+ return value && value.length <= MAX_CAMPAIGN_VALUE_LENGTH ? value : null;
837
+ }
838
+ const CDN_HEADERS = ["cf-ray", "x-amz-cf-id", "fastly-client-ip", "fastly-ff", "fastly-ssl"];
839
+ /**
840
+ * Whether a CDN forwarded the request: Cloudflare's `cf-ray`, CloudFront's
841
+ * `x-amz-cf-id`, or any `fastly-*` header where the header names can be
842
+ * listed. A cache that adds none of them goes undetected.
843
+ */
844
+ function isBehindCdn(headers) {
845
+ try {
846
+ if (CDN_HEADERS.some((name) => headers.get(name) !== null)) {
847
+ return true;
848
+ }
849
+ for (const name of headers.keys?.() ?? []) {
850
+ if (name.toLowerCase().startsWith("fastly-")) {
851
+ return true;
852
+ }
853
+ }
854
+ }
855
+ catch {
856
+ // A custom headers object must not throw into the customer's request path.
857
+ }
858
+ return false;
859
+ }
604
860
  function scheduleBackground(promise, waitUntilContext) {
605
861
  const guardedPromise = promise.catch(() => undefined);
606
862
  if (waitUntilContext) {
@@ -610,7 +866,7 @@ function scheduleBackground(promise, waitUntilContext) {
610
866
  void guardedPromise;
611
867
  }
612
868
  function getWaitUntilContext(value) {
613
- if (!isRecord(value)) {
869
+ if (!(0, registry_js_1.isRecord)(value)) {
614
870
  return undefined;
615
871
  }
616
872
  const waitUntil = value.waitUntil;
@@ -707,7 +963,7 @@ function classifyFailure(error) {
707
963
  ? { kind: error.kind, status: error.status }
708
964
  : { kind: error.kind };
709
965
  }
710
- const name = isRecord(error) ? error.name : undefined;
966
+ const name = (0, registry_js_1.isRecord)(error) ? error.name : undefined;
711
967
  if (name === "TimeoutError") {
712
968
  return { kind: "timeout" };
713
969
  }
@@ -791,7 +1047,4 @@ function truncateString(value, maxLength) {
791
1047
  }
792
1048
  return value.slice(0, maxLength);
793
1049
  }
794
- function isRecord(value) {
795
- return typeof value === "object" && value !== null;
796
- }
797
1050
  //# sourceMappingURL=client.js.map