@devtune/ai-traffic 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +138 -62
- package/dist/cjs/client.js +407 -0
- package/dist/cjs/client.js.map +1 -0
- package/dist/cjs/detection.js +47 -0
- package/dist/cjs/detection.js.map +1 -0
- package/dist/cjs/express.js +61 -0
- package/dist/cjs/express.js.map +1 -0
- package/dist/cjs/index.js +25 -0
- package/dist/cjs/index.js.map +1 -0
- package/dist/cjs/node.js +60 -0
- package/dist/cjs/node.js.map +1 -0
- package/dist/cjs/package.json +3 -0
- package/dist/cjs/registry.js +264 -0
- package/dist/cjs/registry.js.map +1 -0
- package/dist/client.d.ts +1 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +38 -5
- package/dist/client.js.map +1 -1
- package/dist/detection.d.ts +51 -0
- package/dist/detection.d.ts.map +1 -0
- package/dist/detection.js +42 -0
- package/dist/detection.js.map +1 -0
- package/dist/express.d.ts +24 -0
- package/dist/express.d.ts.map +1 -0
- package/dist/express.js +58 -0
- package/dist/express.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/node.d.ts +23 -0
- package/dist/node.d.ts.map +1 -0
- package/dist/node.js +57 -0
- package/dist/node.js.map +1 -0
- package/package.json +15 -2
package/README.md
CHANGED
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
# @devtune/ai-traffic
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
Cloudflare-proxied sites should prefer the Cloudflare pull integration when available. For Vercel and self-hosted sites, this package keeps volume low by filtering at your edge instead of shipping full request logs.
|
|
3
|
+
Measure which AI crawlers visit your site with a dependency-free sensor that filters at the edge and forwards only matched machine traffic to DevTune.
|
|
6
4
|
|
|
7
5
|
## Install
|
|
8
6
|
|
|
@@ -10,118 +8,196 @@ Cloudflare-proxied sites should prefer the Cloudflare pull integration when avai
|
|
|
10
8
|
pnpm add @devtune/ai-traffic
|
|
11
9
|
```
|
|
12
10
|
|
|
13
|
-
|
|
11
|
+
Create a project-scoped server-side ingest key and expose it only to your server runtime:
|
|
14
12
|
|
|
15
13
|
```bash
|
|
16
14
|
DEVTUNE_AI_TRAFFIC_INGEST_KEY=dt_ingest_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
17
15
|
```
|
|
18
16
|
|
|
19
|
-
##
|
|
17
|
+
## Quickstarts
|
|
18
|
+
|
|
19
|
+
### Next.js
|
|
20
20
|
|
|
21
|
-
For Next.js 16, add `proxy.ts
|
|
21
|
+
For Next.js 16, add `proxy.ts`. For older Next.js projects, use the same body in `middleware.ts` and export `middleware()` instead.
|
|
22
22
|
|
|
23
23
|
```ts
|
|
24
24
|
// proxy.ts
|
|
25
|
-
import {
|
|
25
|
+
import { createDevTuneAiTrafficMiddleware } from "@devtune/ai-traffic";
|
|
26
26
|
import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
|
|
27
27
|
|
|
28
|
-
const
|
|
28
|
+
const trackAiTraffic = createDevTuneAiTrafficMiddleware({
|
|
29
29
|
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
30
|
-
batchSize: 1,
|
|
31
|
-
defaultStatus: null,
|
|
32
|
-
flushIntervalMs: 0,
|
|
33
|
-
waitUntilRegistryRefresh: false,
|
|
34
30
|
});
|
|
35
31
|
|
|
36
32
|
export function proxy(request: NextRequest, event: NextFetchEvent) {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
aiTraffic.trackRequest(request, event);
|
|
33
|
+
trackAiTraffic(request, event);
|
|
40
34
|
|
|
41
|
-
return
|
|
35
|
+
return NextResponse.next();
|
|
42
36
|
}
|
|
43
37
|
```
|
|
44
38
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
When your proxy or middleware returns a response directly, pass that exact status:
|
|
39
|
+
### Express
|
|
48
40
|
|
|
49
41
|
```ts
|
|
50
|
-
|
|
51
|
-
|
|
42
|
+
import express from "express";
|
|
43
|
+
import { createDevTuneAiTrafficExpressMiddleware } from "@devtune/ai-traffic/express";
|
|
52
44
|
|
|
53
|
-
|
|
45
|
+
const app = express();
|
|
54
46
|
|
|
55
|
-
|
|
56
|
-
|
|
47
|
+
app.use(
|
|
48
|
+
createDevTuneAiTrafficExpressMiddleware({
|
|
49
|
+
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
50
|
+
}),
|
|
51
|
+
);
|
|
57
52
|
```
|
|
58
53
|
|
|
59
|
-
|
|
54
|
+
The middleware calls `next()` immediately and records the actual response status after Express emits `finish`.
|
|
55
|
+
|
|
56
|
+
### Node HTTP
|
|
60
57
|
|
|
61
58
|
```ts
|
|
62
|
-
import {
|
|
63
|
-
import { NextResponse, type NextFetchEvent, type NextRequest } from "next/server";
|
|
59
|
+
import { createServer } from "node:http";
|
|
64
60
|
|
|
65
|
-
|
|
61
|
+
import { createDevTuneAiTrafficNodeHandler } from "@devtune/ai-traffic/node";
|
|
62
|
+
|
|
63
|
+
const trackAiTraffic = createDevTuneAiTrafficNodeHandler({
|
|
66
64
|
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
67
65
|
});
|
|
68
66
|
|
|
69
|
-
|
|
70
|
-
trackAiTraffic(request,
|
|
67
|
+
createServer((request, response) => {
|
|
68
|
+
trackAiTraffic(request, response);
|
|
71
69
|
|
|
72
|
-
|
|
73
|
-
|
|
70
|
+
response.statusCode = 200;
|
|
71
|
+
response.end("ok");
|
|
72
|
+
}).listen(3000);
|
|
74
73
|
```
|
|
75
74
|
|
|
76
|
-
The
|
|
75
|
+
The hook records the final `response.statusCode` without blocking or changing the response.
|
|
76
|
+
|
|
77
|
+
## Privacy by Design
|
|
77
78
|
|
|
78
|
-
|
|
79
|
+
The sensor filters requests where your application runs. It uses a cheap user-agent hint before consulting the AI bot registry, and only matched machine hits are eligible to be forwarded. It uses no cookies, no fingerprinting, and no full request logs.
|
|
79
80
|
|
|
80
|
-
|
|
81
|
+
For a matched crawler, DevTune receives only:
|
|
82
|
+
|
|
83
|
+
- `path`
|
|
84
|
+
- origin plus path, with query parameters removed
|
|
85
|
+
- user agent
|
|
86
|
+
- response status when the adapter can know it
|
|
87
|
+
- timestamp
|
|
81
88
|
|
|
82
|
-
|
|
89
|
+
Request bodies, cookies, IP addresses, unrelated headers, and query-derived identifiers are not sent.
|
|
90
|
+
|
|
91
|
+
## Detect Without Sending
|
|
92
|
+
|
|
93
|
+
The standalone classifiers need no ingest key and never make a network request or push data to DevTune:
|
|
83
94
|
|
|
84
95
|
```ts
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
+
import { detectAiCrawler, detectAiReferrer } from "@devtune/ai-traffic";
|
|
97
|
+
|
|
98
|
+
const crawler = detectAiCrawler(request);
|
|
99
|
+
const referrer = detectAiReferrer(request);
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
Each function also accepts the relevant string directly: a user-agent for `detectAiCrawler()` or a referrer URL for `detectAiReferrer()`.
|
|
103
|
+
|
|
104
|
+
`detectAiCrawler()` returns the matched bot registry entry or `null`:
|
|
105
|
+
|
|
106
|
+
```ts
|
|
107
|
+
type AiCrawlerDetection = {
|
|
108
|
+
uaPattern: string;
|
|
109
|
+
llmPlatform: string;
|
|
110
|
+
botClass: "training_crawler" | "index_bot" | "answer_fetcher" | "acting_agent";
|
|
111
|
+
label: string;
|
|
112
|
+
status?: "active" | "retired";
|
|
113
|
+
asnHints?: number[] | null;
|
|
114
|
+
notes?: string | null;
|
|
115
|
+
};
|
|
96
116
|
```
|
|
97
117
|
|
|
98
|
-
|
|
118
|
+
`detectAiReferrer()` returns the matched referrer registry entry or `null`:
|
|
119
|
+
|
|
120
|
+
```ts
|
|
121
|
+
type AiReferrerDetection = {
|
|
122
|
+
hostname: string;
|
|
123
|
+
llmPlatform: string;
|
|
124
|
+
label: string;
|
|
125
|
+
};
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
Both use bundled registry snapshots by default, so they work offline and in CI. Pass `registryEntries` as the second argument when you need to classify against a pinned or private registry:
|
|
129
|
+
|
|
130
|
+
```ts
|
|
131
|
+
const match = detectAiCrawler(userAgent, {
|
|
132
|
+
registryEntries: myRegistryEntries,
|
|
133
|
+
});
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
## Machine Traffic and Human Referrals
|
|
137
|
+
|
|
138
|
+
The sensor deliberately measures machine traffic only. DevTune gets human visits from AI products through its GA4 integration, where sessions, engagement, and conversions provide a richer picture than middleware referrer matching. The referrer detector is available for local classification, but the sensor adapters never forward human referral visits.
|
|
139
|
+
|
|
140
|
+
## Production Notes
|
|
141
|
+
|
|
142
|
+
### Batching and Runtime Lifetime
|
|
143
|
+
|
|
144
|
+
The default client configuration sends up to 10 matched events per unchanged ingest payload, with a 250 ms flush window for low-volume traffic. Set `batchSize: 1` and `flushIntervalMs: 0` to opt out and send each matched event immediately.
|
|
145
|
+
|
|
146
|
+
Express and Node servers are long-lived enough to benefit directly from the default. Next.js proxy and middleware pass the delayed send to `event.waitUntil()` when available, which prevents the response from waiting but may keep the middleware invocation alive for the short flush window. Use the explicit opt-out in short-lived runtimes that cannot reliably preserve delayed work, or when minimizing middleware duration matters more than request coalescing:
|
|
99
147
|
|
|
100
148
|
```ts
|
|
101
149
|
const trackAiTraffic = createDevTuneAiTrafficMiddleware({
|
|
102
150
|
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
103
|
-
|
|
151
|
+
batchSize: 1,
|
|
152
|
+
flushIntervalMs: 0,
|
|
104
153
|
});
|
|
105
154
|
```
|
|
106
155
|
|
|
107
|
-
|
|
156
|
+
The middleware helper leaves registry refresh work outside `waitUntil` by default. Ordinary browser user agents are rejected before refresh or ingest work is scheduled.
|
|
108
157
|
|
|
109
|
-
|
|
158
|
+
### Status Semantics
|
|
110
159
|
|
|
111
|
-
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
160
|
+
Express and Node adapters observe the completed response and report its actual status. Next.js proxy and middleware cannot observe the final route status after pass-through, so the convenience middleware omits status instead of guessing a 200.
|
|
161
|
+
|
|
162
|
+
When a Next.js proxy returns a response directly, use the client and pass the known status:
|
|
163
|
+
|
|
164
|
+
```ts
|
|
165
|
+
import { createDevTuneAiTraffic } from "@devtune/ai-traffic";
|
|
166
|
+
import type { NextFetchEvent, NextRequest } from "next/server";
|
|
116
167
|
|
|
117
|
-
|
|
168
|
+
const aiTraffic = createDevTuneAiTraffic({
|
|
169
|
+
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
170
|
+
defaultStatus: null,
|
|
171
|
+
});
|
|
118
172
|
|
|
119
|
-
|
|
173
|
+
export function proxy(request: NextRequest, event: NextFetchEvent) {
|
|
174
|
+
const response = new Response("Forbidden", { status: 403 });
|
|
120
175
|
|
|
121
|
-
|
|
176
|
+
aiTraffic.trackRequest(request, event, response.status);
|
|
177
|
+
|
|
178
|
+
return response;
|
|
179
|
+
}
|
|
180
|
+
```
|
|
122
181
|
|
|
123
|
-
|
|
182
|
+
Use `withDevTuneAiTrafficRoute()` where you own a Fetch-compatible route handler and want its exact response status captured automatically. `notFoundPathPatterns` and `statusResolver` remain available for applications that can provide additional status knowledge.
|
|
183
|
+
|
|
184
|
+
### Registry Refresh and Fallback
|
|
185
|
+
|
|
186
|
+
Clients start with the bundled AI bot snapshot, refresh from `https://devtune.ai/api/v1/llm-traffic/registry`, cache active entries for about an hour, and use `ETag` revalidation. Failed refreshes are guarded and sampled; they never fail the application response, and the bundled snapshot remains usable.
|
|
187
|
+
|
|
188
|
+
User-agent matching is a conservative signal. Some agents spoof ordinary browsers or require network-level signals, so reported counts are a floor rather than exact bot truth. Cloudflare-proxied sites should prefer DevTune's Cloudflare pull integration when available because it can combine request, bot-score, and network signals without running middleware.
|
|
189
|
+
|
|
190
|
+
### Forwarded Origins
|
|
191
|
+
|
|
192
|
+
Express and Node adapters build event URLs from the request protocol and host, preferring the first value in standard `X-Forwarded-Proto` and `X-Forwarded-Host` chains. Only accept those headers from a trusted proxy. If they are unavailable or not trustworthy in your deployment, pass a fixed `origin`:
|
|
193
|
+
|
|
194
|
+
```ts
|
|
195
|
+
const trackAiTraffic = createDevTuneAiTrafficNodeHandler({
|
|
196
|
+
ingestKey: process.env.DEVTUNE_AI_TRAFFIC_INGEST_KEY!,
|
|
197
|
+
origin: "https://www.example.com",
|
|
198
|
+
});
|
|
199
|
+
```
|
|
124
200
|
|
|
125
|
-
|
|
201
|
+
### Failure Behavior
|
|
126
202
|
|
|
127
|
-
|
|
203
|
+
Registry refresh and ingest sends are fire-and-forget and guarded. Network failures may drop telemetry, but they do not block, throw into, or alter the customer response. The ingest endpoint and batched wire format are unchanged.
|
|
@@ -0,0 +1,407 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* ABOUTME: Implements the DevTune AI Traffic edge-push client.
|
|
4
|
+
* Handles registry refresh, AI bot matching, batched ingest beacons, and route wrappers.
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.DevTuneAiTrafficClient = exports.DEFAULT_REGISTRY_URL = exports.DEFAULT_INGEST_ENDPOINT = void 0;
|
|
8
|
+
exports.createDevTuneAiTraffic = createDevTuneAiTraffic;
|
|
9
|
+
exports.createDevTuneAiTrafficMiddleware = createDevTuneAiTrafficMiddleware;
|
|
10
|
+
exports.mightBeAiBotUserAgent = mightBeAiBotUserAgent;
|
|
11
|
+
exports.withDevTuneAiTrafficRoute = withDevTuneAiTrafficRoute;
|
|
12
|
+
const registry_js_1 = require("./registry.js");
|
|
13
|
+
exports.DEFAULT_INGEST_ENDPOINT = "https://devtune.ai/api/v1/llm-traffic/ingest";
|
|
14
|
+
exports.DEFAULT_REGISTRY_URL = "https://devtune.ai/api/v1/llm-traffic/registry";
|
|
15
|
+
const DEFAULT_REGISTRY_REVALIDATE_MS = 60 * 60 * 1000;
|
|
16
|
+
const DEFAULT_WAIT_UNTIL_REGISTRY_REFRESH = true;
|
|
17
|
+
const DEFAULT_BATCH_SIZE = 10;
|
|
18
|
+
const DEFAULT_FLUSH_INTERVAL_MS = 250;
|
|
19
|
+
const DEFAULT_STATUS = 200;
|
|
20
|
+
const MAX_INGEST_BATCH_SIZE = 100;
|
|
21
|
+
const MAX_EVENT_PATH_LENGTH = 2048;
|
|
22
|
+
const MAX_EVENT_UA_LENGTH = 2048;
|
|
23
|
+
const MAX_EVENT_URL_LENGTH = 4096;
|
|
24
|
+
const AI_BOT_USER_AGENT_HINTS = [
|
|
25
|
+
"-ai",
|
|
26
|
+
"ai-",
|
|
27
|
+
"agent",
|
|
28
|
+
"anthropic",
|
|
29
|
+
"bot",
|
|
30
|
+
"bytespider",
|
|
31
|
+
"claude",
|
|
32
|
+
"cohere",
|
|
33
|
+
"crawler",
|
|
34
|
+
"fetch",
|
|
35
|
+
"google-",
|
|
36
|
+
"gpt",
|
|
37
|
+
"meta-external",
|
|
38
|
+
"mistral",
|
|
39
|
+
"notebooklm",
|
|
40
|
+
"oai-",
|
|
41
|
+
"openai",
|
|
42
|
+
"perplexity",
|
|
43
|
+
"quora",
|
|
44
|
+
"spider",
|
|
45
|
+
"xai",
|
|
46
|
+
];
|
|
47
|
+
class DevTuneAiTrafficClient {
|
|
48
|
+
options;
|
|
49
|
+
endpoint;
|
|
50
|
+
registryUrl;
|
|
51
|
+
registryRevalidateMs;
|
|
52
|
+
waitUntilRegistryRefresh;
|
|
53
|
+
batchSize;
|
|
54
|
+
flushIntervalMs;
|
|
55
|
+
defaultStatus;
|
|
56
|
+
fetchImpl;
|
|
57
|
+
logger;
|
|
58
|
+
now;
|
|
59
|
+
random;
|
|
60
|
+
errorSampleRate;
|
|
61
|
+
registry;
|
|
62
|
+
refreshPromise = null;
|
|
63
|
+
flushPromise = null;
|
|
64
|
+
delayedFlushPromise = null;
|
|
65
|
+
queue = [];
|
|
66
|
+
constructor(options) {
|
|
67
|
+
this.options = options;
|
|
68
|
+
if (!options.ingestKey) {
|
|
69
|
+
throw new Error("DevTune AI Traffic ingestKey is required");
|
|
70
|
+
}
|
|
71
|
+
this.endpoint = options.endpoint ?? exports.DEFAULT_INGEST_ENDPOINT;
|
|
72
|
+
this.registryUrl = options.registryUrl ?? exports.DEFAULT_REGISTRY_URL;
|
|
73
|
+
this.registryRevalidateMs = options.registryRevalidateMs ?? DEFAULT_REGISTRY_REVALIDATE_MS;
|
|
74
|
+
this.waitUntilRegistryRefresh =
|
|
75
|
+
options.waitUntilRegistryRefresh ?? DEFAULT_WAIT_UNTIL_REGISTRY_REFRESH;
|
|
76
|
+
this.batchSize = clampInteger(options.batchSize ?? DEFAULT_BATCH_SIZE, 1, MAX_INGEST_BATCH_SIZE);
|
|
77
|
+
this.flushIntervalMs = clampInteger(options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, 0, 60_000);
|
|
78
|
+
this.defaultStatus =
|
|
79
|
+
options.defaultStatus === null
|
|
80
|
+
? null
|
|
81
|
+
: normalizeStatus(options.defaultStatus ?? DEFAULT_STATUS, DEFAULT_STATUS);
|
|
82
|
+
this.fetchImpl = options.fetch ?? fetch;
|
|
83
|
+
this.logger = options.logger ?? null;
|
|
84
|
+
this.now = options.now ?? (() => new Date());
|
|
85
|
+
this.random = options.random ?? Math.random;
|
|
86
|
+
this.errorSampleRate = clampNumber(options.errorSampleRate ?? 0.01, 0, 1);
|
|
87
|
+
const fallbackEntries = (0, registry_js_1.compileAiBotRegistryEntries)(registry_js_1.FALLBACK_AI_BOT_REGISTRY_ENTRIES);
|
|
88
|
+
const fallbackPrefilter = createRegistryPrefilter(fallbackEntries);
|
|
89
|
+
this.registry = {
|
|
90
|
+
entries: fallbackEntries,
|
|
91
|
+
etag: null,
|
|
92
|
+
fetchedAt: 0,
|
|
93
|
+
...fallbackPrefilter,
|
|
94
|
+
version: "bundled",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
mightTrackRequest(request) {
|
|
98
|
+
const userAgent = request.headers.get("user-agent");
|
|
99
|
+
if (!userAgent) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
if (mightBeAiBotUserAgent(userAgent) || this.registry.requiresFullPrefilterScan) {
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
const normalized = userAgent.toLowerCase();
|
|
106
|
+
return this.registry.userAgentHints.some((hint) => normalized.includes(hint));
|
|
107
|
+
}
|
|
108
|
+
trackRequest(request, waitUntilContext, responseStatus) {
|
|
109
|
+
if (!this.mightTrackRequest(request)) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
const userAgent = request.headers.get("user-agent");
|
|
113
|
+
if (!userAgent) {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
const entry = (0, registry_js_1.matchAiBotRegistryEntry)(userAgent, this.registry.entries);
|
|
117
|
+
if (!entry) {
|
|
118
|
+
if (mightBeAiBotUserAgent(userAgent)) {
|
|
119
|
+
this.refreshRegistryIfStale(waitUntilContext);
|
|
120
|
+
}
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
this.refreshRegistryIfStale(waitUntilContext);
|
|
124
|
+
const event = this.createEvent(request, userAgent, responseStatus);
|
|
125
|
+
if (!event) {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
this.enqueue(event, waitUntilContext);
|
|
129
|
+
return {
|
|
130
|
+
entry,
|
|
131
|
+
event,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
async refreshRegistry() {
|
|
135
|
+
const headers = {
|
|
136
|
+
accept: "application/json",
|
|
137
|
+
};
|
|
138
|
+
if (this.registry.etag) {
|
|
139
|
+
headers["if-none-match"] = this.registry.etag;
|
|
140
|
+
}
|
|
141
|
+
const response = await this.fetchImpl(this.registryUrl, {
|
|
142
|
+
headers,
|
|
143
|
+
method: "GET",
|
|
144
|
+
});
|
|
145
|
+
if (response.status === 304) {
|
|
146
|
+
this.registry = {
|
|
147
|
+
...this.registry,
|
|
148
|
+
fetchedAt: Date.now(),
|
|
149
|
+
};
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
if (!response.ok) {
|
|
153
|
+
throw new Error(`Registry request failed with status ${response.status}`);
|
|
154
|
+
}
|
|
155
|
+
const payload = (0, registry_js_1.parseAiBotRegistryPayload)(await response.json());
|
|
156
|
+
if (!payload) {
|
|
157
|
+
throw new Error("Registry response was not a valid DevTune registry");
|
|
158
|
+
}
|
|
159
|
+
const entries = (0, registry_js_1.compileAiBotRegistryEntries)(payload.entries);
|
|
160
|
+
this.registry = {
|
|
161
|
+
entries,
|
|
162
|
+
etag: response.headers.get("etag") ?? `"${payload.version}"`,
|
|
163
|
+
fetchedAt: Date.now(),
|
|
164
|
+
...createRegistryPrefilter(entries),
|
|
165
|
+
version: payload.version,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
async flush() {
|
|
169
|
+
if (this.flushPromise) {
|
|
170
|
+
return this.flushPromise;
|
|
171
|
+
}
|
|
172
|
+
this.flushPromise = this.flushQueuedEvents().finally(() => {
|
|
173
|
+
this.flushPromise = null;
|
|
174
|
+
});
|
|
175
|
+
return this.flushPromise;
|
|
176
|
+
}
|
|
177
|
+
refreshRegistryIfStale(waitUntilContext) {
|
|
178
|
+
if (this.refreshPromise) {
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (Date.now() - this.registry.fetchedAt < this.registryRevalidateMs) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
this.refreshPromise = this.refreshRegistry()
|
|
185
|
+
.catch((error) => {
|
|
186
|
+
this.registry = {
|
|
187
|
+
...this.registry,
|
|
188
|
+
fetchedAt: Date.now(),
|
|
189
|
+
};
|
|
190
|
+
this.sampleWarn("Failed to refresh DevTune AI bot registry", { error });
|
|
191
|
+
})
|
|
192
|
+
.finally(() => {
|
|
193
|
+
this.refreshPromise = null;
|
|
194
|
+
});
|
|
195
|
+
scheduleBackground(this.refreshPromise, this.waitUntilRegistryRefresh ? waitUntilContext : undefined);
|
|
196
|
+
}
|
|
197
|
+
createEvent(request, userAgent, responseStatus) {
|
|
198
|
+
const url = parseRequestUrl(request.url);
|
|
199
|
+
if (!url) {
|
|
200
|
+
return null;
|
|
201
|
+
}
|
|
202
|
+
const pathname = truncateString(url.pathname || "/", MAX_EVENT_PATH_LENGTH);
|
|
203
|
+
const status = this.resolveStatus(request, pathname, responseStatus);
|
|
204
|
+
return {
|
|
205
|
+
path: pathname,
|
|
206
|
+
url: truncateString(`${url.origin}${pathname}`, MAX_EVENT_URL_LENGTH),
|
|
207
|
+
ua: truncateString(userAgent, MAX_EVENT_UA_LENGTH),
|
|
208
|
+
...(status === null ? {} : { status }),
|
|
209
|
+
ts: this.now().toISOString(),
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
resolveStatus(request, pathname, responseStatus) {
|
|
213
|
+
if (responseStatus === null) {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
const explicitStatus = normalizeStatus(responseStatus, null);
|
|
217
|
+
if (explicitStatus !== null) {
|
|
218
|
+
return explicitStatus;
|
|
219
|
+
}
|
|
220
|
+
const resolverStatus = this.options.statusResolver?.(request);
|
|
221
|
+
if (resolverStatus === null) {
|
|
222
|
+
return null;
|
|
223
|
+
}
|
|
224
|
+
const resolvedStatus = normalizeStatus(resolverStatus, null);
|
|
225
|
+
if (resolvedStatus !== null) {
|
|
226
|
+
return resolvedStatus;
|
|
227
|
+
}
|
|
228
|
+
if (this.matchesNotFoundPath(pathname)) {
|
|
229
|
+
return 404;
|
|
230
|
+
}
|
|
231
|
+
return this.defaultStatus;
|
|
232
|
+
}
|
|
233
|
+
matchesNotFoundPath(pathname) {
|
|
234
|
+
for (const pattern of this.options.notFoundPathPatterns ?? []) {
|
|
235
|
+
if (typeof pattern === "string") {
|
|
236
|
+
if (pattern === pathname) {
|
|
237
|
+
return true;
|
|
238
|
+
}
|
|
239
|
+
continue;
|
|
240
|
+
}
|
|
241
|
+
pattern.lastIndex = 0;
|
|
242
|
+
if (pattern.test(pathname)) {
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
enqueue(event, waitUntilContext) {
|
|
249
|
+
this.queue.push(event);
|
|
250
|
+
if (this.queue.length >= this.batchSize) {
|
|
251
|
+
scheduleBackground(this.flush(), waitUntilContext);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
if (this.delayedFlushPromise) {
|
|
255
|
+
scheduleBackground(this.delayedFlushPromise, waitUntilContext);
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
258
|
+
this.delayedFlushPromise = delay(this.flushIntervalMs)
|
|
259
|
+
.then(() => this.flush())
|
|
260
|
+
.finally(() => {
|
|
261
|
+
this.delayedFlushPromise = null;
|
|
262
|
+
});
|
|
263
|
+
scheduleBackground(this.delayedFlushPromise, waitUntilContext);
|
|
264
|
+
}
|
|
265
|
+
async flushQueuedEvents() {
|
|
266
|
+
while (this.queue.length > 0) {
|
|
267
|
+
const batch = this.queue.splice(0, MAX_INGEST_BATCH_SIZE);
|
|
268
|
+
try {
|
|
269
|
+
const response = await this.fetchImpl(this.endpoint, {
|
|
270
|
+
body: JSON.stringify({ events: batch }),
|
|
271
|
+
headers: {
|
|
272
|
+
authorization: `Bearer ${this.options.ingestKey}`,
|
|
273
|
+
"content-type": "application/json",
|
|
274
|
+
},
|
|
275
|
+
keepalive: true,
|
|
276
|
+
method: "POST",
|
|
277
|
+
});
|
|
278
|
+
if (!response.ok) {
|
|
279
|
+
this.sampleWarn("DevTune AI Traffic ingest request failed", {
|
|
280
|
+
status: response.status,
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
this.sampleWarn("DevTune AI Traffic ingest request failed", { error });
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
sampleWarn(message, context) {
|
|
290
|
+
if (this.errorSampleRate === 0 || this.random() > this.errorSampleRate) {
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
this.logger?.warn(message, context);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
exports.DevTuneAiTrafficClient = DevTuneAiTrafficClient;
|
|
297
|
+
function createDevTuneAiTraffic(options) {
|
|
298
|
+
return new DevTuneAiTrafficClient(options);
|
|
299
|
+
}
|
|
300
|
+
function createDevTuneAiTrafficMiddleware(options) {
|
|
301
|
+
const client = createDevTuneAiTraffic({
|
|
302
|
+
...options,
|
|
303
|
+
defaultStatus: options.defaultStatus ?? null,
|
|
304
|
+
waitUntilRegistryRefresh: options.waitUntilRegistryRefresh ?? false,
|
|
305
|
+
});
|
|
306
|
+
return function devTuneAiTrafficMiddleware(request, event) {
|
|
307
|
+
client.trackRequest(request, event);
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
function mightBeAiBotUserAgent(userAgent) {
|
|
311
|
+
if (!userAgent) {
|
|
312
|
+
return false;
|
|
313
|
+
}
|
|
314
|
+
const normalized = userAgent.toLowerCase();
|
|
315
|
+
return AI_BOT_USER_AGENT_HINTS.some((hint) => normalized.includes(hint));
|
|
316
|
+
}
|
|
317
|
+
function withDevTuneAiTrafficRoute(handler, options) {
|
|
318
|
+
const client = createDevTuneAiTraffic(options);
|
|
319
|
+
return async function devTuneAiTrafficRoute(request, context) {
|
|
320
|
+
const response = await handler(request, context);
|
|
321
|
+
const waitUntilContext = getWaitUntilContext(context);
|
|
322
|
+
client.trackRequest(request, waitUntilContext, response.status);
|
|
323
|
+
return response;
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
function scheduleBackground(promise, waitUntilContext) {
|
|
327
|
+
const guardedPromise = promise.catch(() => undefined);
|
|
328
|
+
if (waitUntilContext) {
|
|
329
|
+
waitUntilContext.waitUntil(guardedPromise);
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
void guardedPromise;
|
|
333
|
+
}
|
|
334
|
+
function getWaitUntilContext(value) {
|
|
335
|
+
if (!isRecord(value)) {
|
|
336
|
+
return undefined;
|
|
337
|
+
}
|
|
338
|
+
const waitUntil = value.waitUntil;
|
|
339
|
+
if (typeof waitUntil !== "function") {
|
|
340
|
+
return undefined;
|
|
341
|
+
}
|
|
342
|
+
return {
|
|
343
|
+
waitUntil: (promise) => {
|
|
344
|
+
waitUntil.call(value, promise);
|
|
345
|
+
},
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
function parseRequestUrl(url) {
|
|
349
|
+
try {
|
|
350
|
+
return new URL(url);
|
|
351
|
+
}
|
|
352
|
+
catch {
|
|
353
|
+
return null;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
function delay(ms) {
|
|
357
|
+
if (ms <= 0) {
|
|
358
|
+
return Promise.resolve();
|
|
359
|
+
}
|
|
360
|
+
return new Promise((resolve) => {
|
|
361
|
+
setTimeout(resolve, ms);
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
function createRegistryPrefilter(entries) {
|
|
365
|
+
const userAgentHints = new Set();
|
|
366
|
+
let requiresFullPrefilterScan = false;
|
|
367
|
+
for (const { entry } of entries) {
|
|
368
|
+
const hint = entry.uaPattern.toLowerCase();
|
|
369
|
+
if (hint.length < 3 || /[.^$|]/u.test(hint)) {
|
|
370
|
+
requiresFullPrefilterScan = true;
|
|
371
|
+
continue;
|
|
372
|
+
}
|
|
373
|
+
userAgentHints.add(hint);
|
|
374
|
+
}
|
|
375
|
+
return {
|
|
376
|
+
requiresFullPrefilterScan,
|
|
377
|
+
userAgentHints: [...userAgentHints],
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
function normalizeStatus(value, fallback) {
|
|
381
|
+
if (typeof value === "number" && Number.isInteger(value) && value >= 100 && value <= 599) {
|
|
382
|
+
return value;
|
|
383
|
+
}
|
|
384
|
+
return fallback;
|
|
385
|
+
}
|
|
386
|
+
function clampInteger(value, min, max) {
|
|
387
|
+
if (!Number.isFinite(value)) {
|
|
388
|
+
return min;
|
|
389
|
+
}
|
|
390
|
+
return Math.min(Math.max(Math.trunc(value), min), max);
|
|
391
|
+
}
|
|
392
|
+
function clampNumber(value, min, max) {
|
|
393
|
+
if (!Number.isFinite(value)) {
|
|
394
|
+
return min;
|
|
395
|
+
}
|
|
396
|
+
return Math.min(Math.max(value, min), max);
|
|
397
|
+
}
|
|
398
|
+
function truncateString(value, maxLength) {
|
|
399
|
+
if (value.length <= maxLength) {
|
|
400
|
+
return value;
|
|
401
|
+
}
|
|
402
|
+
return value.slice(0, maxLength);
|
|
403
|
+
}
|
|
404
|
+
function isRecord(value) {
|
|
405
|
+
return typeof value === "object" && value !== null;
|
|
406
|
+
}
|
|
407
|
+
//# sourceMappingURL=client.js.map
|