@scoutline-ai/middleware 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +161 -2
- package/dist/browser-script.d.ts +7 -0
- package/dist/browser-script.js +63 -0
- package/dist/browser.d.ts +33 -0
- package/dist/browser.js +101 -0
- package/dist/classify.d.ts +11 -0
- package/dist/classify.js +48 -0
- package/dist/client-ip.d.ts +31 -0
- package/dist/client-ip.js +83 -0
- package/dist/config.d.ts +2 -0
- package/dist/config.js +4 -0
- package/dist/fetch.d.ts +13 -0
- package/dist/fetch.js +77 -0
- package/dist/index.d.ts +86 -0
- package/dist/index.js +324 -0
- package/dist/next.d.ts +20 -0
- package/dist/next.js +58 -0
- package/dist/node.d.ts +26 -0
- package/dist/node.js +129 -0
- package/dist/outcomes.d.ts +56 -0
- package/dist/outcomes.js +144 -0
- package/package.json +43 -4
package/README.md
CHANGED
|
@@ -1,3 +1,162 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @scoutline-ai/middleware
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Records how agents use your site. It fails open: if Scoutline is down, misconfigured or slow, your site behaves exactly the same. Sending happens after your response, so it adds no latency.
|
|
4
|
+
|
|
5
|
+
Set your key as an environment variable:
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
SCOUTLINE_KEY=sl_live_...
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## Next.js
|
|
12
|
+
|
|
13
|
+
Wrap your proxy in `proxy.ts` (`middleware.ts` before Next.js 16). Your existing handler keeps working.
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { withScoutline } from "@scoutline-ai/middleware/next";
|
|
17
|
+
|
|
18
|
+
export default withScoutline(yourProxy); // or withScoutline() if you have none
|
|
19
|
+
|
|
20
|
+
export const config = {
|
|
21
|
+
matcher: [
|
|
22
|
+
{
|
|
23
|
+
source: "/((?!_next/static|_next/image|favicon.ico).*)",
|
|
24
|
+
missing: [
|
|
25
|
+
{ type: "header", key: "next-router-prefetch" },
|
|
26
|
+
{ type: "header", key: "purpose", value: "prefetch" },
|
|
27
|
+
],
|
|
28
|
+
},
|
|
29
|
+
],
|
|
30
|
+
};
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Next.js removes prefetch headers before your proxy runs, so only the matcher can exclude them. The `missing` rule skips your whole proxy for prefetches, so leave it out if your proxy enforces auth. Prefetches then count as `unknown` traffic. They only come from browsers, so crawler and agent counts are unaffected.
|
|
34
|
+
|
|
35
|
+
Your local and preview deployments can use the same key. Reports only count traffic on your site's domain and its subdomains.
|
|
36
|
+
|
|
37
|
+
A proxy runs before the page renders, so normal page loads have no status or duration. Scoutline leaves those empty and only records a status when the proxy itself responds, such as a redirect.
|
|
38
|
+
|
|
39
|
+
## Cloudflare Workers
|
|
40
|
+
|
|
41
|
+
Pass the key in. Depending on its compatibility date and flags, a Worker may have no `process.env` to read it from.
|
|
42
|
+
|
|
43
|
+
```ts
|
|
44
|
+
import { env } from "cloudflare:workers";
|
|
45
|
+
import { withScoutline } from "@scoutline-ai/middleware/fetch";
|
|
46
|
+
|
|
47
|
+
export default { fetch: withScoutline(app.fetch, { key: env.SCOUTLINE_KEY }) };
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Use `.dev.vars` locally and `npx wrangler secret put SCOUTLINE_KEY` in production. This sees the full response, so it records status, duration and size.
|
|
51
|
+
|
|
52
|
+
## Hono and other fetch handlers
|
|
53
|
+
|
|
54
|
+
```ts
|
|
55
|
+
import { withScoutline } from "@scoutline-ai/middleware/fetch";
|
|
56
|
+
|
|
57
|
+
export default { fetch: withScoutline(app.fetch) };
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
This also records status, duration and size. If Hono runs on Workers, pass the key as shown above.
|
|
61
|
+
|
|
62
|
+
## Express and Fastify
|
|
63
|
+
|
|
64
|
+
```ts
|
|
65
|
+
import { scoutlineExpress, scoutlineFastify } from "@scoutline-ai/middleware/node";
|
|
66
|
+
|
|
67
|
+
app.use(scoutlineExpress()); // Express: before your routes and body parsers
|
|
68
|
+
|
|
69
|
+
app.register(scoutlineFastify); // Fastify
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Both see the full response, so they record status, duration and size. Events send every few seconds.
|
|
73
|
+
|
|
74
|
+
## Browser events
|
|
75
|
+
|
|
76
|
+
The server sees every request, including crawlers that never run JavaScript. The browser script adds what only a page can see: real status and load time, in-page navigation, clicks and form submits, and errors.
|
|
77
|
+
|
|
78
|
+
**Cloudflare Workers:** nothing to do. HTML pages get the script automatically.
|
|
79
|
+
|
|
80
|
+
**Next.js, Hono, Express and Fastify:** add one tag to your root layout or HTML template.
|
|
81
|
+
|
|
82
|
+
```html
|
|
83
|
+
<script src="/_scoutline/s" defer></script>
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The script and its events are served from your own domain by this package, so there is no public key and no third-party request. If your proxy `matcher` is limited to certain paths, include `/_scoutline/:path*`. These routes run before your own proxy, so an auth check won't block them.
|
|
87
|
+
|
|
88
|
+
It stores nothing and sets no cookies. A page links to its server request through a `Server-Timing` header. Each batch of events is one request to your server, sent when the page closes or after 20 events.
|
|
89
|
+
|
|
90
|
+
Browser events can be forged, so they are never `verified` and can't confirm an outcome. Reports keep them out of request counts.
|
|
91
|
+
|
|
92
|
+
## Outcomes
|
|
93
|
+
|
|
94
|
+
Outcomes are what your server confirms really happened, such as an order or a signup. They are never inferred from a 200.
|
|
95
|
+
|
|
96
|
+
Create an outcome key on your Install page. It is separate from your middleware key.
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
SCOUTLINE_OUTCOME_KEY=sl_live_...
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
```ts
|
|
103
|
+
import { createOutcomes } from "@scoutline-ai/middleware/outcomes";
|
|
104
|
+
|
|
105
|
+
const outcomes = createOutcomes();
|
|
106
|
+
|
|
107
|
+
// Runs your action, and confirms the outcome only if it succeeds.
|
|
108
|
+
const order = await outcomes.wrap("order.placed", () => placeOrder(cart), (order) => ({
|
|
109
|
+
id: order.id,
|
|
110
|
+
taskId,
|
|
111
|
+
headers: request.headers,
|
|
112
|
+
}));
|
|
113
|
+
|
|
114
|
+
// Or confirm it yourself.
|
|
115
|
+
await outcomes.confirm("signup.completed", { id: user.id, taskId });
|
|
116
|
+
|
|
117
|
+
// Record a step on the way, or where it failed.
|
|
118
|
+
await outcomes.record("checkout_attempted", { taskId, errorCategory: "card_declined" });
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
- **`id`** makes repeats safe: the same id counts once. It is hashed before sending. Without it, every call counts.
|
|
122
|
+
- **`taskId`** is your id for one piece of work, and puts the event on that Workflow timeline.
|
|
123
|
+
- **`headers`** are the incoming request's headers, which record who acted and the trace id.
|
|
124
|
+
|
|
125
|
+
Nothing here throws, so a Scoutline problem can't fail your action. Each call returns `{ ok: true }` or `{ ok: false, reason }`. Events are sent straight away, at most twice for up to 2 seconds each, so don't await them if your action can't spare that.
|
|
126
|
+
|
|
127
|
+
## Linking requests to a task
|
|
128
|
+
|
|
129
|
+
Add your own ids to requests with `context`. Only what you return here is sent.
|
|
130
|
+
|
|
131
|
+
```ts
|
|
132
|
+
withScoutline(yourProxy, {
|
|
133
|
+
context: (request) => ({
|
|
134
|
+
taskId: request.headers.get("x-your-task-id"),
|
|
135
|
+
resourceId: productIdFrom(request),
|
|
136
|
+
contentVersion: process.env.VERCEL_GIT_COMMIT_SHA,
|
|
137
|
+
}),
|
|
138
|
+
});
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
Requests that share a W3C `traceparent` with an event in the task also appear on its timeline, labelled as linked by trace.
|
|
142
|
+
|
|
143
|
+
## What is sent
|
|
144
|
+
|
|
145
|
+
From the server: the method, pathname, user agent, whether a few standard headers were present, and request and trace ids. Images, video, fonts and similar files are recorded for crawlers and agents, and skipped for everyone else, so one page load counts once. From the browser: the pathname, event type, element type such as `button`, status, and load time.
|
|
146
|
+
|
|
147
|
+
Query strings, cookies, auth headers, bodies, page text and field values are never sent.
|
|
148
|
+
|
|
149
|
+
**The visitor's IP address** is sent to the collector, which uses it to find the country and network, then discards it. It is never stored. Country is where the network is registered, so cloud traffic may not show where the machine really is. Turn this off with `sendClientIp: false`, and country and network stay empty.
|
|
150
|
+
|
|
151
|
+
The collector classifies each request, so new crawlers are recognised without updating this package. A matching user agent is recorded as `declared`, because any client can send one. Other automation is `inferred`, and everything else is `unknown`. A browser claim that doesn't add up is only `inferred` when two different kinds of evidence agree, such as missing browser headers on a hosting network. This package never reports `verified`.
|
|
152
|
+
|
|
153
|
+
## Options
|
|
154
|
+
|
|
155
|
+
| Option | Default | Purpose |
|
|
156
|
+
| --- | --- | --- |
|
|
157
|
+
| `key` | `SCOUTLINE_KEY` | Ingest key |
|
|
158
|
+
| `endpoint` | `SCOUTLINE_ENDPOINT`, then `https://ingest.scoutline.ai` | Collector URL |
|
|
159
|
+
| `sampleUnknown` | `1` | Share of unclassified traffic to send, from 0 to 1 |
|
|
160
|
+
| `ignore` | none | `(pathname) => boolean` to skip a path |
|
|
161
|
+
| `sendClientIp` | `true` | Send the visitor's IP for the country and network lookup. It is never stored |
|
|
162
|
+
| `context` | none | `(request) => ({ taskId, sessionId, resourceId, contentVersion })` adds your own ids |
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Served from the customer's own domain at /_scoutline/s. It stays plain ES5 and
|
|
3
|
+
* dependency free so it can load on any page. It collects no text, field values
|
|
4
|
+
* or query strings, and stores nothing, so no cookie or consent is involved.
|
|
5
|
+
* Everything it sends is an unverified claim and is revalidated in `browser.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const BROWSER_SCRIPT = "(function(){\n var ENDPOINT=\"/_scoutline/e\",MAX=20,queue=[],path=location.pathname;\n var nav=performance.getEntriesByType?performance.getEntriesByType(\"navigation\")[0]:null;\n var automated=navigator.webdriver===true;\n function serverId(){\n var timing=nav&&nav.serverTiming;\n for(var i=0;timing&&i<timing.length;i++){\n if(timing[i].name===\"scoutline\")return timing[i].description;\n }\n }\n function send(){\n if(!queue.length)return;\n var now=Date.now(),events=[];\n for(var i=0;i<queue.length;i++){\n var e=queue[i];e.ageMs=now-e.at;delete e.at;events.push(e);\n }\n queue=[];\n var body=JSON.stringify({automated:automated,events:events});\n if(!navigator.sendBeacon||!navigator.sendBeacon(ENDPOINT,body)){\n fetch(ENDPOINT,{method:\"POST\",body:body,keepalive:true})[\"catch\"](function(){});\n }\n }\n function add(type,extra){\n var e=extra||{};e.type=type;e.path=location.pathname;e.at=Date.now();\n queue.push(e);\n if(queue.length>=MAX)send();\n }\n function firstView(){\n add(\"page_view\",{\n parent:serverId(),\n status:nav&&nav.responseStatus?nav.responseStatus:undefined,\n durationMs:nav&&nav.duration?Math.round(nav.duration):undefined\n });\n }\n function changed(){\n if(location.pathname===path)return;\n path=location.pathname;add(\"page_view\");\n }\n [\"pushState\",\"replaceState\"].forEach(function(name){\n var original=history[name];\n history[name]=function(){var r=original.apply(this,arguments);changed();return r;};\n });\n addEventListener(\"popstate\",changed);\n document.addEventListener(\"click\",function(event){\n var t=event.target&&event.target.closest&&event.target.closest(\"a,button,[role=button],input[type=submit]\");\n if(t)add(\"click\",{element:t.tagName.toLowerCase()});\n },true);\n document.addEventListener(\"submit\",function(){add(\"form_submit\");},true);\n addEventListener(\"error\",function(event){if(event.message)add(\"error\");});\n addEventListener(\"unhandledrejection\",function(){add(\"error\");});\n addEventListener(\"pagehide\",send);\n document.addEventListener(\"visibilitychange\",function(){\n if(document.visibilityState===\"hidden\")send();\n });\n if(document.readyState===\"complete\")setTimeout(firstView);\n else addEventListener(\"load\",function(){setTimeout(firstView);});\n})();";
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Served from the customer's own domain at /_scoutline/s. It stays plain ES5 and
|
|
3
|
+
* dependency free so it can load on any page. It collects no text, field values
|
|
4
|
+
* or query strings, and stores nothing, so no cookie or consent is involved.
|
|
5
|
+
* Everything it sends is an unverified claim and is revalidated in `browser.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export const BROWSER_SCRIPT = `(function(){
|
|
8
|
+
var ENDPOINT="/_scoutline/e",MAX=20,queue=[],path=location.pathname;
|
|
9
|
+
var nav=performance.getEntriesByType?performance.getEntriesByType("navigation")[0]:null;
|
|
10
|
+
var automated=navigator.webdriver===true;
|
|
11
|
+
function serverId(){
|
|
12
|
+
var timing=nav&&nav.serverTiming;
|
|
13
|
+
for(var i=0;timing&&i<timing.length;i++){
|
|
14
|
+
if(timing[i].name==="scoutline")return timing[i].description;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function send(){
|
|
18
|
+
if(!queue.length)return;
|
|
19
|
+
var now=Date.now(),events=[];
|
|
20
|
+
for(var i=0;i<queue.length;i++){
|
|
21
|
+
var e=queue[i];e.ageMs=now-e.at;delete e.at;events.push(e);
|
|
22
|
+
}
|
|
23
|
+
queue=[];
|
|
24
|
+
var body=JSON.stringify({automated:automated,events:events});
|
|
25
|
+
if(!navigator.sendBeacon||!navigator.sendBeacon(ENDPOINT,body)){
|
|
26
|
+
fetch(ENDPOINT,{method:"POST",body:body,keepalive:true})["catch"](function(){});
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function add(type,extra){
|
|
30
|
+
var e=extra||{};e.type=type;e.path=location.pathname;e.at=Date.now();
|
|
31
|
+
queue.push(e);
|
|
32
|
+
if(queue.length>=MAX)send();
|
|
33
|
+
}
|
|
34
|
+
function firstView(){
|
|
35
|
+
add("page_view",{
|
|
36
|
+
parent:serverId(),
|
|
37
|
+
status:nav&&nav.responseStatus?nav.responseStatus:undefined,
|
|
38
|
+
durationMs:nav&&nav.duration?Math.round(nav.duration):undefined
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
function changed(){
|
|
42
|
+
if(location.pathname===path)return;
|
|
43
|
+
path=location.pathname;add("page_view");
|
|
44
|
+
}
|
|
45
|
+
["pushState","replaceState"].forEach(function(name){
|
|
46
|
+
var original=history[name];
|
|
47
|
+
history[name]=function(){var r=original.apply(this,arguments);changed();return r;};
|
|
48
|
+
});
|
|
49
|
+
addEventListener("popstate",changed);
|
|
50
|
+
document.addEventListener("click",function(event){
|
|
51
|
+
var t=event.target&&event.target.closest&&event.target.closest("a,button,[role=button],input[type=submit]");
|
|
52
|
+
if(t)add("click",{element:t.tagName.toLowerCase()});
|
|
53
|
+
},true);
|
|
54
|
+
document.addEventListener("submit",function(){add("form_submit");},true);
|
|
55
|
+
addEventListener("error",function(event){if(event.message)add("error");});
|
|
56
|
+
addEventListener("unhandledrejection",function(){add("error");});
|
|
57
|
+
addEventListener("pagehide",send);
|
|
58
|
+
document.addEventListener("visibilitychange",function(){
|
|
59
|
+
if(document.visibilityState==="hidden")send();
|
|
60
|
+
});
|
|
61
|
+
if(document.readyState==="complete")setTimeout(firstView);
|
|
62
|
+
else addEventListener("load",function(){setTimeout(firstView);});
|
|
63
|
+
})();`;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Actor } from "./classify.js";
|
|
2
|
+
export declare const SCRIPT_PATH = "/_scoutline/s";
|
|
3
|
+
export declare const EVENTS_PATH = "/_scoutline/e";
|
|
4
|
+
export declare const BROWSER_CLASSIFIER_VERSION = "browser-0.1";
|
|
5
|
+
/**
|
|
6
|
+
* Anything in the page can be forged, so only a same-origin browser request is
|
|
7
|
+
* accepted. This keeps out other sites, not a determined client, which is why
|
|
8
|
+
* these events stay low trust however they arrive.
|
|
9
|
+
*/
|
|
10
|
+
export declare function isSameOrigin(headers: Headers): boolean;
|
|
11
|
+
/**
|
|
12
|
+
* Rebuilds each event from an allowlist, so unexpected fields are dropped. The
|
|
13
|
+
* `browser:` action prefix keeps these out of request counts, because a page
|
|
14
|
+
* load also produces a server event, and Googlebot runs this script too.
|
|
15
|
+
*/
|
|
16
|
+
export declare function toBrowserEvent(raw: unknown, actor: Actor, now: number, sampleRate: number): {
|
|
17
|
+
eventId: `${string}-${string}-${string}-${string}-${string}`;
|
|
18
|
+
occurredAt: number;
|
|
19
|
+
family: string;
|
|
20
|
+
actor: Actor;
|
|
21
|
+
action: string;
|
|
22
|
+
url: string;
|
|
23
|
+
interface: string;
|
|
24
|
+
parentActionId: string | undefined;
|
|
25
|
+
status: number | undefined;
|
|
26
|
+
durationMs: number | undefined;
|
|
27
|
+
sampleRate: number;
|
|
28
|
+
classifierVersion: string;
|
|
29
|
+
} | null;
|
|
30
|
+
export declare function parseBrowserBody(text: string): {
|
|
31
|
+
automated: boolean;
|
|
32
|
+
events: unknown[];
|
|
33
|
+
} | null;
|
package/dist/browser.js
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
export const SCRIPT_PATH = "/_scoutline/s";
|
|
2
|
+
export const EVENTS_PATH = "/_scoutline/e";
|
|
3
|
+
export const BROWSER_CLASSIFIER_VERSION = "browser-0.1";
|
|
4
|
+
const MAX_BODY_CHARS = 16 * 1024;
|
|
5
|
+
const MAX_EVENTS = 20;
|
|
6
|
+
const MAX_AGE_MS = 10 * 60 * 1000;
|
|
7
|
+
const MAX_PATH_CHARS = 2048;
|
|
8
|
+
const MAX_DURATION_MS = 60 * 60 * 1000;
|
|
9
|
+
const TYPES = new Set(["page_view", "click", "form_submit", "error"]);
|
|
10
|
+
const ELEMENTS = new Set(["a", "button", "input", "div", "span"]);
|
|
11
|
+
const SERVER_ID = /^[0-9a-f-]{36}$/;
|
|
12
|
+
/**
|
|
13
|
+
* Anything in the page can be forged, so only a same-origin browser request is
|
|
14
|
+
* accepted. This keeps out other sites, not a determined client, which is why
|
|
15
|
+
* these events stay low trust however they arrive.
|
|
16
|
+
*/
|
|
17
|
+
export function isSameOrigin(headers) {
|
|
18
|
+
const site = headers.get("sec-fetch-site");
|
|
19
|
+
if (site && site !== "same-origin")
|
|
20
|
+
return false;
|
|
21
|
+
const origin = headers.get("origin");
|
|
22
|
+
if (!origin)
|
|
23
|
+
return site === "same-origin";
|
|
24
|
+
const host = headers.get("x-forwarded-host") ?? headers.get("host");
|
|
25
|
+
try {
|
|
26
|
+
return new URL(origin).host === host;
|
|
27
|
+
}
|
|
28
|
+
catch {
|
|
29
|
+
return false;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function wholeNumber(value, max) {
|
|
33
|
+
return typeof value === "number" &&
|
|
34
|
+
Number.isInteger(value) &&
|
|
35
|
+
value >= 0 &&
|
|
36
|
+
value <= max
|
|
37
|
+
? value
|
|
38
|
+
: undefined;
|
|
39
|
+
}
|
|
40
|
+
function pathOf(value) {
|
|
41
|
+
if (typeof value !== "string" || value.length > MAX_PATH_CHARS)
|
|
42
|
+
return null;
|
|
43
|
+
try {
|
|
44
|
+
return new URL(value, "https://redacted.invalid").pathname;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Rebuilds each event from an allowlist, so unexpected fields are dropped. The
|
|
52
|
+
* `browser:` action prefix keeps these out of request counts, because a page
|
|
53
|
+
* load also produces a server event, and Googlebot runs this script too.
|
|
54
|
+
*/
|
|
55
|
+
export function toBrowserEvent(raw, actor, now, sampleRate) {
|
|
56
|
+
if (typeof raw !== "object" || raw === null)
|
|
57
|
+
return null;
|
|
58
|
+
const event = raw;
|
|
59
|
+
if (typeof event.type !== "string" || !TYPES.has(event.type))
|
|
60
|
+
return null;
|
|
61
|
+
const url = pathOf(event.path);
|
|
62
|
+
if (url === null)
|
|
63
|
+
return null;
|
|
64
|
+
const element = typeof event.element === "string" && ELEMENTS.has(event.element)
|
|
65
|
+
? `:${event.element}`
|
|
66
|
+
: "";
|
|
67
|
+
const parent = typeof event.parent === "string" && SERVER_ID.test(event.parent)
|
|
68
|
+
? event.parent
|
|
69
|
+
: undefined;
|
|
70
|
+
return {
|
|
71
|
+
eventId: crypto.randomUUID(),
|
|
72
|
+
occurredAt: now - (wholeNumber(event.ageMs, MAX_AGE_MS) ?? 0),
|
|
73
|
+
family: "resource_access",
|
|
74
|
+
actor,
|
|
75
|
+
action: `browser:${event.type}${element}`,
|
|
76
|
+
url,
|
|
77
|
+
interface: "web",
|
|
78
|
+
parentActionId: parent,
|
|
79
|
+
status: wholeNumber(event.status, 599),
|
|
80
|
+
durationMs: wholeNumber(event.durationMs, MAX_DURATION_MS),
|
|
81
|
+
sampleRate,
|
|
82
|
+
classifierVersion: BROWSER_CLASSIFIER_VERSION,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
export function parseBrowserBody(text) {
|
|
86
|
+
if (text.length > MAX_BODY_CHARS)
|
|
87
|
+
return null;
|
|
88
|
+
try {
|
|
89
|
+
const body = JSON.parse(text);
|
|
90
|
+
if (!Array.isArray(body.events) || body.events.length > MAX_EVENTS) {
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
automated: body.automated === true,
|
|
95
|
+
events: body.events,
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export declare const CLASSIFIER_VERSION = "sdk-ua-0.1";
|
|
2
|
+
type Purpose = "indexing" | "research";
|
|
3
|
+
type Mode = "api" | "browser";
|
|
4
|
+
export type Actor = {
|
|
5
|
+
class: "declared" | "inferred" | "unknown";
|
|
6
|
+
operator?: string;
|
|
7
|
+
purpose?: Purpose;
|
|
8
|
+
mode?: Mode;
|
|
9
|
+
};
|
|
10
|
+
export declare function classify(userAgent: string | null): Actor;
|
|
11
|
+
export {};
|
package/dist/classify.js
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
export const CLASSIFIER_VERSION = "sdk-ua-0.1";
|
|
2
|
+
/**
|
|
3
|
+
* Published user agent tokens. A match is only a declaration, since any client
|
|
4
|
+
* can send these strings, so this never returns `verified`.
|
|
5
|
+
*/
|
|
6
|
+
const DECLARED = [
|
|
7
|
+
["GPTBot", "OpenAI", "indexing"],
|
|
8
|
+
["OAI-SearchBot", "OpenAI", "indexing"],
|
|
9
|
+
["ChatGPT-User", "OpenAI", "research"],
|
|
10
|
+
["ClaudeBot", "Anthropic", "indexing"],
|
|
11
|
+
["Claude-SearchBot", "Anthropic", "indexing"],
|
|
12
|
+
["Claude-User", "Anthropic", "research"],
|
|
13
|
+
["PerplexityBot", "Perplexity", "indexing"],
|
|
14
|
+
["Perplexity-User", "Perplexity", "research"],
|
|
15
|
+
["Googlebot", "Google", "indexing"],
|
|
16
|
+
["GoogleOther", "Google", "indexing"],
|
|
17
|
+
["Google-CloudVertexBot", "Google", "indexing"],
|
|
18
|
+
["bingbot", "Microsoft", "indexing"],
|
|
19
|
+
["Applebot", "Apple", "indexing"],
|
|
20
|
+
["Amazonbot", "Amazon", "indexing"],
|
|
21
|
+
["meta-externalagent", "Meta", "indexing"],
|
|
22
|
+
["meta-externalfetcher", "Meta", "research"],
|
|
23
|
+
["Bytespider", "ByteDance", "indexing"],
|
|
24
|
+
["CCBot", "Common Crawl", "indexing"],
|
|
25
|
+
["cohere-ai", "Cohere", "indexing"],
|
|
26
|
+
["MistralAI-User", "Mistral", "research"],
|
|
27
|
+
["DuckAssistBot", "DuckDuckGo", "research"],
|
|
28
|
+
["DuckDuckBot", "DuckDuckGo", "indexing"],
|
|
29
|
+
];
|
|
30
|
+
const AUTOMATED_BROWSER = /headless|playwright|puppeteer|selenium|phantomjs/i;
|
|
31
|
+
const AUTOMATED_CLIENT = /bot\b|crawl|spider|scrape|curl\/|wget\/|python-requests|python-httpx|aiohttp|go-http-client|okhttp|axios\/|node-fetch|undici/i;
|
|
32
|
+
export function classify(userAgent) {
|
|
33
|
+
if (!userAgent)
|
|
34
|
+
return { class: "unknown" };
|
|
35
|
+
const lower = userAgent.toLowerCase();
|
|
36
|
+
for (const [token, operator, purpose] of DECLARED) {
|
|
37
|
+
if (lower.includes(token.toLowerCase())) {
|
|
38
|
+
return { class: "declared", operator, purpose, mode: "api" };
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (AUTOMATED_BROWSER.test(userAgent)) {
|
|
42
|
+
return { class: "inferred", mode: "browser" };
|
|
43
|
+
}
|
|
44
|
+
if (AUTOMATED_CLIENT.test(userAgent)) {
|
|
45
|
+
return { class: "inferred", mode: "api" };
|
|
46
|
+
}
|
|
47
|
+
return { class: "unknown" };
|
|
48
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapted from get-client-ip.ts in OpenPanel's Express SDK. The header order
|
|
3
|
+
* here puts platform-set headers first.
|
|
4
|
+
*
|
|
5
|
+
* MIT License
|
|
6
|
+
*
|
|
7
|
+
* Copyright (c) 2026 OpenPanel
|
|
8
|
+
*
|
|
9
|
+
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
* of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
* in the Software without restriction, including without limitation the rights
|
|
12
|
+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
* copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
* furnished to do so, subject to the following conditions:
|
|
15
|
+
*
|
|
16
|
+
* The above copyright notice and this permission notice shall be included in
|
|
17
|
+
* all copies or substantial portions of the Software.
|
|
18
|
+
*
|
|
19
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
* SOFTWARE.
|
|
26
|
+
*/
|
|
27
|
+
/** Node reports an IPv4 client as `::ffff:1.2.3.4` on a dual-stack socket. */
|
|
28
|
+
export declare function withoutIpv4Prefix(ip: string): string;
|
|
29
|
+
export declare function usableIp(value: string | null | undefined): string | undefined;
|
|
30
|
+
/** The first public address a proxy reported for the visitor. */
|
|
31
|
+
export declare function clientIpFromHeaders(headers: Headers): string | undefined;
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adapted from get-client-ip.ts in OpenPanel's Express SDK. The header order
|
|
3
|
+
* here puts platform-set headers first.
|
|
4
|
+
*
|
|
5
|
+
* MIT License
|
|
6
|
+
*
|
|
7
|
+
* Copyright (c) 2026 OpenPanel
|
|
8
|
+
*
|
|
9
|
+
* Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
10
|
+
* of this software and associated documentation files (the "Software"), to deal
|
|
11
|
+
* in the Software without restriction, including without limitation the rights
|
|
12
|
+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
13
|
+
* copies of the Software, and to permit persons to whom the Software is
|
|
14
|
+
* furnished to do so, subject to the following conditions:
|
|
15
|
+
*
|
|
16
|
+
* The above copyright notice and this permission notice shall be included in
|
|
17
|
+
* all copies or substantial portions of the Software.
|
|
18
|
+
*
|
|
19
|
+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
20
|
+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
21
|
+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
22
|
+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
23
|
+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
24
|
+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
25
|
+
* SOFTWARE.
|
|
26
|
+
*/
|
|
27
|
+
const HEADER_ORDER = [
|
|
28
|
+
"cf-connecting-ip",
|
|
29
|
+
"x-vercel-forwarded-for",
|
|
30
|
+
"true-client-ip",
|
|
31
|
+
"x-real-ip",
|
|
32
|
+
"x-forwarded-for",
|
|
33
|
+
"fastly-client-ip",
|
|
34
|
+
"x-cluster-client-ip",
|
|
35
|
+
"x-appengine-user-ip",
|
|
36
|
+
"do-connecting-ip",
|
|
37
|
+
"x-nf-client-connection-ip",
|
|
38
|
+
"forwarded",
|
|
39
|
+
];
|
|
40
|
+
const IPV4 = /^(\d{1,3}\.){3}\d{1,3}$/;
|
|
41
|
+
const IPV6 = /^([0-9a-fA-F]{0,4}:){2,7}[0-9a-fA-F]{0,4}$/;
|
|
42
|
+
/** Node reports an IPv4 client as `::ffff:1.2.3.4` on a dual-stack socket. */
|
|
43
|
+
export function withoutIpv4Prefix(ip) {
|
|
44
|
+
return ip.startsWith("::ffff:") ? ip.slice("::ffff:".length) : ip;
|
|
45
|
+
}
|
|
46
|
+
function isPublicIp(address) {
|
|
47
|
+
const ip = withoutIpv4Prefix(address);
|
|
48
|
+
if (ip === "::1")
|
|
49
|
+
return false;
|
|
50
|
+
if (/^(fc|fd|fe8|fe9|fea|feb)/i.test(ip) && ip.includes(":"))
|
|
51
|
+
return false;
|
|
52
|
+
if (/^(127|10)\./.test(ip))
|
|
53
|
+
return false;
|
|
54
|
+
if (ip.startsWith("192.168.") || ip.startsWith("169.254."))
|
|
55
|
+
return false;
|
|
56
|
+
if (ip.startsWith("172.")) {
|
|
57
|
+
const second = Number.parseInt(ip.split(".")[1] ?? "0", 10);
|
|
58
|
+
if (second >= 16 && second <= 31)
|
|
59
|
+
return false;
|
|
60
|
+
}
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
export function usableIp(value) {
|
|
64
|
+
const ip = value ? withoutIpv4Prefix(value.trim()) : "";
|
|
65
|
+
if (!(IPV4.test(ip) || IPV6.test(ip)))
|
|
66
|
+
return undefined;
|
|
67
|
+
return isPublicIp(ip) ? ip : undefined;
|
|
68
|
+
}
|
|
69
|
+
/** The first public address a proxy reported for the visitor. */
|
|
70
|
+
export function clientIpFromHeaders(headers) {
|
|
71
|
+
for (const name of HEADER_ORDER) {
|
|
72
|
+
const value = headers.get(name);
|
|
73
|
+
if (!value)
|
|
74
|
+
continue;
|
|
75
|
+
const candidate = name === "forwarded"
|
|
76
|
+
? /for="?\[?([^\]",;]+)/i.exec(value)?.[1]
|
|
77
|
+
: value.split(",")[0];
|
|
78
|
+
const ip = usableIp(candidate);
|
|
79
|
+
if (ip)
|
|
80
|
+
return ip;
|
|
81
|
+
}
|
|
82
|
+
return undefined;
|
|
83
|
+
}
|
package/dist/config.d.ts
ADDED
package/dist/config.js
ADDED
package/dist/fetch.d.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type ScoutlineOptions } from "./index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Wraps any fetch-style handler, such as a Cloudflare Worker or Hono's
|
|
4
|
+
* `app.fetch`. It sees the whole response, so status and duration are real.
|
|
5
|
+
* Bytes stay empty when the response has no Content-Length. On Cloudflare it
|
|
6
|
+
* also adds the browser script to HTML pages, so no code change is needed.
|
|
7
|
+
*
|
|
8
|
+
* export default { fetch: withScoutline(app.fetch) };
|
|
9
|
+
*
|
|
10
|
+
* Workers should pass `{ key: env.SCOUTLINE_KEY }` from `cloudflare:workers`,
|
|
11
|
+
* because older compatibility dates have no `process.env` to read it from.
|
|
12
|
+
*/
|
|
13
|
+
export declare function withScoutline<Args extends unknown[]>(handler: (request: Request, ...args: Args) => Response | Promise<Response>, options?: ScoutlineOptions): (request: Request, ...args: Args) => Promise<Response>;
|
package/dist/fetch.js
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { SCRIPT_PATH } from "./browser.js";
|
|
2
|
+
import { createScoutline, isDocumentRequest, serverTiming, } from "./index.js";
|
|
3
|
+
const SCRIPT_TAG = `<script src="${SCRIPT_PATH}" defer></script>`;
|
|
4
|
+
function forPage(response, eventId) {
|
|
5
|
+
if (!response.headers.get("content-type")?.includes("text/html")) {
|
|
6
|
+
return response;
|
|
7
|
+
}
|
|
8
|
+
const page = new Response(response.body, response);
|
|
9
|
+
if (eventId)
|
|
10
|
+
page.headers.append("Server-Timing", serverTiming(eventId));
|
|
11
|
+
if (typeof HTMLRewriter === "undefined")
|
|
12
|
+
return page;
|
|
13
|
+
return new HTMLRewriter()
|
|
14
|
+
.on("head", {
|
|
15
|
+
element(head) {
|
|
16
|
+
head.append(SCRIPT_TAG, { html: true });
|
|
17
|
+
},
|
|
18
|
+
})
|
|
19
|
+
.transform(page);
|
|
20
|
+
}
|
|
21
|
+
function contentLength(response) {
|
|
22
|
+
const value = Number(response.headers.get("content-length"));
|
|
23
|
+
return response.headers.has("content-length") && Number.isFinite(value)
|
|
24
|
+
? value
|
|
25
|
+
: null;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Wraps any fetch-style handler, such as a Cloudflare Worker or Hono's
|
|
29
|
+
* `app.fetch`. It sees the whole response, so status and duration are real.
|
|
30
|
+
* Bytes stay empty when the response has no Content-Length. On Cloudflare it
|
|
31
|
+
* also adds the browser script to HTML pages, so no code change is needed.
|
|
32
|
+
*
|
|
33
|
+
* export default { fetch: withScoutline(app.fetch) };
|
|
34
|
+
*
|
|
35
|
+
* Workers should pass `{ key: env.SCOUTLINE_KEY }` from `cloudflare:workers`,
|
|
36
|
+
* because older compatibility dates have no `process.env` to read it from.
|
|
37
|
+
*/
|
|
38
|
+
export function withScoutline(handler, options) {
|
|
39
|
+
const scoutline = createScoutline(options);
|
|
40
|
+
return async function scoutlineFetch(request, ...args) {
|
|
41
|
+
const context = args.find((arg) => typeof arg?.waitUntil === "function");
|
|
42
|
+
const own = scoutline.handle(request);
|
|
43
|
+
if (own) {
|
|
44
|
+
const answer = await own;
|
|
45
|
+
context?.waitUntil(scoutline.flush());
|
|
46
|
+
return answer;
|
|
47
|
+
}
|
|
48
|
+
const started = Date.now();
|
|
49
|
+
const finish = (status, bytes) => {
|
|
50
|
+
const eventId = scoutline.observe(request, {
|
|
51
|
+
status,
|
|
52
|
+
bytes,
|
|
53
|
+
durationMs: Date.now() - started,
|
|
54
|
+
cacheStatus: null,
|
|
55
|
+
});
|
|
56
|
+
context?.waitUntil(scoutline.flush());
|
|
57
|
+
return eventId;
|
|
58
|
+
};
|
|
59
|
+
let response;
|
|
60
|
+
try {
|
|
61
|
+
response = await handler(request, ...args);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
finish(null, null);
|
|
65
|
+
throw error;
|
|
66
|
+
}
|
|
67
|
+
const eventId = finish(response.status, contentLength(response));
|
|
68
|
+
if (!isDocumentRequest(request))
|
|
69
|
+
return response;
|
|
70
|
+
try {
|
|
71
|
+
return forPage(response, eventId);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return response;
|
|
75
|
+
}
|
|
76
|
+
};
|
|
77
|
+
}
|