@scoutline-ai/middleware 0.0.0-stage → 0.1.0

package/README.md

@@ -1,3 +1,92 @@
-# Temporary Holding Version
+# @scoutline-ai/middleware
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Records how agents use your site. It fails open: if Scoutline is down, misconfigured or slow, your site behaves exactly the same. Sending happens after your response, so it adds no latency.
+
+Set your key as an environment variable:
+
+```sh
+SCOUTLINE_KEY=sl_live_...
+```
+
+## Next.js
+
+Wrap your proxy in `proxy.ts` (`middleware.ts` before Next.js 16). Your existing handler keeps working.
+
+```ts
+import { withScoutline } from "@scoutline-ai/middleware/next";
+
+export default withScoutline(yourProxy); // or withScoutline() if you have none
+
+export const config = {
+  matcher: [
+    {
+      source: "/((?!_next/static|_next/image).*)",
+      missing: [
+        { type: "header", key: "next-router-prefetch" },
+        { type: "header", key: "purpose", value: "prefetch" },
+      ],
+    },
+  ],
+};
+```
+
+Next.js removes prefetch headers before your proxy runs, so only the matcher can exclude them. The `missing` rule skips your whole proxy for prefetches, so leave it out if your proxy enforces auth. Prefetches then count as `unknown` traffic. They only come from browsers, so crawler and agent counts are unaffected.
+
+A proxy runs before the page renders, so normal page loads have no status or duration. Scoutline leaves those empty and only records a status when the proxy itself responds, such as a redirect.
+
+## Cloudflare Workers and Hono
+
+```ts
+import { withScoutline } from "@scoutline-ai/middleware/fetch";
+
+export default { fetch: withScoutline(app.fetch) };
+```
+
+This sees the full response, so it records status, duration and size.
+
+## Express and Fastify
+
+```ts
+import { scoutlineExpress, scoutlineFastify } from "@scoutline-ai/middleware/node";
+
+app.use(scoutlineExpress()); // Express: before your routes and body parsers
+
+app.register(scoutlineFastify); // Fastify
+```
+
+Both see the full response, so they record status, duration and size. Events send every few seconds.
+
+## Browser events
+
+The server sees every request, including crawlers that never run JavaScript. The browser script adds what only a page can see: real status and load time, in-page navigation, clicks and form submits, and errors.
+
+**Cloudflare Workers:** nothing to do. HTML pages get the script automatically.
+
+**Next.js, Hono, Express and Fastify:** add one tag to your root layout or HTML template.
+
+```html
+<script src="/_scoutline/s" defer></script>
+```
+
+The script and its events are served from your own domain by this package, so there is no public key and no third-party request. If your proxy `matcher` is limited to certain paths, include `/_scoutline/:path*`. These routes run before your own proxy, so an auth check won't block them.
+
+It stores nothing and sets no cookies. A page links to its server request through a `Server-Timing` header. Each batch of events is one request to your server, sent when the page closes or after 20 events.
+
+Browser events can be forged, so they are never `verified` and can't confirm an outcome. Reports keep them out of request counts.
+
+## What is sent
+
+From the server: the method, pathname, classification from the user agent, and request and trace ids. From the browser: the pathname, event type, element type such as `button`, status, and load time.
+
+Query strings, cookies, auth headers, bodies, page text and field values are never sent.
+
+A matching user agent is recorded as `declared`, because any client can send one. Other automation is `inferred`, and everything else is `unknown`. This package never reports `verified`.
+
+## Options
+
+| Option | Default | Purpose |
+| --- | --- | --- |
+| `key` | `SCOUTLINE_KEY` | Ingest key |
+| `endpoint` | `SCOUTLINE_ENDPOINT`, then `https://ingest.scoutline.ai` | Collector URL |
+| `sampleUnknown` | `1` | Share of unclassified traffic to send, from 0 to 1 |
+| `ignore` | none | `(pathname) => boolean` to skip a path |

package/dist/browser-script.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Served from the customer's own domain at /_scoutline/s. It stays plain ES5 and
+ * dependency free so it can load on any page. It collects no text, field values
+ * or query strings, and stores nothing, so no cookie or consent is involved.
+ * Everything it sends is an unverified claim and is revalidated in `browser.ts`.
+ */
+export declare const BROWSER_SCRIPT = "(function(){\n  var ENDPOINT=\"/_scoutline/e\",MAX=20,queue=[],path=location.pathname;\n  var nav=performance.getEntriesByType?performance.getEntriesByType(\"navigation\")[0]:null;\n  var automated=navigator.webdriver===true;\n  function serverId(){\n    var timing=nav&&nav.serverTiming;\n    for(var i=0;timing&&i<timing.length;i++){\n      if(timing[i].name===\"scoutline\")return timing[i].description;\n    }\n  }\n  function send(){\n    if(!queue.length)return;\n    var now=Date.now(),events=[];\n    for(var i=0;i<queue.length;i++){\n      var e=queue[i];e.ageMs=now-e.at;delete e.at;events.push(e);\n    }\n    queue=[];\n    var body=JSON.stringify({automated:automated,events:events});\n    if(!navigator.sendBeacon||!navigator.sendBeacon(ENDPOINT,body)){\n      fetch(ENDPOINT,{method:\"POST\",body:body,keepalive:true})[\"catch\"](function(){});\n    }\n  }\n  function add(type,extra){\n    var e=extra||{};e.type=type;e.path=location.pathname;e.at=Date.now();\n    queue.push(e);\n    if(queue.length>=MAX)send();\n  }\n  function firstView(){\n    add(\"page_view\",{\n      parent:serverId(),\n      status:nav&&nav.responseStatus?nav.responseStatus:undefined,\n      durationMs:nav&&nav.duration?Math.round(nav.duration):undefined\n    });\n  }\n  function changed(){\n    if(location.pathname===path)return;\n    path=location.pathname;add(\"page_view\");\n  }\n  [\"pushState\",\"replaceState\"].forEach(function(name){\n    var original=history[name];\n    history[name]=function(){var r=original.apply(this,arguments);changed();return r;};\n  });\n  addEventListener(\"popstate\",changed);\n  document.addEventListener(\"click\",function(event){\n    var t=event.target&&event.target.closest&&event.target.closest(\"a,button,[role=button],input[type=submit]\");\n    if(t)add(\"click\",{element:t.tagName.toLowerCase()});\n  },true);\n  document.addEventListener(\"submit\",function(){add(\"form_submit\");},true);\n  addEventListener(\"error\",function(event){if(event.message)add(\"error\");});\n  addEventListener(\"unhandledrejection\",function(){add(\"error\");});\n  addEventListener(\"pagehide\",send);\n  document.addEventListener(\"visibilitychange\",function(){\n    if(document.visibilityState===\"hidden\")send();\n  });\n  if(document.readyState===\"complete\")setTimeout(firstView);\n  else addEventListener(\"load\",function(){setTimeout(firstView);});\n})();";

package/dist/browser-script.js

@@ -0,0 +1,63 @@
+/**
+ * Served from the customer's own domain at /_scoutline/s. It stays plain ES5 and
+ * dependency free so it can load on any page. It collects no text, field values
+ * or query strings, and stores nothing, so no cookie or consent is involved.
+ * Everything it sends is an unverified claim and is revalidated in `browser.ts`.
+ */
+export const BROWSER_SCRIPT = `(function(){
+  var ENDPOINT="/_scoutline/e",MAX=20,queue=[],path=location.pathname;
+  var nav=performance.getEntriesByType?performance.getEntriesByType("navigation")[0]:null;
+  var automated=navigator.webdriver===true;
+  function serverId(){
+    var timing=nav&&nav.serverTiming;
+    for(var i=0;timing&&i<timing.length;i++){
+      if(timing[i].name==="scoutline")return timing[i].description;
+    }
+  }
+  function send(){
+    if(!queue.length)return;
+    var now=Date.now(),events=[];
+    for(var i=0;i<queue.length;i++){
+      var e=queue[i];e.ageMs=now-e.at;delete e.at;events.push(e);
+    }
+    queue=[];
+    var body=JSON.stringify({automated:automated,events:events});
+    if(!navigator.sendBeacon||!navigator.sendBeacon(ENDPOINT,body)){
+      fetch(ENDPOINT,{method:"POST",body:body,keepalive:true})["catch"](function(){});
+    }
+  }
+  function add(type,extra){
+    var e=extra||{};e.type=type;e.path=location.pathname;e.at=Date.now();
+    queue.push(e);
+    if(queue.length>=MAX)send();
+  }
+  function firstView(){
+    add("page_view",{
+      parent:serverId(),
+      status:nav&&nav.responseStatus?nav.responseStatus:undefined,
+      durationMs:nav&&nav.duration?Math.round(nav.duration):undefined
+    });
+  }
+  function changed(){
+    if(location.pathname===path)return;
+    path=location.pathname;add("page_view");
+  }
+  ["pushState","replaceState"].forEach(function(name){
+    var original=history[name];
+    history[name]=function(){var r=original.apply(this,arguments);changed();return r;};
+  });
+  addEventListener("popstate",changed);
+  document.addEventListener("click",function(event){
+    var t=event.target&&event.target.closest&&event.target.closest("a,button,[role=button],input[type=submit]");
+    if(t)add("click",{element:t.tagName.toLowerCase()});
+  },true);
+  document.addEventListener("submit",function(){add("form_submit");},true);
+  addEventListener("error",function(event){if(event.message)add("error");});
+  addEventListener("unhandledrejection",function(){add("error");});
+  addEventListener("pagehide",send);
+  document.addEventListener("visibilitychange",function(){
+    if(document.visibilityState==="hidden")send();
+  });
+  if(document.readyState==="complete")setTimeout(firstView);
+  else addEventListener("load",function(){setTimeout(firstView);});
+})();`;

package/dist/browser.d.ts

@@ -0,0 +1,33 @@
+import type { Actor } from "./classify.js";
+export declare const SCRIPT_PATH = "/_scoutline/s";
+export declare const EVENTS_PATH = "/_scoutline/e";
+export declare const BROWSER_CLASSIFIER_VERSION = "browser-0.1";
+/**
+ * Anything in the page can be forged, so only a same-origin browser request is
+ * accepted. This keeps out other sites, not a determined client, which is why
+ * these events stay low trust however they arrive.
+ */
+export declare function isSameOrigin(headers: Headers): boolean;
+/**
+ * Rebuilds each event from an allowlist, so unexpected fields are dropped. The
+ * `browser:` action prefix keeps these out of request counts, because a page
+ * load also produces a server event, and Googlebot runs this script too.
+ */
+export declare function toBrowserEvent(raw: unknown, actor: Actor, now: number, sampleRate: number): {
+    eventId: `${string}-${string}-${string}-${string}-${string}`;
+    occurredAt: number;
+    family: string;
+    actor: Actor;
+    action: string;
+    url: string;
+    interface: string;
+    parentActionId: string | undefined;
+    status: number | undefined;
+    durationMs: number | undefined;
+    sampleRate: number;
+    classifierVersion: string;
+} | null;
+export declare function parseBrowserBody(text: string): {
+    automated: boolean;
+    events: unknown[];
+} | null;

package/dist/browser.js

@@ -0,0 +1,101 @@
+export const SCRIPT_PATH = "/_scoutline/s";
+export const EVENTS_PATH = "/_scoutline/e";
+export const BROWSER_CLASSIFIER_VERSION = "browser-0.1";
+const MAX_BODY_CHARS = 16 * 1024;
+const MAX_EVENTS = 20;
+const MAX_AGE_MS = 10 * 60 * 1000;
+const MAX_PATH_CHARS = 2048;
+const MAX_DURATION_MS = 60 * 60 * 1000;
+const TYPES = new Set(["page_view", "click", "form_submit", "error"]);
+const ELEMENTS = new Set(["a", "button", "input", "div", "span"]);
+const SERVER_ID = /^[0-9a-f-]{36}$/;
+/**
+ * Anything in the page can be forged, so only a same-origin browser request is
+ * accepted. This keeps out other sites, not a determined client, which is why
+ * these events stay low trust however they arrive.
+ */
+export function isSameOrigin(headers) {
+    const site = headers.get("sec-fetch-site");
+    if (site && site !== "same-origin")
+        return false;
+    const origin = headers.get("origin");
+    if (!origin)
+        return site === "same-origin";
+    const host = headers.get("x-forwarded-host") ?? headers.get("host");
+    try {
+        return new URL(origin).host === host;
+    }
+    catch {
+        return false;
+    }
+}
+function wholeNumber(value, max) {
+    return typeof value === "number" &&
+        Number.isInteger(value) &&
+        value >= 0 &&
+        value <= max
+        ? value
+        : undefined;
+}
+function pathOf(value) {
+    if (typeof value !== "string" || value.length > MAX_PATH_CHARS)
+        return null;
+    try {
+        return new URL(value, "https://redacted.invalid").pathname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Rebuilds each event from an allowlist, so unexpected fields are dropped. The
+ * `browser:` action prefix keeps these out of request counts, because a page
+ * load also produces a server event, and Googlebot runs this script too.
+ */
+export function toBrowserEvent(raw, actor, now, sampleRate) {
+    if (typeof raw !== "object" || raw === null)
+        return null;
+    const event = raw;
+    if (typeof event.type !== "string" || !TYPES.has(event.type))
+        return null;
+    const url = pathOf(event.path);
+    if (url === null)
+        return null;
+    const element = typeof event.element === "string" && ELEMENTS.has(event.element)
+        ? `:${event.element}`
+        : "";
+    const parent = typeof event.parent === "string" && SERVER_ID.test(event.parent)
+        ? event.parent
+        : undefined;
+    return {
+        eventId: crypto.randomUUID(),
+        occurredAt: now - (wholeNumber(event.ageMs, MAX_AGE_MS) ?? 0),
+        family: "resource_access",
+        actor,
+        action: `browser:${event.type}${element}`,
+        url,
+        interface: "web",
+        parentActionId: parent,
+        status: wholeNumber(event.status, 599),
+        durationMs: wholeNumber(event.durationMs, MAX_DURATION_MS),
+        sampleRate,
+        classifierVersion: BROWSER_CLASSIFIER_VERSION,
+    };
+}
+export function parseBrowserBody(text) {
+    if (text.length > MAX_BODY_CHARS)
+        return null;
+    try {
+        const body = JSON.parse(text);
+        if (!Array.isArray(body.events) || body.events.length > MAX_EVENTS) {
+            return null;
+        }
+        return {
+            automated: body.automated === true,
+            events: body.events,
+        };
+    }
+    catch {
+        return null;
+    }
+}

package/dist/classify.d.ts

@@ -0,0 +1,11 @@
+export declare const CLASSIFIER_VERSION = "sdk-ua-0.1";
+type Purpose = "indexing" | "research";
+type Mode = "api" | "browser";
+export type Actor = {
+    class: "declared" | "inferred" | "unknown";
+    operator?: string;
+    purpose?: Purpose;
+    mode?: Mode;
+};
+export declare function classify(userAgent: string | null): Actor;
+export {};

package/dist/classify.js

@@ -0,0 +1,48 @@
+export const CLASSIFIER_VERSION = "sdk-ua-0.1";
+/**
+ * Published user agent tokens. A match is only a declaration, since any client
+ * can send these strings, so this never returns `verified`.
+ */
+const DECLARED = [
+    ["GPTBot", "OpenAI", "indexing"],
+    ["OAI-SearchBot", "OpenAI", "indexing"],
+    ["ChatGPT-User", "OpenAI", "research"],
+    ["ClaudeBot", "Anthropic", "indexing"],
+    ["Claude-SearchBot", "Anthropic", "indexing"],
+    ["Claude-User", "Anthropic", "research"],
+    ["PerplexityBot", "Perplexity", "indexing"],
+    ["Perplexity-User", "Perplexity", "research"],
+    ["Googlebot", "Google", "indexing"],
+    ["GoogleOther", "Google", "indexing"],
+    ["Google-CloudVertexBot", "Google", "indexing"],
+    ["bingbot", "Microsoft", "indexing"],
+    ["Applebot", "Apple", "indexing"],
+    ["Amazonbot", "Amazon", "indexing"],
+    ["meta-externalagent", "Meta", "indexing"],
+    ["meta-externalfetcher", "Meta", "research"],
+    ["Bytespider", "ByteDance", "indexing"],
+    ["CCBot", "Common Crawl", "indexing"],
+    ["cohere-ai", "Cohere", "indexing"],
+    ["MistralAI-User", "Mistral", "research"],
+    ["DuckAssistBot", "DuckDuckGo", "research"],
+    ["DuckDuckBot", "DuckDuckGo", "indexing"],
+];
+const AUTOMATED_BROWSER = /headless|playwright|puppeteer|selenium|phantomjs/i;
+const AUTOMATED_CLIENT = /bot\b|crawl|spider|scrape|curl\/|wget\/|python-requests|python-httpx|aiohttp|go-http-client|okhttp|axios\/|node-fetch|undici/i;
+export function classify(userAgent) {
+    if (!userAgent)
+        return { class: "unknown" };
+    const lower = userAgent.toLowerCase();
+    for (const [token, operator, purpose] of DECLARED) {
+        if (lower.includes(token.toLowerCase())) {
+            return { class: "declared", operator, purpose, mode: "api" };
+        }
+    }
+    if (AUTOMATED_BROWSER.test(userAgent)) {
+        return { class: "inferred", mode: "browser" };
+    }
+    if (AUTOMATED_CLIENT.test(userAgent)) {
+        return { class: "inferred", mode: "api" };
+    }
+    return { class: "unknown" };
+}

package/dist/fetch.d.ts

@@ -0,0 +1,10 @@
+import { type ScoutlineOptions } from "./index.js";
+/**
+ * Wraps any fetch-style handler, such as a Cloudflare Worker or Hono's
+ * `app.fetch`. It sees the whole response, so status and duration are real.
+ * Bytes stay empty when the response has no Content-Length. On Cloudflare it
+ * also adds the browser script to HTML pages, so no code change is needed.
+ *
+ *   export default { fetch: withScoutline(app.fetch) };
+ */
+export declare function withScoutline<Args extends unknown[]>(handler: (request: Request, ...args: Args) => Response | Promise<Response>, options?: ScoutlineOptions): (request: Request, ...args: Args) => Promise<Response>;

package/dist/fetch.js

@@ -0,0 +1,74 @@
+import { SCRIPT_PATH } from "./browser.js";
+import { createScoutline, isDocumentRequest, serverTiming, } from "./index.js";
+const SCRIPT_TAG = `<script src="${SCRIPT_PATH}" defer></script>`;
+function forPage(response, eventId) {
+    if (!response.headers.get("content-type")?.includes("text/html")) {
+        return response;
+    }
+    const page = new Response(response.body, response);
+    if (eventId)
+        page.headers.append("Server-Timing", serverTiming(eventId));
+    if (typeof HTMLRewriter === "undefined")
+        return page;
+    return new HTMLRewriter()
+        .on("head", {
+        element(head) {
+            head.append(SCRIPT_TAG, { html: true });
+        },
+    })
+        .transform(page);
+}
+function contentLength(response) {
+    const value = Number(response.headers.get("content-length"));
+    return response.headers.has("content-length") && Number.isFinite(value)
+        ? value
+        : null;
+}
+/**
+ * Wraps any fetch-style handler, such as a Cloudflare Worker or Hono's
+ * `app.fetch`. It sees the whole response, so status and duration are real.
+ * Bytes stay empty when the response has no Content-Length. On Cloudflare it
+ * also adds the browser script to HTML pages, so no code change is needed.
+ *
+ *   export default { fetch: withScoutline(app.fetch) };
+ */
+export function withScoutline(handler, options) {
+    const scoutline = createScoutline(options);
+    return async function scoutlineFetch(request, ...args) {
+        const context = args.find((arg) => typeof arg?.waitUntil === "function");
+        const own = scoutline.handle(request);
+        if (own) {
+            const answer = await own;
+            context?.waitUntil(scoutline.flush());
+            return answer;
+        }
+        const started = Date.now();
+        const finish = (status, bytes) => {
+            const eventId = scoutline.observe(request, {
+                status,
+                bytes,
+                durationMs: Date.now() - started,
+                cacheStatus: null,
+            });
+            context?.waitUntil(scoutline.flush());
+            return eventId;
+        };
+        let response;
+        try {
+            response = await handler(request, ...args);
+        }
+        catch (error) {
+            finish(null, null);
+            throw error;
+        }
+        const eventId = finish(response.status, contentLength(response));
+        if (!isDocumentRequest(request))
+            return response;
+        try {
+            return forPage(response, eventId);
+        }
+        catch {
+            return response;
+        }
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,44 @@
+export { classify } from "./classify.js";
+export type ScoutlineOptions = {
+    /** Ingest key. Falls back to the SCOUTLINE_KEY environment variable. */
+    key?: string;
+    /** Collector URL, for self-hosting or testing. Falls back to SCOUTLINE_ENDPOINT. */
+    endpoint?: string;
+    /** Share of unclassified traffic to send, from 0 to 1. Default 1. */
+    sampleUnknown?: number;
+    /** Skip a path entirely. Runs on the pathname only. */
+    ignore?: (pathname: string) => boolean;
+};
+export type Observation = {
+    /** Set by adapters that must announce the id before the response finishes. */
+    eventId?: string;
+    /** Null when the response isn't visible from this position in the stack. */
+    status?: number | null;
+    durationMs?: number | null;
+    bytes?: number | null;
+    cacheStatus?: string | null;
+};
+export type Scoutline = {
+    /**
+     * Records a request and returns its event id. Synchronous and never throws,
+     * so it can't break a site. Returns nothing if the request was skipped.
+     */
+    observe(request: Request, observation?: Observation): string | undefined;
+    /** Sends what is queued. Never rejects. Pass this to `waitUntil`. */
+    flush(): Promise<void>;
+    /**
+     * Serves the browser script and its events from the site's own domain.
+     * Returns null for every other path. Must run before any auth check.
+     */
+    handle(request: Request): Promise<Response> | null;
+};
+/** Only full page loads carry a server id for the browser script to read. */
+export declare function isDocumentRequest(request: Request): boolean;
+export declare function serverTiming(eventId: string): string;
+/** The site's hostname. A proxy may list several, nearest client first. */
+export declare function hostOf(request: Request): string | undefined;
+/**
+ * Failure here must never reach the customer's site. A missing key turns the
+ * client into a no-op, and a full queue drops the oldest events.
+ */
+export declare function createScoutline(options?: ScoutlineOptions): Scoutline;

package/dist/index.js

@@ -0,0 +1,277 @@
+import { EVENTS_PATH, isSameOrigin, parseBrowserBody, SCRIPT_PATH, toBrowserEvent, } from "./browser.js";
+import { BROWSER_SCRIPT } from "./browser-script.js";
+import { classify, CLASSIFIER_VERSION } from "./classify.js";
+export { classify } from "./classify.js";
+const DEFAULT_ENDPOINT = "https://ingest.scoutline.ai";
+const MAX_QUEUED = 500;
+const MAX_BATCH = 100;
+const SEND_TIMEOUT_MS = 2000;
+const FLUSH_INTERVAL_MS = 5000;
+const HEARTBEAT_INTERVAL_MS = 5 * 60 * 1000;
+const REJECTED_BACKOFF_MS = 60 * 1000;
+const RATE_LIMITED_BACKOFF_MS = 60 * 1000;
+const MAX_FIELD_CHARS = 128;
+const MAX_URL_CHARS = 2048;
+/** Caps browser events per server instance, so a page can't flood the queue. */
+const MAX_BROWSER_EVENTS_PER_MINUTE = 2000;
+const SCRIPT_CACHE_SECONDS = 3600;
+/** Prefetches aren't visits, and assets aren't what agents come for. */
+const IGNORED_PATH = /^\/(?:_next|_vercel|\.well-known\/vercel)\/|\.(?:css|js|mjs|map|png|jpe?g|gif|webp|avif|svg|ico|woff2?|ttf)$/i;
+/** Only full page loads carry a server id for the browser script to read. */
+export function isDocumentRequest(request) {
+    return (request.method === "GET" &&
+        request.headers.get("accept")?.includes("text/html") === true &&
+        !isPrefetch(request.headers));
+}
+export function serverTiming(eventId) {
+    return `scoutline;desc="${eventId}"`;
+}
+function scriptResponse() {
+    return new Response(BROWSER_SCRIPT, {
+        headers: {
+            "Content-Type": "text/javascript; charset=utf-8",
+            "Cache-Control": `public, max-age=${SCRIPT_CACHE_SECONDS}`,
+        },
+    });
+}
+function emptyResponse(status) {
+    return new Response(null, { status });
+}
+function ownRoute(request) {
+    const { pathname } = new URL(request.url);
+    return pathname === SCRIPT_PATH || pathname === EVENTS_PATH ? pathname : null;
+}
+function env(name) {
+    return typeof process === "undefined" ? undefined : process?.env?.[name];
+}
+function limited(value, max = MAX_FIELD_CHARS) {
+    return value && value.length <= max ? value : undefined;
+}
+function interfaceFor(pathname, headers) {
+    if (pathname === "/mcp" || pathname.startsWith("/mcp/"))
+        return "tool";
+    if (pathname.startsWith("/api/"))
+        return "api";
+    return headers.get("accept")?.includes("application/json") ? "api" : "web";
+}
+/** The site's hostname. A proxy may list several, nearest client first. */
+export function hostOf(request) {
+    const forwarded = request.headers.get("x-forwarded-host")?.split(",")[0];
+    const host = forwarded?.trim() ||
+        request.headers.get("host") ||
+        new URL(request.url).host;
+    return limited(host.toLowerCase(), 255);
+}
+function isPrefetch(headers) {
+    return (headers.has("next-router-prefetch") ||
+        headers.get("purpose") === "prefetch" ||
+        headers.get("sec-purpose")?.includes("prefetch") === true);
+}
+/** The W3C trace id from `traceparent`, a claim until something validates it. */
+function traceId(headers) {
+    return limited(headers.get("traceparent")?.split("-")[1] ?? null);
+}
+function toEvent(request, observation, sampleUnknown, ignore) {
+    const { pathname } = new URL(request.url);
+    if (IGNORED_PATH.test(pathname) || ignore?.(pathname))
+        return null;
+    const { headers } = request;
+    if (isPrefetch(headers))
+        return null;
+    const actor = classify(headers.get("user-agent"));
+    const sampled = actor.class === "unknown" && sampleUnknown < 1;
+    if (sampled && Math.random() >= sampleUnknown)
+        return null;
+    return {
+        eventId: observation.eventId ?? crypto.randomUUID(),
+        occurredAt: Date.now(),
+        family: "resource_access",
+        actor,
+        action: request.method.slice(0, 64),
+        // The pathname only: queries, cookies, headers and bodies never leave here.
+        url: pathname.slice(0, MAX_URL_CHARS),
+        host: hostOf(request),
+        interface: interfaceFor(pathname, headers),
+        requestId: limited(headers.get("x-vercel-id") ??
+            headers.get("cf-ray") ??
+            headers.get("x-request-id")),
+        traceId: traceId(headers),
+        status: observation.status ?? undefined,
+        durationMs: observation.durationMs ?? undefined,
+        bytes: observation.bytes ?? undefined,
+        cacheStatus: limited(observation.cacheStatus ?? null, 32),
+        sampleRate: sampled ? sampleUnknown : 1,
+        classifierVersion: CLASSIFIER_VERSION,
+    };
+}
+/**
+ * Failure here must never reach the customer's site. A missing key turns the
+ * client into a no-op, and a full queue drops the oldest events.
+ */
+export function createScoutline(options = {}) {
+    const key = options.key ?? env("SCOUTLINE_KEY");
+    const endpoint = (options.endpoint ||
+        env("SCOUTLINE_ENDPOINT") ||
+        DEFAULT_ENDPOINT).replace(/\/+$/, "");
+    if (!key) {
+        console.warn("[scoutline] Missing SCOUTLINE_KEY, so nothing is collected.");
+        return {
+            observe: () => undefined,
+            flush: async () => { },
+            // The script tag still gets a valid answer, so pages don't log a 404.
+            handle(request) {
+                const route = ownRoute(request);
+                if (route === SCRIPT_PATH)
+                    return Promise.resolve(scriptResponse());
+                return route ? Promise.resolve(emptyResponse(204)) : null;
+            },
+        };
+    }
+    const sampleUnknown = options.sampleUnknown ?? 1;
+    let queue = [];
+    let timer;
+    let lastSentAt = 0;
+    let pausedUntil = 0;
+    let browserWindowStart = 0;
+    let browserEventsInWindow = 0;
+    async function post(events) {
+        const response = await fetch(`${endpoint}/v1/events`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${key}`,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({ events }),
+            signal: AbortSignal.timeout(SEND_TIMEOUT_MS),
+        });
+        await response.body?.cancel();
+        return response.status;
+    }
+    async function drain() {
+        while (queue.length > 0 && Date.now() >= pausedUntil) {
+            const batch = queue.slice(0, MAX_BATCH);
+            queue = queue.slice(batch.length);
+            let status;
+            try {
+                status = await post(batch);
+            }
+            catch {
+                status = undefined;
+            }
+            if (status === 401) {
+                // Retrying can't fix a bad key, so don't spend the site's time on it.
+                console.warn("[scoutline] The collector refused this key.");
+                pausedUntil = Date.now() + REJECTED_BACKOFF_MS;
+                return;
+            }
+            if (status === 429) {
+                // The collector asked us to slow down, so hold these and try later.
+                pausedUntil = Date.now() + RATE_LIMITED_BACKOFF_MS;
+                queue = [...batch, ...queue].slice(-MAX_QUEUED);
+                return;
+            }
+            if (status === undefined || status >= 500) {
+                // Event ids are stable, so a repeat of a partial send dedupes downstream.
+                queue = [...batch, ...queue].slice(-MAX_QUEUED);
+                return;
+            }
+            lastSentAt = Date.now();
+        }
+    }
+    /** A quiet site isn't a broken collector. An empty batch says so. */
+    async function heartbeat() {
+        if (Date.now() - lastSentAt < HEARTBEAT_INTERVAL_MS)
+            return;
+        if (Date.now() < pausedUntil)
+            return;
+        lastSentAt = Date.now();
+        await post([]).catch(() => { });
+    }
+    /**
+     * Each call drains by itself rather than sharing one in-flight promise,
+     * because Cloudflare doesn't let a request wait on another request's work.
+     * The queue hands each batch out once, so overlapping calls are safe.
+     */
+    function flush() {
+        return drain()
+            .then(heartbeat)
+            .catch(() => { });
+    }
+    function enqueue(event) {
+        queue.push(event);
+        if (queue.length > MAX_QUEUED)
+            queue = queue.slice(-MAX_QUEUED);
+        // Keeps long-running servers sending. Serverless flushes via waitUntil.
+        if (!timer) {
+            timer = setTimeout(() => {
+                timer = undefined;
+                void flush();
+            }, FLUSH_INTERVAL_MS);
+            timer.unref?.();
+        }
+    }
+    function observe(request, observation = {}) {
+        try {
+            const event = toEvent(request, observation, sampleUnknown, options.ignore);
+            if (!event)
+                return undefined;
+            enqueue(event);
+            return event.eventId;
+        }
+        catch {
+            // Observation is optional. The customer's request is not.
+            return undefined;
+        }
+    }
+    function withinBrowserLimit(count) {
+        const now = Date.now();
+        if (now - browserWindowStart > 60 * 1000) {
+            browserWindowStart = now;
+            browserEventsInWindow = 0;
+        }
+        browserEventsInWindow += count;
+        return browserEventsInWindow <= MAX_BROWSER_EVENTS_PER_MINUTE;
+    }
+    /** The class comes from the server's view. A page may only make it weaker. */
+    function browserActor(request, automated) {
+        const actor = classify(request.headers.get("user-agent"));
+        if (actor.class === "unknown" && automated) {
+            return { class: "inferred", mode: "browser" };
+        }
+        return actor;
+    }
+    async function acceptBrowserEvents(request) {
+        try {
+            if (request.method !== "POST")
+                return emptyResponse(405);
+            if (!isSameOrigin(request.headers))
+                return emptyResponse(403);
+            const body = parseBrowserBody(await request.text());
+            if (!body)
+                return emptyResponse(400);
+            if (!withinBrowserLimit(body.events.length))
+                return emptyResponse(204);
+            const actor = browserActor(request, body.automated);
+            const sampled = actor.class === "unknown" && sampleUnknown < 1;
+            if (sampled && Math.random() >= sampleUnknown)
+                return emptyResponse(204);
+            const now = Date.now();
+            for (const raw of body.events) {
+                const event = toBrowserEvent(raw, actor, now, sampled ? sampleUnknown : 1);
+                if (event)
+                    enqueue({ ...event, host: hostOf(request) });
+            }
+            return emptyResponse(204);
+        }
+        catch {
+            return emptyResponse(204);
+        }
+    }
+    function handle(request) {
+        const route = ownRoute(request);
+        if (route === SCRIPT_PATH)
+            return Promise.resolve(scriptResponse());
+        return route ? acceptBrowserEvents(request) : null;
+    }
+    return { observe, flush, handle };
+}

package/dist/next.d.ts

@@ -0,0 +1,20 @@
+import { type ScoutlineOptions } from "./index.js";
+type WaitUntil = {
+    waitUntil(promise: Promise<unknown>): void;
+};
+type Proxy<Req extends Request, Event extends WaitUntil> = (request: Req, event: Event) => Response | null | undefined | void | Promise<Response | null | undefined | void>;
+/**
+ * Wraps a Next.js `proxy.ts`. It adds no latency, because sending happens in
+ * `waitUntil` after the response is on its way.
+ *
+ * A proxy runs before the page renders, so a pass-through has no status or
+ * duration to report. Those stay empty rather than guessed. A status is only
+ * recorded when the proxy itself answered, such as a redirect or a 401.
+ *
+ *   export default withScoutline(clerkMiddleware());
+ *
+ * It also serves the browser script and its events on the site's own domain,
+ * ahead of the wrapped proxy, so a signed-out page can still report.
+ */
+export declare function withScoutline<Req extends Request, Event extends WaitUntil>(proxy?: Proxy<Req, Event>, options?: ScoutlineOptions): (request: Req, event: Event) => Promise<void | Response | null>;
+export {};

package/dist/next.js

@@ -0,0 +1,58 @@
+import { createScoutline, isDocumentRequest, serverTiming, } from "./index.js";
+/**
+ * Wraps a Next.js `proxy.ts`. It adds no latency, because sending happens in
+ * `waitUntil` after the response is on its way.
+ *
+ * A proxy runs before the page renders, so a pass-through has no status or
+ * duration to report. Those stay empty rather than guessed. A status is only
+ * recorded when the proxy itself answered, such as a redirect or a 401.
+ *
+ *   export default withScoutline(clerkMiddleware());
+ *
+ * It also serves the browser script and its events on the site's own domain,
+ * ahead of the wrapped proxy, so a signed-out page can still report.
+ */
+export function withScoutline(proxy, options) {
+    const scoutline = createScoutline(options);
+    return async function scoutlineProxy(request, event) {
+        // Before the wrapped proxy, so an auth check can't block the page's events.
+        const own = scoutline.handle(request);
+        if (own) {
+            const answer = await own;
+            event.waitUntil(scoutline.flush());
+            return answer;
+        }
+        let response;
+        try {
+            response = await proxy?.(request, event);
+        }
+        catch (error) {
+            scoutline.observe(request);
+            event.waitUntil(scoutline.flush());
+            throw error;
+        }
+        const answered = response instanceof Response &&
+            !response.headers.has("x-middleware-next") &&
+            !response.headers.has("x-middleware-rewrite");
+        const eventId = scoutline.observe(request, answered ? { status: response.status } : {});
+        event.waitUntil(scoutline.flush());
+        if (!eventId || answered || !isDocumentRequest(request))
+            return response;
+        return withServerId(response, eventId);
+    };
+}
+/**
+ * Hands the page its server event id, so the browser script can link to this
+ * request without a cookie. `x-middleware-next` is what `NextResponse.next()`
+ * sets, so a proxy with nothing to return can still pass the header on.
+ */
+function withServerId(response, id) {
+    try {
+        const target = response ?? new Response(null, { headers: { "x-middleware-next": "1" } });
+        target.headers.append("Server-Timing", serverTiming(id));
+        return target;
+    }
+    catch {
+        return response;
+    }
+}

package/dist/node.d.ts

@@ -0,0 +1,26 @@
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { type ScoutlineOptions } from "./index.js";
+/**
+ * Express middleware. Add it before your routes and body parsers.
+ *
+ *   app.use(scoutlineExpress());
+ */
+export declare function scoutlineExpress(options?: ScoutlineOptions): (request: IncomingMessage, response: ServerResponse, next: (error?: unknown) => void) => void;
+type FastifyLike = {
+    addHook(name: "onRequest", hook: (request: {
+        raw: IncomingMessage;
+    }, reply: {
+        raw: ServerResponse;
+        hijack(): void;
+    }, done: () => void) => void): void;
+};
+/**
+ * Fastify plugin. `skip-override` applies it to the whole app rather than only
+ * the plugin's own scope, the same thing `fastify-plugin` does.
+ *
+ *   app.register(scoutlineFastify);
+ */
+export declare const scoutlineFastify: ((app: FastifyLike, options?: ScoutlineOptions) => Promise<void>) & {
+    [x: symbol]: boolean;
+};
+export {};

package/dist/node.js

@@ -0,0 +1,120 @@
+import { EVENTS_PATH, SCRIPT_PATH } from "./browser.js";
+import { createScoutline, isDocumentRequest, serverTiming, } from "./index.js";
+const MAX_BODY_BYTES = 32 * 1024;
+function toRequest(request, body) {
+    const headers = new Headers();
+    for (const [name, value] of Object.entries(request.headers)) {
+        if (value !== undefined) {
+            headers.set(name, Array.isArray(value) ? value.join(", ") : value);
+        }
+    }
+    const host = headers.get("host") ?? "localhost";
+    const path = request.originalUrl ?? request.url ?? "/";
+    return new Request(`http://${host}${path}`, {
+        method: request.method,
+        headers,
+        body,
+    });
+}
+/** A body parser may have run first, and a stream can only be read once. */
+async function readBody(request) {
+    if (typeof request.body === "string")
+        return request.body;
+    if (request.body !== undefined)
+        return JSON.stringify(request.body);
+    const chunks = [];
+    let size = 0;
+    for await (const chunk of request) {
+        size += chunk.length;
+        if (size > MAX_BODY_BYTES)
+            break;
+        chunks.push(chunk);
+    }
+    return Buffer.concat(chunks).toString("utf8");
+}
+async function serveOwnRoute(scoutline, request, response) {
+    const body = request.method === "POST" ? await readBody(request) : undefined;
+    const answer = await scoutline.handle(toRequest(request, body));
+    const text = (await answer?.text()) ?? "";
+    response.writeHead(answer?.status ?? 204, Object.fromEntries(answer?.headers ?? []));
+    response.end(text);
+    void scoutline.flush();
+}
+function contentLength(response) {
+    const value = Number(response.getHeader("content-length"));
+    return Number.isFinite(value) && value > 0 ? value : null;
+}
+/**
+ * Records the request once the response has finished, so status and duration
+ * are real. A client that hangs up early never got a status, so none is sent.
+ * The event id goes out first, because headers can't change after that.
+ */
+function track(scoutline, request, response) {
+    const started = Date.now();
+    const web = toRequest(request);
+    const eventId = crypto.randomUUID();
+    if (isDocumentRequest(web)) {
+        response.appendHeader("Server-Timing", serverTiming(eventId));
+    }
+    let done = false;
+    const finish = (completed) => {
+        if (done)
+            return;
+        done = true;
+        scoutline.observe(web, {
+            eventId,
+            status: completed ? response.statusCode : null,
+            bytes: completed ? contentLength(response) : null,
+            durationMs: Date.now() - started,
+        });
+    };
+    response.once("finish", () => finish(true));
+    response.once("close", () => finish(false));
+}
+/** Returns true when the request was one of Scoutline's own routes. */
+function handleRequest(scoutline, request, response) {
+    const path = (request.originalUrl ?? request.url ?? "/").split("?")[0];
+    if (path === SCRIPT_PATH || path === EVENTS_PATH) {
+        serveOwnRoute(scoutline, request, response).catch(() => {
+            if (!response.headersSent)
+                response.writeHead(204);
+            response.end();
+        });
+        return true;
+    }
+    try {
+        track(scoutline, request, response);
+    }
+    catch {
+        // Observation is optional. The customer's request is not.
+    }
+    return false;
+}
+/**
+ * Express middleware. Add it before your routes and body parsers.
+ *
+ *   app.use(scoutlineExpress());
+ */
+export function scoutlineExpress(options) {
+    const scoutline = createScoutline(options);
+    return function scoutlineMiddleware(request, response, next) {
+        if (!handleRequest(scoutline, request, response))
+            next();
+    };
+}
+/**
+ * Fastify plugin. `skip-override` applies it to the whole app rather than only
+ * the plugin's own scope, the same thing `fastify-plugin` does.
+ *
+ *   app.register(scoutlineFastify);
+ */
+export const scoutlineFastify = Object.assign(async function scoutlinePlugin(app, options) {
+    const scoutline = createScoutline(options);
+    app.addHook("onRequest", (request, reply, done) => {
+        const path = (request.raw.url ?? "/").split("?")[0];
+        if (path === SCRIPT_PATH || path === EVENTS_PATH)
+            reply.hijack();
+        if (!handleRequest(scoutline, request.raw, reply.raw))
+            done();
+    });
+}, { [Symbol.for("skip-override")]: true });

package/package.json

@@ -1,6 +1,41 @@
 {
   "name": "@scoutline-ai/middleware",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.0",
+  "description": "In-path agent observation for Scoutline. Fails open and adds no work to the hot path.",
+  "type": "module",
+  "files": [
+    "dist"
+  ],
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./next": {
+      "types": "./dist/next.d.ts",
+      "default": "./dist/next.js"
+    },
+    "./fetch": {
+      "types": "./dist/fetch.d.ts",
+      "default": "./dist/fetch.js"
+    },
+    "./node": {
+      "types": "./dist/node.d.ts",
+      "default": "./dist/node.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "prepublishOnly": "tsc -p tsconfig.json"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "scoutline",
+    "ai agents",
+    "analytics",
+    "middleware"
+  ],
+  "homepage": "https://scoutline.ai"
+}