@katrinalaszlo/track 0.4.0 → 0.5.0

package/README.md

@@ -31,7 +31,19 @@ app.use(agentTraffic({ accountIdOf: (req, res) => req.user?.id }));
 
 Register it right after creating the app, before auth and routes. `accountIdOf` returns your customer's id (hashed before it leaves your server), or undefined. Optional `errorOf(req, res)` returns error text for failed responses (400 and above, up to 200 characters), so two errors with the same status show separately. Keys come from `AGENT_TRAFFIC_KEY`, `AGENT_TRAFFIC_PROJECT_ID` and `AGENT_TRAFFIC_ACCOUNT_SALT` in the environment or `.env`; get them with the signup in [SKILL.md](skills/agent-traffic/SKILL.md).
 
-Per request it sends the route pattern, method, status code, the `User-Agent`, `Signature-Agent` and `Accept-Language` headers, the time, the hashed customer id, and the text `errorOf` returns if you set it. When an agent signed its request (Web Bot Auth, `Signature` and `Signature-Input` headers present) it also sends those two headers, the request's full URL, and the other headers the signature covers, so the signature can be checked; never `Authorization` or `Cookie`, whatever the signer covered. Otherwise no bodies, no other headers, no IPs. To see which agents lead to signups and payments, call `traffic.recordOutcome({ accountId, name: "paid", value: 1999, currency: "USD", idempotencyKey: paymentEvent.id })` from your payment webhook (and `name: "signup"` from your signup handler): the outcome joins the agent session of the same customer. Events go out in batches once a second (at once on Vercel, Lambda and Netlify), and it never blocks or fails a request. No dependencies; TypeScript types included.
+Per request it sends the route pattern, method, status code, the `User-Agent`, `Signature-Agent` and `Accept-Language` headers, the time, the hashed customer id, the journey key if the request carried one (see below), and the text `errorOf` returns if you set it. When an agent signed its request (Web Bot Auth, `Signature` and `Signature-Input` headers present) it also sends those two headers, the request's full URL, and the other headers the signature covers, so the signature can be checked; never `Authorization` or `Cookie`, whatever the signer covered. Otherwise no bodies, no other headers, no IPs. To see which agents lead to signups and payments, call `traffic.recordOutcome({ accountId, name: "paid", value: 1999, currency: "USD", idempotencyKey: paymentEvent.id })` from your payment webhook (and `name: "signup"` from your signup handler): the outcome joins the agent session of the same customer. Events go out in batches once a second (at once on Vercel, Lambda and Netlify), and it never blocks or fails a request. No dependencies; TypeScript types included.
+
+### Journey keys
+
+A journey key ties one visitor's requests together across sessions, for example an agent that reads your docs today and signs up tomorrow. The middleware reads it from each request in this order: the `Track-Journey` header, the `?track_j=` query parameter, then the `track_j` cookie. The first value of 8 to 128 letters, digits, `_` or `-` wins and is sent as `journey_id`, with `journey_source` set to `header`, `query` or `cookie`. A value that does not fit is ignored: not sent, and never an error for the request. With no key, neither field is sent (unless `issueJourneyKeys` is on, below). A journey key is never written to a log.
+
+To hand out keys yourself, turn on `issueJourneyKeys`:
+
+```js
+app.use(agentTraffic({ accountIdOf: (req, res) => req.user?.id, issueJourneyKeys: true }));
+```
+
+Then every response carries the visitor's key: the one the request sent, or a new one (`j_` plus 32 hex characters) when it sent none. It goes out as the `Track-Journey` response header and as `Set-Cookie: track_j=<key>; Path=/; Max-Age=1209600; SameSite=Lax; Secure`, with `Cache-Control: private` so a shared cache never serves one visitor's key to another. It is off by default because it sets a cookie on your responses and changes their caching: that can need a line in your cookie policy or consent banner, and a CDN will stop caching those responses. Both are set before your routes run, so a route that sets its own `Cache-Control` replaces `private` (keep it private on any response that carries a key), and a raw `res.setHeader("Set-Cookie", ...)` replaces the `track_j` cookie (Express's `res.cookie` and express-session add to it instead). The request that receives a new key is sent with that key as `journey_id` and `journey_source: "issued"`, so it is the first step of the journey.
 
 Event names use the matched route's pattern and exclude router mount prefixes, which can contain raw IDs. A router mounted at `/customers/:customerId` with route `/orders/:orderId` records `orders_orderid` for every customer. Routes with the same pattern under different mounts share an event name.
 

package/index.d.ts

@@ -13,6 +13,24 @@ export interface AgentTrafficOptions {
   salt?: string;
   /** Defaults to AGENT_TRAFFIC_ENDPOINT, else https://alg.katrinalaszlo.com. */
   endpoint?: string;
+  /**
+   * Off by default. When true, every response carries the visitor's journey key: the one the request sent (Track-Journey
+   * header, ?track_j= query parameter or track_j cookie), else a new "j_" + 32 hex key. It is set as the Track-Journey
+   * response header and as `Set-Cookie: track_j=<key>; Path=/; Max-Age=1209600; SameSite=Lax; Secure`, with
+   * `Cache-Control: private`. Opt-in because it sets a cookie on your responses and stops shared caches from storing them.
+   */
+  issueJourneyKeys?: boolean;
+}
+
+/** The request and response header that carries a journey key: "Track-Journey". */
+export const JOURNEY_HEADER: string;
+
+/** Fields the middleware adds to an event when the request carried a valid journey key; both or neither. */
+export interface JourneyFields {
+  /** 8 to 128 characters of A-Z, a-z, 0-9, _ and -. */
+  journey_id: string;
+  /** "issued" when issueJourneyKeys minted the key on this request's response. */
+  journey_source: "header" | "query" | "cookie" | "issued";
 }
 
 export interface OutcomeInput {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@katrinalaszlo/track",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Express middleware that shows which AI agents call your API, how their sessions go, and which errors end them.",
   "license": "MIT",
   "type": "module",

package/skills/agent-traffic/agent-traffic.cjs

@@ -14,14 +14,23 @@
 // two errors with the same status show as different rows in Fix next. AGENT_TRAFFIC_KEY,
 // AGENT_TRAFFIC_PROJECT_ID and AGENT_TRAFFIC_ACCOUNT_SALT come from the environment, or from a .env in the working
 // directory, this file's folder or any folder above it, when the environment lacks them; only AGENT_TRAFFIC_ lines are
-// read, nothing else in .env is touched.
+// read, nothing else in .env is touched. issueJourneyKeys, optional and off by default, gives each visitor a journey key
+// on the response (a header and a track_j cookie) so its later requests join one journey.
 //
 // Register it right after creating the app, before auth and routes. It never blocks or fails a request: events are
 // queued, sent in batches of up to 25 once a second, and a failed batch is logged and retried with the same keys.
-const { createHmac, randomUUID } = require("node:crypto");
+const { createHmac, randomBytes, randomUUID } = require("node:crypto");
 const { existsSync, readFileSync } = require("node:fs");
 const { dirname, join } = require("node:path");
 
+// The request and response header that carries a journey key.
+const JOURNEY_HEADER = "Track-Journey";
+// The cookie name and the query parameter name.
+const JOURNEY_PARAM = "track_j";
+const JOURNEY_KEY = /^[A-Za-z0-9_-]{8,128}$/;
+// 14 days. Secure so the key never travels over plain http; Lax so it comes along when a person follows a link in.
+const JOURNEY_COOKIE_ATTRIBUTES = "Path=/; Max-Age=1209600; SameSite=Lax; Secure";
+
 const BATCH_MAX = 25;
 const OUTCOME_NAMES = ["signup", "key_created", "activation", "paid", "custom"];
 const FLUSH_MS = 1000;
@@ -107,7 +116,42 @@ function signatureMaterial(req) {
   };
 }
 
-function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint } = {}) {
+// A journey key ties requests from one visitor together across sessions: the journey header, else ?track_j= in the URL,
+// else the track_j cookie. The first value that fits the ingest rule wins; one that does not is ignored, never sent.
+function journeyKeyOf(req) {
+  const header = req.headers[JOURNEY_HEADER.toLowerCase()];
+  if (typeof header === "string" && JOURNEY_KEY.test(header)) return { journey_id: header, journey_source: "header" };
+
+  const url = req.originalUrl ?? req.url ?? "";
+  const queryStart = url.indexOf("?");
+  if (queryStart !== -1) {
+    const fromQuery = new URLSearchParams(url.slice(queryStart + 1)).getAll(JOURNEY_PARAM).find((value) => JOURNEY_KEY.test(value));
+    if (fromQuery) return { journey_id: fromQuery, journey_source: "query" };
+  }
+
+  const cookieHeader = req.headers.cookie;
+  if (typeof cookieHeader === "string") {
+    for (const pair of cookieHeader.split(";")) {
+      const eq = pair.indexOf("=");
+      if (eq === -1 || pair.slice(0, eq).trim() !== JOURNEY_PARAM) continue;
+      const value = pair.slice(eq + 1).trim();
+      if (JOURNEY_KEY.test(value)) return { journey_id: value, journey_source: "cookie" };
+    }
+  }
+  return undefined;
+}
+
+// Cache-Control: private so a shared cache never hands one visitor's key to another. A Set-Cookie an earlier
+// middleware added is kept.
+function setJourneyKey(res, journeyId) {
+  res.setHeader(JOURNEY_HEADER, journeyId);
+  const cookie = `${JOURNEY_PARAM}=${journeyId}; ${JOURNEY_COOKIE_ATTRIBUTES}`;
+  const existing = res.getHeader("set-cookie");
+  res.setHeader("set-cookie", existing === undefined ? cookie : [].concat(existing, cookie));
+  res.setHeader("cache-control", "private");
+}
+
+function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint, issueJourneyKeys = false } = {}) {
   const env = agentTrafficEnv();
   key ??= env.AGENT_TRAFFIC_KEY;
   projectId ??= env.AGENT_TRAFFIC_PROJECT_ID;
@@ -146,7 +190,7 @@ function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint } =
           byReason.set(label, (byReason.get(label) ?? 0) + 1);
         }
         for (const [label, n] of byReason) {
-          const hint = label.startsWith("unknown_field") ? ". The agent traffic host does not know this field yet: it is older than this package (0.3.0). Upgrade the host, or pin @katrinalaszlo/track to 0.2.x. https://alg.katrinalaszlo.com/docs#event-fields" : "";
+          const hint = label.startsWith("unknown_field") ? ". The agent traffic host does not know this field yet: it is older than this package (0.5.0). Upgrade the host, or pin @katrinalaszlo/track to 0.4.x. https://alg.katrinalaszlo.com/docs#event-fields" : "";
           console.error(`agent-traffic: ${n} of ${batch.length} events rejected: ${label}${hint}`);
         }
         return;
@@ -209,14 +253,18 @@ function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint } =
     const occurredAt = new Date().toISOString();
     // Read while the request is current: the context is per request.
     const vercel = globalThis[VERCEL_CONTEXT]?.get?.();
+    let journey = journeyKeyOf(req);
+    if (issueJourneyKeys && !journey) journey = { journey_id: `j_${randomBytes(16).toString("hex")}`, journey_source: "issued" };
+    if (issueJourneyKeys) setJourneyKey(res, journey.journey_id);
     res.on("finish", () => {
       // An exception here would be thrown outside the request, so it is logged instead: the event is lost, the
       // server is not.
       let event;
       try {
-        event = eventOf(req, res, occurredAt);
+        event = eventOf(req, res, occurredAt, journey);
       } catch (err) {
-        console.error(`agent-traffic: event not recorded for ${req.method} ${req.originalUrl ?? req.url}: ${err.stack}`);
+        // The path only: the query string can hold a journey key, which never goes in a log.
+        console.error(`agent-traffic: event not recorded for ${req.method} ${(req.originalUrl ?? req.url ?? "").split("?")[0]}: ${err.stack}`);
         return;
       }
       if (SERVERLESS || vercel?.waitUntil) {
@@ -233,7 +281,7 @@ function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint } =
   agentTrafficMiddleware.recordOutcome = recordOutcome;
   return agentTrafficMiddleware;
 
-  function eventOf(req, res, occurredAt) {
+  function eventOf(req, res, occurredAt, journey) {
     const accountId = accountIdOf ? accountIdOf(req, res) : undefined;
     // Only for failed responses: the text that tells one 402 from another in Fix next. Capped so a long message cannot
     // crowd a batch.
@@ -248,9 +296,10 @@ function agentTraffic({ accountIdOf, errorOf, key, projectId, salt, endpoint } =
       accept_language: req.headers["accept-language"],
       signature_agent: req.headers["signature-agent"],
       ...signatureMaterial(req),
+      ...journey,
       properties: { status_code: res.statusCode, method: req.method, ...(typeof error === "string" && error !== "" ? { error: error.slice(0, ERROR_MAX) } : {}) },
     };
   }
 }
 
-module.exports = { eventName, agentTraffic, signatureMaterial };
+module.exports = { eventName, agentTraffic, signatureMaterial, JOURNEY_HEADER };