@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11

package/src/component/http.ts

@@ -1,30 +1,107 @@
 import { httpRouter } from "convex/server";
 import { ConvexError } from "convex/values";
 import type { Id } from "./_generated/dataModel.js";
-import { api } from "./_generated/api.js";
-import { httpAction } from "./_generated/server.js";
-import { MAX_BATCH_BYTES } from "./constants.js";
+import { api, internal } from "./_generated/api.js";
+import { env, httpAction } from "./_generated/server.js";
+import {
+  MAX_BATCH_BYTES,
+  MAX_BATCH_EVENT_GROUPS,
+  MAX_BATCH_EVENTS,
+  RATE_LIMIT_WINDOW_MS,
+} from "./constants.js";
+import {
+  classifyIp,
+  geoIpDailyLimit,
+  geoIpStatus,
+  isTrustedProxyRequest,
+  readRequestMetadata,
+  resolveClientIp,
+  resolveGeoIp,
+} from "./geo.js";
 import { isPlainRecord } from "./guards.js";
+import { alignClock, sanitizeEvent } from "./sanitize.js";
+import { deriveVisitorKey, randomSecret } from "./identity.js";
+import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
 import {
+  RASTRO_VERSION,
   TRACKER_GZIP_BASE64,
   TRACKER_GZIP_BYTES,
+  TRACKER_HASH,
+  VITALS_GZIP_BASE64,
+  VITALS_GZIP_BYTES,
+  VITALS_HASH,
 } from "../tracker/generated.js";
 
-const TRACKER_GZIP = Uint8Array.from(
-  atob(TRACKER_GZIP_BASE64),
-  (character) => character.charCodeAt(0),
+const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
+);
+const VITALS_GZIP = Uint8Array.from(atob(VITALS_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
 );
 
+// Five minutes fresh, then up to an hour of stale-while-revalidate: a visitor
+// who returns after a deploy gets at most one stale load before the browser's
+// background revalidation, and that revalidation is a 304 against the ETag.
+const SCRIPT_CACHE_CONTROL = "public, max-age=300, stale-while-revalidate=3600";
+
+/**
+ * Serves one pre-gzipped module with a strong ETag from its build-time content
+ * hash. A matching If-None-Match answers 304 with the same caching and CORS
+ * headers, so a revalidation costs a header exchange instead of the script.
+ */
+function scriptResponse(
+  request: Request,
+  body: Uint8Array<ArrayBuffer>,
+  byteLength: number,
+  hash: string,
+): Response {
+  const etag = `"${hash}"`;
+  const headers = {
+    ETag: etag,
+    "Cache-Control": SCRIPT_CACHE_CONTROL,
+    "Access-Control-Allow-Origin": "*",
+    Vary: "Accept-Encoding",
+  };
+  if (etagMatches(request.headers.get("if-none-match"), etag)) {
+    return new Response(null, { status: 304, headers });
+  }
+  return new Response(body, {
+    status: 200,
+    headers: {
+      ...headers,
+      "Content-Type": "text/javascript; charset=utf-8",
+      "Content-Encoding": "gzip",
+      "Content-Length": String(byteLength),
+    },
+  });
+}
+
+/** RFC 9110 If-None-Match: a list of entity tags or "*", weak markers ignored. */
+function etagMatches(header: string | null, etag: string): boolean {
+  if (!header) return false;
+  return header.split(",").some((candidate) => {
+    const value = candidate.trim().replace(/^W\//, "");
+    return value === "*" || value === etag;
+  });
+}
+
 const http = httpRouter();
 
 http.route({
   path: "/events",
   method: "POST",
   handler: httpAction(async (ctx, request) => {
+    const requestId = crypto.randomUUID();
     const origin = request.headers.get("origin");
-    if (!origin) return json({ error: { code: "ORIGIN_REQUIRED" } }, 400);
-    const headers = corsHeaders(origin);
+    if (!origin) {
+      return json(
+        { error: { code: "ORIGIN_REQUIRED" } },
+        400,
+        requestIdHeaders(requestId),
+      );
+    }
+    const headers = corsHeaders(origin, requestId);
     const contentLength = Number(request.headers.get("content-length"));
     if (Number.isFinite(contentLength) && contentLength > MAX_BATCH_BYTES) {
       return json(
@@ -34,7 +111,7 @@ http.route({
       );
     }
 
-    let payload: { siteId: string; events: TrackerEvent[] };
+    let payload: { siteId: string; events: TrackerEvent[]; sentAt?: number };
     try {
       const bytes = await readLimitedBody(request, MAX_BATCH_BYTES);
       const parsed: unknown = JSON.parse(new TextDecoder().decode(bytes));
@@ -51,30 +128,104 @@ http.route({
         {
           error: {
             code: "INVALID_BODY",
-            message: error instanceof Error ? error.message : "invalid JSON body",
+            message:
+              error instanceof Error ? error.message : "invalid JSON body",
           },
         },
         400,
         headers,
       );
     }
+    // The sender's clock says when its events happened, but a clock minutes
+    // off would keep the visitor out of the live view and file the traffic
+    // under another hour. The batch also says when it was sent, and that offset
+    // re-anchors every event here, before the transaction and the geo
+    // enrichment read them, so both see the same timestamps.
+    const events = alignClock(payload.events, payload.sentAt, Date.now());
 
-    const userAgent = request.headers.get("user-agent") ?? "";
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
+    const userAgent =
+      metadata.userAgent ?? request.headers.get("user-agent") ?? "";
+    // Self-declared bots never reach a transaction; the response keeps the
+    // counter shape the tracker expects and names the reason in a header.
+    if (isKnownBot(userAgent)) {
+      return json(
+        { accepted: 0, duplicates: 0, rejected: events.length },
+        200,
+        { ...headers, "X-Rastro-Rejected": "bot" },
+      );
+    }
+    // Anonymous batches (visitorId === sessionId) get a salted daily key so
+    // repeat visits count once; the IP itself stops here.
+    const anonymous = events.some(
+      (event) => event.visitorId === event.sessionId,
+    );
+    const visitorKey =
+      anonymous && clientIp.ip && userAgent
+        ? await deriveVisitorKey({
+            secret: await visitorSecret(ctx),
+            now: Date.now(),
+            siteId: payload.siteId,
+            ip: clientIp.ip,
+            userAgent,
+          })
+        : undefined;
     const context = {
-      country: firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]),
-      city: decodeHeader(firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"])),
-      latitude: numericHeader(request, ["x-vercel-ip-latitude", "cf-iplatitude"]),
-      longitude: numericHeader(request, ["x-vercel-ip-longitude", "cf-iplongitude"]),
-      ...classifyUserAgent(userAgent),
+      ...edgeGeo,
+      ...classifyClient(userAgent, request.headers),
+      ...(visitorKey ? { visitorKey } : {}),
     };
 
     try {
       const result = await ctx.runMutation(api.ingest.ingestBatch, {
         siteId: payload.siteId as Id<"sites">,
         origin,
-        events: payload.events,
+        events,
         context,
       });
+      if (
+        clientIp.ip &&
+        !hasCoordinates(edgeGeo) &&
+        geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
+          "ready"
+      ) {
+        const candidates = enrichmentCandidateEvents(events);
+        if (candidates) {
+          let stage: GeoIpFailureStage = "claim";
+          try {
+            const claim = await ctx.runMutation(internal.geo.claimEnrichment, {
+              siteId: payload.siteId as Id<"sites">,
+              events: candidates,
+              dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+            });
+            if (claim.sessionIds.length > 0) {
+              stage = "resolve";
+              const providerGeo = await resolveGeoIp(
+                clientIp.ip,
+                env.RASTRO_GEOIP_PROVIDER!,
+                env.RASTRO_GEOIP_TOKEN,
+              );
+              if (providerGeo) {
+                stage = "apply";
+                await ctx.runMutation(internal.geo.applyEnrichment, {
+                  siteId: payload.siteId as Id<"sites">,
+                  sessionIds: claim.sessionIds,
+                  events: claim.events,
+                  context: providerGeo,
+                });
+              } else {
+                logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
+              }
+            }
+          } catch {
+            logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
+          }
+        }
+      }
       return json(result, 200, headers);
     } catch (error) {
       if (error instanceof ConvexError) {
@@ -85,9 +236,23 @@ http.route({
             : data.code === "RATE_LIMITED"
               ? 429
               : 400;
-        return json({ error: data }, status, headers);
+        return json(
+          { error: data },
+          status,
+          status === 429
+            ? {
+                ...headers,
+                "Retry-After": String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1_000)),
+              }
+            : headers,
+        );
       }
-      return json({ error: { code: "INVALID_REQUEST" } }, 400, headers);
+      logInternalHttpError(requestId);
+      return json(
+        { error: { code: "INTERNAL_ERROR", requestId } },
+        500,
+        headers,
+      );
     }
   }),
 });
@@ -112,28 +277,65 @@ http.route({
 http.route({
   path: "/tracker.js",
   method: "GET",
-  handler: httpAction(async () => {
-    return new Response(TRACKER_GZIP, {
-      status: 200,
-      headers: {
-        "Content-Type": "text/javascript; charset=utf-8",
-        "Content-Encoding": "gzip",
-        "Content-Length": String(TRACKER_GZIP_BYTES),
-        "Cache-Control": "public, max-age=300, stale-while-revalidate=86400",
-        "Access-Control-Allow-Origin": "*",
-        Vary: "Accept-Encoding",
-      },
-    });
-  }),
+  handler: httpAction(async (_ctx, request) =>
+    scriptResponse(request, TRACKER_GZIP, TRACKER_GZIP_BYTES, TRACKER_HASH),
+  ),
 });
 
+http.route({
+  path: "/vitals.js",
+  method: "GET",
+  handler: httpAction(async (_ctx, request) =>
+    scriptResponse(request, VITALS_GZIP, VITALS_GZIP_BYTES, VITALS_HASH),
+  ),
+});
 
 http.route({
   path: "/health",
   method: "GET",
-  handler: httpAction(async () => {
+  handler: httpAction(async (ctx, request) => {
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const provider = env.RASTRO_GEOIP_PROVIDER;
     return json(
-      { ok: true, component: "@iann29/rastro" },
+      {
+        ok: true,
+        component: "@iann29/rastro",
+        requestMetadata: {
+          runtimeIpKind: classifyIp(metadata.ip),
+          clientIpAvailable: clientIp.ip !== null,
+          clientIpSource: clientIp.source,
+          userAgentAvailable: metadata.userAgent !== null,
+        },
+        geoIp: {
+          provider: provider ?? null,
+          status: geoIpStatus(provider, env.RASTRO_GEOIP_TOKEN),
+          dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+          trustedProxy: trustProxy,
+          proxyRequestTrusted,
+          edgeHeadersAvailable:
+            proxyRequestTrusted &&
+            Object.keys(geoFromHeaders(request)).length > 0,
+        },
+        features: {
+          botDetection: "userAgent",
+          uniqueVisitors:
+            clientIp.ip &&
+            (metadata.userAgent ?? request.headers.get("user-agent"))
+              ? "dailyHash"
+              : "session",
+          webVitals: "optIn",
+          siteMap: "dailyRollup",
+          liveDepartures: "leaveBeacon",
+        },
+        tracker: {
+          version: RASTRO_VERSION,
+          hash: TRACKER_HASH,
+          vitalsHash: VITALS_HASH,
+        },
+      },
       200,
       {
         "Access-Control-Allow-Origin": "*",
@@ -143,11 +345,24 @@ http.route({
   }),
 });
 
-function parsePayload(value: unknown): { siteId: string; events: TrackerEvent[] } {
-  if (!isPlainRecord(value) || typeof value.siteId !== "string" || !Array.isArray(value.events)) {
+function parsePayload(value: unknown): {
+  siteId: string;
+  events: TrackerEvent[];
+  sentAt?: number;
+} {
+  if (
+    !isPlainRecord(value) ||
+    typeof value.siteId !== "string" ||
+    !Array.isArray(value.events)
+  ) {
     throw new Error("body must contain siteId and events");
   }
-  return { siteId: value.siteId, events: value.events.map(parseEvent) };
+  const sentAt = optionalNumber(value.sentAt, "sentAt");
+  return {
+    siteId: value.siteId,
+    events: value.events.map(parseEvent),
+    ...(sentAt === undefined ? {} : { sentAt }),
+  };
 }
 
 class PayloadTooLargeError extends Error {}
@@ -188,7 +403,9 @@ function parseEvent(value: unknown): TrackerEvent {
     type !== "custom" &&
     type !== "conversion" &&
     type !== "heartbeat" &&
-    type !== "outbound"
+    type !== "leave" &&
+    type !== "outbound" &&
+    type !== "vital"
   ) {
     throw new Error("event type is invalid");
   }
@@ -233,6 +450,9 @@ function parseEvent(value: unknown): TrackerEvent {
     ...(optionalString(value.affiliateSlug, "affiliateSlug") !== undefined
       ? { affiliateSlug: optionalString(value.affiliateSlug, "affiliateSlug") }
       : {}),
+    ...(optionalNumber(value.value, "value") !== undefined
+      ? { value: optionalNumber(value.value, "value") }
+      : {}),
   };
 }
 
@@ -266,6 +486,90 @@ function optionalNumber(value: unknown, field: string): number | undefined {
   return value;
 }
 
+function enrichmentCandidateEvents(events: TrackerEvent[]):
+  | Array<{
+      eventId: string;
+      sessionId: string;
+      visitorId: string;
+      timestamp: number;
+      sequence: number;
+    }>
+  | undefined {
+  if (events.length === 0 || events.length > MAX_BATCH_EVENTS) return undefined;
+  const now = Date.now();
+  const sanitized = events.flatMap((event) => {
+    try {
+      return [sanitizeEvent(event, now)];
+    } catch {
+      return [];
+    }
+  });
+  const sessionIds = [...new Set(sanitized.map((event) => event.sessionId))];
+  if (
+    !sanitized.some((event) => event.sequence === 0) ||
+    sessionIds.length > MAX_BATCH_EVENT_GROUPS ||
+    sessionIds.some(
+      (sessionId) => sessionId.length === 0 || sessionId.length > 128,
+    )
+  ) {
+    return undefined;
+  }
+  return sanitized.map(
+    ({ eventId, sessionId, visitorId, timestamp, sequence }) => ({
+      eventId,
+      sessionId,
+      visitorId,
+      timestamp,
+      sequence,
+    }),
+  );
+}
+
+function geoFromHeaders(request: Request): {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+} {
+  const country = firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]);
+  const city = decodeHeader(
+    firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"]),
+  );
+  const latitude = numericHeader(request, [
+    "x-vercel-ip-latitude",
+    "cf-iplatitude",
+  ]);
+  const longitude = numericHeader(request, [
+    "x-vercel-ip-longitude",
+    "cf-iplongitude",
+  ]);
+  return {
+    ...(country ? { country } : {}),
+    ...(city ? { city } : {}),
+    ...(validCoordinates(latitude, longitude) ? { latitude, longitude } : {}),
+  };
+}
+
+function hasCoordinates(value: {
+  latitude?: number;
+  longitude?: number;
+}): boolean {
+  return value.latitude !== undefined && value.longitude !== undefined;
+}
+
+function validCoordinates(
+  latitude: number | undefined,
+  longitude: number | undefined,
+): latitude is number {
+  return (
+    latitude !== undefined &&
+    longitude !== undefined &&
+    latitude >= -90 &&
+    latitude <= 90 &&
+    longitude >= -180 &&
+    longitude <= 180
+  );
+}
 
 function firstHeader(request: Request, names: string[]): string | undefined {
   for (const name of names) {
@@ -291,45 +595,52 @@ function decodeHeader(value: string | undefined): string | undefined {
   }
 }
 
-function classifyUserAgent(userAgent: string): {
-  browser: string;
-  os: string;
-  device: string;
-} {
-  const browser = /Edg\//.test(userAgent)
-    ? "Edge"
-    : /Firefox\//.test(userAgent)
-      ? "Firefox"
-      : /Chrome\//.test(userAgent)
-        ? "Chrome"
-        : /Safari\//.test(userAgent)
-          ? "Safari"
-          : "Other";
-  const os = /Android/.test(userAgent)
-    ? "Android"
-    : /iPhone|iPad/.test(userAgent)
-      ? "iOS"
-      : /Windows/.test(userAgent)
-        ? "Windows"
-        : /Mac OS X/.test(userAgent)
-          ? "macOS"
-          : /Linux/.test(userAgent)
-            ? "Linux"
-            : "Other";
-  const device = /iPad|Tablet/.test(userAgent)
-    ? "tablet"
-    : /Mobile|Android|iPhone/.test(userAgent)
-      ? "mobile"
-      : "desktop";
-  return { browser, os, device };
+type GeoIpFailureStage = "claim" | "resolve" | "apply";
+
+function logInternalHttpError(requestId: string): void {
+  console.error(
+    JSON.stringify({
+      level: "error",
+      event: "rastro.http.internal_error",
+      requestId,
+      route: "/events",
+      method: "POST",
+    }),
+  );
 }
 
-function corsHeaders(origin: string): HeadersInit {
+function logGeoIpFailure(
+  requestId: string,
+  stage: GeoIpFailureStage,
+  provider: "ipinfo" | "ipwhois",
+): void {
+  console.warn(
+    JSON.stringify({
+      level: "warn",
+      event: "rastro.geoip.failure",
+      requestId,
+      route: "/events",
+      stage,
+      provider,
+    }),
+  );
+}
+
+function requestIdHeaders(requestId: string): Record<string, string> {
+  return { "X-Request-ID": requestId };
+}
+
+function corsHeaders(
+  origin: string,
+  requestId?: string,
+): Record<string, string> {
   return {
     "Access-Control-Allow-Origin": origin,
     "Access-Control-Allow-Methods": "POST, OPTIONS",
     "Access-Control-Allow-Headers": "Content-Type",
+    "Access-Control-Expose-Headers": "X-Request-ID, Retry-After",
     "Access-Control-Max-Age": "86400",
+    ...(requestId ? requestIdHeaders(requestId) : {}),
     Vary: "Origin",
   };
 }
@@ -342,3 +653,26 @@ function json(value: unknown, status: number, headers?: HeadersInit): Response {
 }
 
 export default http;
+
+let cachedVisitorSecret: string | undefined;
+
+/** The per-deployment salt behind anonymous visitor keys, created on first use. */
+async function visitorSecret(ctx: {
+  runQuery: (
+    reference: typeof internal.identity.secret,
+    args: Record<string, never>,
+  ) => Promise<string | null>;
+  runMutation: (
+    reference: typeof internal.identity.ensureSecret,
+    args: { candidate: string },
+  ) => Promise<string>;
+}): Promise<string> {
+  if (cachedVisitorSecret) return cachedVisitorSecret;
+  const stored = await ctx.runQuery(internal.identity.secret, {});
+  cachedVisitorSecret =
+    stored ??
+    (await ctx.runMutation(internal.identity.ensureSecret, {
+      candidate: randomSecret(),
+    }));
+  return cachedVisitorSecret;
+}

package/src/component/identity.ts

@@ -0,0 +1,74 @@
+import { v } from "convex/values";
+import { DAY_MS } from "./constants.js";
+import { internalMutation, internalQuery } from "./_generated/server.js";
+
+/**
+ * Anonymous visitors (no host-supplied id) are counted through a salted hash of
+ * the request IP and User-Agent that rotates every UTC day. The hash is derived
+ * in the HTTP action, so the raw IP never reaches a mutation or a table, and the
+ * salt is a per-deployment secret, so nobody holding a key can walk the IPv4
+ * space back to an address. Same device and network on the same day count once;
+ * the next day they are a new visitor by design.
+ */
+const KEY_HEX_LENGTH = 24;
+const SECRET_PATTERN = /^[0-9a-f]{64}$/;
+
+export const secret = internalQuery({
+  args: {},
+  returns: v.union(v.string(), v.null()),
+  handler: async (ctx) => {
+    const row = await ctx.db
+      .query("visitorSecrets")
+      .withIndex("by_key", (range) => range.eq("key", "current"))
+      .unique();
+    return row?.value ?? null;
+  },
+});
+
+export const ensureSecret = internalMutation({
+  args: { candidate: v.string() },
+  returns: v.string(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db
+      .query("visitorSecrets")
+      .withIndex("by_key", (range) => range.eq("key", "current"))
+      .unique();
+    if (existing) return existing.value;
+    if (!SECRET_PATTERN.test(args.candidate)) {
+      throw new Error("visitor secret candidate must be 64 hex characters");
+    }
+    await ctx.db.insert("visitorSecrets", {
+      key: "current",
+      value: args.candidate,
+      createdAt: Date.now(),
+    });
+    return args.candidate;
+  },
+});
+
+export function randomSecret(): string {
+  return hex(crypto.getRandomValues(new Uint8Array(32)));
+}
+
+export async function deriveVisitorKey(input: {
+  secret: string;
+  now: number;
+  siteId: string;
+  ip: string;
+  userAgent: string;
+}): Promise<string> {
+  const dayStart = Math.floor(input.now / DAY_MS) * DAY_MS;
+  const material = new TextEncoder().encode(
+    [input.secret, dayStart, input.siteId, input.ip, input.userAgent].join(" "),
+  );
+  const digest = new Uint8Array(
+    await crypto.subtle.digest("SHA-256", material),
+  );
+  return hex(digest).slice(0, KEY_HEX_LENGTH);
+}
+
+function hex(bytes: Uint8Array): string {
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+    "",
+  );
+}