@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/src/component/goals.ts

@@ -8,10 +8,7 @@ import {
   sanitizePath,
   sanitizeSlug,
 } from "./sanitize.js";
-import {
-  goalFieldsValidator,
-  goalMatchTypeValidator,
-} from "./validators.js";
+import { goalFieldsValidator, goalMatchTypeValidator } from "./validators.js";
 
 const goalDocumentValidator = goalFieldsValidator.extend({
   _id: v.id("goals"),
@@ -31,14 +28,16 @@ export const upsert = mutation({
   },
   returns: v.id("goals"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
     let key: string;
     let matchValue: string;
     try {
       key = sanitizeSlug(args.key, "key");
-      matchValue = args.matchType === "pageview"
-        ? sanitizePath(args.matchValue)
-        : cleanString(args.matchValue, 80);
+      matchValue =
+        args.matchType === "pageview"
+          ? sanitizePath(args.matchValue)
+          : cleanString(args.matchValue, 80);
     } catch (error) {
       fail(
         "INVALID_ARGUMENT",
@@ -49,7 +48,10 @@ export const upsert = mutation({
     if (!name) fail("INVALID_ARGUMENT", "goal name is required");
     if (!matchValue) fail("INVALID_ARGUMENT", "goal match value is required");
     if (args.valueCents !== undefined && !isValidMoney(args.valueCents)) {
-      fail("INVALID_ARGUMENT", "valueCents must be an integer between 0 and 1000000000");
+      fail(
+        "INVALID_ARGUMENT",
+        "valueCents must be an integer between 0 and 1000000000",
+      );
     }
 
     const existingByKey = await ctx.db
@@ -62,10 +64,27 @@ export const upsert = mutation({
     if (args.goalId && existingByKey && existingByKey._id !== args.goalId) {
       fail("CONFLICT", "another goal already uses this key");
     }
+    const goals = await ctx.db
+      .query("goals")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .take(MAX_GOALS_PER_SITE + 1);
     const now = Date.now();
     if (goalId) {
       const goal = await ctx.db.get("goals", goalId);
-      if (!goal || goal.siteId !== args.siteId) fail("NOT_FOUND", "goal not found");
+      if (!goal || goal.siteId !== args.siteId)
+        fail("NOT_FOUND", "goal not found");
+      if (
+        (args.active ?? goal.active) &&
+        goals.some(
+          (candidate) =>
+            candidate._id !== goalId &&
+            candidate.active &&
+            candidate.matchType === args.matchType &&
+            candidate.matchValue === matchValue,
+        )
+      ) {
+        fail("CONFLICT", "another active goal already uses this matcher");
+      }
       await ctx.db.patch("goals", goalId, {
         key,
         name,
@@ -78,12 +97,21 @@ export const upsert = mutation({
       return goalId;
     }
 
-    const goals = await ctx.db
-      .query("goals")
-      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
-      .take(MAX_GOALS_PER_SITE + 1);
+    if (
+      (args.active ?? true) &&
+      goals.some(
+        (candidate) =>
+          candidate.active &&
+          candidate.matchType === args.matchType &&
+          candidate.matchValue === matchValue,
+      )
+    ) {
+      fail("CONFLICT", "another active goal already uses this matcher");
+    }
     if (goals.length >= MAX_GOALS_PER_SITE) {
-      fail("LIMIT_EXCEEDED", "goal limit reached", { limit: MAX_GOALS_PER_SITE });
+      fail("LIMIT_EXCEEDED", "goal limit reached", {
+        limit: MAX_GOALS_PER_SITE,
+      });
     }
     return await ctx.db.insert("goals", {
       siteId: args.siteId,
@@ -115,7 +143,8 @@ export const remove = mutation({
   returns: v.null(),
   handler: async (ctx, args) => {
     const goal = await ctx.db.get("goals", args.goalId);
-    if (!goal || goal.siteId !== args.siteId) fail("NOT_FOUND", "goal not found");
+    if (!goal || goal.siteId !== args.siteId)
+      fail("NOT_FOUND", "goal not found");
     await ctx.db.delete("goals", args.goalId);
     return null;
   },

package/src/component/guards.ts

@@ -1,3 +1,5 @@
-export function isPlainRecord(value: unknown): value is Record<string, unknown> {
+export function isPlainRecord(
+  value: unknown,
+): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }

package/src/component/http.ts

@@ -1,19 +1,40 @@
 import { httpRouter } from "convex/server";
 import { ConvexError } from "convex/values";
 import type { Id } from "./_generated/dataModel.js";
-import { api } from "./_generated/api.js";
-import { httpAction } from "./_generated/server.js";
-import { MAX_BATCH_BYTES } from "./constants.js";
+import { api, internal } from "./_generated/api.js";
+import { env, httpAction } from "./_generated/server.js";
+import {
+  MAX_BATCH_BYTES,
+  MAX_BATCH_EVENT_GROUPS,
+  MAX_BATCH_EVENTS,
+  RATE_LIMIT_WINDOW_MS,
+} from "./constants.js";
+import {
+  classifyIp,
+  geoIpDailyLimit,
+  geoIpStatus,
+  isTrustedProxyRequest,
+  readRequestMetadata,
+  resolveClientIp,
+  resolveGeoIp,
+} from "./geo.js";
 import { isPlainRecord } from "./guards.js";
+import { alignClock, sanitizeEvent } from "./sanitize.js";
+import { deriveVisitorKey, randomSecret } from "./identity.js";
+import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
 import {
   TRACKER_GZIP_BASE64,
   TRACKER_GZIP_BYTES,
+  VITALS_GZIP_BASE64,
+  VITALS_GZIP_BYTES,
 } from "../tracker/generated.js";
 
-const TRACKER_GZIP = Uint8Array.from(
-  atob(TRACKER_GZIP_BASE64),
-  (character) => character.charCodeAt(0),
+const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
+);
+const VITALS_GZIP = Uint8Array.from(atob(VITALS_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
 );
 
 const http = httpRouter();
@@ -22,46 +43,141 @@ http.route({
   path: "/events",
   method: "POST",
   handler: httpAction(async (ctx, request) => {
+    const requestId = crypto.randomUUID();
     const origin = request.headers.get("origin");
-    if (!origin) return json({ error: { code: "ORIGIN_REQUIRED" } }, 400);
-    const bytes = await request.arrayBuffer();
-    if (bytes.byteLength > MAX_BATCH_BYTES) {
-      return json({ error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } }, 413);
+    if (!origin) {
+      return json(
+        { error: { code: "ORIGIN_REQUIRED" } },
+        400,
+        requestIdHeaders(requestId),
+      );
+    }
+    const headers = corsHeaders(origin, requestId);
+    const contentLength = Number(request.headers.get("content-length"));
+    if (Number.isFinite(contentLength) && contentLength > MAX_BATCH_BYTES) {
+      return json(
+        { error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } },
+        413,
+        headers,
+      );
     }
 
-    let payload: { siteId: string; events: TrackerEvent[] };
+    let payload: { siteId: string; events: TrackerEvent[]; sentAt?: number };
     try {
+      const bytes = await readLimitedBody(request, MAX_BATCH_BYTES);
       const parsed: unknown = JSON.parse(new TextDecoder().decode(bytes));
       payload = parsePayload(parsed);
     } catch (error) {
+      if (error instanceof PayloadTooLargeError) {
+        return json(
+          { error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } },
+          413,
+          headers,
+        );
+      }
       return json(
         {
           error: {
             code: "INVALID_BODY",
-            message: error instanceof Error ? error.message : "invalid JSON body",
+            message:
+              error instanceof Error ? error.message : "invalid JSON body",
           },
         },
         400,
+        headers,
       );
     }
+    // The sender's clock says when its events happened, but a clock minutes
+    // off would keep the visitor out of the live view and file the traffic
+    // under another hour. The batch also says when it was sent, and that offset
+    // re-anchors every event here, before the transaction and the geo
+    // enrichment read them, so both see the same timestamps.
+    const events = alignClock(payload.events, payload.sentAt, Date.now());
 
-    const userAgent = request.headers.get("user-agent") ?? "";
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
+    const userAgent =
+      metadata.userAgent ?? request.headers.get("user-agent") ?? "";
+    // Self-declared bots never reach a transaction; the response keeps the
+    // counter shape the tracker expects and names the reason in a header.
+    if (isKnownBot(userAgent)) {
+      return json(
+        { accepted: 0, duplicates: 0, rejected: events.length },
+        200,
+        { ...headers, "X-Rastro-Rejected": "bot" },
+      );
+    }
+    // Anonymous batches (visitorId === sessionId) get a salted daily key so
+    // repeat visits count once; the IP itself stops here.
+    const anonymous = events.some(
+      (event) => event.visitorId === event.sessionId,
+    );
+    const visitorKey =
+      anonymous && clientIp.ip && userAgent
+        ? await deriveVisitorKey({
+            secret: await visitorSecret(ctx),
+            now: Date.now(),
+            siteId: payload.siteId,
+            ip: clientIp.ip,
+            userAgent,
+          })
+        : undefined;
     const context = {
-      country: firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]),
-      city: decodeHeader(firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"])),
-      latitude: numericHeader(request, ["x-vercel-ip-latitude", "cf-iplatitude"]),
-      longitude: numericHeader(request, ["x-vercel-ip-longitude", "cf-iplongitude"]),
-      ...classifyUserAgent(userAgent),
+      ...edgeGeo,
+      ...classifyClient(userAgent, request.headers),
+      ...(visitorKey ? { visitorKey } : {}),
     };
 
     try {
       const result = await ctx.runMutation(api.ingest.ingestBatch, {
         siteId: payload.siteId as Id<"sites">,
         origin,
-        events: payload.events,
+        events,
         context,
       });
-      return json(result, 200, corsHeaders(origin));
+      if (
+        clientIp.ip &&
+        !hasCoordinates(edgeGeo) &&
+        geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
+          "ready"
+      ) {
+        const candidates = enrichmentCandidateEvents(events);
+        if (candidates) {
+          let stage: GeoIpFailureStage = "claim";
+          try {
+            const claim = await ctx.runMutation(internal.geo.claimEnrichment, {
+              siteId: payload.siteId as Id<"sites">,
+              events: candidates,
+              dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+            });
+            if (claim.sessionIds.length > 0) {
+              stage = "resolve";
+              const providerGeo = await resolveGeoIp(
+                clientIp.ip,
+                env.RASTRO_GEOIP_PROVIDER!,
+                env.RASTRO_GEOIP_TOKEN,
+              );
+              if (providerGeo) {
+                stage = "apply";
+                await ctx.runMutation(internal.geo.applyEnrichment, {
+                  siteId: payload.siteId as Id<"sites">,
+                  sessionIds: claim.sessionIds,
+                  events: claim.events,
+                  context: providerGeo,
+                });
+              } else {
+                logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
+              }
+            }
+          } catch {
+            logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
+          }
+        }
+      }
+      return json(result, 200, headers);
     } catch (error) {
       if (error instanceof ConvexError) {
         const data = error.data as { code?: string; message?: string };
@@ -71,9 +187,23 @@ http.route({
             : data.code === "RATE_LIMITED"
               ? 429
               : 400;
-        return json({ error: data }, status);
+        return json(
+          { error: data },
+          status,
+          status === 429
+            ? {
+                ...headers,
+                "Retry-After": String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1_000)),
+              }
+            : headers,
+        );
       }
-      return json({ error: { code: "INVALID_REQUEST" } }, 400);
+      logInternalHttpError(requestId);
+      return json(
+        { error: { code: "INTERNAL_ERROR", requestId } },
+        500,
+        headers,
+      );
     }
   }),
 });
@@ -113,13 +243,65 @@ http.route({
   }),
 });
 
+http.route({
+  path: "/vitals.js",
+  method: "GET",
+  handler: httpAction(async () => {
+    return new Response(VITALS_GZIP, {
+      status: 200,
+      headers: {
+        "Content-Type": "text/javascript; charset=utf-8",
+        "Content-Encoding": "gzip",
+        "Content-Length": String(VITALS_GZIP_BYTES),
+        "Cache-Control": "public, max-age=300, stale-while-revalidate=86400",
+        "Access-Control-Allow-Origin": "*",
+        Vary: "Accept-Encoding",
+      },
+    });
+  }),
+});
 
 http.route({
   path: "/health",
   method: "GET",
-  handler: httpAction(async () => {
+  handler: httpAction(async (ctx, request) => {
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const provider = env.RASTRO_GEOIP_PROVIDER;
     return json(
-      { ok: true, component: "@iann29/rastro" },
+      {
+        ok: true,
+        component: "@iann29/rastro",
+        requestMetadata: {
+          runtimeIpKind: classifyIp(metadata.ip),
+          clientIpAvailable: clientIp.ip !== null,
+          clientIpSource: clientIp.source,
+          userAgentAvailable: metadata.userAgent !== null,
+        },
+        geoIp: {
+          provider: provider ?? null,
+          status: geoIpStatus(provider, env.RASTRO_GEOIP_TOKEN),
+          dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+          trustedProxy: trustProxy,
+          proxyRequestTrusted,
+          edgeHeadersAvailable:
+            proxyRequestTrusted &&
+            Object.keys(geoFromHeaders(request)).length > 0,
+        },
+        features: {
+          botDetection: "userAgent",
+          uniqueVisitors:
+            clientIp.ip &&
+            (metadata.userAgent ?? request.headers.get("user-agent"))
+              ? "dailyHash"
+              : "session",
+          webVitals: "optIn",
+          siteMap: "dailyRollup",
+          liveDepartures: "leaveBeacon",
+        },
+      },
       200,
       {
         "Access-Control-Allow-Origin": "*",
@@ -129,11 +311,53 @@ http.route({
   }),
 });
 
-function parsePayload(value: unknown): { siteId: string; events: TrackerEvent[] } {
-  if (!isPlainRecord(value) || typeof value.siteId !== "string" || !Array.isArray(value.events)) {
+function parsePayload(value: unknown): {
+  siteId: string;
+  events: TrackerEvent[];
+  sentAt?: number;
+} {
+  if (
+    !isPlainRecord(value) ||
+    typeof value.siteId !== "string" ||
+    !Array.isArray(value.events)
+  ) {
     throw new Error("body must contain siteId and events");
   }
-  return { siteId: value.siteId, events: value.events.map(parseEvent) };
+  const sentAt = optionalNumber(value.sentAt, "sentAt");
+  return {
+    siteId: value.siteId,
+    events: value.events.map(parseEvent),
+    ...(sentAt === undefined ? {} : { sentAt }),
+  };
+}
+
+class PayloadTooLargeError extends Error {}
+
+async function readLimitedBody(
+  request: Request,
+  maximumBytes: number,
+): Promise<Uint8Array> {
+  if (!request.body) return new Uint8Array();
+  const reader = request.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let length = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    length += value.byteLength;
+    if (length > maximumBytes) {
+      await reader.cancel();
+      throw new PayloadTooLargeError();
+    }
+    chunks.push(value);
+  }
+  const result = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return result;
 }
 
 function parseEvent(value: unknown): TrackerEvent {
@@ -145,7 +369,9 @@ function parseEvent(value: unknown): TrackerEvent {
     type !== "custom" &&
     type !== "conversion" &&
     type !== "heartbeat" &&
-    type !== "outbound"
+    type !== "leave" &&
+    type !== "outbound" &&
+    type !== "vital"
   ) {
     throw new Error("event type is invalid");
   }
@@ -190,6 +416,9 @@ function parseEvent(value: unknown): TrackerEvent {
     ...(optionalString(value.affiliateSlug, "affiliateSlug") !== undefined
       ? { affiliateSlug: optionalString(value.affiliateSlug, "affiliateSlug") }
       : {}),
+    ...(optionalNumber(value.value, "value") !== undefined
+      ? { value: optionalNumber(value.value, "value") }
+      : {}),
   };
 }
 
@@ -223,6 +452,90 @@ function optionalNumber(value: unknown, field: string): number | undefined {
   return value;
 }
 
+function enrichmentCandidateEvents(events: TrackerEvent[]):
+  | Array<{
+      eventId: string;
+      sessionId: string;
+      visitorId: string;
+      timestamp: number;
+      sequence: number;
+    }>
+  | undefined {
+  if (events.length === 0 || events.length > MAX_BATCH_EVENTS) return undefined;
+  const now = Date.now();
+  const sanitized = events.flatMap((event) => {
+    try {
+      return [sanitizeEvent(event, now)];
+    } catch {
+      return [];
+    }
+  });
+  const sessionIds = [...new Set(sanitized.map((event) => event.sessionId))];
+  if (
+    !sanitized.some((event) => event.sequence === 0) ||
+    sessionIds.length > MAX_BATCH_EVENT_GROUPS ||
+    sessionIds.some(
+      (sessionId) => sessionId.length === 0 || sessionId.length > 128,
+    )
+  ) {
+    return undefined;
+  }
+  return sanitized.map(
+    ({ eventId, sessionId, visitorId, timestamp, sequence }) => ({
+      eventId,
+      sessionId,
+      visitorId,
+      timestamp,
+      sequence,
+    }),
+  );
+}
+
+function geoFromHeaders(request: Request): {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+} {
+  const country = firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]);
+  const city = decodeHeader(
+    firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"]),
+  );
+  const latitude = numericHeader(request, [
+    "x-vercel-ip-latitude",
+    "cf-iplatitude",
+  ]);
+  const longitude = numericHeader(request, [
+    "x-vercel-ip-longitude",
+    "cf-iplongitude",
+  ]);
+  return {
+    ...(country ? { country } : {}),
+    ...(city ? { city } : {}),
+    ...(validCoordinates(latitude, longitude) ? { latitude, longitude } : {}),
+  };
+}
+
+function hasCoordinates(value: {
+  latitude?: number;
+  longitude?: number;
+}): boolean {
+  return value.latitude !== undefined && value.longitude !== undefined;
+}
+
+function validCoordinates(
+  latitude: number | undefined,
+  longitude: number | undefined,
+): latitude is number {
+  return (
+    latitude !== undefined &&
+    longitude !== undefined &&
+    latitude >= -90 &&
+    latitude <= 90 &&
+    longitude >= -180 &&
+    longitude <= 180
+  );
+}
 
 function firstHeader(request: Request, names: string[]): string | undefined {
   for (const name of names) {
@@ -248,45 +561,52 @@ function decodeHeader(value: string | undefined): string | undefined {
   }
 }
 
-function classifyUserAgent(userAgent: string): {
-  browser: string;
-  os: string;
-  device: string;
-} {
-  const browser = /Edg\//.test(userAgent)
-    ? "Edge"
-    : /Firefox\//.test(userAgent)
-      ? "Firefox"
-      : /Chrome\//.test(userAgent)
-        ? "Chrome"
-        : /Safari\//.test(userAgent)
-          ? "Safari"
-          : "Other";
-  const os = /Android/.test(userAgent)
-    ? "Android"
-    : /iPhone|iPad/.test(userAgent)
-      ? "iOS"
-      : /Windows/.test(userAgent)
-        ? "Windows"
-        : /Mac OS X/.test(userAgent)
-          ? "macOS"
-          : /Linux/.test(userAgent)
-            ? "Linux"
-            : "Other";
-  const device = /iPad|Tablet/.test(userAgent)
-    ? "tablet"
-    : /Mobile|Android|iPhone/.test(userAgent)
-      ? "mobile"
-      : "desktop";
-  return { browser, os, device };
+type GeoIpFailureStage = "claim" | "resolve" | "apply";
+
+function logInternalHttpError(requestId: string): void {
+  console.error(
+    JSON.stringify({
+      level: "error",
+      event: "rastro.http.internal_error",
+      requestId,
+      route: "/events",
+      method: "POST",
+    }),
+  );
 }
 
-function corsHeaders(origin: string): HeadersInit {
+function logGeoIpFailure(
+  requestId: string,
+  stage: GeoIpFailureStage,
+  provider: "ipinfo" | "ipwhois",
+): void {
+  console.warn(
+    JSON.stringify({
+      level: "warn",
+      event: "rastro.geoip.failure",
+      requestId,
+      route: "/events",
+      stage,
+      provider,
+    }),
+  );
+}
+
+function requestIdHeaders(requestId: string): Record<string, string> {
+  return { "X-Request-ID": requestId };
+}
+
+function corsHeaders(
+  origin: string,
+  requestId?: string,
+): Record<string, string> {
   return {
     "Access-Control-Allow-Origin": origin,
     "Access-Control-Allow-Methods": "POST, OPTIONS",
     "Access-Control-Allow-Headers": "Content-Type",
+    "Access-Control-Expose-Headers": "X-Request-ID, Retry-After",
     "Access-Control-Max-Age": "86400",
+    ...(requestId ? requestIdHeaders(requestId) : {}),
     Vary: "Origin",
   };
 }
@@ -299,3 +619,26 @@ function json(value: unknown, status: number, headers?: HeadersInit): Response {
 }
 
 export default http;
+
+let cachedVisitorSecret: string | undefined;
+
+/** The per-deployment salt behind anonymous visitor keys, created on first use. */
+async function visitorSecret(ctx: {
+  runQuery: (
+    reference: typeof internal.identity.secret,
+    args: Record<string, never>,
+  ) => Promise<string | null>;
+  runMutation: (
+    reference: typeof internal.identity.ensureSecret,
+    args: { candidate: string },
+  ) => Promise<string>;
+}): Promise<string> {
+  if (cachedVisitorSecret) return cachedVisitorSecret;
+  const stored = await ctx.runQuery(internal.identity.secret, {});
+  cachedVisitorSecret =
+    stored ??
+    (await ctx.runMutation(internal.identity.ensureSecret, {
+      candidate: randomSecret(),
+    }));
+  return cachedVisitorSecret;
+}