@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/src/component/goals.ts

@@ -8,10 +8,7 @@ import {
   sanitizePath,
   sanitizeSlug,
 } from "./sanitize.js";
-import {
-  goalFieldsValidator,
-  goalMatchTypeValidator,
-} from "./validators.js";
+import { goalFieldsValidator, goalMatchTypeValidator } from "./validators.js";
 
 const goalDocumentValidator = goalFieldsValidator.extend({
   _id: v.id("goals"),
@@ -31,14 +28,16 @@ export const upsert = mutation({
   },
   returns: v.id("goals"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
     let key: string;
     let matchValue: string;
     try {
       key = sanitizeSlug(args.key, "key");
-      matchValue = args.matchType === "pageview"
-        ? sanitizePath(args.matchValue)
-        : cleanString(args.matchValue, 80);
+      matchValue =
+        args.matchType === "pageview"
+          ? sanitizePath(args.matchValue)
+          : cleanString(args.matchValue, 80);
     } catch (error) {
       fail(
         "INVALID_ARGUMENT",
@@ -49,7 +48,10 @@ export const upsert = mutation({
     if (!name) fail("INVALID_ARGUMENT", "goal name is required");
     if (!matchValue) fail("INVALID_ARGUMENT", "goal match value is required");
     if (args.valueCents !== undefined && !isValidMoney(args.valueCents)) {
-      fail("INVALID_ARGUMENT", "valueCents must be an integer between 0 and 1000000000");
+      fail(
+        "INVALID_ARGUMENT",
+        "valueCents must be an integer between 0 and 1000000000",
+      );
     }
 
     const existingByKey = await ctx.db
@@ -69,14 +71,16 @@ export const upsert = mutation({
     const now = Date.now();
     if (goalId) {
       const goal = await ctx.db.get("goals", goalId);
-      if (!goal || goal.siteId !== args.siteId) fail("NOT_FOUND", "goal not found");
+      if (!goal || goal.siteId !== args.siteId)
+        fail("NOT_FOUND", "goal not found");
       if (
         (args.active ?? goal.active) &&
-        goals.some((candidate) =>
-          candidate._id !== goalId &&
-          candidate.active &&
-          candidate.matchType === args.matchType &&
-          candidate.matchValue === matchValue
+        goals.some(
+          (candidate) =>
+            candidate._id !== goalId &&
+            candidate.active &&
+            candidate.matchType === args.matchType &&
+            candidate.matchValue === matchValue,
         )
       ) {
         fail("CONFLICT", "another active goal already uses this matcher");
@@ -95,16 +99,19 @@ export const upsert = mutation({
 
     if (
       (args.active ?? true) &&
-      goals.some((candidate) =>
-        candidate.active &&
-        candidate.matchType === args.matchType &&
-        candidate.matchValue === matchValue
+      goals.some(
+        (candidate) =>
+          candidate.active &&
+          candidate.matchType === args.matchType &&
+          candidate.matchValue === matchValue,
       )
     ) {
       fail("CONFLICT", "another active goal already uses this matcher");
     }
     if (goals.length >= MAX_GOALS_PER_SITE) {
-      fail("LIMIT_EXCEEDED", "goal limit reached", { limit: MAX_GOALS_PER_SITE });
+      fail("LIMIT_EXCEEDED", "goal limit reached", {
+        limit: MAX_GOALS_PER_SITE,
+      });
     }
     return await ctx.db.insert("goals", {
       siteId: args.siteId,
@@ -136,7 +143,8 @@ export const remove = mutation({
   returns: v.null(),
   handler: async (ctx, args) => {
     const goal = await ctx.db.get("goals", args.goalId);
-    if (!goal || goal.siteId !== args.siteId) fail("NOT_FOUND", "goal not found");
+    if (!goal || goal.siteId !== args.siteId)
+      fail("NOT_FOUND", "goal not found");
     await ctx.db.delete("goals", args.goalId);
     return null;
   },

package/src/component/guards.ts

@@ -1,3 +1,5 @@
-export function isPlainRecord(value: unknown): value is Record<string, unknown> {
+export function isPlainRecord(
+  value: unknown,
+): value is Record<string, unknown> {
   return typeof value === "object" && value !== null && !Array.isArray(value);
 }

package/src/component/http.ts

@@ -20,15 +20,22 @@ import {
 } from "./geo.js";
 import { isPlainRecord } from "./guards.js";
 import { sanitizeEvent } from "./sanitize.js";
+import { deriveVisitorKey, randomSecret } from "./identity.js";
+import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
 import {
   TRACKER_GZIP_BASE64,
   TRACKER_GZIP_BYTES,
+  VITALS_GZIP_BASE64,
+  VITALS_GZIP_BYTES,
 } from "../tracker/generated.js";
 
 const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
   character.charCodeAt(0),
 );
+const VITALS_GZIP = Uint8Array.from(atob(VITALS_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
+);
 
 const http = httpRouter();
 
@@ -88,9 +95,34 @@ http.route({
     const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
     const userAgent =
       metadata.userAgent ?? request.headers.get("user-agent") ?? "";
+    // Self-declared bots never reach a transaction; the response keeps the
+    // counter shape the tracker expects and names the reason in a header.
+    if (isKnownBot(userAgent)) {
+      return json(
+        { accepted: 0, duplicates: 0, rejected: payload.events.length },
+        200,
+        { ...headers, "X-Rastro-Rejected": "bot" },
+      );
+    }
+    // Anonymous batches (visitorId === sessionId) get a salted daily key so
+    // repeat visits count once; the IP itself stops here.
+    const anonymous = payload.events.some(
+      (event) => event.visitorId === event.sessionId,
+    );
+    const visitorKey =
+      anonymous && clientIp.ip && userAgent
+        ? await deriveVisitorKey({
+            secret: await visitorSecret(ctx),
+            now: Date.now(),
+            siteId: payload.siteId,
+            ip: clientIp.ip,
+            userAgent,
+          })
+        : undefined;
     const context = {
       ...edgeGeo,
-      ...classifyUserAgent(userAgent),
+      ...classifyClient(userAgent, request.headers),
+      ...(visitorKey ? { visitorKey } : {}),
     };
 
     try {
@@ -110,14 +142,11 @@ http.route({
         if (events) {
           let stage: GeoIpFailureStage = "claim";
           try {
-            const claim = await ctx.runMutation(
-              internal.geo.claimEnrichment,
-              {
-                siteId: payload.siteId as Id<"sites">,
-                events,
-                dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
-              },
-            );
+            const claim = await ctx.runMutation(internal.geo.claimEnrichment, {
+              siteId: payload.siteId as Id<"sites">,
+              events,
+              dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+            });
             if (claim.sessionIds.length > 0) {
               stage = "resolve";
               const providerGeo = await resolveGeoIp(
@@ -134,19 +163,11 @@ http.route({
                   context: providerGeo,
                 });
               } else {
-                logGeoIpFailure(
-                  requestId,
-                  stage,
-                  env.RASTRO_GEOIP_PROVIDER!,
-                );
+                logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
               }
             }
           } catch {
-            logGeoIpFailure(
-              requestId,
-              stage,
-              env.RASTRO_GEOIP_PROVIDER!,
-            );
+            logGeoIpFailure(requestId, stage, env.RASTRO_GEOIP_PROVIDER!);
           }
         }
       }
@@ -166,9 +187,7 @@ http.route({
           status === 429
             ? {
                 ...headers,
-                "Retry-After": String(
-                  Math.ceil(RATE_LIMIT_WINDOW_MS / 1_000),
-                ),
+                "Retry-After": String(Math.ceil(RATE_LIMIT_WINDOW_MS / 1_000)),
               }
             : headers,
         );
@@ -218,6 +237,24 @@ http.route({
   }),
 });
 
+http.route({
+  path: "/vitals.js",
+  method: "GET",
+  handler: httpAction(async () => {
+    return new Response(VITALS_GZIP, {
+      status: 200,
+      headers: {
+        "Content-Type": "text/javascript; charset=utf-8",
+        "Content-Encoding": "gzip",
+        "Content-Length": String(VITALS_GZIP_BYTES),
+        "Cache-Control": "public, max-age=300, stale-while-revalidate=86400",
+        "Access-Control-Allow-Origin": "*",
+        Vary: "Accept-Encoding",
+      },
+    });
+  }),
+});
+
 http.route({
   path: "/health",
   method: "GET",
@@ -244,7 +281,17 @@ http.route({
           trustedProxy: trustProxy,
           proxyRequestTrusted,
           edgeHeadersAvailable:
-            proxyRequestTrusted && Object.keys(geoFromHeaders(request)).length > 0,
+            proxyRequestTrusted &&
+            Object.keys(geoFromHeaders(request)).length > 0,
+        },
+        features: {
+          botDetection: "userAgent",
+          uniqueVisitors:
+            clientIp.ip &&
+            (metadata.userAgent ?? request.headers.get("user-agent"))
+              ? "dailyHash"
+              : "session",
+          webVitals: "optIn",
         },
       },
       200,
@@ -308,7 +355,8 @@ function parseEvent(value: unknown): TrackerEvent {
     type !== "custom" &&
     type !== "conversion" &&
     type !== "heartbeat" &&
-    type !== "outbound"
+    type !== "outbound" &&
+    type !== "vital"
   ) {
     throw new Error("event type is invalid");
   }
@@ -353,6 +401,9 @@ function parseEvent(value: unknown): TrackerEvent {
     ...(optionalString(value.affiliateSlug, "affiliateSlug") !== undefined
       ? { affiliateSlug: optionalString(value.affiliateSlug, "affiliateSlug") }
       : {}),
+    ...(optionalNumber(value.value, "value") !== undefined
+      ? { value: optionalNumber(value.value, "value") }
+      : {}),
   };
 }
 
@@ -446,9 +497,7 @@ function geoFromHeaders(request: Request): {
   return {
     ...(country ? { country } : {}),
     ...(city ? { city } : {}),
-    ...(validCoordinates(latitude, longitude)
-      ? { latitude, longitude }
-      : {}),
+    ...(validCoordinates(latitude, longitude) ? { latitude, longitude } : {}),
   };
 }
 
@@ -463,12 +512,14 @@ function validCoordinates(
   latitude: number | undefined,
   longitude: number | undefined,
 ): latitude is number {
-  return latitude !== undefined &&
+  return (
+    latitude !== undefined &&
     longitude !== undefined &&
     latitude >= -90 &&
     latitude <= 90 &&
     longitude >= -180 &&
-    longitude <= 180;
+    longitude <= 180
+  );
 }
 
 function firstHeader(request: Request, names: string[]): string | undefined {
@@ -495,49 +546,18 @@ function decodeHeader(value: string | undefined): string | undefined {
   }
 }
 
-function classifyUserAgent(userAgent: string): {
-  browser: string;
-  os: string;
-  device: string;
-} {
-  const browser = /Edg\//.test(userAgent)
-    ? "Edge"
-    : /Firefox\//.test(userAgent)
-      ? "Firefox"
-      : /Chrome\//.test(userAgent)
-        ? "Chrome"
-        : /Safari\//.test(userAgent)
-          ? "Safari"
-          : "Other";
-  const os = /Android/.test(userAgent)
-    ? "Android"
-    : /iPhone|iPad/.test(userAgent)
-      ? "iOS"
-      : /Windows/.test(userAgent)
-        ? "Windows"
-        : /Mac OS X/.test(userAgent)
-          ? "macOS"
-          : /Linux/.test(userAgent)
-            ? "Linux"
-            : "Other";
-  const device = /iPad|Tablet/.test(userAgent)
-    ? "tablet"
-    : /Mobile|Android|iPhone/.test(userAgent)
-      ? "mobile"
-      : "desktop";
-  return { browser, os, device };
-}
-
 type GeoIpFailureStage = "claim" | "resolve" | "apply";
 
 function logInternalHttpError(requestId: string): void {
-  console.error(JSON.stringify({
-    level: "error",
-    event: "rastro.http.internal_error",
-    requestId,
-    route: "/events",
-    method: "POST",
-  }));
+  console.error(
+    JSON.stringify({
+      level: "error",
+      event: "rastro.http.internal_error",
+      requestId,
+      route: "/events",
+      method: "POST",
+    }),
+  );
 }
 
 function logGeoIpFailure(
@@ -545,21 +565,26 @@ function logGeoIpFailure(
   stage: GeoIpFailureStage,
   provider: "ipinfo" | "ipwhois",
 ): void {
-  console.warn(JSON.stringify({
-    level: "warn",
-    event: "rastro.geoip.failure",
-    requestId,
-    route: "/events",
-    stage,
-    provider,
-  }));
+  console.warn(
+    JSON.stringify({
+      level: "warn",
+      event: "rastro.geoip.failure",
+      requestId,
+      route: "/events",
+      stage,
+      provider,
+    }),
+  );
 }
 
 function requestIdHeaders(requestId: string): Record<string, string> {
   return { "X-Request-ID": requestId };
 }
 
-function corsHeaders(origin: string, requestId?: string): Record<string, string> {
+function corsHeaders(
+  origin: string,
+  requestId?: string,
+): Record<string, string> {
   return {
     "Access-Control-Allow-Origin": origin,
     "Access-Control-Allow-Methods": "POST, OPTIONS",
@@ -579,3 +604,26 @@ function json(value: unknown, status: number, headers?: HeadersInit): Response {
 }
 
 export default http;
+
+let cachedVisitorSecret: string | undefined;
+
+/** The per-deployment salt behind anonymous visitor keys, created on first use. */
+async function visitorSecret(ctx: {
+  runQuery: (
+    reference: typeof internal.identity.secret,
+    args: Record<string, never>,
+  ) => Promise<string | null>;
+  runMutation: (
+    reference: typeof internal.identity.ensureSecret,
+    args: { candidate: string },
+  ) => Promise<string>;
+}): Promise<string> {
+  if (cachedVisitorSecret) return cachedVisitorSecret;
+  const stored = await ctx.runQuery(internal.identity.secret, {});
+  cachedVisitorSecret =
+    stored ??
+    (await ctx.runMutation(internal.identity.ensureSecret, {
+      candidate: randomSecret(),
+    }));
+  return cachedVisitorSecret;
+}

package/src/component/identity.ts

@@ -0,0 +1,74 @@
+import { v } from "convex/values";
+import { DAY_MS } from "./constants.js";
+import { internalMutation, internalQuery } from "./_generated/server.js";
+
+/**
+ * Anonymous visitors (no host-supplied id) are counted through a salted hash of
+ * the request IP and User-Agent that rotates every UTC day. The hash is derived
+ * in the HTTP action, so the raw IP never reaches a mutation or a table, and the
+ * salt is a per-deployment secret, so nobody holding a key can walk the IPv4
+ * space back to an address. Same device and network on the same day count once;
+ * the next day they are a new visitor by design.
+ */
+const KEY_HEX_LENGTH = 24;
+const SECRET_PATTERN = /^[0-9a-f]{64}$/;
+
+export const secret = internalQuery({
+  args: {},
+  returns: v.union(v.string(), v.null()),
+  handler: async (ctx) => {
+    const row = await ctx.db
+      .query("visitorSecrets")
+      .withIndex("by_key", (range) => range.eq("key", "current"))
+      .unique();
+    return row?.value ?? null;
+  },
+});
+
+export const ensureSecret = internalMutation({
+  args: { candidate: v.string() },
+  returns: v.string(),
+  handler: async (ctx, args) => {
+    const existing = await ctx.db
+      .query("visitorSecrets")
+      .withIndex("by_key", (range) => range.eq("key", "current"))
+      .unique();
+    if (existing) return existing.value;
+    if (!SECRET_PATTERN.test(args.candidate)) {
+      throw new Error("visitor secret candidate must be 64 hex characters");
+    }
+    await ctx.db.insert("visitorSecrets", {
+      key: "current",
+      value: args.candidate,
+      createdAt: Date.now(),
+    });
+    return args.candidate;
+  },
+});
+
+export function randomSecret(): string {
+  return hex(crypto.getRandomValues(new Uint8Array(32)));
+}
+
+export async function deriveVisitorKey(input: {
+  secret: string;
+  now: number;
+  siteId: string;
+  ip: string;
+  userAgent: string;
+}): Promise<string> {
+  const dayStart = Math.floor(input.now / DAY_MS) * DAY_MS;
+  const material = new TextEncoder().encode(
+    [input.secret, dayStart, input.siteId, input.ip, input.userAgent].join(" "),
+  );
+  const digest = new Uint8Array(
+    await crypto.subtle.digest("SHA-256", material),
+  );
+  return hex(digest).slice(0, KEY_HEX_LENGTH);
+}
+
+function hex(bytes: Uint8Array): string {
+  return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(
+    "",
+  );
+}