@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.3

package/src/component/http.ts

@@ -1,19 +1,32 @@
 import { httpRouter } from "convex/server";
 import { ConvexError } from "convex/values";
 import type { Id } from "./_generated/dataModel.js";
-import { api } from "./_generated/api.js";
-import { httpAction } from "./_generated/server.js";
-import { MAX_BATCH_BYTES } from "./constants.js";
+import { api, internal } from "./_generated/api.js";
+import { env, httpAction } from "./_generated/server.js";
+import {
+  MAX_BATCH_BYTES,
+  MAX_BATCH_EVENT_GROUPS,
+  MAX_BATCH_EVENTS,
+} from "./constants.js";
+import {
+  classifyIp,
+  geoIpDailyLimit,
+  geoIpStatus,
+  isTrustedProxyRequest,
+  readRequestMetadata,
+  resolveClientIp,
+  resolveGeoIp,
+} from "./geo.js";
 import { isPlainRecord } from "./guards.js";
+import { sanitizeEvent } from "./sanitize.js";
 import type { TrackerEvent } from "./validators.js";
 import {
   TRACKER_GZIP_BASE64,
   TRACKER_GZIP_BYTES,
 } from "../tracker/generated.js";
 
-const TRACKER_GZIP = Uint8Array.from(
-  atob(TRACKER_GZIP_BASE64),
-  (character) => character.charCodeAt(0),
+const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
+  character.charCodeAt(0),
 );
 
 const http = httpRouter();
@@ -51,7 +64,8 @@ http.route({
         {
           error: {
             code: "INVALID_BODY",
-            message: error instanceof Error ? error.message : "invalid JSON body",
+            message:
+              error instanceof Error ? error.message : "invalid JSON body",
           },
         },
         400,
@@ -59,12 +73,15 @@ http.route({
       );
     }
 
-    const userAgent = request.headers.get("user-agent") ?? "";
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
+    const userAgent =
+      metadata.userAgent ?? request.headers.get("user-agent") ?? "";
     const context = {
-      country: firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]),
-      city: decodeHeader(firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"])),
-      latitude: numericHeader(request, ["x-vercel-ip-latitude", "cf-iplatitude"]),
-      longitude: numericHeader(request, ["x-vercel-ip-longitude", "cf-iplongitude"]),
+      ...edgeGeo,
       ...classifyUserAgent(userAgent),
     };
 
@@ -75,6 +92,43 @@ http.route({
         events: payload.events,
         context,
       });
+      if (
+        clientIp.ip &&
+        !hasCoordinates(edgeGeo) &&
+        geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
+          "ready"
+      ) {
+        const events = enrichmentCandidateEvents(payload.events);
+        if (events) {
+          try {
+            const claim = await ctx.runMutation(
+              internal.geo.claimEnrichment,
+              {
+                siteId: payload.siteId as Id<"sites">,
+                events,
+                dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+              },
+            );
+            if (claim.sessionIds.length > 0) {
+              const providerGeo = await resolveGeoIp(
+                clientIp.ip,
+                env.RASTRO_GEOIP_PROVIDER!,
+                env.RASTRO_GEOIP_TOKEN,
+              );
+              if (providerGeo) {
+                await ctx.runMutation(internal.geo.applyEnrichment, {
+                  siteId: payload.siteId as Id<"sites">,
+                  sessionIds: claim.sessionIds,
+                  events: claim.events,
+                  context: providerGeo,
+                });
+              }
+            }
+          } catch {
+            // Analytics ingestion remains available if optional enrichment fails.
+          }
+        }
+      }
       return json(result, 200, headers);
     } catch (error) {
       if (error instanceof ConvexError) {
@@ -127,13 +181,35 @@ http.route({
   }),
 });
 
-
 http.route({
   path: "/health",
   method: "GET",
-  handler: httpAction(async () => {
+  handler: httpAction(async (ctx, request) => {
+    const trustProxy = env.RASTRO_TRUST_PROXY === "true";
+    const metadata = await readRequestMetadata(ctx);
+    const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
+    const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
+    const provider = env.RASTRO_GEOIP_PROVIDER;
     return json(
-      { ok: true, component: "@iann29/rastro" },
+      {
+        ok: true,
+        component: "@iann29/rastro",
+        requestMetadata: {
+          runtimeIpKind: classifyIp(metadata.ip),
+          clientIpAvailable: clientIp.ip !== null,
+          clientIpSource: clientIp.source,
+          userAgentAvailable: metadata.userAgent !== null,
+        },
+        geoIp: {
+          provider: provider ?? null,
+          status: geoIpStatus(provider, env.RASTRO_GEOIP_TOKEN),
+          dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
+          trustedProxy: trustProxy,
+          proxyRequestTrusted,
+          edgeHeadersAvailable:
+            proxyRequestTrusted && Object.keys(geoFromHeaders(request)).length > 0,
+        },
+      },
       200,
       {
         "Access-Control-Allow-Origin": "*",
@@ -143,8 +219,15 @@ http.route({
   }),
 });
 
-function parsePayload(value: unknown): { siteId: string; events: TrackerEvent[] } {
-  if (!isPlainRecord(value) || typeof value.siteId !== "string" || !Array.isArray(value.events)) {
+function parsePayload(value: unknown): {
+  siteId: string;
+  events: TrackerEvent[];
+} {
+  if (
+    !isPlainRecord(value) ||
+    typeof value.siteId !== "string" ||
+    !Array.isArray(value.events)
+  ) {
     throw new Error("body must contain siteId and events");
   }
   return { siteId: value.siteId, events: value.events.map(parseEvent) };
@@ -266,6 +349,90 @@ function optionalNumber(value: unknown, field: string): number | undefined {
   return value;
 }
 
+function enrichmentCandidateEvents(events: TrackerEvent[]):
+  | Array<{
+      eventId: string;
+      sessionId: string;
+      visitorId: string;
+      timestamp: number;
+      sequence: number;
+    }>
+  | undefined {
+  if (events.length === 0 || events.length > MAX_BATCH_EVENTS) return undefined;
+  const now = Date.now();
+  const sanitized = events.flatMap((event) => {
+    try {
+      return [sanitizeEvent(event, now)];
+    } catch {
+      return [];
+    }
+  });
+  const sessionIds = [...new Set(sanitized.map((event) => event.sessionId))];
+  if (
+    !sanitized.some((event) => event.sequence === 0) ||
+    sessionIds.length > MAX_BATCH_EVENT_GROUPS ||
+    sessionIds.some(
+      (sessionId) => sessionId.length === 0 || sessionId.length > 128,
+    )
+  ) {
+    return undefined;
+  }
+  return sanitized.map(
+    ({ eventId, sessionId, visitorId, timestamp, sequence }) => ({
+      eventId,
+      sessionId,
+      visitorId,
+      timestamp,
+      sequence,
+    }),
+  );
+}
+
+function geoFromHeaders(request: Request): {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+} {
+  const country = firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]);
+  const city = decodeHeader(
+    firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"]),
+  );
+  const latitude = numericHeader(request, [
+    "x-vercel-ip-latitude",
+    "cf-iplatitude",
+  ]);
+  const longitude = numericHeader(request, [
+    "x-vercel-ip-longitude",
+    "cf-iplongitude",
+  ]);
+  return {
+    ...(country ? { country } : {}),
+    ...(city ? { city } : {}),
+    ...(validCoordinates(latitude, longitude)
+      ? { latitude, longitude }
+      : {}),
+  };
+}
+
+function hasCoordinates(value: {
+  latitude?: number;
+  longitude?: number;
+}): boolean {
+  return value.latitude !== undefined && value.longitude !== undefined;
+}
+
+function validCoordinates(
+  latitude: number | undefined,
+  longitude: number | undefined,
+): latitude is number {
+  return latitude !== undefined &&
+    longitude !== undefined &&
+    latitude >= -90 &&
+    latitude <= 90 &&
+    longitude >= -180 &&
+    longitude <= 180;
+}
 
 function firstHeader(request: Request, names: string[]): string | undefined {
   for (const name of names) {

package/src/component/ingest.ts

@@ -99,6 +99,10 @@ type SessionState = {
   isNewVisitorDay: boolean;
   startedAt: number;
   source: string;
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
   pageviewCount: number;
   eventCount: number;
   affiliateId?: Id<"affiliates">;
@@ -236,7 +240,6 @@ export const ingestBatch = mutation({
           siteId: args.siteId,
           sessionId: group.sessionId,
           bucketStart: group.bucketStart,
-          candidateEventIds: group.eventIds,
         });
         const newCount = [...group.eventIds].filter((eventId) =>
           !eventIds.has(eventId)
@@ -356,7 +359,7 @@ export const ingestBatch = mutation({
           .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
           .take(MAX_FUNNELS_PER_SITE);
       }
-      await accountEvent(ctx, {
+      const session = await accountEvent(ctx, {
         siteId: args.siteId,
         event,
         context,
@@ -370,7 +373,12 @@ export const ingestBatch = mutation({
         now,
         batchState,
       });
-      bufferEventBatch(args.siteId, event, batchState.eventBatches);
+      bufferEventBatch(
+        args.siteId,
+        event,
+        batchState.eventBatches,
+        session.country ?? "unknown",
+      );
       accepted += 1;
     }
     await flushBatchState(ctx, batchState);
@@ -467,7 +475,7 @@ export const trackConversion = mutation({
     };
 
     const aggregateDeltas: AggregateDeltas = new Map();
-    await accountEvent(ctx, {
+    const session = await accountEvent(ctx, {
       siteId: args.siteId,
       event,
       context: conversionContext,
@@ -481,7 +489,12 @@ export const trackConversion = mutation({
       now: Date.now(),
     });
     if (!existingTelemetry && !telemetry.saturated) {
-      await insertEventBatch(ctx, args.siteId, event);
+      await insertEventBatch(
+        ctx,
+        args.siteId,
+        event,
+        session.country ?? "unknown",
+      );
     }
     await flushAggregates(ctx, aggregateDeltas);
     return { accepted: 1, duplicates: 0, rejected: 0 };
@@ -612,7 +625,7 @@ async function accountEvent(
     now: number;
     batchState?: BatchState;
   },
-) {
+): Promise<SessionState> {
   const { event } = input;
   const session = input.recordTelemetry
     ? await upsertSession(
@@ -700,7 +713,7 @@ async function accountEvent(
   const dimensions: Dimension[] = [
     { type: "source", value: session.source },
     { type: "page", value: event.path },
-    { type: "country", value: input.context.country ?? "unknown" },
+    { type: "country", value: session.country ?? "unknown" },
     { type: "device", value: input.context.device ?? "unknown" },
     { type: "browser", value: input.context.browser ?? "Other" },
     {
@@ -733,6 +746,7 @@ async function accountEvent(
       input.batchState,
     );
   }
+  return session;
 }
 
 async function accountTrustedSession(
@@ -755,6 +769,10 @@ async function accountTrustedSession(
     isNewVisitorDay: false,
     startedAt: existing.startedAt,
     source: existing.source,
+    country: existing.country,
+    city: existing.city,
+    latitude: existing.latitude,
+    longitude: existing.longitude,
     pageviewCount: existing.pageviewCount,
     eventCount: existing.eventCount,
     affiliateId: existing.affiliateId,
@@ -832,6 +850,10 @@ async function upsertSession(
         Math.floor(event.timestamp / DAY_MS),
       startedAt,
       source: existing.source,
+      country: context.country ?? existing.country,
+      city: context.city ?? existing.city,
+      latitude: context.latitude ?? existing.latitude,
+      longitude: context.longitude ?? existing.longitude,
       pageviewCount:
         existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
       eventCount: existing.eventCount + 1,
@@ -910,6 +932,10 @@ async function upsertSession(
         Math.floor(event.timestamp / DAY_MS),
     startedAt: event.timestamp,
     source,
+    country: context.country,
+    city: context.city,
+    latitude: context.latitude,
+    longitude: context.longitude,
     pageviewCount: event.type === "pageview" ? 1 : 0,
     eventCount: 1,
     ...(attribution ?? {}),
@@ -1102,6 +1128,10 @@ async function accountHeartbeat(
       isNewVisitorDay: false,
       startedAt,
       source: session.source,
+      country: context.country ?? session.country,
+      city: context.city ?? session.city,
+      latitude: context.latitude ?? session.latitude,
+      longitude: context.longitude ?? session.longitude,
       pageviewCount: session.pageviewCount,
       eventCount: session.eventCount,
       affiliateSlug: session.affiliateSlug,
@@ -1289,6 +1319,10 @@ async function updateLiveSession(
         )
         .unique();
   const deadline = now + LIVE_SESSION_TTL_MS;
+  const country = context.country ?? session.country;
+  const city = context.city ?? session.city;
+  const latitude = context.latitude ?? session.latitude;
+  const longitude = context.longitude ?? session.longitude;
   if (existing) {
     const recentHeartbeatEventIds = event.type === "heartbeat"
       ? [
@@ -1304,10 +1338,10 @@ async function updateLiveSession(
       path: event.path,
       lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
       expiresAt: deadline,
-      ...(context.country ? { country: context.country } : {}),
-      ...(context.city ? { city: context.city } : {}),
-      ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-      ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+      ...(country ? { country } : {}),
+      ...(city ? { city } : {}),
+      ...(latitude !== undefined ? { latitude } : {}),
+      ...(longitude !== undefined ? { longitude } : {}),
       browser: context.browser ?? existing.browser,
       os: context.os ?? existing.os,
       source: session.source,
@@ -1338,10 +1372,10 @@ async function updateLiveSession(
     startedAt: event.timestamp,
     lastSeenAt: event.timestamp,
     expiresAt: deadline,
-    ...(context.country ? { country: context.country } : {}),
-    ...(context.city ? { city: context.city } : {}),
-    ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-    ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+    ...(country ? { country } : {}),
+    ...(city ? { city } : {}),
+    ...(latitude !== undefined ? { latitude } : {}),
+    ...(longitude !== undefined ? { longitude } : {}),
     browser: context.browser ?? "Other",
     os: context.os ?? "Other",
     source: session.source,

package/src/component/reports.ts

@@ -25,6 +25,7 @@ import {
   hydrateEventContexts,
   type JourneyCursor,
   loadSessionJourneyEvents,
+  loadVisitorJourneyEvents,
   type RawJourneyEvent,
 } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
@@ -330,7 +331,10 @@ export const listSessions = query({
           .take(take)),
       );
     }
-    return keysetPaginationResult(rows, pagination, (row) => ({
+    const publicRows = rows.map(
+      ({ geoLookupAttemptedAt: _attemptedAt, ...row }) => row,
+    );
+    return keysetPaginationResult(publicRows, pagination, (row) => ({
       timestamp: row.lastSeenAt,
       creationTime: row._creationTime,
     }));
@@ -450,47 +454,17 @@ export const visitorJourney = query({
     if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
       fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
     }
-    const sessionGroups = await Promise.all(
-      siteIds.map((siteId) =>
-        ctx.db
-          .query("sessions")
-          .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
-            range
-              .eq("siteId", siteId)
-              .eq("visitorId", args.visitorId)
-              .gte("lastSeenAt", args.from),
-          )
-          .order("asc")
-          .take(MAX_JOURNEY_EVENTS + 1),
-      ),
-    );
-    const sessions = sessionGroups
-      .flat()
-      .filter((session) => session.startedAt <= args.to)
-      .sort((left, right) => left.startedAt - right.startedAt)
-      .slice(0, MAX_JOURNEY_EVENTS);
     const rows: RawJourneyEvent[] = [];
-    let overlappingSessionsRead = 0;
-    for (const session of sessions) {
-      const resultIsFull = rows.length >= limit;
-      const cutoff = resultIsFull
+    for (const siteId of siteIds) {
+      const cutoff = rows.length >= limit
         ? rows[limit - 1]!.timestamp
         : args.to;
-      if (session.startedAt > cutoff) break;
-      if (resultIsFull && ++overlappingSessionsRead > 8) {
-        fail(
-          "LIMIT_EXCEEDED",
-          "visitor journey has too many overlapping sessions",
-          { maximumOverlappingSessions: 8 },
-        );
-      }
-      rows.push(...await loadSessionJourneyEvents(ctx, {
-        siteId: session.siteId,
-        sessionId: session.sessionId,
+      rows.push(...await loadVisitorJourneyEvents(ctx, {
+        siteId,
+        visitorId: args.visitorId,
         from: args.from,
-        to: Math.min(args.to, cutoff),
-        cursor: null,
-        resultLimit: resultIsFull ? limit : limit - rows.length,
+        to: cutoff,
+        resultLimit: limit,
       }));
       rows.sort(compareJourneyEvents);
       if (rows.length > limit) rows.length = limit;

package/src/component/schema.ts

@@ -3,12 +3,11 @@ import { v } from "convex/values";
 import {
   affiliateFieldsValidator,
   aggregateFieldsValidator,
-  storedEventFieldsValidator,
+  batchedEventValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
-  trackerEventValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -17,7 +16,11 @@ export default defineSchema({
     .index("by_ownerId_and_name", ["ownerId", "name"])
     .index("by_networkId", ["networkId"]),
 
-  sessions: defineTable(sessionFieldsValidator)
+  sessions: defineTable(
+    sessionFieldsValidator.extend({
+      geoLookupAttemptedAt: v.optional(v.number()),
+    }),
+  )
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
     .index("by_siteId_and_startedAt", ["siteId", "startedAt"])
@@ -59,25 +62,19 @@ export default defineSchema({
     scheduledAt: v.number(),
   }).index("by_siteId", ["siteId"]),
 
-  events: defineTable(storedEventFieldsValidator)
-    .index("by_siteId_and_eventId", ["siteId", "eventId"])
-    .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
-    .index("by_siteId_and_sessionId_and_timestamp", [
-      "siteId",
-      "sessionId",
-      "timestamp",
-    ])
-    .index("by_siteId_and_visitorId_and_timestamp", {
-      fields: ["siteId", "visitorId", "timestamp"],
-      staged: true,
-    }),
+  geoLookupBudgets: defineTable({
+    siteId: v.id("sites"),
+    dayStart: v.number(),
+    count: v.number(),
+  }).index("by_siteId", ["siteId"]),
 
   eventBatches: defineTable({
     siteId: v.id("sites"),
     sessionId: v.string(),
     visitorId: v.string(),
     bucketStart: v.number(),
-    events: v.array(trackerEventValidator),
+    events: v.array(batchedEventValidator),
+    migrationVersion: v.optional(v.literal(1)),
   })
     .index("by_siteId_and_sessionId_and_bucketStart", [
       "siteId",
@@ -85,10 +82,11 @@ export default defineSchema({
       "bucketStart",
     ])
     .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
-    .index("by_siteId_and_visitorId_and_bucketStart", {
-      fields: ["siteId", "visitorId", "bucketStart"],
-      staged: true,
-    }),
+    .index("by_siteId_and_visitorId_and_bucketStart", [
+      "siteId",
+      "visitorId",
+      "bucketStart",
+    ]),
 
   aggregateBuckets: defineTable(aggregateFieldsValidator)
     .index("by_siteId_and_granularity_and_bucketStart_and_shard", [
@@ -119,7 +117,11 @@ export default defineSchema({
     valueCents: v.number(),
   })
     .index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
-    .index("by_siteId_and_goalId_and_timestamp", ["siteId", "goalId", "timestamp"])
+    .index("by_siteId_and_goalId_and_timestamp", [
+      "siteId",
+      "goalId",
+      "timestamp",
+    ])
     .index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
 
   funnels: defineTable(funnelFieldsValidator)

package/src/component/validators.ts

@@ -49,6 +49,13 @@ export const ingestContextValidator = v.object({
   device: v.optional(v.string()),
 });
 
+export const batchedEventValidator = trackerEventValidator.extend(
+  {
+    ...ingestContextValidator.fields,
+    aggregateCountry: v.optional(v.string()),
+  },
+);
+
 export const siteFieldsValidator = v.object({
   ownerId: v.string(),
   name: v.string(),
@@ -119,18 +126,6 @@ export const eventFieldsValidator = v.object({
   device: v.string(),
 });
 
-export const storedEventFieldsValidator = eventFieldsValidator
-  .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
-  .extend({
-    country: v.optional(v.string()),
-    city: v.optional(v.string()),
-    latitude: v.optional(v.number()),
-    longitude: v.optional(v.number()),
-    browser: v.optional(v.string()),
-    os: v.optional(v.string()),
-    device: v.optional(v.string()),
-  });
-
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
   v.literal("page"),