@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.3

package/src/component/convex.config.ts

@@ -1,3 +1,13 @@
 import { defineComponent } from "convex/server";
+import { v } from "convex/values";
 
-export default defineComponent("rastroAnalytics");
+export default defineComponent("rastroAnalytics", {
+  env: {
+    RASTRO_GEOIP_PROVIDER: v.optional(
+      v.union(v.literal("ipinfo"), v.literal("ipwhois")),
+    ),
+    RASTRO_GEOIP_TOKEN: v.optional(v.string()),
+    RASTRO_GEOIP_DAILY_LIMIT: v.optional(v.string()),
+    RASTRO_TRUST_PROXY: v.optional(v.literal("true")),
+  },
+});

package/src/component/diagnostics.ts

@@ -0,0 +1,64 @@
+import { v } from "convex/values";
+import { query } from "./_generated/server.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+
+const MAX_SESSION_TOTAL_IDS = 100;
+
+export const sessionEventTotals = query({
+  args: {
+    siteId: v.id("sites"),
+    sessionIds: v.array(v.string()),
+  },
+  returns: v.object({
+    requestedSessions: v.number(),
+    foundSessions: v.number(),
+    storedEvents: v.number(),
+  }),
+  handler: async (ctx, args) => {
+    if (
+      args.sessionIds.length < 1 ||
+      args.sessionIds.length > MAX_SESSION_TOTAL_IDS
+    ) {
+      fail(
+        "INVALID_ARGUMENT",
+        `sessionIds must contain between 1 and ${MAX_SESSION_TOTAL_IDS} IDs`,
+      );
+    }
+    let sessionIds: string[];
+    try {
+      sessionIds = args.sessionIds.map((sessionId) => {
+        const sanitized = sanitizeOpaqueId(sessionId, "sessionId");
+        if (sanitized !== sessionId) throw new Error("sessionId is not canonical");
+        return sanitized;
+      });
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid sessionId",
+      );
+    }
+    if (new Set(sessionIds).size !== sessionIds.length) {
+      fail("INVALID_ARGUMENT", "sessionIds must be unique");
+    }
+
+    const sessions = await Promise.all(
+      sessionIds.map((sessionId) =>
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", args.siteId).eq("sessionId", sessionId),
+          )
+          .unique()
+      ),
+    );
+    return {
+      requestedSessions: sessionIds.length,
+      foundSessions: sessions.filter((session) => session !== null).length,
+      storedEvents: sessions.reduce(
+        (total, session) => total + (session?.eventCount ?? 0),
+        0,
+      ),
+    };
+  },
+});

package/src/component/eventStore.ts

@@ -1,9 +1,12 @@
 import type { Doc, Id } from "./_generated/dataModel.js";
 import type { MutationCtx, QueryCtx } from "./_generated/server.js";
 import {
+  JOURNEY_READ_BYTE_RESERVE,
+  JOURNEY_READ_DOCUMENT_RESERVE,
   MAX_EVENTS_PER_SESSION_WINDOW,
   RATE_LIMIT_WINDOW_MS,
 } from "./constants.js";
+import { fail } from "./errors.js";
 import type { TrackerEvent } from "./validators.js";
 
 export type TelemetryMatch = {
@@ -34,7 +37,16 @@ export type JourneyCursor = {
   id: string;
 };
 
-export type RawJourneyEvent = Omit<Doc<"events">, "_id"> & { _id: string };
+export type RawJourneyEvent = Omit<
+  Doc<"eventBatches">["events"][number],
+  "aggregateCountry"
+> & {
+  _id: string;
+  _creationTime: number;
+  siteId: Id<"sites">;
+  sessionId: string;
+  visitorId: string;
+};
 
 export function eventBucketStart(timestamp: number): number {
   return Math.floor(timestamp / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -50,32 +62,15 @@ export async function inspectTelemetryMinute(
     siteId: Id<"sites">;
     sessionId: string;
     bucketStart: number;
-    candidateEventIds: Set<string>;
   },
 ) {
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, input);
-  const eventIds = new Set([
-    ...legacyEvents.map((event) => event.eventId),
-    ...batches.flatMap((batch) =>
-      batch.events.map((event) => event.eventId),
-    ),
-  ]);
-  if (legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW) {
-    await Promise.all(
-      [...input.candidateEventIds]
-        .filter((eventId) => !eventIds.has(eventId))
-        .map(async (eventId) => {
-          const legacy = await findLegacyTelemetry(ctx, {
-            ...input,
-            eventId,
-          });
-          if (legacy) eventIds.add(eventId);
-        }),
-    );
-  }
+  const batches = await loadTelemetryMinute(ctx, input);
+  const eventIds = new Set(
+    batches.flatMap((batch) => batch.events.map((event) => event.eventId)),
+  );
   return {
     eventIds,
-    persistedCount: telemetryMinuteCount(legacyEvents, batches),
+    persistedCount: telemetryMinuteCount(batches),
   };
 }
 
@@ -85,34 +80,31 @@ export async function findTelemetry(
   event: TrackerEvent,
 ): Promise<{ match: TelemetryMatch | null; saturated: boolean }> {
   const bucketStart = eventBucketStart(event.timestamp);
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, {
+  const batches = await loadTelemetryMinute(ctx, {
     siteId,
     sessionId: event.sessionId,
     bucketStart,
   });
-  const persistedCount = telemetryMinuteCount(legacyEvents, batches);
-  let legacyEvent = legacyEvents.find((candidate) =>
-    candidate.eventId === event.eventId
-  );
-  if (
-    !legacyEvent &&
-    legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW
-  ) {
-    legacyEvent = await findLegacyTelemetry(ctx, {
-      siteId,
-      sessionId: event.sessionId,
-      bucketStart,
-      eventId: event.eventId,
-    }) ?? undefined;
-  }
+  const persistedCount = telemetryMinuteCount(batches);
   const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
-  if (legacyEvent) return { match: legacyEvent, saturated };
-  const batch = batches.find((candidate) =>
-    candidate.events.some((nested) => nested.eventId === event.eventId)
-  );
+  let batch: Doc<"eventBatches"> | undefined;
+  let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
+  for (const candidate of batches) {
+    nestedEvent = candidate.events.find((nested) =>
+      nested.eventId === event.eventId
+    );
+    if (nestedEvent) {
+      batch = candidate;
+      break;
+    }
+  }
   return {
-    match: batch
-      ? { sessionId: batch.sessionId, visitorId: batch.visitorId }
+    match: batch && nestedEvent
+      ? {
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+        ...eventContext(nestedEvent),
+      }
       : null,
     saturated,
   };
@@ -122,6 +114,7 @@ export function bufferEventBatch(
   siteId: Id<"sites">,
   event: TrackerEvent,
   buffer: EventBatchBuffer,
+  country: string,
 ) {
   const bucketStart = eventBucketStart(event.timestamp);
   const key = eventBatchKey(event.sessionId, bucketStart);
@@ -130,7 +123,7 @@ export function bufferEventBatch(
     if (existing.visitorId !== event.visitorId) {
       throw new Error("event batch visitor does not match its session");
     }
-    existing.events.push(eventForBatch(event));
+    existing.events.push(eventForBatch(event, country));
     return;
   }
   buffer.set(key, {
@@ -138,7 +131,7 @@ export function bufferEventBatch(
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart,
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -146,13 +139,14 @@ export async function insertEventBatch(
   ctx: MutationCtx,
   siteId: Id<"sites">,
   event: TrackerEvent,
+  country: string,
 ) {
   await ctx.db.insert("eventBatches", {
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart: eventBucketStart(event.timestamp),
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -183,8 +177,7 @@ export async function loadSessionJourneyEvents(
   // be read completely before applying the cursor.
   const loadBatches = async () => {
     const batches: Doc<"eventBatches">[] = [];
-    let eventCount = 0;
-    const requiredEvents = input.resultLimit + MAX_EVENTS_PER_SESSION_WINDOW;
+    let qualifyingEvents = 0;
     let completedBucket: number | undefined;
     const query = ctx.db
       .query("eventBatches")
@@ -201,78 +194,81 @@ export async function loadSessionJourneyEvents(
         break;
       }
       batches.push(batch);
-      eventCount += batch.events.length;
-      if (eventCount >= requiredEvents) completedBucket = batch.bucketStart;
+      qualifyingEvents += batch.events.reduce((count, _event, index) => {
+        const event = eventFromBatch(batch, index);
+        return count + Number(
+          event.timestamp >= input.from &&
+            event.timestamp <= input.to &&
+            (!input.cursor ||
+              compareJourneyEventToCursor(event, input.cursor) > 0),
+        );
+      }, 0);
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
     }
     return batches;
   };
-  const loadLegacyEvents = async () => {
-    const take = input.resultLimit + 1;
-    if (!input.cursor) {
-      return ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gte("timestamp", input.from)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take);
-    }
-    const rows = await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = [];
+  appendBatchEvents(rows, batches);
+  return rows
+    .filter((event) =>
+      event.timestamp >= input.from &&
+      event.timestamp <= input.to &&
+      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
+export async function loadVisitorJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    visitorId: string;
+    from: number;
+    to: number;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const firstBucket = eventBucketStart(input.from);
+  const lastBucket = eventBucketStart(input.to);
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let qualifyingEvents = 0;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_visitorId_and_bucketStart", (range) =>
         range
           .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("timestamp", input.cursor!.timestamp)
-          .gt("_creationTime", input.cursor!.creationTime),
+          .eq("visitorId", input.visitorId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
       )
-      .order("asc")
-      .take(take);
-    if (rows.length < take) {
-      rows.push(...await ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gt("timestamp", input.cursor!.timestamp)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take - rows.length));
+      .order("asc");
+    for await (const batch of query) {
+      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+        break;
+      }
+      batches.push(batch);
+      await requireJourneyReadHeadroom(ctx);
+      qualifyingEvents += batch.events.filter((event) =>
+        event.timestamp >= input.from && event.timestamp <= input.to
+      ).length;
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
     }
-    return rows;
+    return batches;
   };
-  const [legacyEvents, batches] = await Promise.all([
-    loadLegacyEvents(),
-    loadBatches(),
-  ]);
-  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
-    ...event,
-    _id: String(event._id),
-  }));
-  for (const batch of batches) {
-    for (let index = 0; index < batch.events.length; index += 1) {
-      const event = batch.events[index];
-      rows.push({
-        ...event,
-        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
-        _creationTime: batch._creationTime,
-        siteId: batch.siteId,
-        sessionId: batch.sessionId,
-        visitorId: batch.visitorId,
-      });
-    }
-  }
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = [];
+  appendBatchEvents(rows, batches);
   return rows
     .filter((event) =>
-      event.timestamp >= input.from &&
-      event.timestamp <= input.to &&
-      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+      event.timestamp >= input.from && event.timestamp <= input.to
     )
     .sort(compareJourneyEvents)
     .slice(0, input.resultLimit);
@@ -342,32 +338,19 @@ export async function deleteTelemetryBefore(
 ) {
   const batchCutoff = eventBucketStart(input.before);
   const limit = input.batchSize + 1;
-  const [legacyEvents, batches] = await Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range.eq("siteId", input.siteId).lt("timestamp", input.before),
-      )
-      .take(limit),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_bucketStart", (range) =>
-        range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
-      )
-      .take(limit),
-  ]);
-  const rows = [...legacyEvents, ...batches].slice(0, input.batchSize);
+  const batches = await ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_bucketStart", (range) =>
+      range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
+    )
+    .take(limit);
+  const rows = batches.slice(0, input.batchSize);
   for (const row of rows) {
-    if ("bucketStart" in row) {
-      await ctx.db.delete("eventBatches", row._id);
-    } else {
-      await ctx.db.delete("events", row._id);
-    }
+    await ctx.db.delete("eventBatches", row._id);
   }
   return {
     deleted: rows.length,
-    hasMore:
-      legacyEvents.length + batches.length > input.batchSize,
+    hasMore: batches.length > input.batchSize,
   };
 }
 
@@ -379,61 +362,30 @@ async function loadTelemetryMinute(
     bucketStart: number;
   },
 ) {
-  return Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .gte("timestamp", input.bucketStart)
-          .lt("timestamp", input.bucketStart + RATE_LIMIT_WINDOW_MS),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("bucketStart", input.bucketStart),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-  ]);
-}
-
-async function findLegacyTelemetry(
-  ctx: MutationCtx,
-  input: {
-    siteId: Id<"sites">;
-    sessionId: string;
-    bucketStart: number;
-    eventId: string;
-  },
-) {
-  const candidates = await ctx.db
-    .query("events")
-    .withIndex("by_siteId_and_eventId", (range) =>
-      range.eq("siteId", input.siteId).eq("eventId", input.eventId),
+  return ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+      range
+        .eq("siteId", input.siteId)
+        .eq("sessionId", input.sessionId)
+        .eq("bucketStart", input.bucketStart),
     )
-    .take(2);
-  return candidates.find((candidate) =>
-    candidate.sessionId === input.sessionId &&
-    eventBucketStart(candidate.timestamp) === input.bucketStart
-  ) ?? null;
+    .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
 }
 
 function telemetryMinuteCount(
-  legacyEvents: Array<{ eventId: string }>,
   batches: Array<{ events: TrackerEvent[] }>,
 ) {
-  return legacyEvents.length + batches.reduce(
+  return batches.reduce(
     (total, batch) => total + batch.events.length,
     0,
   );
 }
 
-function eventForBatch(event: TrackerEvent): TrackerEvent {
+function eventForBatch(
+  event: TrackerEvent,
+  country: string,
+): TrackerEvent & { aggregateCountry?: string } {
   return {
     eventId: event.eventId,
     sessionId: event.sessionId,
@@ -452,9 +404,74 @@ function eventForBatch(event: TrackerEvent): TrackerEvent {
       : {}),
     ...(event.currency ? { currency: event.currency } : {}),
     ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+    aggregateCountry: country,
   };
 }
 
+function appendBatchEvents(
+  rows: RawJourneyEvent[],
+  batches: Doc<"eventBatches">[],
+) {
+  for (const batch of batches) {
+    for (let index = 0; index < batch.events.length; index += 1) {
+      rows.push(eventFromBatch(batch, index));
+    }
+  }
+}
+
+function eventFromBatch(
+  batch: Doc<"eventBatches">,
+  index: number,
+): RawJourneyEvent {
+  const { aggregateCountry: _aggregateCountry, ...event } = batch.events[index]!;
+  void _aggregateCountry;
+  return {
+    ...event,
+    _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+    _creationTime: batch._creationTime,
+    siteId: batch.siteId,
+    sessionId: batch.sessionId,
+    visitorId: batch.visitorId,
+  };
+}
+
+function eventContext(event: {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+}) {
+  return {
+    ...(event.country ? { country: event.country } : {}),
+    ...(event.city ? { city: event.city } : {}),
+    ...(event.latitude !== undefined ? { latitude: event.latitude } : {}),
+    ...(event.longitude !== undefined ? { longitude: event.longitude } : {}),
+    ...(event.browser ? { browser: event.browser } : {}),
+    ...(event.os ? { os: event.os } : {}),
+    ...(event.device ? { device: event.device } : {}),
+  };
+}
+
+async function requireJourneyReadHeadroom(ctx: QueryCtx) {
+  const metrics = await ctx.meta.getTransactionMetrics();
+  if (
+    metrics.bytesRead.remaining < JOURNEY_READ_BYTE_RESERVE ||
+    metrics.documentsRead.remaining < JOURNEY_READ_DOCUMENT_RESERVE
+  ) {
+    fail(
+      "LIMIT_EXCEEDED",
+      "visitor journey exceeds the safe read budget",
+      {
+        bytesRead: metrics.bytesRead.used,
+        documentsRead: metrics.documentsRead.used,
+      },
+    );
+  }
+}
+
 function compareJourneyEventToCursor(
   event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   cursor: JourneyCursor,