@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.3

package/src/component/eventStore.ts

@@ -3,7 +3,6 @@ import type { MutationCtx, QueryCtx } from "./_generated/server.js";
 import {
   JOURNEY_READ_BYTE_RESERVE,
   JOURNEY_READ_DOCUMENT_RESERVE,
-  MAX_BATCH_EVENTS,
   MAX_EVENTS_PER_SESSION_WINDOW,
   RATE_LIMIT_WINDOW_MS,
 } from "./constants.js";
@@ -32,23 +31,22 @@ type BufferedEventBatch = {
 
 export type EventBatchBuffer = Map<string, BufferedEventBatch>;
 
-type MigrationBatch = Pick<
-  Doc<"eventBatches">,
-  "_id" | "visitorId" | "events" | "migrationVersion"
->;
-
-const migrationBatchCache = new WeakMap<
-  MutationCtx,
-  Map<string, MigrationBatch[]>
->();
-
 export type JourneyCursor = {
   timestamp: number;
   creationTime: number;
   id: string;
 };
 
-export type RawJourneyEvent = Omit<Doc<"events">, "_id"> & { _id: string };
+export type RawJourneyEvent = Omit<
+  Doc<"eventBatches">["events"][number],
+  "aggregateCountry"
+> & {
+  _id: string;
+  _creationTime: number;
+  siteId: Id<"sites">;
+  sessionId: string;
+  visitorId: string;
+};
 
 export function eventBucketStart(timestamp: number): number {
   return Math.floor(timestamp / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -64,32 +62,15 @@ export async function inspectTelemetryMinute(
     siteId: Id<"sites">;
     sessionId: string;
     bucketStart: number;
-    candidateEventIds: Set<string>;
   },
 ) {
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, input);
-  const eventIds = new Set([
-    ...legacyEvents.map((event) => event.eventId),
-    ...batches.flatMap((batch) =>
-      batch.events.map((event) => event.eventId),
-    ),
-  ]);
-  if (legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW) {
-    await Promise.all(
-      [...input.candidateEventIds]
-        .filter((eventId) => !eventIds.has(eventId))
-        .map(async (eventId) => {
-          const legacy = await findLegacyTelemetry(ctx, {
-            ...input,
-            eventId,
-          });
-          if (legacy) eventIds.add(eventId);
-        }),
-    );
-  }
+  const batches = await loadTelemetryMinute(ctx, input);
+  const eventIds = new Set(
+    batches.flatMap((batch) => batch.events.map((event) => event.eventId)),
+  );
   return {
     eventIds,
-    persistedCount: telemetryMinuteCount(legacyEvents, batches),
+    persistedCount: telemetryMinuteCount(batches),
   };
 }
 
@@ -99,28 +80,13 @@ export async function findTelemetry(
   event: TrackerEvent,
 ): Promise<{ match: TelemetryMatch | null; saturated: boolean }> {
   const bucketStart = eventBucketStart(event.timestamp);
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, {
+  const batches = await loadTelemetryMinute(ctx, {
     siteId,
     sessionId: event.sessionId,
     bucketStart,
   });
-  const persistedCount = telemetryMinuteCount(legacyEvents, batches);
-  let legacyEvent = legacyEvents.find((candidate) =>
-    candidate.eventId === event.eventId
-  );
-  if (
-    !legacyEvent &&
-    legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW
-  ) {
-    legacyEvent = await findLegacyTelemetry(ctx, {
-      siteId,
-      sessionId: event.sessionId,
-      bucketStart,
-      eventId: event.eventId,
-    }) ?? undefined;
-  }
+  const persistedCount = telemetryMinuteCount(batches);
   const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
-  if (legacyEvent) return { match: legacyEvent, saturated };
   let batch: Doc<"eventBatches"> | undefined;
   let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
   for (const candidate of batches) {
@@ -148,6 +114,7 @@ export function bufferEventBatch(
   siteId: Id<"sites">,
   event: TrackerEvent,
   buffer: EventBatchBuffer,
+  country: string,
 ) {
   const bucketStart = eventBucketStart(event.timestamp);
   const key = eventBatchKey(event.sessionId, bucketStart);
@@ -156,7 +123,7 @@ export function bufferEventBatch(
     if (existing.visitorId !== event.visitorId) {
       throw new Error("event batch visitor does not match its session");
     }
-    existing.events.push(eventForBatch(event));
+    existing.events.push(eventForBatch(event, country));
     return;
   }
   buffer.set(key, {
@@ -164,7 +131,7 @@ export function bufferEventBatch(
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart,
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -172,13 +139,14 @@ export async function insertEventBatch(
   ctx: MutationCtx,
   siteId: Id<"sites">,
   event: TrackerEvent,
+  country: string,
 ) {
   await ctx.db.insert("eventBatches", {
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart: eventBucketStart(event.timestamp),
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -191,84 +159,6 @@ export async function flushEventBatches(
   }
 }
 
-export async function migrateLegacyEventToBatch(
-  ctx: MutationCtx,
-  event: Doc<"events">,
-) {
-  const bucketStart = eventBucketStart(event.timestamp);
-  let transactionCache = migrationBatchCache.get(ctx);
-  if (!transactionCache) {
-    transactionCache = new Map();
-    migrationBatchCache.set(ctx, transactionCache);
-  }
-  const cacheKey = JSON.stringify([
-    event.siteId,
-    event.sessionId,
-    bucketStart,
-  ]);
-  let batches = transactionCache.get(cacheKey);
-  if (!batches) {
-    batches = await ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
-        range
-          .eq("siteId", event.siteId)
-          .eq("sessionId", event.sessionId)
-          .eq("bucketStart", bucketStart),
-      )
-      .order("desc")
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
-    transactionCache.set(cacheKey, batches);
-  }
-  const duplicateBatch = batches.find((batch) =>
-    batch.events.some((candidate) => candidate.eventId === event.eventId)
-  );
-  const context = eventContext(event);
-  if (duplicateBatch) {
-    if (Object.keys(context).length > 0) {
-      const events = duplicateBatch.events.map((candidate) =>
-        candidate.eventId === event.eventId
-          ? { ...candidate, ...context }
-          : candidate
-      );
-      await ctx.db.patch("eventBatches", duplicateBatch._id, {
-        events,
-      });
-      duplicateBatch.events = events;
-    }
-    await ctx.db.delete("events", event._id);
-    return;
-  }
-  const migratedEvent = {
-    ...eventForBatch(event),
-    ...context,
-  };
-  const latestBatch = batches[0];
-  if (
-    latestBatch?.migrationVersion === 1 &&
-    latestBatch.visitorId === event.visitorId &&
-    latestBatch.events.length < MAX_BATCH_EVENTS
-  ) {
-    const events = [...latestBatch.events, migratedEvent];
-    await ctx.db.patch("eventBatches", latestBatch._id, {
-      events,
-    });
-    latestBatch.events = events;
-  } else {
-    const batch = {
-      siteId: event.siteId,
-      sessionId: event.sessionId,
-      visitorId: event.visitorId,
-      bucketStart,
-      events: [migratedEvent],
-      migrationVersion: 1 as const,
-    };
-    const batchId = await ctx.db.insert("eventBatches", batch);
-    batches.unshift({ _id: batchId, ...batch });
-  }
-  await ctx.db.delete("events", event._id);
-}
-
 export async function loadSessionJourneyEvents(
   ctx: QueryCtx,
   input: {
@@ -287,8 +177,7 @@ export async function loadSessionJourneyEvents(
   // be read completely before applying the cursor.
   const loadBatches = async () => {
     const batches: Doc<"eventBatches">[] = [];
-    let eventCount = 0;
-    const requiredEvents = input.resultLimit + MAX_EVENTS_PER_SESSION_WINDOW;
+    let qualifyingEvents = 0;
     let completedBucket: number | undefined;
     const query = ctx.db
       .query("eventBatches")
@@ -305,60 +194,23 @@ export async function loadSessionJourneyEvents(
         break;
       }
       batches.push(batch);
-      eventCount += batch.events.length;
-      if (eventCount >= requiredEvents) completedBucket = batch.bucketStart;
+      qualifyingEvents += batch.events.reduce((count, _event, index) => {
+        const event = eventFromBatch(batch, index);
+        return count + Number(
+          event.timestamp >= input.from &&
+            event.timestamp <= input.to &&
+            (!input.cursor ||
+              compareJourneyEventToCursor(event, input.cursor) > 0),
+        );
+      }, 0);
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
     }
     return batches;
   };
-  const loadLegacyEvents = async () => {
-    const take = input.resultLimit + 1;
-    if (!input.cursor) {
-      return ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gte("timestamp", input.from)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take);
-    }
-    const rows = await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("timestamp", input.cursor!.timestamp)
-          .gt("_creationTime", input.cursor!.creationTime),
-      )
-      .order("asc")
-      .take(take);
-    if (rows.length < take) {
-      rows.push(...await ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gt("timestamp", input.cursor!.timestamp)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take - rows.length));
-    }
-    return rows;
-  };
-  const [legacyEvents, batches] = await Promise.all([
-    loadLegacyEvents(),
-    loadBatches(),
-  ]);
-  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
-    ...event,
-    _id: String(event._id),
-  }));
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = [];
   appendBatchEvents(rows, batches);
   return rows
     .filter((event) =>
@@ -411,27 +263,8 @@ export async function loadVisitorJourneyEvents(
     }
     return batches;
   };
-  const legacyEvents: Doc<"events">[] = [];
-  const legacyQuery = ctx.db
-    .query("events")
-    .withIndex("by_siteId_and_visitorId_and_timestamp", (range) =>
-      range
-        .eq("siteId", input.siteId)
-        .eq("visitorId", input.visitorId)
-        .gte("timestamp", input.from)
-        .lte("timestamp", input.to),
-    )
-    .order("asc");
-  for await (const event of legacyQuery) {
-    legacyEvents.push(event);
-    await requireJourneyReadHeadroom(ctx);
-    if (legacyEvents.length >= input.resultLimit) break;
-  }
   const batches = await loadBatches();
-  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
-    ...event,
-    _id: String(event._id),
-  }));
+  const rows: RawJourneyEvent[] = [];
   appendBatchEvents(rows, batches);
   return rows
     .filter((event) =>
@@ -505,32 +338,19 @@ export async function deleteTelemetryBefore(
 ) {
   const batchCutoff = eventBucketStart(input.before);
   const limit = input.batchSize + 1;
-  const [legacyEvents, batches] = await Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range.eq("siteId", input.siteId).lt("timestamp", input.before),
-      )
-      .take(limit),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_bucketStart", (range) =>
-        range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
-      )
-      .take(limit),
-  ]);
-  const rows = [...legacyEvents, ...batches].slice(0, input.batchSize);
+  const batches = await ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_bucketStart", (range) =>
+      range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
+    )
+    .take(limit);
+  const rows = batches.slice(0, input.batchSize);
   for (const row of rows) {
-    if ("bucketStart" in row) {
-      await ctx.db.delete("eventBatches", row._id);
-    } else {
-      await ctx.db.delete("events", row._id);
-    }
+    await ctx.db.delete("eventBatches", row._id);
   }
   return {
     deleted: rows.length,
-    hasMore:
-      legacyEvents.length + batches.length > input.batchSize,
+    hasMore: batches.length > input.batchSize,
   };
 }
 
@@ -542,61 +362,30 @@ async function loadTelemetryMinute(
     bucketStart: number;
   },
 ) {
-  return Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .gte("timestamp", input.bucketStart)
-          .lt("timestamp", input.bucketStart + RATE_LIMIT_WINDOW_MS),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("bucketStart", input.bucketStart),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-  ]);
-}
-
-async function findLegacyTelemetry(
-  ctx: MutationCtx,
-  input: {
-    siteId: Id<"sites">;
-    sessionId: string;
-    bucketStart: number;
-    eventId: string;
-  },
-) {
-  const candidates = await ctx.db
-    .query("events")
-    .withIndex("by_siteId_and_eventId", (range) =>
-      range.eq("siteId", input.siteId).eq("eventId", input.eventId),
+  return ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+      range
+        .eq("siteId", input.siteId)
+        .eq("sessionId", input.sessionId)
+        .eq("bucketStart", input.bucketStart),
     )
-    .take(2);
-  return candidates.find((candidate) =>
-    candidate.sessionId === input.sessionId &&
-    eventBucketStart(candidate.timestamp) === input.bucketStart
-  ) ?? null;
+    .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
 }
 
 function telemetryMinuteCount(
-  legacyEvents: Array<{ eventId: string }>,
   batches: Array<{ events: TrackerEvent[] }>,
 ) {
-  return legacyEvents.length + batches.reduce(
+  return batches.reduce(
     (total, batch) => total + batch.events.length,
     0,
   );
 }
 
-function eventForBatch(event: TrackerEvent): TrackerEvent {
+function eventForBatch(
+  event: TrackerEvent,
+  country: string,
+): TrackerEvent & { aggregateCountry?: string } {
   return {
     eventId: event.eventId,
     sessionId: event.sessionId,
@@ -615,6 +404,7 @@ function eventForBatch(event: TrackerEvent): TrackerEvent {
       : {}),
     ...(event.currency ? { currency: event.currency } : {}),
     ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+    aggregateCountry: country,
   };
 }
 
@@ -624,19 +414,27 @@ function appendBatchEvents(
 ) {
   for (const batch of batches) {
     for (let index = 0; index < batch.events.length; index += 1) {
-      const event = batch.events[index];
-      rows.push({
-        ...event,
-        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
-        _creationTime: batch._creationTime,
-        siteId: batch.siteId,
-        sessionId: batch.sessionId,
-        visitorId: batch.visitorId,
-      });
+      rows.push(eventFromBatch(batch, index));
     }
   }
 }
 
+function eventFromBatch(
+  batch: Doc<"eventBatches">,
+  index: number,
+): RawJourneyEvent {
+  const { aggregateCountry: _aggregateCountry, ...event } = batch.events[index]!;
+  void _aggregateCountry;
+  return {
+    ...event,
+    _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+    _creationTime: batch._creationTime,
+    siteId: batch.siteId,
+    sessionId: batch.sessionId,
+    visitorId: batch.visitorId,
+  };
+}
+
 function eventContext(event: {
   country?: string;
   city?: string;