@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11

package/src/component/eventStore.ts

@@ -1,9 +1,12 @@
 import type { Doc, Id } from "./_generated/dataModel.js";
 import type { MutationCtx, QueryCtx } from "./_generated/server.js";
 import {
+  JOURNEY_READ_BYTE_RESERVE,
+  JOURNEY_READ_DOCUMENT_RESERVE,
   MAX_EVENTS_PER_SESSION_WINDOW,
   RATE_LIMIT_WINDOW_MS,
 } from "./constants.js";
+import { fail } from "./errors.js";
 import type { TrackerEvent } from "./validators.js";
 
 export type TelemetryMatch = {
@@ -34,7 +37,16 @@ export type JourneyCursor = {
   id: string;
 };
 
-export type RawJourneyEvent = Omit<Doc<"events">, "_id"> & { _id: string };
+export type RawJourneyEvent = Omit<
+  Doc<"eventBatches">["events"][number],
+  "aggregateCountry"
+> & {
+  _id: string;
+  _creationTime: number;
+  siteId: Id<"sites">;
+  sessionId: string;
+  visitorId: string;
+};
 
 export function eventBucketStart(timestamp: number): number {
   return Math.floor(timestamp / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -50,32 +62,15 @@ export async function inspectTelemetryMinute(
     siteId: Id<"sites">;
     sessionId: string;
     bucketStart: number;
-    candidateEventIds: Set<string>;
   },
 ) {
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, input);
-  const eventIds = new Set([
-    ...legacyEvents.map((event) => event.eventId),
-    ...batches.flatMap((batch) =>
-      batch.events.map((event) => event.eventId),
-    ),
-  ]);
-  if (legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW) {
-    await Promise.all(
-      [...input.candidateEventIds]
-        .filter((eventId) => !eventIds.has(eventId))
-        .map(async (eventId) => {
-          const legacy = await findLegacyTelemetry(ctx, {
-            ...input,
-            eventId,
-          });
-          if (legacy) eventIds.add(eventId);
-        }),
-    );
-  }
+  const batches = await loadTelemetryMinute(ctx, input);
+  const eventIds = new Set(
+    batches.flatMap((batch) => batch.events.map((event) => event.eventId)),
+  );
   return {
     eventIds,
-    persistedCount: telemetryMinuteCount(legacyEvents, batches),
+    persistedCount: telemetryMinuteCount(batches),
   };
 }
 
@@ -85,35 +80,33 @@ export async function findTelemetry(
   event: TrackerEvent,
 ): Promise<{ match: TelemetryMatch | null; saturated: boolean }> {
   const bucketStart = eventBucketStart(event.timestamp);
-  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, {
+  const batches = await loadTelemetryMinute(ctx, {
     siteId,
     sessionId: event.sessionId,
     bucketStart,
   });
-  const persistedCount = telemetryMinuteCount(legacyEvents, batches);
-  let legacyEvent = legacyEvents.find((candidate) =>
-    candidate.eventId === event.eventId
-  );
-  if (
-    !legacyEvent &&
-    legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW
-  ) {
-    legacyEvent = await findLegacyTelemetry(ctx, {
-      siteId,
-      sessionId: event.sessionId,
-      bucketStart,
-      eventId: event.eventId,
-    }) ?? undefined;
-  }
+  const persistedCount = telemetryMinuteCount(batches);
   const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
-  if (legacyEvent) return { match: legacyEvent, saturated };
-  const batch = batches.find((candidate) =>
-    candidate.events.some((nested) => nested.eventId === event.eventId)
-  );
+  let batch: Doc<"eventBatches"> | undefined;
+  let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
+  for (const candidate of batches) {
+    nestedEvent = candidate.events.find(
+      (nested) => nested.eventId === event.eventId,
+    );
+    if (nestedEvent) {
+      batch = candidate;
+      break;
+    }
+  }
   return {
-    match: batch
-      ? { sessionId: batch.sessionId, visitorId: batch.visitorId }
-      : null,
+    match:
+      batch && nestedEvent
+        ? {
+            sessionId: batch.sessionId,
+            visitorId: batch.visitorId,
+            ...eventContext(nestedEvent),
+          }
+        : null,
     saturated,
   };
 }
@@ -122,6 +115,7 @@ export function bufferEventBatch(
   siteId: Id<"sites">,
   event: TrackerEvent,
   buffer: EventBatchBuffer,
+  country: string,
 ) {
   const bucketStart = eventBucketStart(event.timestamp);
   const key = eventBatchKey(event.sessionId, bucketStart);
@@ -130,7 +124,7 @@ export function bufferEventBatch(
     if (existing.visitorId !== event.visitorId) {
       throw new Error("event batch visitor does not match its session");
     }
-    existing.events.push(eventForBatch(event));
+    existing.events.push(eventForBatch(event, country));
     return;
   }
   buffer.set(key, {
@@ -138,7 +132,7 @@ export function bufferEventBatch(
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart,
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -146,13 +140,14 @@ export async function insertEventBatch(
   ctx: MutationCtx,
   siteId: Id<"sites">,
   event: TrackerEvent,
+  country: string,
 ) {
   await ctx.db.insert("eventBatches", {
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
     bucketStart: eventBucketStart(event.timestamp),
-    events: [eventForBatch(event)],
+    events: [eventForBatch(event, country)],
   });
 }
 
@@ -183,8 +178,7 @@ export async function loadSessionJourneyEvents(
   // be read completely before applying the cursor.
   const loadBatches = async () => {
     const batches: Doc<"eventBatches">[] = [];
-    let eventCount = 0;
-    const requiredEvents = input.resultLimit + MAX_EVENTS_PER_SESSION_WINDOW;
+    let qualifyingEvents = 0;
     let completedBucket: number | undefined;
     const query = ctx.db
       .query("eventBatches")
@@ -197,82 +191,95 @@ export async function loadSessionJourneyEvents(
       )
       .order("asc");
     for await (const batch of query) {
-      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+      if (
+        completedBucket !== undefined &&
+        batch.bucketStart !== completedBucket
+      ) {
         break;
       }
       batches.push(batch);
-      eventCount += batch.events.length;
-      if (eventCount >= requiredEvents) completedBucket = batch.bucketStart;
+      qualifyingEvents += batch.events.reduce((count, _event, index) => {
+        const event = eventFromBatch(batch, index);
+        return (
+          count +
+          Number(
+            event.timestamp >= input.from &&
+              event.timestamp <= input.to &&
+              (!input.cursor ||
+                compareJourneyEventToCursor(event, input.cursor) > 0),
+          )
+        );
+      }, 0);
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
     }
     return batches;
   };
-  const loadLegacyEvents = async () => {
-    const take = input.resultLimit + 1;
-    if (!input.cursor) {
-      return ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gte("timestamp", input.from)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take);
-    }
-    const rows = await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = [];
+  appendBatchEvents(rows, batches);
+  return rows
+    .filter(
+      (event) =>
+        event.timestamp >= input.from &&
+        event.timestamp <= input.to &&
+        (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0),
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
+export async function loadVisitorJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    visitorId: string;
+    from: number;
+    to: number;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const firstBucket = eventBucketStart(input.from);
+  const lastBucket = eventBucketStart(input.to);
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let qualifyingEvents = 0;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_visitorId_and_bucketStart", (range) =>
         range
           .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("timestamp", input.cursor!.timestamp)
-          .gt("_creationTime", input.cursor!.creationTime),
+          .eq("visitorId", input.visitorId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
       )
-      .order("asc")
-      .take(take);
-    if (rows.length < take) {
-      rows.push(...await ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-          range
-            .eq("siteId", input.siteId)
-            .eq("sessionId", input.sessionId)
-            .gt("timestamp", input.cursor!.timestamp)
-            .lte("timestamp", input.to),
-        )
-        .order("asc")
-        .take(take - rows.length));
+      .order("asc");
+    for await (const batch of query) {
+      if (
+        completedBucket !== undefined &&
+        batch.bucketStart !== completedBucket
+      ) {
+        break;
+      }
+      batches.push(batch);
+      await requireJourneyReadHeadroom(ctx);
+      qualifyingEvents += batch.events.filter(
+        (event) => event.timestamp >= input.from && event.timestamp <= input.to,
+      ).length;
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
     }
-    return rows;
+    return batches;
   };
-  const [legacyEvents, batches] = await Promise.all([
-    loadLegacyEvents(),
-    loadBatches(),
-  ]);
-  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
-    ...event,
-    _id: String(event._id),
-  }));
-  for (const batch of batches) {
-    for (let index = 0; index < batch.events.length; index += 1) {
-      const event = batch.events[index];
-      rows.push({
-        ...event,
-        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
-        _creationTime: batch._creationTime,
-        siteId: batch.siteId,
-        sessionId: batch.sessionId,
-        visitorId: batch.visitorId,
-      });
-    }
-  }
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = [];
+  appendBatchEvents(rows, batches);
   return rows
-    .filter((event) =>
-      event.timestamp >= input.from &&
-      event.timestamp <= input.to &&
-      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+    .filter(
+      (event) => event.timestamp >= input.from && event.timestamp <= input.to,
     )
     .sort(compareJourneyEvents)
     .slice(0, input.resultLimit);
@@ -282,9 +289,11 @@ export function compareJourneyEvents(
   left: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   right: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
 ): number {
-  return left.timestamp - right.timestamp ||
+  return (
+    left.timestamp - right.timestamp ||
     left._creationTime - right._creationTime ||
-    left._id.localeCompare(right._id);
+    left._id.localeCompare(right._id)
+  );
 }
 
 export async function hydrateEventContexts(
@@ -308,8 +317,8 @@ export async function hydrateEventContexts(
   }
   const sessions = new Map(
     await Promise.all(
-      [...sessionPromises].map(async ([key, session]) =>
-        [key, await session] as const
+      [...sessionPromises].map(
+        async ([key, session]) => [key, await session] as const,
       ),
     ),
   );
@@ -342,32 +351,19 @@ export async function deleteTelemetryBefore(
 ) {
   const batchCutoff = eventBucketStart(input.before);
   const limit = input.batchSize + 1;
-  const [legacyEvents, batches] = await Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range.eq("siteId", input.siteId).lt("timestamp", input.before),
-      )
-      .take(limit),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_bucketStart", (range) =>
-        range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
-      )
-      .take(limit),
-  ]);
-  const rows = [...legacyEvents, ...batches].slice(0, input.batchSize);
+  const batches = await ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_bucketStart", (range) =>
+      range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
+    )
+    .take(limit);
+  const rows = batches.slice(0, input.batchSize);
   for (const row of rows) {
-    if ("bucketStart" in row) {
-      await ctx.db.delete("eventBatches", row._id);
-    } else {
-      await ctx.db.delete("events", row._id);
-    }
+    await ctx.db.delete("eventBatches", row._id);
   }
   return {
     deleted: rows.length,
-    hasMore:
-      legacyEvents.length + batches.length > input.batchSize,
+    hasMore: batches.length > input.batchSize,
   };
 }
 
@@ -379,61 +375,25 @@ async function loadTelemetryMinute(
     bucketStart: number;
   },
 ) {
-  return Promise.all([
-    ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .gte("timestamp", input.bucketStart)
-          .lt("timestamp", input.bucketStart + RATE_LIMIT_WINDOW_MS),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-    ctx.db
-      .query("eventBatches")
-      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
-        range
-          .eq("siteId", input.siteId)
-          .eq("sessionId", input.sessionId)
-          .eq("bucketStart", input.bucketStart),
-      )
-      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
-  ]);
-}
-
-async function findLegacyTelemetry(
-  ctx: MutationCtx,
-  input: {
-    siteId: Id<"sites">;
-    sessionId: string;
-    bucketStart: number;
-    eventId: string;
-  },
-) {
-  const candidates = await ctx.db
-    .query("events")
-    .withIndex("by_siteId_and_eventId", (range) =>
-      range.eq("siteId", input.siteId).eq("eventId", input.eventId),
+  return ctx.db
+    .query("eventBatches")
+    .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+      range
+        .eq("siteId", input.siteId)
+        .eq("sessionId", input.sessionId)
+        .eq("bucketStart", input.bucketStart),
     )
-    .take(2);
-  return candidates.find((candidate) =>
-    candidate.sessionId === input.sessionId &&
-    eventBucketStart(candidate.timestamp) === input.bucketStart
-  ) ?? null;
+    .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
 }
 
-function telemetryMinuteCount(
-  legacyEvents: Array<{ eventId: string }>,
-  batches: Array<{ events: TrackerEvent[] }>,
-) {
-  return legacyEvents.length + batches.reduce(
-    (total, batch) => total + batch.events.length,
-    0,
-  );
+function telemetryMinuteCount(batches: Array<{ events: TrackerEvent[] }>) {
+  return batches.reduce((total, batch) => total + batch.events.length, 0);
 }
 
-function eventForBatch(event: TrackerEvent): TrackerEvent {
+function eventForBatch(
+  event: TrackerEvent,
+  country: string,
+): TrackerEvent & { aggregateCountry?: string } {
   return {
     eventId: event.eventId,
     sessionId: event.sessionId,
@@ -452,14 +412,78 @@ function eventForBatch(event: TrackerEvent): TrackerEvent {
       : {}),
     ...(event.currency ? { currency: event.currency } : {}),
     ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+    aggregateCountry: country,
+  };
+}
+
+function appendBatchEvents(
+  rows: RawJourneyEvent[],
+  batches: Doc<"eventBatches">[],
+) {
+  for (const batch of batches) {
+    for (let index = 0; index < batch.events.length; index += 1) {
+      rows.push(eventFromBatch(batch, index));
+    }
+  }
+}
+
+function eventFromBatch(
+  batch: Doc<"eventBatches">,
+  index: number,
+): RawJourneyEvent {
+  const { aggregateCountry: _aggregateCountry, ...event } =
+    batch.events[index]!;
+  void _aggregateCountry;
+  return {
+    ...event,
+    _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+    _creationTime: batch._creationTime,
+    siteId: batch.siteId,
+    sessionId: batch.sessionId,
+    visitorId: batch.visitorId,
+  };
+}
+
+function eventContext(event: {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+}) {
+  return {
+    ...(event.country ? { country: event.country } : {}),
+    ...(event.city ? { city: event.city } : {}),
+    ...(event.latitude !== undefined ? { latitude: event.latitude } : {}),
+    ...(event.longitude !== undefined ? { longitude: event.longitude } : {}),
+    ...(event.browser ? { browser: event.browser } : {}),
+    ...(event.os ? { os: event.os } : {}),
+    ...(event.device ? { device: event.device } : {}),
   };
 }
 
+async function requireJourneyReadHeadroom(ctx: QueryCtx) {
+  const metrics = await ctx.meta.getTransactionMetrics();
+  if (
+    metrics.bytesRead.remaining < JOURNEY_READ_BYTE_RESERVE ||
+    metrics.documentsRead.remaining < JOURNEY_READ_DOCUMENT_RESERVE
+  ) {
+    fail("LIMIT_EXCEEDED", "visitor journey exceeds the safe read budget", {
+      bytesRead: metrics.bytesRead.used,
+      documentsRead: metrics.documentsRead.used,
+    });
+  }
+}
+
 function compareJourneyEventToCursor(
   event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   cursor: JourneyCursor,
 ) {
-  return event.timestamp - cursor.timestamp ||
+  return (
+    event.timestamp - cursor.timestamp ||
     event._creationTime - cursor.creationTime ||
-    event._id.localeCompare(cursor.id);
+    event._id.localeCompare(cursor.id)
+  );
 }

package/src/component/funnels.ts

@@ -1,15 +1,9 @@
 import { v } from "convex/values";
 import { mutation, query } from "./_generated/server.js";
-import {
-  MAX_FUNNELS_PER_SITE,
-  MAX_FUNNEL_STEPS,
-} from "./constants.js";
+import { MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS } from "./constants.js";
 import { fail } from "./errors.js";
 import { cleanString, sanitizePath, sanitizeSlug } from "./sanitize.js";
-import {
-  funnelFieldsValidator,
-  funnelStepValidator,
-} from "./validators.js";
+import { funnelFieldsValidator, funnelStepValidator } from "./validators.js";
 
 const funnelDocumentValidator = funnelFieldsValidator.extend({
   _id: v.id("funnels"),
@@ -28,7 +22,8 @@ export const upsert = mutation({
   },
   returns: v.id("funnels"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
     if (args.steps.length < 2 || args.steps.length > MAX_FUNNEL_STEPS) {
       fail("INVALID_ARGUMENT", `funnels require 2-${MAX_FUNNEL_STEPS} steps`);
     }
@@ -37,7 +32,10 @@ export const upsert = mutation({
       args.conversionWindowMs < 60_000 ||
       args.conversionWindowMs > 90 * 86_400_000
     ) {
-      fail("INVALID_ARGUMENT", "conversionWindowMs must be between one minute and 90 days");
+      fail(
+        "INVALID_ARGUMENT",
+        "conversionWindowMs must be between one minute and 90 days",
+      );
     }
     let key: string;
     let steps: Array<{
@@ -49,9 +47,10 @@ export const upsert = mutation({
       key = sanitizeSlug(args.key, "key");
       steps = args.steps.map((step) => {
         const stepName = cleanString(step.name, 80);
-        const value = step.type === "pageview"
-          ? sanitizePath(step.value)
-          : cleanString(step.value, 80);
+        const value =
+          step.type === "pageview"
+            ? sanitizePath(step.value)
+            : cleanString(step.value, 80);
         if (!stepName || !value) {
           throw new Error("funnel steps require name and value");
         }
@@ -79,7 +78,8 @@ export const upsert = mutation({
     const now = Date.now();
     if (funnelId) {
       const funnel = await ctx.db.get("funnels", funnelId);
-      if (!funnel || funnel.siteId !== args.siteId) fail("NOT_FOUND", "funnel not found");
+      if (!funnel || funnel.siteId !== args.siteId)
+        fail("NOT_FOUND", "funnel not found");
       await ctx.db.patch("funnels", funnelId, {
         key,
         name,
@@ -96,7 +96,9 @@ export const upsert = mutation({
       .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
       .take(MAX_FUNNELS_PER_SITE + 1);
     if (funnels.length >= MAX_FUNNELS_PER_SITE) {
-      fail("LIMIT_EXCEEDED", "funnel limit reached", { limit: MAX_FUNNELS_PER_SITE });
+      fail("LIMIT_EXCEEDED", "funnel limit reached", {
+        limit: MAX_FUNNELS_PER_SITE,
+      });
     }
     return await ctx.db.insert("funnels", {
       siteId: args.siteId,
@@ -127,7 +129,8 @@ export const remove = mutation({
   returns: v.null(),
   handler: async (ctx, args) => {
     const funnel = await ctx.db.get("funnels", args.funnelId);
-    if (!funnel || funnel.siteId !== args.siteId) fail("NOT_FOUND", "funnel not found");
+    if (!funnel || funnel.siteId !== args.siteId)
+      fail("NOT_FOUND", "funnel not found");
     await ctx.db.delete("funnels", args.funnelId);
     return null;
   },