@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.2

package/src/component/eventStore.ts

@@ -1,9 +1,13 @@
 import type { Doc, Id } from "./_generated/dataModel.js";
 import type { MutationCtx, QueryCtx } from "./_generated/server.js";
 import {
+  JOURNEY_READ_BYTE_RESERVE,
+  JOURNEY_READ_DOCUMENT_RESERVE,
+  MAX_BATCH_EVENTS,
   MAX_EVENTS_PER_SESSION_WINDOW,
   RATE_LIMIT_WINDOW_MS,
 } from "./constants.js";
+import { fail } from "./errors.js";
 import type { TrackerEvent } from "./validators.js";
 
 export type TelemetryMatch = {
@@ -28,6 +32,16 @@ type BufferedEventBatch = {
 
 export type EventBatchBuffer = Map<string, BufferedEventBatch>;
 
+type MigrationBatch = Pick<
+  Doc<"eventBatches">,
+  "_id" | "visitorId" | "events" | "migrationVersion"
+>;
+
+const migrationBatchCache = new WeakMap<
+  MutationCtx,
+  Map<string, MigrationBatch[]>
+>();
+
 export type JourneyCursor = {
   timestamp: number;
   creationTime: number;
@@ -107,12 +121,24 @@ export async function findTelemetry(
   }
   const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
   if (legacyEvent) return { match: legacyEvent, saturated };
-  const batch = batches.find((candidate) =>
-    candidate.events.some((nested) => nested.eventId === event.eventId)
-  );
+  let batch: Doc<"eventBatches"> | undefined;
+  let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
+  for (const candidate of batches) {
+    nestedEvent = candidate.events.find((nested) =>
+      nested.eventId === event.eventId
+    );
+    if (nestedEvent) {
+      batch = candidate;
+      break;
+    }
+  }
   return {
-    match: batch
-      ? { sessionId: batch.sessionId, visitorId: batch.visitorId }
+    match: batch && nestedEvent
+      ? {
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+        ...eventContext(nestedEvent),
+      }
       : null,
     saturated,
   };
@@ -165,6 +191,84 @@ export async function flushEventBatches(
   }
 }
 
+export async function migrateLegacyEventToBatch(
+  ctx: MutationCtx,
+  event: Doc<"events">,
+) {
+  const bucketStart = eventBucketStart(event.timestamp);
+  let transactionCache = migrationBatchCache.get(ctx);
+  if (!transactionCache) {
+    transactionCache = new Map();
+    migrationBatchCache.set(ctx, transactionCache);
+  }
+  const cacheKey = JSON.stringify([
+    event.siteId,
+    event.sessionId,
+    bucketStart,
+  ]);
+  let batches = transactionCache.get(cacheKey);
+  if (!batches) {
+    batches = await ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", event.siteId)
+          .eq("sessionId", event.sessionId)
+          .eq("bucketStart", bucketStart),
+      )
+      .order("desc")
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
+    transactionCache.set(cacheKey, batches);
+  }
+  const duplicateBatch = batches.find((batch) =>
+    batch.events.some((candidate) => candidate.eventId === event.eventId)
+  );
+  const context = eventContext(event);
+  if (duplicateBatch) {
+    if (Object.keys(context).length > 0) {
+      const events = duplicateBatch.events.map((candidate) =>
+        candidate.eventId === event.eventId
+          ? { ...candidate, ...context }
+          : candidate
+      );
+      await ctx.db.patch("eventBatches", duplicateBatch._id, {
+        events,
+      });
+      duplicateBatch.events = events;
+    }
+    await ctx.db.delete("events", event._id);
+    return;
+  }
+  const migratedEvent = {
+    ...eventForBatch(event),
+    ...context,
+  };
+  const latestBatch = batches[0];
+  if (
+    latestBatch?.migrationVersion === 1 &&
+    latestBatch.visitorId === event.visitorId &&
+    latestBatch.events.length < MAX_BATCH_EVENTS
+  ) {
+    const events = [...latestBatch.events, migratedEvent];
+    await ctx.db.patch("eventBatches", latestBatch._id, {
+      events,
+    });
+    latestBatch.events = events;
+  } else {
+    const batch = {
+      siteId: event.siteId,
+      sessionId: event.sessionId,
+      visitorId: event.visitorId,
+      bucketStart,
+      events: [migratedEvent],
+      migrationVersion: 1 as const,
+    };
+    const batchId = await ctx.db.insert("eventBatches", batch);
+    batches.unshift({ _id: batchId, ...batch });
+  }
+  await ctx.db.delete("events", event._id);
+}
+
 export async function loadSessionJourneyEvents(
   ctx: QueryCtx,
   input: {
@@ -255,19 +359,7 @@ export async function loadSessionJourneyEvents(
     ...event,
     _id: String(event._id),
   }));
-  for (const batch of batches) {
-    for (let index = 0; index < batch.events.length; index += 1) {
-      const event = batch.events[index];
-      rows.push({
-        ...event,
-        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
-        _creationTime: batch._creationTime,
-        siteId: batch.siteId,
-        sessionId: batch.sessionId,
-        visitorId: batch.visitorId,
-      });
-    }
-  }
+  appendBatchEvents(rows, batches);
   return rows
     .filter((event) =>
       event.timestamp >= input.from &&
@@ -278,6 +370,77 @@ export async function loadSessionJourneyEvents(
     .slice(0, input.resultLimit);
 }
 
+export async function loadVisitorJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    visitorId: string;
+    from: number;
+    to: number;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const firstBucket = eventBucketStart(input.from);
+  const lastBucket = eventBucketStart(input.to);
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let qualifyingEvents = 0;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_visitorId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("visitorId", input.visitorId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
+      )
+      .order("asc");
+    for await (const batch of query) {
+      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+        break;
+      }
+      batches.push(batch);
+      await requireJourneyReadHeadroom(ctx);
+      qualifyingEvents += batch.events.filter((event) =>
+        event.timestamp >= input.from && event.timestamp <= input.to
+      ).length;
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
+    }
+    return batches;
+  };
+  const legacyEvents: Doc<"events">[] = [];
+  const legacyQuery = ctx.db
+    .query("events")
+    .withIndex("by_siteId_and_visitorId_and_timestamp", (range) =>
+      range
+        .eq("siteId", input.siteId)
+        .eq("visitorId", input.visitorId)
+        .gte("timestamp", input.from)
+        .lte("timestamp", input.to),
+    )
+    .order("asc");
+  for await (const event of legacyQuery) {
+    legacyEvents.push(event);
+    await requireJourneyReadHeadroom(ctx);
+    if (legacyEvents.length >= input.resultLimit) break;
+  }
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
+    ...event,
+    _id: String(event._id),
+  }));
+  appendBatchEvents(rows, batches);
+  return rows
+    .filter((event) =>
+      event.timestamp >= input.from && event.timestamp <= input.to
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
 export function compareJourneyEvents(
   left: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   right: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
@@ -455,6 +618,62 @@ function eventForBatch(event: TrackerEvent): TrackerEvent {
   };
 }
 
+function appendBatchEvents(
+  rows: RawJourneyEvent[],
+  batches: Doc<"eventBatches">[],
+) {
+  for (const batch of batches) {
+    for (let index = 0; index < batch.events.length; index += 1) {
+      const event = batch.events[index];
+      rows.push({
+        ...event,
+        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+        _creationTime: batch._creationTime,
+        siteId: batch.siteId,
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+      });
+    }
+  }
+}
+
+function eventContext(event: {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+}) {
+  return {
+    ...(event.country ? { country: event.country } : {}),
+    ...(event.city ? { city: event.city } : {}),
+    ...(event.latitude !== undefined ? { latitude: event.latitude } : {}),
+    ...(event.longitude !== undefined ? { longitude: event.longitude } : {}),
+    ...(event.browser ? { browser: event.browser } : {}),
+    ...(event.os ? { os: event.os } : {}),
+    ...(event.device ? { device: event.device } : {}),
+  };
+}
+
+async function requireJourneyReadHeadroom(ctx: QueryCtx) {
+  const metrics = await ctx.meta.getTransactionMetrics();
+  if (
+    metrics.bytesRead.remaining < JOURNEY_READ_BYTE_RESERVE ||
+    metrics.documentsRead.remaining < JOURNEY_READ_DOCUMENT_RESERVE
+  ) {
+    fail(
+      "LIMIT_EXCEEDED",
+      "visitor journey exceeds the safe read budget",
+      {
+        bytesRead: metrics.bytesRead.used,
+        documentsRead: metrics.documentsRead.used,
+      },
+    );
+  }
+}
+
 function compareJourneyEventToCursor(
   event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   cursor: JourneyCursor,

package/src/component/migrations.ts

@@ -0,0 +1,52 @@
+import { Migrations } from "@convex-dev/migrations";
+import { v } from "convex/values";
+import { components } from "./_generated/api.js";
+import { internalMutation, internalQuery } from "./_generated/server.js";
+import { migrateLegacyEventToBatch } from "./eventStore.js";
+import schema from "./schema.js";
+
+const migrations = new Migrations(components.migrations, {
+  internalMutation,
+  schema,
+});
+
+export const migrateLegacyEvents = migrations.define({
+  table: "events",
+  batchSize: 16,
+  migrateOne: migrateLegacyEventToBatch,
+});
+
+export const legacyEventsStatus = internalQuery({
+  args: {},
+  returns: v.object({
+    complete: v.boolean(),
+    sourceEmpty: v.boolean(),
+    migrationState: v.union(
+      v.literal("inProgress"),
+      v.literal("success"),
+      v.literal("failed"),
+      v.literal("canceled"),
+      v.literal("unknown"),
+    ),
+    processed: v.number(),
+    sampleRemainingId: v.union(v.id("events"), v.null()),
+  }),
+  handler: async (ctx) => {
+    const [remaining, statuses] = await Promise.all([
+      ctx.db.query("events").first(),
+      migrations.getStatus(ctx, {
+        migrations: ["migrations:migrateLegacyEvents"],
+      }),
+    ]);
+    const migrationStatus = statuses[0];
+    const migrationState = migrationStatus?.state ?? "unknown";
+    const sourceEmpty = remaining === null;
+    return {
+      complete: sourceEmpty && migrationState === "success",
+      sourceEmpty,
+      migrationState,
+      processed: migrationStatus?.processed ?? 0,
+      sampleRemainingId: remaining?._id ?? null,
+    };
+  },
+});

package/src/component/reports.ts

@@ -25,6 +25,7 @@ import {
   hydrateEventContexts,
   type JourneyCursor,
   loadSessionJourneyEvents,
+  loadVisitorJourneyEvents,
   type RawJourneyEvent,
 } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
@@ -450,47 +451,17 @@ export const visitorJourney = query({
     if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
       fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
     }
-    const sessionGroups = await Promise.all(
-      siteIds.map((siteId) =>
-        ctx.db
-          .query("sessions")
-          .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
-            range
-              .eq("siteId", siteId)
-              .eq("visitorId", args.visitorId)
-              .gte("lastSeenAt", args.from),
-          )
-          .order("asc")
-          .take(MAX_JOURNEY_EVENTS + 1),
-      ),
-    );
-    const sessions = sessionGroups
-      .flat()
-      .filter((session) => session.startedAt <= args.to)
-      .sort((left, right) => left.startedAt - right.startedAt)
-      .slice(0, MAX_JOURNEY_EVENTS);
     const rows: RawJourneyEvent[] = [];
-    let overlappingSessionsRead = 0;
-    for (const session of sessions) {
-      const resultIsFull = rows.length >= limit;
-      const cutoff = resultIsFull
+    for (const siteId of siteIds) {
+      const cutoff = rows.length >= limit
         ? rows[limit - 1]!.timestamp
         : args.to;
-      if (session.startedAt > cutoff) break;
-      if (resultIsFull && ++overlappingSessionsRead > 8) {
-        fail(
-          "LIMIT_EXCEEDED",
-          "visitor journey has too many overlapping sessions",
-          { maximumOverlappingSessions: 8 },
-        );
-      }
-      rows.push(...await loadSessionJourneyEvents(ctx, {
-        siteId: session.siteId,
-        sessionId: session.sessionId,
+      rows.push(...await loadVisitorJourneyEvents(ctx, {
+        siteId,
+        visitorId: args.visitorId,
         from: args.from,
-        to: Math.min(args.to, cutoff),
-        cursor: null,
-        resultLimit: resultIsFull ? limit : limit - rows.length,
+        to: cutoff,
+        resultLimit: limit,
       }));
       rows.sort(compareJourneyEvents);
       if (rows.length > limit) rows.length = limit;

package/src/component/schema.ts

@@ -3,12 +3,12 @@ import { v } from "convex/values";
 import {
   affiliateFieldsValidator,
   aggregateFieldsValidator,
+  batchedEventValidator,
   storedEventFieldsValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
-  trackerEventValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -67,17 +67,19 @@ export default defineSchema({
       "sessionId",
       "timestamp",
     ])
-    .index("by_siteId_and_visitorId_and_timestamp", {
-      fields: ["siteId", "visitorId", "timestamp"],
-      staged: true,
-    }),
+    .index("by_siteId_and_visitorId_and_timestamp", [
+      "siteId",
+      "visitorId",
+      "timestamp",
+    ]),
 
   eventBatches: defineTable({
     siteId: v.id("sites"),
     sessionId: v.string(),
     visitorId: v.string(),
     bucketStart: v.number(),
-    events: v.array(trackerEventValidator),
+    events: v.array(batchedEventValidator),
+    migrationVersion: v.optional(v.literal(1)),
   })
     .index("by_siteId_and_sessionId_and_bucketStart", [
       "siteId",
@@ -85,10 +87,11 @@ export default defineSchema({
       "bucketStart",
     ])
     .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
-    .index("by_siteId_and_visitorId_and_bucketStart", {
-      fields: ["siteId", "visitorId", "bucketStart"],
-      staged: true,
-    }),
+    .index("by_siteId_and_visitorId_and_bucketStart", [
+      "siteId",
+      "visitorId",
+      "bucketStart",
+    ]),
 
   aggregateBuckets: defineTable(aggregateFieldsValidator)
     .index("by_siteId_and_granularity_and_bucketStart_and_shard", [

package/src/component/validators.ts

@@ -49,6 +49,10 @@ export const ingestContextValidator = v.object({
   device: v.optional(v.string()),
 });
 
+export const batchedEventValidator = trackerEventValidator.extend(
+  ingestContextValidator.fields,
+);
+
 export const siteFieldsValidator = v.object({
   ownerId: v.string(),
   name: v.string(),

package/src/test.ts

@@ -1,4 +1,5 @@
 /// <reference types="vite/client" />
+import migrationsComponent from "@convex-dev/migrations/test";
 import type { TestConvex } from "convex-test";
 import type { GenericSchema, SchemaDefinition } from "convex/server";
 import schema from "./component/schema.js";
@@ -15,5 +16,6 @@ export function register(
   name: string = "rastroAnalytics",
 ) {
   t.registerComponent(name, schema, modules);
+  migrationsComponent.register(t, `${name}/migrations`);
 }
 export default { register, schema, modules };