@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.2

package/src/component/eventStore.ts

@@ -0,0 +1,684 @@
+import type { Doc, Id } from "./_generated/dataModel.js";
+import type { MutationCtx, QueryCtx } from "./_generated/server.js";
+import {
+  JOURNEY_READ_BYTE_RESERVE,
+  JOURNEY_READ_DOCUMENT_RESERVE,
+  MAX_BATCH_EVENTS,
+  MAX_EVENTS_PER_SESSION_WINDOW,
+  RATE_LIMIT_WINDOW_MS,
+} from "./constants.js";
+import { fail } from "./errors.js";
+import type { TrackerEvent } from "./validators.js";
+
+export type TelemetryMatch = {
+  sessionId: string;
+  visitorId: string;
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+};
+
+type BufferedEventBatch = {
+  siteId: Id<"sites">;
+  sessionId: string;
+  visitorId: string;
+  bucketStart: number;
+  events: TrackerEvent[];
+};
+
+export type EventBatchBuffer = Map<string, BufferedEventBatch>;
+
+type MigrationBatch = Pick<
+  Doc<"eventBatches">,
+  "_id" | "visitorId" | "events" | "migrationVersion"
+>;
+
+const migrationBatchCache = new WeakMap<
+  MutationCtx,
+  Map<string, MigrationBatch[]>
+>();
+
+export type JourneyCursor = {
+  timestamp: number;
+  creationTime: number;
+  id: string;
+};
+
+export type RawJourneyEvent = Omit<Doc<"events">, "_id"> & { _id: string };
+
+export function eventBucketStart(timestamp: number): number {
+  return Math.floor(timestamp / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+}
+
+export function eventBatchKey(sessionId: string, bucketStart: number): string {
+  return JSON.stringify([sessionId, bucketStart]);
+}
+
+export async function inspectTelemetryMinute(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+    candidateEventIds: Set<string>;
+  },
+) {
+  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, input);
+  const eventIds = new Set([
+    ...legacyEvents.map((event) => event.eventId),
+    ...batches.flatMap((batch) =>
+      batch.events.map((event) => event.eventId),
+    ),
+  ]);
+  if (legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW) {
+    await Promise.all(
+      [...input.candidateEventIds]
+        .filter((eventId) => !eventIds.has(eventId))
+        .map(async (eventId) => {
+          const legacy = await findLegacyTelemetry(ctx, {
+            ...input,
+            eventId,
+          });
+          if (legacy) eventIds.add(eventId);
+        }),
+    );
+  }
+  return {
+    eventIds,
+    persistedCount: telemetryMinuteCount(legacyEvents, batches),
+  };
+}
+
+export async function findTelemetry(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+): Promise<{ match: TelemetryMatch | null; saturated: boolean }> {
+  const bucketStart = eventBucketStart(event.timestamp);
+  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, {
+    siteId,
+    sessionId: event.sessionId,
+    bucketStart,
+  });
+  const persistedCount = telemetryMinuteCount(legacyEvents, batches);
+  let legacyEvent = legacyEvents.find((candidate) =>
+    candidate.eventId === event.eventId
+  );
+  if (
+    !legacyEvent &&
+    legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW
+  ) {
+    legacyEvent = await findLegacyTelemetry(ctx, {
+      siteId,
+      sessionId: event.sessionId,
+      bucketStart,
+      eventId: event.eventId,
+    }) ?? undefined;
+  }
+  const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
+  if (legacyEvent) return { match: legacyEvent, saturated };
+  let batch: Doc<"eventBatches"> | undefined;
+  let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
+  for (const candidate of batches) {
+    nestedEvent = candidate.events.find((nested) =>
+      nested.eventId === event.eventId
+    );
+    if (nestedEvent) {
+      batch = candidate;
+      break;
+    }
+  }
+  return {
+    match: batch && nestedEvent
+      ? {
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+        ...eventContext(nestedEvent),
+      }
+      : null,
+    saturated,
+  };
+}
+
+export function bufferEventBatch(
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  buffer: EventBatchBuffer,
+) {
+  const bucketStart = eventBucketStart(event.timestamp);
+  const key = eventBatchKey(event.sessionId, bucketStart);
+  const existing = buffer.get(key);
+  if (existing) {
+    if (existing.visitorId !== event.visitorId) {
+      throw new Error("event batch visitor does not match its session");
+    }
+    existing.events.push(eventForBatch(event));
+    return;
+  }
+  buffer.set(key, {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    bucketStart,
+    events: [eventForBatch(event)],
+  });
+}
+
+export async function insertEventBatch(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+) {
+  await ctx.db.insert("eventBatches", {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    bucketStart: eventBucketStart(event.timestamp),
+    events: [eventForBatch(event)],
+  });
+}
+
+export async function flushEventBatches(
+  ctx: MutationCtx,
+  buffer: EventBatchBuffer,
+) {
+  for (const batch of buffer.values()) {
+    await ctx.db.insert("eventBatches", batch);
+  }
+}
+
+export async function migrateLegacyEventToBatch(
+  ctx: MutationCtx,
+  event: Doc<"events">,
+) {
+  const bucketStart = eventBucketStart(event.timestamp);
+  let transactionCache = migrationBatchCache.get(ctx);
+  if (!transactionCache) {
+    transactionCache = new Map();
+    migrationBatchCache.set(ctx, transactionCache);
+  }
+  const cacheKey = JSON.stringify([
+    event.siteId,
+    event.sessionId,
+    bucketStart,
+  ]);
+  let batches = transactionCache.get(cacheKey);
+  if (!batches) {
+    batches = await ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", event.siteId)
+          .eq("sessionId", event.sessionId)
+          .eq("bucketStart", bucketStart),
+      )
+      .order("desc")
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
+    transactionCache.set(cacheKey, batches);
+  }
+  const duplicateBatch = batches.find((batch) =>
+    batch.events.some((candidate) => candidate.eventId === event.eventId)
+  );
+  const context = eventContext(event);
+  if (duplicateBatch) {
+    if (Object.keys(context).length > 0) {
+      const events = duplicateBatch.events.map((candidate) =>
+        candidate.eventId === event.eventId
+          ? { ...candidate, ...context }
+          : candidate
+      );
+      await ctx.db.patch("eventBatches", duplicateBatch._id, {
+        events,
+      });
+      duplicateBatch.events = events;
+    }
+    await ctx.db.delete("events", event._id);
+    return;
+  }
+  const migratedEvent = {
+    ...eventForBatch(event),
+    ...context,
+  };
+  const latestBatch = batches[0];
+  if (
+    latestBatch?.migrationVersion === 1 &&
+    latestBatch.visitorId === event.visitorId &&
+    latestBatch.events.length < MAX_BATCH_EVENTS
+  ) {
+    const events = [...latestBatch.events, migratedEvent];
+    await ctx.db.patch("eventBatches", latestBatch._id, {
+      events,
+    });
+    latestBatch.events = events;
+  } else {
+    const batch = {
+      siteId: event.siteId,
+      sessionId: event.sessionId,
+      visitorId: event.visitorId,
+      bucketStart,
+      events: [migratedEvent],
+      migrationVersion: 1 as const,
+    };
+    const batchId = await ctx.db.insert("eventBatches", batch);
+    batches.unshift({ _id: batchId, ...batch });
+  }
+  await ctx.db.delete("events", event._id);
+}
+
+export async function loadSessionJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    from: number;
+    to: number;
+    cursor: JourneyCursor | null;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const startTimestamp = input.cursor?.timestamp ?? input.from;
+  const firstBucket = eventBucketStart(startTimestamp);
+  const lastBucket = eventBucketStart(input.to);
+  // Nested event tiebreakers are not index fields, so the boundary minute must
+  // be read completely before applying the cursor.
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let eventCount = 0;
+    const requiredEvents = input.resultLimit + MAX_EVENTS_PER_SESSION_WINDOW;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
+      )
+      .order("asc");
+    for await (const batch of query) {
+      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+        break;
+      }
+      batches.push(batch);
+      eventCount += batch.events.length;
+      if (eventCount >= requiredEvents) completedBucket = batch.bucketStart;
+    }
+    return batches;
+  };
+  const loadLegacyEvents = async () => {
+    const take = input.resultLimit + 1;
+    if (!input.cursor) {
+      return ctx.db
+        .query("events")
+        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+          range
+            .eq("siteId", input.siteId)
+            .eq("sessionId", input.sessionId)
+            .gte("timestamp", input.from)
+            .lte("timestamp", input.to),
+        )
+        .order("asc")
+        .take(take);
+    }
+    const rows = await ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .eq("timestamp", input.cursor!.timestamp)
+          .gt("_creationTime", input.cursor!.creationTime),
+      )
+      .order("asc")
+      .take(take);
+    if (rows.length < take) {
+      rows.push(...await ctx.db
+        .query("events")
+        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+          range
+            .eq("siteId", input.siteId)
+            .eq("sessionId", input.sessionId)
+            .gt("timestamp", input.cursor!.timestamp)
+            .lte("timestamp", input.to),
+        )
+        .order("asc")
+        .take(take - rows.length));
+    }
+    return rows;
+  };
+  const [legacyEvents, batches] = await Promise.all([
+    loadLegacyEvents(),
+    loadBatches(),
+  ]);
+  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
+    ...event,
+    _id: String(event._id),
+  }));
+  appendBatchEvents(rows, batches);
+  return rows
+    .filter((event) =>
+      event.timestamp >= input.from &&
+      event.timestamp <= input.to &&
+      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
+export async function loadVisitorJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    visitorId: string;
+    from: number;
+    to: number;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const firstBucket = eventBucketStart(input.from);
+  const lastBucket = eventBucketStart(input.to);
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let qualifyingEvents = 0;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_visitorId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("visitorId", input.visitorId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
+      )
+      .order("asc");
+    for await (const batch of query) {
+      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+        break;
+      }
+      batches.push(batch);
+      await requireJourneyReadHeadroom(ctx);
+      qualifyingEvents += batch.events.filter((event) =>
+        event.timestamp >= input.from && event.timestamp <= input.to
+      ).length;
+      if (qualifyingEvents >= input.resultLimit) {
+        completedBucket = batch.bucketStart;
+      }
+    }
+    return batches;
+  };
+  const legacyEvents: Doc<"events">[] = [];
+  const legacyQuery = ctx.db
+    .query("events")
+    .withIndex("by_siteId_and_visitorId_and_timestamp", (range) =>
+      range
+        .eq("siteId", input.siteId)
+        .eq("visitorId", input.visitorId)
+        .gte("timestamp", input.from)
+        .lte("timestamp", input.to),
+    )
+    .order("asc");
+  for await (const event of legacyQuery) {
+    legacyEvents.push(event);
+    await requireJourneyReadHeadroom(ctx);
+    if (legacyEvents.length >= input.resultLimit) break;
+  }
+  const batches = await loadBatches();
+  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
+    ...event,
+    _id: String(event._id),
+  }));
+  appendBatchEvents(rows, batches);
+  return rows
+    .filter((event) =>
+      event.timestamp >= input.from && event.timestamp <= input.to
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
+export function compareJourneyEvents(
+  left: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+  right: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+): number {
+  return left.timestamp - right.timestamp ||
+    left._creationTime - right._creationTime ||
+    left._id.localeCompare(right._id);
+}
+
+export async function hydrateEventContexts(
+  ctx: QueryCtx,
+  rows: RawJourneyEvent[],
+) {
+  const sessionPromises = new Map<string, Promise<Doc<"sessions"> | null>>();
+  for (const row of rows) {
+    const key = `${row.siteId}:${row.sessionId}`;
+    if (!sessionPromises.has(key)) {
+      sessionPromises.set(
+        key,
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", row.siteId).eq("sessionId", row.sessionId),
+          )
+          .unique(),
+      );
+    }
+  }
+  const sessions = new Map(
+    await Promise.all(
+      [...sessionPromises].map(async ([key, session]) =>
+        [key, await session] as const
+      ),
+    ),
+  );
+  return rows.map((row) => {
+    const session = sessions.get(`${row.siteId}:${row.sessionId}`);
+    const country = row.country ?? session?.country;
+    const city = row.city ?? session?.city;
+    const latitude = row.latitude ?? session?.latitude;
+    const longitude = row.longitude ?? session?.longitude;
+    return {
+      ...row,
+      ...(country ? { country } : {}),
+      ...(city ? { city } : {}),
+      ...(latitude !== undefined ? { latitude } : {}),
+      ...(longitude !== undefined ? { longitude } : {}),
+      browser: row.browser ?? session?.browser ?? "Other",
+      os: row.os ?? session?.os ?? "Other",
+      device: row.device ?? session?.device ?? "unknown",
+    };
+  });
+}
+
+export async function deleteTelemetryBefore(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    before: number;
+    batchSize: number;
+  },
+) {
+  const batchCutoff = eventBucketStart(input.before);
+  const limit = input.batchSize + 1;
+  const [legacyEvents, batches] = await Promise.all([
+    ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_timestamp", (range) =>
+        range.eq("siteId", input.siteId).lt("timestamp", input.before),
+      )
+      .take(limit),
+    ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_bucketStart", (range) =>
+        range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
+      )
+      .take(limit),
+  ]);
+  const rows = [...legacyEvents, ...batches].slice(0, input.batchSize);
+  for (const row of rows) {
+    if ("bucketStart" in row) {
+      await ctx.db.delete("eventBatches", row._id);
+    } else {
+      await ctx.db.delete("events", row._id);
+    }
+  }
+  return {
+    deleted: rows.length,
+    hasMore:
+      legacyEvents.length + batches.length > input.batchSize,
+  };
+}
+
+async function loadTelemetryMinute(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+  },
+) {
+  return Promise.all([
+    ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .gte("timestamp", input.bucketStart)
+          .lt("timestamp", input.bucketStart + RATE_LIMIT_WINDOW_MS),
+      )
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
+    ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .eq("bucketStart", input.bucketStart),
+      )
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
+  ]);
+}
+
+async function findLegacyTelemetry(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+    eventId: string;
+  },
+) {
+  const candidates = await ctx.db
+    .query("events")
+    .withIndex("by_siteId_and_eventId", (range) =>
+      range.eq("siteId", input.siteId).eq("eventId", input.eventId),
+    )
+    .take(2);
+  return candidates.find((candidate) =>
+    candidate.sessionId === input.sessionId &&
+    eventBucketStart(candidate.timestamp) === input.bucketStart
+  ) ?? null;
+}
+
+function telemetryMinuteCount(
+  legacyEvents: Array<{ eventId: string }>,
+  batches: Array<{ events: TrackerEvent[] }>,
+) {
+  return legacyEvents.length + batches.reduce(
+    (total, batch) => total + batch.events.length,
+    0,
+  );
+}
+
+function eventForBatch(event: TrackerEvent): TrackerEvent {
+  return {
+    eventId: event.eventId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    type: event.type,
+    ...(event.name ? { name: event.name } : {}),
+    path: event.path,
+    ...(event.referrer ? { referrer: event.referrer } : {}),
+    timestamp: event.timestamp,
+    sequence: event.sequence,
+    ...(event.target ? { target: event.target } : {}),
+    ...(event.href ? { href: event.href } : {}),
+    ...(event.properties ? { properties: event.properties } : {}),
+    ...(event.revenueCents !== undefined
+      ? { revenueCents: event.revenueCents }
+      : {}),
+    ...(event.currency ? { currency: event.currency } : {}),
+    ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+  };
+}
+
+function appendBatchEvents(
+  rows: RawJourneyEvent[],
+  batches: Doc<"eventBatches">[],
+) {
+  for (const batch of batches) {
+    for (let index = 0; index < batch.events.length; index += 1) {
+      const event = batch.events[index];
+      rows.push({
+        ...event,
+        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+        _creationTime: batch._creationTime,
+        siteId: batch.siteId,
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+      });
+    }
+  }
+}
+
+function eventContext(event: {
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+}) {
+  return {
+    ...(event.country ? { country: event.country } : {}),
+    ...(event.city ? { city: event.city } : {}),
+    ...(event.latitude !== undefined ? { latitude: event.latitude } : {}),
+    ...(event.longitude !== undefined ? { longitude: event.longitude } : {}),
+    ...(event.browser ? { browser: event.browser } : {}),
+    ...(event.os ? { os: event.os } : {}),
+    ...(event.device ? { device: event.device } : {}),
+  };
+}
+
+async function requireJourneyReadHeadroom(ctx: QueryCtx) {
+  const metrics = await ctx.meta.getTransactionMetrics();
+  if (
+    metrics.bytesRead.remaining < JOURNEY_READ_BYTE_RESERVE ||
+    metrics.documentsRead.remaining < JOURNEY_READ_DOCUMENT_RESERVE
+  ) {
+    fail(
+      "LIMIT_EXCEEDED",
+      "visitor journey exceeds the safe read budget",
+      {
+        bytesRead: metrics.bytesRead.used,
+        documentsRead: metrics.documentsRead.used,
+      },
+    );
+  }
+}
+
+function compareJourneyEventToCursor(
+  event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+  cursor: JourneyCursor,
+) {
+  return event.timestamp - cursor.timestamp ||
+    event._creationTime - cursor.creationTime ||
+    event._id.localeCompare(cursor.id);
+}