@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.2

package/src/component/ingest.ts

@@ -5,10 +5,13 @@ import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
 import {
   AGGREGATE_SHARDS,
+  MAX_BATCH_EVENT_GROUPS,
   DAY_MS,
+  HOURLY_AGGREGATE_SHARDS,
   DIMENSION_SLOTS,
   HOUR_MS,
   LIVE_SESSION_TTL_MS,
+  LIVE_SWEEP_INTERVAL_MS,
   MAX_BATCH_EVENTS,
   MAX_SITE_BYTES_PER_SHARD_WINDOW,
   MAX_SITE_EVENTS_PER_SHARD_WINDOW,
@@ -17,9 +20,20 @@ import {
   MAX_FUNNELS_PER_SITE,
   MAX_GOALS_PER_SITE,
   RATE_LIMIT_WINDOW_MS,
+  SECONDARY_AGGREGATE_SHARDS,
   SITE_INGEST_SHARDS,
 } from "./constants.js";
 import { fail } from "./errors.js";
+import {
+  bufferEventBatch,
+  eventBatchKey,
+  eventBucketStart,
+  type EventBatchBuffer,
+  findTelemetry,
+  flushEventBatches,
+  insertEventBatch,
+  inspectTelemetryMinute,
+} from "./eventStore.js";
 import {
   cleanString,
   ensureBatchBytes,
@@ -93,6 +107,35 @@ type SessionState = {
   affiliateExpiresAt?: number;
 };
 
+type SessionRateWindowState = {
+  siteId: Id<"sites">;
+  sessionId: string;
+  window: Doc<"ingestWindows"> | null;
+  eventCount: number;
+  byteCount: number;
+  dirty: boolean;
+};
+
+type CachedRow<T> = {
+  current: T | null;
+  dirty: boolean;
+};
+
+type CachedDocument<TableName extends keyof import("./_generated/dataModel.js").DataModel> =
+  Omit<Doc<TableName>, "_creationTime"> & { _creationTime?: number };
+
+type BatchState = {
+  sessions: Map<string, CachedRow<CachedDocument<"sessions">>>;
+  liveSessions: Map<string, CachedRow<CachedDocument<"liveSessions">>>;
+  visitorAttributions: Map<
+    string,
+    CachedRow<CachedDocument<"visitorAttributions">>
+  >;
+  funnelProgress: Map<string, CachedRow<CachedDocument<"funnelProgress">>>;
+  eventBatches: EventBatchBuffer;
+  liveSweepSites: Set<string>;
+};
+
 export const ingestBatch = mutation({
   args: {
     siteId: v.id("sites"),
@@ -129,27 +172,101 @@ export const ingestBatch = mutation({
     );
     const context = sanitizeContext(args.context);
     const aggregateDeltas: AggregateDeltas = new Map();
+    const sessionRateWindows = new Map<string, SessionRateWindowState>();
+    const batchState: BatchState = {
+      sessions: new Map(),
+      liveSessions: new Map(),
+      visitorAttributions: new Map(),
+      funnelProgress: new Map(),
+      eventBatches: new Map(),
+      liveSweepSites: new Set(),
+    };
     let goals: Doc<"goals">[] | null = null;
     let funnels: Doc<"funnels">[] | null = null;
     let accepted = 0;
     let duplicates = 0;
     let rejected = 0;
-    for (let eventIndex = 0; eventIndex < args.events.length; eventIndex += 1) {
-      const input = args.events[eventIndex];
-      let event: TrackerEvent;
+    const events = args.events.map((input) => {
       try {
-        event = sanitizeEvent(input, now);
+        return sanitizeEvent(input, now);
       } catch {
+        return null;
+      }
+    });
+    const duplicateInBatch = new Set<number>();
+    const eventIdsByGroup = new Map<string, Set<string>>();
+    const requestGroups = new Set<string>();
+    const dedupeGroups = new Map<
+      string,
+      { sessionId: string; bucketStart: number; eventIds: Set<string> }
+    >();
+    for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+      const event = events[eventIndex];
+      if (!event) continue;
+      const bucketStart = eventBucketStart(event.timestamp);
+      const groupKey = eventBatchKey(event.sessionId, bucketStart);
+      requestGroups.add(groupKey);
+      if (event.type === "heartbeat") continue;
+      const eventIds = eventIdsByGroup.get(groupKey) ?? new Set<string>();
+      if (eventIds.has(event.eventId)) {
+        duplicateInBatch.add(eventIndex);
+        continue;
+      }
+      eventIds.add(event.eventId);
+      eventIdsByGroup.set(groupKey, eventIds);
+      const group = dedupeGroups.get(groupKey) ?? {
+        sessionId: event.sessionId,
+        bucketStart,
+        eventIds: new Set<string>(),
+      };
+      group.eventIds.add(event.eventId);
+      dedupeGroups.set(groupKey, group);
+    }
+    if (requestGroups.size > MAX_BATCH_EVENT_GROUPS) {
+      fail(
+        "BATCH_TOO_LARGE",
+        `events may span at most ${MAX_BATCH_EVENT_GROUPS} session-minute groups`,
+        { limit: MAX_BATCH_EVENT_GROUPS },
+      );
+    }
+    const persistedEventIds = new Map<string, Set<string>>();
+    await Promise.all(
+      [...dedupeGroups].map(async ([key, group]) => {
+        const { eventIds, persistedCount } = await inspectTelemetryMinute(ctx, {
+          siteId: args.siteId,
+          sessionId: group.sessionId,
+          bucketStart: group.bucketStart,
+          candidateEventIds: group.eventIds,
+        });
+        const newCount = [...group.eventIds].filter((eventId) =>
+          !eventIds.has(eventId)
+        ).length;
+        if (
+          newCount > 0 &&
+          persistedCount + newCount > MAX_EVENTS_PER_SESSION_WINDOW
+        ) {
+          fail("RATE_LIMITED", "session event minute limit exceeded", {
+            limit: MAX_EVENTS_PER_SESSION_WINDOW,
+          });
+        }
+        persistedEventIds.set(key, eventIds);
+      }),
+    );
+
+    for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+      const event = events[eventIndex];
+      if (!event) {
         rejected += 1;
         continue;
       }
 
-      const existingSession = await ctx.db
-        .query("sessions")
-        .withIndex("by_siteId_and_sessionId", (range) =>
-          range.eq("siteId", args.siteId).eq("sessionId", event.sessionId),
-        )
-        .unique();
+      const sessionRow = await loadSession(
+        ctx,
+        args.siteId,
+        event.sessionId,
+        batchState,
+      );
+      const existingSession = sessionRow.current;
       if (existingSession && existingSession.visitorId !== event.visitorId) {
         rejected += 1;
         continue;
@@ -160,13 +277,19 @@ export const ingestBatch = mutation({
           rejected += 1;
           continue;
         }
-        const liveSession = await ctx.db
-          .query("liveSessions")
-          .withIndex("by_siteId_and_sessionId", (range) =>
-            range.eq("siteId", args.siteId).eq("sessionId", event.sessionId),
+        const liveSession = (
+          await loadLiveSession(
+            ctx,
+            args.siteId,
+            event.sessionId,
+            batchState,
           )
-          .unique();
-        if (liveSession?.lastHeartbeatEventId === event.eventId) {
+        ).current;
+        if (
+          liveSession &&
+          (liveSession.lastHeartbeatEventId === event.eventId ||
+            liveSession.recentHeartbeatEventIds?.includes(event.eventId))
+        ) {
           duplicates += 1;
           continue;
         }
@@ -177,6 +300,7 @@ export const ingestBatch = mutation({
             event.sessionId,
             eventByteCounts[eventIndex],
             now,
+            sessionRateWindows,
           ))
         ) {
           rejected += 1;
@@ -189,18 +313,18 @@ export const ingestBatch = mutation({
           event,
           context,
           now,
+          batchState,
         );
         accepted += 1;
         continue;
       }
 
-      const duplicate = await ctx.db
-        .query("events")
-        .withIndex("by_siteId_and_eventId", (range) =>
-          range.eq("siteId", args.siteId).eq("eventId", event.eventId),
-        )
-        .unique();
-      if (duplicate) {
+      if (
+        duplicateInBatch.has(eventIndex) ||
+        persistedEventIds
+          .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
+          ?.has(event.eventId)
+      ) {
         duplicates += 1;
         continue;
       }
@@ -212,6 +336,7 @@ export const ingestBatch = mutation({
           event.sessionId,
           eventByteCounts[eventIndex],
           now,
+          sessionRateWindows,
         ))
       ) {
         rejected += 1;
@@ -243,9 +368,13 @@ export const ingestBatch = mutation({
         financialConversion: false,
         trustedGoalKey: undefined,
         now,
+        batchState,
       });
+      bufferEventBatch(args.siteId, event, batchState.eventBatches);
       accepted += 1;
     }
+    await flushBatchState(ctx, batchState);
+    await flushSessionRateLimits(ctx, sessionRateWindows, now);
     await flushAggregates(ctx, aggregateDeltas);
     return { accepted, duplicates, rejected };
   },
@@ -261,14 +390,18 @@ export const trackConversion = mutation({
       fail("INVALID_ARGUMENT", "revenueCents must be an integer between 0 and 1000000000");
     }
     let event: TrackerEvent;
+    let trustedGoalKey: string | undefined;
     try {
+      trustedGoalKey = args.goalKey
+        ? sanitizeSlug(args.goalKey, "goalKey")
+        : undefined;
       event = sanitizeEvent(
         {
           eventId: sanitizeOpaqueId(args.eventId, "eventId"),
           sessionId: sanitizeOpaqueId(args.sessionId, "sessionId"),
           visitorId: sanitizeOpaqueId(args.visitorId, "visitorId"),
           type: "conversion",
-          name: args.goalKey ? sanitizeSlug(args.goalKey, "goalKey") : "conversion",
+          name: trustedGoalKey ?? "conversion",
           path: sanitizePath(args.path ?? "/conversion"),
           timestamp: args.timestamp,
           sequence: 0,
@@ -283,12 +416,6 @@ export const trackConversion = mutation({
     }
 
     const siteCurrency = site.currency ?? "USD";
-    if (event.currency !== siteCurrency) {
-      fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
-        siteCurrency,
-        conversionCurrency: event.currency ?? "USD",
-      });
-    }
     const priorConversion = await ctx.db
       .query("conversions")
       .withIndex("by_siteId_and_eventId", (range) =>
@@ -296,15 +423,23 @@ export const trackConversion = mutation({
       )
       .unique();
     if (priorConversion) {
+      if (!sameTrustedConversion(priorConversion, event, trustedGoalKey)) {
+        fail(
+          "CONFLICT",
+          "conversion eventId is already associated with a different payload",
+        );
+      }
       return { accepted: 0, duplicates: 1, rejected: 0 };
     }
+    if (event.currency !== siteCurrency) {
+      fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
+        siteCurrency,
+        conversionCurrency: event.currency ?? "USD",
+      });
+    }
 
-    const existingTelemetry = await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_eventId", (range) =>
-        range.eq("siteId", args.siteId).eq("eventId", event.eventId),
-      )
-      .unique();
+    const telemetry = await findTelemetry(ctx, args.siteId, event);
+    const existingTelemetry = telemetry.match;
     if (
       existingTelemetry &&
       (existingTelemetry.sessionId !== event.sessionId ||
@@ -321,17 +456,15 @@ export const trackConversion = mutation({
     if (existingSession && existingSession.visitorId !== event.visitorId) {
       fail("CONFLICT", "session belongs to a different visitor");
     }
-    const conversionContext: IngestContext = existingTelemetry
-      ? {
-          country: existingTelemetry.country,
-          city: existingTelemetry.city,
-          latitude: existingTelemetry.latitude,
-          longitude: existingTelemetry.longitude,
-          browser: existingTelemetry.browser,
-          os: existingTelemetry.os,
-          device: existingTelemetry.device,
-        }
-      : sanitizeContext(undefined);
+    const conversionContext: IngestContext = {
+      country: existingTelemetry?.country ?? existingSession?.country,
+      city: existingTelemetry?.city ?? existingSession?.city,
+      latitude: existingTelemetry?.latitude ?? existingSession?.latitude,
+      longitude: existingTelemetry?.longitude ?? existingSession?.longitude,
+      browser: existingTelemetry?.browser ?? existingSession?.browser,
+      os: existingTelemetry?.os ?? existingSession?.os,
+      device: existingTelemetry?.device ?? existingSession?.device,
+    };
 
     const aggregateDeltas: AggregateDeltas = new Map();
     await accountEvent(ctx, {
@@ -342,25 +475,134 @@ export const trackConversion = mutation({
       goals: [],
       funnels: [],
       aggregateDeltas,
-      recordTelemetry: existingTelemetry === null,
+      recordTelemetry: existingTelemetry === null && !telemetry.saturated,
       financialConversion: true,
-      trustedGoalKey: args.goalKey
-        ? sanitizeSlug(args.goalKey, "goalKey")
-        : undefined,
+      trustedGoalKey,
       now: Date.now(),
     });
+    if (!existingTelemetry && !telemetry.saturated) {
+      await insertEventBatch(ctx, args.siteId, event);
+    }
     await flushAggregates(ctx, aggregateDeltas);
     return { accepted: 1, duplicates: 0, rejected: 0 };
   },
 });
 
+function sameTrustedConversion(
+  existing: Doc<"conversions">,
+  event: TrackerEvent,
+  goalKey: string | undefined,
+): boolean {
+  return (
+    existing.source === "trusted" &&
+    existing.sessionId === event.sessionId &&
+    existing.visitorId === event.visitorId &&
+    existing.timestamp === event.timestamp &&
+    existing.path === event.path &&
+    existing.revenueCents === (event.revenueCents ?? 0) &&
+    existing.currency === (event.currency ?? "USD") &&
+    existing.goalKey === goalKey
+  );
+}
+
+async function loadSession(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  sessionId: string,
+  batchState: BatchState,
+): Promise<CachedRow<CachedDocument<"sessions">>> {
+  const cached = batchState.sessions.get(sessionId);
+  if (cached) return cached;
+  const current = await ctx.db
+    .query("sessions")
+    .withIndex("by_siteId_and_sessionId", (range) =>
+      range.eq("siteId", siteId).eq("sessionId", sessionId),
+    )
+    .unique();
+  const loaded = { current, dirty: false };
+  batchState.sessions.set(sessionId, loaded);
+  return loaded;
+}
+
+async function loadLiveSession(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  sessionId: string,
+  batchState: BatchState,
+): Promise<CachedRow<CachedDocument<"liveSessions">>> {
+  const cached = batchState.liveSessions.get(sessionId);
+  if (cached) return cached;
+  const current = await ctx.db
+    .query("liveSessions")
+    .withIndex("by_siteId_and_sessionId", (range) =>
+      range.eq("siteId", siteId).eq("sessionId", sessionId),
+    )
+    .unique();
+  const loaded = { current, dirty: false };
+  batchState.liveSessions.set(sessionId, loaded);
+  return loaded;
+}
+
+async function flushBatchState(ctx: MutationCtx, batchState: BatchState) {
+  for (const state of batchState.sessions.values()) {
+    if (state.dirty && state.current) {
+      await ctx.db.replace(
+        "sessions",
+        state.current._id,
+        withoutSystemFields(state.current),
+      );
+    }
+  }
+  for (const state of batchState.liveSessions.values()) {
+    if (state.dirty && state.current) {
+      await ctx.db.replace(
+        "liveSessions",
+        state.current._id,
+        withoutSystemFields(state.current),
+      );
+    }
+  }
+  for (const state of batchState.visitorAttributions.values()) {
+    if (state.dirty && state.current) {
+      await ctx.db.replace(
+        "visitorAttributions",
+        state.current._id,
+        withoutSystemFields(state.current),
+      );
+    }
+  }
+  for (const state of batchState.funnelProgress.values()) {
+    if (state.dirty && state.current) {
+      await ctx.db.replace(
+        "funnelProgress",
+        state.current._id,
+        withoutSystemFields(state.current),
+      );
+    }
+  }
+  await flushEventBatches(ctx, batchState.eventBatches);
+}
+
+function withoutSystemFields<
+  T extends { _id: unknown; _creationTime?: number },
+>(document: T): Omit<T, "_id" | "_creationTime"> {
+  const {
+    _id,
+    _creationTime,
+    ...fields
+  } = document;
+  void _id;
+  void _creationTime;
+  return fields;
+}
+
 async function accountEvent(
   ctx: MutationCtx,
   input: {
     siteId: Id<"sites">;
     event: TrackerEvent;
     context: IngestContext;
-    existingSession: Doc<"sessions"> | null;
+    existingSession: CachedDocument<"sessions"> | null;
     goals: Doc<"goals">[];
     funnels: Doc<"funnels">[];
     aggregateDeltas: AggregateDeltas;
@@ -368,6 +610,7 @@ async function accountEvent(
     financialConversion: boolean;
     trustedGoalKey: string | undefined;
     now: number;
+    batchState?: BatchState;
   },
 ) {
   const { event } = input;
@@ -379,61 +622,57 @@ async function accountEvent(
         input.context,
         input.existingSession,
         input.financialConversion,
+        input.batchState,
       )
     : await accountTrustedSession(ctx, input.existingSession, event);
 
-  if (input.recordTelemetry) {
-    await ctx.db.insert("events", {
-      siteId: input.siteId,
-      eventId: event.eventId,
-      sessionId: event.sessionId,
-      visitorId: event.visitorId,
-      type: event.type,
-      ...(event.name ? { name: event.name } : {}),
-      path: event.path,
-      ...(event.referrer ? { referrer: event.referrer } : {}),
-      timestamp: event.timestamp,
-      sequence: event.sequence,
-      ...(event.target ? { target: event.target } : {}),
-      ...(event.href ? { href: event.href } : {}),
-      ...(event.properties ? { properties: event.properties } : {}),
-      ...(event.revenueCents !== undefined
-        ? { revenueCents: event.revenueCents }
-        : {}),
-      ...(event.currency ? { currency: event.currency } : {}),
-      ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
-      ...(input.context.country ? { country: input.context.country } : {}),
-      ...(input.context.city ? { city: input.context.city } : {}),
-      ...(input.context.latitude !== undefined
-        ? { latitude: input.context.latitude }
-        : {}),
-      ...(input.context.longitude !== undefined
-        ? { longitude: input.context.longitude }
-        : {}),
-      browser: input.context.browser ?? "Other",
-      os: input.context.os ?? "Other",
-      device: input.context.device ?? "unknown",
-    });
-  }
-
   const matchedGoalKeys: string[] = [];
-  for (const goal of input.goals) {
-    if (!goal.active || !goalMatches(goal, event)) continue;
-    const prior = await ctx.db
+  const matchingGoals = input.goals.filter((goal) =>
+    goal.active && goalMatches(goal, event)
+  );
+  if (matchingGoals.length > 1) {
+    fail("LIMIT_EXCEEDED", "active goals must use unique matchers");
+  }
+  for (const goal of matchingGoals) {
+    const completionEventId = JSON.stringify([
+      event.sessionId,
+      eventBucketStart(event.timestamp),
+      event.eventId,
+    ]);
+    const scopedPrior = (await ctx.db
       .query("goalCompletions")
       .withIndex("by_siteId_and_goalId_and_eventId", (range) =>
         range
           .eq("siteId", input.siteId)
           .eq("goalId", goal._id)
-          .eq("eventId", event.eventId),
+          .eq("eventId", completionEventId),
       )
-      .unique();
-    if (prior) continue;
+      .take(2))
+      .find((completion) => completion.idempotencyVersion === 2);
+    const legacyPrior = scopedPrior
+      ? undefined
+      : (await ctx.db
+          .query("goalCompletions")
+          .withIndex("by_siteId_and_goalId_and_eventId", (range) =>
+            range
+              .eq("siteId", input.siteId)
+              .eq("goalId", goal._id)
+              .eq("eventId", event.eventId),
+          )
+          .take(2))
+          .find((completion) =>
+            completion.idempotencyVersion === undefined &&
+            completion.sessionId === event.sessionId &&
+            eventBucketStart(completion.timestamp) ===
+              eventBucketStart(event.timestamp)
+          );
+    if (scopedPrior || legacyPrior) continue;
     await ctx.db.insert("goalCompletions", {
       siteId: input.siteId,
       goalId: goal._id,
       goalKey: goal.key,
-      eventId: event.eventId,
+      eventId: completionEventId,
+      idempotencyVersion: 2,
       sessionId: event.sessionId,
       visitorId: event.visitorId,
       timestamp: event.timestamp,
@@ -444,7 +683,7 @@ async function accountEvent(
 
   for (const funnel of input.funnels) {
     if (funnel.active) {
-      await advanceFunnel(ctx, input.siteId, funnel, event);
+      await advanceFunnel(ctx, input.siteId, funnel, event, input.batchState);
     }
   }
 
@@ -491,13 +730,14 @@ async function accountEvent(
       input.context,
       session,
       input.now,
+      input.batchState,
     );
   }
 }
 
 async function accountTrustedSession(
   ctx: MutationCtx,
-  existing: Doc<"sessions"> | null,
+  existing: CachedDocument<"sessions"> | null,
   event: TrackerEvent,
 ): Promise<SessionState> {
   if (!existing) {
@@ -529,18 +769,25 @@ async function upsertSession(
   siteId: Id<"sites">,
   event: TrackerEvent,
   context: IngestContext,
-  existing: Doc<"sessions"> | null,
+  existing: CachedDocument<"sessions"> | null,
   financialConversion: boolean,
+  batchState?: BatchState,
 ): Promise<SessionState> {
   const source = sourceFromReferrer(event.referrer);
   const utmSource = stringProperty(event.properties, "utm_source");
   const utmMedium = stringProperty(event.properties, "utm_medium");
   const utmCampaign = stringProperty(event.properties, "utm_campaign");
-  const attribution = await attributionForEvent(ctx, siteId, event, existing);
+  const attribution = await attributionForEvent(
+    ctx,
+    siteId,
+    event,
+    existing,
+    batchState,
+  );
   if (existing) {
     const startedAt = Math.min(existing.startedAt, event.timestamp);
     const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
-    await ctx.db.patch("sessions", existing._id, {
+    const updates = {
       startedAt,
       lastSeenAt,
       entryPath: event.timestamp < existing.startedAt ? event.path : existing.entryPath,
@@ -566,7 +813,15 @@ async function upsertSession(
       browser: context.browser ?? existing.browser,
       os: context.os ?? existing.os,
       device: context.device ?? existing.device,
-    });
+    };
+    if (batchState) {
+      const state = batchState.sessions.get(event.sessionId);
+      if (!state) throw new Error("session cache was not initialized");
+      state.current = { ...existing, ...updates };
+      state.dirty = true;
+    } else {
+      await ctx.db.patch("sessions", existing._id, updates);
+    }
     return {
       isNewSession: false,
       isNewVisitorHour:
@@ -584,14 +839,27 @@ async function upsertSession(
     };
   }
 
-  const priorVisit = await ctx.db
+  const priorVisits = await ctx.db
     .query("sessions")
     .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
       range.eq("siteId", siteId).eq("visitorId", event.visitorId),
     )
     .order("desc")
     .take(1);
-  await ctx.db.insert("sessions", {
+  let priorVisit: CachedDocument<"sessions"> | null = priorVisits[0] ?? null;
+  if (batchState) {
+    for (const state of batchState.sessions.values()) {
+      const candidate = state.current;
+      if (
+        candidate?.visitorId === event.visitorId &&
+        candidate.sessionId !== event.sessionId &&
+        (!priorVisit || candidate.lastSeenAt > priorVisit.lastSeenAt)
+      ) {
+        priorVisit = candidate;
+      }
+    }
+  }
+  const fields: Omit<Doc<"sessions">, "_id" | "_creationTime"> = {
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
@@ -620,16 +888,25 @@ async function upsertSession(
       ? { currency: event.currency }
       : {}),
     ...(attribution ?? {}),
-  });
+  };
+  const sessionId = await ctx.db.insert("sessions", fields);
+  if (batchState) {
+    const state = batchState.sessions.get(event.sessionId);
+    if (!state) throw new Error("session cache was not initialized");
+    state.current = {
+      ...fields,
+      _id: sessionId,
+    };
+  }
   return {
     isNewSession: true,
     isNewVisitorHour:
-      priorVisit.length === 0 ||
-      Math.floor(priorVisit[0].lastSeenAt / HOUR_MS) <
+      priorVisit === null ||
+      Math.floor(priorVisit.lastSeenAt / HOUR_MS) <
         Math.floor(event.timestamp / HOUR_MS),
     isNewVisitorDay:
-      priorVisit.length === 0 ||
-      Math.floor(priorVisit[0].lastSeenAt / DAY_MS) <
+      priorVisit === null ||
+      Math.floor(priorVisit.lastSeenAt / DAY_MS) <
         Math.floor(event.timestamp / DAY_MS),
     startedAt: event.timestamp,
     source,
@@ -680,7 +957,8 @@ async function attributionForEvent(
   ctx: MutationCtx,
   siteId: Id<"sites">,
   event: TrackerEvent,
-  existingSession: Doc<"sessions"> | null,
+  existingSession: CachedDocument<"sessions"> | null,
+  batchState?: BatchState,
 ): Promise<AttributionSnapshot | undefined> {
   const sessionAttribution = existingSession
     ? attributionSnapshot(existingSession)
@@ -696,12 +974,25 @@ async function attributionForEvent(
     return sessionAttribution;
   }
 
-  const current = await ctx.db
-    .query("visitorAttributions")
-    .withIndex("by_siteId_and_visitorId", (range) =>
-      range.eq("siteId", siteId).eq("visitorId", event.visitorId),
-    )
-    .unique();
+  let attributionState = batchState?.visitorAttributions.get(event.visitorId);
+  if (batchState && !attributionState) {
+    const current = await ctx.db
+      .query("visitorAttributions")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", siteId).eq("visitorId", event.visitorId),
+      )
+      .unique();
+    attributionState = { current, dirty: false };
+    batchState.visitorAttributions.set(event.visitorId, attributionState);
+  }
+  const current = attributionState
+    ? attributionState.current
+    : await ctx.db
+        .query("visitorAttributions")
+        .withIndex("by_siteId_and_visitorId", (range) =>
+          range.eq("siteId", siteId).eq("visitorId", event.visitorId),
+        )
+        .unique();
   if (current) {
     const snapshot: AttributionSnapshot = {
       affiliateId: current.affiliateId,
@@ -733,22 +1024,30 @@ async function attributionForEvent(
 
   const attributedAt = event.timestamp;
   const expiresAt = attributedAt + affiliate.attributionWindowDays * DAY_MS;
+  const fields = {
+    siteId,
+    visitorId: event.visitorId,
+    affiliateId: affiliate._id,
+    affiliateSlug: affiliate.slug,
+    attributedAt,
+    expiresAt,
+  };
   if (current) {
-    await ctx.db.patch("visitorAttributions", current._id, {
-      affiliateId: affiliate._id,
-      affiliateSlug: affiliate.slug,
-      attributedAt,
-      expiresAt,
-    });
+    if (attributionState) {
+      attributionState.current = { ...current, ...fields };
+      attributionState.dirty = true;
+    } else {
+      await ctx.db.patch("visitorAttributions", current._id, fields);
+    }
   } else {
-    await ctx.db.insert("visitorAttributions", {
-      siteId,
-      visitorId: event.visitorId,
-      affiliateId: affiliate._id,
-      affiliateSlug: affiliate.slug,
-      attributedAt,
-      expiresAt,
-    });
+    const attributionId = await ctx.db.insert("visitorAttributions", fields);
+    if (attributionState) {
+      attributionState.current = {
+        ...fields,
+        _id: attributionId,
+        _creationTime: event.timestamp,
+      };
+    }
   }
   return {
     affiliateId: affiliate._id,
@@ -761,14 +1060,15 @@ async function attributionForEvent(
 async function accountHeartbeat(
   ctx: MutationCtx,
   siteId: Id<"sites">,
-  session: Doc<"sessions">,
+  session: CachedDocument<"sessions">,
   event: TrackerEvent,
   context: IngestContext,
   now: number,
+  batchState?: BatchState,
 ) {
   const startedAt = Math.min(session.startedAt, event.timestamp);
   const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
-  await ctx.db.patch("sessions", session._id, {
+  const updates = {
     startedAt,
     lastSeenAt,
     exitPath: event.timestamp >= session.lastSeenAt
@@ -782,7 +1082,15 @@ async function accountHeartbeat(
     browser: context.browser ?? session.browser,
     os: context.os ?? session.os,
     device: context.device ?? session.device,
-  });
+  };
+  if (batchState) {
+    const state = batchState.sessions.get(event.sessionId);
+    if (!state) throw new Error("session cache was not initialized");
+    state.current = { ...session, ...updates };
+    state.dirty = true;
+  } else {
+    await ctx.db.patch("sessions", session._id, updates);
+  }
   await updateLiveSession(
     ctx,
     siteId,
@@ -799,6 +1107,7 @@ async function accountHeartbeat(
       affiliateSlug: session.affiliateSlug,
     },
     now,
+    batchState,
   );
 }
 
@@ -830,15 +1139,17 @@ async function consumeSiteRateLimit(
     window: Doc<"siteIngestWindows"> | null;
   }> = [];
   for (const [shard, increment] of increments) {
-    const window = await ctx.db
+    const rows = await ctx.db
       .query("siteIngestWindows")
       .withIndex("by_siteId_and_shard_and_windowStart", (range) =>
-        range
-          .eq("siteId", siteId)
-          .eq("shard", shard)
-          .eq("windowStart", windowStart),
+        range.eq("siteId", siteId).eq("shard", shard),
       )
-      .unique();
+      .order("desc")
+      .take(1);
+    const storedWindow = rows[0] ?? null;
+    const window = storedWindow?.windowStart === windowStart
+      ? storedWindow
+      : null;
     if (
       (window?.eventCount ?? 0) + increment.eventCount >
         MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
@@ -856,14 +1167,21 @@ async function consumeSiteRateLimit(
           SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
       });
     }
-    windows.push({ shard, increment, window });
+    windows.push({ shard, increment, window: storedWindow });
   }
 
   for (const { shard, increment, window } of windows) {
     if (window) {
       await ctx.db.patch("siteIngestWindows", window._id, {
-        eventCount: window.eventCount + increment.eventCount,
-        byteCount: window.byteCount + increment.byteCount,
+        windowStart,
+        eventCount:
+          window.windowStart === windowStart
+            ? window.eventCount + increment.eventCount
+            : increment.eventCount,
+        byteCount:
+          window.windowStart === windowStart
+            ? window.byteCount + increment.byteCount
+            : increment.byteCount,
         updatedAt: now,
       });
     } else {
@@ -886,43 +1204,69 @@ async function consumeRateLimit(
   sessionId: string,
   byteCount: number,
   now: number,
+  states: Map<string, SessionRateWindowState>,
 ): Promise<boolean> {
   const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
-  const window = await ctx.db
-    .query("ingestWindows")
-    .withIndex("by_siteId_and_sessionId_and_windowStart", (range) =>
-      range
-        .eq("siteId", siteId)
-        .eq("sessionId", sessionId)
-        .eq("windowStart", windowStart),
-    )
-    .unique();
+  let state = states.get(sessionId);
+  if (!state) {
+    const rows = await ctx.db
+      .query("ingestWindows")
+      .withIndex("by_siteId_and_sessionId_and_windowStart", (range) =>
+        range.eq("siteId", siteId).eq("sessionId", sessionId),
+      )
+      .order("desc")
+      .take(1);
+    const window = rows[0] ?? null;
+    state = {
+      siteId,
+      sessionId,
+      window,
+      eventCount: window?.windowStart === windowStart ? window.eventCount : 0,
+      byteCount: window?.windowStart === windowStart ? window.byteCount : 0,
+      dirty: false,
+    };
+    states.set(sessionId, state);
+  }
   if (
-    window &&
-    (window.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
-      window.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW)
+    state.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
+    state.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW
   ) {
     return false;
   }
-  if (window) {
-    await ctx.db.patch("ingestWindows", window._id, {
-      eventCount: window.eventCount + 1,
-      byteCount: window.byteCount + byteCount,
-      updatedAt: now,
-    });
-  } else {
-    await ctx.db.insert("ingestWindows", {
-      siteId,
-      sessionId,
-      windowStart,
-      eventCount: 1,
-      byteCount,
-      updatedAt: now,
-    });
-  }
+  state.eventCount += 1;
+  state.byteCount += byteCount;
+  state.dirty = true;
   return true;
 }
 
+async function flushSessionRateLimits(
+  ctx: MutationCtx,
+  states: Map<string, SessionRateWindowState>,
+  now: number,
+) {
+  const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+  for (const state of states.values()) {
+    if (!state.dirty) continue;
+    if (state.window) {
+      await ctx.db.patch("ingestWindows", state.window._id, {
+        windowStart,
+        eventCount: state.eventCount,
+        byteCount: state.byteCount,
+        updatedAt: now,
+      });
+    } else {
+      await ctx.db.insert("ingestWindows", {
+        siteId: state.siteId,
+        sessionId: state.sessionId,
+        windowStart,
+        eventCount: state.eventCount,
+        byteCount: state.byteCount,
+        updatedAt: now,
+      });
+    }
+  }
+}
+
 async function updateLiveSession(
   ctx: MutationCtx,
   siteId: Id<"sites">,
@@ -930,23 +1274,32 @@ async function updateLiveSession(
   context: IngestContext,
   session: SessionState,
   now: number,
+  batchState?: BatchState,
 ) {
-  const existing = await ctx.db
-    .query("liveSessions")
-    .withIndex("by_siteId_and_sessionId", (range) =>
-      range.eq("siteId", siteId).eq("sessionId", event.sessionId),
-    )
-    .unique();
-  const renewsExpiry =
-    !existing ||
-    existing.expiresAt <= now ||
-    event.type === "pageview" ||
-    event.type === "heartbeat";
-  const deadline = renewsExpiry
-    ? now + LIVE_SESSION_TTL_MS
-    : existing.expiresAt;
+  await ensureLiveSweep(ctx, siteId, now, batchState);
+  const liveState = batchState
+    ? await loadLiveSession(ctx, siteId, event.sessionId, batchState)
+    : undefined;
+  const existing = liveState
+    ? liveState.current
+    : await ctx.db
+        .query("liveSessions")
+        .withIndex("by_siteId_and_sessionId", (range) =>
+          range.eq("siteId", siteId).eq("sessionId", event.sessionId),
+        )
+        .unique();
+  const deadline = now + LIVE_SESSION_TTL_MS;
   if (existing) {
-    await ctx.db.patch("liveSessions", existing._id, {
+    const recentHeartbeatEventIds = event.type === "heartbeat"
+      ? [
+          ...(existing.recentHeartbeatEventIds ??
+            (existing.lastHeartbeatEventId
+              ? [existing.lastHeartbeatEventId]
+              : [])),
+          event.eventId,
+        ].slice(-8)
+      : undefined;
+    const updates = {
       visitorId: event.visitorId,
       path: event.path,
       lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
@@ -962,13 +1315,22 @@ async function updateLiveSession(
       pageviewCount: session.pageviewCount,
       eventCount: session.eventCount,
       ...(event.type === "heartbeat"
-        ? { lastHeartbeatEventId: event.eventId }
+        ? {
+            lastHeartbeatEventId: event.eventId,
+            recentHeartbeatEventIds,
+          }
         : {}),
-    });
+    };
+    if (liveState) {
+      liveState.current = { ...existing, ...updates };
+      liveState.dirty = true;
+    } else {
+      await ctx.db.patch("liveSessions", existing._id, updates);
+    }
     return;
   }
 
-  const liveSessionId = await ctx.db.insert("liveSessions", {
+  const fields: Omit<Doc<"liveSessions">, "_id" | "_creationTime"> = {
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
@@ -976,7 +1338,6 @@ async function updateLiveSession(
     startedAt: event.timestamp,
     lastSeenAt: event.timestamp,
     expiresAt: deadline,
-    expiryChainDeadline: deadline,
     ...(context.country ? { country: context.country } : {}),
     ...(context.city ? { city: context.city } : {}),
     ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
@@ -988,13 +1349,42 @@ async function updateLiveSession(
     pageviewCount: session.pageviewCount,
     eventCount: session.eventCount,
     ...(event.type === "heartbeat"
-      ? { lastHeartbeatEventId: event.eventId }
+      ? {
+          lastHeartbeatEventId: event.eventId,
+          recentHeartbeatEventIds: [event.eventId],
+        }
       : {}),
-  });
-  await ctx.scheduler.runAt(deadline, internal.live.expire, {
-    liveSessionId,
-    scheduledDeadline: deadline,
-  });
+  };
+  const liveSessionId = await ctx.db.insert("liveSessions", fields);
+  if (liveState) {
+    liveState.current = {
+      ...fields,
+      _id: liveSessionId,
+    };
+  }
+}
+
+async function ensureLiveSweep(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+  batchState?: BatchState,
+) {
+  const key = String(siteId);
+  if (batchState?.liveSweepSites.has(key)) return;
+  const existing = await ctx.db
+    .query("liveSweepStates")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (!existing) {
+    const scheduledAt = now + LIVE_SWEEP_INTERVAL_MS;
+    await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+    await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
+      siteId,
+      scheduledAt,
+    });
+  }
+  batchState?.liveSweepSites.add(key);
 }
 
 function goalMatches(goal: Doc<"goals">, event: TrackerEvent): boolean {
@@ -1009,20 +1399,38 @@ async function advanceFunnel(
   siteId: Id<"sites">,
   funnel: Doc<"funnels">,
   event: TrackerEvent,
+  batchState?: BatchState,
 ) {
-  const progress = await ctx.db
-    .query("funnelProgress")
-    .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
-      range
-        .eq("siteId", siteId)
-        .eq("funnelId", funnel._id)
-        .eq("visitorId", event.visitorId),
-    )
-    .unique();
+  const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
+  let progressState = batchState?.funnelProgress.get(cacheKey);
+  if (batchState && !progressState) {
+    const current = await ctx.db
+      .query("funnelProgress")
+      .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
+        range
+          .eq("siteId", siteId)
+          .eq("funnelId", funnel._id)
+          .eq("visitorId", event.visitorId),
+      )
+      .unique();
+    progressState = { current, dirty: false };
+    batchState.funnelProgress.set(cacheKey, progressState);
+  }
+  const progress = progressState
+    ? progressState.current
+    : await ctx.db
+        .query("funnelProgress")
+        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
+          range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnel._id)
+            .eq("visitorId", event.visitorId),
+        )
+        .unique();
   const matchesFirst = stepMatches(funnel.steps[0], event);
   if (!progress) {
     if (!matchesFirst) return;
-    await ctx.db.insert("funnelProgress", {
+    const fields: Omit<Doc<"funnelProgress">, "_id" | "_creationTime"> = {
       siteId,
       funnelId: funnel._id,
       visitorId: event.visitorId,
@@ -1030,20 +1438,40 @@ async function advanceFunnel(
       currentStep: 1,
       startedAt: event.timestamp,
       lastStepAt: event.timestamp,
-    });
+    };
+    const progressId = await ctx.db.insert("funnelProgress", fields);
+    if (progressState) {
+      progressState.current = {
+        ...fields,
+        _id: progressId,
+      };
+    }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
     return;
   }
   if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt) return;
   if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
     if (!matchesFirst) return;
-    await ctx.db.patch("funnelProgress", progress._id, {
+    const updated = {
+      ...progress,
       sessionId: event.sessionId,
       currentStep: 1,
       startedAt: event.timestamp,
       lastStepAt: event.timestamp,
-      completedAt: undefined,
-    });
+    };
+    delete updated.completedAt;
+    if (progressState) {
+      progressState.current = updated;
+      progressState.dirty = true;
+    } else {
+      await ctx.db.patch("funnelProgress", progress._id, {
+        sessionId: event.sessionId,
+        currentStep: 1,
+        startedAt: event.timestamp,
+        lastStepAt: event.timestamp,
+        completedAt: undefined,
+      });
+    }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
     return;
   }
@@ -1051,12 +1479,18 @@ async function advanceFunnel(
   if (!nextStep || !stepMatches(nextStep, event)) return;
   const currentStep = progress.currentStep + 1;
   const completed = currentStep === funnel.steps.length;
-  await ctx.db.patch("funnelProgress", progress._id, {
+  const updates = {
     sessionId: event.sessionId,
     currentStep,
     lastStepAt: event.timestamp,
     ...(completed ? { completedAt: event.timestamp } : {}),
-  });
+  };
+  if (progressState) {
+    progressState.current = { ...progress, ...updates };
+    progressState.dirty = true;
+  } else {
+    await ctx.db.patch("funnelProgress", progress._id, updates);
+  }
   await incrementFunnelStep(
     ctx,
     siteId,
@@ -1085,7 +1519,7 @@ async function incrementFunnelStep(
   completed: boolean,
 ) {
   const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-  const shard = stableHash(event.visitorId) % AGGREGATE_SHARDS;
+  const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
   const bucket = await ctx.db
     .query("funnelStepBuckets")
     .withIndex(
@@ -1155,7 +1589,7 @@ async function accountConversion(
   });
   if (commissionAffiliate && attribution) {
     const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-    const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+    const shard = stableHash(event.sessionId) % SECONDARY_AGGREGATE_SHARDS;
     const stats = await ctx.db
       .query("affiliateDailyStats")
       .withIndex(
@@ -1198,7 +1632,6 @@ function foldAggregates(
   recordTelemetry: boolean,
   financialConversion: boolean,
 ) {
-  const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
   const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
   const dimensionDeltas = dimensions.map(
     (dimension): DimensionDelta => ({
@@ -1208,6 +1641,8 @@ function foldAggregates(
     }),
   );
   for (const granularity of ["hour", "day"] as const) {
+    const shard = stableHash(event.sessionId) %
+      (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
     const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
     const bucketStart = Math.floor(event.timestamp / interval) * interval;
     const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
@@ -1325,7 +1760,7 @@ function mergeDimensions(
     const typeSlots = result.filter(
       (candidate) => candidate.type === addition.type,
     );
-    if (typeSlots.length < DIMENSION_SLOTS) {
+    if (typeSlots.length < DIMENSION_SLOTS - 1) {
       const occupied = Array.from(
         { length: DIMENSION_SLOTS },
         () => false,
@@ -1342,15 +1777,36 @@ function mergeDimensions(
       });
       continue;
     }
-    if (addition.count === 0) continue;
+    const overflow = typeSlots.find(
+      (candidate) => candidate.value === "(other)",
+    );
+    if (overflow) {
+      overflow.count += addition.count;
+      overflow.revenueCents += addition.revenueCents;
+      continue;
+    }
+    if (addition.count === 0 && addition.revenueCents === 0) continue;
+    if (typeSlots.length < DIMENSION_SLOTS) {
+      const occupied = new Set(typeSlots.map((candidate) => candidate.slot));
+      let slot = stableHash("(other)") % DIMENSION_SLOTS;
+      while (occupied.has(slot)) slot = (slot + 1) % DIMENSION_SLOTS;
+      result.push({
+        type: addition.type,
+        slot,
+        value: "(other)",
+        count: addition.count,
+        revenueCents: addition.revenueCents,
+      });
+      continue;
+    }
 
     let replacement = typeSlots[0];
     for (const candidate of typeSlots.slice(1)) {
       if (candidate.count < replacement.count) replacement = candidate;
     }
-    replacement.value = addition.value;
+    replacement.value = "(other)";
     replacement.count += addition.count;
-    replacement.revenueCents = addition.revenueCents;
+    replacement.revenueCents += addition.revenueCents;
   }
   return result;
 }