@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.2

package/dist/component/ingest.js

@@ -1,8 +1,9 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, DAY_MS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
 import { fail } from "./errors.js";
+import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
 import { cleanString, ensureBatchBytes, isValidMoney, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
 export const ingestBatch = mutation({
@@ -37,25 +38,84 @@ export const ingestBatch = mutation({
         const eventByteCounts = await consumeSiteRateLimit(ctx, args.siteId, args.events, now);
         const context = sanitizeContext(args.context);
         const aggregateDeltas = new Map();
+        const sessionRateWindows = new Map();
+        const batchState = {
+            sessions: new Map(),
+            liveSessions: new Map(),
+            visitorAttributions: new Map(),
+            funnelProgress: new Map(),
+            eventBatches: new Map(),
+            liveSweepSites: new Set(),
+        };
         let goals = null;
         let funnels = null;
         let accepted = 0;
         let duplicates = 0;
         let rejected = 0;
-        for (let eventIndex = 0; eventIndex < args.events.length; eventIndex += 1) {
-            const input = args.events[eventIndex];
-            let event;
+        const events = args.events.map((input) => {
             try {
-                event = sanitizeEvent(input, now);
+                return sanitizeEvent(input, now);
             }
             catch {
+                return null;
+            }
+        });
+        const duplicateInBatch = new Set();
+        const eventIdsByGroup = new Map();
+        const requestGroups = new Set();
+        const dedupeGroups = new Map();
+        for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+            const event = events[eventIndex];
+            if (!event)
+                continue;
+            const bucketStart = eventBucketStart(event.timestamp);
+            const groupKey = eventBatchKey(event.sessionId, bucketStart);
+            requestGroups.add(groupKey);
+            if (event.type === "heartbeat")
+                continue;
+            const eventIds = eventIdsByGroup.get(groupKey) ?? new Set();
+            if (eventIds.has(event.eventId)) {
+                duplicateInBatch.add(eventIndex);
+                continue;
+            }
+            eventIds.add(event.eventId);
+            eventIdsByGroup.set(groupKey, eventIds);
+            const group = dedupeGroups.get(groupKey) ?? {
+                sessionId: event.sessionId,
+                bucketStart,
+                eventIds: new Set(),
+            };
+            group.eventIds.add(event.eventId);
+            dedupeGroups.set(groupKey, group);
+        }
+        if (requestGroups.size > MAX_BATCH_EVENT_GROUPS) {
+            fail("BATCH_TOO_LARGE", `events may span at most ${MAX_BATCH_EVENT_GROUPS} session-minute groups`, { limit: MAX_BATCH_EVENT_GROUPS });
+        }
+        const persistedEventIds = new Map();
+        await Promise.all([...dedupeGroups].map(async ([key, group]) => {
+            const { eventIds, persistedCount } = await inspectTelemetryMinute(ctx, {
+                siteId: args.siteId,
+                sessionId: group.sessionId,
+                bucketStart: group.bucketStart,
+                candidateEventIds: group.eventIds,
+            });
+            const newCount = [...group.eventIds].filter((eventId) => !eventIds.has(eventId)).length;
+            if (newCount > 0 &&
+                persistedCount + newCount > MAX_EVENTS_PER_SESSION_WINDOW) {
+                fail("RATE_LIMITED", "session event minute limit exceeded", {
+                    limit: MAX_EVENTS_PER_SESSION_WINDOW,
+                });
+            }
+            persistedEventIds.set(key, eventIds);
+        }));
+        for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+            const event = events[eventIndex];
+            if (!event) {
                 rejected += 1;
                 continue;
             }
-            const existingSession = await ctx.db
-                .query("sessions")
-                .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
-                .unique();
+            const sessionRow = await loadSession(ctx, args.siteId, event.sessionId, batchState);
+            const existingSession = sessionRow.current;
             if (existingSession && existingSession.visitorId !== event.visitorId) {
                 rejected += 1;
                 continue;
@@ -65,31 +125,29 @@ export const ingestBatch = mutation({
                     rejected += 1;
                     continue;
                 }
-                const liveSession = await ctx.db
-                    .query("liveSessions")
-                    .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
-                    .unique();
-                if (liveSession?.lastHeartbeatEventId === event.eventId) {
+                const liveSession = (await loadLiveSession(ctx, args.siteId, event.sessionId, batchState)).current;
+                if (liveSession &&
+                    (liveSession.lastHeartbeatEventId === event.eventId ||
+                        liveSession.recentHeartbeatEventIds?.includes(event.eventId))) {
                     duplicates += 1;
                     continue;
                 }
-                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
                     rejected += 1;
                     continue;
                 }
-                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now);
+                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
                 accepted += 1;
                 continue;
             }
-            const duplicate = await ctx.db
-                .query("events")
-                .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
-                .unique();
-            if (duplicate) {
+            if (duplicateInBatch.has(eventIndex) ||
+                persistedEventIds
+                    .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
+                    ?.has(event.eventId)) {
                 duplicates += 1;
                 continue;
             }
-            if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+            if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
                 rejected += 1;
                 continue;
             }
@@ -118,9 +176,13 @@ export const ingestBatch = mutation({
                 financialConversion: false,
                 trustedGoalKey: undefined,
                 now,
+                batchState,
             });
+            bufferEventBatch(args.siteId, event, batchState.eventBatches);
             accepted += 1;
         }
+        await flushBatchState(ctx, batchState);
+        await flushSessionRateLimits(ctx, sessionRateWindows, now);
         await flushAggregates(ctx, aggregateDeltas);
         return { accepted, duplicates, rejected };
     },
@@ -136,13 +198,17 @@ export const trackConversion = mutation({
             fail("INVALID_ARGUMENT", "revenueCents must be an integer between 0 and 1000000000");
         }
         let event;
+        let trustedGoalKey;
         try {
+            trustedGoalKey = args.goalKey
+                ? sanitizeSlug(args.goalKey, "goalKey")
+                : undefined;
             event = sanitizeEvent({
                 eventId: sanitizeOpaqueId(args.eventId, "eventId"),
                 sessionId: sanitizeOpaqueId(args.sessionId, "sessionId"),
                 visitorId: sanitizeOpaqueId(args.visitorId, "visitorId"),
                 type: "conversion",
-                name: args.goalKey ? sanitizeSlug(args.goalKey, "goalKey") : "conversion",
+                name: trustedGoalKey ?? "conversion",
                 path: sanitizePath(args.path ?? "/conversion"),
                 timestamp: args.timestamp,
                 sequence: 0,
@@ -155,23 +221,24 @@ export const trackConversion = mutation({
             fail("INVALID_ARGUMENT", error instanceof Error ? error.message : "invalid conversion");
         }
         const siteCurrency = site.currency ?? "USD";
-        if (event.currency !== siteCurrency) {
-            fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
-                siteCurrency,
-                conversionCurrency: event.currency ?? "USD",
-            });
-        }
         const priorConversion = await ctx.db
             .query("conversions")
             .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
             .unique();
         if (priorConversion) {
+            if (!sameTrustedConversion(priorConversion, event, trustedGoalKey)) {
+                fail("CONFLICT", "conversion eventId is already associated with a different payload");
+            }
             return { accepted: 0, duplicates: 1, rejected: 0 };
         }
-        const existingTelemetry = await ctx.db
-            .query("events")
-            .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
-            .unique();
+        if (event.currency !== siteCurrency) {
+            fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
+                siteCurrency,
+                conversionCurrency: event.currency ?? "USD",
+            });
+        }
+        const telemetry = await findTelemetry(ctx, args.siteId, event);
+        const existingTelemetry = telemetry.match;
         if (existingTelemetry &&
             (existingTelemetry.sessionId !== event.sessionId ||
                 existingTelemetry.visitorId !== event.visitorId)) {
@@ -184,17 +251,15 @@ export const trackConversion = mutation({
         if (existingSession && existingSession.visitorId !== event.visitorId) {
             fail("CONFLICT", "session belongs to a different visitor");
         }
-        const conversionContext = existingTelemetry
-            ? {
-                country: existingTelemetry.country,
-                city: existingTelemetry.city,
-                latitude: existingTelemetry.latitude,
-                longitude: existingTelemetry.longitude,
-                browser: existingTelemetry.browser,
-                os: existingTelemetry.os,
-                device: existingTelemetry.device,
-            }
-            : sanitizeContext(undefined);
+        const conversionContext = {
+            country: existingTelemetry?.country ?? existingSession?.country,
+            city: existingTelemetry?.city ?? existingSession?.city,
+            latitude: existingTelemetry?.latitude ?? existingSession?.latitude,
+            longitude: existingTelemetry?.longitude ?? existingSession?.longitude,
+            browser: existingTelemetry?.browser ?? existingSession?.browser,
+            os: existingTelemetry?.os ?? existingSession?.os,
+            device: existingTelemetry?.device ?? existingSession?.device,
+        };
         const aggregateDeltas = new Map();
         await accountEvent(ctx, {
             siteId: args.siteId,
@@ -204,73 +269,126 @@ export const trackConversion = mutation({
             goals: [],
             funnels: [],
             aggregateDeltas,
-            recordTelemetry: existingTelemetry === null,
+            recordTelemetry: existingTelemetry === null && !telemetry.saturated,
             financialConversion: true,
-            trustedGoalKey: args.goalKey
-                ? sanitizeSlug(args.goalKey, "goalKey")
-                : undefined,
+            trustedGoalKey,
             now: Date.now(),
         });
+        if (!existingTelemetry && !telemetry.saturated) {
+            await insertEventBatch(ctx, args.siteId, event);
+        }
         await flushAggregates(ctx, aggregateDeltas);
         return { accepted: 1, duplicates: 0, rejected: 0 };
     },
 });
+function sameTrustedConversion(existing, event, goalKey) {
+    return (existing.source === "trusted" &&
+        existing.sessionId === event.sessionId &&
+        existing.visitorId === event.visitorId &&
+        existing.timestamp === event.timestamp &&
+        existing.path === event.path &&
+        existing.revenueCents === (event.revenueCents ?? 0) &&
+        existing.currency === (event.currency ?? "USD") &&
+        existing.goalKey === goalKey);
+}
+async function loadSession(ctx, siteId, sessionId, batchState) {
+    const cached = batchState.sessions.get(sessionId);
+    if (cached)
+        return cached;
+    const current = await ctx.db
+        .query("sessions")
+        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+        .unique();
+    const loaded = { current, dirty: false };
+    batchState.sessions.set(sessionId, loaded);
+    return loaded;
+}
+async function loadLiveSession(ctx, siteId, sessionId, batchState) {
+    const cached = batchState.liveSessions.get(sessionId);
+    if (cached)
+        return cached;
+    const current = await ctx.db
+        .query("liveSessions")
+        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+        .unique();
+    const loaded = { current, dirty: false };
+    batchState.liveSessions.set(sessionId, loaded);
+    return loaded;
+}
+async function flushBatchState(ctx, batchState) {
+    for (const state of batchState.sessions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("sessions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.liveSessions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("liveSessions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.visitorAttributions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("visitorAttributions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.funnelProgress.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("funnelProgress", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    await flushEventBatches(ctx, batchState.eventBatches);
+}
+function withoutSystemFields(document) {
+    const { _id, _creationTime, ...fields } = document;
+    void _id;
+    void _creationTime;
+    return fields;
+}
 async function accountEvent(ctx, input) {
     const { event } = input;
     const session = input.recordTelemetry
-        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion)
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.batchState)
         : await accountTrustedSession(ctx, input.existingSession, event);
-    if (input.recordTelemetry) {
-        await ctx.db.insert("events", {
-            siteId: input.siteId,
-            eventId: event.eventId,
-            sessionId: event.sessionId,
-            visitorId: event.visitorId,
-            type: event.type,
-            ...(event.name ? { name: event.name } : {}),
-            path: event.path,
-            ...(event.referrer ? { referrer: event.referrer } : {}),
-            timestamp: event.timestamp,
-            sequence: event.sequence,
-            ...(event.target ? { target: event.target } : {}),
-            ...(event.href ? { href: event.href } : {}),
-            ...(event.properties ? { properties: event.properties } : {}),
-            ...(event.revenueCents !== undefined
-                ? { revenueCents: event.revenueCents }
-                : {}),
-            ...(event.currency ? { currency: event.currency } : {}),
-            ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
-            ...(input.context.country ? { country: input.context.country } : {}),
-            ...(input.context.city ? { city: input.context.city } : {}),
-            ...(input.context.latitude !== undefined
-                ? { latitude: input.context.latitude }
-                : {}),
-            ...(input.context.longitude !== undefined
-                ? { longitude: input.context.longitude }
-                : {}),
-            browser: input.context.browser ?? "Other",
-            os: input.context.os ?? "Other",
-            device: input.context.device ?? "unknown",
-        });
-    }
     const matchedGoalKeys = [];
-    for (const goal of input.goals) {
-        if (!goal.active || !goalMatches(goal, event))
-            continue;
-        const prior = await ctx.db
+    const matchingGoals = input.goals.filter((goal) => goal.active && goalMatches(goal, event));
+    if (matchingGoals.length > 1) {
+        fail("LIMIT_EXCEEDED", "active goals must use unique matchers");
+    }
+    for (const goal of matchingGoals) {
+        const completionEventId = JSON.stringify([
+            event.sessionId,
+            eventBucketStart(event.timestamp),
+            event.eventId,
+        ]);
+        const scopedPrior = (await ctx.db
             .query("goalCompletions")
             .withIndex("by_siteId_and_goalId_and_eventId", (range) => range
             .eq("siteId", input.siteId)
             .eq("goalId", goal._id)
-            .eq("eventId", event.eventId))
-            .unique();
-        if (prior)
+            .eq("eventId", completionEventId))
+            .take(2))
+            .find((completion) => completion.idempotencyVersion === 2);
+        const legacyPrior = scopedPrior
+            ? undefined
+            : (await ctx.db
+                .query("goalCompletions")
+                .withIndex("by_siteId_and_goalId_and_eventId", (range) => range
+                .eq("siteId", input.siteId)
+                .eq("goalId", goal._id)
+                .eq("eventId", event.eventId))
+                .take(2))
+                .find((completion) => completion.idempotencyVersion === undefined &&
+                completion.sessionId === event.sessionId &&
+                eventBucketStart(completion.timestamp) ===
+                    eventBucketStart(event.timestamp));
+        if (scopedPrior || legacyPrior)
             continue;
         await ctx.db.insert("goalCompletions", {
             siteId: input.siteId,
             goalId: goal._id,
             goalKey: goal.key,
-            eventId: event.eventId,
+            eventId: completionEventId,
+            idempotencyVersion: 2,
             sessionId: event.sessionId,
             visitorId: event.visitorId,
             timestamp: event.timestamp,
@@ -280,7 +398,7 @@ async function accountEvent(ctx, input) {
     }
     for (const funnel of input.funnels) {
         if (funnel.active) {
-            await advanceFunnel(ctx, input.siteId, funnel, event);
+            await advanceFunnel(ctx, input.siteId, funnel, event, input.batchState);
         }
     }
     if (input.financialConversion) {
@@ -301,7 +419,7 @@ async function accountEvent(ctx, input) {
     foldAggregates(input.aggregateDeltas, input.siteId, event, session, dimensions, input.recordTelemetry, input.financialConversion);
     if (input.recordTelemetry &&
         event.timestamp >= input.now - LIVE_SESSION_TTL_MS) {
-        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now);
+        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now, input.batchState);
     }
 }
 async function accountTrustedSession(ctx, existing, event) {
@@ -328,16 +446,16 @@ async function accountTrustedSession(ctx, existing, event) {
         affiliateExpiresAt: existing.affiliateExpiresAt,
     };
 }
-async function upsertSession(ctx, siteId, event, context, existing, financialConversion) {
+async function upsertSession(ctx, siteId, event, context, existing, financialConversion, batchState) {
     const source = sourceFromReferrer(event.referrer);
     const utmSource = stringProperty(event.properties, "utm_source");
     const utmMedium = stringProperty(event.properties, "utm_medium");
     const utmCampaign = stringProperty(event.properties, "utm_campaign");
-    const attribution = await attributionForEvent(ctx, siteId, event, existing);
+    const attribution = await attributionForEvent(ctx, siteId, event, existing, batchState);
     if (existing) {
         const startedAt = Math.min(existing.startedAt, event.timestamp);
         const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
-        await ctx.db.patch("sessions", existing._id, {
+        const updates = {
             startedAt,
             lastSeenAt,
             entryPath: event.timestamp < existing.startedAt ? event.path : existing.entryPath,
@@ -362,7 +480,17 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             browser: context.browser ?? existing.browser,
             os: context.os ?? existing.os,
             device: context.device ?? existing.device,
-        });
+        };
+        if (batchState) {
+            const state = batchState.sessions.get(event.sessionId);
+            if (!state)
+                throw new Error("session cache was not initialized");
+            state.current = { ...existing, ...updates };
+            state.dirty = true;
+        }
+        else {
+            await ctx.db.patch("sessions", existing._id, updates);
+        }
         return {
             isNewSession: false,
             isNewVisitorHour: Math.floor(existing.lastSeenAt / HOUR_MS) <
@@ -376,12 +504,23 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             ...(attribution ?? {}),
         };
     }
-    const priorVisit = await ctx.db
+    const priorVisits = await ctx.db
         .query("sessions")
         .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
         .order("desc")
         .take(1);
-    await ctx.db.insert("sessions", {
+    let priorVisit = priorVisits[0] ?? null;
+    if (batchState) {
+        for (const state of batchState.sessions.values()) {
+            const candidate = state.current;
+            if (candidate?.visitorId === event.visitorId &&
+                candidate.sessionId !== event.sessionId &&
+                (!priorVisit || candidate.lastSeenAt > priorVisit.lastSeenAt)) {
+                priorVisit = candidate;
+            }
+        }
+    }
+    const fields = {
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
@@ -410,14 +549,24 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             ? { currency: event.currency }
             : {}),
         ...(attribution ?? {}),
-    });
+    };
+    const sessionId = await ctx.db.insert("sessions", fields);
+    if (batchState) {
+        const state = batchState.sessions.get(event.sessionId);
+        if (!state)
+            throw new Error("session cache was not initialized");
+        state.current = {
+            ...fields,
+            _id: sessionId,
+        };
+    }
     return {
         isNewSession: true,
-        isNewVisitorHour: priorVisit.length === 0 ||
-            Math.floor(priorVisit[0].lastSeenAt / HOUR_MS) <
+        isNewVisitorHour: priorVisit === null ||
+            Math.floor(priorVisit.lastSeenAt / HOUR_MS) <
                 Math.floor(event.timestamp / HOUR_MS),
-        isNewVisitorDay: priorVisit.length === 0 ||
-            Math.floor(priorVisit[0].lastSeenAt / DAY_MS) <
+        isNewVisitorDay: priorVisit === null ||
+            Math.floor(priorVisit.lastSeenAt / DAY_MS) <
                 Math.floor(event.timestamp / DAY_MS),
         startedAt: event.timestamp,
         source,
@@ -446,7 +595,7 @@ function attributionApplies(attribution, timestamp) {
 }
 // The first valid touch wins through its inclusive expiry. Only a later,
 // in-order valid touch may replace the visitor's single bounded record.
-async function attributionForEvent(ctx, siteId, event, existingSession) {
+async function attributionForEvent(ctx, siteId, event, existingSession, batchState) {
     const sessionAttribution = existingSession
         ? attributionSnapshot(existingSession)
         : undefined;
@@ -458,10 +607,21 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
         attributionApplies(sessionAttribution, event.timestamp)) {
         return sessionAttribution;
     }
-    const current = await ctx.db
-        .query("visitorAttributions")
-        .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
-        .unique();
+    let attributionState = batchState?.visitorAttributions.get(event.visitorId);
+    if (batchState && !attributionState) {
+        const current = await ctx.db
+            .query("visitorAttributions")
+            .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+            .unique();
+        attributionState = { current, dirty: false };
+        batchState.visitorAttributions.set(event.visitorId, attributionState);
+    }
+    const current = attributionState
+        ? attributionState.current
+        : await ctx.db
+            .query("visitorAttributions")
+            .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+            .unique();
     if (current) {
         const snapshot = {
             affiliateId: current.affiliateId,
@@ -489,23 +649,32 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
     }
     const attributedAt = event.timestamp;
     const expiresAt = attributedAt + affiliate.attributionWindowDays * DAY_MS;
+    const fields = {
+        siteId,
+        visitorId: event.visitorId,
+        affiliateId: affiliate._id,
+        affiliateSlug: affiliate.slug,
+        attributedAt,
+        expiresAt,
+    };
     if (current) {
-        await ctx.db.patch("visitorAttributions", current._id, {
-            affiliateId: affiliate._id,
-            affiliateSlug: affiliate.slug,
-            attributedAt,
-            expiresAt,
-        });
+        if (attributionState) {
+            attributionState.current = { ...current, ...fields };
+            attributionState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("visitorAttributions", current._id, fields);
+        }
     }
     else {
-        await ctx.db.insert("visitorAttributions", {
-            siteId,
-            visitorId: event.visitorId,
-            affiliateId: affiliate._id,
-            affiliateSlug: affiliate.slug,
-            attributedAt,
-            expiresAt,
-        });
+        const attributionId = await ctx.db.insert("visitorAttributions", fields);
+        if (attributionState) {
+            attributionState.current = {
+                ...fields,
+                _id: attributionId,
+                _creationTime: event.timestamp,
+            };
+        }
     }
     return {
         affiliateId: affiliate._id,
@@ -514,10 +683,10 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
         affiliateExpiresAt: expiresAt,
     };
 }
-async function accountHeartbeat(ctx, siteId, session, event, context, now) {
+async function accountHeartbeat(ctx, siteId, session, event, context, now, batchState) {
     const startedAt = Math.min(session.startedAt, event.timestamp);
     const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
-    await ctx.db.patch("sessions", session._id, {
+    const updates = {
         startedAt,
         lastSeenAt,
         exitPath: event.timestamp >= session.lastSeenAt
@@ -531,7 +700,17 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now) {
         browser: context.browser ?? session.browser,
         os: context.os ?? session.os,
         device: context.device ?? session.device,
-    });
+    };
+    if (batchState) {
+        const state = batchState.sessions.get(event.sessionId);
+        if (!state)
+            throw new Error("session cache was not initialized");
+        state.current = { ...session, ...updates };
+        state.dirty = true;
+    }
+    else {
+        await ctx.db.patch("sessions", session._id, updates);
+    }
     await updateLiveSession(ctx, siteId, event, context, {
         isNewSession: false,
         isNewVisitorHour: false,
@@ -541,7 +720,7 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now) {
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         affiliateSlug: session.affiliateSlug,
-    }, now);
+    }, now, batchState);
 }
 async function consumeSiteRateLimit(ctx, siteId, events, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -560,13 +739,15 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
     }
     const windows = [];
     for (const [shard, increment] of increments) {
-        const window = await ctx.db
+        const rows = await ctx.db
             .query("siteIngestWindows")
-            .withIndex("by_siteId_and_shard_and_windowStart", (range) => range
-            .eq("siteId", siteId)
-            .eq("shard", shard)
-            .eq("windowStart", windowStart))
-            .unique();
+            .withIndex("by_siteId_and_shard_and_windowStart", (range) => range.eq("siteId", siteId).eq("shard", shard))
+            .order("desc")
+            .take(1);
+        const storedWindow = rows[0] ?? null;
+        const window = storedWindow?.windowStart === windowStart
+            ? storedWindow
+            : null;
         if ((window?.eventCount ?? 0) + increment.eventCount >
             MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
             (window?.byteCount ?? 0) + increment.byteCount >
@@ -580,13 +761,18 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
                 siteByteLimit: SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
             });
         }
-        windows.push({ shard, increment, window });
+        windows.push({ shard, increment, window: storedWindow });
     }
     for (const { shard, increment, window } of windows) {
         if (window) {
             await ctx.db.patch("siteIngestWindows", window._id, {
-                eventCount: window.eventCount + increment.eventCount,
-                byteCount: window.byteCount + increment.byteCount,
+                windowStart,
+                eventCount: window.windowStart === windowStart
+                    ? window.eventCount + increment.eventCount
+                    : increment.eventCount,
+                byteCount: window.windowStart === windowStart
+                    ? window.byteCount + increment.byteCount
+                    : increment.byteCount,
                 updatedAt: now,
             });
         }
@@ -603,53 +789,83 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
     }
     return eventByteCounts;
 }
-async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now) {
+async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now, states) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
-    const window = await ctx.db
-        .query("ingestWindows")
-        .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range
-        .eq("siteId", siteId)
-        .eq("sessionId", sessionId)
-        .eq("windowStart", windowStart))
-        .unique();
-    if (window &&
-        (window.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
-            window.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW)) {
-        return false;
-    }
-    if (window) {
-        await ctx.db.patch("ingestWindows", window._id, {
-            eventCount: window.eventCount + 1,
-            byteCount: window.byteCount + byteCount,
-            updatedAt: now,
-        });
-    }
-    else {
-        await ctx.db.insert("ingestWindows", {
+    let state = states.get(sessionId);
+    if (!state) {
+        const rows = await ctx.db
+            .query("ingestWindows")
+            .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+            .order("desc")
+            .take(1);
+        const window = rows[0] ?? null;
+        state = {
             siteId,
             sessionId,
-            windowStart,
-            eventCount: 1,
-            byteCount,
-            updatedAt: now,
-        });
+            window,
+            eventCount: window?.windowStart === windowStart ? window.eventCount : 0,
+            byteCount: window?.windowStart === windowStart ? window.byteCount : 0,
+            dirty: false,
+        };
+        states.set(sessionId, state);
+    }
+    if (state.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
+        state.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW) {
+        return false;
     }
+    state.eventCount += 1;
+    state.byteCount += byteCount;
+    state.dirty = true;
     return true;
 }
-async function updateLiveSession(ctx, siteId, event, context, session, now) {
-    const existing = await ctx.db
-        .query("liveSessions")
-        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
-        .unique();
-    const renewsExpiry = !existing ||
-        existing.expiresAt <= now ||
-        event.type === "pageview" ||
-        event.type === "heartbeat";
-    const deadline = renewsExpiry
-        ? now + LIVE_SESSION_TTL_MS
-        : existing.expiresAt;
+async function flushSessionRateLimits(ctx, states, now) {
+    const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+    for (const state of states.values()) {
+        if (!state.dirty)
+            continue;
+        if (state.window) {
+            await ctx.db.patch("ingestWindows", state.window._id, {
+                windowStart,
+                eventCount: state.eventCount,
+                byteCount: state.byteCount,
+                updatedAt: now,
+            });
+        }
+        else {
+            await ctx.db.insert("ingestWindows", {
+                siteId: state.siteId,
+                sessionId: state.sessionId,
+                windowStart,
+                eventCount: state.eventCount,
+                byteCount: state.byteCount,
+                updatedAt: now,
+            });
+        }
+    }
+}
+async function updateLiveSession(ctx, siteId, event, context, session, now, batchState) {
+    await ensureLiveSweep(ctx, siteId, now, batchState);
+    const liveState = batchState
+        ? await loadLiveSession(ctx, siteId, event.sessionId, batchState)
+        : undefined;
+    const existing = liveState
+        ? liveState.current
+        : await ctx.db
+            .query("liveSessions")
+            .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
+            .unique();
+    const deadline = now + LIVE_SESSION_TTL_MS;
     if (existing) {
-        await ctx.db.patch("liveSessions", existing._id, {
+        const recentHeartbeatEventIds = event.type === "heartbeat"
+            ? [
+                ...(existing.recentHeartbeatEventIds ??
+                    (existing.lastHeartbeatEventId
+                        ? [existing.lastHeartbeatEventId]
+                        : [])),
+                event.eventId,
+            ].slice(-8)
+            : undefined;
+        const updates = {
             visitorId: event.visitorId,
             path: event.path,
             lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
@@ -665,12 +881,22 @@ async function updateLiveSession(ctx, siteId, event, context, session, now) {
             pageviewCount: session.pageviewCount,
             eventCount: session.eventCount,
             ...(event.type === "heartbeat"
-                ? { lastHeartbeatEventId: event.eventId }
+                ? {
+                    lastHeartbeatEventId: event.eventId,
+                    recentHeartbeatEventIds,
+                }
                 : {}),
-        });
+        };
+        if (liveState) {
+            liveState.current = { ...existing, ...updates };
+            liveState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("liveSessions", existing._id, updates);
+        }
         return;
     }
-    const liveSessionId = await ctx.db.insert("liveSessions", {
+    const fields = {
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
@@ -678,7 +904,6 @@ async function updateLiveSession(ctx, siteId, event, context, session, now) {
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         expiresAt: deadline,
-        expiryChainDeadline: deadline,
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
@@ -690,13 +915,37 @@ async function updateLiveSession(ctx, siteId, event, context, session, now) {
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         ...(event.type === "heartbeat"
-            ? { lastHeartbeatEventId: event.eventId }
+            ? {
+                lastHeartbeatEventId: event.eventId,
+                recentHeartbeatEventIds: [event.eventId],
+            }
             : {}),
-    });
-    await ctx.scheduler.runAt(deadline, internal.live.expire, {
-        liveSessionId,
-        scheduledDeadline: deadline,
-    });
+    };
+    const liveSessionId = await ctx.db.insert("liveSessions", fields);
+    if (liveState) {
+        liveState.current = {
+            ...fields,
+            _id: liveSessionId,
+        };
+    }
+}
+async function ensureLiveSweep(ctx, siteId, now, batchState) {
+    const key = String(siteId);
+    if (batchState?.liveSweepSites.has(key))
+        return;
+    const existing = await ctx.db
+        .query("liveSweepStates")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    if (!existing) {
+        const scheduledAt = now + LIVE_SWEEP_INTERVAL_MS;
+        await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
+            siteId,
+            scheduledAt,
+        });
+    }
+    batchState?.liveSweepSites.add(key);
 }
 function goalMatches(goal, event) {
     if (goal.matchType === "pageview") {
@@ -704,19 +953,34 @@ function goalMatches(goal, event) {
     }
     return event.type === "custom" && event.name === goal.matchValue;
 }
-async function advanceFunnel(ctx, siteId, funnel, event) {
-    const progress = await ctx.db
-        .query("funnelProgress")
-        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
-        .eq("siteId", siteId)
-        .eq("funnelId", funnel._id)
-        .eq("visitorId", event.visitorId))
-        .unique();
+async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
+    const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
+    let progressState = batchState?.funnelProgress.get(cacheKey);
+    if (batchState && !progressState) {
+        const current = await ctx.db
+            .query("funnelProgress")
+            .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnel._id)
+            .eq("visitorId", event.visitorId))
+            .unique();
+        progressState = { current, dirty: false };
+        batchState.funnelProgress.set(cacheKey, progressState);
+    }
+    const progress = progressState
+        ? progressState.current
+        : await ctx.db
+            .query("funnelProgress")
+            .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnel._id)
+            .eq("visitorId", event.visitorId))
+            .unique();
     const matchesFirst = stepMatches(funnel.steps[0], event);
     if (!progress) {
         if (!matchesFirst)
             return;
-        await ctx.db.insert("funnelProgress", {
+        const fields = {
             siteId,
             funnelId: funnel._id,
             visitorId: event.visitorId,
@@ -724,7 +988,14 @@ async function advanceFunnel(ctx, siteId, funnel, event) {
             currentStep: 1,
             startedAt: event.timestamp,
             lastStepAt: event.timestamp,
-        });
+        };
+        const progressId = await ctx.db.insert("funnelProgress", fields);
+        if (progressState) {
+            progressState.current = {
+                ...fields,
+                _id: progressId,
+            };
+        }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
         return;
     }
@@ -733,13 +1004,27 @@ async function advanceFunnel(ctx, siteId, funnel, event) {
     if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
         if (!matchesFirst)
             return;
-        await ctx.db.patch("funnelProgress", progress._id, {
+        const updated = {
+            ...progress,
             sessionId: event.sessionId,
             currentStep: 1,
             startedAt: event.timestamp,
             lastStepAt: event.timestamp,
-            completedAt: undefined,
-        });
+        };
+        delete updated.completedAt;
+        if (progressState) {
+            progressState.current = updated;
+            progressState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("funnelProgress", progress._id, {
+                sessionId: event.sessionId,
+                currentStep: 1,
+                startedAt: event.timestamp,
+                lastStepAt: event.timestamp,
+                completedAt: undefined,
+            });
+        }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
         return;
     }
@@ -748,12 +1033,19 @@ async function advanceFunnel(ctx, siteId, funnel, event) {
         return;
     const currentStep = progress.currentStep + 1;
     const completed = currentStep === funnel.steps.length;
-    await ctx.db.patch("funnelProgress", progress._id, {
+    const updates = {
         sessionId: event.sessionId,
         currentStep,
         lastStepAt: event.timestamp,
         ...(completed ? { completedAt: event.timestamp } : {}),
-    });
+    };
+    if (progressState) {
+        progressState.current = { ...progress, ...updates };
+        progressState.dirty = true;
+    }
+    else {
+        await ctx.db.patch("funnelProgress", progress._id, updates);
+    }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
 }
 function stepMatches(step, event) {
@@ -763,7 +1055,7 @@ function stepMatches(step, event) {
 }
 async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed) {
     const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-    const shard = stableHash(event.visitorId) % AGGREGATE_SHARDS;
+    const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
     const bucket = await ctx.db
         .query("funnelStepBuckets")
         .withIndex("by_siteId_and_funnelId_and_bucketStart_and_step_and_shard", (range) => range
@@ -821,7 +1113,7 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
     });
     if (commissionAffiliate && attribution) {
         const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-        const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+        const shard = stableHash(event.sessionId) % SECONDARY_AGGREGATE_SHARDS;
         const stats = await ctx.db
             .query("affiliateDailyStats")
             .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
@@ -852,7 +1144,6 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
     }
 }
 function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
-    const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
     const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
         ...dimension,
@@ -860,6 +1151,8 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
         revenueCents,
     }));
     for (const granularity of ["hour", "day"]) {
+        const shard = stableHash(event.sessionId) %
+            (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
         const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
         const bucketStart = Math.floor(event.timestamp / interval) * interval;
         const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
@@ -955,7 +1248,7 @@ function mergeDimensions(current, additions) {
             continue;
         }
         const typeSlots = result.filter((candidate) => candidate.type === addition.type);
-        if (typeSlots.length < DIMENSION_SLOTS) {
+        if (typeSlots.length < DIMENSION_SLOTS - 1) {
             const occupied = Array.from({ length: DIMENSION_SLOTS }, () => false);
             for (const candidate of typeSlots)
                 occupied[candidate.slot] = true;
@@ -971,16 +1264,36 @@ function mergeDimensions(current, additions) {
             });
             continue;
         }
-        if (addition.count === 0)
+        const overflow = typeSlots.find((candidate) => candidate.value === "(other)");
+        if (overflow) {
+            overflow.count += addition.count;
+            overflow.revenueCents += addition.revenueCents;
             continue;
+        }
+        if (addition.count === 0 && addition.revenueCents === 0)
+            continue;
+        if (typeSlots.length < DIMENSION_SLOTS) {
+            const occupied = new Set(typeSlots.map((candidate) => candidate.slot));
+            let slot = stableHash("(other)") % DIMENSION_SLOTS;
+            while (occupied.has(slot))
+                slot = (slot + 1) % DIMENSION_SLOTS;
+            result.push({
+                type: addition.type,
+                slot,
+                value: "(other)",
+                count: addition.count,
+                revenueCents: addition.revenueCents,
+            });
+            continue;
+        }
         let replacement = typeSlots[0];
         for (const candidate of typeSlots.slice(1)) {
             if (candidate.count < replacement.count)
                 replacement = candidate;
         }
-        replacement.value = addition.value;
+        replacement.value = "(other)";
         replacement.count += addition.count;
-        replacement.revenueCents = addition.revenueCents;
+        replacement.revenueCents += addition.revenueCents;
     }
     return result;
 }