@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/dist/component/ingest.js

@@ -1,9 +1,13 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, DAY_MS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_LEAVE_GRACE_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, MAX_SITE_MAP_ROUTES_PER_DAY, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, ROUTE_TRANSITION_SLOTS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, SITE_MAP_OTHER_ROUTE, VITAL_PAGE_SLOTS, } from "./constants.js";
+import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
+import { addToVitalHistogram, emptyVitalHistogram, mergeVitalHistograms, vitalDevice, VITAL_ALL, VITAL_OTHER_PAGES, } from "./vitals.js";
 import { fail } from "./errors.js";
-import { cleanString, ensureBatchBytes, isValidMoney, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { ensureAnalyticsControl } from "./coverage.js";
+import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
+import { cleanString, isValidMoney, measureBatchBytes, normalizeRoute, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
 export const ingestBatch = mutation({
     args: {
@@ -27,35 +31,98 @@ export const ingestBatch = mutation({
                 limit: MAX_BATCH_EVENTS,
             });
         }
+        let eventByteCounts;
         try {
-            ensureBatchBytes({ siteId: String(args.siteId), events: args.events });
+            ({ eventByteCounts } = measureBatchBytes(String(args.siteId), args.events));
         }
         catch {
             fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
         }
         const now = Date.now();
-        const eventByteCounts = await consumeSiteRateLimit(ctx, args.siteId, args.events, now);
+        await ensureAnalyticsControl(ctx, args.siteId, now);
+        await consumeSiteRateLimit(ctx, args.siteId, args.events, eventByteCounts, now);
         const context = sanitizeContext(args.context);
         const aggregateDeltas = new Map();
+        const reportRollupDeltas = createReportRollupDeltas();
+        const sessionRateWindows = new Map();
+        const batchState = {
+            sessions: new Map(),
+            liveSessions: new Map(),
+            visitorAttributions: new Map(),
+            funnelProgress: new Map(),
+            eventBatches: new Map(),
+            liveSweepSites: new Set(),
+        };
         let goals = null;
         let funnels = null;
         let accepted = 0;
         let duplicates = 0;
         let rejected = 0;
-        for (let eventIndex = 0; eventIndex < args.events.length; eventIndex += 1) {
-            const input = args.events[eventIndex];
-            let event;
+        const events = args.events.map((input) => {
             try {
-                event = sanitizeEvent(input, now);
+                return sanitizeEvent(input, now);
             }
             catch {
+                return null;
+            }
+        });
+        const duplicateInBatch = new Set();
+        const eventIdsByGroup = new Map();
+        const requestGroups = new Set();
+        const dedupeGroups = new Map();
+        for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+            const event = events[eventIndex];
+            if (!event)
+                continue;
+            const bucketStart = eventBucketStart(event.timestamp);
+            const groupKey = eventBatchKey(event.sessionId, bucketStart);
+            requestGroups.add(groupKey);
+            if (event.type === "heartbeat" ||
+                event.type === "vital" ||
+                event.type === "leave")
+                continue;
+            const eventIds = eventIdsByGroup.get(groupKey) ?? new Set();
+            if (eventIds.has(event.eventId)) {
+                duplicateInBatch.add(eventIndex);
+                continue;
+            }
+            eventIds.add(event.eventId);
+            eventIdsByGroup.set(groupKey, eventIds);
+            const group = dedupeGroups.get(groupKey) ?? {
+                sessionId: event.sessionId,
+                bucketStart,
+                eventIds: new Set(),
+            };
+            group.eventIds.add(event.eventId);
+            dedupeGroups.set(groupKey, group);
+        }
+        if (requestGroups.size > MAX_BATCH_EVENT_GROUPS) {
+            fail("BATCH_TOO_LARGE", `events may span at most ${MAX_BATCH_EVENT_GROUPS} session-minute groups`, { limit: MAX_BATCH_EVENT_GROUPS });
+        }
+        const persistedEventIds = new Map();
+        await Promise.all([...dedupeGroups].map(async ([key, group]) => {
+            const { eventIds, persistedCount } = await inspectTelemetryMinute(ctx, {
+                siteId: args.siteId,
+                sessionId: group.sessionId,
+                bucketStart: group.bucketStart,
+            });
+            const newCount = [...group.eventIds].filter((eventId) => !eventIds.has(eventId)).length;
+            if (newCount > 0 &&
+                persistedCount + newCount > MAX_EVENTS_PER_SESSION_WINDOW) {
+                fail("RATE_LIMITED", "session event minute limit exceeded", {
+                    limit: MAX_EVENTS_PER_SESSION_WINDOW,
+                });
+            }
+            persistedEventIds.set(key, eventIds);
+        }));
+        for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) {
+            const event = events[eventIndex];
+            if (!event) {
                 rejected += 1;
                 continue;
             }
-            const existingSession = await ctx.db
-                .query("sessions")
-                .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
-                .unique();
+            const sessionRow = await loadSession(ctx, args.siteId, event.sessionId, batchState);
+            const existingSession = sessionRow.current;
             if (existingSession && existingSession.visitorId !== event.visitorId) {
                 rejected += 1;
                 continue;
@@ -65,31 +132,70 @@ export const ingestBatch = mutation({
                     rejected += 1;
                     continue;
                 }
-                const liveSession = await ctx.db
-                    .query("liveSessions")
-                    .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
-                    .unique();
-                if (liveSession?.lastHeartbeatEventId === event.eventId) {
+                const liveSession = (await loadLiveSession(ctx, args.siteId, event.sessionId, batchState)).current;
+                if (liveSession &&
+                    (liveSession.lastHeartbeatEventId === event.eventId ||
+                        liveSession.recentHeartbeatEventIds?.includes(event.eventId))) {
                     duplicates += 1;
                     continue;
                 }
-                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
                     rejected += 1;
                     continue;
                 }
-                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now);
+                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
                 accepted += 1;
                 continue;
             }
-            const duplicate = await ctx.db
-                .query("events")
-                .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
-                .unique();
-            if (duplicate) {
+            if (event.type === "leave") {
+                // A departure only shortens live presence: it stores no telemetry,
+                // touches no aggregates, and never extends anything, so a replayed
+                // or late leave is inert. The read side filters by expiresAt and the
+                // sweep deletes the row, so the dashboard sees the exit in seconds.
+                if (!existingSession) {
+                    rejected += 1;
+                    continue;
+                }
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
+                    rejected += 1;
+                    continue;
+                }
+                const liveState = await loadLiveSession(ctx, args.siteId, event.sessionId, batchState);
+                const liveSession = liveState.current;
+                if (liveSession) {
+                    const expiresAt = Math.min(liveSession.expiresAt, now + LIVE_LEAVE_GRACE_MS);
+                    if (expiresAt < liveSession.expiresAt) {
+                        await ctx.db.patch("liveSessions", liveSession._id, { expiresAt });
+                        liveState.current = { ...liveSession, expiresAt };
+                    }
+                    await ensureLiveSweep(ctx, args.siteId, now, batchState);
+                }
+                accepted += 1;
+                continue;
+            }
+            if (event.type === "vital") {
+                // A vital annotates telemetry its session already produced; it never
+                // creates sessions, live presence, event batches, or aggregate counts.
+                if (!existingSession) {
+                    rejected += 1;
+                    continue;
+                }
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
+                    rejected += 1;
+                    continue;
+                }
+                foldVitalRollups(reportRollupDeltas, args.siteId, event, vitalDevice(context.device));
+                accepted += 1;
+                continue;
+            }
+            if (duplicateInBatch.has(eventIndex) ||
+                persistedEventIds
+                    .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
+                    ?.has(event.eventId)) {
                 duplicates += 1;
                 continue;
             }
-            if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+            if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
                 rejected += 1;
                 continue;
             }
@@ -106,22 +212,28 @@ export const ingestBatch = mutation({
                     .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
                     .take(MAX_FUNNELS_PER_SITE);
             }
-            await accountEvent(ctx, {
+            const session = await accountEvent(ctx, {
                 siteId: args.siteId,
                 event,
                 context,
                 existingSession,
-                goals: canMatchDefinition ? goals ?? [] : [],
-                funnels: canMatchDefinition ? funnels ?? [] : [],
+                goals: canMatchDefinition ? (goals ?? []) : [],
+                funnels: canMatchDefinition ? (funnels ?? []) : [],
                 aggregateDeltas,
+                reportRollupDeltas,
                 recordTelemetry: true,
                 financialConversion: false,
                 trustedGoalKey: undefined,
                 now,
+                batchState,
             });
+            bufferEventBatch(args.siteId, event, batchState.eventBatches, session.country ?? "unknown");
             accepted += 1;
         }
+        await flushBatchState(ctx, batchState);
+        await flushSessionRateLimits(ctx, sessionRateWindows, now);
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
         return { accepted, duplicates, rejected };
     },
 });
@@ -136,13 +248,17 @@ export const trackConversion = mutation({
             fail("INVALID_ARGUMENT", "revenueCents must be an integer between 0 and 1000000000");
         }
         let event;
+        let trustedGoalKey;
         try {
+            trustedGoalKey = args.goalKey
+                ? sanitizeSlug(args.goalKey, "goalKey")
+                : undefined;
             event = sanitizeEvent({
                 eventId: sanitizeOpaqueId(args.eventId, "eventId"),
                 sessionId: sanitizeOpaqueId(args.sessionId, "sessionId"),
                 visitorId: sanitizeOpaqueId(args.visitorId, "visitorId"),
                 type: "conversion",
-                name: args.goalKey ? sanitizeSlug(args.goalKey, "goalKey") : "conversion",
+                name: trustedGoalKey ?? "conversion",
                 path: sanitizePath(args.path ?? "/conversion"),
                 timestamp: args.timestamp,
                 sequence: 0,
@@ -155,23 +271,24 @@ export const trackConversion = mutation({
             fail("INVALID_ARGUMENT", error instanceof Error ? error.message : "invalid conversion");
         }
         const siteCurrency = site.currency ?? "USD";
-        if (event.currency !== siteCurrency) {
-            fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
-                siteCurrency,
-                conversionCurrency: event.currency ?? "USD",
-            });
-        }
         const priorConversion = await ctx.db
             .query("conversions")
             .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
             .unique();
         if (priorConversion) {
+            if (!sameTrustedConversion(priorConversion, event, trustedGoalKey)) {
+                fail("CONFLICT", "conversion eventId is already associated with a different payload");
+            }
             return { accepted: 0, duplicates: 1, rejected: 0 };
         }
-        const existingTelemetry = await ctx.db
-            .query("events")
-            .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
-            .unique();
+        if (event.currency !== siteCurrency) {
+            fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
+                siteCurrency,
+                conversionCurrency: event.currency ?? "USD",
+            });
+        }
+        const telemetry = await findTelemetry(ctx, args.siteId, event);
+        const existingTelemetry = telemetry.match;
         if (existingTelemetry &&
             (existingTelemetry.sessionId !== event.sessionId ||
                 existingTelemetry.visitorId !== event.visitorId)) {
@@ -184,19 +301,18 @@ export const trackConversion = mutation({
         if (existingSession && existingSession.visitorId !== event.visitorId) {
             fail("CONFLICT", "session belongs to a different visitor");
         }
-        const conversionContext = existingTelemetry
-            ? {
-                country: existingTelemetry.country,
-                city: existingTelemetry.city,
-                latitude: existingTelemetry.latitude,
-                longitude: existingTelemetry.longitude,
-                browser: existingTelemetry.browser,
-                os: existingTelemetry.os,
-                device: existingTelemetry.device,
-            }
-            : sanitizeContext(undefined);
+        const conversionContext = {
+            country: existingTelemetry?.country ?? existingSession?.country,
+            city: existingTelemetry?.city ?? existingSession?.city,
+            latitude: existingTelemetry?.latitude ?? existingSession?.latitude,
+            longitude: existingTelemetry?.longitude ?? existingSession?.longitude,
+            browser: existingTelemetry?.browser ?? existingSession?.browser,
+            os: existingTelemetry?.os ?? existingSession?.os,
+            device: existingTelemetry?.device ?? existingSession?.device,
+        };
         const aggregateDeltas = new Map();
-        await accountEvent(ctx, {
+        const reportRollupDeltas = createReportRollupDeltas();
+        const session = await accountEvent(ctx, {
             siteId: args.siteId,
             event,
             context: conversionContext,
@@ -204,105 +320,170 @@ export const trackConversion = mutation({
             goals: [],
             funnels: [],
             aggregateDeltas,
-            recordTelemetry: existingTelemetry === null,
+            reportRollupDeltas,
+            recordTelemetry: existingTelemetry === null && !telemetry.saturated,
             financialConversion: true,
-            trustedGoalKey: args.goalKey
-                ? sanitizeSlug(args.goalKey, "goalKey")
-                : undefined,
+            trustedGoalKey,
             now: Date.now(),
         });
+        if (!existingTelemetry && !telemetry.saturated) {
+            await insertEventBatch(ctx, args.siteId, event, session.country ?? "unknown");
+        }
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
+        await ensureAnalyticsControl(ctx, args.siteId, Date.now());
         return { accepted: 1, duplicates: 0, rejected: 0 };
     },
 });
+function sameTrustedConversion(existing, event, goalKey) {
+    return (existing.source === "trusted" &&
+        existing.sessionId === event.sessionId &&
+        existing.visitorId === event.visitorId &&
+        existing.timestamp === event.timestamp &&
+        existing.path === event.path &&
+        existing.revenueCents === (event.revenueCents ?? 0) &&
+        existing.currency === (event.currency ?? "USD") &&
+        existing.goalKey === goalKey);
+}
+async function loadSession(ctx, siteId, sessionId, batchState) {
+    const cached = batchState.sessions.get(sessionId);
+    if (cached)
+        return cached;
+    const current = await ctx.db
+        .query("sessions")
+        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+        .unique();
+    const loaded = { current, dirty: false };
+    batchState.sessions.set(sessionId, loaded);
+    return loaded;
+}
+async function loadLiveSession(ctx, siteId, sessionId, batchState) {
+    const cached = batchState.liveSessions.get(sessionId);
+    if (cached)
+        return cached;
+    const current = await ctx.db
+        .query("liveSessions")
+        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+        .unique();
+    const loaded = { current, dirty: false };
+    batchState.liveSessions.set(sessionId, loaded);
+    return loaded;
+}
+async function flushBatchState(ctx, batchState) {
+    for (const state of batchState.sessions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("sessions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.liveSessions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("liveSessions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.visitorAttributions.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("visitorAttributions", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    for (const state of batchState.funnelProgress.values()) {
+        if (state.dirty && state.current) {
+            await ctx.db.replace("funnelProgress", state.current._id, withoutSystemFields(state.current));
+        }
+    }
+    await flushEventBatches(ctx, batchState.eventBatches);
+}
+function withoutSystemFields(document) {
+    const { _id, _creationTime, ...fields } = document;
+    void _id;
+    void _creationTime;
+    return fields;
+}
 async function accountEvent(ctx, input) {
     const { event } = input;
     const session = input.recordTelemetry
-        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion)
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.reportRollupDeltas, input.batchState)
         : await accountTrustedSession(ctx, input.existingSession, event);
-    if (input.recordTelemetry) {
-        await ctx.db.insert("events", {
-            siteId: input.siteId,
-            eventId: event.eventId,
-            sessionId: event.sessionId,
-            visitorId: event.visitorId,
-            type: event.type,
-            ...(event.name ? { name: event.name } : {}),
-            path: event.path,
-            ...(event.referrer ? { referrer: event.referrer } : {}),
-            timestamp: event.timestamp,
-            sequence: event.sequence,
-            ...(event.target ? { target: event.target } : {}),
-            ...(event.href ? { href: event.href } : {}),
-            ...(event.properties ? { properties: event.properties } : {}),
-            ...(event.revenueCents !== undefined
-                ? { revenueCents: event.revenueCents }
-                : {}),
-            ...(event.currency ? { currency: event.currency } : {}),
-            ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
-            ...(input.context.country ? { country: input.context.country } : {}),
-            ...(input.context.city ? { city: input.context.city } : {}),
-            ...(input.context.latitude !== undefined
-                ? { latitude: input.context.latitude }
-                : {}),
-            ...(input.context.longitude !== undefined
-                ? { longitude: input.context.longitude }
-                : {}),
-            browser: input.context.browser ?? "Other",
-            os: input.context.os ?? "Other",
-            device: input.context.device ?? "unknown",
-        });
+    if (input.recordTelemetry && event.type === "pageview") {
+        // `existingSession` is the snapshot from before `upsertSession`, so it
+        // still carries the route the visitor is leaving.
+        foldRouteRollups(input.reportRollupDeltas, input.siteId, event, session, input.existingSession);
     }
     const matchedGoalKeys = [];
-    for (const goal of input.goals) {
-        if (!goal.active || !goalMatches(goal, event))
-            continue;
-        const prior = await ctx.db
+    const matchingGoals = input.goals.filter((goal) => goal.active && goalMatches(goal, event));
+    if (matchingGoals.length > 1) {
+        fail("LIMIT_EXCEEDED", "active goals must use unique matchers");
+    }
+    for (const goal of matchingGoals) {
+        const completionEventId = JSON.stringify([
+            event.sessionId,
+            eventBucketStart(event.timestamp),
+            event.eventId,
+        ]);
+        const scopedPrior = (await ctx.db
             .query("goalCompletions")
             .withIndex("by_siteId_and_goalId_and_eventId", (range) => range
             .eq("siteId", input.siteId)
             .eq("goalId", goal._id)
-            .eq("eventId", event.eventId))
-            .unique();
-        if (prior)
+            .eq("eventId", completionEventId))
+            .take(2)).find((completion) => completion.idempotencyVersion === 2);
+        const legacyPrior = scopedPrior
+            ? undefined
+            : (await ctx.db
+                .query("goalCompletions")
+                .withIndex("by_siteId_and_goalId_and_eventId", (range) => range
+                .eq("siteId", input.siteId)
+                .eq("goalId", goal._id)
+                .eq("eventId", event.eventId))
+                .take(2)).find((completion) => completion.idempotencyVersion === undefined &&
+                completion.sessionId === event.sessionId &&
+                eventBucketStart(completion.timestamp) ===
+                    eventBucketStart(event.timestamp));
+        if (scopedPrior || legacyPrior)
             continue;
         await ctx.db.insert("goalCompletions", {
             siteId: input.siteId,
             goalId: goal._id,
             goalKey: goal.key,
-            eventId: event.eventId,
+            eventId: completionEventId,
+            idempotencyVersion: 2,
             sessionId: event.sessionId,
             visitorId: event.visitorId,
             timestamp: event.timestamp,
             valueCents: goal.valueCents ?? 0,
         });
+        foldGoalRollup(input.reportRollupDeltas, input.siteId, event, goal);
         matchedGoalKeys.push(goal.key);
     }
     for (const funnel of input.funnels) {
         if (funnel.active) {
-            await advanceFunnel(ctx, input.siteId, funnel, event);
+            await advanceFunnel(ctx, input.siteId, funnel, event, input.reportRollupDeltas, input.batchState);
         }
     }
     if (input.financialConversion) {
-        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey);
+        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey, input.reportRollupDeltas);
     }
+    const campaignDimensions = session.utmCampaign
+        ? [{ type: "campaign", value: session.utmCampaign }]
+        : [];
     const dimensions = [
         { type: "source", value: session.source },
+        ...campaignDimensions,
         { type: "page", value: event.path },
-        { type: "country", value: input.context.country ?? "unknown" },
+        { type: "country", value: session.country ?? "unknown" },
         { type: "device", value: input.context.device ?? "unknown" },
         { type: "browser", value: input.context.browser ?? "Other" },
         {
             type: "event",
-            value: event.type === "custom" ? event.name ?? "custom" : event.type,
+            value: event.type === "custom" ? (event.name ?? "custom") : event.type,
         },
         ...matchedGoalKeys.map((value) => ({ type: "goal", value })),
     ];
     foldAggregates(input.aggregateDeltas, input.siteId, event, session, dimensions, input.recordTelemetry, input.financialConversion);
     if (input.recordTelemetry &&
         event.timestamp >= input.now - LIVE_SESSION_TTL_MS) {
-        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now);
+        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now, input.batchState);
     }
+    return session;
 }
 async function accountTrustedSession(ctx, existing, event) {
     if (!existing) {
@@ -320,6 +501,12 @@ async function accountTrustedSession(ctx, existing, event) {
         isNewVisitorDay: false,
         startedAt: existing.startedAt,
         source: existing.source,
+        utmCampaign: existing.utmCampaign,
+        visitorKey: existing.visitorKey ?? existing.visitorId,
+        country: existing.country,
+        city: existing.city,
+        latitude: existing.latitude,
+        longitude: existing.longitude,
         pageviewCount: existing.pageviewCount,
         eventCount: existing.eventCount,
         affiliateId: existing.affiliateId,
@@ -328,26 +515,41 @@ async function accountTrustedSession(ctx, existing, event) {
         affiliateExpiresAt: existing.affiliateExpiresAt,
     };
 }
-async function upsertSession(ctx, siteId, event, context, existing, financialConversion) {
-    const source = sourceFromReferrer(event.referrer);
+async function upsertSession(ctx, siteId, event, context, existing, financialConversion, reportRollupDeltas, batchState) {
+    // A host-supplied id is the identity; anonymous sessions count through the
+    // daily key derived by the HTTP action, or once per session without one.
+    const visitorKey = existing?.visitorKey ??
+        (event.visitorId !== event.sessionId
+            ? event.visitorId
+            : (context.visitorKey ?? event.sessionId));
     const utmSource = stringProperty(event.properties, "utm_source");
+    // Campaign traffic names its own source; the referrer host is the fallback.
+    const source = utmSource?.toLowerCase() ?? sourceFromReferrer(event.referrer);
     const utmMedium = stringProperty(event.properties, "utm_medium");
     const utmCampaign = stringProperty(event.properties, "utm_campaign");
-    const attribution = await attributionForEvent(ctx, siteId, event, existing);
+    const attribution = await attributionForEvent(ctx, siteId, event, existing, batchState);
     if (existing) {
+        if (attribution && existing.affiliateId !== attribution.affiliateId) {
+            foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
+        }
         const startedAt = Math.min(existing.startedAt, event.timestamp);
         const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
-        await ctx.db.patch("sessions", existing._id, {
+        const updates = {
             startedAt,
             lastSeenAt,
             entryPath: event.timestamp < existing.startedAt ? event.path : existing.entryPath,
             exitPath: event.timestamp >= existing.lastSeenAt ? event.path : existing.exitPath,
             eventCount: existing.eventCount + 1,
             pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+            ...(event.type === "pageview"
+                ? {
+                    lastPageviewAt: Math.max(existing.lastPageviewAt ?? 0, event.timestamp),
+                }
+                : {}),
             durationMs: Math.max(0, lastSeenAt - startedAt),
             conversionCount: existing.conversionCount + Number(financialConversion),
             revenueCents: existing.revenueCents +
-                (financialConversion ? event.revenueCents ?? 0 : 0),
+                (financialConversion ? (event.revenueCents ?? 0) : 0),
             ...(financialConversion && event.currency
                 ? { currency: event.currency }
                 : {}),
@@ -358,11 +560,23 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             ...(context.country ? { country: context.country } : {}),
             ...(context.city ? { city: context.city } : {}),
             ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            ...(context.longitude !== undefined
+                ? { longitude: context.longitude }
+                : {}),
             browser: context.browser ?? existing.browser,
             os: context.os ?? existing.os,
             device: context.device ?? existing.device,
-        });
+        };
+        if (batchState) {
+            const state = batchState.sessions.get(event.sessionId);
+            if (!state)
+                throw new Error("session cache was not initialized");
+            state.current = { ...existing, ...updates };
+            state.dirty = true;
+        }
+        else {
+            await ctx.db.patch("sessions", existing._id, updates);
+        }
         return {
             isNewSession: false,
             isNewVisitorHour: Math.floor(existing.lastSeenAt / HOUR_MS) <
@@ -371,20 +585,38 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
             startedAt,
             source: existing.source,
+            utmCampaign: existing.utmCampaign,
+            visitorKey,
+            country: context.country ?? existing.country,
+            city: context.city ?? existing.city,
+            latitude: context.latitude ?? existing.latitude,
+            longitude: context.longitude ?? existing.longitude,
             pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
             eventCount: existing.eventCount + 1,
             ...(attribution ?? {}),
         };
     }
-    const priorVisit = await ctx.db
+    const priorVisits = await ctx.db
         .query("sessions")
         .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
         .order("desc")
         .take(1);
-    await ctx.db.insert("sessions", {
+    let priorVisit = priorVisits[0] ?? null;
+    if (batchState) {
+        for (const state of batchState.sessions.values()) {
+            const candidate = state.current;
+            if (candidate?.visitorId === event.visitorId &&
+                candidate.sessionId !== event.sessionId &&
+                (!priorVisit || candidate.lastSeenAt > priorVisit.lastSeenAt)) {
+                priorVisit = candidate;
+            }
+        }
+    }
+    const fields = {
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
+        visitorKey,
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         entryPath: event.path,
@@ -397,30 +629,52 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(context.longitude !== undefined
+            ? { longitude: context.longitude }
+            : {}),
         browser: context.browser ?? "Other",
         os: context.os ?? "Other",
         device: context.device ?? "unknown",
         eventCount: 1,
         pageviewCount: event.type === "pageview" ? 1 : 0,
+        ...(event.type === "pageview" ? { lastPageviewAt: event.timestamp } : {}),
         durationMs: 0,
         conversionCount: Number(financialConversion),
-        revenueCents: financialConversion ? event.revenueCents ?? 0 : 0,
+        revenueCents: financialConversion ? (event.revenueCents ?? 0) : 0,
         ...(financialConversion && event.currency
             ? { currency: event.currency }
             : {}),
         ...(attribution ?? {}),
-    });
+    };
+    const sessionId = await ctx.db.insert("sessions", fields);
+    if (attribution) {
+        foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
+    }
+    if (batchState) {
+        const state = batchState.sessions.get(event.sessionId);
+        if (!state)
+            throw new Error("session cache was not initialized");
+        state.current = {
+            ...fields,
+            _id: sessionId,
+        };
+    }
     return {
         isNewSession: true,
-        isNewVisitorHour: priorVisit.length === 0 ||
-            Math.floor(priorVisit[0].lastSeenAt / HOUR_MS) <
+        isNewVisitorHour: priorVisit === null ||
+            Math.floor(priorVisit.lastSeenAt / HOUR_MS) <
                 Math.floor(event.timestamp / HOUR_MS),
-        isNewVisitorDay: priorVisit.length === 0 ||
-            Math.floor(priorVisit[0].lastSeenAt / DAY_MS) <
+        isNewVisitorDay: priorVisit === null ||
+            Math.floor(priorVisit.lastSeenAt / DAY_MS) <
                 Math.floor(event.timestamp / DAY_MS),
         startedAt: event.timestamp,
         source,
+        utmCampaign,
+        visitorKey,
+        country: context.country,
+        city: context.city,
+        latitude: context.latitude,
+        longitude: context.longitude,
         pageviewCount: event.type === "pageview" ? 1 : 0,
         eventCount: 1,
         ...(attribution ?? {}),
@@ -446,7 +700,7 @@ function attributionApplies(attribution, timestamp) {
 }
 // The first valid touch wins through its inclusive expiry. Only a later,
 // in-order valid touch may replace the visitor's single bounded record.
-async function attributionForEvent(ctx, siteId, event, existingSession) {
+async function attributionForEvent(ctx, siteId, event, existingSession, batchState) {
     const sessionAttribution = existingSession
         ? attributionSnapshot(existingSession)
         : undefined;
@@ -458,10 +712,21 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
         attributionApplies(sessionAttribution, event.timestamp)) {
         return sessionAttribution;
     }
-    const current = await ctx.db
-        .query("visitorAttributions")
-        .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
-        .unique();
+    let attributionState = batchState?.visitorAttributions.get(event.visitorId);
+    if (batchState && !attributionState) {
+        const current = await ctx.db
+            .query("visitorAttributions")
+            .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+            .unique();
+        attributionState = { current, dirty: false };
+        batchState.visitorAttributions.set(event.visitorId, attributionState);
+    }
+    const current = attributionState
+        ? attributionState.current
+        : await ctx.db
+            .query("visitorAttributions")
+            .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+            .unique();
     if (current) {
         const snapshot = {
             affiliateId: current.affiliateId,
@@ -489,23 +754,32 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
     }
     const attributedAt = event.timestamp;
     const expiresAt = attributedAt + affiliate.attributionWindowDays * DAY_MS;
+    const fields = {
+        siteId,
+        visitorId: event.visitorId,
+        affiliateId: affiliate._id,
+        affiliateSlug: affiliate.slug,
+        attributedAt,
+        expiresAt,
+    };
     if (current) {
-        await ctx.db.patch("visitorAttributions", current._id, {
-            affiliateId: affiliate._id,
-            affiliateSlug: affiliate.slug,
-            attributedAt,
-            expiresAt,
-        });
+        if (attributionState) {
+            attributionState.current = { ...current, ...fields };
+            attributionState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("visitorAttributions", current._id, fields);
+        }
     }
     else {
-        await ctx.db.insert("visitorAttributions", {
-            siteId,
-            visitorId: event.visitorId,
-            affiliateId: affiliate._id,
-            affiliateSlug: affiliate.slug,
-            attributedAt,
-            expiresAt,
-        });
+        const attributionId = await ctx.db.insert("visitorAttributions", fields);
+        if (attributionState) {
+            attributionState.current = {
+                ...fields,
+                _id: attributionId,
+                _creationTime: event.timestamp,
+            };
+        }
     }
     return {
         affiliateId: affiliate._id,
@@ -514,44 +788,56 @@ async function attributionForEvent(ctx, siteId, event, existingSession) {
         affiliateExpiresAt: expiresAt,
     };
 }
-async function accountHeartbeat(ctx, siteId, session, event, context, now) {
+async function accountHeartbeat(ctx, siteId, session, event, context, now, batchState) {
     const startedAt = Math.min(session.startedAt, event.timestamp);
     const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
-    await ctx.db.patch("sessions", session._id, {
+    const updates = {
         startedAt,
         lastSeenAt,
-        exitPath: event.timestamp >= session.lastSeenAt
-            ? event.path
-            : session.exitPath,
+        exitPath: event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
         durationMs: Math.max(0, lastSeenAt - startedAt),
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(context.longitude !== undefined
+            ? { longitude: context.longitude }
+            : {}),
         browser: context.browser ?? session.browser,
         os: context.os ?? session.os,
         device: context.device ?? session.device,
-    });
+    };
+    if (batchState) {
+        const state = batchState.sessions.get(event.sessionId);
+        if (!state)
+            throw new Error("session cache was not initialized");
+        state.current = { ...session, ...updates };
+        state.dirty = true;
+    }
+    else {
+        await ctx.db.patch("sessions", session._id, updates);
+    }
     await updateLiveSession(ctx, siteId, event, context, {
         isNewSession: false,
         isNewVisitorHour: false,
         isNewVisitorDay: false,
         startedAt,
         source: session.source,
+        visitorKey: session.visitorKey ?? session.visitorId,
+        country: context.country ?? session.country,
+        city: context.city ?? session.city,
+        latitude: context.latitude ?? session.latitude,
+        longitude: context.longitude ?? session.longitude,
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         affiliateSlug: session.affiliateSlug,
-    }, now);
+    }, now, batchState);
 }
-async function consumeSiteRateLimit(ctx, siteId, events, now) {
+async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
     const increments = new Map();
-    const eventByteCounts = [];
-    const encoder = new TextEncoder();
     for (let index = 0; index < events.length; index += 1) {
         const event = events[index];
-        const byteCount = encoder.encode(JSON.stringify(event)).byteLength;
-        eventByteCounts.push(byteCount);
+        const byteCount = eventByteCounts[index];
         const shard = stableHash(event.sessionId) % SITE_INGEST_SHARDS;
         const current = increments.get(shard) ?? { eventCount: 0, byteCount: 0 };
         current.eventCount += 1;
@@ -560,13 +846,13 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
     }
     const windows = [];
     for (const [shard, increment] of increments) {
-        const window = await ctx.db
+        const rows = await ctx.db
             .query("siteIngestWindows")
-            .withIndex("by_siteId_and_shard_and_windowStart", (range) => range
-            .eq("siteId", siteId)
-            .eq("shard", shard)
-            .eq("windowStart", windowStart))
-            .unique();
+            .withIndex("by_siteId_and_shard_and_windowStart", (range) => range.eq("siteId", siteId).eq("shard", shard))
+            .order("desc")
+            .take(1);
+        const storedWindow = rows[0] ?? null;
+        const window = storedWindow?.windowStart === windowStart ? storedWindow : null;
         if ((window?.eventCount ?? 0) + increment.eventCount >
             MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
             (window?.byteCount ?? 0) + increment.byteCount >
@@ -580,13 +866,18 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
                 siteByteLimit: SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
             });
         }
-        windows.push({ shard, increment, window });
+        windows.push({ shard, increment, window: storedWindow });
     }
     for (const { shard, increment, window } of windows) {
         if (window) {
             await ctx.db.patch("siteIngestWindows", window._id, {
-                eventCount: window.eventCount + increment.eventCount,
-                byteCount: window.byteCount + increment.byteCount,
+                windowStart,
+                eventCount: window.windowStart === windowStart
+                    ? window.eventCount + increment.eventCount
+                    : increment.eventCount,
+                byteCount: window.windowStart === windowStart
+                    ? window.byteCount + increment.byteCount
+                    : increment.byteCount,
                 updatedAt: now,
             });
         }
@@ -601,76 +892,124 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
             });
         }
     }
-    return eventByteCounts;
 }
-async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now) {
+async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now, states) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
-    const window = await ctx.db
-        .query("ingestWindows")
-        .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range
-        .eq("siteId", siteId)
-        .eq("sessionId", sessionId)
-        .eq("windowStart", windowStart))
-        .unique();
-    if (window &&
-        (window.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
-            window.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW)) {
-        return false;
-    }
-    if (window) {
-        await ctx.db.patch("ingestWindows", window._id, {
-            eventCount: window.eventCount + 1,
-            byteCount: window.byteCount + byteCount,
-            updatedAt: now,
-        });
-    }
-    else {
-        await ctx.db.insert("ingestWindows", {
+    let state = states.get(sessionId);
+    if (!state) {
+        const rows = await ctx.db
+            .query("ingestWindows")
+            .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+            .order("desc")
+            .take(1);
+        const window = rows[0] ?? null;
+        state = {
             siteId,
             sessionId,
-            windowStart,
-            eventCount: 1,
-            byteCount,
-            updatedAt: now,
-        });
+            window,
+            eventCount: window?.windowStart === windowStart ? window.eventCount : 0,
+            byteCount: window?.windowStart === windowStart ? window.byteCount : 0,
+            dirty: false,
+        };
+        states.set(sessionId, state);
     }
+    if (state.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
+        state.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW) {
+        return false;
+    }
+    state.eventCount += 1;
+    state.byteCount += byteCount;
+    state.dirty = true;
     return true;
 }
-async function updateLiveSession(ctx, siteId, event, context, session, now) {
-    const existing = await ctx.db
-        .query("liveSessions")
-        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
-        .unique();
-    const renewsExpiry = !existing ||
-        existing.expiresAt <= now ||
-        event.type === "pageview" ||
-        event.type === "heartbeat";
-    const deadline = renewsExpiry
-        ? now + LIVE_SESSION_TTL_MS
-        : existing.expiresAt;
+async function flushSessionRateLimits(ctx, states, now) {
+    const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+    for (const state of states.values()) {
+        if (!state.dirty)
+            continue;
+        if (state.window) {
+            await ctx.db.patch("ingestWindows", state.window._id, {
+                windowStart,
+                eventCount: state.eventCount,
+                byteCount: state.byteCount,
+                updatedAt: now,
+            });
+        }
+        else {
+            await ctx.db.insert("ingestWindows", {
+                siteId: state.siteId,
+                sessionId: state.sessionId,
+                windowStart,
+                eventCount: state.eventCount,
+                byteCount: state.byteCount,
+                updatedAt: now,
+            });
+        }
+    }
+}
+async function updateLiveSession(ctx, siteId, event, context, session, now, batchState) {
+    await ensureLiveSweep(ctx, siteId, now, batchState);
+    const liveState = batchState
+        ? await loadLiveSession(ctx, siteId, event.sessionId, batchState)
+        : undefined;
+    const existing = liveState
+        ? liveState.current
+        : await ctx.db
+            .query("liveSessions")
+            .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
+            .unique();
+    const deadline = now + LIVE_SESSION_TTL_MS;
+    const country = context.country ?? session.country;
+    const city = context.city ?? session.city;
+    const latitude = context.latitude ?? session.latitude;
+    const longitude = context.longitude ?? session.longitude;
     if (existing) {
-        await ctx.db.patch("liveSessions", existing._id, {
+        const recentHeartbeatEventIds = event.type === "heartbeat"
+            ? [
+                ...(existing.recentHeartbeatEventIds ??
+                    (existing.lastHeartbeatEventId
+                        ? [existing.lastHeartbeatEventId]
+                        : [])),
+                event.eventId,
+            ].slice(-8)
+            : undefined;
+        const updates = {
             visitorId: event.visitorId,
             path: event.path,
             lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
             expiresAt: deadline,
-            ...(context.country ? { country: context.country } : {}),
-            ...(context.city ? { city: context.city } : {}),
-            ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            ...(country ? { country } : {}),
+            ...(city ? { city } : {}),
+            ...(latitude !== undefined ? { latitude } : {}),
+            ...(longitude !== undefined ? { longitude } : {}),
             browser: context.browser ?? existing.browser,
             os: context.os ?? existing.os,
             source: session.source,
             device: context.device ?? existing.device,
             pageviewCount: session.pageviewCount,
             eventCount: session.eventCount,
+            // The live map animates a visitor from the route they left to the one
+            // they reached; a heartbeat or click on the same page is not a move.
+            ...(event.type === "pageview" && event.path !== existing.path
+                ? { previousPath: existing.path, movedAt: event.timestamp }
+                : {}),
             ...(event.type === "heartbeat"
-                ? { lastHeartbeatEventId: event.eventId }
+                ? {
+                    lastHeartbeatEventId: event.eventId,
+                    recentHeartbeatEventIds,
+                }
                 : {}),
-        });
+        };
+        if (liveState) {
+            liveState.current = { ...existing, ...updates };
+            liveState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("liveSessions", existing._id, updates);
+        }
         return;
     }
-    const liveSessionId = await ctx.db.insert("liveSessions", {
+    const fields = {
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
@@ -678,11 +1017,10 @@ async function updateLiveSession(ctx, siteId, event, context, session, now) {
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         expiresAt: deadline,
-        expiryChainDeadline: deadline,
-        ...(context.country ? { country: context.country } : {}),
-        ...(context.city ? { city: context.city } : {}),
-        ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(country ? { country } : {}),
+        ...(city ? { city } : {}),
+        ...(latitude !== undefined ? { latitude } : {}),
+        ...(longitude !== undefined ? { longitude } : {}),
         browser: context.browser ?? "Other",
         os: context.os ?? "Other",
         source: session.source,
@@ -690,13 +1028,42 @@ async function updateLiveSession(ctx, siteId, event, context, session, now) {
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         ...(event.type === "heartbeat"
-            ? { lastHeartbeatEventId: event.eventId }
+            ? {
+                lastHeartbeatEventId: event.eventId,
+                recentHeartbeatEventIds: [event.eventId],
+            }
             : {}),
-    });
-    await ctx.scheduler.runAt(deadline, internal.live.expire, {
-        liveSessionId,
-        scheduledDeadline: deadline,
-    });
+    };
+    const liveSessionId = await ctx.db.insert("liveSessions", fields);
+    if (liveState) {
+        liveState.current = {
+            ...fields,
+            _id: liveSessionId,
+        };
+    }
+}
+async function ensureLiveSweep(ctx, siteId, now, batchState) {
+    const key = String(siteId);
+    if (batchState?.liveSweepSites.has(key))
+        return;
+    const existing = await ctx.db
+        .query("liveSweepStates")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    if (!existing || existing.scheduledAt <= now) {
+        const scheduledAt = now + LIVE_SWEEP_INTERVAL_MS;
+        if (existing) {
+            await ctx.db.patch("liveSweepStates", existing._id, { scheduledAt });
+        }
+        else {
+            await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        }
+        await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
+            siteId,
+            scheduledAt,
+        });
+    }
+    batchState?.liveSweepSites.add(key);
 }
 function goalMatches(goal, event) {
     if (goal.matchType === "pageview") {
@@ -704,19 +1071,34 @@ function goalMatches(goal, event) {
     }
     return event.type === "custom" && event.name === goal.matchValue;
 }
-async function advanceFunnel(ctx, siteId, funnel, event) {
-    const progress = await ctx.db
-        .query("funnelProgress")
-        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
-        .eq("siteId", siteId)
-        .eq("funnelId", funnel._id)
-        .eq("visitorId", event.visitorId))
-        .unique();
+async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, batchState) {
+    const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
+    let progressState = batchState?.funnelProgress.get(cacheKey);
+    if (batchState && !progressState) {
+        const current = await ctx.db
+            .query("funnelProgress")
+            .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnel._id)
+            .eq("visitorId", event.visitorId))
+            .unique();
+        progressState = { current, dirty: false };
+        batchState.funnelProgress.set(cacheKey, progressState);
+    }
+    const progress = progressState
+        ? progressState.current
+        : await ctx.db
+            .query("funnelProgress")
+            .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnel._id)
+            .eq("visitorId", event.visitorId))
+            .unique();
     const matchesFirst = stepMatches(funnel.steps[0], event);
     if (!progress) {
         if (!matchesFirst)
             return;
-        await ctx.db.insert("funnelProgress", {
+        const fields = {
             siteId,
             funnelId: funnel._id,
             visitorId: event.visitorId,
@@ -724,23 +1106,47 @@ async function advanceFunnel(ctx, siteId, funnel, event) {
             currentStep: 1,
             startedAt: event.timestamp,
             lastStepAt: event.timestamp,
-        });
+        };
+        const progressId = await ctx.db.insert("funnelProgress", fields);
+        if (progressState) {
+            progressState.current = {
+                ...fields,
+                _id: progressId,
+            };
+        }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
-    if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt)
+    if (progress.completedAt !== undefined ||
+        event.timestamp < progress.lastStepAt)
         return;
     if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
         if (!matchesFirst)
             return;
-        await ctx.db.patch("funnelProgress", progress._id, {
+        const updated = {
+            ...progress,
             sessionId: event.sessionId,
             currentStep: 1,
             startedAt: event.timestamp,
             lastStepAt: event.timestamp,
-            completedAt: undefined,
-        });
+        };
+        delete updated.completedAt;
+        if (progressState) {
+            progressState.current = updated;
+            progressState.dirty = true;
+        }
+        else {
+            await ctx.db.patch("funnelProgress", progress._id, {
+                sessionId: event.sessionId,
+                currentStep: 1,
+                startedAt: event.timestamp,
+                lastStepAt: event.timestamp,
+                completedAt: undefined,
+            });
+        }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
     const nextStep = funnel.steps[progress.currentStep];
@@ -748,13 +1154,21 @@ async function advanceFunnel(ctx, siteId, funnel, event) {
         return;
     const currentStep = progress.currentStep + 1;
     const completed = currentStep === funnel.steps.length;
-    await ctx.db.patch("funnelProgress", progress._id, {
+    const updates = {
         sessionId: event.sessionId,
         currentStep,
         lastStepAt: event.timestamp,
         ...(completed ? { completedAt: event.timestamp } : {}),
-    });
+    };
+    if (progressState) {
+        progressState.current = { ...progress, ...updates };
+        progressState.dirty = true;
+    }
+    else {
+        await ctx.db.patch("funnelProgress", progress._id, updates);
+    }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
+    foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, progress.currentStep, completed);
 }
 function stepMatches(step, event) {
     return step.type === "pageview"
@@ -763,7 +1177,7 @@ function stepMatches(step, event) {
 }
 async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed) {
     const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-    const shard = stableHash(event.visitorId) % AGGREGATE_SHARDS;
+    const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
     const bucket = await ctx.db
         .query("funnelStepBuckets")
         .withIndex("by_siteId_and_funnelId_and_bucketStart_and_step_and_shard", (range) => range
@@ -791,7 +1205,7 @@ async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed
         });
     }
 }
-async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
+async function accountConversion(ctx, siteId, session, event, trustedGoalKey, reportRollupDeltas) {
     const revenueCents = event.revenueCents ?? 0;
     const currency = event.currency ?? "USD";
     const candidate = attributionSnapshot(session);
@@ -820,8 +1234,9 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         commissionCents,
     });
     if (commissionAffiliate && attribution) {
+        foldAffiliateConversionRollup(reportRollupDeltas, siteId, event, commissionAffiliate, attribution.affiliateSlug, revenueCents, commissionCents);
         const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-        const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+        const shard = stableHash(event.sessionId) % SECONDARY_AGGREGATE_SHARDS;
         const stats = await ctx.db
             .query("affiliateDailyStats")
             .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
@@ -851,15 +1266,425 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         }
     }
 }
+function createReportRollupDeltas() {
+    return {
+        goals: new Map(),
+        funnels: new Map(),
+        affiliates: new Map(),
+        vitals: new Map(),
+        routes: new Map(),
+    };
+}
+function reportRollupCoordinates(siteId, definitionId, timestamp, visitorId) {
+    const bucketStart = Math.floor(timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(visitorId) % REPORT_ROLLUP_SHARDS;
+    return {
+        bucketStart,
+        shard,
+        key: `${siteId}:${definitionId}:${bucketStart}:${shard}`,
+    };
+}
+function foldGoalRollup(deltas, siteId, event, goal) {
+    const coordinates = reportRollupCoordinates(siteId, String(goal._id), event.timestamp, event.visitorId);
+    const current = deltas.goals.get(coordinates.key) ?? {
+        siteId,
+        goalId: goal._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        goalKey: goal.key,
+        completions: 0,
+        valueCents: 0,
+    };
+    current.completions += 1;
+    current.valueCents += goal.valueCents ?? 0;
+    deltas.goals.set(coordinates.key, current);
+}
+function foldFunnelRollup(deltas, siteId, funnelId, event, step, completed) {
+    const coordinates = reportRollupCoordinates(siteId, String(funnelId), event.timestamp, event.visitorId);
+    const rollup = deltas.funnels.get(coordinates.key) ?? {
+        siteId,
+        funnelId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        steps: new Map(),
+    };
+    const current = rollup.steps.get(step) ?? {
+        step,
+        entrants: 0,
+        completions: 0,
+    };
+    current.entrants += 1;
+    current.completions += Number(completed);
+    rollup.steps.set(step, current);
+    deltas.funnels.set(coordinates.key, rollup);
+}
+function foldAffiliateVisitRollup(deltas, siteId, event, attribution, visitorKey) {
+    const coordinates = reportRollupCoordinates(siteId, String(attribution.affiliateId), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: attribution.affiliateId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug: attribution.affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.visits += 1;
+    current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
+    deltas.affiliates.set(coordinates.key, current);
+}
+function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affiliateSlug, revenueCents, commissionCents) {
+    const coordinates = reportRollupCoordinates(siteId, String(affiliate._id), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: affiliate._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.conversions += 1;
+    current.revenueCents += revenueCents;
+    current.commissionCents += commissionCents;
+    deltas.affiliates.set(coordinates.key, current);
+}
+function foldVitalRollups(deltas, siteId, event, device) {
+    const metric = event.name;
+    const value = event.value ?? 0;
+    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(event.sessionId) % REPORT_ROLLUP_SHARDS;
+    // Each measurement lands twice: on its page (across devices) and on its
+    // device class (across pages), so both breakdowns stay exact and bounded.
+    for (const target of [
+        { device: VITAL_ALL, page: event.path },
+        { device, page: VITAL_ALL },
+    ]) {
+        const key = JSON.stringify([
+            siteId,
+            bucketStart,
+            target.device,
+            target.page,
+            shard,
+        ]);
+        const row = deltas.vitals.get(key) ?? {
+            siteId,
+            bucketStart,
+            device: target.device,
+            page: target.page,
+            shard,
+            metrics: new Map(),
+        };
+        const entry = row.metrics.get(metric) ?? {
+            count: 0,
+            sum: 0,
+            histogram: emptyVitalHistogram(),
+        };
+        entry.count += 1;
+        entry.sum += value;
+        entry.histogram = addToVitalHistogram(entry.histogram, value);
+        row.metrics.set(metric, entry);
+        deltas.vitals.set(key, row);
+    }
+}
+function foldRouteDelta(deltas, siteId, route, timestamp, visitorId, apply) {
+    const coordinates = reportRollupCoordinates(siteId, `route:${route}`, timestamp, visitorId);
+    const current = deltas.routes.get(coordinates.key) ?? {
+        siteId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        route,
+        pageviews: 0,
+        entries: 0,
+        continued: 0,
+        dwellMs: 0,
+        dwellSamples: 0,
+        visitorSketch: createVisitorSketch(),
+        outgoing: new Map(),
+    };
+    apply(current);
+    deltas.routes.set(coordinates.key, current);
+}
+/**
+ * One pageview feeds the site map three ways: the route it lands on gains a
+ * view (and an entry when it opens the session), the route it leaves gains
+ * the transition and the time spent there, and the session's entry route is
+ * marked as continued the first time its visitor moves on — which is what
+ * lets the report derive bounces and exits without a session-end hook.
+ */
+function foldRouteRollups(deltas, siteId, event, session, existing) {
+    const route = normalizeRoute(event.path);
+    const priorPageviews = existing?.pageviewCount ?? 0;
+    const firstPageview = existing === null || priorPageviews === 0;
+    foldRouteDelta(deltas, siteId, route, event.timestamp, event.visitorId, (delta) => {
+        delta.pageviews += 1;
+        if (firstPageview)
+            delta.entries += 1;
+        delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+    });
+    // Out-of-order telemetry keeps its view but cannot say where it came from.
+    if (existing === null || firstPageview)
+        return;
+    if (event.timestamp < existing.lastSeenAt)
+        return;
+    const previous = normalizeRoute(existing.exitPath);
+    const lastPageviewAt = existing.lastPageviewAt;
+    foldRouteDelta(deltas, siteId, previous, event.timestamp, event.visitorId, (delta) => {
+        delta.outgoing.set(route, (delta.outgoing.get(route) ?? 0) + 1);
+        if (lastPageviewAt !== undefined && event.timestamp >= lastPageviewAt) {
+            delta.dwellMs += event.timestamp - lastPageviewAt;
+            delta.dwellSamples += 1;
+        }
+    });
+    if (priorPageviews === 1) {
+        foldRouteDelta(deltas, siteId, normalizeRoute(existing.entryPath), existing.startedAt, event.visitorId, (delta) => {
+            delta.continued += 1;
+        });
+    }
+}
+async function upsertRouteRollup(ctx, delta, dayRowCounts) {
+    const existing = await ctx.db
+        .query("routeDailyRollups")
+        .withIndex("by_siteId_and_route_and_bucketStart_and_shard", (range) => range
+        .eq("siteId", delta.siteId)
+        .eq("route", delta.route)
+        .eq("bucketStart", delta.bucketStart)
+        .eq("shard", delta.shard))
+        .unique();
+    if (!existing && delta.route !== SITE_MAP_OTHER_ROUTE) {
+        // A new route row spends one bounded read per batch to respect the daily
+        // cap; past it, the day's remaining routes fold into "(other)".
+        const capRows = MAX_SITE_MAP_ROUTES_PER_DAY * REPORT_ROLLUP_SHARDS;
+        const dayKey = `${delta.siteId}:${delta.bucketStart}`;
+        let dayRows = dayRowCounts.get(dayKey);
+        if (dayRows === undefined) {
+            dayRows = (await ctx.db
+                .query("routeDailyRollups")
+                .withIndex("by_siteId_and_bucketStart", (range) => range
+                .eq("siteId", delta.siteId)
+                .eq("bucketStart", delta.bucketStart))
+                .take(capRows + 1)).length;
+        }
+        if (dayRows >= capRows) {
+            await upsertRouteRollup(ctx, { ...delta, route: SITE_MAP_OTHER_ROUTE }, dayRowCounts);
+            return;
+        }
+        dayRowCounts.set(dayKey, dayRows + 1);
+    }
+    const outgoing = existing?.outgoing.map((entry) => ({ ...entry })) ?? [];
+    for (const [target, count] of delta.outgoing) {
+        let entry = outgoing.find((candidate) => candidate.route === target);
+        if (!entry) {
+            const named = outgoing.filter((candidate) => candidate.route !== SITE_MAP_OTHER_ROUTE).length;
+            const route = target !== SITE_MAP_OTHER_ROUTE && named >= ROUTE_TRANSITION_SLOTS
+                ? SITE_MAP_OTHER_ROUTE
+                : target;
+            entry = outgoing.find((candidate) => candidate.route === route);
+            if (!entry) {
+                entry = { route, count: 0 };
+                outgoing.push(entry);
+            }
+        }
+        entry.count += count;
+    }
+    const fields = {
+        siteId: delta.siteId,
+        bucketStart: delta.bucketStart,
+        shard: delta.shard,
+        generation: 1,
+        route: delta.route,
+        pageviews: (existing?.pageviews ?? 0) + delta.pageviews,
+        entries: (existing?.entries ?? 0) + delta.entries,
+        continued: (existing?.continued ?? 0) + delta.continued,
+        dwellMs: (existing?.dwellMs ?? 0) + delta.dwellMs,
+        dwellSamples: (existing?.dwellSamples ?? 0) + delta.dwellSamples,
+        visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
+        outgoing,
+    };
+    if (existing) {
+        await ctx.db.replace("routeDailyRollups", existing._id, fields);
+    }
+    else {
+        await ctx.db.insert("routeDailyRollups", fields);
+    }
+}
+async function upsertVitalRollup(ctx, delta) {
+    const existing = await ctx.db
+        .query("vitalRollups")
+        .withIndex("by_key", (range) => range
+        .eq("siteId", delta.siteId)
+        .eq("bucketStart", delta.bucketStart)
+        .eq("device", delta.device)
+        .eq("page", delta.page)
+        .eq("shard", delta.shard))
+        .unique();
+    if (!existing &&
+        delta.device === VITAL_ALL &&
+        delta.page !== VITAL_OTHER_PAGES) {
+        // A new page row spends one bounded read to respect the daily page slots;
+        // past the cap, the day's remaining pages fold into "(other)".
+        const pageRows = await ctx.db
+            .query("vitalRollups")
+            .withIndex("by_key", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("device", VITAL_ALL))
+            .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
+        if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
+            await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
+            return;
+        }
+    }
+    const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
+    for (const [metric, addition] of delta.metrics) {
+        const entry = metrics.find((candidate) => candidate.metric === metric);
+        if (entry) {
+            entry.count += addition.count;
+            entry.sum += addition.sum;
+            entry.histogram = mergeVitalHistograms(entry.histogram, addition.histogram);
+        }
+        else {
+            metrics.push({
+                metric,
+                count: addition.count,
+                sum: addition.sum,
+                histogram: [...addition.histogram],
+            });
+        }
+    }
+    const fields = {
+        siteId: delta.siteId,
+        bucketStart: delta.bucketStart,
+        device: delta.device,
+        page: delta.page,
+        shard: delta.shard,
+        metrics,
+    };
+    if (existing) {
+        await ctx.db.replace("vitalRollups", existing._id, fields);
+    }
+    else {
+        await ctx.db.insert("vitalRollups", fields);
+    }
+}
+async function flushReportRollups(ctx, deltas) {
+    for (const delta of deltas.goals.values()) {
+        const existing = await ctx.db
+            .query("goalDailyRollups")
+            .withIndex("by_siteId_and_goalId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("goalId", delta.goalId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            goalId: delta.goalId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            goalKey: delta.goalKey,
+            completions: (existing?.completions ?? 0) + delta.completions,
+            valueCents: (existing?.valueCents ?? 0) + delta.valueCents,
+        };
+        if (existing) {
+            await ctx.db.replace("goalDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("goalDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.funnels.values()) {
+        const existing = await ctx.db
+            .query("funnelDailyRollups")
+            .withIndex("by_siteId_and_funnelId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("funnelId", delta.funnelId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const steps = existing?.steps.map((step) => ({ ...step })) ?? [];
+        for (const addition of delta.steps.values()) {
+            const current = steps.find((step) => step.step === addition.step);
+            if (current) {
+                current.entrants += addition.entrants;
+                current.completions += addition.completions;
+            }
+            else {
+                if (steps.length >= MAX_FUNNEL_STEPS) {
+                    fail("LIMIT_EXCEEDED", "daily funnel rollup step limit exceeded");
+                }
+                steps.push(addition);
+            }
+        }
+        const fields = {
+            siteId: delta.siteId,
+            funnelId: delta.funnelId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            steps,
+        };
+        if (existing) {
+            await ctx.db.replace("funnelDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("funnelDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.affiliates.values()) {
+        const existing = await ctx.db
+            .query("affiliateDailyRollups")
+            .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("affiliateId", delta.affiliateId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            affiliateId: delta.affiliateId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            affiliateSlug: delta.affiliateSlug,
+            visits: (existing?.visits ?? 0) + delta.visits,
+            visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
+            conversions: (existing?.conversions ?? 0) + delta.conversions,
+            revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
+            commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
+        };
+        if (existing) {
+            await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("affiliateDailyRollups", fields);
+        }
+    }
+    const routeDayRowCounts = new Map();
+    for (const delta of deltas.routes.values()) {
+        await upsertRouteRollup(ctx, delta, routeDayRowCounts);
+    }
+    for (const delta of deltas.vitals.values()) {
+        await upsertVitalRollup(ctx, delta);
+    }
+}
 function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
-    const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
-    const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
+    const revenueCents = financialConversion ? (event.revenueCents ?? 0) : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
         ...dimension,
         count: Number(recordTelemetry),
         revenueCents,
     }));
     for (const granularity of ["hour", "day"]) {
+        const shard = stableHash(event.sessionId) %
+            (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
         const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
         const bucketStart = Math.floor(event.timestamp / interval) * interval;
         const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
@@ -877,6 +1702,7 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 outboundClicks: 0,
                 sessions: 0,
                 visitors: 0,
+                visitorSketch: createVisitorSketch(),
                 conversions: 0,
                 revenueCents: 0,
                 dimensions: [],
@@ -895,6 +1721,9 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 ? Number(session.isNewVisitorHour)
                 : Number(session.isNewVisitorDay)
             : 0;
+        if (recordTelemetry) {
+            delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+        }
         delta.conversions += Number(financialConversion);
         delta.revenueCents += revenueCents;
         delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
@@ -911,6 +1740,13 @@ async function flushAggregates(ctx, aggregateDeltas) {
             .eq("shard", delta.shard))
             .unique();
         const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions);
+        const priorSketch = bucket?.visitorSketch;
+        const visitorSketch = priorSketch
+            ? unionVisitorSketches(priorSketch, delta.visitorSketch)
+            : delta.visitorSketch;
+        const visitorSketchComplete = bucket
+            ? bucket.visitorSketchComplete === true && priorSketch !== undefined
+            : true;
         if (bucket) {
             await ctx.db.patch("aggregateBuckets", bucket._id, {
                 events: bucket.events + delta.events,
@@ -920,6 +1756,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: bucket.outboundClicks + delta.outboundClicks,
                 sessions: bucket.sessions + delta.sessions,
                 visitors: bucket.visitors + delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: bucket.conversions + delta.conversions,
                 revenueCents: bucket.revenueCents + delta.revenueCents,
                 dimensions,
@@ -938,6 +1776,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: delta.outboundClicks,
                 sessions: delta.sessions,
                 visitors: delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: delta.conversions,
                 revenueCents: delta.revenueCents,
                 dimensions,
@@ -955,7 +1795,7 @@ function mergeDimensions(current, additions) {
             continue;
         }
         const typeSlots = result.filter((candidate) => candidate.type === addition.type);
-        if (typeSlots.length < DIMENSION_SLOTS) {
+        if (typeSlots.length < DIMENSION_SLOTS - 1) {
             const occupied = Array.from({ length: DIMENSION_SLOTS }, () => false);
             for (const candidate of typeSlots)
                 occupied[candidate.slot] = true;
@@ -971,21 +1811,43 @@ function mergeDimensions(current, additions) {
             });
             continue;
         }
-        if (addition.count === 0)
+        const overflow = typeSlots.find((candidate) => candidate.value === "(other)");
+        if (overflow) {
+            overflow.count += addition.count;
+            overflow.revenueCents += addition.revenueCents;
             continue;
+        }
+        if (addition.count === 0 && addition.revenueCents === 0)
+            continue;
+        if (typeSlots.length < DIMENSION_SLOTS) {
+            const occupied = new Set(typeSlots.map((candidate) => candidate.slot));
+            let slot = stableHash("(other)") % DIMENSION_SLOTS;
+            while (occupied.has(slot))
+                slot = (slot + 1) % DIMENSION_SLOTS;
+            result.push({
+                type: addition.type,
+                slot,
+                value: "(other)",
+                count: addition.count,
+                revenueCents: addition.revenueCents,
+            });
+            continue;
+        }
         let replacement = typeSlots[0];
         for (const candidate of typeSlots.slice(1)) {
             if (candidate.count < replacement.count)
                 replacement = candidate;
         }
-        replacement.value = addition.value;
+        replacement.value = "(other)";
         replacement.count += addition.count;
-        replacement.revenueCents = addition.revenueCents;
+        replacement.revenueCents += addition.revenueCents;
     }
     return result;
 }
 function stringProperty(properties, key) {
     const value = properties?.[key];
-    return typeof value === "string" ? cleanString(value, 120) || undefined : undefined;
+    return typeof value === "string"
+        ? cleanString(value, 120) || undefined
+        : undefined;
 }
 //# sourceMappingURL=ingest.js.map