@iann29/rastro 0.1.0-alpha.8 → 0.1.0

package/dist/component/ingest.js

@@ -1,14 +1,25 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, MAX_SITE_MAP_ROUTES_PER_DAY, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, ROUTE_TRANSITION_SLOTS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, SITE_MAP_OTHER_ROUTE, VITAL_PAGE_SLOTS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_LEAVE_GRACE_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
 import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
-import { addToVitalHistogram, emptyVitalHistogram, mergeVitalHistograms, vitalDevice, VITAL_ALL, VITAL_OTHER_PAGES, } from "./vitals.js";
+import { addToVitalHistogram, emptyVitalHistogram, vitalDevice, VITAL_ALL, } from "./vitals.js";
 import { fail } from "./errors.js";
 import { ensureAnalyticsControl } from "./coverage.js";
 import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
-import { cleanString, isValidMoney, measureBatchBytes, normalizeRoute, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { cleanString, isValidMoney, measureBatchBytes, normalizeRoute, originAllowed, publicUrlHost, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
+import { createReportRollupDeltas, enqueueReportRollups, reportRollupCoordinates, } from "./rollupStore.js";
+import { localDayStart, localDayTimezone } from "./localTime.js";
+function furtherReach(current, next) {
+    if (!next)
+        return current;
+    if (!current)
+        return next;
+    return next.step / next.steps >= current.step / current.steps
+        ? next
+        : current;
+}
 export const ingestBatch = mutation({
     args: {
         siteId: v.id("sites"),
@@ -42,6 +53,7 @@ export const ingestBatch = mutation({
         await ensureAnalyticsControl(ctx, args.siteId, now);
         await consumeSiteRateLimit(ctx, args.siteId, args.events, eventByteCounts, now);
         const context = sanitizeContext(args.context);
+        const timezone = localDayTimezone(site.timezone);
         const aggregateDeltas = new Map();
         const reportRollupDeltas = createReportRollupDeltas();
         const sessionRateWindows = new Map();
@@ -77,7 +89,9 @@ export const ingestBatch = mutation({
             const bucketStart = eventBucketStart(event.timestamp);
             const groupKey = eventBatchKey(event.sessionId, bucketStart);
             requestGroups.add(groupKey);
-            if (event.type === "heartbeat" || event.type === "vital")
+            if (event.type === "heartbeat" ||
+                event.type === "vital" ||
+                event.type === "leave")
                 continue;
             const eventIds = eventIdsByGroup.get(groupKey) ?? new Set();
             if (eventIds.has(event.eventId)) {
@@ -141,7 +155,37 @@ export const ingestBatch = mutation({
                     rejected += 1;
                     continue;
                 }
-                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
+                const durationDeltaMs = await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
+                // A heartbeat is presence, not telemetry, but it is what keeps a
+                // session's duration honest while the visitor reads: the growth is
+                // folded into the buckets the heartbeat landed in.
+                foldSessionDuration(aggregateDeltas, args.siteId, event, durationDeltaMs, timezone);
+                accepted += 1;
+                continue;
+            }
+            if (event.type === "leave") {
+                // A departure only shortens live presence: it stores no telemetry,
+                // touches no aggregates, and never extends anything, so a replayed
+                // or late leave is inert. The read side filters by expiresAt and the
+                // sweep deletes the row, so the dashboard sees the exit in seconds.
+                if (!existingSession) {
+                    rejected += 1;
+                    continue;
+                }
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
+                    rejected += 1;
+                    continue;
+                }
+                const liveState = await loadLiveSession(ctx, args.siteId, event.sessionId, batchState);
+                const liveSession = liveState.current;
+                if (liveSession) {
+                    const expiresAt = Math.min(liveSession.expiresAt, now + LIVE_LEAVE_GRACE_MS);
+                    if (expiresAt < liveSession.expiresAt) {
+                        await ctx.db.patch("liveSessions", liveSession._id, { expiresAt });
+                        liveState.current = { ...liveSession, expiresAt };
+                    }
+                    await ensureLiveSweep(ctx, args.siteId, now, batchState);
+                }
                 accepted += 1;
                 continue;
             }
@@ -194,6 +238,7 @@ export const ingestBatch = mutation({
                 aggregateDeltas,
                 reportRollupDeltas,
                 recordTelemetry: true,
+                timezone,
                 financialConversion: false,
                 trustedGoalKey: undefined,
                 now,
@@ -205,7 +250,7 @@ export const ingestBatch = mutation({
         await flushBatchState(ctx, batchState);
         await flushSessionRateLimits(ctx, sessionRateWindows, now);
         await flushAggregates(ctx, aggregateDeltas);
-        await flushReportRollups(ctx, reportRollupDeltas);
+        await enqueueReportRollups(ctx, args.siteId, reportRollupDeltas, now);
         return { accepted, duplicates, rejected };
     },
 });
@@ -294,6 +339,7 @@ export const trackConversion = mutation({
             aggregateDeltas,
             reportRollupDeltas,
             recordTelemetry: existingTelemetry === null && !telemetry.saturated,
+            timezone: localDayTimezone(site.timezone),
             financialConversion: true,
             trustedGoalKey,
             now: Date.now(),
@@ -302,8 +348,9 @@ export const trackConversion = mutation({
             await insertEventBatch(ctx, args.siteId, event, session.country ?? "unknown");
         }
         await flushAggregates(ctx, aggregateDeltas);
-        await flushReportRollups(ctx, reportRollupDeltas);
-        await ensureAnalyticsControl(ctx, args.siteId, Date.now());
+        const now = Date.now();
+        await enqueueReportRollups(ctx, args.siteId, reportRollupDeltas, now);
+        await ensureAnalyticsControl(ctx, args.siteId, now);
         return { accepted: 1, duplicates: 0, rejected: 0 };
     },
 });
@@ -426,20 +473,34 @@ async function accountEvent(ctx, input) {
         foldGoalRollup(input.reportRollupDeltas, input.siteId, event, goal);
         matchedGoalKeys.push(goal.key);
     }
+    let reach;
     for (const funnel of input.funnels) {
         if (funnel.active) {
-            await advanceFunnel(ctx, input.siteId, funnel, event, input.reportRollupDeltas, input.batchState);
+            reach = furtherReach(reach, await advanceFunnel(ctx, input.siteId, funnel, event, input.reportRollupDeltas, input.batchState));
         }
     }
     if (input.financialConversion) {
         await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey, input.reportRollupDeltas);
     }
-    const campaignDimensions = session.utmCampaign
-        ? [{ type: "campaign", value: session.utmCampaign }]
-        : [];
+    const campaignDimensions = [];
+    if (session.utmCampaign) {
+        campaignDimensions.push({ type: "campaign", value: session.utmCampaign });
+    }
+    if (session.utmMedium) {
+        // A medium is a taxonomy ("social", "cpc", "email"), so its case is noise.
+        campaignDimensions.push({
+            type: "medium",
+            value: session.utmMedium.toLowerCase(),
+        });
+    }
+    // An outbound click ranks by where it went: the host alone, never the path.
+    const outboundHost = event.type === "outbound" ? publicUrlHost(event.href) : undefined;
     const dimensions = [
         { type: "source", value: session.source },
         ...campaignDimensions,
+        ...(outboundHost
+            ? [{ type: "outbound", value: outboundHost }]
+            : []),
         { type: "page", value: event.path },
         { type: "country", value: session.country ?? "unknown" },
         { type: "device", value: input.context.device ?? "unknown" },
@@ -450,10 +511,10 @@ async function accountEvent(ctx, input) {
         },
         ...matchedGoalKeys.map((value) => ({ type: "goal", value })),
     ];
-    foldAggregates(input.aggregateDeltas, input.siteId, event, session, dimensions, input.recordTelemetry, input.financialConversion);
+    foldAggregates(input.aggregateDeltas, input.siteId, event, session, dimensions, input.recordTelemetry, input.financialConversion, input.timezone);
     if (input.recordTelemetry &&
         event.timestamp >= input.now - LIVE_SESSION_TTL_MS) {
-        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now, input.batchState);
+        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now, input.batchState, reach);
     }
     return session;
 }
@@ -471,9 +532,16 @@ async function accountTrustedSession(ctx, existing, event) {
         isNewSession: false,
         isNewVisitorHour: false,
         isNewVisitorDay: false,
+        isNewVisitor: false,
+        durationDeltaMs: 0,
+        continued: false,
         startedAt: existing.startedAt,
         source: existing.source,
         utmCampaign: existing.utmCampaign,
+        utmMedium: existing.utmMedium,
+        entryPath: existing.entryPath,
+        conversionCount: existing.conversionCount + 1,
+        newVisitor: existing.newVisitor,
         visitorKey: existing.visitorKey ?? existing.visitorId,
         country: existing.country,
         city: existing.city,
@@ -555,9 +623,16 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / HOUR_MS),
             isNewVisitorDay: Math.floor(existing.lastSeenAt / DAY_MS) <
                 Math.floor(event.timestamp / DAY_MS),
+            isNewVisitor: false,
+            durationDeltaMs: Math.max(0, updates.durationMs - existing.durationMs),
+            continued: event.type === "pageview" && existing.pageviewCount === 1,
             startedAt,
             source: existing.source,
             utmCampaign: existing.utmCampaign,
+            utmMedium: existing.utmMedium,
+            entryPath: existing.entryPath,
+            conversionCount: updates.conversionCount,
+            newVisitor: existing.newVisitor,
             visitorKey,
             country: context.country ?? existing.country,
             city: context.city ?? existing.city,
@@ -584,6 +659,18 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             }
         }
     }
+    // A visitor is new when nothing was ever recorded under their id. A host
+    // id whose anonymous history was linked at sign-up is a return, not a new
+    // person; a browser without a persistent id is new every session.
+    let isNewVisitor = priorVisit === null;
+    if (isNewVisitor && event.visitorId !== event.sessionId) {
+        const alias = await ctx.db
+            .query("visitorAliases")
+            .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+            .first();
+        if (alias)
+            isNewVisitor = false;
+    }
     const fields = {
         siteId,
         sessionId: event.sessionId,
@@ -593,6 +680,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         lastSeenAt: event.timestamp,
         entryPath: event.path,
         exitPath: event.path,
+        newVisitor: isNewVisitor,
         ...(event.referrer ? { referrer: event.referrer } : {}),
         source,
         ...(utmSource ? { utmSource } : {}),
@@ -639,9 +727,16 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         isNewVisitorDay: priorVisit === null ||
             Math.floor(priorVisit.lastSeenAt / DAY_MS) <
                 Math.floor(event.timestamp / DAY_MS),
+        isNewVisitor,
+        durationDeltaMs: 0,
+        continued: false,
         startedAt: event.timestamp,
         source,
         utmCampaign,
+        utmMedium,
+        entryPath: event.path,
+        conversionCount: Number(financialConversion),
+        newVisitor: isNewVisitor,
         visitorKey,
         country: context.country,
         city: context.city,
@@ -788,10 +883,14 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
     else {
         await ctx.db.patch("sessions", session._id, updates);
     }
+    const durationDeltaMs = Math.max(0, updates.durationMs - session.durationMs);
     await updateLiveSession(ctx, siteId, event, context, {
         isNewSession: false,
         isNewVisitorHour: false,
         isNewVisitorDay: false,
+        isNewVisitor: false,
+        durationDeltaMs: 0,
+        continued: false,
         startedAt,
         source: session.source,
         visitorKey: session.visitorKey ?? session.visitorId,
@@ -802,7 +901,11 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         affiliateSlug: session.affiliateSlug,
+        entryPath: session.entryPath,
+        conversionCount: session.conversionCount,
+        newVisitor: session.newVisitor,
     }, now, batchState);
+    return durationDeltaMs;
 }
 async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -919,7 +1022,26 @@ async function flushSessionRateLimits(ctx, states, now) {
         }
     }
 }
-async function updateLiveSession(ctx, siteId, event, context, session, now, batchState) {
+/**
+ * The intent facts a live row carries for the dashboard's heat score. A
+ * session row from before 0.1.0 has no first-visit flag, so `returning` stays
+ * unknown rather than guessed.
+ */
+function liveIntentFields(session, funnel) {
+    return {
+        ...(session.entryPath ? { entryPath: session.entryPath } : {}),
+        ...(session.newVisitor !== undefined
+            ? { returning: !session.newVisitor }
+            : {}),
+        ...(session.utmCampaign ? { utmCampaign: session.utmCampaign } : {}),
+        ...(session.affiliateSlug ? { affiliateSlug: session.affiliateSlug } : {}),
+        ...(session.conversionCount !== undefined
+            ? { conversionCount: session.conversionCount }
+            : {}),
+        ...(funnel ? { funnel } : {}),
+    };
+}
+async function updateLiveSession(ctx, siteId, event, context, session, now, batchState, reach) {
     await ensureLiveSweep(ctx, siteId, now, batchState);
     const liveState = batchState
         ? await loadLiveSession(ctx, siteId, event.sessionId, batchState)
@@ -960,6 +1082,7 @@ async function updateLiveSession(ctx, siteId, event, context, session, now, batc
             device: context.device ?? existing.device,
             pageviewCount: session.pageviewCount,
             eventCount: session.eventCount,
+            ...liveIntentFields(session, furtherReach(existing.funnel, reach)),
             // The live map animates a visitor from the route they left to the one
             // they reached; a heartbeat or click on the same page is not a move.
             ...(event.type === "pageview" && event.path !== existing.path
@@ -999,6 +1122,7 @@ async function updateLiveSession(ctx, siteId, event, context, session, now, batc
         device: context.device ?? "unknown",
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
+        ...liveIntentFields(session, reach),
         ...(event.type === "heartbeat"
             ? {
                 lastHeartbeatEventId: event.eventId,
@@ -1086,9 +1210,9 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
                 _id: progressId,
             };
         }
-        await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelStepDelta(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
-        return;
+        return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
     }
     if (progress.completedAt !== undefined ||
         event.timestamp < progress.lastStepAt)
@@ -1117,9 +1241,9 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
                 completedAt: undefined,
             });
         }
-        await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelStepDelta(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
-        return;
+        return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
     }
     const nextStep = funnel.steps[progress.currentStep];
     if (!nextStep || !stepMatches(nextStep, event))
@@ -1139,44 +1263,19 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
     else {
         await ctx.db.patch("funnelProgress", progress._id, updates);
     }
-    await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
+    foldFunnelStepDelta(reportRollupDeltas, siteId, funnel._id, event, progress.currentStep, completed);
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, progress.currentStep, completed);
+    return {
+        funnelId: funnel._id,
+        step: currentStep,
+        steps: funnel.steps.length,
+    };
 }
 function stepMatches(step, event) {
     return step.type === "pageview"
         ? event.type === "pageview" && event.path === step.value
         : event.type === "custom" && event.name === step.value;
 }
-async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed) {
-    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-    const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
-    const bucket = await ctx.db
-        .query("funnelStepBuckets")
-        .withIndex("by_siteId_and_funnelId_and_bucketStart_and_step_and_shard", (range) => range
-        .eq("siteId", siteId)
-        .eq("funnelId", funnelId)
-        .eq("bucketStart", bucketStart)
-        .eq("step", step)
-        .eq("shard", shard))
-        .unique();
-    if (bucket) {
-        await ctx.db.patch("funnelStepBuckets", bucket._id, {
-            entrants: bucket.entrants + 1,
-            completions: bucket.completions + (completed ? 1 : 0),
-        });
-    }
-    else {
-        await ctx.db.insert("funnelStepBuckets", {
-            siteId,
-            funnelId,
-            bucketStart,
-            step,
-            shard,
-            entrants: 1,
-            completions: completed ? 1 : 0,
-        });
-    }
-}
 async function accountConversion(ctx, siteId, session, event, trustedGoalKey, reportRollupDeltas) {
     const revenueCents = event.revenueCents ?? 0;
     const currency = event.currency ?? "USD";
@@ -1238,24 +1337,6 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey, re
         }
     }
 }
-function createReportRollupDeltas() {
-    return {
-        goals: new Map(),
-        funnels: new Map(),
-        affiliates: new Map(),
-        vitals: new Map(),
-        routes: new Map(),
-    };
-}
-function reportRollupCoordinates(siteId, definitionId, timestamp, visitorId) {
-    const bucketStart = Math.floor(timestamp / DAY_MS) * DAY_MS;
-    const shard = stableHash(visitorId) % REPORT_ROLLUP_SHARDS;
-    return {
-        bucketStart,
-        shard,
-        key: `${siteId}:${definitionId}:${bucketStart}:${shard}`,
-    };
-}
 function foldGoalRollup(deltas, siteId, event, goal) {
     const coordinates = reportRollupCoordinates(siteId, String(goal._id), event.timestamp, event.visitorId);
     const current = deltas.goals.get(coordinates.key) ?? {
@@ -1290,6 +1371,23 @@ function foldFunnelRollup(deltas, siteId, funnelId, event, step, completed) {
     rollup.steps.set(step, current);
     deltas.funnels.set(coordinates.key, rollup);
 }
+function foldFunnelStepDelta(deltas, siteId, funnelId, event, step, completed) {
+    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
+    const key = `${siteId}:${funnelId}:${bucketStart}:${step}:${shard}`;
+    const current = deltas.funnelSteps.get(key) ?? {
+        siteId,
+        funnelId,
+        bucketStart,
+        step,
+        shard,
+        entrants: 0,
+        completions: 0,
+    };
+    current.entrants += 1;
+    current.completions += Number(completed);
+    deltas.funnelSteps.set(key, current);
+}
 function foldAffiliateVisitRollup(deltas, siteId, event, attribution, visitorKey) {
     const coordinates = reportRollupCoordinates(siteId, String(attribution.affiliateId), event.timestamp, event.visitorId);
     const current = deltas.affiliates.get(coordinates.key) ?? {
@@ -1299,13 +1397,13 @@ function foldAffiliateVisitRollup(deltas, siteId, event, attribution, visitorKey
         shard: coordinates.shard,
         affiliateSlug: attribution.affiliateSlug,
         visits: 0,
-        visitorSketch: createVisitorSketch(),
+        visitorKeys: new Set(),
         conversions: 0,
         revenueCents: 0,
         commissionCents: 0,
     };
     current.visits += 1;
-    current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
+    current.visitorKeys.add(visitorKey);
     deltas.affiliates.set(coordinates.key, current);
 }
 function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affiliateSlug, revenueCents, commissionCents) {
@@ -1317,7 +1415,7 @@ function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affilia
         shard: coordinates.shard,
         affiliateSlug,
         visits: 0,
-        visitorSketch: createVisitorSketch(),
+        visitorKeys: new Set(),
         conversions: 0,
         revenueCents: 0,
         commissionCents: 0,
@@ -1377,7 +1475,7 @@ function foldRouteDelta(deltas, siteId, route, timestamp, visitorId, apply) {
         continued: 0,
         dwellMs: 0,
         dwellSamples: 0,
-        visitorSketch: createVisitorSketch(),
+        visitorKeys: new Set(),
         outgoing: new Map(),
     };
     apply(current);
@@ -1398,7 +1496,7 @@ function foldRouteRollups(deltas, siteId, event, session, existing) {
         delta.pageviews += 1;
         if (firstPageview)
             delta.entries += 1;
-        delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+        delta.visitorKeys.add(session.visitorKey);
     });
     // Out-of-order telemetry keeps its view but cannot say where it came from.
     if (existing === null || firstPageview)
@@ -1420,245 +1518,19 @@ function foldRouteRollups(deltas, siteId, event, session, existing) {
         });
     }
 }
-async function upsertRouteRollup(ctx, delta, dayRowCounts) {
-    const existing = await ctx.db
-        .query("routeDailyRollups")
-        .withIndex("by_siteId_and_route_and_bucketStart_and_shard", (range) => range
-        .eq("siteId", delta.siteId)
-        .eq("route", delta.route)
-        .eq("bucketStart", delta.bucketStart)
-        .eq("shard", delta.shard))
-        .unique();
-    if (!existing && delta.route !== SITE_MAP_OTHER_ROUTE) {
-        // A new route row spends one bounded read per batch to respect the daily
-        // cap; past it, the day's remaining routes fold into "(other)".
-        const capRows = MAX_SITE_MAP_ROUTES_PER_DAY * REPORT_ROLLUP_SHARDS;
-        const dayKey = `${delta.siteId}:${delta.bucketStart}`;
-        let dayRows = dayRowCounts.get(dayKey);
-        if (dayRows === undefined) {
-            dayRows = (await ctx.db
-                .query("routeDailyRollups")
-                .withIndex("by_siteId_and_bucketStart", (range) => range
-                .eq("siteId", delta.siteId)
-                .eq("bucketStart", delta.bucketStart))
-                .take(capRows + 1)).length;
-        }
-        if (dayRows >= capRows) {
-            await upsertRouteRollup(ctx, { ...delta, route: SITE_MAP_OTHER_ROUTE }, dayRowCounts);
-            return;
-        }
-        dayRowCounts.set(dayKey, dayRows + 1);
-    }
-    const outgoing = existing?.outgoing.map((entry) => ({ ...entry })) ?? [];
-    for (const [target, count] of delta.outgoing) {
-        let entry = outgoing.find((candidate) => candidate.route === target);
-        if (!entry) {
-            const named = outgoing.filter((candidate) => candidate.route !== SITE_MAP_OTHER_ROUTE).length;
-            const route = target !== SITE_MAP_OTHER_ROUTE && named >= ROUTE_TRANSITION_SLOTS
-                ? SITE_MAP_OTHER_ROUTE
-                : target;
-            entry = outgoing.find((candidate) => candidate.route === route);
-            if (!entry) {
-                entry = { route, count: 0 };
-                outgoing.push(entry);
-            }
-        }
-        entry.count += count;
-    }
-    const fields = {
-        siteId: delta.siteId,
-        bucketStart: delta.bucketStart,
-        shard: delta.shard,
-        generation: 1,
-        route: delta.route,
-        pageviews: (existing?.pageviews ?? 0) + delta.pageviews,
-        entries: (existing?.entries ?? 0) + delta.entries,
-        continued: (existing?.continued ?? 0) + delta.continued,
-        dwellMs: (existing?.dwellMs ?? 0) + delta.dwellMs,
-        dwellSamples: (existing?.dwellSamples ?? 0) + delta.dwellSamples,
-        visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
-        outgoing,
-    };
-    if (existing) {
-        await ctx.db.replace("routeDailyRollups", existing._id, fields);
-    }
-    else {
-        await ctx.db.insert("routeDailyRollups", fields);
-    }
-}
-async function upsertVitalRollup(ctx, delta) {
-    const existing = await ctx.db
-        .query("vitalRollups")
-        .withIndex("by_key", (range) => range
-        .eq("siteId", delta.siteId)
-        .eq("bucketStart", delta.bucketStart)
-        .eq("device", delta.device)
-        .eq("page", delta.page)
-        .eq("shard", delta.shard))
-        .unique();
-    if (!existing &&
-        delta.device === VITAL_ALL &&
-        delta.page !== VITAL_OTHER_PAGES) {
-        // A new page row spends one bounded read to respect the daily page slots;
-        // past the cap, the day's remaining pages fold into "(other)".
-        const pageRows = await ctx.db
-            .query("vitalRollups")
-            .withIndex("by_key", (range) => range
-            .eq("siteId", delta.siteId)
-            .eq("bucketStart", delta.bucketStart)
-            .eq("device", VITAL_ALL))
-            .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
-        if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
-            await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
-            return;
-        }
-    }
-    const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
-    for (const [metric, addition] of delta.metrics) {
-        const entry = metrics.find((candidate) => candidate.metric === metric);
-        if (entry) {
-            entry.count += addition.count;
-            entry.sum += addition.sum;
-            entry.histogram = mergeVitalHistograms(entry.histogram, addition.histogram);
-        }
-        else {
-            metrics.push({
-                metric,
-                count: addition.count,
-                sum: addition.sum,
-                histogram: [...addition.histogram],
-            });
-        }
-    }
-    const fields = {
-        siteId: delta.siteId,
-        bucketStart: delta.bucketStart,
-        device: delta.device,
-        page: delta.page,
-        shard: delta.shard,
-        metrics,
-    };
-    if (existing) {
-        await ctx.db.replace("vitalRollups", existing._id, fields);
-    }
-    else {
-        await ctx.db.insert("vitalRollups", fields);
-    }
-}
-async function flushReportRollups(ctx, deltas) {
-    for (const delta of deltas.goals.values()) {
-        const existing = await ctx.db
-            .query("goalDailyRollups")
-            .withIndex("by_siteId_and_goalId_and_bucketStart_and_shard", (range) => range
-            .eq("siteId", delta.siteId)
-            .eq("goalId", delta.goalId)
-            .eq("bucketStart", delta.bucketStart)
-            .eq("shard", delta.shard))
-            .unique();
-        const fields = {
-            siteId: delta.siteId,
-            goalId: delta.goalId,
-            bucketStart: delta.bucketStart,
-            shard: delta.shard,
-            generation: 1,
-            goalKey: delta.goalKey,
-            completions: (existing?.completions ?? 0) + delta.completions,
-            valueCents: (existing?.valueCents ?? 0) + delta.valueCents,
-        };
-        if (existing) {
-            await ctx.db.replace("goalDailyRollups", existing._id, fields);
-        }
-        else {
-            await ctx.db.insert("goalDailyRollups", fields);
-        }
-    }
-    for (const delta of deltas.funnels.values()) {
-        const existing = await ctx.db
-            .query("funnelDailyRollups")
-            .withIndex("by_siteId_and_funnelId_and_bucketStart_and_shard", (range) => range
-            .eq("siteId", delta.siteId)
-            .eq("funnelId", delta.funnelId)
-            .eq("bucketStart", delta.bucketStart)
-            .eq("shard", delta.shard))
-            .unique();
-        const steps = existing?.steps.map((step) => ({ ...step })) ?? [];
-        for (const addition of delta.steps.values()) {
-            const current = steps.find((step) => step.step === addition.step);
-            if (current) {
-                current.entrants += addition.entrants;
-                current.completions += addition.completions;
-            }
-            else {
-                if (steps.length >= MAX_FUNNEL_STEPS) {
-                    fail("LIMIT_EXCEEDED", "daily funnel rollup step limit exceeded");
-                }
-                steps.push(addition);
-            }
-        }
-        const fields = {
-            siteId: delta.siteId,
-            funnelId: delta.funnelId,
-            bucketStart: delta.bucketStart,
-            shard: delta.shard,
-            generation: 1,
-            steps,
-        };
-        if (existing) {
-            await ctx.db.replace("funnelDailyRollups", existing._id, fields);
-        }
-        else {
-            await ctx.db.insert("funnelDailyRollups", fields);
-        }
-    }
-    for (const delta of deltas.affiliates.values()) {
-        const existing = await ctx.db
-            .query("affiliateDailyRollups")
-            .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
-            .eq("siteId", delta.siteId)
-            .eq("affiliateId", delta.affiliateId)
-            .eq("bucketStart", delta.bucketStart)
-            .eq("shard", delta.shard))
-            .unique();
-        const fields = {
-            siteId: delta.siteId,
-            affiliateId: delta.affiliateId,
-            bucketStart: delta.bucketStart,
-            shard: delta.shard,
-            generation: 1,
-            affiliateSlug: delta.affiliateSlug,
-            visits: (existing?.visits ?? 0) + delta.visits,
-            visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
-            conversions: (existing?.conversions ?? 0) + delta.conversions,
-            revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
-            commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
-        };
-        if (existing) {
-            await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
-        }
-        else {
-            await ctx.db.insert("affiliateDailyRollups", fields);
-        }
-    }
-    const routeDayRowCounts = new Map();
-    for (const delta of deltas.routes.values()) {
-        await upsertRouteRollup(ctx, delta, routeDayRowCounts);
-    }
-    for (const delta of deltas.vitals.values()) {
-        await upsertVitalRollup(ctx, delta);
-    }
-}
-function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
+function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion, timezone) {
     const revenueCents = financialConversion ? (event.revenueCents ?? 0) : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
         ...dimension,
         count: Number(recordTelemetry),
         revenueCents,
+        pageviews: Number(recordTelemetry && event.type === "pageview"),
+        sessions: Number(recordTelemetry && session.isNewSession),
     }));
-    for (const granularity of ["hour", "day"]) {
+    for (const granularity of aggregateGranularities(timezone)) {
         const shard = stableHash(event.sessionId) %
             (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
-        const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
-        const bucketStart = Math.floor(event.timestamp / interval) * interval;
+        const bucketStart = aggregateBucketStart(granularity, event.timestamp, timezone);
         const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
         let delta = aggregateDeltas.get(key);
         if (!delta) {
@@ -1678,6 +1550,9 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 conversions: 0,
                 revenueCents: 0,
                 dimensions: [],
+                durationMs: 0,
+                continuedSessions: 0,
+                newVisitors: 0,
             };
             aggregateDeltas.set(key, delta);
         }
@@ -1699,6 +1574,58 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
         delta.conversions += Number(financialConversion);
         delta.revenueCents += revenueCents;
         delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
+        if (recordTelemetry) {
+            delta.durationMs += session.durationDeltaMs;
+            delta.continuedSessions += Number(session.continued);
+            delta.newVisitors += Number(session.isNewSession && session.isNewVisitor);
+        }
+    }
+}
+/**
+ * The buckets an event lands in: UTC hour and UTC day always, plus the
+ * calendar day in the site's timezone when the site keeps one.
+ */
+function aggregateGranularities(timezone) {
+    return timezone ? ["hour", "day", "localDay"] : ["hour", "day"];
+}
+function aggregateBucketStart(granularity, timestamp, timezone) {
+    if (granularity === "localDay" && timezone) {
+        return localDayStart(timestamp, timezone);
+    }
+    const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
+    return Math.floor(timestamp / interval) * interval;
+}
+/** Folds only session duration growth, for heartbeats. */
+function foldSessionDuration(aggregateDeltas, siteId, event, durationDeltaMs, timezone) {
+    if (durationDeltaMs <= 0)
+        return;
+    for (const granularity of aggregateGranularities(timezone)) {
+        const shard = stableHash(event.sessionId) %
+            (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
+        const bucketStart = aggregateBucketStart(granularity, event.timestamp, timezone);
+        const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
+        const delta = aggregateDeltas.get(key) ?? {
+            siteId,
+            granularity,
+            bucketStart,
+            shard,
+            events: 0,
+            pageviews: 0,
+            clicks: 0,
+            customEvents: 0,
+            outboundClicks: 0,
+            sessions: 0,
+            visitors: 0,
+            visitorSketch: createVisitorSketch(),
+            conversions: 0,
+            revenueCents: 0,
+            dimensions: [],
+            durationMs: 0,
+            continuedSessions: 0,
+            newVisitors: 0,
+        };
+        delta.durationMs += durationDeltaMs;
+        aggregateDeltas.set(key, delta);
     }
 }
 async function flushAggregates(ctx, aggregateDeltas) {
@@ -1711,7 +1638,11 @@ async function flushAggregates(ctx, aggregateDeltas) {
             .eq("bucketStart", delta.bucketStart)
             .eq("shard", delta.shard))
             .unique();
-        const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions);
+        const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions.map((slot) => ({
+            ...slot,
+            pageviews: slot.pageviews ?? 0,
+            sessions: slot.sessions ?? 0,
+        })));
         const priorSketch = bucket?.visitorSketch;
         const visitorSketch = priorSketch
             ? unionVisitorSketches(priorSketch, delta.visitorSketch)
@@ -1733,6 +1664,9 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 conversions: bucket.conversions + delta.conversions,
                 revenueCents: bucket.revenueCents + delta.revenueCents,
                 dimensions,
+                durationMs: (bucket.durationMs ?? 0) + delta.durationMs,
+                continuedSessions: (bucket.continuedSessions ?? 0) + delta.continuedSessions,
+                newVisitors: (bucket.newVisitors ?? 0) + delta.newVisitors,
             });
         }
         else {
@@ -1753,6 +1687,9 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 conversions: delta.conversions,
                 revenueCents: delta.revenueCents,
                 dimensions,
+                durationMs: delta.durationMs,
+                continuedSessions: delta.continuedSessions,
+                newVisitors: delta.newVisitors,
             });
         }
     }
@@ -1764,6 +1701,8 @@ function mergeDimensions(current, additions) {
         if (exact) {
             exact.count += addition.count;
             exact.revenueCents += addition.revenueCents;
+            exact.pageviews = (exact.pageviews ?? 0) + addition.pageviews;
+            exact.sessions = (exact.sessions ?? 0) + addition.sessions;
             continue;
         }
         const typeSlots = result.filter((candidate) => candidate.type === addition.type);
@@ -1780,6 +1719,8 @@ function mergeDimensions(current, additions) {
                 value: addition.value,
                 count: addition.count,
                 revenueCents: addition.revenueCents,
+                pageviews: addition.pageviews,
+                sessions: addition.sessions,
             });
             continue;
         }
@@ -1787,6 +1728,8 @@ function mergeDimensions(current, additions) {
         if (overflow) {
             overflow.count += addition.count;
             overflow.revenueCents += addition.revenueCents;
+            overflow.pageviews = (overflow.pageviews ?? 0) + addition.pageviews;
+            overflow.sessions = (overflow.sessions ?? 0) + addition.sessions;
             continue;
         }
         if (addition.count === 0 && addition.revenueCents === 0)
@@ -1802,6 +1745,8 @@ function mergeDimensions(current, additions) {
                 value: "(other)",
                 count: addition.count,
                 revenueCents: addition.revenueCents,
+                pageviews: addition.pageviews,
+                sessions: addition.sessions,
             });
             continue;
         }
@@ -1813,6 +1758,8 @@ function mergeDimensions(current, additions) {
         replacement.value = "(other)";
         replacement.count += addition.count;
         replacement.revenueCents += addition.revenueCents;
+        replacement.pageviews = (replacement.pageviews ?? 0) + addition.pageviews;
+        replacement.sessions = (replacement.sessions ?? 0) + addition.sessions;
     }
     return result;
 }