@iann29/rastro 0.1.0-alpha.8 → 0.1.0

package/src/component/ingest.ts

@@ -10,6 +10,7 @@ import {
   HOURLY_AGGREGATE_SHARDS,
   DIMENSION_SLOTS,
   HOUR_MS,
+  LIVE_LEAVE_GRACE_MS,
   LIVE_SESSION_TTL_MS,
   LIVE_SWEEP_INTERVAL_MS,
   MAX_BATCH_EVENTS,
@@ -18,16 +19,11 @@ import {
   MAX_BYTES_PER_SESSION_WINDOW,
   MAX_EVENTS_PER_SESSION_WINDOW,
   MAX_FUNNELS_PER_SITE,
-  MAX_FUNNEL_STEPS,
   MAX_GOALS_PER_SITE,
-  MAX_SITE_MAP_ROUTES_PER_DAY,
   RATE_LIMIT_WINDOW_MS,
   REPORT_ROLLUP_SHARDS,
-  ROUTE_TRANSITION_SLOTS,
   SECONDARY_AGGREGATE_SHARDS,
   SITE_INGEST_SHARDS,
-  SITE_MAP_OTHER_ROUTE,
-  VITAL_PAGE_SLOTS,
 } from "./constants.js";
 import {
   addVisitor,
@@ -38,10 +34,8 @@ import {
 import {
   addToVitalHistogram,
   emptyVitalHistogram,
-  mergeVitalHistograms,
   vitalDevice,
   VITAL_ALL,
-  VITAL_OTHER_PAGES,
   type VitalMetric,
 } from "./vitals.js";
 import { fail } from "./errors.js";
@@ -62,6 +56,7 @@ import {
   measureBatchBytes,
   normalizeRoute,
   originAllowed,
+  publicUrlHost,
   sanitizeContext,
   sanitizeCurrency,
   sanitizeEvent,
@@ -80,6 +75,14 @@ import {
   type IngestContext,
   type TrackerEvent,
 } from "./validators.js";
+import {
+  createReportRollupDeltas,
+  enqueueReportRollups,
+  reportRollupCoordinates,
+  type ReportRollupDeltas,
+  type RouteRollupDelta,
+} from "./rollupStore.js";
+import { localDayStart, localDayTimezone } from "./localTime.js";
 
 type DimensionType =
   | "source"
@@ -89,13 +92,20 @@ type DimensionType =
   | "device"
   | "browser"
   | "event"
-  | "goal";
+  | "goal"
+  | "medium"
+  | "outbound";
 
 type Dimension = { type: DimensionType; value: string };
-type DimensionDelta = Dimension & { count: number; revenueCents: number };
+type DimensionDelta = Dimension & {
+  count: number;
+  revenueCents: number;
+  pageviews: number;
+  sessions: number;
+};
 type AggregateDelta = {
   siteId: Id<"sites">;
-  granularity: "hour" | "day";
+  granularity: "hour" | "day" | "localDay";
   bucketStart: number;
   shard: number;
   events: number;
@@ -109,74 +119,11 @@ type AggregateDelta = {
   conversions: number;
   revenueCents: number;
   dimensions: Doc<"aggregateBuckets">["dimensions"];
+  durationMs: number;
+  continuedSessions: number;
+  newVisitors: number;
 };
 type AggregateDeltas = Map<string, AggregateDelta>;
-type GoalRollupDelta = {
-  siteId: Id<"sites">;
-  goalId: Id<"goals">;
-  bucketStart: number;
-  shard: number;
-  goalKey: string;
-  completions: number;
-  valueCents: number;
-};
-type FunnelRollupDelta = {
-  siteId: Id<"sites">;
-  funnelId: Id<"funnels">;
-  bucketStart: number;
-  shard: number;
-  steps: Map<
-    number,
-    {
-      step: number;
-      entrants: number;
-      completions: number;
-    }
-  >;
-};
-type AffiliateRollupDelta = {
-  siteId: Id<"sites">;
-  affiliateId: Id<"affiliates">;
-  bucketStart: number;
-  shard: number;
-  affiliateSlug: string;
-  visits: number;
-  visitorSketch: VisitorSketch;
-  conversions: number;
-  revenueCents: number;
-  commissionCents: number;
-};
-type VitalRollupDelta = {
-  siteId: Id<"sites">;
-  bucketStart: number;
-  device: string;
-  page: string;
-  shard: number;
-  metrics: Map<
-    VitalMetric,
-    { count: number; sum: number; histogram: number[] }
-  >;
-};
-type RouteRollupDelta = {
-  siteId: Id<"sites">;
-  bucketStart: number;
-  shard: number;
-  route: string;
-  pageviews: number;
-  entries: number;
-  continued: number;
-  dwellMs: number;
-  dwellSamples: number;
-  visitorSketch: VisitorSketch;
-  outgoing: Map<string, number>;
-};
-type ReportRollupDeltas = {
-  goals: Map<string, GoalRollupDelta>;
-  funnels: Map<string, FunnelRollupDelta>;
-  affiliates: Map<string, AffiliateRollupDelta>;
-  vitals: Map<string, VitalRollupDelta>;
-  routes: Map<string, RouteRollupDelta>;
-};
 type AttributionSnapshot = {
   affiliateId: Id<"affiliates">;
   affiliateSlug: string;
@@ -188,10 +135,17 @@ type SessionState = {
   isNewSession: boolean;
   isNewVisitorHour: boolean;
   isNewVisitorDay: boolean;
+  /** The session is new and its visitor has no earlier session or alias. */
+  isNewVisitor: boolean;
+  /** How much this event grew the session's duration. */
+  durationDeltaMs: number;
+  /** This event is the session's second pageview: it did not bounce. */
+  continued: boolean;
   startedAt: number;
   source: string;
   visitorKey: string;
   utmCampaign?: string;
+  utmMedium?: string;
   country?: string;
   city?: string;
   latitude?: number;
@@ -202,8 +156,28 @@ type SessionState = {
   affiliateSlug?: string;
   affiliateAttributedAt?: number;
   affiliateExpiresAt?: number;
+  /** The session's landing path, for the live row. */
+  entryPath?: string;
+  /** Conversions the session holds after this event, for the live row. */
+  conversionCount?: number;
+  /** The session's stored first-visit flag; unknown for older rows. */
+  newVisitor?: boolean;
 };
 
+/** The furthest step of a funnel an event took its visitor to. */
+type FunnelReach = { funnelId: Id<"funnels">; step: number; steps: number };
+
+function furtherReach(
+  current: FunnelReach | undefined,
+  next: FunnelReach | undefined,
+): FunnelReach | undefined {
+  if (!next) return current;
+  if (!current) return next;
+  return next.step / next.steps >= current.step / current.steps
+    ? next
+    : current;
+}
+
 type SessionRateWindowState = {
   siteId: Id<"sites">;
   sessionId: string;
@@ -279,6 +253,7 @@ export const ingestBatch = mutation({
       now,
     );
     const context = sanitizeContext(args.context);
+    const timezone = localDayTimezone(site.timezone);
     const aggregateDeltas: AggregateDeltas = new Map();
     const reportRollupDeltas = createReportRollupDeltas();
     const sessionRateWindows = new Map<string, SessionRateWindowState>();
@@ -315,7 +290,12 @@ export const ingestBatch = mutation({
       const bucketStart = eventBucketStart(event.timestamp);
       const groupKey = eventBatchKey(event.sessionId, bucketStart);
       requestGroups.add(groupKey);
-      if (event.type === "heartbeat" || event.type === "vital") continue;
+      if (
+        event.type === "heartbeat" ||
+        event.type === "vital" ||
+        event.type === "leave"
+      )
+        continue;
       const eventIds = eventIdsByGroup.get(groupKey) ?? new Set<string>();
       if (eventIds.has(event.eventId)) {
         duplicateInBatch.add(eventIndex);
@@ -409,7 +389,7 @@ export const ingestBatch = mutation({
           rejected += 1;
           continue;
         }
-        await accountHeartbeat(
+        const durationDeltaMs = await accountHeartbeat(
           ctx,
           args.siteId,
           existingSession,
@@ -418,6 +398,60 @@ export const ingestBatch = mutation({
           now,
           batchState,
         );
+        // A heartbeat is presence, not telemetry, but it is what keeps a
+        // session's duration honest while the visitor reads: the growth is
+        // folded into the buckets the heartbeat landed in.
+        foldSessionDuration(
+          aggregateDeltas,
+          args.siteId,
+          event,
+          durationDeltaMs,
+          timezone,
+        );
+        accepted += 1;
+        continue;
+      }
+
+      if (event.type === "leave") {
+        // A departure only shortens live presence: it stores no telemetry,
+        // touches no aggregates, and never extends anything, so a replayed
+        // or late leave is inert. The read side filters by expiresAt and the
+        // sweep deletes the row, so the dashboard sees the exit in seconds.
+        if (!existingSession) {
+          rejected += 1;
+          continue;
+        }
+        if (
+          !(await consumeRateLimit(
+            ctx,
+            args.siteId,
+            event.sessionId,
+            eventByteCounts[eventIndex],
+            now,
+            sessionRateWindows,
+          ))
+        ) {
+          rejected += 1;
+          continue;
+        }
+        const liveState = await loadLiveSession(
+          ctx,
+          args.siteId,
+          event.sessionId,
+          batchState,
+        );
+        const liveSession = liveState.current;
+        if (liveSession) {
+          const expiresAt = Math.min(
+            liveSession.expiresAt,
+            now + LIVE_LEAVE_GRACE_MS,
+          );
+          if (expiresAt < liveSession.expiresAt) {
+            await ctx.db.patch("liveSessions", liveSession._id, { expiresAt });
+            liveState.current = { ...liveSession, expiresAt };
+          }
+          await ensureLiveSweep(ctx, args.siteId, now, batchState);
+        }
         accepted += 1;
         continue;
       }
@@ -501,6 +535,7 @@ export const ingestBatch = mutation({
         aggregateDeltas,
         reportRollupDeltas,
         recordTelemetry: true,
+        timezone,
         financialConversion: false,
         trustedGoalKey: undefined,
         now,
@@ -517,7 +552,7 @@ export const ingestBatch = mutation({
     await flushBatchState(ctx, batchState);
     await flushSessionRateLimits(ctx, sessionRateWindows, now);
     await flushAggregates(ctx, aggregateDeltas);
-    await flushReportRollups(ctx, reportRollupDeltas);
+    await enqueueReportRollups(ctx, args.siteId, reportRollupDeltas, now);
     return { accepted, duplicates, rejected };
   },
 });
@@ -630,6 +665,7 @@ export const trackConversion = mutation({
       aggregateDeltas,
       reportRollupDeltas,
       recordTelemetry: existingTelemetry === null && !telemetry.saturated,
+      timezone: localDayTimezone(site.timezone),
       financialConversion: true,
       trustedGoalKey,
       now: Date.now(),
@@ -643,8 +679,9 @@ export const trackConversion = mutation({
       );
     }
     await flushAggregates(ctx, aggregateDeltas);
-    await flushReportRollups(ctx, reportRollupDeltas);
-    await ensureAnalyticsControl(ctx, args.siteId, Date.now());
+    const now = Date.now();
+    await enqueueReportRollups(ctx, args.siteId, reportRollupDeltas, now);
+    await ensureAnalyticsControl(ctx, args.siteId, now);
     return { accepted: 1, duplicates: 0, rejected: 0 };
   },
 });
@@ -765,6 +802,8 @@ async function accountEvent(
     aggregateDeltas: AggregateDeltas;
     reportRollupDeltas: ReportRollupDeltas;
     recordTelemetry: boolean;
+    /** The site's timezone when it keeps local-day buckets. */
+    timezone?: string;
     financialConversion: boolean;
     trustedGoalKey: string | undefined;
     now: number;
@@ -855,15 +894,19 @@ async function accountEvent(
     matchedGoalKeys.push(goal.key);
   }
 
+  let reach: FunnelReach | undefined;
   for (const funnel of input.funnels) {
     if (funnel.active) {
-      await advanceFunnel(
-        ctx,
-        input.siteId,
-        funnel,
-        event,
-        input.reportRollupDeltas,
-        input.batchState,
+      reach = furtherReach(
+        reach,
+        await advanceFunnel(
+          ctx,
+          input.siteId,
+          funnel,
+          event,
+          input.reportRollupDeltas,
+          input.batchState,
+        ),
       );
     }
   }
@@ -879,12 +922,26 @@ async function accountEvent(
     );
   }
 
-  const campaignDimensions: Dimension[] = session.utmCampaign
-    ? [{ type: "campaign", value: session.utmCampaign }]
-    : [];
+  const campaignDimensions: Dimension[] = [];
+  if (session.utmCampaign) {
+    campaignDimensions.push({ type: "campaign", value: session.utmCampaign });
+  }
+  if (session.utmMedium) {
+    // A medium is a taxonomy ("social", "cpc", "email"), so its case is noise.
+    campaignDimensions.push({
+      type: "medium",
+      value: session.utmMedium.toLowerCase(),
+    });
+  }
+  // An outbound click ranks by where it went: the host alone, never the path.
+  const outboundHost =
+    event.type === "outbound" ? publicUrlHost(event.href) : undefined;
   const dimensions: Dimension[] = [
     { type: "source", value: session.source },
     ...campaignDimensions,
+    ...(outboundHost
+      ? [{ type: "outbound", value: outboundHost } satisfies Dimension]
+      : []),
     { type: "page", value: event.path },
     { type: "country", value: session.country ?? "unknown" },
     { type: "device", value: input.context.device ?? "unknown" },
@@ -903,6 +960,7 @@ async function accountEvent(
     dimensions,
     input.recordTelemetry,
     input.financialConversion,
+    input.timezone,
   );
 
   if (
@@ -917,6 +975,7 @@ async function accountEvent(
       session,
       input.now,
       input.batchState,
+      reach,
     );
   }
   return session;
@@ -940,9 +999,16 @@ async function accountTrustedSession(
     isNewSession: false,
     isNewVisitorHour: false,
     isNewVisitorDay: false,
+    isNewVisitor: false,
+    durationDeltaMs: 0,
+    continued: false,
     startedAt: existing.startedAt,
     source: existing.source,
     utmCampaign: existing.utmCampaign,
+    utmMedium: existing.utmMedium,
+    entryPath: existing.entryPath,
+    conversionCount: existing.conversionCount + 1,
+    newVisitor: existing.newVisitor,
     visitorKey: existing.visitorKey ?? existing.visitorId,
     country: existing.country,
     city: existing.city,
@@ -1054,9 +1120,16 @@ async function upsertSession(
       isNewVisitorDay:
         Math.floor(existing.lastSeenAt / DAY_MS) <
         Math.floor(event.timestamp / DAY_MS),
+      isNewVisitor: false,
+      durationDeltaMs: Math.max(0, updates.durationMs - existing.durationMs),
+      continued: event.type === "pageview" && existing.pageviewCount === 1,
       startedAt,
       source: existing.source,
       utmCampaign: existing.utmCampaign,
+      utmMedium: existing.utmMedium,
+      entryPath: existing.entryPath,
+      conversionCount: updates.conversionCount,
+      newVisitor: existing.newVisitor,
       visitorKey,
       country: context.country ?? existing.country,
       city: context.city ?? existing.city,
@@ -1089,6 +1162,19 @@ async function upsertSession(
       }
     }
   }
+  // A visitor is new when nothing was ever recorded under their id. A host
+  // id whose anonymous history was linked at sign-up is a return, not a new
+  // person; a browser without a persistent id is new every session.
+  let isNewVisitor = priorVisit === null;
+  if (isNewVisitor && event.visitorId !== event.sessionId) {
+    const alias = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", siteId).eq("visitorId", event.visitorId),
+      )
+      .first();
+    if (alias) isNewVisitor = false;
+  }
   const fields: Omit<Doc<"sessions">, "_id" | "_creationTime"> = {
     siteId,
     sessionId: event.sessionId,
@@ -1098,6 +1184,7 @@ async function upsertSession(
     lastSeenAt: event.timestamp,
     entryPath: event.path,
     exitPath: event.path,
+    newVisitor: isNewVisitor,
     ...(event.referrer ? { referrer: event.referrer } : {}),
     source,
     ...(utmSource ? { utmSource } : {}),
@@ -1151,9 +1238,16 @@ async function upsertSession(
       priorVisit === null ||
       Math.floor(priorVisit.lastSeenAt / DAY_MS) <
         Math.floor(event.timestamp / DAY_MS),
+    isNewVisitor,
+    durationDeltaMs: 0,
+    continued: false,
     startedAt: event.timestamp,
     source,
     utmCampaign,
+    utmMedium,
+    entryPath: event.path,
+    conversionCount: Number(financialConversion),
+    newVisitor: isNewVisitor,
     visitorKey,
     country: context.country,
     city: context.city,
@@ -1311,7 +1405,7 @@ async function accountHeartbeat(
   context: IngestContext,
   now: number,
   batchState?: BatchState,
-) {
+): Promise<number> {
   const startedAt = Math.min(session.startedAt, event.timestamp);
   const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
   const updates = {
@@ -1338,6 +1432,7 @@ async function accountHeartbeat(
   } else {
     await ctx.db.patch("sessions", session._id, updates);
   }
+  const durationDeltaMs = Math.max(0, updates.durationMs - session.durationMs);
   await updateLiveSession(
     ctx,
     siteId,
@@ -1347,6 +1442,9 @@ async function accountHeartbeat(
       isNewSession: false,
       isNewVisitorHour: false,
       isNewVisitorDay: false,
+      isNewVisitor: false,
+      durationDeltaMs: 0,
+      continued: false,
       startedAt,
       source: session.source,
       visitorKey: session.visitorKey ?? session.visitorId,
@@ -1357,10 +1455,14 @@ async function accountHeartbeat(
       pageviewCount: session.pageviewCount,
       eventCount: session.eventCount,
       affiliateSlug: session.affiliateSlug,
+      entryPath: session.entryPath,
+      conversionCount: session.conversionCount,
+      newVisitor: session.newVisitor,
     },
     now,
     batchState,
   );
+  return durationDeltaMs;
 }
 
 async function consumeSiteRateLimit(
@@ -1518,6 +1620,26 @@ async function flushSessionRateLimits(
   }
 }
 
+/**
+ * The intent facts a live row carries for the dashboard's heat score. A
+ * session row from before 0.1.0 has no first-visit flag, so `returning` stays
+ * unknown rather than guessed.
+ */
+function liveIntentFields(session: SessionState, funnel?: FunnelReach) {
+  return {
+    ...(session.entryPath ? { entryPath: session.entryPath } : {}),
+    ...(session.newVisitor !== undefined
+      ? { returning: !session.newVisitor }
+      : {}),
+    ...(session.utmCampaign ? { utmCampaign: session.utmCampaign } : {}),
+    ...(session.affiliateSlug ? { affiliateSlug: session.affiliateSlug } : {}),
+    ...(session.conversionCount !== undefined
+      ? { conversionCount: session.conversionCount }
+      : {}),
+    ...(funnel ? { funnel } : {}),
+  };
+}
+
 async function updateLiveSession(
   ctx: MutationCtx,
   siteId: Id<"sites">,
@@ -1526,6 +1648,7 @@ async function updateLiveSession(
   session: SessionState,
   now: number,
   batchState?: BatchState,
+  reach?: FunnelReach,
 ) {
   await ensureLiveSweep(ctx, siteId, now, batchState);
   const liveState = batchState
@@ -1570,6 +1693,7 @@ async function updateLiveSession(
       device: context.device ?? existing.device,
       pageviewCount: session.pageviewCount,
       eventCount: session.eventCount,
+      ...liveIntentFields(session, furtherReach(existing.funnel, reach)),
       // The live map animates a visitor from the route they left to the one
       // they reached; a heartbeat or click on the same page is not a move.
       ...(event.type === "pageview" && event.path !== existing.path
@@ -1609,6 +1733,7 @@ async function updateLiveSession(
     device: context.device ?? "unknown",
     pageviewCount: session.pageviewCount,
     eventCount: session.eventCount,
+    ...liveIntentFields(session, reach),
     ...(event.type === "heartbeat"
       ? {
           lastHeartbeatEventId: event.eventId,
@@ -1666,7 +1791,7 @@ async function advanceFunnel(
   event: TrackerEvent,
   reportRollupDeltas: ReportRollupDeltas,
   batchState?: BatchState,
-) {
+): Promise<FunnelReach | undefined> {
   const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
   let progressState = batchState?.funnelProgress.get(cacheKey);
   if (batchState && !progressState) {
@@ -1712,9 +1837,16 @@ async function advanceFunnel(
         _id: progressId,
       };
     }
-    await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+    foldFunnelStepDelta(
+      reportRollupDeltas,
+      siteId,
+      funnel._id,
+      event,
+      0,
+      false,
+    );
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
-    return;
+    return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
   if (
     progress.completedAt !== undefined ||
@@ -1743,9 +1875,16 @@ async function advanceFunnel(
         completedAt: undefined,
       });
     }
-    await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+    foldFunnelStepDelta(
+      reportRollupDeltas,
+      siteId,
+      funnel._id,
+      event,
+      0,
+      false,
+    );
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
-    return;
+    return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
   const nextStep = funnel.steps[progress.currentStep];
   if (!nextStep || !stepMatches(nextStep, event)) return;
@@ -1763,8 +1902,8 @@ async function advanceFunnel(
   } else {
     await ctx.db.patch("funnelProgress", progress._id, updates);
   }
-  await incrementFunnelStep(
-    ctx,
+  foldFunnelStepDelta(
+    reportRollupDeltas,
     siteId,
     funnel._id,
     event,
@@ -1779,6 +1918,11 @@ async function advanceFunnel(
     progress.currentStep,
     completed,
   );
+  return {
+    funnelId: funnel._id,
+    step: currentStep,
+    steps: funnel.steps.length,
+  };
 }
 
 function stepMatches(
@@ -1790,47 +1934,6 @@ function stepMatches(
     : event.type === "custom" && event.name === step.value;
 }
 
-async function incrementFunnelStep(
-  ctx: MutationCtx,
-  siteId: Id<"sites">,
-  funnelId: Id<"funnels">,
-  event: TrackerEvent,
-  step: number,
-  completed: boolean,
-) {
-  const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
-  const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
-  const bucket = await ctx.db
-    .query("funnelStepBuckets")
-    .withIndex(
-      "by_siteId_and_funnelId_and_bucketStart_and_step_and_shard",
-      (range) =>
-        range
-          .eq("siteId", siteId)
-          .eq("funnelId", funnelId)
-          .eq("bucketStart", bucketStart)
-          .eq("step", step)
-          .eq("shard", shard),
-    )
-    .unique();
-  if (bucket) {
-    await ctx.db.patch("funnelStepBuckets", bucket._id, {
-      entrants: bucket.entrants + 1,
-      completions: bucket.completions + (completed ? 1 : 0),
-    });
-  } else {
-    await ctx.db.insert("funnelStepBuckets", {
-      siteId,
-      funnelId,
-      bucketStart,
-      step,
-      shard,
-      entrants: 1,
-      completions: completed ? 1 : 0,
-    });
-  }
-}
-
 async function accountConversion(
   ctx: MutationCtx,
   siteId: Id<"sites">,
@@ -1913,31 +2016,6 @@ async function accountConversion(
   }
 }
 
-function createReportRollupDeltas(): ReportRollupDeltas {
-  return {
-    goals: new Map(),
-    funnels: new Map(),
-    affiliates: new Map(),
-    vitals: new Map(),
-    routes: new Map(),
-  };
-}
-
-function reportRollupCoordinates(
-  siteId: Id<"sites">,
-  definitionId: string,
-  timestamp: number,
-  visitorId: string,
-) {
-  const bucketStart = Math.floor(timestamp / DAY_MS) * DAY_MS;
-  const shard = stableHash(visitorId) % REPORT_ROLLUP_SHARDS;
-  return {
-    bucketStart,
-    shard,
-    key: `${siteId}:${definitionId}:${bucketStart}:${shard}`,
-  };
-}
-
 function foldGoalRollup(
   deltas: ReportRollupDeltas,
   siteId: Id<"sites">,
@@ -1996,6 +2074,31 @@ function foldFunnelRollup(
   deltas.funnels.set(coordinates.key, rollup);
 }
 
+function foldFunnelStepDelta(
+  deltas: ReportRollupDeltas,
+  siteId: Id<"sites">,
+  funnelId: Id<"funnels">,
+  event: TrackerEvent,
+  step: number,
+  completed: boolean,
+) {
+  const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+  const shard = stableHash(event.visitorId) % SECONDARY_AGGREGATE_SHARDS;
+  const key = `${siteId}:${funnelId}:${bucketStart}:${step}:${shard}`;
+  const current = deltas.funnelSteps.get(key) ?? {
+    siteId,
+    funnelId,
+    bucketStart,
+    step,
+    shard,
+    entrants: 0,
+    completions: 0,
+  };
+  current.entrants += 1;
+  current.completions += Number(completed);
+  deltas.funnelSteps.set(key, current);
+}
+
 function foldAffiliateVisitRollup(
   deltas: ReportRollupDeltas,
   siteId: Id<"sites">,
@@ -2016,13 +2119,13 @@ function foldAffiliateVisitRollup(
     shard: coordinates.shard,
     affiliateSlug: attribution.affiliateSlug,
     visits: 0,
-    visitorSketch: createVisitorSketch(),
+    visitorKeys: new Set<string>(),
     conversions: 0,
     revenueCents: 0,
     commissionCents: 0,
   };
   current.visits += 1;
-  current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
+  current.visitorKeys.add(visitorKey);
   deltas.affiliates.set(coordinates.key, current);
 }
 
@@ -2048,7 +2151,7 @@ function foldAffiliateConversionRollup(
     shard: coordinates.shard,
     affiliateSlug,
     visits: 0,
-    visitorSketch: createVisitorSketch(),
+    visitorKeys: new Set<string>(),
     conversions: 0,
     revenueCents: 0,
     commissionCents: 0,
@@ -2130,7 +2233,7 @@ function foldRouteDelta(
     continued: 0,
     dwellMs: 0,
     dwellSamples: 0,
-    visitorSketch: createVisitorSketch(),
+    visitorKeys: new Set<string>(),
     outgoing: new Map<string, number>(),
   };
   apply(current);
@@ -2163,7 +2266,7 @@ function foldRouteRollups(
     (delta) => {
       delta.pageviews += 1;
       if (firstPageview) delta.entries += 1;
-      delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+      delta.visitorKeys.add(session.visitorKey);
     },
   );
   // Out-of-order telemetry keeps its view but cannot say where it came from.
@@ -2199,272 +2302,6 @@ function foldRouteRollups(
   }
 }
 
-async function upsertRouteRollup(
-  ctx: MutationCtx,
-  delta: RouteRollupDelta,
-  dayRowCounts: Map<string, number>,
-) {
-  const existing = await ctx.db
-    .query("routeDailyRollups")
-    .withIndex("by_siteId_and_route_and_bucketStart_and_shard", (range) =>
-      range
-        .eq("siteId", delta.siteId)
-        .eq("route", delta.route)
-        .eq("bucketStart", delta.bucketStart)
-        .eq("shard", delta.shard),
-    )
-    .unique();
-  if (!existing && delta.route !== SITE_MAP_OTHER_ROUTE) {
-    // A new route row spends one bounded read per batch to respect the daily
-    // cap; past it, the day's remaining routes fold into "(other)".
-    const capRows = MAX_SITE_MAP_ROUTES_PER_DAY * REPORT_ROLLUP_SHARDS;
-    const dayKey = `${delta.siteId}:${delta.bucketStart}`;
-    let dayRows = dayRowCounts.get(dayKey);
-    if (dayRows === undefined) {
-      dayRows = (
-        await ctx.db
-          .query("routeDailyRollups")
-          .withIndex("by_siteId_and_bucketStart", (range) =>
-            range
-              .eq("siteId", delta.siteId)
-              .eq("bucketStart", delta.bucketStart),
-          )
-          .take(capRows + 1)
-      ).length;
-    }
-    if (dayRows >= capRows) {
-      await upsertRouteRollup(
-        ctx,
-        { ...delta, route: SITE_MAP_OTHER_ROUTE },
-        dayRowCounts,
-      );
-      return;
-    }
-    dayRowCounts.set(dayKey, dayRows + 1);
-  }
-  const outgoing = existing?.outgoing.map((entry) => ({ ...entry })) ?? [];
-  for (const [target, count] of delta.outgoing) {
-    let entry = outgoing.find((candidate) => candidate.route === target);
-    if (!entry) {
-      const named = outgoing.filter(
-        (candidate) => candidate.route !== SITE_MAP_OTHER_ROUTE,
-      ).length;
-      const route =
-        target !== SITE_MAP_OTHER_ROUTE && named >= ROUTE_TRANSITION_SLOTS
-          ? SITE_MAP_OTHER_ROUTE
-          : target;
-      entry = outgoing.find((candidate) => candidate.route === route);
-      if (!entry) {
-        entry = { route, count: 0 };
-        outgoing.push(entry);
-      }
-    }
-    entry.count += count;
-  }
-  const fields = {
-    siteId: delta.siteId,
-    bucketStart: delta.bucketStart,
-    shard: delta.shard,
-    generation: 1 as const,
-    route: delta.route,
-    pageviews: (existing?.pageviews ?? 0) + delta.pageviews,
-    entries: (existing?.entries ?? 0) + delta.entries,
-    continued: (existing?.continued ?? 0) + delta.continued,
-    dwellMs: (existing?.dwellMs ?? 0) + delta.dwellMs,
-    dwellSamples: (existing?.dwellSamples ?? 0) + delta.dwellSamples,
-    visitorSketch: unionVisitorSketches(
-      existing?.visitorSketch ?? createVisitorSketch(),
-      delta.visitorSketch,
-    ),
-    outgoing,
-  };
-  if (existing) {
-    await ctx.db.replace("routeDailyRollups", existing._id, fields);
-  } else {
-    await ctx.db.insert("routeDailyRollups", fields);
-  }
-}
-
-async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
-  const existing = await ctx.db
-    .query("vitalRollups")
-    .withIndex("by_key", (range) =>
-      range
-        .eq("siteId", delta.siteId)
-        .eq("bucketStart", delta.bucketStart)
-        .eq("device", delta.device)
-        .eq("page", delta.page)
-        .eq("shard", delta.shard),
-    )
-    .unique();
-  if (
-    !existing &&
-    delta.device === VITAL_ALL &&
-    delta.page !== VITAL_OTHER_PAGES
-  ) {
-    // A new page row spends one bounded read to respect the daily page slots;
-    // past the cap, the day's remaining pages fold into "(other)".
-    const pageRows = await ctx.db
-      .query("vitalRollups")
-      .withIndex("by_key", (range) =>
-        range
-          .eq("siteId", delta.siteId)
-          .eq("bucketStart", delta.bucketStart)
-          .eq("device", VITAL_ALL),
-      )
-      .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
-    if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
-      await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
-      return;
-    }
-  }
-  const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
-  for (const [metric, addition] of delta.metrics) {
-    const entry = metrics.find((candidate) => candidate.metric === metric);
-    if (entry) {
-      entry.count += addition.count;
-      entry.sum += addition.sum;
-      entry.histogram = mergeVitalHistograms(
-        entry.histogram,
-        addition.histogram,
-      );
-    } else {
-      metrics.push({
-        metric,
-        count: addition.count,
-        sum: addition.sum,
-        histogram: [...addition.histogram],
-      });
-    }
-  }
-  const fields = {
-    siteId: delta.siteId,
-    bucketStart: delta.bucketStart,
-    device: delta.device,
-    page: delta.page,
-    shard: delta.shard,
-    metrics,
-  };
-  if (existing) {
-    await ctx.db.replace("vitalRollups", existing._id, fields);
-  } else {
-    await ctx.db.insert("vitalRollups", fields);
-  }
-}
-
-async function flushReportRollups(
-  ctx: MutationCtx,
-  deltas: ReportRollupDeltas,
-) {
-  for (const delta of deltas.goals.values()) {
-    const existing = await ctx.db
-      .query("goalDailyRollups")
-      .withIndex("by_siteId_and_goalId_and_bucketStart_and_shard", (range) =>
-        range
-          .eq("siteId", delta.siteId)
-          .eq("goalId", delta.goalId)
-          .eq("bucketStart", delta.bucketStart)
-          .eq("shard", delta.shard),
-      )
-      .unique();
-    const fields = {
-      siteId: delta.siteId,
-      goalId: delta.goalId,
-      bucketStart: delta.bucketStart,
-      shard: delta.shard,
-      generation: 1 as const,
-      goalKey: delta.goalKey,
-      completions: (existing?.completions ?? 0) + delta.completions,
-      valueCents: (existing?.valueCents ?? 0) + delta.valueCents,
-    };
-    if (existing) {
-      await ctx.db.replace("goalDailyRollups", existing._id, fields);
-    } else {
-      await ctx.db.insert("goalDailyRollups", fields);
-    }
-  }
-  for (const delta of deltas.funnels.values()) {
-    const existing = await ctx.db
-      .query("funnelDailyRollups")
-      .withIndex("by_siteId_and_funnelId_and_bucketStart_and_shard", (range) =>
-        range
-          .eq("siteId", delta.siteId)
-          .eq("funnelId", delta.funnelId)
-          .eq("bucketStart", delta.bucketStart)
-          .eq("shard", delta.shard),
-      )
-      .unique();
-    const steps = existing?.steps.map((step) => ({ ...step })) ?? [];
-    for (const addition of delta.steps.values()) {
-      const current = steps.find((step) => step.step === addition.step);
-      if (current) {
-        current.entrants += addition.entrants;
-        current.completions += addition.completions;
-      } else {
-        if (steps.length >= MAX_FUNNEL_STEPS) {
-          fail("LIMIT_EXCEEDED", "daily funnel rollup step limit exceeded");
-        }
-        steps.push(addition);
-      }
-    }
-    const fields = {
-      siteId: delta.siteId,
-      funnelId: delta.funnelId,
-      bucketStart: delta.bucketStart,
-      shard: delta.shard,
-      generation: 1 as const,
-      steps,
-    };
-    if (existing) {
-      await ctx.db.replace("funnelDailyRollups", existing._id, fields);
-    } else {
-      await ctx.db.insert("funnelDailyRollups", fields);
-    }
-  }
-  for (const delta of deltas.affiliates.values()) {
-    const existing = await ctx.db
-      .query("affiliateDailyRollups")
-      .withIndex(
-        "by_siteId_and_affiliateId_and_bucketStart_and_shard",
-        (range) =>
-          range
-            .eq("siteId", delta.siteId)
-            .eq("affiliateId", delta.affiliateId)
-            .eq("bucketStart", delta.bucketStart)
-            .eq("shard", delta.shard),
-      )
-      .unique();
-    const fields = {
-      siteId: delta.siteId,
-      affiliateId: delta.affiliateId,
-      bucketStart: delta.bucketStart,
-      shard: delta.shard,
-      generation: 1 as const,
-      affiliateSlug: delta.affiliateSlug,
-      visits: (existing?.visits ?? 0) + delta.visits,
-      visitorSketch: unionVisitorSketches(
-        existing?.visitorSketch ?? createVisitorSketch(),
-        delta.visitorSketch,
-      ),
-      conversions: (existing?.conversions ?? 0) + delta.conversions,
-      revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
-      commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
-    };
-    if (existing) {
-      await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
-    } else {
-      await ctx.db.insert("affiliateDailyRollups", fields);
-    }
-  }
-  const routeDayRowCounts = new Map<string, number>();
-  for (const delta of deltas.routes.values()) {
-    await upsertRouteRollup(ctx, delta, routeDayRowCounts);
-  }
-  for (const delta of deltas.vitals.values()) {
-    await upsertVitalRollup(ctx, delta);
-  }
-}
-
 function foldAggregates(
   aggregateDeltas: AggregateDeltas,
   siteId: Id<"sites">,
@@ -2473,19 +2310,25 @@ function foldAggregates(
   dimensions: Dimension[],
   recordTelemetry: boolean,
   financialConversion: boolean,
+  timezone?: string,
 ) {
   const revenueCents = financialConversion ? (event.revenueCents ?? 0) : 0;
   const dimensionDeltas = dimensions.map((dimension): DimensionDelta => ({
     ...dimension,
     count: Number(recordTelemetry),
     revenueCents,
+    pageviews: Number(recordTelemetry && event.type === "pageview"),
+    sessions: Number(recordTelemetry && session.isNewSession),
   }));
-  for (const granularity of ["hour", "day"] as const) {
+  for (const granularity of aggregateGranularities(timezone)) {
     const shard =
       stableHash(event.sessionId) %
       (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
-    const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
-    const bucketStart = Math.floor(event.timestamp / interval) * interval;
+    const bucketStart = aggregateBucketStart(
+      granularity,
+      event.timestamp,
+      timezone,
+    );
     const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
     let delta = aggregateDeltas.get(key);
     if (!delta) {
@@ -2505,6 +2348,9 @@ function foldAggregates(
         conversions: 0,
         revenueCents: 0,
         dimensions: [],
+        durationMs: 0,
+        continuedSessions: 0,
+        newVisitors: 0,
       };
       aggregateDeltas.set(key, delta);
     }
@@ -2528,6 +2374,77 @@ function foldAggregates(
     delta.conversions += Number(financialConversion);
     delta.revenueCents += revenueCents;
     delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
+    if (recordTelemetry) {
+      delta.durationMs += session.durationDeltaMs;
+      delta.continuedSessions += Number(session.continued);
+      delta.newVisitors += Number(session.isNewSession && session.isNewVisitor);
+    }
+  }
+}
+
+/**
+ * The buckets an event lands in: UTC hour and UTC day always, plus the
+ * calendar day in the site's timezone when the site keeps one.
+ */
+function aggregateGranularities(
+  timezone: string | undefined,
+): ReadonlyArray<"hour" | "day" | "localDay"> {
+  return timezone ? ["hour", "day", "localDay"] : ["hour", "day"];
+}
+
+function aggregateBucketStart(
+  granularity: "hour" | "day" | "localDay",
+  timestamp: number,
+  timezone: string | undefined,
+): number {
+  if (granularity === "localDay" && timezone) {
+    return localDayStart(timestamp, timezone);
+  }
+  const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
+  return Math.floor(timestamp / interval) * interval;
+}
+
+/** Folds only session duration growth, for heartbeats. */
+function foldSessionDuration(
+  aggregateDeltas: AggregateDeltas,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  durationDeltaMs: number,
+  timezone?: string,
+) {
+  if (durationDeltaMs <= 0) return;
+  for (const granularity of aggregateGranularities(timezone)) {
+    const shard =
+      stableHash(event.sessionId) %
+      (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
+    const bucketStart = aggregateBucketStart(
+      granularity,
+      event.timestamp,
+      timezone,
+    );
+    const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
+    const delta = aggregateDeltas.get(key) ?? {
+      siteId,
+      granularity,
+      bucketStart,
+      shard,
+      events: 0,
+      pageviews: 0,
+      clicks: 0,
+      customEvents: 0,
+      outboundClicks: 0,
+      sessions: 0,
+      visitors: 0,
+      visitorSketch: createVisitorSketch(),
+      conversions: 0,
+      revenueCents: 0,
+      dimensions: [],
+      durationMs: 0,
+      continuedSessions: 0,
+      newVisitors: 0,
+    };
+    delta.durationMs += durationDeltaMs;
+    aggregateDeltas.set(key, delta);
   }
 }
 
@@ -2550,7 +2467,11 @@ async function flushAggregates(
       .unique();
     const dimensions = mergeDimensions(
       bucket?.dimensions ?? [],
-      delta.dimensions,
+      delta.dimensions.map((slot) => ({
+        ...slot,
+        pageviews: slot.pageviews ?? 0,
+        sessions: slot.sessions ?? 0,
+      })),
     );
     const priorSketch = bucket?.visitorSketch;
     const visitorSketch = priorSketch
@@ -2573,6 +2494,10 @@ async function flushAggregates(
         conversions: bucket.conversions + delta.conversions,
         revenueCents: bucket.revenueCents + delta.revenueCents,
         dimensions,
+        durationMs: (bucket.durationMs ?? 0) + delta.durationMs,
+        continuedSessions:
+          (bucket.continuedSessions ?? 0) + delta.continuedSessions,
+        newVisitors: (bucket.newVisitors ?? 0) + delta.newVisitors,
       });
     } else {
       await ctx.db.insert("aggregateBuckets", {
@@ -2592,6 +2517,9 @@ async function flushAggregates(
         conversions: delta.conversions,
         revenueCents: delta.revenueCents,
         dimensions,
+        durationMs: delta.durationMs,
+        continuedSessions: delta.continuedSessions,
+        newVisitors: delta.newVisitors,
       });
     }
   }
@@ -2610,6 +2538,8 @@ function mergeDimensions(
     if (exact) {
       exact.count += addition.count;
       exact.revenueCents += addition.revenueCents;
+      exact.pageviews = (exact.pageviews ?? 0) + addition.pageviews;
+      exact.sessions = (exact.sessions ?? 0) + addition.sessions;
       continue;
     }
 
@@ -2627,6 +2557,8 @@ function mergeDimensions(
         value: addition.value,
         count: addition.count,
         revenueCents: addition.revenueCents,
+        pageviews: addition.pageviews,
+        sessions: addition.sessions,
       });
       continue;
     }
@@ -2636,6 +2568,8 @@ function mergeDimensions(
     if (overflow) {
       overflow.count += addition.count;
       overflow.revenueCents += addition.revenueCents;
+      overflow.pageviews = (overflow.pageviews ?? 0) + addition.pageviews;
+      overflow.sessions = (overflow.sessions ?? 0) + addition.sessions;
       continue;
     }
     if (addition.count === 0 && addition.revenueCents === 0) continue;
@@ -2649,6 +2583,8 @@ function mergeDimensions(
         value: "(other)",
         count: addition.count,
         revenueCents: addition.revenueCents,
+        pageviews: addition.pageviews,
+        sessions: addition.sessions,
       });
       continue;
     }
@@ -2660,6 +2596,8 @@ function mergeDimensions(
     replacement.value = "(other)";
     replacement.count += addition.count;
     replacement.revenueCents += addition.revenueCents;
+    replacement.pageviews = (replacement.pageviews ?? 0) + addition.pageviews;
+    replacement.sessions = (replacement.sessions ?? 0) + addition.sessions;
   }
   return result;
 }