@iann29/rastro 0.3.0 → 0.4.0

package/src/component/constants.ts

@@ -4,9 +4,20 @@ export const MAX_BATCH_BYTES = 64 * 1024;
 export const MAX_EVENTS_PER_SESSION_WINDOW = 120;
 export const RATE_LIMIT_WINDOW_MS = 60_000;
 export const MAX_BYTES_PER_SESSION_WINDOW = 256 * 1024;
-export const SITE_INGEST_SHARDS = 4_096;
-export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 300;
-export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 512 * 1024;
+// A site admits at most SITE_INGEST_SHARDS × the per-shard caps a minute:
+// 384,000 events and 512 MiB. Shards keep every POST off one shared row per
+// site; random session IDs can only fill them evenly, while a real peak lands
+// unevenly — the realistic benchmark (~100k events/min) peaks near 1,050
+// events in its hottest shard-minute, so fewer shards would need a looser cap
+// per shard and more would need a looser cap per site.
+export const SITE_INGEST_SHARDS = 256;
+export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 1_500;
+export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 2 * 1024 * 1024;
+// Per client network (an IPv4 address or an IPv6 /64, keyed by a salted daily
+// hash) per site and minute. Generous on purpose: an office or carrier-grade
+// NAT puts many visitors behind one address, and they all share this budget.
+export const MAX_EVENTS_PER_CLIENT_WINDOW = 1_200;
+export const MAX_BYTES_PER_CLIENT_WINDOW = 2 * 1024 * 1024;
 export const LIVE_SESSION_TTL_MS = 90_000;
 export const LIVE_LEAVE_GRACE_MS = 10_000;
 export const LIVE_SWEEP_INTERVAL_MS = 5_000;
@@ -44,6 +55,16 @@ export const RETENTION_LEASE_MS = 60_000;
 export const ROLLUP_FOLD_BATCH_DOCS = 200;
 export const ROLLUP_FOLD_MAX_ROWS = 1_500;
 export const ROLLUP_FOLD_LEASE_MS = 60_000;
+// A batch that finds no fold pending schedules one this far out, and a run
+// that folded anything schedules the next one as far again, so one run per
+// second serves every batch of that second; only a run that finds the queue
+// empty releases the lease.
+export const ROLLUP_FOLD_DEBOUNCE_MS = 1_000;
+// A run that throws retries its oldest batch alone; a batch that fails this
+// many runs in a row is set aside as a dead letter, kept this long (30 days)
+// for inspection, so one bad batch cannot stall a site's reports.
+export const ROLLUP_FOLD_MAX_FAILURES = 3;
+export const ROLLUP_DEAD_LETTER_RETENTION_MS = 30 * 86_400_000;
 // Shared Web Vitals histogram edges. Every Google rating threshold (LCP
 // 2500/4000, CLS×1000 100/250, INP 200/500, FCP 1800/3000, TTFB 800/1800) is
 // an edge, so rating counts stay exact while one layout serves all metrics.

package/src/component/convex.config.ts

@@ -9,5 +9,8 @@ export default defineComponent("rastroAnalytics", {
     RASTRO_GEOIP_TOKEN: v.optional(v.string()),
     RASTRO_GEOIP_DAILY_LIMIT: v.optional(v.string()),
     RASTRO_TRUST_PROXY: v.optional(v.literal("true")),
+    // "off" drops the per-client ingest budget, for a load-test canary whose
+    // driver sends every request from one address.
+    RASTRO_CLIENT_RATE_LIMIT: v.optional(v.literal("off")),
   },
 });

package/src/component/geo.ts

@@ -510,7 +510,7 @@ function classifyIpv6(ip: string): Exclude<IpKind, "unavailable"> {
   return (groups[0]! & 0xe000) === 0x2000 ? "public" : "private";
 }
 
-function parseIpv6(ip: string): number[] | undefined {
+export function parseIpv6(ip: string): number[] | undefined {
   let normalized = ip;
   const dottedTail = normalized.match(/(\d+\.\d+\.\d+\.\d+)$/)?.[1];
   if (dottedTail) {

package/src/component/http.ts

@@ -20,7 +20,7 @@ import {
 } from "./geo.js";
 import { isPlainRecord } from "./guards.js";
 import { alignClock, sanitizeEvent } from "./sanitize.js";
-import { deriveVisitorKey, randomSecret } from "./identity.js";
+import { deriveClientKey, deriveVisitorKey, randomSecret } from "./identity.js";
 import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
 import {
@@ -179,6 +179,19 @@ http.route({
       ...classifyClient(userAgent, request.headers),
       ...(visitorKey ? { visitorKey } : {}),
     };
+    // Origin is a browser control: any other sender writes the header it
+    // likes. The per-client budget is keyed like the visitor key above (salted,
+    // daily, never the address); without a usable client IP only the per-site
+    // shards bound the batch.
+    const clientKey =
+      clientIp.ip && env.RASTRO_CLIENT_RATE_LIMIT !== "off"
+        ? await deriveClientKey({
+            secret: await visitorSecret(ctx),
+            now: Date.now(),
+            siteId: payload.siteId,
+            ip: clientIp.ip,
+          })
+        : undefined;
 
     try {
       const result = await ctx.runMutation(api.ingest.ingestBatch, {
@@ -186,6 +199,7 @@ http.route({
         origin,
         events,
         context,
+        ...(clientKey ? { clientKey } : {}),
       });
       if (
         clientIp.ip &&

package/src/component/identity.ts

@@ -1,5 +1,6 @@
 import { v } from "convex/values";
 import { DAY_MS } from "./constants.js";
+import { parseIpv6 } from "./geo.js";
 import { internalMutation, internalQuery } from "./_generated/server.js";
 
 /**
@@ -57,9 +58,47 @@ export async function deriveVisitorKey(input: {
   ip: string;
   userAgent: string;
 }): Promise<string> {
-  const dayStart = Math.floor(input.now / DAY_MS) * DAY_MS;
+  return saltedDailyKey(input.secret, input.now, [
+    input.siteId,
+    input.ip,
+    input.userAgent,
+  ]);
+}
+
+/**
+ * The per-client ingest budget's key: the same salted daily hash, over the
+ * client's network instead of its device — the IPv4 address, or the /64 of an
+ * IPv6 address, the block one subscriber gets and could otherwise walk through
+ * one address per request.
+ */
+export async function deriveClientKey(input: {
+  secret: string;
+  now: number;
+  siteId: string;
+  ip: string;
+}): Promise<string> {
+  const groups = parseIpv6(input.ip.toLowerCase());
+  const network = groups
+    ? `${groups
+        .slice(0, 4)
+        .map((group) => group.toString(16))
+        .join(":")}::/64`
+    : input.ip;
+  return saltedDailyKey(input.secret, input.now, [
+    "client",
+    input.siteId,
+    network,
+  ]);
+}
+
+async function saltedDailyKey(
+  secret: string,
+  now: number,
+  parts: string[],
+): Promise<string> {
+  const dayStart = Math.floor(now / DAY_MS) * DAY_MS;
   const material = new TextEncoder().encode(
-    [input.secret, dayStart, input.siteId, input.ip, input.userAgent].join(" "),
+    [secret, dayStart, ...parts].join(" "),
   );
   const digest = new Uint8Array(
     await crypto.subtle.digest("SHA-256", material),

package/src/component/ingest.ts

@@ -14,6 +14,8 @@ import {
   LIVE_SESSION_TTL_MS,
   LIVE_SWEEP_INTERVAL_MS,
   MAX_BATCH_EVENTS,
+  MAX_BYTES_PER_CLIENT_WINDOW,
+  MAX_EVENTS_PER_CLIENT_WINDOW,
   MAX_SITE_BYTES_PER_SHARD_WINDOW,
   MAX_SITE_EVENTS_PER_SHARD_WINDOW,
   MAX_BYTES_PER_SESSION_WINDOW,
@@ -123,7 +125,7 @@ type AggregateDelta = {
   continuedSessions: number;
   newVisitors: number;
 };
-type AggregateDeltas = Map<string, AggregateDelta>;
+export type AggregateDeltas = Map<string, AggregateDelta>;
 type AttributionSnapshot = {
   affiliateId: Id<"affiliates">;
   affiliateSlug: string;
@@ -214,6 +216,8 @@ export const ingestBatch = mutation({
     origin: v.string(),
     events: v.array(trackerEventValidator),
     context: v.optional(ingestContextValidator),
+    // The HTTP action's salted daily client-network key (see identity.ts).
+    clientKey: v.optional(v.string()),
   },
   returns: ingestResultValidator,
   handler: async (ctx, args) => {
@@ -245,6 +249,15 @@ export const ingestBatch = mutation({
 
     const now = Date.now();
     await ensureAnalyticsControl(ctx, args.siteId, now);
+    if (args.clientKey !== undefined) {
+      await consumeClientRateLimit(
+        ctx,
+        args.siteId,
+        args.clientKey,
+        eventByteCounts,
+        now,
+      );
+    }
     await consumeSiteRateLimit(
       ctx,
       args.siteId,
@@ -389,7 +402,10 @@ export const ingestBatch = mutation({
           rejected += 1;
           continue;
         }
-        const durationDeltaMs = await accountHeartbeat(
+        // A heartbeat is presence, not telemetry: it moves the session's
+        // lastSeenAt and live row only. The time it proves reaches the
+        // duration counters once, when the visitor departs (live.ts).
+        await accountHeartbeat(
           ctx,
           args.siteId,
           existingSession,
@@ -398,16 +414,6 @@ export const ingestBatch = mutation({
           now,
           batchState,
         );
-        // A heartbeat is presence, not telemetry, but it is what keeps a
-        // session's duration honest while the visitor reads: the growth is
-        // folded into the buckets the heartbeat landed in.
-        foldSessionDuration(
-          aggregateDeltas,
-          args.siteId,
-          event,
-          durationDeltaMs,
-          timezone,
-        );
         accepted += 1;
         continue;
       }
@@ -461,7 +467,8 @@ export const ingestBatch = mutation({
             foldSessionDuration(
               aggregateDeltas,
               args.siteId,
-              event,
+              event.sessionId,
+              event.timestamp,
               durationMs - existingSession.durationMs,
               timezone,
             );
@@ -1421,6 +1428,12 @@ async function attributionForEvent(
   };
 }
 
+/**
+ * Moves the session's lastSeenAt and its live row. `durationMs` stays: it is
+ * what the duration counters already hold, and the time a heartbeat proves
+ * is counted when the visitor departs (live.ts), so an idle tab never
+ * rewrites an aggregate bucket.
+ */
 async function accountHeartbeat(
   ctx: MutationCtx,
   siteId: Id<"sites">,
@@ -1429,7 +1442,7 @@ async function accountHeartbeat(
   context: IngestContext,
   now: number,
   batchState?: BatchState,
-): Promise<number> {
+): Promise<void> {
   const startedAt = Math.min(session.startedAt, event.timestamp);
   const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
   const updates = {
@@ -1437,7 +1450,6 @@ async function accountHeartbeat(
     lastSeenAt,
     exitPath:
       event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
-    durationMs: Math.max(0, lastSeenAt - startedAt),
     ...(context.country ? { country: context.country } : {}),
     ...(context.city ? { city: context.city } : {}),
     ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
@@ -1456,7 +1468,6 @@ async function accountHeartbeat(
   } else {
     await ctx.db.patch("sessions", session._id, updates);
   }
-  const durationDeltaMs = Math.max(0, updates.durationMs - session.durationMs);
   await updateLiveSession(
     ctx,
     siteId,
@@ -1486,7 +1497,54 @@ async function accountHeartbeat(
     now,
     batchState,
   );
-  return durationDeltaMs;
+}
+
+/**
+ * One client network's budget per site and minute, kept in the session
+ * windows' table under `~client:<key>`, a key no sanitized session ID can take.
+ * The key is the salted daily hash the HTTP action derived; the address never
+ * reaches this mutation. It is one row per client, so only that client's own
+ * concurrent requests contend on it.
+ */
+async function consumeClientRateLimit(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  clientKey: string,
+  eventByteCounts: number[],
+  now: number,
+): Promise<void> {
+  const windowStart =
+    Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+  const sessionId = `~client:${clientKey}`;
+  const rows = await ctx.db
+    .query("ingestWindows")
+    .withIndex("by_siteId_and_sessionId_and_windowStart", (range) =>
+      range.eq("siteId", siteId).eq("sessionId", sessionId),
+    )
+    .order("desc")
+    .take(1);
+  const window = rows[0] ?? null;
+  const current = window?.windowStart === windowStart ? window : null;
+  const eventCount = (current?.eventCount ?? 0) + eventByteCounts.length;
+  const byteCount =
+    (current?.byteCount ?? 0) +
+    eventByteCounts.reduce((total, bytes) => total + bytes, 0);
+  if (
+    eventCount > MAX_EVENTS_PER_CLIENT_WINDOW ||
+    byteCount > MAX_BYTES_PER_CLIENT_WINDOW
+  ) {
+    fail("RATE_LIMITED", "client ingest budget exceeded", {
+      windowMs: RATE_LIMIT_WINDOW_MS,
+      eventLimitPerClient: MAX_EVENTS_PER_CLIENT_WINDOW,
+      byteLimitPerClient: MAX_BYTES_PER_CLIENT_WINDOW,
+    });
+  }
+  const fields = { windowStart, eventCount, byteCount, updatedAt: now };
+  if (window) {
+    await ctx.db.patch("ingestWindows", window._id, fields);
+  } else {
+    await ctx.db.insert("ingestWindows", { siteId, sessionId, ...fields });
+  }
 }
 
 async function consumeSiteRateLimit(
@@ -1872,11 +1930,10 @@ async function advanceFunnel(
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
     return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
-  if (
-    progress.completedAt !== undefined ||
-    event.timestamp < progress.lastStepAt
-  )
-    return;
+  if (event.timestamp < progress.lastStepAt) return;
+  // An expired window restarts on a first-step event even after a completion,
+  // so a returning buyer can complete again; inside the window a completed
+  // funnel stays done.
   if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
     if (!matchesFirst) return;
     const updated = {
@@ -1910,6 +1967,7 @@ async function advanceFunnel(
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
     return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
+  if (progress.completedAt !== undefined) return;
   const nextStep = funnel.steps[progress.currentStep];
   if (!nextStep || !stepMatches(nextStep, event)) return;
   const currentStep = progress.currentStep + 1;
@@ -2428,24 +2486,26 @@ function aggregateBucketStart(
   return Math.floor(timestamp / interval) * interval;
 }
 
-/** Folds session duration growth without telemetry, for heartbeats and leaves. */
-function foldSessionDuration(
+/**
+ * Folds session duration growth without telemetry: a departure's final
+ * interval, or the heartbeat time of a visitor the live sweep retired. The
+ * shard follows `shardKey`: the session for a departure, the site for a
+ * sweep.
+ */
+export function foldSessionDuration(
   aggregateDeltas: AggregateDeltas,
   siteId: Id<"sites">,
-  event: TrackerEvent,
+  shardKey: string,
+  timestamp: number,
   durationDeltaMs: number,
   timezone?: string,
 ) {
   if (durationDeltaMs <= 0) return;
   for (const granularity of aggregateGranularities(timezone)) {
     const shard =
-      stableHash(event.sessionId) %
+      stableHash(shardKey) %
       (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
-    const bucketStart = aggregateBucketStart(
-      granularity,
-      event.timestamp,
-      timezone,
-    );
+    const bucketStart = aggregateBucketStart(granularity, timestamp, timezone);
     const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
     const delta = aggregateDeltas.get(key) ?? {
       siteId,
@@ -2472,7 +2532,7 @@ function foldSessionDuration(
   }
 }
 
-async function flushAggregates(
+export async function flushAggregates(
   ctx: MutationCtx,
   aggregateDeltas: AggregateDeltas,
 ) {

package/src/component/live.ts

@@ -1,7 +1,15 @@
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
 import { internalMutation } from "./_generated/server.js";
 import { LIVE_SWEEP_INTERVAL_MS } from "./constants.js";
+import {
+  flushAggregates,
+  foldSessionDuration,
+  type AggregateDeltas,
+} from "./ingest.js";
+import { localDayTimezone } from "./localTime.js";
 
 const SWEEP_BATCH_SIZE = 500;
 
@@ -58,7 +66,7 @@ export const expire = internalMutation({
       }
       return { expired: false, rescheduled: false, stale: false };
     }
-    await ctx.db.delete("liveSessions", liveSession._id);
+    await retireLiveSessions(ctx, liveSession.siteId, [liveSession]);
     return { expired: true, rescheduled: false, stale: false };
   },
 });
@@ -89,9 +97,7 @@ export const sweep = internalMutation({
         range.eq("siteId", args.siteId).lte("expiresAt", now),
       )
       .take(SWEEP_BATCH_SIZE);
-    for (const session of expired) {
-      await ctx.db.delete("liveSessions", session._id);
-    }
+    await retireLiveSessions(ctx, args.siteId, expired);
 
     const next = await ctx.db
       .query("liveSessions")
@@ -115,3 +121,44 @@ export const sweep = internalMutation({
     return { expired: expired.length, rescheduled: true, stale: false };
   },
 });
+
+/**
+ * Deletes departed live rows and counts the time their heartbeats proved. A
+ * heartbeat only moves its session's `lastSeenAt`, and `durationMs` is what
+ * the duration counters already hold, so the difference reaches the buckets
+ * here, once, in the hour the visitor was last seen. It lands in one shard
+ * per site, so a sweep rewrites a few buckets however many visitors it
+ * retires.
+ */
+async function retireLiveSessions(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  rows: Doc<"liveSessions">[],
+) {
+  if (rows.length === 0) return;
+  const site = await ctx.db.get("sites", siteId);
+  const timezone = localDayTimezone(site?.timezone);
+  const aggregateDeltas: AggregateDeltas = new Map();
+  for (const row of rows) {
+    await ctx.db.delete("liveSessions", row._id);
+    const session = await ctx.db
+      .query("sessions")
+      .withIndex("by_siteId_and_sessionId", (range) =>
+        range.eq("siteId", siteId).eq("sessionId", row.sessionId),
+      )
+      .unique();
+    if (!session) continue;
+    const durationMs = session.lastSeenAt - session.startedAt;
+    if (durationMs <= session.durationMs) continue;
+    await ctx.db.patch("sessions", session._id, { durationMs });
+    foldSessionDuration(
+      aggregateDeltas,
+      siteId,
+      siteId,
+      session.lastSeenAt,
+      durationMs - session.durationMs,
+      timezone,
+    );
+  }
+  await flushAggregates(ctx, aggregateDeltas);
+}

package/src/component/reports.ts

@@ -346,14 +346,19 @@ export const overview = query({
     const bucketCount = localDays
       ? bucketStarts.length
       : Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
-    const perSiteLimit =
-      bucketCount *
-      (interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
+    const shards =
+      interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS;
+    const perSiteLimit = bucketCount * shards;
     if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
+      // The row budget, not MAX_REPORT_RANGE_DAYS, bounds the overview: 101
+      // days for one site and 10 for ten; 24 hours for two, 5 for ten.
+      const maximumBuckets = Math.floor(
+        MAX_OVERVIEW_AGGREGATE_ROWS / (shards * siteIds.length),
+      );
       fail(
         "LIMIT_EXCEEDED",
-        "overview range is too wide for the requested number of sites",
-        { maximumAggregateRows: MAX_OVERVIEW_AGGREGATE_ROWS },
+        `overview range is too wide: ${siteIds.length === 1 ? "one site" : `${siteIds.length} sites`} can read at most ${maximumBuckets} ${interval}s`,
+        { maximumAggregateRows: MAX_OVERVIEW_AGGREGATE_ROWS, maximumBuckets },
       );
     }
     const totals = emptyTotals();
@@ -382,7 +387,13 @@ export const overview = query({
     await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
 
     for (const siteId of siteIds) {
-      const buckets = await ctx.db
+      // Iterated rather than taken whole: within the row budget a busy range
+      // can still cross the bytes a query may read. Values are length-capped,
+      // so sixteen buckets stay far below the reserve, and running out fails
+      // with REPORT_INCOMPLETE, as the feature reports do, instead of a raw
+      // limit error.
+      const buckets: Doc<"aggregateBuckets">[] = [];
+      for await (const bucket of ctx.db
         .query("aggregateBuckets")
         .withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
           range
@@ -390,8 +401,13 @@ export const overview = query({
             .eq("granularity", granularity)
             .gte("bucketStart", firstBucket)
             .lte("bucketStart", lastBucket),
-        )
-        .take(perSiteLimit + 1);
+        )) {
+        buckets.push(bucket);
+        if (buckets.length > perSiteLimit) break;
+        if (buckets.length % 16 === 0) {
+          await requireFeatureReadHeadroom(ctx, "overview aggregate buckets");
+        }
+      }
       failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
       for (const bucket of buckets) {
         if (

package/src/component/rollupStore.ts

@@ -13,6 +13,7 @@ import {
   MAX_FUNNEL_STEPS,
   MAX_SITE_MAP_ROUTES_PER_DAY,
   REPORT_ROLLUP_SHARDS,
+  ROLLUP_FOLD_DEBOUNCE_MS,
   ROLLUP_FOLD_LEASE_MS,
   ROUTE_TRANSITION_SLOTS,
   SITE_MAP_OTHER_ROUTE,
@@ -36,8 +37,8 @@ import {
  * whole batches are lost (#52). Ingestion therefore never touches a rollup
  * row. It appends the batch's deltas to `reportRollupQueue` — an insert
  * conflicts with nothing — and one leased fold job per site merges the queue
- * into the rows a few hundred milliseconds later, so every rollup row has a
- * single writer.
+ * into the rows about a second later (see rollups.ts), so every rollup row
+ * has a single writer.
  */
 
 export type GoalRollupDelta = {
@@ -221,6 +222,10 @@ const queuedVitalValidator = v.object({
 export const reportRollupQueueFields = {
   siteId: v.id("sites"),
   createdAt: v.number(),
+  // Failed runs in a row with this batch at the head of the queue, and when
+  // it was set aside after too many (see rollups.ts).
+  foldFailures: v.optional(v.number()),
+  deadLetteredAt: v.optional(v.number()),
   goals: v.array(queuedGoalValidator),
   funnels: v.array(queuedFunnelValidator),
   funnelSteps: v.array(queuedFunnelStepValidator),
@@ -456,8 +461,9 @@ export function mergeQueuedRollups(
 /**
  * Appends a batch's rollup deltas to the site's queue and makes sure a fold
  * job will pick them up. The only document this reads is the site's fold
- * state, which the fold job rewrites about twice a minute under load, so
- * concurrent ingests almost never retry because of it.
+ * state, which no batch rewrites while a fold is pending and the fold job
+ * rewrites about twice a minute under load, so concurrent ingests almost
+ * never retry because of it.
  */
 export async function enqueueReportRollups(
   ctx: MutationCtx,
@@ -482,6 +488,7 @@ export async function ensureRollupFold(
     .query("rollupFoldStates")
     .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
     .unique();
+  // A held lease means a chain is running and will fold this batch.
   if (state && state.leaseUntil > now) return;
   const token = crypto.randomUUID();
   const leaseUntil = now + ROLLUP_FOLD_LEASE_MS;
@@ -490,7 +497,11 @@ export async function ensureRollupFold(
   } else {
     await ctx.db.insert("rollupFoldStates", { siteId, leaseUntil, token });
   }
-  await ctx.scheduler.runAfter(0, internal.rollups.fold, { siteId, token });
+  // Debounced, so every batch of the next second joins this run.
+  await ctx.scheduler.runAfter(ROLLUP_FOLD_DEBOUNCE_MS, internal.rollups.fold, {
+    siteId,
+    token,
+  });
 }
 
 // ---------------------------------------------------------------------------
@@ -632,8 +643,9 @@ export async function flushReportRollups(
   for (const delta of deltas.routes.values()) {
     await upsertRouteRollup(ctx, delta, routeDayRowCounts);
   }
+  const vitalDayPageRows = new Map<string, number>();
   for (const delta of deltas.vitals.values()) {
-    await upsertVitalRollup(ctx, delta);
+    await upsertVitalRollup(ctx, delta, vitalDayPageRows);
   }
 }
 
@@ -731,7 +743,11 @@ async function upsertRouteRollup(
   }
 }
 
-async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
+async function upsertVitalRollup(
+  ctx: MutationCtx,
+  delta: VitalRollupDelta,
+  dayPageRows: Map<string, number>,
+) {
   const existing = await ctx.db
     .query("vitalRollups")
     .withIndex("by_key", (range) =>
@@ -748,21 +764,36 @@ async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
     delta.device === VITAL_ALL &&
     delta.page !== VITAL_OTHER_PAGES
   ) {
-    // A new page row spends one bounded read to respect the daily page slots;
-    // past the cap, the day's remaining pages fold into "(other)".
-    const pageRows = await ctx.db
-      .query("vitalRollups")
-      .withIndex("by_key", (range) =>
-        range
-          .eq("siteId", delta.siteId)
-          .eq("bucketStart", delta.bucketStart)
-          .eq("device", VITAL_ALL),
-      )
-      .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
-    if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
-      await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
+    // New page rows spend one bounded read per day and fold run to respect
+    // the daily page slots, as routes do: rereading it for every new page let
+    // a fold of many distinct URLs pass the read limit and fail every retry.
+    // Past the cap, the day's remaining pages fold into "(other)".
+    const capRows = VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS;
+    const dayKey = `${delta.siteId}:${delta.bucketStart}`;
+    let pageRows = dayPageRows.get(dayKey);
+    if (pageRows === undefined) {
+      pageRows = (
+        await ctx.db
+          .query("vitalRollups")
+          .withIndex("by_key", (range) =>
+            range
+              .eq("siteId", delta.siteId)
+              .eq("bucketStart", delta.bucketStart)
+              .eq("device", VITAL_ALL),
+          )
+          .take(capRows + 1)
+      ).length;
+      dayPageRows.set(dayKey, pageRows);
+    }
+    if (pageRows > capRows) {
+      await upsertVitalRollup(
+        ctx,
+        { ...delta, page: VITAL_OTHER_PAGES },
+        dayPageRows,
+      );
       return;
     }
+    dayPageRows.set(dayKey, pageRows + 1);
   }
   const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
   for (const [metric, addition] of delta.metrics) {