@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.4

package/dist/component/ingest.js

@@ -1,10 +1,12 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
 import { fail } from "./errors.js";
+import { ensureAnalyticsControl } from "./coverage.js";
 import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
-import { cleanString, ensureBatchBytes, isValidMoney, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { cleanString, isValidMoney, measureBatchBytes, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
 export const ingestBatch = mutation({
     args: {
@@ -28,16 +30,19 @@ export const ingestBatch = mutation({
                 limit: MAX_BATCH_EVENTS,
             });
         }
+        let eventByteCounts;
         try {
-            ensureBatchBytes({ siteId: String(args.siteId), events: args.events });
+            ({ eventByteCounts } = measureBatchBytes(String(args.siteId), args.events));
         }
         catch {
             fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
         }
         const now = Date.now();
-        const eventByteCounts = await consumeSiteRateLimit(ctx, args.siteId, args.events, now);
+        await ensureAnalyticsControl(ctx, args.siteId, now);
+        await consumeSiteRateLimit(ctx, args.siteId, args.events, eventByteCounts, now);
         const context = sanitizeContext(args.context);
         const aggregateDeltas = new Map();
+        const reportRollupDeltas = createReportRollupDeltas();
         const sessionRateWindows = new Map();
         const batchState = {
             sessions: new Map(),
@@ -97,7 +102,6 @@ export const ingestBatch = mutation({
                 siteId: args.siteId,
                 sessionId: group.sessionId,
                 bucketStart: group.bucketStart,
-                candidateEventIds: group.eventIds,
             });
             const newCount = [...group.eventIds].filter((eventId) => !eventIds.has(eventId)).length;
             if (newCount > 0 &&
@@ -164,7 +168,7 @@ export const ingestBatch = mutation({
                     .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
                     .take(MAX_FUNNELS_PER_SITE);
             }
-            await accountEvent(ctx, {
+            const session = await accountEvent(ctx, {
                 siteId: args.siteId,
                 event,
                 context,
@@ -172,18 +176,20 @@ export const ingestBatch = mutation({
                 goals: canMatchDefinition ? goals ?? [] : [],
                 funnels: canMatchDefinition ? funnels ?? [] : [],
                 aggregateDeltas,
+                reportRollupDeltas,
                 recordTelemetry: true,
                 financialConversion: false,
                 trustedGoalKey: undefined,
                 now,
                 batchState,
             });
-            bufferEventBatch(args.siteId, event, batchState.eventBatches);
+            bufferEventBatch(args.siteId, event, batchState.eventBatches, session.country ?? "unknown");
             accepted += 1;
         }
         await flushBatchState(ctx, batchState);
         await flushSessionRateLimits(ctx, sessionRateWindows, now);
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
         return { accepted, duplicates, rejected };
     },
 });
@@ -261,7 +267,8 @@ export const trackConversion = mutation({
             device: existingTelemetry?.device ?? existingSession?.device,
         };
         const aggregateDeltas = new Map();
-        await accountEvent(ctx, {
+        const reportRollupDeltas = createReportRollupDeltas();
+        const session = await accountEvent(ctx, {
             siteId: args.siteId,
             event,
             context: conversionContext,
@@ -269,15 +276,18 @@ export const trackConversion = mutation({
             goals: [],
             funnels: [],
             aggregateDeltas,
+            reportRollupDeltas,
             recordTelemetry: existingTelemetry === null && !telemetry.saturated,
             financialConversion: true,
             trustedGoalKey,
             now: Date.now(),
         });
         if (!existingTelemetry && !telemetry.saturated) {
-            await insertEventBatch(ctx, args.siteId, event);
+            await insertEventBatch(ctx, args.siteId, event, session.country ?? "unknown");
         }
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
+        await ensureAnalyticsControl(ctx, args.siteId, Date.now());
         return { accepted: 1, duplicates: 0, rejected: 0 };
     },
 });
@@ -347,7 +357,7 @@ function withoutSystemFields(document) {
 async function accountEvent(ctx, input) {
     const { event } = input;
     const session = input.recordTelemetry
-        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.batchState)
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.reportRollupDeltas, input.batchState)
         : await accountTrustedSession(ctx, input.existingSession, event);
     const matchedGoalKeys = [];
     const matchingGoals = input.goals.filter((goal) => goal.active && goalMatches(goal, event));
@@ -394,20 +404,21 @@ async function accountEvent(ctx, input) {
             timestamp: event.timestamp,
             valueCents: goal.valueCents ?? 0,
         });
+        foldGoalRollup(input.reportRollupDeltas, input.siteId, event, goal);
         matchedGoalKeys.push(goal.key);
     }
     for (const funnel of input.funnels) {
         if (funnel.active) {
-            await advanceFunnel(ctx, input.siteId, funnel, event, input.batchState);
+            await advanceFunnel(ctx, input.siteId, funnel, event, input.reportRollupDeltas, input.batchState);
         }
     }
     if (input.financialConversion) {
-        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey);
+        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey, input.reportRollupDeltas);
     }
     const dimensions = [
         { type: "source", value: session.source },
         { type: "page", value: event.path },
-        { type: "country", value: input.context.country ?? "unknown" },
+        { type: "country", value: session.country ?? "unknown" },
         { type: "device", value: input.context.device ?? "unknown" },
         { type: "browser", value: input.context.browser ?? "Other" },
         {
@@ -421,6 +432,7 @@ async function accountEvent(ctx, input) {
         event.timestamp >= input.now - LIVE_SESSION_TTL_MS) {
         await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now, input.batchState);
     }
+    return session;
 }
 async function accountTrustedSession(ctx, existing, event) {
     if (!existing) {
@@ -438,6 +450,10 @@ async function accountTrustedSession(ctx, existing, event) {
         isNewVisitorDay: false,
         startedAt: existing.startedAt,
         source: existing.source,
+        country: existing.country,
+        city: existing.city,
+        latitude: existing.latitude,
+        longitude: existing.longitude,
         pageviewCount: existing.pageviewCount,
         eventCount: existing.eventCount,
         affiliateId: existing.affiliateId,
@@ -446,13 +462,16 @@ async function accountTrustedSession(ctx, existing, event) {
         affiliateExpiresAt: existing.affiliateExpiresAt,
     };
 }
-async function upsertSession(ctx, siteId, event, context, existing, financialConversion, batchState) {
+async function upsertSession(ctx, siteId, event, context, existing, financialConversion, reportRollupDeltas, batchState) {
     const source = sourceFromReferrer(event.referrer);
     const utmSource = stringProperty(event.properties, "utm_source");
     const utmMedium = stringProperty(event.properties, "utm_medium");
     const utmCampaign = stringProperty(event.properties, "utm_campaign");
     const attribution = await attributionForEvent(ctx, siteId, event, existing, batchState);
     if (existing) {
+        if (attribution && existing.affiliateId !== attribution.affiliateId) {
+            foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution);
+        }
         const startedAt = Math.min(existing.startedAt, event.timestamp);
         const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
         const updates = {
@@ -499,6 +518,10 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
             startedAt,
             source: existing.source,
+            country: context.country ?? existing.country,
+            city: context.city ?? existing.city,
+            latitude: context.latitude ?? existing.latitude,
+            longitude: context.longitude ?? existing.longitude,
             pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
             eventCount: existing.eventCount + 1,
             ...(attribution ?? {}),
@@ -551,6 +574,9 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         ...(attribution ?? {}),
     };
     const sessionId = await ctx.db.insert("sessions", fields);
+    if (attribution) {
+        foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution);
+    }
     if (batchState) {
         const state = batchState.sessions.get(event.sessionId);
         if (!state)
@@ -570,6 +596,10 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
         startedAt: event.timestamp,
         source,
+        country: context.country,
+        city: context.city,
+        latitude: context.latitude,
+        longitude: context.longitude,
         pageviewCount: event.type === "pageview" ? 1 : 0,
         eventCount: 1,
         ...(attribution ?? {}),
@@ -717,20 +747,21 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         isNewVisitorDay: false,
         startedAt,
         source: session.source,
+        country: context.country ?? session.country,
+        city: context.city ?? session.city,
+        latitude: context.latitude ?? session.latitude,
+        longitude: context.longitude ?? session.longitude,
         pageviewCount: session.pageviewCount,
         eventCount: session.eventCount,
         affiliateSlug: session.affiliateSlug,
     }, now, batchState);
 }
-async function consumeSiteRateLimit(ctx, siteId, events, now) {
+async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
     const increments = new Map();
-    const eventByteCounts = [];
-    const encoder = new TextEncoder();
     for (let index = 0; index < events.length; index += 1) {
         const event = events[index];
-        const byteCount = encoder.encode(JSON.stringify(event)).byteLength;
-        eventByteCounts.push(byteCount);
+        const byteCount = eventByteCounts[index];
         const shard = stableHash(event.sessionId) % SITE_INGEST_SHARDS;
         const current = increments.get(shard) ?? { eventCount: 0, byteCount: 0 };
         current.eventCount += 1;
@@ -787,7 +818,6 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
             });
         }
     }
-    return eventByteCounts;
 }
 async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now, states) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -855,6 +885,10 @@ async function updateLiveSession(ctx, siteId, event, context, session, now, batc
             .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
             .unique();
     const deadline = now + LIVE_SESSION_TTL_MS;
+    const country = context.country ?? session.country;
+    const city = context.city ?? session.city;
+    const latitude = context.latitude ?? session.latitude;
+    const longitude = context.longitude ?? session.longitude;
     if (existing) {
         const recentHeartbeatEventIds = event.type === "heartbeat"
             ? [
@@ -870,10 +904,10 @@ async function updateLiveSession(ctx, siteId, event, context, session, now, batc
             path: event.path,
             lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
             expiresAt: deadline,
-            ...(context.country ? { country: context.country } : {}),
-            ...(context.city ? { city: context.city } : {}),
-            ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            ...(country ? { country } : {}),
+            ...(city ? { city } : {}),
+            ...(latitude !== undefined ? { latitude } : {}),
+            ...(longitude !== undefined ? { longitude } : {}),
             browser: context.browser ?? existing.browser,
             os: context.os ?? existing.os,
             source: session.source,
@@ -904,10 +938,10 @@ async function updateLiveSession(ctx, siteId, event, context, session, now, batc
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         expiresAt: deadline,
-        ...(context.country ? { country: context.country } : {}),
-        ...(context.city ? { city: context.city } : {}),
-        ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(country ? { country } : {}),
+        ...(city ? { city } : {}),
+        ...(latitude !== undefined ? { latitude } : {}),
+        ...(longitude !== undefined ? { longitude } : {}),
         browser: context.browser ?? "Other",
         os: context.os ?? "Other",
         source: session.source,
@@ -937,9 +971,14 @@ async function ensureLiveSweep(ctx, siteId, now, batchState) {
         .query("liveSweepStates")
         .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
         .unique();
-    if (!existing) {
+    if (!existing || existing.scheduledAt <= now) {
         const scheduledAt = now + LIVE_SWEEP_INTERVAL_MS;
-        await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        if (existing) {
+            await ctx.db.patch("liveSweepStates", existing._id, { scheduledAt });
+        }
+        else {
+            await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        }
         await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
             siteId,
             scheduledAt,
@@ -953,7 +992,7 @@ function goalMatches(goal, event) {
     }
     return event.type === "custom" && event.name === goal.matchValue;
 }
-async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
+async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, batchState) {
     const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
     let progressState = batchState?.funnelProgress.get(cacheKey);
     if (batchState && !progressState) {
@@ -997,6 +1036,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
             };
         }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
     if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt)
@@ -1026,6 +1066,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
             });
         }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
     const nextStep = funnel.steps[progress.currentStep];
@@ -1047,6 +1088,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
         await ctx.db.patch("funnelProgress", progress._id, updates);
     }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
+    foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, progress.currentStep, completed);
 }
 function stepMatches(step, event) {
     return step.type === "pageview"
@@ -1083,7 +1125,7 @@ async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed
         });
     }
 }
-async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
+async function accountConversion(ctx, siteId, session, event, trustedGoalKey, reportRollupDeltas) {
     const revenueCents = event.revenueCents ?? 0;
     const currency = event.currency ?? "USD";
     const candidate = attributionSnapshot(session);
@@ -1112,6 +1154,7 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         commissionCents,
     });
     if (commissionAffiliate && attribution) {
+        foldAffiliateConversionRollup(reportRollupDeltas, siteId, event, commissionAffiliate, attribution.affiliateSlug, revenueCents, commissionCents);
         const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
         const shard = stableHash(event.sessionId) % SECONDARY_AGGREGATE_SHARDS;
         const stats = await ctx.db
@@ -1143,6 +1186,188 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         }
     }
 }
+function createReportRollupDeltas() {
+    return {
+        goals: new Map(),
+        funnels: new Map(),
+        affiliates: new Map(),
+    };
+}
+function reportRollupCoordinates(siteId, definitionId, timestamp, visitorId) {
+    const bucketStart = Math.floor(timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(visitorId) % REPORT_ROLLUP_SHARDS;
+    return {
+        bucketStart,
+        shard,
+        key: `${siteId}:${definitionId}:${bucketStart}:${shard}`,
+    };
+}
+function foldGoalRollup(deltas, siteId, event, goal) {
+    const coordinates = reportRollupCoordinates(siteId, String(goal._id), event.timestamp, event.visitorId);
+    const current = deltas.goals.get(coordinates.key) ?? {
+        siteId,
+        goalId: goal._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        goalKey: goal.key,
+        completions: 0,
+        valueCents: 0,
+    };
+    current.completions += 1;
+    current.valueCents += goal.valueCents ?? 0;
+    deltas.goals.set(coordinates.key, current);
+}
+function foldFunnelRollup(deltas, siteId, funnelId, event, step, completed) {
+    const coordinates = reportRollupCoordinates(siteId, String(funnelId), event.timestamp, event.visitorId);
+    const rollup = deltas.funnels.get(coordinates.key) ?? {
+        siteId,
+        funnelId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        steps: new Map(),
+    };
+    const current = rollup.steps.get(step) ?? {
+        step,
+        entrants: 0,
+        completions: 0,
+    };
+    current.entrants += 1;
+    current.completions += Number(completed);
+    rollup.steps.set(step, current);
+    deltas.funnels.set(coordinates.key, rollup);
+}
+function foldAffiliateVisitRollup(deltas, siteId, event, attribution) {
+    const coordinates = reportRollupCoordinates(siteId, String(attribution.affiliateId), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: attribution.affiliateId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug: attribution.affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.visits += 1;
+    current.visitorSketch = addVisitor(current.visitorSketch, event.visitorId);
+    deltas.affiliates.set(coordinates.key, current);
+}
+function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affiliateSlug, revenueCents, commissionCents) {
+    const coordinates = reportRollupCoordinates(siteId, String(affiliate._id), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: affiliate._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.conversions += 1;
+    current.revenueCents += revenueCents;
+    current.commissionCents += commissionCents;
+    deltas.affiliates.set(coordinates.key, current);
+}
+async function flushReportRollups(ctx, deltas) {
+    for (const delta of deltas.goals.values()) {
+        const existing = await ctx.db
+            .query("goalDailyRollups")
+            .withIndex("by_siteId_and_goalId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("goalId", delta.goalId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            goalId: delta.goalId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            goalKey: delta.goalKey,
+            completions: (existing?.completions ?? 0) + delta.completions,
+            valueCents: (existing?.valueCents ?? 0) + delta.valueCents,
+        };
+        if (existing) {
+            await ctx.db.replace("goalDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("goalDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.funnels.values()) {
+        const existing = await ctx.db
+            .query("funnelDailyRollups")
+            .withIndex("by_siteId_and_funnelId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("funnelId", delta.funnelId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const steps = existing?.steps.map((step) => ({ ...step })) ?? [];
+        for (const addition of delta.steps.values()) {
+            const current = steps.find((step) => step.step === addition.step);
+            if (current) {
+                current.entrants += addition.entrants;
+                current.completions += addition.completions;
+            }
+            else {
+                if (steps.length >= MAX_FUNNEL_STEPS) {
+                    fail("LIMIT_EXCEEDED", "daily funnel rollup step limit exceeded");
+                }
+                steps.push(addition);
+            }
+        }
+        const fields = {
+            siteId: delta.siteId,
+            funnelId: delta.funnelId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            steps,
+        };
+        if (existing) {
+            await ctx.db.replace("funnelDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("funnelDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.affiliates.values()) {
+        const existing = await ctx.db
+            .query("affiliateDailyRollups")
+            .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("affiliateId", delta.affiliateId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            affiliateId: delta.affiliateId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            affiliateSlug: delta.affiliateSlug,
+            visits: (existing?.visits ?? 0) + delta.visits,
+            visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
+            conversions: (existing?.conversions ?? 0) + delta.conversions,
+            revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
+            commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
+        };
+        if (existing) {
+            await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("affiliateDailyRollups", fields);
+        }
+    }
+}
 function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
     const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
@@ -1170,6 +1395,7 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 outboundClicks: 0,
                 sessions: 0,
                 visitors: 0,
+                visitorSketch: createVisitorSketch(),
                 conversions: 0,
                 revenueCents: 0,
                 dimensions: [],
@@ -1188,6 +1414,9 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 ? Number(session.isNewVisitorHour)
                 : Number(session.isNewVisitorDay)
             : 0;
+        if (recordTelemetry) {
+            delta.visitorSketch = addVisitor(delta.visitorSketch, event.visitorId);
+        }
         delta.conversions += Number(financialConversion);
         delta.revenueCents += revenueCents;
         delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
@@ -1204,6 +1433,13 @@ async function flushAggregates(ctx, aggregateDeltas) {
             .eq("shard", delta.shard))
             .unique();
         const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions);
+        const priorSketch = bucket?.visitorSketch;
+        const visitorSketch = priorSketch
+            ? unionVisitorSketches(priorSketch, delta.visitorSketch)
+            : delta.visitorSketch;
+        const visitorSketchComplete = bucket
+            ? bucket.visitorSketchComplete === true && priorSketch !== undefined
+            : true;
         if (bucket) {
             await ctx.db.patch("aggregateBuckets", bucket._id, {
                 events: bucket.events + delta.events,
@@ -1213,6 +1449,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: bucket.outboundClicks + delta.outboundClicks,
                 sessions: bucket.sessions + delta.sessions,
                 visitors: bucket.visitors + delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: bucket.conversions + delta.conversions,
                 revenueCents: bucket.revenueCents + delta.revenueCents,
                 dimensions,
@@ -1231,6 +1469,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: delta.outboundClicks,
                 sessions: delta.sessions,
                 visitors: delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: delta.conversions,
                 revenueCents: delta.revenueCents,
                 dimensions,