@iann29/rastro 0.1.0-alpha.3 → 0.1.0-alpha.5

package/dist/component/ingest.js

@@ -1,10 +1,12 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
 import { fail } from "./errors.js";
+import { ensureAnalyticsControl } from "./coverage.js";
 import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
-import { cleanString, ensureBatchBytes, isValidMoney, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { cleanString, isValidMoney, measureBatchBytes, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
 export const ingestBatch = mutation({
     args: {
@@ -28,16 +30,19 @@ export const ingestBatch = mutation({
                 limit: MAX_BATCH_EVENTS,
             });
         }
+        let eventByteCounts;
         try {
-            ensureBatchBytes({ siteId: String(args.siteId), events: args.events });
+            ({ eventByteCounts } = measureBatchBytes(String(args.siteId), args.events));
         }
         catch {
             fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
         }
         const now = Date.now();
-        const eventByteCounts = await consumeSiteRateLimit(ctx, args.siteId, args.events, now);
+        await ensureAnalyticsControl(ctx, args.siteId, now);
+        await consumeSiteRateLimit(ctx, args.siteId, args.events, eventByteCounts, now);
         const context = sanitizeContext(args.context);
         const aggregateDeltas = new Map();
+        const reportRollupDeltas = createReportRollupDeltas();
         const sessionRateWindows = new Map();
         const batchState = {
             sessions: new Map(),
@@ -171,6 +176,7 @@ export const ingestBatch = mutation({
                 goals: canMatchDefinition ? goals ?? [] : [],
                 funnels: canMatchDefinition ? funnels ?? [] : [],
                 aggregateDeltas,
+                reportRollupDeltas,
                 recordTelemetry: true,
                 financialConversion: false,
                 trustedGoalKey: undefined,
@@ -183,6 +189,7 @@ export const ingestBatch = mutation({
         await flushBatchState(ctx, batchState);
         await flushSessionRateLimits(ctx, sessionRateWindows, now);
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
         return { accepted, duplicates, rejected };
     },
 });
@@ -260,6 +267,7 @@ export const trackConversion = mutation({
             device: existingTelemetry?.device ?? existingSession?.device,
         };
         const aggregateDeltas = new Map();
+        const reportRollupDeltas = createReportRollupDeltas();
         const session = await accountEvent(ctx, {
             siteId: args.siteId,
             event,
@@ -268,6 +276,7 @@ export const trackConversion = mutation({
             goals: [],
             funnels: [],
             aggregateDeltas,
+            reportRollupDeltas,
             recordTelemetry: existingTelemetry === null && !telemetry.saturated,
             financialConversion: true,
             trustedGoalKey,
@@ -277,6 +286,8 @@ export const trackConversion = mutation({
             await insertEventBatch(ctx, args.siteId, event, session.country ?? "unknown");
         }
         await flushAggregates(ctx, aggregateDeltas);
+        await flushReportRollups(ctx, reportRollupDeltas);
+        await ensureAnalyticsControl(ctx, args.siteId, Date.now());
         return { accepted: 1, duplicates: 0, rejected: 0 };
     },
 });
@@ -346,7 +357,7 @@ function withoutSystemFields(document) {
 async function accountEvent(ctx, input) {
     const { event } = input;
     const session = input.recordTelemetry
-        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.batchState)
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.reportRollupDeltas, input.batchState)
         : await accountTrustedSession(ctx, input.existingSession, event);
     const matchedGoalKeys = [];
     const matchingGoals = input.goals.filter((goal) => goal.active && goalMatches(goal, event));
@@ -393,18 +404,23 @@ async function accountEvent(ctx, input) {
             timestamp: event.timestamp,
             valueCents: goal.valueCents ?? 0,
         });
+        foldGoalRollup(input.reportRollupDeltas, input.siteId, event, goal);
         matchedGoalKeys.push(goal.key);
     }
     for (const funnel of input.funnels) {
         if (funnel.active) {
-            await advanceFunnel(ctx, input.siteId, funnel, event, input.batchState);
+            await advanceFunnel(ctx, input.siteId, funnel, event, input.reportRollupDeltas, input.batchState);
         }
     }
     if (input.financialConversion) {
-        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey);
+        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey, input.reportRollupDeltas);
     }
+    const campaignDimensions = session.utmCampaign
+        ? [{ type: "campaign", value: session.utmCampaign }]
+        : [];
     const dimensions = [
         { type: "source", value: session.source },
+        ...campaignDimensions,
         { type: "page", value: event.path },
         { type: "country", value: session.country ?? "unknown" },
         { type: "device", value: input.context.device ?? "unknown" },
@@ -438,6 +454,8 @@ async function accountTrustedSession(ctx, existing, event) {
         isNewVisitorDay: false,
         startedAt: existing.startedAt,
         source: existing.source,
+        utmCampaign: existing.utmCampaign,
+        visitorKey: existing.visitorKey ?? existing.visitorId,
         country: existing.country,
         city: existing.city,
         latitude: existing.latitude,
@@ -450,13 +468,23 @@ async function accountTrustedSession(ctx, existing, event) {
         affiliateExpiresAt: existing.affiliateExpiresAt,
     };
 }
-async function upsertSession(ctx, siteId, event, context, existing, financialConversion, batchState) {
-    const source = sourceFromReferrer(event.referrer);
+async function upsertSession(ctx, siteId, event, context, existing, financialConversion, reportRollupDeltas, batchState) {
+    // A host-supplied id is the identity; anonymous sessions count through the
+    // daily key derived by the HTTP action, or once per session without one.
+    const visitorKey = existing?.visitorKey ??
+        (event.visitorId !== event.sessionId
+            ? event.visitorId
+            : (context.visitorKey ?? event.sessionId));
     const utmSource = stringProperty(event.properties, "utm_source");
+    // Campaign traffic names its own source; the referrer host is the fallback.
+    const source = utmSource?.toLowerCase() ?? sourceFromReferrer(event.referrer);
     const utmMedium = stringProperty(event.properties, "utm_medium");
     const utmCampaign = stringProperty(event.properties, "utm_campaign");
     const attribution = await attributionForEvent(ctx, siteId, event, existing, batchState);
     if (existing) {
+        if (attribution && existing.affiliateId !== attribution.affiliateId) {
+            foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
+        }
         const startedAt = Math.min(existing.startedAt, event.timestamp);
         const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
         const updates = {
@@ -503,6 +531,8 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
             startedAt,
             source: existing.source,
+            utmCampaign: existing.utmCampaign,
+            visitorKey,
             country: context.country ?? existing.country,
             city: context.city ?? existing.city,
             latitude: context.latitude ?? existing.latitude,
@@ -532,6 +562,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
+        visitorKey,
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         entryPath: event.path,
@@ -559,6 +590,9 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         ...(attribution ?? {}),
     };
     const sessionId = await ctx.db.insert("sessions", fields);
+    if (attribution) {
+        foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
+    }
     if (batchState) {
         const state = batchState.sessions.get(event.sessionId);
         if (!state)
@@ -578,6 +612,8 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
         startedAt: event.timestamp,
         source,
+        utmCampaign,
+        visitorKey,
         country: context.country,
         city: context.city,
         latitude: context.latitude,
@@ -729,6 +765,7 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         isNewVisitorDay: false,
         startedAt,
         source: session.source,
+        visitorKey: session.visitorKey ?? session.visitorId,
         country: context.country ?? session.country,
         city: context.city ?? session.city,
         latitude: context.latitude ?? session.latitude,
@@ -738,15 +775,12 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         affiliateSlug: session.affiliateSlug,
     }, now, batchState);
 }
-async function consumeSiteRateLimit(ctx, siteId, events, now) {
+async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
     const increments = new Map();
-    const eventByteCounts = [];
-    const encoder = new TextEncoder();
     for (let index = 0; index < events.length; index += 1) {
         const event = events[index];
-        const byteCount = encoder.encode(JSON.stringify(event)).byteLength;
-        eventByteCounts.push(byteCount);
+        const byteCount = eventByteCounts[index];
         const shard = stableHash(event.sessionId) % SITE_INGEST_SHARDS;
         const current = increments.get(shard) ?? { eventCount: 0, byteCount: 0 };
         current.eventCount += 1;
@@ -803,7 +837,6 @@ async function consumeSiteRateLimit(ctx, siteId, events, now) {
             });
         }
     }
-    return eventByteCounts;
 }
 async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now, states) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -957,9 +990,14 @@ async function ensureLiveSweep(ctx, siteId, now, batchState) {
         .query("liveSweepStates")
         .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
         .unique();
-    if (!existing) {
+    if (!existing || existing.scheduledAt <= now) {
         const scheduledAt = now + LIVE_SWEEP_INTERVAL_MS;
-        await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        if (existing) {
+            await ctx.db.patch("liveSweepStates", existing._id, { scheduledAt });
+        }
+        else {
+            await ctx.db.insert("liveSweepStates", { siteId, scheduledAt });
+        }
         await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
             siteId,
             scheduledAt,
@@ -973,7 +1011,7 @@ function goalMatches(goal, event) {
     }
     return event.type === "custom" && event.name === goal.matchValue;
 }
-async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
+async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, batchState) {
     const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
     let progressState = batchState?.funnelProgress.get(cacheKey);
     if (batchState && !progressState) {
@@ -1017,6 +1055,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
             };
         }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
     if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt)
@@ -1046,6 +1085,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
             });
         }
         await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
     const nextStep = funnel.steps[progress.currentStep];
@@ -1067,6 +1107,7 @@ async function advanceFunnel(ctx, siteId, funnel, event, batchState) {
         await ctx.db.patch("funnelProgress", progress._id, updates);
     }
     await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
+    foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, progress.currentStep, completed);
 }
 function stepMatches(step, event) {
     return step.type === "pageview"
@@ -1103,7 +1144,7 @@ async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed
         });
     }
 }
-async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
+async function accountConversion(ctx, siteId, session, event, trustedGoalKey, reportRollupDeltas) {
     const revenueCents = event.revenueCents ?? 0;
     const currency = event.currency ?? "USD";
     const candidate = attributionSnapshot(session);
@@ -1132,6 +1173,7 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         commissionCents,
     });
     if (commissionAffiliate && attribution) {
+        foldAffiliateConversionRollup(reportRollupDeltas, siteId, event, commissionAffiliate, attribution.affiliateSlug, revenueCents, commissionCents);
         const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
         const shard = stableHash(event.sessionId) % SECONDARY_AGGREGATE_SHARDS;
         const stats = await ctx.db
@@ -1163,6 +1205,188 @@ async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
         }
     }
 }
+function createReportRollupDeltas() {
+    return {
+        goals: new Map(),
+        funnels: new Map(),
+        affiliates: new Map(),
+    };
+}
+function reportRollupCoordinates(siteId, definitionId, timestamp, visitorId) {
+    const bucketStart = Math.floor(timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(visitorId) % REPORT_ROLLUP_SHARDS;
+    return {
+        bucketStart,
+        shard,
+        key: `${siteId}:${definitionId}:${bucketStart}:${shard}`,
+    };
+}
+function foldGoalRollup(deltas, siteId, event, goal) {
+    const coordinates = reportRollupCoordinates(siteId, String(goal._id), event.timestamp, event.visitorId);
+    const current = deltas.goals.get(coordinates.key) ?? {
+        siteId,
+        goalId: goal._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        goalKey: goal.key,
+        completions: 0,
+        valueCents: 0,
+    };
+    current.completions += 1;
+    current.valueCents += goal.valueCents ?? 0;
+    deltas.goals.set(coordinates.key, current);
+}
+function foldFunnelRollup(deltas, siteId, funnelId, event, step, completed) {
+    const coordinates = reportRollupCoordinates(siteId, String(funnelId), event.timestamp, event.visitorId);
+    const rollup = deltas.funnels.get(coordinates.key) ?? {
+        siteId,
+        funnelId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        steps: new Map(),
+    };
+    const current = rollup.steps.get(step) ?? {
+        step,
+        entrants: 0,
+        completions: 0,
+    };
+    current.entrants += 1;
+    current.completions += Number(completed);
+    rollup.steps.set(step, current);
+    deltas.funnels.set(coordinates.key, rollup);
+}
+function foldAffiliateVisitRollup(deltas, siteId, event, attribution, visitorKey) {
+    const coordinates = reportRollupCoordinates(siteId, String(attribution.affiliateId), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: attribution.affiliateId,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug: attribution.affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.visits += 1;
+    current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
+    deltas.affiliates.set(coordinates.key, current);
+}
+function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affiliateSlug, revenueCents, commissionCents) {
+    const coordinates = reportRollupCoordinates(siteId, String(affiliate._id), event.timestamp, event.visitorId);
+    const current = deltas.affiliates.get(coordinates.key) ?? {
+        siteId,
+        affiliateId: affiliate._id,
+        bucketStart: coordinates.bucketStart,
+        shard: coordinates.shard,
+        affiliateSlug,
+        visits: 0,
+        visitorSketch: createVisitorSketch(),
+        conversions: 0,
+        revenueCents: 0,
+        commissionCents: 0,
+    };
+    current.conversions += 1;
+    current.revenueCents += revenueCents;
+    current.commissionCents += commissionCents;
+    deltas.affiliates.set(coordinates.key, current);
+}
+async function flushReportRollups(ctx, deltas) {
+    for (const delta of deltas.goals.values()) {
+        const existing = await ctx.db
+            .query("goalDailyRollups")
+            .withIndex("by_siteId_and_goalId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("goalId", delta.goalId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            goalId: delta.goalId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            goalKey: delta.goalKey,
+            completions: (existing?.completions ?? 0) + delta.completions,
+            valueCents: (existing?.valueCents ?? 0) + delta.valueCents,
+        };
+        if (existing) {
+            await ctx.db.replace("goalDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("goalDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.funnels.values()) {
+        const existing = await ctx.db
+            .query("funnelDailyRollups")
+            .withIndex("by_siteId_and_funnelId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("funnelId", delta.funnelId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const steps = existing?.steps.map((step) => ({ ...step })) ?? [];
+        for (const addition of delta.steps.values()) {
+            const current = steps.find((step) => step.step === addition.step);
+            if (current) {
+                current.entrants += addition.entrants;
+                current.completions += addition.completions;
+            }
+            else {
+                if (steps.length >= MAX_FUNNEL_STEPS) {
+                    fail("LIMIT_EXCEEDED", "daily funnel rollup step limit exceeded");
+                }
+                steps.push(addition);
+            }
+        }
+        const fields = {
+            siteId: delta.siteId,
+            funnelId: delta.funnelId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            steps,
+        };
+        if (existing) {
+            await ctx.db.replace("funnelDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("funnelDailyRollups", fields);
+        }
+    }
+    for (const delta of deltas.affiliates.values()) {
+        const existing = await ctx.db
+            .query("affiliateDailyRollups")
+            .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("affiliateId", delta.affiliateId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const fields = {
+            siteId: delta.siteId,
+            affiliateId: delta.affiliateId,
+            bucketStart: delta.bucketStart,
+            shard: delta.shard,
+            generation: 1,
+            affiliateSlug: delta.affiliateSlug,
+            visits: (existing?.visits ?? 0) + delta.visits,
+            visitorSketch: unionVisitorSketches(existing?.visitorSketch ?? createVisitorSketch(), delta.visitorSketch),
+            conversions: (existing?.conversions ?? 0) + delta.conversions,
+            revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
+            commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
+        };
+        if (existing) {
+            await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
+        }
+        else {
+            await ctx.db.insert("affiliateDailyRollups", fields);
+        }
+    }
+}
 function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
     const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
@@ -1190,6 +1414,7 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 outboundClicks: 0,
                 sessions: 0,
                 visitors: 0,
+                visitorSketch: createVisitorSketch(),
                 conversions: 0,
                 revenueCents: 0,
                 dimensions: [],
@@ -1208,6 +1433,9 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 ? Number(session.isNewVisitorHour)
                 : Number(session.isNewVisitorDay)
             : 0;
+        if (recordTelemetry) {
+            delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+        }
         delta.conversions += Number(financialConversion);
         delta.revenueCents += revenueCents;
         delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
@@ -1224,6 +1452,13 @@ async function flushAggregates(ctx, aggregateDeltas) {
             .eq("shard", delta.shard))
             .unique();
         const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions);
+        const priorSketch = bucket?.visitorSketch;
+        const visitorSketch = priorSketch
+            ? unionVisitorSketches(priorSketch, delta.visitorSketch)
+            : delta.visitorSketch;
+        const visitorSketchComplete = bucket
+            ? bucket.visitorSketchComplete === true && priorSketch !== undefined
+            : true;
         if (bucket) {
             await ctx.db.patch("aggregateBuckets", bucket._id, {
                 events: bucket.events + delta.events,
@@ -1233,6 +1468,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: bucket.outboundClicks + delta.outboundClicks,
                 sessions: bucket.sessions + delta.sessions,
                 visitors: bucket.visitors + delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: bucket.conversions + delta.conversions,
                 revenueCents: bucket.revenueCents + delta.revenueCents,
                 dimensions,
@@ -1251,6 +1488,8 @@ async function flushAggregates(ctx, aggregateDeltas) {
                 outboundClicks: delta.outboundClicks,
                 sessions: delta.sessions,
                 visitors: delta.visitors,
+                visitorSketch,
+                visitorSketchComplete,
                 conversions: delta.conversions,
                 revenueCents: delta.revenueCents,
                 dimensions,