@iann29/rastro 0.1.0-alpha.0

package/dist/component/ingest.js

@@ -0,0 +1,991 @@
+import { v } from "convex/values";
+import { mutation } from "./_generated/server.js";
+import { internal } from "./_generated/api.js";
+import { AGGREGATE_SHARDS, DAY_MS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { fail } from "./errors.js";
+import { cleanString, ensureBatchBytes, isValidMoney, originAllowed, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
+export const ingestBatch = mutation({
+    args: {
+        siteId: v.id("sites"),
+        origin: v.string(),
+        events: v.array(trackerEventValidator),
+        context: v.optional(ingestContextValidator),
+    },
+    returns: ingestResultValidator,
+    handler: async (ctx, args) => {
+        const site = await ctx.db.get("sites", args.siteId);
+        if (!site)
+            fail("NOT_FOUND", "site not found");
+        if (!originAllowed(args.origin, site.domains)) {
+            fail("ORIGIN_NOT_ALLOWED", "request origin is not allowed", {
+                siteId: String(args.siteId),
+            });
+        }
+        if (args.events.length < 1 || args.events.length > MAX_BATCH_EVENTS) {
+            fail("BATCH_TOO_LARGE", `events must contain 1-${MAX_BATCH_EVENTS} items`, {
+                limit: MAX_BATCH_EVENTS,
+            });
+        }
+        try {
+            ensureBatchBytes({ siteId: String(args.siteId), events: args.events });
+        }
+        catch {
+            fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
+        }
+        const now = Date.now();
+        const eventByteCounts = await consumeSiteRateLimit(ctx, args.siteId, args.events, now);
+        const context = sanitizeContext(args.context);
+        const aggregateDeltas = new Map();
+        let goals = null;
+        let funnels = null;
+        let accepted = 0;
+        let duplicates = 0;
+        let rejected = 0;
+        for (let eventIndex = 0; eventIndex < args.events.length; eventIndex += 1) {
+            const input = args.events[eventIndex];
+            let event;
+            try {
+                event = sanitizeEvent(input, now);
+            }
+            catch {
+                rejected += 1;
+                continue;
+            }
+            const existingSession = await ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
+                .unique();
+            if (existingSession && existingSession.visitorId !== event.visitorId) {
+                rejected += 1;
+                continue;
+            }
+            if (event.type === "heartbeat") {
+                if (!existingSession || event.timestamp < now - LIVE_SESSION_TTL_MS) {
+                    rejected += 1;
+                    continue;
+                }
+                const liveSession = await ctx.db
+                    .query("liveSessions")
+                    .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
+                    .unique();
+                if (liveSession?.lastHeartbeatEventId === event.eventId) {
+                    duplicates += 1;
+                    continue;
+                }
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+                    rejected += 1;
+                    continue;
+                }
+                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now);
+                accepted += 1;
+                continue;
+            }
+            const duplicate = await ctx.db
+                .query("events")
+                .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
+                .unique();
+            if (duplicate) {
+                duplicates += 1;
+                continue;
+            }
+            if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now))) {
+                rejected += 1;
+                continue;
+            }
+            const canMatchDefinition = event.type === "pageview" || event.type === "custom";
+            if (canMatchDefinition && goals === null) {
+                goals = await ctx.db
+                    .query("goals")
+                    .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+                    .take(MAX_GOALS_PER_SITE);
+            }
+            if (canMatchDefinition && funnels === null) {
+                funnels = await ctx.db
+                    .query("funnels")
+                    .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+                    .take(MAX_FUNNELS_PER_SITE);
+            }
+            await accountEvent(ctx, {
+                siteId: args.siteId,
+                event,
+                context,
+                existingSession,
+                goals: canMatchDefinition ? goals ?? [] : [],
+                funnels: canMatchDefinition ? funnels ?? [] : [],
+                aggregateDeltas,
+                recordTelemetry: true,
+                financialConversion: false,
+                trustedGoalKey: undefined,
+                now,
+            });
+            accepted += 1;
+        }
+        await flushAggregates(ctx, aggregateDeltas);
+        return { accepted, duplicates, rejected };
+    },
+});
+export const trackConversion = mutation({
+    args: trustedConversionValidator.fields,
+    returns: ingestResultValidator,
+    handler: async (ctx, args) => {
+        const site = await ctx.db.get("sites", args.siteId);
+        if (!site)
+            fail("NOT_FOUND", "site not found");
+        if (!isValidMoney(args.revenueCents)) {
+            fail("INVALID_ARGUMENT", "revenueCents must be an integer between 0 and 1000000000");
+        }
+        let event;
+        try {
+            event = sanitizeEvent({
+                eventId: sanitizeOpaqueId(args.eventId, "eventId"),
+                sessionId: sanitizeOpaqueId(args.sessionId, "sessionId"),
+                visitorId: sanitizeOpaqueId(args.visitorId, "visitorId"),
+                type: "conversion",
+                name: args.goalKey ? sanitizeSlug(args.goalKey, "goalKey") : "conversion",
+                path: sanitizePath(args.path ?? "/conversion"),
+                timestamp: args.timestamp,
+                sequence: 0,
+                properties: sanitizeProperties(args.properties),
+                revenueCents: args.revenueCents,
+                currency: sanitizeCurrency(args.currency),
+            }, Date.now());
+        }
+        catch (error) {
+            fail("INVALID_ARGUMENT", error instanceof Error ? error.message : "invalid conversion");
+        }
+        const siteCurrency = site.currency ?? "USD";
+        if (event.currency !== siteCurrency) {
+            fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
+                siteCurrency,
+                conversionCurrency: event.currency ?? "USD",
+            });
+        }
+        const priorConversion = await ctx.db
+            .query("conversions")
+            .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
+            .unique();
+        if (priorConversion) {
+            return { accepted: 0, duplicates: 1, rejected: 0 };
+        }
+        const existingTelemetry = await ctx.db
+            .query("events")
+            .withIndex("by_siteId_and_eventId", (range) => range.eq("siteId", args.siteId).eq("eventId", event.eventId))
+            .unique();
+        if (existingTelemetry &&
+            (existingTelemetry.sessionId !== event.sessionId ||
+                existingTelemetry.visitorId !== event.visitorId)) {
+            fail("CONFLICT", "event belongs to a different session or visitor");
+        }
+        const existingSession = await ctx.db
+            .query("sessions")
+            .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", args.siteId).eq("sessionId", event.sessionId))
+            .unique();
+        if (existingSession && existingSession.visitorId !== event.visitorId) {
+            fail("CONFLICT", "session belongs to a different visitor");
+        }
+        const conversionContext = existingTelemetry
+            ? {
+                country: existingTelemetry.country,
+                city: existingTelemetry.city,
+                latitude: existingTelemetry.latitude,
+                longitude: existingTelemetry.longitude,
+                browser: existingTelemetry.browser,
+                os: existingTelemetry.os,
+                device: existingTelemetry.device,
+            }
+            : sanitizeContext(undefined);
+        const aggregateDeltas = new Map();
+        await accountEvent(ctx, {
+            siteId: args.siteId,
+            event,
+            context: conversionContext,
+            existingSession,
+            goals: [],
+            funnels: [],
+            aggregateDeltas,
+            recordTelemetry: existingTelemetry === null,
+            financialConversion: true,
+            trustedGoalKey: args.goalKey
+                ? sanitizeSlug(args.goalKey, "goalKey")
+                : undefined,
+            now: Date.now(),
+        });
+        await flushAggregates(ctx, aggregateDeltas);
+        return { accepted: 1, duplicates: 0, rejected: 0 };
+    },
+});
+async function accountEvent(ctx, input) {
+    const { event } = input;
+    const session = input.recordTelemetry
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion)
+        : await accountTrustedSession(ctx, input.existingSession, event);
+    if (input.recordTelemetry) {
+        await ctx.db.insert("events", {
+            siteId: input.siteId,
+            eventId: event.eventId,
+            sessionId: event.sessionId,
+            visitorId: event.visitorId,
+            type: event.type,
+            ...(event.name ? { name: event.name } : {}),
+            path: event.path,
+            ...(event.referrer ? { referrer: event.referrer } : {}),
+            timestamp: event.timestamp,
+            sequence: event.sequence,
+            ...(event.target ? { target: event.target } : {}),
+            ...(event.href ? { href: event.href } : {}),
+            ...(event.properties ? { properties: event.properties } : {}),
+            ...(event.revenueCents !== undefined
+                ? { revenueCents: event.revenueCents }
+                : {}),
+            ...(event.currency ? { currency: event.currency } : {}),
+            ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+            ...(input.context.country ? { country: input.context.country } : {}),
+            ...(input.context.city ? { city: input.context.city } : {}),
+            ...(input.context.latitude !== undefined
+                ? { latitude: input.context.latitude }
+                : {}),
+            ...(input.context.longitude !== undefined
+                ? { longitude: input.context.longitude }
+                : {}),
+            browser: input.context.browser ?? "Other",
+            os: input.context.os ?? "Other",
+            device: input.context.device ?? "unknown",
+        });
+    }
+    const matchedGoalKeys = [];
+    for (const goal of input.goals) {
+        if (!goal.active || !goalMatches(goal, event))
+            continue;
+        const prior = await ctx.db
+            .query("goalCompletions")
+            .withIndex("by_siteId_and_goalId_and_eventId", (range) => range
+            .eq("siteId", input.siteId)
+            .eq("goalId", goal._id)
+            .eq("eventId", event.eventId))
+            .unique();
+        if (prior)
+            continue;
+        await ctx.db.insert("goalCompletions", {
+            siteId: input.siteId,
+            goalId: goal._id,
+            goalKey: goal.key,
+            eventId: event.eventId,
+            sessionId: event.sessionId,
+            visitorId: event.visitorId,
+            timestamp: event.timestamp,
+            valueCents: goal.valueCents ?? 0,
+        });
+        matchedGoalKeys.push(goal.key);
+    }
+    for (const funnel of input.funnels) {
+        if (funnel.active) {
+            await advanceFunnel(ctx, input.siteId, funnel, event);
+        }
+    }
+    if (input.financialConversion) {
+        await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey);
+    }
+    const dimensions = [
+        { type: "source", value: session.source },
+        { type: "page", value: event.path },
+        { type: "country", value: input.context.country ?? "unknown" },
+        { type: "device", value: input.context.device ?? "unknown" },
+        { type: "browser", value: input.context.browser ?? "Other" },
+        {
+            type: "event",
+            value: event.type === "custom" ? event.name ?? "custom" : event.type,
+        },
+        ...matchedGoalKeys.map((value) => ({ type: "goal", value })),
+    ];
+    foldAggregates(input.aggregateDeltas, input.siteId, event, session, dimensions, input.recordTelemetry, input.financialConversion);
+    if (input.recordTelemetry &&
+        event.timestamp >= input.now - LIVE_SESSION_TTL_MS) {
+        await updateLiveSession(ctx, input.siteId, event, input.context, session, input.now);
+    }
+}
+async function accountTrustedSession(ctx, existing, event) {
+    if (!existing) {
+        fail("CONFLICT", "trusted conversion telemetry has no matching session");
+    }
+    const revenueCents = event.revenueCents ?? 0;
+    await ctx.db.patch("sessions", existing._id, {
+        conversionCount: existing.conversionCount + 1,
+        revenueCents: existing.revenueCents + revenueCents,
+        currency: event.currency ?? "USD",
+    });
+    return {
+        isNewSession: false,
+        isNewVisitorHour: false,
+        isNewVisitorDay: false,
+        startedAt: existing.startedAt,
+        source: existing.source,
+        pageviewCount: existing.pageviewCount,
+        eventCount: existing.eventCount,
+        affiliateId: existing.affiliateId,
+        affiliateSlug: existing.affiliateSlug,
+        affiliateAttributedAt: existing.affiliateAttributedAt,
+        affiliateExpiresAt: existing.affiliateExpiresAt,
+    };
+}
+async function upsertSession(ctx, siteId, event, context, existing, financialConversion) {
+    const source = sourceFromReferrer(event.referrer);
+    const utmSource = stringProperty(event.properties, "utm_source");
+    const utmMedium = stringProperty(event.properties, "utm_medium");
+    const utmCampaign = stringProperty(event.properties, "utm_campaign");
+    const attribution = await attributionForEvent(ctx, siteId, event, existing);
+    if (existing) {
+        const startedAt = Math.min(existing.startedAt, event.timestamp);
+        const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
+        await ctx.db.patch("sessions", existing._id, {
+            startedAt,
+            lastSeenAt,
+            entryPath: event.timestamp < existing.startedAt ? event.path : existing.entryPath,
+            exitPath: event.timestamp >= existing.lastSeenAt ? event.path : existing.exitPath,
+            eventCount: existing.eventCount + 1,
+            pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+            durationMs: Math.max(0, lastSeenAt - startedAt),
+            conversionCount: existing.conversionCount + Number(financialConversion),
+            revenueCents: existing.revenueCents +
+                (financialConversion ? event.revenueCents ?? 0 : 0),
+            ...(financialConversion && event.currency
+                ? { currency: event.currency }
+                : {}),
+            ...(attribution ?? {}),
+            ...(utmSource ? { utmSource } : {}),
+            ...(utmMedium ? { utmMedium } : {}),
+            ...(utmCampaign ? { utmCampaign } : {}),
+            ...(context.country ? { country: context.country } : {}),
+            ...(context.city ? { city: context.city } : {}),
+            ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            browser: context.browser ?? existing.browser,
+            os: context.os ?? existing.os,
+            device: context.device ?? existing.device,
+        });
+        return {
+            isNewSession: false,
+            isNewVisitorHour: Math.floor(existing.lastSeenAt / HOUR_MS) <
+                Math.floor(event.timestamp / HOUR_MS),
+            isNewVisitorDay: Math.floor(existing.lastSeenAt / DAY_MS) <
+                Math.floor(event.timestamp / DAY_MS),
+            startedAt,
+            source: existing.source,
+            pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+            eventCount: existing.eventCount + 1,
+            ...(attribution ?? {}),
+        };
+    }
+    const priorVisit = await ctx.db
+        .query("sessions")
+        .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+        .order("desc")
+        .take(1);
+    await ctx.db.insert("sessions", {
+        siteId,
+        sessionId: event.sessionId,
+        visitorId: event.visitorId,
+        startedAt: event.timestamp,
+        lastSeenAt: event.timestamp,
+        entryPath: event.path,
+        exitPath: event.path,
+        ...(event.referrer ? { referrer: event.referrer } : {}),
+        source,
+        ...(utmSource ? { utmSource } : {}),
+        ...(utmMedium ? { utmMedium } : {}),
+        ...(utmCampaign ? { utmCampaign } : {}),
+        ...(context.country ? { country: context.country } : {}),
+        ...(context.city ? { city: context.city } : {}),
+        ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        browser: context.browser ?? "Other",
+        os: context.os ?? "Other",
+        device: context.device ?? "unknown",
+        eventCount: 1,
+        pageviewCount: event.type === "pageview" ? 1 : 0,
+        durationMs: 0,
+        conversionCount: Number(financialConversion),
+        revenueCents: financialConversion ? event.revenueCents ?? 0 : 0,
+        ...(financialConversion && event.currency
+            ? { currency: event.currency }
+            : {}),
+        ...(attribution ?? {}),
+    });
+    return {
+        isNewSession: true,
+        isNewVisitorHour: priorVisit.length === 0 ||
+            Math.floor(priorVisit[0].lastSeenAt / HOUR_MS) <
+                Math.floor(event.timestamp / HOUR_MS),
+        isNewVisitorDay: priorVisit.length === 0 ||
+            Math.floor(priorVisit[0].lastSeenAt / DAY_MS) <
+                Math.floor(event.timestamp / DAY_MS),
+        startedAt: event.timestamp,
+        source,
+        pageviewCount: event.type === "pageview" ? 1 : 0,
+        eventCount: 1,
+        ...(attribution ?? {}),
+    };
+}
+function attributionSnapshot(value) {
+    if (!value.affiliateId ||
+        !value.affiliateSlug ||
+        value.affiliateAttributedAt === undefined ||
+        value.affiliateExpiresAt === undefined) {
+        return undefined;
+    }
+    return {
+        affiliateId: value.affiliateId,
+        affiliateSlug: value.affiliateSlug,
+        affiliateAttributedAt: value.affiliateAttributedAt,
+        affiliateExpiresAt: value.affiliateExpiresAt,
+    };
+}
+function attributionApplies(attribution, timestamp) {
+    return (timestamp >= attribution.affiliateAttributedAt &&
+        timestamp <= attribution.affiliateExpiresAt);
+}
+// The first valid touch wins through its inclusive expiry. Only a later,
+// in-order valid touch may replace the visitor's single bounded record.
+async function attributionForEvent(ctx, siteId, event, existingSession) {
+    const sessionAttribution = existingSession
+        ? attributionSnapshot(existingSession)
+        : undefined;
+    if (!event.affiliateSlug && sessionAttribution) {
+        return sessionAttribution;
+    }
+    if (event.affiliateSlug &&
+        sessionAttribution &&
+        attributionApplies(sessionAttribution, event.timestamp)) {
+        return sessionAttribution;
+    }
+    const current = await ctx.db
+        .query("visitorAttributions")
+        .withIndex("by_siteId_and_visitorId", (range) => range.eq("siteId", siteId).eq("visitorId", event.visitorId))
+        .unique();
+    if (current) {
+        const snapshot = {
+            affiliateId: current.affiliateId,
+            affiliateSlug: current.affiliateSlug,
+            affiliateAttributedAt: current.attributedAt,
+            affiliateExpiresAt: current.expiresAt,
+        };
+        if (attributionApplies(snapshot, event.timestamp)) {
+            return snapshot;
+        }
+        if (event.timestamp <= current.expiresAt) {
+            return sessionAttribution;
+        }
+    }
+    const affiliateSlug = event.affiliateSlug;
+    if (!affiliateSlug) {
+        return sessionAttribution;
+    }
+    const affiliate = await ctx.db
+        .query("affiliates")
+        .withIndex("by_siteId_and_slug", (range) => range.eq("siteId", siteId).eq("slug", affiliateSlug))
+        .unique();
+    if (!affiliate?.active) {
+        return sessionAttribution;
+    }
+    const attributedAt = event.timestamp;
+    const expiresAt = attributedAt + affiliate.attributionWindowDays * DAY_MS;
+    if (current) {
+        await ctx.db.patch("visitorAttributions", current._id, {
+            affiliateId: affiliate._id,
+            affiliateSlug: affiliate.slug,
+            attributedAt,
+            expiresAt,
+        });
+    }
+    else {
+        await ctx.db.insert("visitorAttributions", {
+            siteId,
+            visitorId: event.visitorId,
+            affiliateId: affiliate._id,
+            affiliateSlug: affiliate.slug,
+            attributedAt,
+            expiresAt,
+        });
+    }
+    return {
+        affiliateId: affiliate._id,
+        affiliateSlug: affiliate.slug,
+        affiliateAttributedAt: attributedAt,
+        affiliateExpiresAt: expiresAt,
+    };
+}
+async function accountHeartbeat(ctx, siteId, session, event, context, now) {
+    const startedAt = Math.min(session.startedAt, event.timestamp);
+    const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
+    await ctx.db.patch("sessions", session._id, {
+        startedAt,
+        lastSeenAt,
+        exitPath: event.timestamp >= session.lastSeenAt
+            ? event.path
+            : session.exitPath,
+        durationMs: Math.max(0, lastSeenAt - startedAt),
+        ...(context.country ? { country: context.country } : {}),
+        ...(context.city ? { city: context.city } : {}),
+        ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        browser: context.browser ?? session.browser,
+        os: context.os ?? session.os,
+        device: context.device ?? session.device,
+    });
+    await updateLiveSession(ctx, siteId, event, context, {
+        isNewSession: false,
+        isNewVisitorHour: false,
+        isNewVisitorDay: false,
+        startedAt,
+        source: session.source,
+        pageviewCount: session.pageviewCount,
+        eventCount: session.eventCount,
+        affiliateSlug: session.affiliateSlug,
+    }, now);
+}
+async function consumeSiteRateLimit(ctx, siteId, events, now) {
+    const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+    const increments = new Map();
+    const eventByteCounts = [];
+    const encoder = new TextEncoder();
+    for (let index = 0; index < events.length; index += 1) {
+        const event = events[index];
+        const byteCount = encoder.encode(JSON.stringify(event)).byteLength;
+        eventByteCounts.push(byteCount);
+        const shard = stableHash(event.sessionId) % SITE_INGEST_SHARDS;
+        const current = increments.get(shard) ?? { eventCount: 0, byteCount: 0 };
+        current.eventCount += 1;
+        current.byteCount += byteCount;
+        increments.set(shard, current);
+    }
+    const windows = [];
+    for (const [shard, increment] of increments) {
+        const window = await ctx.db
+            .query("siteIngestWindows")
+            .withIndex("by_siteId_and_shard_and_windowStart", (range) => range
+            .eq("siteId", siteId)
+            .eq("shard", shard)
+            .eq("windowStart", windowStart))
+            .unique();
+        if ((window?.eventCount ?? 0) + increment.eventCount >
+            MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
+            (window?.byteCount ?? 0) + increment.byteCount >
+                MAX_SITE_BYTES_PER_SHARD_WINDOW) {
+            fail("RATE_LIMITED", "site ingest budget exceeded", {
+                siteShard: shard,
+                windowMs: RATE_LIMIT_WINDOW_MS,
+                eventLimitPerShard: MAX_SITE_EVENTS_PER_SHARD_WINDOW,
+                byteLimitPerShard: MAX_SITE_BYTES_PER_SHARD_WINDOW,
+                siteEventLimit: SITE_INGEST_SHARDS * MAX_SITE_EVENTS_PER_SHARD_WINDOW,
+                siteByteLimit: SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
+            });
+        }
+        windows.push({ shard, increment, window });
+    }
+    for (const { shard, increment, window } of windows) {
+        if (window) {
+            await ctx.db.patch("siteIngestWindows", window._id, {
+                eventCount: window.eventCount + increment.eventCount,
+                byteCount: window.byteCount + increment.byteCount,
+                updatedAt: now,
+            });
+        }
+        else {
+            await ctx.db.insert("siteIngestWindows", {
+                siteId,
+                shard,
+                windowStart,
+                eventCount: increment.eventCount,
+                byteCount: increment.byteCount,
+                updatedAt: now,
+            });
+        }
+    }
+    return eventByteCounts;
+}
+async function consumeRateLimit(ctx, siteId, sessionId, byteCount, now) {
+    const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+    const window = await ctx.db
+        .query("ingestWindows")
+        .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range
+        .eq("siteId", siteId)
+        .eq("sessionId", sessionId)
+        .eq("windowStart", windowStart))
+        .unique();
+    if (window &&
+        (window.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
+            window.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW)) {
+        return false;
+    }
+    if (window) {
+        await ctx.db.patch("ingestWindows", window._id, {
+            eventCount: window.eventCount + 1,
+            byteCount: window.byteCount + byteCount,
+            updatedAt: now,
+        });
+    }
+    else {
+        await ctx.db.insert("ingestWindows", {
+            siteId,
+            sessionId,
+            windowStart,
+            eventCount: 1,
+            byteCount,
+            updatedAt: now,
+        });
+    }
+    return true;
+}
+async function updateLiveSession(ctx, siteId, event, context, session, now) {
+    const existing = await ctx.db
+        .query("liveSessions")
+        .withIndex("by_siteId_and_sessionId", (range) => range.eq("siteId", siteId).eq("sessionId", event.sessionId))
+        .unique();
+    const renewsExpiry = !existing ||
+        existing.expiresAt <= now ||
+        event.type === "pageview" ||
+        event.type === "heartbeat";
+    const deadline = renewsExpiry
+        ? now + LIVE_SESSION_TTL_MS
+        : existing.expiresAt;
+    if (existing) {
+        await ctx.db.patch("liveSessions", existing._id, {
+            visitorId: event.visitorId,
+            path: event.path,
+            lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
+            expiresAt: deadline,
+            ...(context.country ? { country: context.country } : {}),
+            ...(context.city ? { city: context.city } : {}),
+            ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            browser: context.browser ?? existing.browser,
+            os: context.os ?? existing.os,
+            source: session.source,
+            device: context.device ?? existing.device,
+            pageviewCount: session.pageviewCount,
+            eventCount: session.eventCount,
+            ...(event.type === "heartbeat"
+                ? { lastHeartbeatEventId: event.eventId }
+                : {}),
+        });
+        return;
+    }
+    const liveSessionId = await ctx.db.insert("liveSessions", {
+        siteId,
+        sessionId: event.sessionId,
+        visitorId: event.visitorId,
+        path: event.path,
+        startedAt: event.timestamp,
+        lastSeenAt: event.timestamp,
+        expiresAt: deadline,
+        expiryChainDeadline: deadline,
+        ...(context.country ? { country: context.country } : {}),
+        ...(context.city ? { city: context.city } : {}),
+        ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        browser: context.browser ?? "Other",
+        os: context.os ?? "Other",
+        source: session.source,
+        device: context.device ?? "unknown",
+        pageviewCount: session.pageviewCount,
+        eventCount: session.eventCount,
+        ...(event.type === "heartbeat"
+            ? { lastHeartbeatEventId: event.eventId }
+            : {}),
+    });
+    await ctx.scheduler.runAt(deadline, internal.live.expire, {
+        liveSessionId,
+        scheduledDeadline: deadline,
+    });
+}
+function goalMatches(goal, event) {
+    if (goal.matchType === "pageview") {
+        return event.type === "pageview" && event.path === goal.matchValue;
+    }
+    return event.type === "custom" && event.name === goal.matchValue;
+}
+async function advanceFunnel(ctx, siteId, funnel, event) {
+    const progress = await ctx.db
+        .query("funnelProgress")
+        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) => range
+        .eq("siteId", siteId)
+        .eq("funnelId", funnel._id)
+        .eq("visitorId", event.visitorId))
+        .unique();
+    const matchesFirst = stepMatches(funnel.steps[0], event);
+    if (!progress) {
+        if (!matchesFirst)
+            return;
+        await ctx.db.insert("funnelProgress", {
+            siteId,
+            funnelId: funnel._id,
+            visitorId: event.visitorId,
+            sessionId: event.sessionId,
+            currentStep: 1,
+            startedAt: event.timestamp,
+            lastStepAt: event.timestamp,
+        });
+        await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        return;
+    }
+    if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt)
+        return;
+    if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
+        if (!matchesFirst)
+            return;
+        await ctx.db.patch("funnelProgress", progress._id, {
+            sessionId: event.sessionId,
+            currentStep: 1,
+            startedAt: event.timestamp,
+            lastStepAt: event.timestamp,
+            completedAt: undefined,
+        });
+        await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+        return;
+    }
+    const nextStep = funnel.steps[progress.currentStep];
+    if (!nextStep || !stepMatches(nextStep, event))
+        return;
+    const currentStep = progress.currentStep + 1;
+    const completed = currentStep === funnel.steps.length;
+    await ctx.db.patch("funnelProgress", progress._id, {
+        sessionId: event.sessionId,
+        currentStep,
+        lastStepAt: event.timestamp,
+        ...(completed ? { completedAt: event.timestamp } : {}),
+    });
+    await incrementFunnelStep(ctx, siteId, funnel._id, event, progress.currentStep, completed);
+}
+function stepMatches(step, event) {
+    return step.type === "pageview"
+        ? event.type === "pageview" && event.path === step.value
+        : event.type === "custom" && event.name === step.value;
+}
+async function incrementFunnelStep(ctx, siteId, funnelId, event, step, completed) {
+    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(event.visitorId) % AGGREGATE_SHARDS;
+    const bucket = await ctx.db
+        .query("funnelStepBuckets")
+        .withIndex("by_siteId_and_funnelId_and_bucketStart_and_step_and_shard", (range) => range
+        .eq("siteId", siteId)
+        .eq("funnelId", funnelId)
+        .eq("bucketStart", bucketStart)
+        .eq("step", step)
+        .eq("shard", shard))
+        .unique();
+    if (bucket) {
+        await ctx.db.patch("funnelStepBuckets", bucket._id, {
+            entrants: bucket.entrants + 1,
+            completions: bucket.completions + (completed ? 1 : 0),
+        });
+    }
+    else {
+        await ctx.db.insert("funnelStepBuckets", {
+            siteId,
+            funnelId,
+            bucketStart,
+            step,
+            shard,
+            entrants: 1,
+            completions: completed ? 1 : 0,
+        });
+    }
+}
+async function accountConversion(ctx, siteId, session, event, trustedGoalKey) {
+    const revenueCents = event.revenueCents ?? 0;
+    const currency = event.currency ?? "USD";
+    const candidate = attributionSnapshot(session);
+    const attribution = candidate && attributionApplies(candidate, event.timestamp)
+        ? candidate
+        : undefined;
+    const affiliate = attribution
+        ? await ctx.db.get("affiliates", attribution.affiliateId)
+        : null;
+    const commissionAffiliate = affiliate?.active && affiliate.siteId === siteId ? affiliate : null;
+    const commissionCents = commissionAffiliate
+        ? Math.round((revenueCents * commissionAffiliate.commissionBps) / 10_000)
+        : 0;
+    await ctx.db.insert("conversions", {
+        siteId,
+        eventId: event.eventId,
+        sessionId: event.sessionId,
+        visitorId: event.visitorId,
+        timestamp: event.timestamp,
+        path: event.path,
+        revenueCents,
+        currency,
+        source: "trusted",
+        ...(trustedGoalKey ? { goalKey: trustedGoalKey } : {}),
+        ...(attribution ?? {}),
+        commissionCents,
+    });
+    if (commissionAffiliate && attribution) {
+        const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+        const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+        const stats = await ctx.db
+            .query("affiliateDailyStats")
+            .withIndex("by_siteId_and_affiliateId_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", siteId)
+            .eq("affiliateId", commissionAffiliate._id)
+            .eq("bucketStart", bucketStart)
+            .eq("shard", shard))
+            .unique();
+        if (stats) {
+            await ctx.db.patch("affiliateDailyStats", stats._id, {
+                conversions: stats.conversions + 1,
+                revenueCents: stats.revenueCents + revenueCents,
+                commissionCents: stats.commissionCents + commissionCents,
+            });
+        }
+        else {
+            await ctx.db.insert("affiliateDailyStats", {
+                siteId,
+                affiliateId: commissionAffiliate._id,
+                affiliateSlug: attribution.affiliateSlug,
+                bucketStart,
+                shard,
+                conversions: 1,
+                revenueCents,
+                commissionCents,
+            });
+        }
+    }
+}
+function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
+    const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+    const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
+    const dimensionDeltas = dimensions.map((dimension) => ({
+        ...dimension,
+        count: Number(recordTelemetry),
+        revenueCents,
+    }));
+    for (const granularity of ["hour", "day"]) {
+        const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
+        const bucketStart = Math.floor(event.timestamp / interval) * interval;
+        const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
+        let delta = aggregateDeltas.get(key);
+        if (!delta) {
+            delta = {
+                siteId,
+                granularity,
+                bucketStart,
+                shard,
+                events: 0,
+                pageviews: 0,
+                clicks: 0,
+                customEvents: 0,
+                outboundClicks: 0,
+                sessions: 0,
+                visitors: 0,
+                conversions: 0,
+                revenueCents: 0,
+                dimensions: [],
+            };
+            aggregateDeltas.set(key, delta);
+        }
+        const eventCount = Number(recordTelemetry);
+        delta.events += eventCount;
+        delta.pageviews += Number(recordTelemetry && event.type === "pageview");
+        delta.clicks += Number(recordTelemetry && event.type === "click");
+        delta.customEvents += Number(recordTelemetry && event.type === "custom");
+        delta.outboundClicks += Number(recordTelemetry && event.type === "outbound");
+        delta.sessions += Number(recordTelemetry && session.isNewSession);
+        delta.visitors += recordTelemetry
+            ? granularity === "hour"
+                ? Number(session.isNewVisitorHour)
+                : Number(session.isNewVisitorDay)
+            : 0;
+        delta.conversions += Number(financialConversion);
+        delta.revenueCents += revenueCents;
+        delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
+    }
+}
+async function flushAggregates(ctx, aggregateDeltas) {
+    for (const delta of aggregateDeltas.values()) {
+        const bucket = await ctx.db
+            .query("aggregateBuckets")
+            .withIndex("by_siteId_and_granularity_and_bucketStart_and_shard", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("granularity", delta.granularity)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard))
+            .unique();
+        const dimensions = mergeDimensions(bucket?.dimensions ?? [], delta.dimensions);
+        if (bucket) {
+            await ctx.db.patch("aggregateBuckets", bucket._id, {
+                events: bucket.events + delta.events,
+                pageviews: bucket.pageviews + delta.pageviews,
+                clicks: bucket.clicks + delta.clicks,
+                customEvents: bucket.customEvents + delta.customEvents,
+                outboundClicks: bucket.outboundClicks + delta.outboundClicks,
+                sessions: bucket.sessions + delta.sessions,
+                visitors: bucket.visitors + delta.visitors,
+                conversions: bucket.conversions + delta.conversions,
+                revenueCents: bucket.revenueCents + delta.revenueCents,
+                dimensions,
+            });
+        }
+        else {
+            await ctx.db.insert("aggregateBuckets", {
+                siteId: delta.siteId,
+                granularity: delta.granularity,
+                bucketStart: delta.bucketStart,
+                shard: delta.shard,
+                events: delta.events,
+                pageviews: delta.pageviews,
+                clicks: delta.clicks,
+                customEvents: delta.customEvents,
+                outboundClicks: delta.outboundClicks,
+                sessions: delta.sessions,
+                visitors: delta.visitors,
+                conversions: delta.conversions,
+                revenueCents: delta.revenueCents,
+                dimensions,
+            });
+        }
+    }
+}
+function mergeDimensions(current, additions) {
+    const result = current.map((slot) => ({ ...slot }));
+    for (const addition of additions) {
+        const exact = result.find((candidate) => candidate.type === addition.type && candidate.value === addition.value);
+        if (exact) {
+            exact.count += addition.count;
+            exact.revenueCents += addition.revenueCents;
+            continue;
+        }
+        const typeSlots = result.filter((candidate) => candidate.type === addition.type);
+        if (typeSlots.length < DIMENSION_SLOTS) {
+            const occupied = Array.from({ length: DIMENSION_SLOTS }, () => false);
+            for (const candidate of typeSlots)
+                occupied[candidate.slot] = true;
+            let slot = stableHash(addition.value) % DIMENSION_SLOTS;
+            while (occupied[slot])
+                slot = (slot + 1) % DIMENSION_SLOTS;
+            result.push({
+                type: addition.type,
+                slot,
+                value: addition.value,
+                count: addition.count,
+                revenueCents: addition.revenueCents,
+            });
+            continue;
+        }
+        if (addition.count === 0)
+            continue;
+        let replacement = typeSlots[0];
+        for (const candidate of typeSlots.slice(1)) {
+            if (candidate.count < replacement.count)
+                replacement = candidate;
+        }
+        replacement.value = addition.value;
+        replacement.count += addition.count;
+        replacement.revenueCents = addition.revenueCents;
+    }
+    return result;
+}
+function stringProperty(properties, key) {
+    const value = properties?.[key];
+    return typeof value === "string" ? cleanString(value, 120) || undefined : undefined;
+}
+//# sourceMappingURL=ingest.js.map