@iann29/rastro 0.1.0-alpha.0

package/src/component/ingest.ts

@@ -0,0 +1,1364 @@
+import { v } from "convex/values";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
+import { mutation } from "./_generated/server.js";
+import { internal } from "./_generated/api.js";
+import {
+  AGGREGATE_SHARDS,
+  DAY_MS,
+  DIMENSION_SLOTS,
+  HOUR_MS,
+  LIVE_SESSION_TTL_MS,
+  MAX_BATCH_EVENTS,
+  MAX_SITE_BYTES_PER_SHARD_WINDOW,
+  MAX_SITE_EVENTS_PER_SHARD_WINDOW,
+  MAX_BYTES_PER_SESSION_WINDOW,
+  MAX_EVENTS_PER_SESSION_WINDOW,
+  MAX_FUNNELS_PER_SITE,
+  MAX_GOALS_PER_SITE,
+  RATE_LIMIT_WINDOW_MS,
+  SITE_INGEST_SHARDS,
+} from "./constants.js";
+import { fail } from "./errors.js";
+import {
+  cleanString,
+  ensureBatchBytes,
+  isValidMoney,
+  originAllowed,
+  sanitizeContext,
+  sanitizeCurrency,
+  sanitizeEvent,
+  sanitizeOpaqueId,
+  sanitizePath,
+  sanitizeProperties,
+  sanitizeSlug,
+  sourceFromReferrer,
+  stableHash,
+} from "./sanitize.js";
+import {
+  ingestContextValidator,
+  ingestResultValidator,
+  trackerEventValidator,
+  trustedConversionValidator,
+  type IngestContext,
+  type TrackerEvent,
+} from "./validators.js";
+
+type DimensionType =
+  | "source"
+  | "page"
+  | "country"
+  | "device"
+  | "browser"
+  | "event"
+  | "goal";
+
+type Dimension = { type: DimensionType; value: string };
+type DimensionDelta = Dimension & { count: number; revenueCents: number };
+type AggregateDelta = {
+  siteId: Id<"sites">;
+  granularity: "hour" | "day";
+  bucketStart: number;
+  shard: number;
+  events: number;
+  pageviews: number;
+  clicks: number;
+  customEvents: number;
+  outboundClicks: number;
+  sessions: number;
+  visitors: number;
+  conversions: number;
+  revenueCents: number;
+  dimensions: Doc<"aggregateBuckets">["dimensions"];
+};
+type AggregateDeltas = Map<string, AggregateDelta>;
+type AttributionSnapshot = {
+  affiliateId: Id<"affiliates">;
+  affiliateSlug: string;
+  affiliateAttributedAt: number;
+  affiliateExpiresAt: number;
+};
+
+type SessionState = {
+  isNewSession: boolean;
+  isNewVisitorHour: boolean;
+  isNewVisitorDay: boolean;
+  startedAt: number;
+  source: string;
+  pageviewCount: number;
+  eventCount: number;
+  affiliateId?: Id<"affiliates">;
+  affiliateSlug?: string;
+  affiliateAttributedAt?: number;
+  affiliateExpiresAt?: number;
+};
+
+export const ingestBatch = mutation({
+  args: {
+    siteId: v.id("sites"),
+    origin: v.string(),
+    events: v.array(trackerEventValidator),
+    context: v.optional(ingestContextValidator),
+  },
+  returns: ingestResultValidator,
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    if (!originAllowed(args.origin, site.domains)) {
+      fail("ORIGIN_NOT_ALLOWED", "request origin is not allowed", {
+        siteId: String(args.siteId),
+      });
+    }
+    if (args.events.length < 1 || args.events.length > MAX_BATCH_EVENTS) {
+      fail("BATCH_TOO_LARGE", `events must contain 1-${MAX_BATCH_EVENTS} items`, {
+        limit: MAX_BATCH_EVENTS,
+      });
+    }
+    try {
+      ensureBatchBytes({ siteId: String(args.siteId), events: args.events });
+    } catch {
+      fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
+    }
+
+    const now = Date.now();
+    const eventByteCounts = await consumeSiteRateLimit(
+      ctx,
+      args.siteId,
+      args.events,
+      now,
+    );
+    const context = sanitizeContext(args.context);
+    const aggregateDeltas: AggregateDeltas = new Map();
+    let goals: Doc<"goals">[] | null = null;
+    let funnels: Doc<"funnels">[] | null = null;
+    let accepted = 0;
+    let duplicates = 0;
+    let rejected = 0;
+    for (let eventIndex = 0; eventIndex < args.events.length; eventIndex += 1) {
+      const input = args.events[eventIndex];
+      let event: TrackerEvent;
+      try {
+        event = sanitizeEvent(input, now);
+      } catch {
+        rejected += 1;
+        continue;
+      }
+
+      const existingSession = await ctx.db
+        .query("sessions")
+        .withIndex("by_siteId_and_sessionId", (range) =>
+          range.eq("siteId", args.siteId).eq("sessionId", event.sessionId),
+        )
+        .unique();
+      if (existingSession && existingSession.visitorId !== event.visitorId) {
+        rejected += 1;
+        continue;
+      }
+
+      if (event.type === "heartbeat") {
+        if (!existingSession || event.timestamp < now - LIVE_SESSION_TTL_MS) {
+          rejected += 1;
+          continue;
+        }
+        const liveSession = await ctx.db
+          .query("liveSessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", args.siteId).eq("sessionId", event.sessionId),
+          )
+          .unique();
+        if (liveSession?.lastHeartbeatEventId === event.eventId) {
+          duplicates += 1;
+          continue;
+        }
+        if (
+          !(await consumeRateLimit(
+            ctx,
+            args.siteId,
+            event.sessionId,
+            eventByteCounts[eventIndex],
+            now,
+          ))
+        ) {
+          rejected += 1;
+          continue;
+        }
+        await accountHeartbeat(
+          ctx,
+          args.siteId,
+          existingSession,
+          event,
+          context,
+          now,
+        );
+        accepted += 1;
+        continue;
+      }
+
+      const duplicate = await ctx.db
+        .query("events")
+        .withIndex("by_siteId_and_eventId", (range) =>
+          range.eq("siteId", args.siteId).eq("eventId", event.eventId),
+        )
+        .unique();
+      if (duplicate) {
+        duplicates += 1;
+        continue;
+      }
+
+      if (
+        !(await consumeRateLimit(
+          ctx,
+          args.siteId,
+          event.sessionId,
+          eventByteCounts[eventIndex],
+          now,
+        ))
+      ) {
+        rejected += 1;
+        continue;
+      }
+      const canMatchDefinition =
+        event.type === "pageview" || event.type === "custom";
+      if (canMatchDefinition && goals === null) {
+        goals = await ctx.db
+          .query("goals")
+          .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+          .take(MAX_GOALS_PER_SITE);
+      }
+      if (canMatchDefinition && funnels === null) {
+        funnels = await ctx.db
+          .query("funnels")
+          .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+          .take(MAX_FUNNELS_PER_SITE);
+      }
+      await accountEvent(ctx, {
+        siteId: args.siteId,
+        event,
+        context,
+        existingSession,
+        goals: canMatchDefinition ? goals ?? [] : [],
+        funnels: canMatchDefinition ? funnels ?? [] : [],
+        aggregateDeltas,
+        recordTelemetry: true,
+        financialConversion: false,
+        trustedGoalKey: undefined,
+        now,
+      });
+      accepted += 1;
+    }
+    await flushAggregates(ctx, aggregateDeltas);
+    return { accepted, duplicates, rejected };
+  },
+});
+
+export const trackConversion = mutation({
+  args: trustedConversionValidator.fields,
+  returns: ingestResultValidator,
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    if (!isValidMoney(args.revenueCents)) {
+      fail("INVALID_ARGUMENT", "revenueCents must be an integer between 0 and 1000000000");
+    }
+    let event: TrackerEvent;
+    try {
+      event = sanitizeEvent(
+        {
+          eventId: sanitizeOpaqueId(args.eventId, "eventId"),
+          sessionId: sanitizeOpaqueId(args.sessionId, "sessionId"),
+          visitorId: sanitizeOpaqueId(args.visitorId, "visitorId"),
+          type: "conversion",
+          name: args.goalKey ? sanitizeSlug(args.goalKey, "goalKey") : "conversion",
+          path: sanitizePath(args.path ?? "/conversion"),
+          timestamp: args.timestamp,
+          sequence: 0,
+          properties: sanitizeProperties(args.properties),
+          revenueCents: args.revenueCents,
+          currency: sanitizeCurrency(args.currency),
+        },
+        Date.now(),
+      );
+    } catch (error) {
+      fail("INVALID_ARGUMENT", error instanceof Error ? error.message : "invalid conversion");
+    }
+
+    const siteCurrency = site.currency ?? "USD";
+    if (event.currency !== siteCurrency) {
+      fail("CURRENCY_MISMATCH", "conversion currency does not match site currency", {
+        siteCurrency,
+        conversionCurrency: event.currency ?? "USD",
+      });
+    }
+    const priorConversion = await ctx.db
+      .query("conversions")
+      .withIndex("by_siteId_and_eventId", (range) =>
+        range.eq("siteId", args.siteId).eq("eventId", event.eventId),
+      )
+      .unique();
+    if (priorConversion) {
+      return { accepted: 0, duplicates: 1, rejected: 0 };
+    }
+
+    const existingTelemetry = await ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_eventId", (range) =>
+        range.eq("siteId", args.siteId).eq("eventId", event.eventId),
+      )
+      .unique();
+    if (
+      existingTelemetry &&
+      (existingTelemetry.sessionId !== event.sessionId ||
+        existingTelemetry.visitorId !== event.visitorId)
+    ) {
+      fail("CONFLICT", "event belongs to a different session or visitor");
+    }
+    const existingSession = await ctx.db
+      .query("sessions")
+      .withIndex("by_siteId_and_sessionId", (range) =>
+        range.eq("siteId", args.siteId).eq("sessionId", event.sessionId),
+      )
+      .unique();
+    if (existingSession && existingSession.visitorId !== event.visitorId) {
+      fail("CONFLICT", "session belongs to a different visitor");
+    }
+    const conversionContext: IngestContext = existingTelemetry
+      ? {
+          country: existingTelemetry.country,
+          city: existingTelemetry.city,
+          latitude: existingTelemetry.latitude,
+          longitude: existingTelemetry.longitude,
+          browser: existingTelemetry.browser,
+          os: existingTelemetry.os,
+          device: existingTelemetry.device,
+        }
+      : sanitizeContext(undefined);
+
+    const aggregateDeltas: AggregateDeltas = new Map();
+    await accountEvent(ctx, {
+      siteId: args.siteId,
+      event,
+      context: conversionContext,
+      existingSession,
+      goals: [],
+      funnels: [],
+      aggregateDeltas,
+      recordTelemetry: existingTelemetry === null,
+      financialConversion: true,
+      trustedGoalKey: args.goalKey
+        ? sanitizeSlug(args.goalKey, "goalKey")
+        : undefined,
+      now: Date.now(),
+    });
+    await flushAggregates(ctx, aggregateDeltas);
+    return { accepted: 1, duplicates: 0, rejected: 0 };
+  },
+});
+
+async function accountEvent(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    event: TrackerEvent;
+    context: IngestContext;
+    existingSession: Doc<"sessions"> | null;
+    goals: Doc<"goals">[];
+    funnels: Doc<"funnels">[];
+    aggregateDeltas: AggregateDeltas;
+    recordTelemetry: boolean;
+    financialConversion: boolean;
+    trustedGoalKey: string | undefined;
+    now: number;
+  },
+) {
+  const { event } = input;
+  const session = input.recordTelemetry
+    ? await upsertSession(
+        ctx,
+        input.siteId,
+        event,
+        input.context,
+        input.existingSession,
+        input.financialConversion,
+      )
+    : await accountTrustedSession(ctx, input.existingSession, event);
+
+  if (input.recordTelemetry) {
+    await ctx.db.insert("events", {
+      siteId: input.siteId,
+      eventId: event.eventId,
+      sessionId: event.sessionId,
+      visitorId: event.visitorId,
+      type: event.type,
+      ...(event.name ? { name: event.name } : {}),
+      path: event.path,
+      ...(event.referrer ? { referrer: event.referrer } : {}),
+      timestamp: event.timestamp,
+      sequence: event.sequence,
+      ...(event.target ? { target: event.target } : {}),
+      ...(event.href ? { href: event.href } : {}),
+      ...(event.properties ? { properties: event.properties } : {}),
+      ...(event.revenueCents !== undefined
+        ? { revenueCents: event.revenueCents }
+        : {}),
+      ...(event.currency ? { currency: event.currency } : {}),
+      ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+      ...(input.context.country ? { country: input.context.country } : {}),
+      ...(input.context.city ? { city: input.context.city } : {}),
+      ...(input.context.latitude !== undefined
+        ? { latitude: input.context.latitude }
+        : {}),
+      ...(input.context.longitude !== undefined
+        ? { longitude: input.context.longitude }
+        : {}),
+      browser: input.context.browser ?? "Other",
+      os: input.context.os ?? "Other",
+      device: input.context.device ?? "unknown",
+    });
+  }
+
+  const matchedGoalKeys: string[] = [];
+  for (const goal of input.goals) {
+    if (!goal.active || !goalMatches(goal, event)) continue;
+    const prior = await ctx.db
+      .query("goalCompletions")
+      .withIndex("by_siteId_and_goalId_and_eventId", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("goalId", goal._id)
+          .eq("eventId", event.eventId),
+      )
+      .unique();
+    if (prior) continue;
+    await ctx.db.insert("goalCompletions", {
+      siteId: input.siteId,
+      goalId: goal._id,
+      goalKey: goal.key,
+      eventId: event.eventId,
+      sessionId: event.sessionId,
+      visitorId: event.visitorId,
+      timestamp: event.timestamp,
+      valueCents: goal.valueCents ?? 0,
+    });
+    matchedGoalKeys.push(goal.key);
+  }
+
+  for (const funnel of input.funnels) {
+    if (funnel.active) {
+      await advanceFunnel(ctx, input.siteId, funnel, event);
+    }
+  }
+
+  if (input.financialConversion) {
+    await accountConversion(
+      ctx,
+      input.siteId,
+      session,
+      event,
+      input.trustedGoalKey,
+    );
+  }
+
+  const dimensions: Dimension[] = [
+    { type: "source", value: session.source },
+    { type: "page", value: event.path },
+    { type: "country", value: input.context.country ?? "unknown" },
+    { type: "device", value: input.context.device ?? "unknown" },
+    { type: "browser", value: input.context.browser ?? "Other" },
+    {
+      type: "event",
+      value: event.type === "custom" ? event.name ?? "custom" : event.type,
+    },
+    ...matchedGoalKeys.map((value): Dimension => ({ type: "goal", value })),
+  ];
+  foldAggregates(
+    input.aggregateDeltas,
+    input.siteId,
+    event,
+    session,
+    dimensions,
+    input.recordTelemetry,
+    input.financialConversion,
+  );
+
+  if (
+    input.recordTelemetry &&
+    event.timestamp >= input.now - LIVE_SESSION_TTL_MS
+  ) {
+    await updateLiveSession(
+      ctx,
+      input.siteId,
+      event,
+      input.context,
+      session,
+      input.now,
+    );
+  }
+}
+
+async function accountTrustedSession(
+  ctx: MutationCtx,
+  existing: Doc<"sessions"> | null,
+  event: TrackerEvent,
+): Promise<SessionState> {
+  if (!existing) {
+    fail("CONFLICT", "trusted conversion telemetry has no matching session");
+  }
+  const revenueCents = event.revenueCents ?? 0;
+  await ctx.db.patch("sessions", existing._id, {
+    conversionCount: existing.conversionCount + 1,
+    revenueCents: existing.revenueCents + revenueCents,
+    currency: event.currency ?? "USD",
+  });
+  return {
+    isNewSession: false,
+    isNewVisitorHour: false,
+    isNewVisitorDay: false,
+    startedAt: existing.startedAt,
+    source: existing.source,
+    pageviewCount: existing.pageviewCount,
+    eventCount: existing.eventCount,
+    affiliateId: existing.affiliateId,
+    affiliateSlug: existing.affiliateSlug,
+    affiliateAttributedAt: existing.affiliateAttributedAt,
+    affiliateExpiresAt: existing.affiliateExpiresAt,
+  };
+}
+
+async function upsertSession(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  context: IngestContext,
+  existing: Doc<"sessions"> | null,
+  financialConversion: boolean,
+): Promise<SessionState> {
+  const source = sourceFromReferrer(event.referrer);
+  const utmSource = stringProperty(event.properties, "utm_source");
+  const utmMedium = stringProperty(event.properties, "utm_medium");
+  const utmCampaign = stringProperty(event.properties, "utm_campaign");
+  const attribution = await attributionForEvent(ctx, siteId, event, existing);
+  if (existing) {
+    const startedAt = Math.min(existing.startedAt, event.timestamp);
+    const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
+    await ctx.db.patch("sessions", existing._id, {
+      startedAt,
+      lastSeenAt,
+      entryPath: event.timestamp < existing.startedAt ? event.path : existing.entryPath,
+      exitPath: event.timestamp >= existing.lastSeenAt ? event.path : existing.exitPath,
+      eventCount: existing.eventCount + 1,
+      pageviewCount: existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+      durationMs: Math.max(0, lastSeenAt - startedAt),
+      conversionCount: existing.conversionCount + Number(financialConversion),
+      revenueCents:
+        existing.revenueCents +
+        (financialConversion ? event.revenueCents ?? 0 : 0),
+      ...(financialConversion && event.currency
+        ? { currency: event.currency }
+        : {}),
+      ...(attribution ?? {}),
+      ...(utmSource ? { utmSource } : {}),
+      ...(utmMedium ? { utmMedium } : {}),
+      ...(utmCampaign ? { utmCampaign } : {}),
+      ...(context.country ? { country: context.country } : {}),
+      ...(context.city ? { city: context.city } : {}),
+      ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+      ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+      browser: context.browser ?? existing.browser,
+      os: context.os ?? existing.os,
+      device: context.device ?? existing.device,
+    });
+    return {
+      isNewSession: false,
+      isNewVisitorHour:
+        Math.floor(existing.lastSeenAt / HOUR_MS) <
+        Math.floor(event.timestamp / HOUR_MS),
+      isNewVisitorDay:
+        Math.floor(existing.lastSeenAt / DAY_MS) <
+        Math.floor(event.timestamp / DAY_MS),
+      startedAt,
+      source: existing.source,
+      pageviewCount:
+        existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+      eventCount: existing.eventCount + 1,
+      ...(attribution ?? {}),
+    };
+  }
+
+  const priorVisit = await ctx.db
+    .query("sessions")
+    .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", event.visitorId),
+    )
+    .order("desc")
+    .take(1);
+  await ctx.db.insert("sessions", {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    startedAt: event.timestamp,
+    lastSeenAt: event.timestamp,
+    entryPath: event.path,
+    exitPath: event.path,
+    ...(event.referrer ? { referrer: event.referrer } : {}),
+    source,
+    ...(utmSource ? { utmSource } : {}),
+    ...(utmMedium ? { utmMedium } : {}),
+    ...(utmCampaign ? { utmCampaign } : {}),
+    ...(context.country ? { country: context.country } : {}),
+    ...(context.city ? { city: context.city } : {}),
+    ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+    ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+    browser: context.browser ?? "Other",
+    os: context.os ?? "Other",
+    device: context.device ?? "unknown",
+    eventCount: 1,
+    pageviewCount: event.type === "pageview" ? 1 : 0,
+    durationMs: 0,
+    conversionCount: Number(financialConversion),
+    revenueCents: financialConversion ? event.revenueCents ?? 0 : 0,
+    ...(financialConversion && event.currency
+      ? { currency: event.currency }
+      : {}),
+    ...(attribution ?? {}),
+  });
+  return {
+    isNewSession: true,
+    isNewVisitorHour:
+      priorVisit.length === 0 ||
+      Math.floor(priorVisit[0].lastSeenAt / HOUR_MS) <
+        Math.floor(event.timestamp / HOUR_MS),
+    isNewVisitorDay:
+      priorVisit.length === 0 ||
+      Math.floor(priorVisit[0].lastSeenAt / DAY_MS) <
+        Math.floor(event.timestamp / DAY_MS),
+    startedAt: event.timestamp,
+    source,
+    pageviewCount: event.type === "pageview" ? 1 : 0,
+    eventCount: 1,
+    ...(attribution ?? {}),
+  };
+}
+
+function attributionSnapshot(
+  value: {
+    affiliateId?: Id<"affiliates">;
+    affiliateSlug?: string;
+    affiliateAttributedAt?: number;
+    affiliateExpiresAt?: number;
+  },
+): AttributionSnapshot | undefined {
+  if (
+    !value.affiliateId ||
+    !value.affiliateSlug ||
+    value.affiliateAttributedAt === undefined ||
+    value.affiliateExpiresAt === undefined
+  ) {
+    return undefined;
+  }
+  return {
+    affiliateId: value.affiliateId,
+    affiliateSlug: value.affiliateSlug,
+    affiliateAttributedAt: value.affiliateAttributedAt,
+    affiliateExpiresAt: value.affiliateExpiresAt,
+  };
+}
+
+
+function attributionApplies(
+  attribution: AttributionSnapshot,
+  timestamp: number,
+): boolean {
+  return (
+    timestamp >= attribution.affiliateAttributedAt &&
+    timestamp <= attribution.affiliateExpiresAt
+  );
+}
+
+// The first valid touch wins through its inclusive expiry. Only a later,
+// in-order valid touch may replace the visitor's single bounded record.
+async function attributionForEvent(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  existingSession: Doc<"sessions"> | null,
+): Promise<AttributionSnapshot | undefined> {
+  const sessionAttribution = existingSession
+    ? attributionSnapshot(existingSession)
+    : undefined;
+  if (!event.affiliateSlug && sessionAttribution) {
+    return sessionAttribution;
+  }
+  if (
+    event.affiliateSlug &&
+    sessionAttribution &&
+    attributionApplies(sessionAttribution, event.timestamp)
+  ) {
+    return sessionAttribution;
+  }
+
+  const current = await ctx.db
+    .query("visitorAttributions")
+    .withIndex("by_siteId_and_visitorId", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", event.visitorId),
+    )
+    .unique();
+  if (current) {
+    const snapshot: AttributionSnapshot = {
+      affiliateId: current.affiliateId,
+      affiliateSlug: current.affiliateSlug,
+      affiliateAttributedAt: current.attributedAt,
+      affiliateExpiresAt: current.expiresAt,
+    };
+    if (attributionApplies(snapshot, event.timestamp)) {
+      return snapshot;
+    }
+    if (event.timestamp <= current.expiresAt) {
+      return sessionAttribution;
+    }
+  }
+  const affiliateSlug = event.affiliateSlug;
+  if (!affiliateSlug) {
+    return sessionAttribution;
+  }
+
+  const affiliate = await ctx.db
+    .query("affiliates")
+    .withIndex("by_siteId_and_slug", (range) =>
+      range.eq("siteId", siteId).eq("slug", affiliateSlug),
+    )
+    .unique();
+  if (!affiliate?.active) {
+    return sessionAttribution;
+  }
+
+  const attributedAt = event.timestamp;
+  const expiresAt = attributedAt + affiliate.attributionWindowDays * DAY_MS;
+  if (current) {
+    await ctx.db.patch("visitorAttributions", current._id, {
+      affiliateId: affiliate._id,
+      affiliateSlug: affiliate.slug,
+      attributedAt,
+      expiresAt,
+    });
+  } else {
+    await ctx.db.insert("visitorAttributions", {
+      siteId,
+      visitorId: event.visitorId,
+      affiliateId: affiliate._id,
+      affiliateSlug: affiliate.slug,
+      attributedAt,
+      expiresAt,
+    });
+  }
+  return {
+    affiliateId: affiliate._id,
+    affiliateSlug: affiliate.slug,
+    affiliateAttributedAt: attributedAt,
+    affiliateExpiresAt: expiresAt,
+  };
+}
+
+async function accountHeartbeat(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  session: Doc<"sessions">,
+  event: TrackerEvent,
+  context: IngestContext,
+  now: number,
+) {
+  const startedAt = Math.min(session.startedAt, event.timestamp);
+  const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
+  await ctx.db.patch("sessions", session._id, {
+    startedAt,
+    lastSeenAt,
+    exitPath: event.timestamp >= session.lastSeenAt
+      ? event.path
+      : session.exitPath,
+    durationMs: Math.max(0, lastSeenAt - startedAt),
+    ...(context.country ? { country: context.country } : {}),
+    ...(context.city ? { city: context.city } : {}),
+    ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+    ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+    browser: context.browser ?? session.browser,
+    os: context.os ?? session.os,
+    device: context.device ?? session.device,
+  });
+  await updateLiveSession(
+    ctx,
+    siteId,
+    event,
+    context,
+    {
+      isNewSession: false,
+      isNewVisitorHour: false,
+      isNewVisitorDay: false,
+      startedAt,
+      source: session.source,
+      pageviewCount: session.pageviewCount,
+      eventCount: session.eventCount,
+      affiliateSlug: session.affiliateSlug,
+    },
+    now,
+  );
+}
+
+async function consumeSiteRateLimit(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  events: TrackerEvent[],
+  now: number,
+): Promise<number[]> {
+  const windowStart =
+    Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+  const increments = new Map<number, { eventCount: number; byteCount: number }>();
+  const eventByteCounts: number[] = [];
+  const encoder = new TextEncoder();
+  for (let index = 0; index < events.length; index += 1) {
+    const event = events[index];
+    const byteCount = encoder.encode(JSON.stringify(event)).byteLength;
+    eventByteCounts.push(byteCount);
+    const shard = stableHash(event.sessionId) % SITE_INGEST_SHARDS;
+    const current = increments.get(shard) ?? { eventCount: 0, byteCount: 0 };
+    current.eventCount += 1;
+    current.byteCount += byteCount;
+    increments.set(shard, current);
+  }
+
+  const windows: Array<{
+    shard: number;
+    increment: { eventCount: number; byteCount: number };
+    window: Doc<"siteIngestWindows"> | null;
+  }> = [];
+  for (const [shard, increment] of increments) {
+    const window = await ctx.db
+      .query("siteIngestWindows")
+      .withIndex("by_siteId_and_shard_and_windowStart", (range) =>
+        range
+          .eq("siteId", siteId)
+          .eq("shard", shard)
+          .eq("windowStart", windowStart),
+      )
+      .unique();
+    if (
+      (window?.eventCount ?? 0) + increment.eventCount >
+        MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
+      (window?.byteCount ?? 0) + increment.byteCount >
+        MAX_SITE_BYTES_PER_SHARD_WINDOW
+    ) {
+      fail("RATE_LIMITED", "site ingest budget exceeded", {
+        siteShard: shard,
+        windowMs: RATE_LIMIT_WINDOW_MS,
+        eventLimitPerShard: MAX_SITE_EVENTS_PER_SHARD_WINDOW,
+        byteLimitPerShard: MAX_SITE_BYTES_PER_SHARD_WINDOW,
+        siteEventLimit:
+          SITE_INGEST_SHARDS * MAX_SITE_EVENTS_PER_SHARD_WINDOW,
+        siteByteLimit:
+          SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
+      });
+    }
+    windows.push({ shard, increment, window });
+  }
+
+  for (const { shard, increment, window } of windows) {
+    if (window) {
+      await ctx.db.patch("siteIngestWindows", window._id, {
+        eventCount: window.eventCount + increment.eventCount,
+        byteCount: window.byteCount + increment.byteCount,
+        updatedAt: now,
+      });
+    } else {
+      await ctx.db.insert("siteIngestWindows", {
+        siteId,
+        shard,
+        windowStart,
+        eventCount: increment.eventCount,
+        byteCount: increment.byteCount,
+        updatedAt: now,
+      });
+    }
+  }
+  return eventByteCounts;
+}
+
+async function consumeRateLimit(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  sessionId: string,
+  byteCount: number,
+  now: number,
+): Promise<boolean> {
+  const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+  const window = await ctx.db
+    .query("ingestWindows")
+    .withIndex("by_siteId_and_sessionId_and_windowStart", (range) =>
+      range
+        .eq("siteId", siteId)
+        .eq("sessionId", sessionId)
+        .eq("windowStart", windowStart),
+    )
+    .unique();
+  if (
+    window &&
+    (window.eventCount >= MAX_EVENTS_PER_SESSION_WINDOW ||
+      window.byteCount + byteCount > MAX_BYTES_PER_SESSION_WINDOW)
+  ) {
+    return false;
+  }
+  if (window) {
+    await ctx.db.patch("ingestWindows", window._id, {
+      eventCount: window.eventCount + 1,
+      byteCount: window.byteCount + byteCount,
+      updatedAt: now,
+    });
+  } else {
+    await ctx.db.insert("ingestWindows", {
+      siteId,
+      sessionId,
+      windowStart,
+      eventCount: 1,
+      byteCount,
+      updatedAt: now,
+    });
+  }
+  return true;
+}
+
+async function updateLiveSession(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  context: IngestContext,
+  session: SessionState,
+  now: number,
+) {
+  const existing = await ctx.db
+    .query("liveSessions")
+    .withIndex("by_siteId_and_sessionId", (range) =>
+      range.eq("siteId", siteId).eq("sessionId", event.sessionId),
+    )
+    .unique();
+  const renewsExpiry =
+    !existing ||
+    existing.expiresAt <= now ||
+    event.type === "pageview" ||
+    event.type === "heartbeat";
+  const deadline = renewsExpiry
+    ? now + LIVE_SESSION_TTL_MS
+    : existing.expiresAt;
+  if (existing) {
+    await ctx.db.patch("liveSessions", existing._id, {
+      visitorId: event.visitorId,
+      path: event.path,
+      lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
+      expiresAt: deadline,
+      ...(context.country ? { country: context.country } : {}),
+      ...(context.city ? { city: context.city } : {}),
+      ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+      ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+      browser: context.browser ?? existing.browser,
+      os: context.os ?? existing.os,
+      source: session.source,
+      device: context.device ?? existing.device,
+      pageviewCount: session.pageviewCount,
+      eventCount: session.eventCount,
+      ...(event.type === "heartbeat"
+        ? { lastHeartbeatEventId: event.eventId }
+        : {}),
+    });
+    return;
+  }
+
+  const liveSessionId = await ctx.db.insert("liveSessions", {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    path: event.path,
+    startedAt: event.timestamp,
+    lastSeenAt: event.timestamp,
+    expiresAt: deadline,
+    expiryChainDeadline: deadline,
+    ...(context.country ? { country: context.country } : {}),
+    ...(context.city ? { city: context.city } : {}),
+    ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
+    ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+    browser: context.browser ?? "Other",
+    os: context.os ?? "Other",
+    source: session.source,
+    device: context.device ?? "unknown",
+    pageviewCount: session.pageviewCount,
+    eventCount: session.eventCount,
+    ...(event.type === "heartbeat"
+      ? { lastHeartbeatEventId: event.eventId }
+      : {}),
+  });
+  await ctx.scheduler.runAt(deadline, internal.live.expire, {
+    liveSessionId,
+    scheduledDeadline: deadline,
+  });
+}
+
+function goalMatches(goal: Doc<"goals">, event: TrackerEvent): boolean {
+  if (goal.matchType === "pageview") {
+    return event.type === "pageview" && event.path === goal.matchValue;
+  }
+  return event.type === "custom" && event.name === goal.matchValue;
+}
+
+async function advanceFunnel(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  funnel: Doc<"funnels">,
+  event: TrackerEvent,
+) {
+  const progress = await ctx.db
+    .query("funnelProgress")
+    .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
+      range
+        .eq("siteId", siteId)
+        .eq("funnelId", funnel._id)
+        .eq("visitorId", event.visitorId),
+    )
+    .unique();
+  const matchesFirst = stepMatches(funnel.steps[0], event);
+  if (!progress) {
+    if (!matchesFirst) return;
+    await ctx.db.insert("funnelProgress", {
+      siteId,
+      funnelId: funnel._id,
+      visitorId: event.visitorId,
+      sessionId: event.sessionId,
+      currentStep: 1,
+      startedAt: event.timestamp,
+      lastStepAt: event.timestamp,
+    });
+    await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+    return;
+  }
+  if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt) return;
+  if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
+    if (!matchesFirst) return;
+    await ctx.db.patch("funnelProgress", progress._id, {
+      sessionId: event.sessionId,
+      currentStep: 1,
+      startedAt: event.timestamp,
+      lastStepAt: event.timestamp,
+      completedAt: undefined,
+    });
+    await incrementFunnelStep(ctx, siteId, funnel._id, event, 0, false);
+    return;
+  }
+  const nextStep = funnel.steps[progress.currentStep];
+  if (!nextStep || !stepMatches(nextStep, event)) return;
+  const currentStep = progress.currentStep + 1;
+  const completed = currentStep === funnel.steps.length;
+  await ctx.db.patch("funnelProgress", progress._id, {
+    sessionId: event.sessionId,
+    currentStep,
+    lastStepAt: event.timestamp,
+    ...(completed ? { completedAt: event.timestamp } : {}),
+  });
+  await incrementFunnelStep(
+    ctx,
+    siteId,
+    funnel._id,
+    event,
+    progress.currentStep,
+    completed,
+  );
+}
+
+function stepMatches(
+  step: { type: "pageview" | "custom"; value: string },
+  event: TrackerEvent,
+): boolean {
+  return step.type === "pageview"
+    ? event.type === "pageview" && event.path === step.value
+    : event.type === "custom" && event.name === step.value;
+}
+
+async function incrementFunnelStep(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  funnelId: Id<"funnels">,
+  event: TrackerEvent,
+  step: number,
+  completed: boolean,
+) {
+  const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+  const shard = stableHash(event.visitorId) % AGGREGATE_SHARDS;
+  const bucket = await ctx.db
+    .query("funnelStepBuckets")
+    .withIndex(
+      "by_siteId_and_funnelId_and_bucketStart_and_step_and_shard",
+      (range) =>
+        range
+          .eq("siteId", siteId)
+          .eq("funnelId", funnelId)
+          .eq("bucketStart", bucketStart)
+          .eq("step", step)
+          .eq("shard", shard),
+    )
+    .unique();
+  if (bucket) {
+    await ctx.db.patch("funnelStepBuckets", bucket._id, {
+      entrants: bucket.entrants + 1,
+      completions: bucket.completions + (completed ? 1 : 0),
+    });
+  } else {
+    await ctx.db.insert("funnelStepBuckets", {
+      siteId,
+      funnelId,
+      bucketStart,
+      step,
+      shard,
+      entrants: 1,
+      completions: completed ? 1 : 0,
+    });
+  }
+}
+
+async function accountConversion(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  session: SessionState,
+  event: TrackerEvent,
+  trustedGoalKey: string | undefined,
+) {
+  const revenueCents = event.revenueCents ?? 0;
+  const currency = event.currency ?? "USD";
+  const candidate = attributionSnapshot(session);
+  const attribution =
+    candidate && attributionApplies(candidate, event.timestamp)
+      ? candidate
+      : undefined;
+  const affiliate = attribution
+    ? await ctx.db.get("affiliates", attribution.affiliateId)
+    : null;
+  const commissionAffiliate =
+    affiliate?.active && affiliate.siteId === siteId ? affiliate : null;
+  const commissionCents = commissionAffiliate
+    ? Math.round((revenueCents * commissionAffiliate.commissionBps) / 10_000)
+    : 0;
+  await ctx.db.insert("conversions", {
+    siteId,
+    eventId: event.eventId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    timestamp: event.timestamp,
+    path: event.path,
+    revenueCents,
+    currency,
+    source: "trusted",
+    ...(trustedGoalKey ? { goalKey: trustedGoalKey } : {}),
+    ...(attribution ?? {}),
+    commissionCents,
+  });
+  if (commissionAffiliate && attribution) {
+    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+    const stats = await ctx.db
+      .query("affiliateDailyStats")
+      .withIndex(
+        "by_siteId_and_affiliateId_and_bucketStart_and_shard",
+        (range) =>
+          range
+            .eq("siteId", siteId)
+            .eq("affiliateId", commissionAffiliate._id)
+            .eq("bucketStart", bucketStart)
+            .eq("shard", shard),
+      )
+      .unique();
+    if (stats) {
+      await ctx.db.patch("affiliateDailyStats", stats._id, {
+        conversions: stats.conversions + 1,
+        revenueCents: stats.revenueCents + revenueCents,
+        commissionCents: stats.commissionCents + commissionCents,
+      });
+    } else {
+      await ctx.db.insert("affiliateDailyStats", {
+        siteId,
+        affiliateId: commissionAffiliate._id,
+        affiliateSlug: attribution.affiliateSlug,
+        bucketStart,
+        shard,
+        conversions: 1,
+        revenueCents,
+        commissionCents,
+      });
+    }
+  }
+}
+
+function foldAggregates(
+  aggregateDeltas: AggregateDeltas,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  session: SessionState,
+  dimensions: Dimension[],
+  recordTelemetry: boolean,
+  financialConversion: boolean,
+) {
+  const shard = stableHash(event.sessionId) % AGGREGATE_SHARDS;
+  const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
+  const dimensionDeltas = dimensions.map(
+    (dimension): DimensionDelta => ({
+      ...dimension,
+      count: Number(recordTelemetry),
+      revenueCents,
+    }),
+  );
+  for (const granularity of ["hour", "day"] as const) {
+    const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
+    const bucketStart = Math.floor(event.timestamp / interval) * interval;
+    const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
+    let delta = aggregateDeltas.get(key);
+    if (!delta) {
+      delta = {
+        siteId,
+        granularity,
+        bucketStart,
+        shard,
+        events: 0,
+        pageviews: 0,
+        clicks: 0,
+        customEvents: 0,
+        outboundClicks: 0,
+        sessions: 0,
+        visitors: 0,
+        conversions: 0,
+        revenueCents: 0,
+        dimensions: [],
+      };
+      aggregateDeltas.set(key, delta);
+    }
+    const eventCount = Number(recordTelemetry);
+    delta.events += eventCount;
+    delta.pageviews += Number(recordTelemetry && event.type === "pageview");
+    delta.clicks += Number(recordTelemetry && event.type === "click");
+    delta.customEvents += Number(recordTelemetry && event.type === "custom");
+    delta.outboundClicks += Number(
+      recordTelemetry && event.type === "outbound",
+    );
+    delta.sessions += Number(recordTelemetry && session.isNewSession);
+    delta.visitors += recordTelemetry
+      ? granularity === "hour"
+        ? Number(session.isNewVisitorHour)
+        : Number(session.isNewVisitorDay)
+      : 0;
+    delta.conversions += Number(financialConversion);
+    delta.revenueCents += revenueCents;
+    delta.dimensions = mergeDimensions(delta.dimensions, dimensionDeltas);
+  }
+}
+
+async function flushAggregates(
+  ctx: MutationCtx,
+  aggregateDeltas: AggregateDeltas,
+) {
+  for (const delta of aggregateDeltas.values()) {
+    const bucket = await ctx.db
+      .query("aggregateBuckets")
+      .withIndex(
+        "by_siteId_and_granularity_and_bucketStart_and_shard",
+        (range) =>
+          range
+            .eq("siteId", delta.siteId)
+            .eq("granularity", delta.granularity)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("shard", delta.shard),
+      )
+      .unique();
+    const dimensions = mergeDimensions(
+      bucket?.dimensions ?? [],
+      delta.dimensions,
+    );
+    if (bucket) {
+      await ctx.db.patch("aggregateBuckets", bucket._id, {
+        events: bucket.events + delta.events,
+        pageviews: bucket.pageviews + delta.pageviews,
+        clicks: bucket.clicks + delta.clicks,
+        customEvents: bucket.customEvents + delta.customEvents,
+        outboundClicks: bucket.outboundClicks + delta.outboundClicks,
+        sessions: bucket.sessions + delta.sessions,
+        visitors: bucket.visitors + delta.visitors,
+        conversions: bucket.conversions + delta.conversions,
+        revenueCents: bucket.revenueCents + delta.revenueCents,
+        dimensions,
+      });
+    } else {
+      await ctx.db.insert("aggregateBuckets", {
+        siteId: delta.siteId,
+        granularity: delta.granularity,
+        bucketStart: delta.bucketStart,
+        shard: delta.shard,
+        events: delta.events,
+        pageviews: delta.pageviews,
+        clicks: delta.clicks,
+        customEvents: delta.customEvents,
+        outboundClicks: delta.outboundClicks,
+        sessions: delta.sessions,
+        visitors: delta.visitors,
+        conversions: delta.conversions,
+        revenueCents: delta.revenueCents,
+        dimensions,
+      });
+    }
+  }
+}
+
+function mergeDimensions(
+  current: Doc<"aggregateBuckets">["dimensions"],
+  additions: DimensionDelta[],
+): Doc<"aggregateBuckets">["dimensions"] {
+  const result = current.map((slot) => ({ ...slot }));
+  for (const addition of additions) {
+    const exact = result.find(
+      (candidate) =>
+        candidate.type === addition.type && candidate.value === addition.value,
+    );
+    if (exact) {
+      exact.count += addition.count;
+      exact.revenueCents += addition.revenueCents;
+      continue;
+    }
+
+    const typeSlots = result.filter(
+      (candidate) => candidate.type === addition.type,
+    );
+    if (typeSlots.length < DIMENSION_SLOTS) {
+      const occupied = Array.from(
+        { length: DIMENSION_SLOTS },
+        () => false,
+      );
+      for (const candidate of typeSlots) occupied[candidate.slot] = true;
+      let slot = stableHash(addition.value) % DIMENSION_SLOTS;
+      while (occupied[slot]) slot = (slot + 1) % DIMENSION_SLOTS;
+      result.push({
+        type: addition.type,
+        slot,
+        value: addition.value,
+        count: addition.count,
+        revenueCents: addition.revenueCents,
+      });
+      continue;
+    }
+    if (addition.count === 0) continue;
+
+    let replacement = typeSlots[0];
+    for (const candidate of typeSlots.slice(1)) {
+      if (candidate.count < replacement.count) replacement = candidate;
+    }
+    replacement.value = addition.value;
+    replacement.count += addition.count;
+    replacement.revenueCents = addition.revenueCents;
+  }
+  return result;
+}
+
+function stringProperty(
+  properties: TrackerEvent["properties"],
+  key: string,
+): string | undefined {
+  const value = properties?.[key];
+  return typeof value === "string" ? cleanString(value, 120) || undefined : undefined;
+}