@iann29/rastro 0.1.0-alpha.7 → 0.1.0-alpha.9

package/src/component/ingest.ts

@@ -10,6 +10,7 @@ import {
   HOURLY_AGGREGATE_SHARDS,
   DIMENSION_SLOTS,
   HOUR_MS,
+  LIVE_LEAVE_GRACE_MS,
   LIVE_SESSION_TTL_MS,
   LIVE_SWEEP_INTERVAL_MS,
   MAX_BATCH_EVENTS,
@@ -20,10 +21,13 @@ import {
   MAX_FUNNELS_PER_SITE,
   MAX_FUNNEL_STEPS,
   MAX_GOALS_PER_SITE,
+  MAX_SITE_MAP_ROUTES_PER_DAY,
   RATE_LIMIT_WINDOW_MS,
   REPORT_ROLLUP_SHARDS,
+  ROUTE_TRANSITION_SLOTS,
   SECONDARY_AGGREGATE_SHARDS,
   SITE_INGEST_SHARDS,
+  SITE_MAP_OTHER_ROUTE,
   VITAL_PAGE_SLOTS,
 } from "./constants.js";
 import {
@@ -57,6 +61,7 @@ import {
   cleanString,
   isValidMoney,
   measureBatchBytes,
+  normalizeRoute,
   originAllowed,
   sanitizeContext,
   sanitizeCurrency,
@@ -153,11 +158,25 @@ type VitalRollupDelta = {
     { count: number; sum: number; histogram: number[] }
   >;
 };
+type RouteRollupDelta = {
+  siteId: Id<"sites">;
+  bucketStart: number;
+  shard: number;
+  route: string;
+  pageviews: number;
+  entries: number;
+  continued: number;
+  dwellMs: number;
+  dwellSamples: number;
+  visitorSketch: VisitorSketch;
+  outgoing: Map<string, number>;
+};
 type ReportRollupDeltas = {
   goals: Map<string, GoalRollupDelta>;
   funnels: Map<string, FunnelRollupDelta>;
   affiliates: Map<string, AffiliateRollupDelta>;
   vitals: Map<string, VitalRollupDelta>;
+  routes: Map<string, RouteRollupDelta>;
 };
 type AttributionSnapshot = {
   affiliateId: Id<"affiliates">;
@@ -297,7 +316,12 @@ export const ingestBatch = mutation({
       const bucketStart = eventBucketStart(event.timestamp);
       const groupKey = eventBatchKey(event.sessionId, bucketStart);
       requestGroups.add(groupKey);
-      if (event.type === "heartbeat" || event.type === "vital") continue;
+      if (
+        event.type === "heartbeat" ||
+        event.type === "vital" ||
+        event.type === "leave"
+      )
+        continue;
       const eventIds = eventIdsByGroup.get(groupKey) ?? new Set<string>();
       if (eventIds.has(event.eventId)) {
         duplicateInBatch.add(eventIndex);
@@ -404,6 +428,50 @@ export const ingestBatch = mutation({
         continue;
       }
 
+      if (event.type === "leave") {
+        // A departure only shortens live presence: it stores no telemetry,
+        // touches no aggregates, and never extends anything, so a replayed
+        // or late leave is inert. The read side filters by expiresAt and the
+        // sweep deletes the row, so the dashboard sees the exit in seconds.
+        if (!existingSession) {
+          rejected += 1;
+          continue;
+        }
+        if (
+          !(await consumeRateLimit(
+            ctx,
+            args.siteId,
+            event.sessionId,
+            eventByteCounts[eventIndex],
+            now,
+            sessionRateWindows,
+          ))
+        ) {
+          rejected += 1;
+          continue;
+        }
+        const liveState = await loadLiveSession(
+          ctx,
+          args.siteId,
+          event.sessionId,
+          batchState,
+        );
+        const liveSession = liveState.current;
+        if (liveSession) {
+          const expiresAt = Math.min(
+            liveSession.expiresAt,
+            now + LIVE_LEAVE_GRACE_MS,
+          );
+          if (expiresAt < liveSession.expiresAt) {
+            await ctx.db.patch("liveSessions", liveSession._id, { expiresAt });
+            liveState.current = { ...liveSession, expiresAt };
+          }
+          await ensureLiveSweep(ctx, args.siteId, now, batchState);
+        }
+        accepted += 1;
+        continue;
+      }
+
       if (event.type === "vital") {
         // A vital annotates telemetry its session already produced; it never
         // creates sessions, live presence, event batches, or aggregate counts.
@@ -766,6 +834,17 @@ async function accountEvent(
         input.batchState,
       )
     : await accountTrustedSession(ctx, input.existingSession, event);
+  if (input.recordTelemetry && event.type === "pageview") {
+    // `existingSession` is the snapshot from before `upsertSession`, so it
+    // still carries the route the visitor is leaving.
+    foldRouteRollups(
+      input.reportRollupDeltas,
+      input.siteId,
+      event,
+      session,
+      input.existingSession,
+    );
+  }
 
   const matchedGoalKeys: string[] = [];
   const matchingGoals = input.goals.filter(
@@ -979,6 +1058,14 @@ async function upsertSession(
       eventCount: existing.eventCount + 1,
       pageviewCount:
         existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
+      ...(event.type === "pageview"
+        ? {
+            lastPageviewAt: Math.max(
+              existing.lastPageviewAt ?? 0,
+              event.timestamp,
+            ),
+          }
+        : {}),
       durationMs: Math.max(0, lastSeenAt - startedAt),
       conversionCount: existing.conversionCount + Number(financialConversion),
       revenueCents:
@@ -1077,6 +1164,7 @@ async function upsertSession(
     device: context.device ?? "unknown",
     eventCount: 1,
     pageviewCount: event.type === "pageview" ? 1 : 0,
+    ...(event.type === "pageview" ? { lastPageviewAt: event.timestamp } : {}),
     durationMs: 0,
     conversionCount: Number(financialConversion),
     revenueCents: financialConversion ? (event.revenueCents ?? 0) : 0,
@@ -1532,6 +1620,11 @@ async function updateLiveSession(
       device: context.device ?? existing.device,
       pageviewCount: session.pageviewCount,
       eventCount: session.eventCount,
+      // The live map animates a visitor from the route they left to the one
+      // they reached; a heartbeat or click on the same page is not a move.
+      ...(event.type === "pageview" && event.path !== existing.path
+        ? { previousPath: existing.path, movedAt: event.timestamp }
+        : {}),
       ...(event.type === "heartbeat"
         ? {
             lastHeartbeatEventId: event.eventId,
@@ -1876,6 +1969,7 @@ function createReportRollupDeltas(): ReportRollupDeltas {
     funnels: new Map(),
     affiliates: new Map(),
     vitals: new Map(),
+    routes: new Map(),
   };
 }
 
@@ -2062,6 +2156,185 @@ function foldVitalRollups(
   }
 }
 
+function foldRouteDelta(
+  deltas: ReportRollupDeltas,
+  siteId: Id<"sites">,
+  route: string,
+  timestamp: number,
+  visitorId: string,
+  apply: (delta: RouteRollupDelta) => void,
+) {
+  const coordinates = reportRollupCoordinates(
+    siteId,
+    `route:${route}`,
+    timestamp,
+    visitorId,
+  );
+  const current = deltas.routes.get(coordinates.key) ?? {
+    siteId,
+    bucketStart: coordinates.bucketStart,
+    shard: coordinates.shard,
+    route,
+    pageviews: 0,
+    entries: 0,
+    continued: 0,
+    dwellMs: 0,
+    dwellSamples: 0,
+    visitorSketch: createVisitorSketch(),
+    outgoing: new Map<string, number>(),
+  };
+  apply(current);
+  deltas.routes.set(coordinates.key, current);
+}
+
+/**
+ * One pageview feeds the site map three ways: the route it lands on gains a
+ * view (and an entry when it opens the session), the route it leaves gains
+ * the transition and the time spent there, and the session's entry route is
+ * marked as continued the first time its visitor moves on — which is what
+ * lets the report derive bounces and exits without a session-end hook.
+ */
+function foldRouteRollups(
+  deltas: ReportRollupDeltas,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  session: SessionState,
+  existing: CachedDocument<"sessions"> | null,
+) {
+  const route = normalizeRoute(event.path);
+  const priorPageviews = existing?.pageviewCount ?? 0;
+  const firstPageview = existing === null || priorPageviews === 0;
+  foldRouteDelta(
+    deltas,
+    siteId,
+    route,
+    event.timestamp,
+    event.visitorId,
+    (delta) => {
+      delta.pageviews += 1;
+      if (firstPageview) delta.entries += 1;
+      delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
+    },
+  );
+  // Out-of-order telemetry keeps its view but cannot say where it came from.
+  if (existing === null || firstPageview) return;
+  if (event.timestamp < existing.lastSeenAt) return;
+  const previous = normalizeRoute(existing.exitPath);
+  const lastPageviewAt = existing.lastPageviewAt;
+  foldRouteDelta(
+    deltas,
+    siteId,
+    previous,
+    event.timestamp,
+    event.visitorId,
+    (delta) => {
+      delta.outgoing.set(route, (delta.outgoing.get(route) ?? 0) + 1);
+      if (lastPageviewAt !== undefined && event.timestamp >= lastPageviewAt) {
+        delta.dwellMs += event.timestamp - lastPageviewAt;
+        delta.dwellSamples += 1;
+      }
+    },
+  );
+  if (priorPageviews === 1) {
+    foldRouteDelta(
+      deltas,
+      siteId,
+      normalizeRoute(existing.entryPath),
+      existing.startedAt,
+      event.visitorId,
+      (delta) => {
+        delta.continued += 1;
+      },
+    );
+  }
+}
+
+async function upsertRouteRollup(
+  ctx: MutationCtx,
+  delta: RouteRollupDelta,
+  dayRowCounts: Map<string, number>,
+) {
+  const existing = await ctx.db
+    .query("routeDailyRollups")
+    .withIndex("by_siteId_and_route_and_bucketStart_and_shard", (range) =>
+      range
+        .eq("siteId", delta.siteId)
+        .eq("route", delta.route)
+        .eq("bucketStart", delta.bucketStart)
+        .eq("shard", delta.shard),
+    )
+    .unique();
+  if (!existing && delta.route !== SITE_MAP_OTHER_ROUTE) {
+    // A new route row spends one bounded read per batch to respect the daily
+    // cap; past it, the day's remaining routes fold into "(other)".
+    const capRows = MAX_SITE_MAP_ROUTES_PER_DAY * REPORT_ROLLUP_SHARDS;
+    const dayKey = `${delta.siteId}:${delta.bucketStart}`;
+    let dayRows = dayRowCounts.get(dayKey);
+    if (dayRows === undefined) {
+      dayRows = (
+        await ctx.db
+          .query("routeDailyRollups")
+          .withIndex("by_siteId_and_bucketStart", (range) =>
+            range
+              .eq("siteId", delta.siteId)
+              .eq("bucketStart", delta.bucketStart),
+          )
+          .take(capRows + 1)
+      ).length;
+    }
+    if (dayRows >= capRows) {
+      await upsertRouteRollup(
+        ctx,
+        { ...delta, route: SITE_MAP_OTHER_ROUTE },
+        dayRowCounts,
+      );
+      return;
+    }
+    dayRowCounts.set(dayKey, dayRows + 1);
+  }
+  const outgoing = existing?.outgoing.map((entry) => ({ ...entry })) ?? [];
+  for (const [target, count] of delta.outgoing) {
+    let entry = outgoing.find((candidate) => candidate.route === target);
+    if (!entry) {
+      const named = outgoing.filter(
+        (candidate) => candidate.route !== SITE_MAP_OTHER_ROUTE,
+      ).length;
+      const route =
+        target !== SITE_MAP_OTHER_ROUTE && named >= ROUTE_TRANSITION_SLOTS
+          ? SITE_MAP_OTHER_ROUTE
+          : target;
+      entry = outgoing.find((candidate) => candidate.route === route);
+      if (!entry) {
+        entry = { route, count: 0 };
+        outgoing.push(entry);
+      }
+    }
+    entry.count += count;
+  }
+  const fields = {
+    siteId: delta.siteId,
+    bucketStart: delta.bucketStart,
+    shard: delta.shard,
+    generation: 1 as const,
+    route: delta.route,
+    pageviews: (existing?.pageviews ?? 0) + delta.pageviews,
+    entries: (existing?.entries ?? 0) + delta.entries,
+    continued: (existing?.continued ?? 0) + delta.continued,
+    dwellMs: (existing?.dwellMs ?? 0) + delta.dwellMs,
+    dwellSamples: (existing?.dwellSamples ?? 0) + delta.dwellSamples,
+    visitorSketch: unionVisitorSketches(
+      existing?.visitorSketch ?? createVisitorSketch(),
+      delta.visitorSketch,
+    ),
+    outgoing,
+  };
+  if (existing) {
+    await ctx.db.replace("routeDailyRollups", existing._id, fields);
+  } else {
+    await ctx.db.insert("routeDailyRollups", fields);
+  }
+}
+
 async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
   const existing = await ctx.db
     .query("vitalRollups")
@@ -2233,6 +2506,10 @@ async function flushReportRollups(
       await ctx.db.insert("affiliateDailyRollups", fields);
     }
   }
+  const routeDayRowCounts = new Map<string, number>();
+  for (const delta of deltas.routes.values()) {
+    await upsertRouteRollup(ctx, delta, routeDayRowCounts);
+  }
   for (const delta of deltas.vitals.values()) {
     await upsertVitalRollup(ctx, delta);
   }