@iann29/rastro 0.3.0 → 0.5.0

package/src/component/ingest.ts

@@ -14,6 +14,8 @@ import {
   LIVE_SESSION_TTL_MS,
   LIVE_SWEEP_INTERVAL_MS,
   MAX_BATCH_EVENTS,
+  MAX_BYTES_PER_CLIENT_WINDOW,
+  MAX_EVENTS_PER_CLIENT_WINDOW,
   MAX_SITE_BYTES_PER_SHARD_WINDOW,
   MAX_SITE_EVENTS_PER_SHARD_WINDOW,
   MAX_BYTES_PER_SESSION_WINDOW,
@@ -52,6 +54,7 @@ import {
 } from "./eventStore.js";
 import {
   cleanString,
+  externalReferrer,
   isValidMoney,
   measureBatchBytes,
   normalizeRoute,
@@ -72,6 +75,8 @@ import {
   ingestResultValidator,
   trackerEventValidator,
   trustedConversionValidator,
+  type Channel,
+  type Evidence,
   type IngestContext,
   type TrackerEvent,
 } from "./validators.js";
@@ -83,6 +88,7 @@ import {
   type RouteRollupDelta,
 } from "./rollupStore.js";
 import { localDayStart, localDayTimezone } from "./localTime.js";
+import { classifyOrigin, platformDimensionValue } from "./origin.js";
 
 type DimensionType =
   | "source"
@@ -94,7 +100,10 @@ type DimensionType =
   | "event"
   | "goal"
   | "medium"
-  | "outbound";
+  | "outbound"
+  | "platform"
+  | "channel"
+  | "evidence";
 
 type Dimension = { type: DimensionType; value: string };
 type DimensionDelta = Dimension & {
@@ -123,7 +132,7 @@ type AggregateDelta = {
   continuedSessions: number;
   newVisitors: number;
 };
-type AggregateDeltas = Map<string, AggregateDelta>;
+export type AggregateDeltas = Map<string, AggregateDelta>;
 type AttributionSnapshot = {
   affiliateId: Id<"affiliates">;
   affiliateSlug: string;
@@ -143,6 +152,10 @@ type SessionState = {
   continued: boolean;
   startedAt: number;
   source: string;
+  /** The first touch's origin; absent on sessions recorded before it. */
+  platform?: string;
+  channel?: Channel;
+  evidence?: Evidence;
   visitorKey: string;
   utmCampaign?: string;
   utmMedium?: string;
@@ -214,6 +227,8 @@ export const ingestBatch = mutation({
     origin: v.string(),
     events: v.array(trackerEventValidator),
     context: v.optional(ingestContextValidator),
+    // The HTTP action's salted daily client-network key (see identity.ts).
+    clientKey: v.optional(v.string()),
   },
   returns: ingestResultValidator,
   handler: async (ctx, args) => {
@@ -245,6 +260,15 @@ export const ingestBatch = mutation({
 
     const now = Date.now();
     await ensureAnalyticsControl(ctx, args.siteId, now);
+    if (args.clientKey !== undefined) {
+      await consumeClientRateLimit(
+        ctx,
+        args.siteId,
+        args.clientKey,
+        eventByteCounts,
+        now,
+      );
+    }
     await consumeSiteRateLimit(
       ctx,
       args.siteId,
@@ -389,7 +413,10 @@ export const ingestBatch = mutation({
           rejected += 1;
           continue;
         }
-        const durationDeltaMs = await accountHeartbeat(
+        // A heartbeat is presence, not telemetry: it moves the session's
+        // lastSeenAt and live row only. The time it proves reaches the
+        // duration counters once, when the visitor departs (live.ts).
+        await accountHeartbeat(
           ctx,
           args.siteId,
           existingSession,
@@ -398,16 +425,6 @@ export const ingestBatch = mutation({
           now,
           batchState,
         );
-        // A heartbeat is presence, not telemetry, but it is what keeps a
-        // session's duration honest while the visitor reads: the growth is
-        // folded into the buckets the heartbeat landed in.
-        foldSessionDuration(
-          aggregateDeltas,
-          args.siteId,
-          event,
-          durationDeltaMs,
-          timezone,
-        );
         accepted += 1;
         continue;
       }
@@ -461,7 +478,8 @@ export const ingestBatch = mutation({
             foldSessionDuration(
               aggregateDeltas,
               args.siteId,
-              event,
+              event.sessionId,
+              event.timestamp,
               durationMs - existingSession.durationMs,
               timezone,
             );
@@ -560,6 +578,7 @@ export const ingestBatch = mutation({
         reportRollupDeltas,
         recordTelemetry: true,
         timezone,
+        siteDomains: site.domains,
         financialConversion: false,
         trustedGoalKey: undefined,
         now,
@@ -690,6 +709,7 @@ export const trackConversion = mutation({
       reportRollupDeltas,
       recordTelemetry: existingTelemetry === null && !telemetry.saturated,
       timezone: localDayTimezone(site.timezone),
+      siteDomains: site.domains,
       financialConversion: true,
       trustedGoalKey,
       now: Date.now(),
@@ -828,6 +848,8 @@ async function accountEvent(
     recordTelemetry: boolean;
     /** The site's timezone when it keeps local-day buckets. */
     timezone?: string;
+    /** The site's own domains: a referrer on them is not an origin. */
+    siteDomains: string[];
     financialConversion: boolean;
     trustedGoalKey: string | undefined;
     now: number;
@@ -841,6 +863,7 @@ async function accountEvent(
         input.siteId,
         event,
         input.context,
+        input.siteDomains,
         input.existingSession,
         input.financialConversion,
         input.reportRollupDeltas,
@@ -962,6 +985,23 @@ async function accountEvent(
     event.type === "outbound" ? publicUrlHost(event.href) : undefined;
   const dimensions: Dimension[] = [
     { type: "source", value: session.source },
+    // A session recorded before origins carries none, and fills no slot.
+    ...(session.platform && session.channel && session.evidence
+      ? ([
+          {
+            type: "platform",
+            // The value carries the channel too when it is not the
+            // platform's default, so the pair survives without its own
+            // dimension (splitPlatformDimension reads it back).
+            value: platformDimensionValue({
+              platform: session.platform,
+              channel: session.channel,
+            }),
+          },
+          { type: "channel", value: session.channel },
+          { type: "evidence", value: session.evidence },
+        ] satisfies Dimension[])
+      : []),
     ...campaignDimensions,
     ...(outboundHost
       ? [{ type: "outbound", value: outboundHost } satisfies Dimension]
@@ -1028,6 +1068,9 @@ async function accountTrustedSession(
     continued: false,
     startedAt: existing.startedAt,
     source: existing.source,
+    platform: existing.platform,
+    channel: existing.channel,
+    evidence: existing.evidence,
     utmCampaign: existing.utmCampaign,
     utmMedium: existing.utmMedium,
     entryPath: existing.entryPath,
@@ -1052,6 +1095,7 @@ async function upsertSession(
   siteId: Id<"sites">,
   event: TrackerEvent,
   context: IngestContext,
+  siteDomains: string[],
   existing: CachedDocument<"sessions"> | null,
   financialConversion: boolean,
   reportRollupDeltas: ReportRollupDeltas,
@@ -1149,6 +1193,9 @@ async function upsertSession(
       continued: event.type === "pageview" && existing.pageviewCount === 1,
       startedAt,
       source: existing.source,
+      platform: existing.platform,
+      channel: existing.channel,
+      evidence: existing.evidence,
       utmCampaign: existing.utmCampaign,
       utmMedium: existing.utmMedium,
       entryPath: existing.entryPath,
@@ -1199,6 +1246,17 @@ async function upsertSession(
       .first();
     if (alias) isNewVisitor = false;
   }
+  // First touch only, like `source`: the session keeps where it came from.
+  // An affiliate channel needs this landing's own `ref` to have resolved; a
+  // visitor's standing attribution alone is not how this visit arrived.
+  const origin = classifyOrigin({
+    utmSource,
+    utmMedium,
+    clickIds: event.clid,
+    referrer: externalReferrer(event.referrer, siteDomains),
+    inApp: context.inApp,
+    affiliate: event.affiliateSlug !== undefined && attribution !== undefined,
+  });
   const fields: Omit<Doc<"sessions">, "_id" | "_creationTime"> = {
     siteId,
     sessionId: event.sessionId,
@@ -1211,6 +1269,7 @@ async function upsertSession(
     newVisitor: isNewVisitor,
     ...(event.referrer ? { referrer: event.referrer } : {}),
     source,
+    ...origin,
     ...(utmSource ? { utmSource } : {}),
     ...(utmMedium ? { utmMedium } : {}),
     ...(utmCampaign ? { utmCampaign } : {}),
@@ -1267,6 +1326,7 @@ async function upsertSession(
     continued: false,
     startedAt: event.timestamp,
     source,
+    ...origin,
     utmCampaign,
     utmMedium,
     entryPath: event.path,
@@ -1421,6 +1481,12 @@ async function attributionForEvent(
   };
 }
 
+/**
+ * Moves the session's lastSeenAt and its live row. `durationMs` stays: it is
+ * what the duration counters already hold, and the time a heartbeat proves
+ * is counted when the visitor departs (live.ts), so an idle tab never
+ * rewrites an aggregate bucket.
+ */
 async function accountHeartbeat(
   ctx: MutationCtx,
   siteId: Id<"sites">,
@@ -1429,7 +1495,7 @@ async function accountHeartbeat(
   context: IngestContext,
   now: number,
   batchState?: BatchState,
-): Promise<number> {
+): Promise<void> {
   const startedAt = Math.min(session.startedAt, event.timestamp);
   const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
   const updates = {
@@ -1437,7 +1503,6 @@ async function accountHeartbeat(
     lastSeenAt,
     exitPath:
       event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
-    durationMs: Math.max(0, lastSeenAt - startedAt),
     ...(context.country ? { country: context.country } : {}),
     ...(context.city ? { city: context.city } : {}),
     ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
@@ -1456,7 +1521,6 @@ async function accountHeartbeat(
   } else {
     await ctx.db.patch("sessions", session._id, updates);
   }
-  const durationDeltaMs = Math.max(0, updates.durationMs - session.durationMs);
   await updateLiveSession(
     ctx,
     siteId,
@@ -1486,7 +1550,54 @@ async function accountHeartbeat(
     now,
     batchState,
   );
-  return durationDeltaMs;
+}
+
+/**
+ * One client network's budget per site and minute, kept in the session
+ * windows' table under `~client:<key>`, a key no sanitized session ID can take.
+ * The key is the salted daily hash the HTTP action derived; the address never
+ * reaches this mutation. It is one row per client, so only that client's own
+ * concurrent requests contend on it.
+ */
+async function consumeClientRateLimit(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  clientKey: string,
+  eventByteCounts: number[],
+  now: number,
+): Promise<void> {
+  const windowStart =
+    Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+  const sessionId = `~client:${clientKey}`;
+  const rows = await ctx.db
+    .query("ingestWindows")
+    .withIndex("by_siteId_and_sessionId_and_windowStart", (range) =>
+      range.eq("siteId", siteId).eq("sessionId", sessionId),
+    )
+    .order("desc")
+    .take(1);
+  const window = rows[0] ?? null;
+  const current = window?.windowStart === windowStart ? window : null;
+  const eventCount = (current?.eventCount ?? 0) + eventByteCounts.length;
+  const byteCount =
+    (current?.byteCount ?? 0) +
+    eventByteCounts.reduce((total, bytes) => total + bytes, 0);
+  if (
+    eventCount > MAX_EVENTS_PER_CLIENT_WINDOW ||
+    byteCount > MAX_BYTES_PER_CLIENT_WINDOW
+  ) {
+    fail("RATE_LIMITED", "client ingest budget exceeded", {
+      windowMs: RATE_LIMIT_WINDOW_MS,
+      eventLimitPerClient: MAX_EVENTS_PER_CLIENT_WINDOW,
+      byteLimitPerClient: MAX_BYTES_PER_CLIENT_WINDOW,
+    });
+  }
+  const fields = { windowStart, eventCount, byteCount, updatedAt: now };
+  if (window) {
+    await ctx.db.patch("ingestWindows", window._id, fields);
+  } else {
+    await ctx.db.insert("ingestWindows", { siteId, sessionId, ...fields });
+  }
 }
 
 async function consumeSiteRateLimit(
@@ -1872,11 +1983,10 @@ async function advanceFunnel(
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
     return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
-  if (
-    progress.completedAt !== undefined ||
-    event.timestamp < progress.lastStepAt
-  )
-    return;
+  if (event.timestamp < progress.lastStepAt) return;
+  // An expired window restarts on a first-step event even after a completion,
+  // so a returning buyer can complete again; inside the window a completed
+  // funnel stays done.
   if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
     if (!matchesFirst) return;
     const updated = {
@@ -1910,6 +2020,7 @@ async function advanceFunnel(
     foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
     return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
   }
+  if (progress.completedAt !== undefined) return;
   const nextStep = funnel.steps[progress.currentStep];
   if (!nextStep || !stepMatches(nextStep, event)) return;
   const currentStep = progress.currentStep + 1;
@@ -2428,24 +2539,26 @@ function aggregateBucketStart(
   return Math.floor(timestamp / interval) * interval;
 }
 
-/** Folds session duration growth without telemetry, for heartbeats and leaves. */
-function foldSessionDuration(
+/**
+ * Folds session duration growth without telemetry: a departure's final
+ * interval, or the heartbeat time of a visitor the live sweep retired. The
+ * shard follows `shardKey`: the session for a departure, the site for a
+ * sweep.
+ */
+export function foldSessionDuration(
   aggregateDeltas: AggregateDeltas,
   siteId: Id<"sites">,
-  event: TrackerEvent,
+  shardKey: string,
+  timestamp: number,
   durationDeltaMs: number,
   timezone?: string,
 ) {
   if (durationDeltaMs <= 0) return;
   for (const granularity of aggregateGranularities(timezone)) {
     const shard =
-      stableHash(event.sessionId) %
+      stableHash(shardKey) %
       (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
-    const bucketStart = aggregateBucketStart(
-      granularity,
-      event.timestamp,
-      timezone,
-    );
+    const bucketStart = aggregateBucketStart(granularity, timestamp, timezone);
     const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
     const delta = aggregateDeltas.get(key) ?? {
       siteId,
@@ -2472,7 +2585,7 @@ function foldSessionDuration(
   }
 }
 
-async function flushAggregates(
+export async function flushAggregates(
   ctx: MutationCtx,
   aggregateDeltas: AggregateDeltas,
 ) {

package/src/component/live.ts

@@ -1,7 +1,15 @@
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
 import { internalMutation } from "./_generated/server.js";
 import { LIVE_SWEEP_INTERVAL_MS } from "./constants.js";
+import {
+  flushAggregates,
+  foldSessionDuration,
+  type AggregateDeltas,
+} from "./ingest.js";
+import { localDayTimezone } from "./localTime.js";
 
 const SWEEP_BATCH_SIZE = 500;
 
@@ -58,7 +66,7 @@ export const expire = internalMutation({
       }
       return { expired: false, rescheduled: false, stale: false };
     }
-    await ctx.db.delete("liveSessions", liveSession._id);
+    await retireLiveSessions(ctx, liveSession.siteId, [liveSession]);
     return { expired: true, rescheduled: false, stale: false };
   },
 });
@@ -89,9 +97,7 @@ export const sweep = internalMutation({
         range.eq("siteId", args.siteId).lte("expiresAt", now),
       )
       .take(SWEEP_BATCH_SIZE);
-    for (const session of expired) {
-      await ctx.db.delete("liveSessions", session._id);
-    }
+    await retireLiveSessions(ctx, args.siteId, expired);
 
     const next = await ctx.db
       .query("liveSessions")
@@ -115,3 +121,44 @@ export const sweep = internalMutation({
     return { expired: expired.length, rescheduled: true, stale: false };
   },
 });
+
+/**
+ * Deletes departed live rows and counts the time their heartbeats proved. A
+ * heartbeat only moves its session's `lastSeenAt`, and `durationMs` is what
+ * the duration counters already hold, so the difference reaches the buckets
+ * here, once, in the hour the visitor was last seen. It lands in one shard
+ * per site, so a sweep rewrites a few buckets however many visitors it
+ * retires.
+ */
+async function retireLiveSessions(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  rows: Doc<"liveSessions">[],
+) {
+  if (rows.length === 0) return;
+  const site = await ctx.db.get("sites", siteId);
+  const timezone = localDayTimezone(site?.timezone);
+  const aggregateDeltas: AggregateDeltas = new Map();
+  for (const row of rows) {
+    await ctx.db.delete("liveSessions", row._id);
+    const session = await ctx.db
+      .query("sessions")
+      .withIndex("by_siteId_and_sessionId", (range) =>
+        range.eq("siteId", siteId).eq("sessionId", row.sessionId),
+      )
+      .unique();
+    if (!session) continue;
+    const durationMs = session.lastSeenAt - session.startedAt;
+    if (durationMs <= session.durationMs) continue;
+    await ctx.db.patch("sessions", session._id, { durationMs });
+    foldSessionDuration(
+      aggregateDeltas,
+      siteId,
+      siteId,
+      session.lastSeenAt,
+      durationMs - session.durationMs,
+      timezone,
+    );
+  }
+  await flushAggregates(ctx, aggregateDeltas);
+}