@iann29/rastro 0.3.0 → 0.5.0

package/dist/component/ingest.js

@@ -1,16 +1,17 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_LEAVE_GRACE_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_LEAVE_GRACE_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_BYTES_PER_CLIENT_WINDOW, MAX_EVENTS_PER_CLIENT_WINDOW, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
 import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
 import { addToVitalHistogram, emptyVitalHistogram, vitalDevice, VITAL_ALL, } from "./vitals.js";
 import { fail } from "./errors.js";
 import { ensureAnalyticsControl } from "./coverage.js";
 import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
-import { cleanString, isValidMoney, measureBatchBytes, normalizeRoute, originAllowed, publicUrlHost, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
+import { cleanString, externalReferrer, isValidMoney, measureBatchBytes, normalizeRoute, originAllowed, publicUrlHost, sanitizeContext, sanitizeCurrency, sanitizeEvent, sanitizeOpaqueId, sanitizePath, sanitizeProperties, sanitizeSlug, sourceFromReferrer, stableHash, } from "./sanitize.js";
 import { ingestContextValidator, ingestResultValidator, trackerEventValidator, trustedConversionValidator, } from "./validators.js";
 import { createReportRollupDeltas, enqueueReportRollups, reportRollupCoordinates, } from "./rollupStore.js";
 import { localDayStart, localDayTimezone } from "./localTime.js";
+import { classifyOrigin, platformDimensionValue } from "./origin.js";
 function furtherReach(current, next) {
     if (!next)
         return current;
@@ -26,6 +27,8 @@ export const ingestBatch = mutation({
         origin: v.string(),
         events: v.array(trackerEventValidator),
         context: v.optional(ingestContextValidator),
+        // The HTTP action's salted daily client-network key (see identity.ts).
+        clientKey: v.optional(v.string()),
     },
     returns: ingestResultValidator,
     handler: async (ctx, args) => {
@@ -51,6 +54,9 @@ export const ingestBatch = mutation({
         }
         const now = Date.now();
         await ensureAnalyticsControl(ctx, args.siteId, now);
+        if (args.clientKey !== undefined) {
+            await consumeClientRateLimit(ctx, args.siteId, args.clientKey, eventByteCounts, now);
+        }
         await consumeSiteRateLimit(ctx, args.siteId, args.events, eventByteCounts, now);
         const context = sanitizeContext(args.context);
         const timezone = localDayTimezone(site.timezone);
@@ -155,11 +161,10 @@ export const ingestBatch = mutation({
                     rejected += 1;
                     continue;
                 }
-                const durationDeltaMs = await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
-                // A heartbeat is presence, not telemetry, but it is what keeps a
-                // session's duration honest while the visitor reads: the growth is
-                // folded into the buckets the heartbeat landed in.
-                foldSessionDuration(aggregateDeltas, args.siteId, event, durationDeltaMs, timezone);
+                // A heartbeat is presence, not telemetry: it moves the session's
+                // lastSeenAt and live row only. The time it proves reaches the
+                // duration counters once, when the visitor departs (live.ts).
+                await accountHeartbeat(ctx, args.siteId, existingSession, event, context, now, batchState);
                 accepted += 1;
                 continue;
             }
@@ -190,7 +195,7 @@ export const ingestBatch = mutation({
                             durationMs,
                         };
                         sessionRow.dirty = true;
-                        foldSessionDuration(aggregateDeltas, args.siteId, event, durationMs - existingSession.durationMs, timezone);
+                        foldSessionDuration(aggregateDeltas, args.siteId, event.sessionId, event.timestamp, durationMs - existingSession.durationMs, timezone);
                     }
                     const expiresAt = Math.min(liveSession.expiresAt, now + LIVE_LEAVE_GRACE_MS);
                     if (expiresAt < liveSession.expiresAt) {
@@ -252,6 +257,7 @@ export const ingestBatch = mutation({
                 reportRollupDeltas,
                 recordTelemetry: true,
                 timezone,
+                siteDomains: site.domains,
                 financialConversion: false,
                 trustedGoalKey: undefined,
                 now,
@@ -353,6 +359,7 @@ export const trackConversion = mutation({
             reportRollupDeltas,
             recordTelemetry: existingTelemetry === null && !telemetry.saturated,
             timezone: localDayTimezone(site.timezone),
+            siteDomains: site.domains,
             financialConversion: true,
             trustedGoalKey,
             now: Date.now(),
@@ -433,7 +440,7 @@ function withoutSystemFields(document) {
 async function accountEvent(ctx, input) {
     const { event } = input;
     const session = input.recordTelemetry
-        ? await upsertSession(ctx, input.siteId, event, input.context, input.existingSession, input.financialConversion, input.reportRollupDeltas, input.batchState)
+        ? await upsertSession(ctx, input.siteId, event, input.context, input.siteDomains, input.existingSession, input.financialConversion, input.reportRollupDeltas, input.batchState)
         : await accountTrustedSession(ctx, input.existingSession, event);
     if (input.recordTelemetry && event.type === "pageview") {
         // `existingSession` is the snapshot from before `upsertSession`, so it
@@ -510,6 +517,23 @@ async function accountEvent(ctx, input) {
     const outboundHost = event.type === "outbound" ? publicUrlHost(event.href) : undefined;
     const dimensions = [
         { type: "source", value: session.source },
+        // A session recorded before origins carries none, and fills no slot.
+        ...(session.platform && session.channel && session.evidence
+            ? [
+                {
+                    type: "platform",
+                    // The value carries the channel too when it is not the
+                    // platform's default, so the pair survives without its own
+                    // dimension (splitPlatformDimension reads it back).
+                    value: platformDimensionValue({
+                        platform: session.platform,
+                        channel: session.channel,
+                    }),
+                },
+                { type: "channel", value: session.channel },
+                { type: "evidence", value: session.evidence },
+            ]
+            : []),
         ...campaignDimensions,
         ...(outboundHost
             ? [{ type: "outbound", value: outboundHost }]
@@ -550,6 +574,9 @@ async function accountTrustedSession(ctx, existing, event) {
         continued: false,
         startedAt: existing.startedAt,
         source: existing.source,
+        platform: existing.platform,
+        channel: existing.channel,
+        evidence: existing.evidence,
         utmCampaign: existing.utmCampaign,
         utmMedium: existing.utmMedium,
         entryPath: existing.entryPath,
@@ -568,7 +595,7 @@ async function accountTrustedSession(ctx, existing, event) {
         affiliateExpiresAt: existing.affiliateExpiresAt,
     };
 }
-async function upsertSession(ctx, siteId, event, context, existing, financialConversion, reportRollupDeltas, batchState) {
+async function upsertSession(ctx, siteId, event, context, siteDomains, existing, financialConversion, reportRollupDeltas, batchState) {
     // A host-supplied id is the identity; anonymous sessions count through the
     // daily key derived by the HTTP action, or once per session without one.
     const visitorKey = existing?.visitorKey ??
@@ -641,6 +668,9 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             continued: event.type === "pageview" && existing.pageviewCount === 1,
             startedAt,
             source: existing.source,
+            platform: existing.platform,
+            channel: existing.channel,
+            evidence: existing.evidence,
             utmCampaign: existing.utmCampaign,
             utmMedium: existing.utmMedium,
             entryPath: existing.entryPath,
@@ -684,6 +714,17 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         if (alias)
             isNewVisitor = false;
     }
+    // First touch only, like `source`: the session keeps where it came from.
+    // An affiliate channel needs this landing's own `ref` to have resolved; a
+    // visitor's standing attribution alone is not how this visit arrived.
+    const origin = classifyOrigin({
+        utmSource,
+        utmMedium,
+        clickIds: event.clid,
+        referrer: externalReferrer(event.referrer, siteDomains),
+        inApp: context.inApp,
+        affiliate: event.affiliateSlug !== undefined && attribution !== undefined,
+    });
     const fields = {
         siteId,
         sessionId: event.sessionId,
@@ -696,6 +737,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         newVisitor: isNewVisitor,
         ...(event.referrer ? { referrer: event.referrer } : {}),
         source,
+        ...origin,
         ...(utmSource ? { utmSource } : {}),
         ...(utmMedium ? { utmMedium } : {}),
         ...(utmCampaign ? { utmCampaign } : {}),
@@ -745,6 +787,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         continued: false,
         startedAt: event.timestamp,
         source,
+        ...origin,
         utmCampaign,
         utmMedium,
         entryPath: event.path,
@@ -868,6 +911,12 @@ async function attributionForEvent(ctx, siteId, event, existingSession, batchSta
         affiliateExpiresAt: expiresAt,
     };
 }
+/**
+ * Moves the session's lastSeenAt and its live row. `durationMs` stays: it is
+ * what the duration counters already hold, and the time a heartbeat proves
+ * is counted when the visitor departs (live.ts), so an idle tab never
+ * rewrites an aggregate bucket.
+ */
 async function accountHeartbeat(ctx, siteId, session, event, context, now, batchState) {
     const startedAt = Math.min(session.startedAt, event.timestamp);
     const lastSeenAt = Math.max(session.lastSeenAt, event.timestamp);
@@ -875,7 +924,6 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         startedAt,
         lastSeenAt,
         exitPath: event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
-        durationMs: Math.max(0, lastSeenAt - startedAt),
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
@@ -896,7 +944,6 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
     else {
         await ctx.db.patch("sessions", session._id, updates);
     }
-    const durationDeltaMs = Math.max(0, updates.durationMs - session.durationMs);
     await updateLiveSession(ctx, siteId, event, context, {
         isNewSession: false,
         isNewVisitorHour: false,
@@ -918,7 +965,42 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         conversionCount: session.conversionCount,
         newVisitor: session.newVisitor,
     }, now, batchState);
-    return durationDeltaMs;
+}
+/**
+ * One client network's budget per site and minute, kept in the session
+ * windows' table under `~client:<key>`, a key no sanitized session ID can take.
+ * The key is the salted daily hash the HTTP action derived; the address never
+ * reaches this mutation. It is one row per client, so only that client's own
+ * concurrent requests contend on it.
+ */
+async function consumeClientRateLimit(ctx, siteId, clientKey, eventByteCounts, now) {
+    const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+    const sessionId = `~client:${clientKey}`;
+    const rows = await ctx.db
+        .query("ingestWindows")
+        .withIndex("by_siteId_and_sessionId_and_windowStart", (range) => range.eq("siteId", siteId).eq("sessionId", sessionId))
+        .order("desc")
+        .take(1);
+    const window = rows[0] ?? null;
+    const current = window?.windowStart === windowStart ? window : null;
+    const eventCount = (current?.eventCount ?? 0) + eventByteCounts.length;
+    const byteCount = (current?.byteCount ?? 0) +
+        eventByteCounts.reduce((total, bytes) => total + bytes, 0);
+    if (eventCount > MAX_EVENTS_PER_CLIENT_WINDOW ||
+        byteCount > MAX_BYTES_PER_CLIENT_WINDOW) {
+        fail("RATE_LIMITED", "client ingest budget exceeded", {
+            windowMs: RATE_LIMIT_WINDOW_MS,
+            eventLimitPerClient: MAX_EVENTS_PER_CLIENT_WINDOW,
+            byteLimitPerClient: MAX_BYTES_PER_CLIENT_WINDOW,
+        });
+    }
+    const fields = { windowStart, eventCount, byteCount, updatedAt: now };
+    if (window) {
+        await ctx.db.patch("ingestWindows", window._id, fields);
+    }
+    else {
+        await ctx.db.insert("ingestWindows", { siteId, sessionId, ...fields });
+    }
 }
 async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
     const windowStart = Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
@@ -1227,9 +1309,11 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
         foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
     }
-    if (progress.completedAt !== undefined ||
-        event.timestamp < progress.lastStepAt)
+    if (event.timestamp < progress.lastStepAt)
         return;
+    // An expired window restarts on a first-step event even after a completion,
+    // so a returning buyer can complete again; inside the window a completed
+    // funnel stays done.
     if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
         if (!matchesFirst)
             return;
@@ -1258,6 +1342,8 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
         foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return { funnelId: funnel._id, step: 1, steps: funnel.steps.length };
     }
+    if (progress.completedAt !== undefined)
+        return;
     const nextStep = funnel.steps[progress.currentStep];
     if (!nextStep || !stepMatches(nextStep, event))
         return;
@@ -1608,14 +1694,19 @@ function aggregateBucketStart(granularity, timestamp, timezone) {
     const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
     return Math.floor(timestamp / interval) * interval;
 }
-/** Folds session duration growth without telemetry, for heartbeats and leaves. */
-function foldSessionDuration(aggregateDeltas, siteId, event, durationDeltaMs, timezone) {
+/**
+ * Folds session duration growth without telemetry: a departure's final
+ * interval, or the heartbeat time of a visitor the live sweep retired. The
+ * shard follows `shardKey`: the session for a departure, the site for a
+ * sweep.
+ */
+export function foldSessionDuration(aggregateDeltas, siteId, shardKey, timestamp, durationDeltaMs, timezone) {
     if (durationDeltaMs <= 0)
         return;
     for (const granularity of aggregateGranularities(timezone)) {
-        const shard = stableHash(event.sessionId) %
+        const shard = stableHash(shardKey) %
             (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
-        const bucketStart = aggregateBucketStart(granularity, event.timestamp, timezone);
+        const bucketStart = aggregateBucketStart(granularity, timestamp, timezone);
         const key = `${siteId}:${granularity}:${bucketStart}:${shard}`;
         const delta = aggregateDeltas.get(key) ?? {
             siteId,
@@ -1641,7 +1732,7 @@ function foldSessionDuration(aggregateDeltas, siteId, event, durationDeltaMs, ti
         aggregateDeltas.set(key, delta);
     }
 }
-async function flushAggregates(ctx, aggregateDeltas) {
+export async function flushAggregates(ctx, aggregateDeltas) {
     for (const delta of aggregateDeltas.values()) {
         const bucket = await ctx.db
             .query("aggregateBuckets")