@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/dist/component/ingest.js

@@ -1,8 +1,9 @@
 import { v } from "convex/values";
 import { mutation } from "./_generated/server.js";
 import { internal } from "./_generated/api.js";
-import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, MAX_BATCH_EVENT_GROUPS, DAY_MS, HOURLY_AGGREGATE_SHARDS, DIMENSION_SLOTS, HOUR_MS, LIVE_SESSION_TTL_MS, LIVE_SWEEP_INTERVAL_MS, MAX_BATCH_EVENTS, MAX_SITE_BYTES_PER_SHARD_WINDOW, MAX_SITE_EVENTS_PER_SHARD_WINDOW, MAX_BYTES_PER_SESSION_WINDOW, MAX_EVENTS_PER_SESSION_WINDOW, MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS, MAX_GOALS_PER_SITE, RATE_LIMIT_WINDOW_MS, REPORT_ROLLUP_SHARDS, SECONDARY_AGGREGATE_SHARDS, SITE_INGEST_SHARDS, VITAL_PAGE_SLOTS, } from "./constants.js";
 import { addVisitor, createVisitorSketch, unionVisitorSketches, } from "./cardinality.js";
+import { addToVitalHistogram, emptyVitalHistogram, mergeVitalHistograms, vitalDevice, VITAL_ALL, VITAL_OTHER_PAGES, } from "./vitals.js";
 import { fail } from "./errors.js";
 import { ensureAnalyticsControl } from "./coverage.js";
 import { bufferEventBatch, eventBatchKey, eventBucketStart, findTelemetry, flushEventBatches, insertEventBatch, inspectTelemetryMinute, } from "./eventStore.js";
@@ -76,7 +77,7 @@ export const ingestBatch = mutation({
             const bucketStart = eventBucketStart(event.timestamp);
             const groupKey = eventBatchKey(event.sessionId, bucketStart);
             requestGroups.add(groupKey);
-            if (event.type === "heartbeat")
+            if (event.type === "heartbeat" || event.type === "vital")
                 continue;
             const eventIds = eventIdsByGroup.get(groupKey) ?? new Set();
             if (eventIds.has(event.eventId)) {
@@ -144,6 +145,21 @@ export const ingestBatch = mutation({
                 accepted += 1;
                 continue;
             }
+            if (event.type === "vital") {
+                // A vital annotates telemetry its session already produced; it never
+                // creates sessions, live presence, event batches, or aggregate counts.
+                if (!existingSession) {
+                    rejected += 1;
+                    continue;
+                }
+                if (!(await consumeRateLimit(ctx, args.siteId, event.sessionId, eventByteCounts[eventIndex], now, sessionRateWindows))) {
+                    rejected += 1;
+                    continue;
+                }
+                foldVitalRollups(reportRollupDeltas, args.siteId, event, vitalDevice(context.device));
+                accepted += 1;
+                continue;
+            }
             if (duplicateInBatch.has(eventIndex) ||
                 persistedEventIds
                     .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
@@ -173,8 +189,8 @@ export const ingestBatch = mutation({
                 event,
                 context,
                 existingSession,
-                goals: canMatchDefinition ? goals ?? [] : [],
-                funnels: canMatchDefinition ? funnels ?? [] : [],
+                goals: canMatchDefinition ? (goals ?? []) : [],
+                funnels: canMatchDefinition ? (funnels ?? []) : [],
                 aggregateDeltas,
                 reportRollupDeltas,
                 recordTelemetry: true,
@@ -376,8 +392,7 @@ async function accountEvent(ctx, input) {
             .eq("siteId", input.siteId)
             .eq("goalId", goal._id)
             .eq("eventId", completionEventId))
-            .take(2))
-            .find((completion) => completion.idempotencyVersion === 2);
+            .take(2)).find((completion) => completion.idempotencyVersion === 2);
         const legacyPrior = scopedPrior
             ? undefined
             : (await ctx.db
@@ -386,8 +401,7 @@ async function accountEvent(ctx, input) {
                 .eq("siteId", input.siteId)
                 .eq("goalId", goal._id)
                 .eq("eventId", event.eventId))
-                .take(2))
-                .find((completion) => completion.idempotencyVersion === undefined &&
+                .take(2)).find((completion) => completion.idempotencyVersion === undefined &&
                 completion.sessionId === event.sessionId &&
                 eventBucketStart(completion.timestamp) ===
                     eventBucketStart(event.timestamp));
@@ -415,15 +429,19 @@ async function accountEvent(ctx, input) {
     if (input.financialConversion) {
         await accountConversion(ctx, input.siteId, session, event, input.trustedGoalKey, input.reportRollupDeltas);
     }
+    const campaignDimensions = session.utmCampaign
+        ? [{ type: "campaign", value: session.utmCampaign }]
+        : [];
     const dimensions = [
         { type: "source", value: session.source },
+        ...campaignDimensions,
         { type: "page", value: event.path },
         { type: "country", value: session.country ?? "unknown" },
         { type: "device", value: input.context.device ?? "unknown" },
         { type: "browser", value: input.context.browser ?? "Other" },
         {
             type: "event",
-            value: event.type === "custom" ? event.name ?? "custom" : event.type,
+            value: event.type === "custom" ? (event.name ?? "custom") : event.type,
         },
         ...matchedGoalKeys.map((value) => ({ type: "goal", value })),
     ];
@@ -450,6 +468,8 @@ async function accountTrustedSession(ctx, existing, event) {
         isNewVisitorDay: false,
         startedAt: existing.startedAt,
         source: existing.source,
+        utmCampaign: existing.utmCampaign,
+        visitorKey: existing.visitorKey ?? existing.visitorId,
         country: existing.country,
         city: existing.city,
         latitude: existing.latitude,
@@ -463,14 +483,21 @@ async function accountTrustedSession(ctx, existing, event) {
     };
 }
 async function upsertSession(ctx, siteId, event, context, existing, financialConversion, reportRollupDeltas, batchState) {
-    const source = sourceFromReferrer(event.referrer);
+    // A host-supplied id is the identity; anonymous sessions count through the
+    // daily key derived by the HTTP action, or once per session without one.
+    const visitorKey = existing?.visitorKey ??
+        (event.visitorId !== event.sessionId
+            ? event.visitorId
+            : (context.visitorKey ?? event.sessionId));
     const utmSource = stringProperty(event.properties, "utm_source");
+    // Campaign traffic names its own source; the referrer host is the fallback.
+    const source = utmSource?.toLowerCase() ?? sourceFromReferrer(event.referrer);
     const utmMedium = stringProperty(event.properties, "utm_medium");
     const utmCampaign = stringProperty(event.properties, "utm_campaign");
     const attribution = await attributionForEvent(ctx, siteId, event, existing, batchState);
     if (existing) {
         if (attribution && existing.affiliateId !== attribution.affiliateId) {
-            foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution);
+            foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
         }
         const startedAt = Math.min(existing.startedAt, event.timestamp);
         const lastSeenAt = Math.max(existing.lastSeenAt, event.timestamp);
@@ -484,7 +511,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             durationMs: Math.max(0, lastSeenAt - startedAt),
             conversionCount: existing.conversionCount + Number(financialConversion),
             revenueCents: existing.revenueCents +
-                (financialConversion ? event.revenueCents ?? 0 : 0),
+                (financialConversion ? (event.revenueCents ?? 0) : 0),
             ...(financialConversion && event.currency
                 ? { currency: event.currency }
                 : {}),
@@ -495,7 +522,9 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
             ...(context.country ? { country: context.country } : {}),
             ...(context.city ? { city: context.city } : {}),
             ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-            ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+            ...(context.longitude !== undefined
+                ? { longitude: context.longitude }
+                : {}),
             browser: context.browser ?? existing.browser,
             os: context.os ?? existing.os,
             device: context.device ?? existing.device,
@@ -518,6 +547,8 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
             startedAt,
             source: existing.source,
+            utmCampaign: existing.utmCampaign,
+            visitorKey,
             country: context.country ?? existing.country,
             city: context.city ?? existing.city,
             latitude: context.latitude ?? existing.latitude,
@@ -547,6 +578,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         siteId,
         sessionId: event.sessionId,
         visitorId: event.visitorId,
+        visitorKey,
         startedAt: event.timestamp,
         lastSeenAt: event.timestamp,
         entryPath: event.path,
@@ -559,7 +591,9 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(context.longitude !== undefined
+            ? { longitude: context.longitude }
+            : {}),
         browser: context.browser ?? "Other",
         os: context.os ?? "Other",
         device: context.device ?? "unknown",
@@ -567,7 +601,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
         pageviewCount: event.type === "pageview" ? 1 : 0,
         durationMs: 0,
         conversionCount: Number(financialConversion),
-        revenueCents: financialConversion ? event.revenueCents ?? 0 : 0,
+        revenueCents: financialConversion ? (event.revenueCents ?? 0) : 0,
         ...(financialConversion && event.currency
             ? { currency: event.currency }
             : {}),
@@ -575,7 +609,7 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
     };
     const sessionId = await ctx.db.insert("sessions", fields);
     if (attribution) {
-        foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution);
+        foldAffiliateVisitRollup(reportRollupDeltas, siteId, event, attribution, visitorKey);
     }
     if (batchState) {
         const state = batchState.sessions.get(event.sessionId);
@@ -596,6 +630,8 @@ async function upsertSession(ctx, siteId, event, context, existing, financialCon
                 Math.floor(event.timestamp / DAY_MS),
         startedAt: event.timestamp,
         source,
+        utmCampaign,
+        visitorKey,
         country: context.country,
         city: context.city,
         latitude: context.latitude,
@@ -719,14 +755,14 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
     const updates = {
         startedAt,
         lastSeenAt,
-        exitPath: event.timestamp >= session.lastSeenAt
-            ? event.path
-            : session.exitPath,
+        exitPath: event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
         durationMs: Math.max(0, lastSeenAt - startedAt),
         ...(context.country ? { country: context.country } : {}),
         ...(context.city ? { city: context.city } : {}),
         ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
-        ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
+        ...(context.longitude !== undefined
+            ? { longitude: context.longitude }
+            : {}),
         browser: context.browser ?? session.browser,
         os: context.os ?? session.os,
         device: context.device ?? session.device,
@@ -747,6 +783,7 @@ async function accountHeartbeat(ctx, siteId, session, event, context, now, batch
         isNewVisitorDay: false,
         startedAt,
         source: session.source,
+        visitorKey: session.visitorKey ?? session.visitorId,
         country: context.country ?? session.country,
         city: context.city ?? session.city,
         latitude: context.latitude ?? session.latitude,
@@ -776,9 +813,7 @@ async function consumeSiteRateLimit(ctx, siteId, events, eventByteCounts, now) {
             .order("desc")
             .take(1);
         const storedWindow = rows[0] ?? null;
-        const window = storedWindow?.windowStart === windowStart
-            ? storedWindow
-            : null;
+        const window = storedWindow?.windowStart === windowStart ? storedWindow : null;
         if ((window?.eventCount ?? 0) + increment.eventCount >
             MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
             (window?.byteCount ?? 0) + increment.byteCount >
@@ -1039,7 +1074,8 @@ async function advanceFunnel(ctx, siteId, funnel, event, reportRollupDeltas, bat
         foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
         return;
     }
-    if (progress.completedAt !== undefined || event.timestamp < progress.lastStepAt)
+    if (progress.completedAt !== undefined ||
+        event.timestamp < progress.lastStepAt)
         return;
     if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
         if (!matchesFirst)
@@ -1191,6 +1227,7 @@ function createReportRollupDeltas() {
         goals: new Map(),
         funnels: new Map(),
         affiliates: new Map(),
+        vitals: new Map(),
     };
 }
 function reportRollupCoordinates(siteId, definitionId, timestamp, visitorId) {
@@ -1236,7 +1273,7 @@ function foldFunnelRollup(deltas, siteId, funnelId, event, step, completed) {
     rollup.steps.set(step, current);
     deltas.funnels.set(coordinates.key, rollup);
 }
-function foldAffiliateVisitRollup(deltas, siteId, event, attribution) {
+function foldAffiliateVisitRollup(deltas, siteId, event, attribution, visitorKey) {
     const coordinates = reportRollupCoordinates(siteId, String(attribution.affiliateId), event.timestamp, event.visitorId);
     const current = deltas.affiliates.get(coordinates.key) ?? {
         siteId,
@@ -1251,7 +1288,7 @@ function foldAffiliateVisitRollup(deltas, siteId, event, attribution) {
         commissionCents: 0,
     };
     current.visits += 1;
-    current.visitorSketch = addVisitor(current.visitorSketch, event.visitorId);
+    current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
     deltas.affiliates.set(coordinates.key, current);
 }
 function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affiliateSlug, revenueCents, commissionCents) {
@@ -1273,6 +1310,103 @@ function foldAffiliateConversionRollup(deltas, siteId, event, affiliate, affilia
     current.commissionCents += commissionCents;
     deltas.affiliates.set(coordinates.key, current);
 }
+function foldVitalRollups(deltas, siteId, event, device) {
+    const metric = event.name;
+    const value = event.value ?? 0;
+    const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
+    const shard = stableHash(event.sessionId) % REPORT_ROLLUP_SHARDS;
+    // Each measurement lands twice: on its page (across devices) and on its
+    // device class (across pages), so both breakdowns stay exact and bounded.
+    for (const target of [
+        { device: VITAL_ALL, page: event.path },
+        { device, page: VITAL_ALL },
+    ]) {
+        const key = JSON.stringify([
+            siteId,
+            bucketStart,
+            target.device,
+            target.page,
+            shard,
+        ]);
+        const row = deltas.vitals.get(key) ?? {
+            siteId,
+            bucketStart,
+            device: target.device,
+            page: target.page,
+            shard,
+            metrics: new Map(),
+        };
+        const entry = row.metrics.get(metric) ?? {
+            count: 0,
+            sum: 0,
+            histogram: emptyVitalHistogram(),
+        };
+        entry.count += 1;
+        entry.sum += value;
+        entry.histogram = addToVitalHistogram(entry.histogram, value);
+        row.metrics.set(metric, entry);
+        deltas.vitals.set(key, row);
+    }
+}
+async function upsertVitalRollup(ctx, delta) {
+    const existing = await ctx.db
+        .query("vitalRollups")
+        .withIndex("by_key", (range) => range
+        .eq("siteId", delta.siteId)
+        .eq("bucketStart", delta.bucketStart)
+        .eq("device", delta.device)
+        .eq("page", delta.page)
+        .eq("shard", delta.shard))
+        .unique();
+    if (!existing &&
+        delta.device === VITAL_ALL &&
+        delta.page !== VITAL_OTHER_PAGES) {
+        // A new page row spends one bounded read to respect the daily page slots;
+        // past the cap, the day's remaining pages fold into "(other)".
+        const pageRows = await ctx.db
+            .query("vitalRollups")
+            .withIndex("by_key", (range) => range
+            .eq("siteId", delta.siteId)
+            .eq("bucketStart", delta.bucketStart)
+            .eq("device", VITAL_ALL))
+            .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
+        if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
+            await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
+            return;
+        }
+    }
+    const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
+    for (const [metric, addition] of delta.metrics) {
+        const entry = metrics.find((candidate) => candidate.metric === metric);
+        if (entry) {
+            entry.count += addition.count;
+            entry.sum += addition.sum;
+            entry.histogram = mergeVitalHistograms(entry.histogram, addition.histogram);
+        }
+        else {
+            metrics.push({
+                metric,
+                count: addition.count,
+                sum: addition.sum,
+                histogram: [...addition.histogram],
+            });
+        }
+    }
+    const fields = {
+        siteId: delta.siteId,
+        bucketStart: delta.bucketStart,
+        device: delta.device,
+        page: delta.page,
+        shard: delta.shard,
+        metrics,
+    };
+    if (existing) {
+        await ctx.db.replace("vitalRollups", existing._id, fields);
+    }
+    else {
+        await ctx.db.insert("vitalRollups", fields);
+    }
+}
 async function flushReportRollups(ctx, deltas) {
     for (const delta of deltas.goals.values()) {
         const existing = await ctx.db
@@ -1367,9 +1501,12 @@ async function flushReportRollups(ctx, deltas) {
             await ctx.db.insert("affiliateDailyRollups", fields);
         }
     }
+    for (const delta of deltas.vitals.values()) {
+        await upsertVitalRollup(ctx, delta);
+    }
 }
 function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, recordTelemetry, financialConversion) {
-    const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
+    const revenueCents = financialConversion ? (event.revenueCents ?? 0) : 0;
     const dimensionDeltas = dimensions.map((dimension) => ({
         ...dimension,
         count: Number(recordTelemetry),
@@ -1415,7 +1552,7 @@ function foldAggregates(aggregateDeltas, siteId, event, session, dimensions, rec
                 : Number(session.isNewVisitorDay)
             : 0;
         if (recordTelemetry) {
-            delta.visitorSketch = addVisitor(delta.visitorSketch, event.visitorId);
+            delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
         }
         delta.conversions += Number(financialConversion);
         delta.revenueCents += revenueCents;
@@ -1539,6 +1676,8 @@ function mergeDimensions(current, additions) {
 }
 function stringProperty(properties, key) {
     const value = properties?.[key];
-    return typeof value === "string" ? cleanString(value, 120) || undefined : undefined;
+    return typeof value === "string"
+        ? cleanString(value, 120) || undefined
+        : undefined;
 }
 //# sourceMappingURL=ingest.js.map