@iann29/rastro 0.3.0 → 0.5.0

package/src/component/reports.ts

@@ -55,6 +55,8 @@ import {
   type RawJourneyEvent,
 } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
+import { classifyOrigin } from "./origin.js";
+import { externalReferrer } from "./sanitize.js";
 import {
   affiliateFieldsValidator,
   eventFieldsValidator,
@@ -220,6 +222,11 @@ export const overview = query({
     topMediums: v.array(topItemValidator),
     // Destination hosts of outbound clicks, ranked by clicks.
     topOutbound: v.array(topItemValidator),
+    // The session's origin (origin.ts), ranked like sources: platform ids,
+    // channel ids, and the evidence that decided them.
+    topPlatforms: v.array(topItemValidator),
+    topChannels: v.array(topItemValidator),
+    topEvidence: v.array(topItemValidator),
     metadata: v.object({
       visitors: v.object({
         basis: v.union(
@@ -242,6 +249,9 @@ export const overview = query({
         topGoals: v.literal("goalCompletionVolume"),
         topMediums: breakdownBasisValidator,
         topOutbound: v.literal("eventVolume"),
+        topPlatforms: breakdownBasisValidator,
+        topChannels: breakdownBasisValidator,
+        topEvidence: breakdownBasisValidator,
       }),
       // Since when every site in the request records the medium and outbound
       // slots; null while one of them has not stamped it. A range that starts
@@ -346,14 +356,19 @@ export const overview = query({
     const bucketCount = localDays
       ? bucketStarts.length
       : Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
-    const perSiteLimit =
-      bucketCount *
-      (interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
+    const shards =
+      interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS;
+    const perSiteLimit = bucketCount * shards;
     if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
+      // The row budget, not MAX_REPORT_RANGE_DAYS, bounds the overview: 101
+      // days for one site and 10 for ten; 24 hours for two, 5 for ten.
+      const maximumBuckets = Math.floor(
+        MAX_OVERVIEW_AGGREGATE_ROWS / (shards * siteIds.length),
+      );
       fail(
         "LIMIT_EXCEEDED",
-        "overview range is too wide for the requested number of sites",
-        { maximumAggregateRows: MAX_OVERVIEW_AGGREGATE_ROWS },
+        `overview range is too wide: ${siteIds.length === 1 ? "one site" : `${siteIds.length} sites`} can read at most ${maximumBuckets} ${interval}s`,
+        { maximumAggregateRows: MAX_OVERVIEW_AGGREGATE_ROWS, maximumBuckets },
       );
     }
     const totals = emptyTotals();
@@ -382,7 +397,13 @@ export const overview = query({
     await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
 
     for (const siteId of siteIds) {
-      const buckets = await ctx.db
+      // Iterated rather than taken whole: within the row budget a busy range
+      // can still cross the bytes a query may read. Values are length-capped,
+      // so sixteen buckets stay far below the reserve, and running out fails
+      // with REPORT_INCOMPLETE, as the feature reports do, instead of a raw
+      // limit error.
+      const buckets: Doc<"aggregateBuckets">[] = [];
+      for await (const bucket of ctx.db
         .query("aggregateBuckets")
         .withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
           range
@@ -390,8 +411,13 @@ export const overview = query({
             .eq("granularity", granularity)
             .gte("bucketStart", firstBucket)
             .lte("bucketStart", lastBucket),
-        )
-        .take(perSiteLimit + 1);
+        )) {
+        buckets.push(bucket);
+        if (buckets.length > perSiteLimit) break;
+        if (buckets.length % 16 === 0) {
+          await requireFeatureReadHeadroom(ctx, "overview aggregate buckets");
+        }
+      }
       failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
       for (const bucket of buckets) {
         if (
@@ -520,6 +546,9 @@ export const overview = query({
       topGoals: top("goal", "eventVolume"),
       topMediums: top("medium", sessionBasis),
       topOutbound: top("outbound", "eventVolume"),
+      topPlatforms: top("platform", sessionBasis),
+      topChannels: top("channel", sessionBasis),
+      topEvidence: top("evidence", sessionBasis),
       metadata: {
         visitors: {
           basis: visitorBasis,
@@ -539,6 +568,9 @@ export const overview = query({
           topGoals: "goalCompletionVolume" as const,
           topMediums: sessionBasis,
           topOutbound: "eventVolume" as const,
+          topPlatforms: sessionBasis,
+          topChannels: sessionBasis,
+          topEvidence: sessionBasis,
         },
         dimensionsSince: controls.every(
           (control) => control?.dimensionsSince !== undefined,
@@ -932,6 +964,27 @@ async function readRouteRollups(
   return { rows, completeFrom: cutAt + DAY_MS };
 }
 
+type SessionRow = Omit<
+  Doc<"sessions">,
+  "geoLookupAttemptedAt" | "lastPageviewAt"
+>;
+
+/**
+ * A session recorded before origins existed, classified on read from what it
+ * kept (utm fields, referrer, affiliate) and marked `legacy`: it carries no
+ * click IDs or in-app signal, and it never enters the aggregates.
+ */
+function withOrigin(session: SessionRow, domains: string[]): SessionRow {
+  if (session.platform !== undefined) return session;
+  const origin = classifyOrigin({
+    utmSource: session.utmSource,
+    utmMedium: session.utmMedium,
+    referrer: externalReferrer(session.referrer, domains),
+    affiliate: session.affiliateSlug !== undefined,
+  });
+  return { ...session, ...origin, evidence: "legacy" };
+}
+
 /** Metadata for a selected session, independent of list pagination. */
 export const getSession = query({
   args: { siteId: v.id("sites"), sessionId: v.string() },
@@ -949,7 +1002,8 @@ export const getSession = query({
       lastPageviewAt: _pageview,
       ...result
     } = session;
-    return result;
+    const site = await ctx.db.get("sites", args.siteId);
+    return withOrigin(result, site?.domains ?? []);
   },
 });
 
@@ -1088,12 +1142,13 @@ export const listSessions = query({
           .take(take)),
       );
     }
+    const site = await ctx.db.get("sites", args.siteId);
     const publicRows = rows.map(
       ({
         geoLookupAttemptedAt: _attemptedAt,
         lastPageviewAt: _lastPageviewAt,
         ...row
-      }) => row,
+      }) => withOrigin(row, site?.domains ?? []),
     );
     return keysetPaginationResult(publicRows, pagination, (row) => ({
       timestamp: row.lastSeenAt,
@@ -1810,6 +1865,7 @@ const coverageDatasetValidator = v.union(
   v.literal("affiliates"),
   v.literal("vitals"),
   v.literal("siteMap"),
+  v.literal("origin"),
 );
 
 export const dataCoverage = query({
@@ -1843,6 +1899,7 @@ export const dataCoverage = query({
       "affiliates",
       "vitals",
       "siteMap",
+      "origin",
     ] as const;
     return {
       siteId: args.siteId,
@@ -2147,7 +2204,26 @@ async function datasetCoverageForRange(
   to: number,
 ) {
   if (dataset === "siteMap") {
-    return siteMapDatasetCoverage(ctx, siteId, from, to);
+    return stampedDatasetCoverage(
+      ctx,
+      siteId,
+      from,
+      to,
+      "siteMapSince",
+      siteMapCoverageDataset,
+    );
+  }
+  // Origins exist in sessions and aggregate slots from the site's stamp, and
+  // the overview reads them from the daily aggregates retention trims.
+  if (dataset === "origin") {
+    return stampedDatasetCoverage(
+      ctx,
+      siteId,
+      from,
+      to,
+      "originSince",
+      "overviewDay",
+    );
   }
   // Vitals have no legacy fallback source; the rollup either covers the range
   // or the coverage states say how much of it exists.
@@ -2189,23 +2265,26 @@ async function datasetCoverageForRange(
 }
 
 /**
- * The site map's coverage without reading its rows: it starts at the site's
- * `siteMapSince` and ends where retention cut it. The report itself may still
- * narrow the window when the range exceeds its row budget.
+ * A stamped dataset's coverage without reading its rows: it starts at the
+ * site's stamp (`siteMapSince`, `originSince`) and ends where retention cut
+ * it. The site map report may still narrow the window when the range exceeds
+ * its row budget.
  */
-async function siteMapDatasetCoverage(
+async function stampedDatasetCoverage(
   ctx: QueryCtx,
   siteId: Id<"sites">,
   from: number,
   to: number,
+  stamp: "siteMapSince" | "originSince",
+  retainedDataset: string,
 ) {
   const control = await ctx.db
     .query("analyticsCoverage")
     .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
     .unique();
-  const since = control?.siteMapSince ?? null;
+  const since = control?.[stamp] ?? null;
   const retainedBefore =
-    control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
+    control?.retained.find((item) => item.dataset === retainedDataset)
       ?.before ?? null;
   const unavailable =
     since === null ||
@@ -2236,7 +2315,8 @@ type PublicCoverageDataset =
   | "funnels"
   | "affiliates"
   | "vitals"
-  | "siteMap";
+  | "siteMap"
+  | "origin";
 
 type CoverageSource =
   | "overviewHour"

package/src/component/rollupStore.ts

@@ -13,6 +13,7 @@ import {
   MAX_FUNNEL_STEPS,
   MAX_SITE_MAP_ROUTES_PER_DAY,
   REPORT_ROLLUP_SHARDS,
+  ROLLUP_FOLD_DEBOUNCE_MS,
   ROLLUP_FOLD_LEASE_MS,
   ROUTE_TRANSITION_SLOTS,
   SITE_MAP_OTHER_ROUTE,
@@ -36,8 +37,8 @@ import {
  * whole batches are lost (#52). Ingestion therefore never touches a rollup
  * row. It appends the batch's deltas to `reportRollupQueue` — an insert
  * conflicts with nothing — and one leased fold job per site merges the queue
- * into the rows a few hundred milliseconds later, so every rollup row has a
- * single writer.
+ * into the rows about a second later (see rollups.ts), so every rollup row
+ * has a single writer.
  */
 
 export type GoalRollupDelta = {
@@ -221,6 +222,10 @@ const queuedVitalValidator = v.object({
 export const reportRollupQueueFields = {
   siteId: v.id("sites"),
   createdAt: v.number(),
+  // Failed runs in a row with this batch at the head of the queue, and when
+  // it was set aside after too many (see rollups.ts).
+  foldFailures: v.optional(v.number()),
+  deadLetteredAt: v.optional(v.number()),
   goals: v.array(queuedGoalValidator),
   funnels: v.array(queuedFunnelValidator),
   funnelSteps: v.array(queuedFunnelStepValidator),
@@ -456,8 +461,9 @@ export function mergeQueuedRollups(
 /**
  * Appends a batch's rollup deltas to the site's queue and makes sure a fold
  * job will pick them up. The only document this reads is the site's fold
- * state, which the fold job rewrites about twice a minute under load, so
- * concurrent ingests almost never retry because of it.
+ * state, which no batch rewrites while a fold is pending and the fold job
+ * rewrites about twice a minute under load, so concurrent ingests almost
+ * never retry because of it.
  */
 export async function enqueueReportRollups(
   ctx: MutationCtx,
@@ -482,6 +488,7 @@ export async function ensureRollupFold(
     .query("rollupFoldStates")
     .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
     .unique();
+  // A held lease means a chain is running and will fold this batch.
   if (state && state.leaseUntil > now) return;
   const token = crypto.randomUUID();
   const leaseUntil = now + ROLLUP_FOLD_LEASE_MS;
@@ -490,7 +497,11 @@ export async function ensureRollupFold(
   } else {
     await ctx.db.insert("rollupFoldStates", { siteId, leaseUntil, token });
   }
-  await ctx.scheduler.runAfter(0, internal.rollups.fold, { siteId, token });
+  // Debounced, so every batch of the next second joins this run.
+  await ctx.scheduler.runAfter(ROLLUP_FOLD_DEBOUNCE_MS, internal.rollups.fold, {
+    siteId,
+    token,
+  });
 }
 
 // ---------------------------------------------------------------------------
@@ -632,8 +643,9 @@ export async function flushReportRollups(
   for (const delta of deltas.routes.values()) {
     await upsertRouteRollup(ctx, delta, routeDayRowCounts);
   }
+  const vitalDayPageRows = new Map<string, number>();
   for (const delta of deltas.vitals.values()) {
-    await upsertVitalRollup(ctx, delta);
+    await upsertVitalRollup(ctx, delta, vitalDayPageRows);
   }
 }
 
@@ -731,7 +743,11 @@ async function upsertRouteRollup(
   }
 }
 
-async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
+async function upsertVitalRollup(
+  ctx: MutationCtx,
+  delta: VitalRollupDelta,
+  dayPageRows: Map<string, number>,
+) {
   const existing = await ctx.db
     .query("vitalRollups")
     .withIndex("by_key", (range) =>
@@ -748,21 +764,36 @@ async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
     delta.device === VITAL_ALL &&
     delta.page !== VITAL_OTHER_PAGES
   ) {
-    // A new page row spends one bounded read to respect the daily page slots;
-    // past the cap, the day's remaining pages fold into "(other)".
-    const pageRows = await ctx.db
-      .query("vitalRollups")
-      .withIndex("by_key", (range) =>
-        range
-          .eq("siteId", delta.siteId)
-          .eq("bucketStart", delta.bucketStart)
-          .eq("device", VITAL_ALL),
-      )
-      .take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
-    if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
-      await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
+    // New page rows spend one bounded read per day and fold run to respect
+    // the daily page slots, as routes do: rereading it for every new page let
+    // a fold of many distinct URLs pass the read limit and fail every retry.
+    // Past the cap, the day's remaining pages fold into "(other)".
+    const capRows = VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS;
+    const dayKey = `${delta.siteId}:${delta.bucketStart}`;
+    let pageRows = dayPageRows.get(dayKey);
+    if (pageRows === undefined) {
+      pageRows = (
+        await ctx.db
+          .query("vitalRollups")
+          .withIndex("by_key", (range) =>
+            range
+              .eq("siteId", delta.siteId)
+              .eq("bucketStart", delta.bucketStart)
+              .eq("device", VITAL_ALL),
+          )
+          .take(capRows + 1)
+      ).length;
+      dayPageRows.set(dayKey, pageRows);
+    }
+    if (pageRows > capRows) {
+      await upsertVitalRollup(
+        ctx,
+        { ...delta, page: VITAL_OTHER_PAGES },
+        dayPageRows,
+      );
       return;
     }
+    dayPageRows.set(dayKey, pageRows + 1);
   }
   const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
   for (const [metric, addition] of delta.metrics) {

package/src/component/rollups.ts

@@ -1,9 +1,14 @@
-import { v } from "convex/values";
+import { v, type Infer } from "convex/values";
 import { internal } from "./_generated/api.js";
+import type { Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
 import { internalMutation } from "./_generated/server.js";
 import {
+  ROLLUP_DEAD_LETTER_RETENTION_MS,
   ROLLUP_FOLD_BATCH_DOCS,
+  ROLLUP_FOLD_DEBOUNCE_MS,
   ROLLUP_FOLD_LEASE_MS,
+  ROLLUP_FOLD_MAX_FAILURES,
   ROLLUP_FOLD_MAX_ROWS,
 } from "./constants.js";
 import {
@@ -13,24 +18,65 @@ import {
   mergeQueuedRollups,
 } from "./rollupStore.js";
 
+const foldArgs = { siteId: v.id("sites"), token: v.string() };
+const foldResult = v.object({
+  folded: v.number(),
+  remaining: v.boolean(),
+  stale: v.boolean(),
+});
+type FoldResult = Infer<typeof foldResult>;
+
+// A failed run's reads still count against the transaction that catches it,
+// so a run stops short of the platform's read limits (16 MiB, 32,000
+// documents, 4,096 queries) and leaves room to record the failure.
+const RUN_READ_LIMITS = {
+  bytesRead: 15 * 1024 * 1024,
+  documentsRead: 31_000,
+  databaseQueries: 4_000,
+};
+
 /**
  * Folds a site's queued report rollup deltas into the rollup rows. Exactly
- * one chain runs per site: ingestion starts one when no lease is held, every
- * run carries the token of the lease it belongs to, and a run whose token is
- * no longer current exits without touching anything. A run that finds more
- * queued documents than it could take continues at once; a run that drains
- * the queue releases the lease. A batch that lands while a run is draining
- * is never stranded: the run read the queue range the batch inserted into
- * and the batch read the lease the run released, so one of the two retries
- * and sees the other's work.
+ * one chain runs per site: a batch that finds the lease free takes it and
+ * schedules the first run a debounce away (see ensureRollupFold), every run
+ * carries the token of that lease, and a run whose token is no longer
+ * current exits without touching anything. Batches that land while the
+ * lease is held only append to the queue; they never write the lease or
+ * schedule anything.
+ *
+ * A run folds the batches created before it began and, if it folded any,
+ * schedules the next run a debounce later (at once while a backlog remains),
+ * so a steady stream costs one run per second and batches landing during a
+ * run never conflict with it. Only a run that finds nothing left releases
+ * the lease, and that run reads the whole queue range: a batch landing
+ * concurrently either commits first, and the run retries and folds it, or
+ * reads the released lease and starts a new chain.
+ *
+ * A run executes as a nested transaction, so one that throws is rolled back
+ * whole and its batches stay queued; the oldest is charged the failure and
+ * the next run retries it alone. After ROLLUP_FOLD_MAX_FAILURES runs in a
+ * row it is set aside as a dead letter and the queue moves on.
  */
 export const fold = internalMutation({
-  args: { siteId: v.id("sites"), token: v.string() },
-  returns: v.object({
-    folded: v.number(),
-    remaining: v.boolean(),
-    stale: v.boolean(),
-  }),
+  args: foldArgs,
+  returns: foldResult,
+  handler: async (ctx, args) => {
+    try {
+      const result: FoldResult = await ctx.runMutation(
+        internal.rollups.foldRun,
+        args,
+        { transactionLimits: RUN_READ_LIMITS },
+      );
+      return result;
+    } catch (error) {
+      return await setAsideFailedRun(ctx, args, error);
+    }
+  },
+});
+
+export const foldRun = internalMutation({
+  args: foldArgs,
+  returns: foldResult,
   handler: async (ctx, args) => {
     const state = await ctx.db
       .query("rollupFoldStates")
@@ -42,20 +88,31 @@ export const fold = internalMutation({
     const now = Date.now();
     const queued = await ctx.db
       .query("reportRollupQueue")
-      .withIndex("by_siteId_and_createdAt", (range) =>
-        range.eq("siteId", args.siteId),
+      .withIndex("by_siteId_and_deadLetteredAt_and_createdAt", (range) =>
+        range
+          .eq("siteId", args.siteId)
+          .eq("deadLetteredAt", undefined)
+          .lte("createdAt", now),
       )
-      .order("asc")
       .take(ROLLUP_FOLD_BATCH_DOCS + 1);
     if (queued.length === 0) {
-      if (state.leaseUntil > now) {
+      if (await oldestQueued(ctx, args.siteId)) {
+        await ctx.scheduler.runAfter(
+          ROLLUP_FOLD_DEBOUNCE_MS,
+          internal.rollups.fold,
+          args,
+        );
+      } else if (state.leaseUntil > now) {
         await ctx.db.patch("rollupFoldStates", state._id, { leaseUntil: now });
       }
       return { folded: 0, remaining: false, stale: false };
     }
+    // A batch that already failed a run is retried alone, so the next
+    // failure is charged to the batch that causes it.
+    const batchDocs = queued[0].foldFailures ? 1 : ROLLUP_FOLD_BATCH_DOCS;
     const deltas = createReportRollupDeltas();
     let taken = 0;
-    for (const document of queued.slice(0, ROLLUP_FOLD_BATCH_DOCS)) {
+    for (const document of queued.slice(0, batchDocs)) {
       mergeQueuedRollups(deltas, document);
       taken += 1;
       if (countReportRollupRows(deltas) >= ROLLUP_FOLD_MAX_ROWS) break;
@@ -64,19 +121,71 @@ export const fold = internalMutation({
     for (const document of queued.slice(0, taken)) {
       await ctx.db.delete("reportRollupQueue", document._id);
     }
-    const remaining = queued.length > taken;
-    if (remaining) {
-      // Renew only when the lease is halfway gone, so the state document is
-      // rewritten about twice a minute under load rather than on every run.
-      if (state.leaseUntil - now < ROLLUP_FOLD_LEASE_MS / 2) {
-        await ctx.db.patch("rollupFoldStates", state._id, {
-          leaseUntil: now + ROLLUP_FOLD_LEASE_MS,
-        });
-      }
-      await ctx.scheduler.runAfter(0, internal.rollups.fold, args);
-    } else {
-      await ctx.db.patch("rollupFoldStates", state._id, { leaseUntil: now });
+    // Renew only when the lease is halfway gone, so the state document is
+    // rewritten about twice a minute while batches keep arriving.
+    if (state.leaseUntil - now < ROLLUP_FOLD_LEASE_MS / 2) {
+      await ctx.db.patch("rollupFoldStates", state._id, {
+        leaseUntil: now + ROLLUP_FOLD_LEASE_MS,
+      });
     }
+    const remaining = queued.length > taken;
+    await ctx.scheduler.runAfter(
+      remaining ? 0 : ROLLUP_FOLD_DEBOUNCE_MS,
+      internal.rollups.fold,
+      args,
+    );
     return { folded: taken, remaining, stale: false };
   },
 });
+
+/** The oldest batch still waiting to fold; dead letters are skipped. */
+function oldestQueued(ctx: MutationCtx, siteId: Id<"sites">) {
+  return ctx.db
+    .query("reportRollupQueue")
+    .withIndex("by_siteId_and_deadLetteredAt_and_createdAt", (range) =>
+      range.eq("siteId", siteId).eq("deadLetteredAt", undefined),
+    )
+    .first();
+}
+
+async function setAsideFailedRun(
+  ctx: MutationCtx,
+  args: { siteId: Id<"sites">; token: string },
+  error: unknown,
+): Promise<FoldResult> {
+  const state = await ctx.db
+    .query("rollupFoldStates")
+    .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+    .unique();
+  const head = await oldestQueued(ctx, args.siteId);
+  if (!head || state?.token !== args.token) throw error;
+  const now = Date.now();
+  const foldFailures = (head.foldFailures ?? 0) + 1;
+  if (foldFailures < ROLLUP_FOLD_MAX_FAILURES) {
+    await ctx.db.patch("reportRollupQueue", head._id, { foldFailures });
+  } else {
+    console.error(
+      `rollup fold set aside queued batch ${head._id} of site ${args.siteId} after ${foldFailures} failed runs`,
+      error,
+    );
+    await ctx.db.patch("reportRollupQueue", head._id, {
+      foldFailures,
+      deadLetteredAt: now,
+    });
+    // Dead letters stay for inspection; each new one retires expired ones.
+    const expired = await ctx.db
+      .query("reportRollupQueue")
+      .withIndex("by_siteId_and_deadLetteredAt_and_createdAt", (range) =>
+        range
+          .eq("siteId", args.siteId)
+          .gte("deadLetteredAt", 0)
+          .lt("deadLetteredAt", now - ROLLUP_DEAD_LETTER_RETENTION_MS),
+      )
+      .take(ROLLUP_FOLD_BATCH_DOCS);
+    for (const document of expired) {
+      await ctx.db.delete("reportRollupQueue", document._id);
+    }
+  }
+  await ctx.scheduler.runAfter(0, internal.rollups.fold, args);
+  return { folded: 0, remaining: true, stale: false };
+}

package/src/component/sanitize.ts

@@ -10,6 +10,7 @@ import type {
   TrackerEvent,
 } from "./validators.js";
 import { isValidVitalValue, isVitalMetric } from "./vitals.js";
+import { isPlatform, knownClickIds } from "./origin.js";
 
 const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -121,6 +122,27 @@ export function sanitizePublicUrl(value: string): string | undefined {
   }
 }
 
+const ANDROID_PACKAGE = /^[a-z][a-z0-9_]*(?:\.[a-z0-9_]+)+$/;
+
+/**
+ * A referrer as sanitizePublicUrl keeps it, or the Android app that opened
+ * the link: `android-app://<package>/…` keeps the package alone.
+ */
+function sanitizeReferrer(value: string): string | undefined {
+  const input = cleanString(value, 2048);
+  if (!input.toLowerCase().startsWith("android-app://")) {
+    return sanitizePublicUrl(input);
+  }
+  try {
+    const pkg = new URL(input).hostname.toLowerCase();
+    return pkg.length <= 120 && ANDROID_PACKAGE.test(pkg)
+      ? `android-app://${pkg}`
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 /**
  * The host of a public URL the way the outbound dimension names it: lowercase,
  * without port, path, or query. A relative path has no host.
@@ -229,7 +251,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     referrer:
       event.referrer === undefined
         ? undefined
-        : sanitizePublicUrl(event.referrer),
+        : sanitizeReferrer(event.referrer),
     timestamp: Math.trunc(event.timestamp),
     sequence: event.sequence,
     target:
@@ -246,6 +268,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
       event.type === "vital" && event.value !== undefined
         ? Math.round(event.value)
         : undefined,
+    clid: event.clid === undefined ? undefined : knownClickIds(event.clid),
   };
 }
 
@@ -353,6 +376,8 @@ export function sanitizeContext(
       context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
         ? cleanString(context.visitorKey, 128) || undefined
         : undefined,
+    inApp:
+      context?.inApp && isPlatform(context.inApp) ? context.inApp : undefined,
   };
 }
 
@@ -393,6 +418,19 @@ export function sourceFromReferrer(referrer: string | undefined): string {
   }
 }
 
+/**
+ * The referrer unless it is internal: a relative path or one of the site's
+ * own domains never names where a session came from.
+ */
+export function externalReferrer(
+  referrer: string | undefined,
+  domains: string[],
+): string | undefined {
+  if (!referrer || referrer.startsWith("/") || originAllowed(referrer, domains))
+    return undefined;
+  return referrer;
+}
+
 export function stableHash(value: string): number {
   let hash = 2_166_136_261;
   for (let index = 0; index < value.length; index += 1) {