@iann29/rastro 0.4.0 → 0.6.0

package/src/component/ingest.ts

@@ -54,6 +54,7 @@ import {
 } from "./eventStore.js";
 import {
   cleanString,
+  externalReferrer,
   isValidMoney,
   measureBatchBytes,
   normalizeRoute,
@@ -74,6 +75,8 @@ import {
   ingestResultValidator,
   trackerEventValidator,
   trustedConversionValidator,
+  type Channel,
+  type Evidence,
   type IngestContext,
   type TrackerEvent,
 } from "./validators.js";
@@ -85,6 +88,8 @@ import {
   type RouteRollupDelta,
 } from "./rollupStore.js";
 import { localDayStart, localDayTimezone } from "./localTime.js";
+import { classifyOrigin, platformDimensionValue } from "./origin.js";
+import { resolveVisitorIdentities } from "./visitors.js";
 
 type DimensionType =
   | "source"
@@ -96,7 +101,10 @@ type DimensionType =
   | "event"
   | "goal"
   | "medium"
-  | "outbound";
+  | "outbound"
+  | "platform"
+  | "channel"
+  | "evidence";
 
 type Dimension = { type: DimensionType; value: string };
 type DimensionDelta = Dimension & {
@@ -145,6 +153,10 @@ type SessionState = {
   continued: boolean;
   startedAt: number;
   source: string;
+  /** The first touch's origin; absent on sessions recorded before it. */
+  platform?: string;
+  channel?: Channel;
+  evidence?: Evidence;
   visitorKey: string;
   utmCampaign?: string;
   utmMedium?: string;
@@ -206,6 +218,7 @@ type BatchState = {
     CachedRow<CachedDocument<"visitorAttributions">>
   >;
   funnelProgress: Map<string, CachedRow<CachedDocument<"funnelProgress">>>;
+  funnelIdentities: Map<string, string[]>;
   eventBatches: EventBatchBuffer;
   liveSweepSites: Set<string>;
 };
@@ -275,6 +288,7 @@ export const ingestBatch = mutation({
       liveSessions: new Map(),
       visitorAttributions: new Map(),
       funnelProgress: new Map(),
+      funnelIdentities: new Map(),
       eventBatches: new Map(),
       liveSweepSites: new Set(),
     };
@@ -567,6 +581,7 @@ export const ingestBatch = mutation({
         reportRollupDeltas,
         recordTelemetry: true,
         timezone,
+        siteDomains: site.domains,
         financialConversion: false,
         trustedGoalKey: undefined,
         now,
@@ -697,6 +712,7 @@ export const trackConversion = mutation({
       reportRollupDeltas,
       recordTelemetry: existingTelemetry === null && !telemetry.saturated,
       timezone: localDayTimezone(site.timezone),
+      siteDomains: site.domains,
       financialConversion: true,
       trustedGoalKey,
       now: Date.now(),
@@ -835,6 +851,8 @@ async function accountEvent(
     recordTelemetry: boolean;
     /** The site's timezone when it keeps local-day buckets. */
     timezone?: string;
+    /** The site's own domains: a referrer on them is not an origin. */
+    siteDomains: string[];
     financialConversion: boolean;
     trustedGoalKey: string | undefined;
     now: number;
@@ -848,6 +866,7 @@ async function accountEvent(
         input.siteId,
         event,
         input.context,
+        input.siteDomains,
         input.existingSession,
         input.financialConversion,
         input.reportRollupDeltas,
@@ -969,6 +988,23 @@ async function accountEvent(
     event.type === "outbound" ? publicUrlHost(event.href) : undefined;
   const dimensions: Dimension[] = [
     { type: "source", value: session.source },
+    // A session recorded before origins carries none, and fills no slot.
+    ...(session.platform && session.channel && session.evidence
+      ? ([
+          {
+            type: "platform",
+            // The value carries the channel too when it is not the
+            // platform's default, so the pair survives without its own
+            // dimension (splitPlatformDimension reads it back).
+            value: platformDimensionValue({
+              platform: session.platform,
+              channel: session.channel,
+            }),
+          },
+          { type: "channel", value: session.channel },
+          { type: "evidence", value: session.evidence },
+        ] satisfies Dimension[])
+      : []),
     ...campaignDimensions,
     ...(outboundHost
       ? [{ type: "outbound", value: outboundHost } satisfies Dimension]
@@ -1035,6 +1071,9 @@ async function accountTrustedSession(
     continued: false,
     startedAt: existing.startedAt,
     source: existing.source,
+    platform: existing.platform,
+    channel: existing.channel,
+    evidence: existing.evidence,
     utmCampaign: existing.utmCampaign,
     utmMedium: existing.utmMedium,
     entryPath: existing.entryPath,
@@ -1059,6 +1098,7 @@ async function upsertSession(
   siteId: Id<"sites">,
   event: TrackerEvent,
   context: IngestContext,
+  siteDomains: string[],
   existing: CachedDocument<"sessions"> | null,
   financialConversion: boolean,
   reportRollupDeltas: ReportRollupDeltas,
@@ -1156,6 +1196,9 @@ async function upsertSession(
       continued: event.type === "pageview" && existing.pageviewCount === 1,
       startedAt,
       source: existing.source,
+      platform: existing.platform,
+      channel: existing.channel,
+      evidence: existing.evidence,
       utmCampaign: existing.utmCampaign,
       utmMedium: existing.utmMedium,
       entryPath: existing.entryPath,
@@ -1206,6 +1249,17 @@ async function upsertSession(
       .first();
     if (alias) isNewVisitor = false;
   }
+  // First touch only, like `source`: the session keeps where it came from.
+  // An affiliate channel needs this landing's own `ref` to have resolved; a
+  // visitor's standing attribution alone is not how this visit arrived.
+  const origin = classifyOrigin({
+    utmSource,
+    utmMedium,
+    clickIds: event.clid,
+    referrer: externalReferrer(event.referrer, siteDomains),
+    inApp: context.inApp,
+    affiliate: event.affiliateSlug !== undefined && attribution !== undefined,
+  });
   const fields: Omit<Doc<"sessions">, "_id" | "_creationTime"> = {
     siteId,
     sessionId: event.sessionId,
@@ -1218,6 +1272,7 @@ async function upsertSession(
     newVisitor: isNewVisitor,
     ...(event.referrer ? { referrer: event.referrer } : {}),
     source,
+    ...origin,
     ...(utmSource ? { utmSource } : {}),
     ...(utmMedium ? { utmMedium } : {}),
     ...(utmCampaign ? { utmCampaign } : {}),
@@ -1274,6 +1329,7 @@ async function upsertSession(
     continued: false,
     startedAt: event.timestamp,
     source,
+    ...origin,
     utmCampaign,
     utmMedium,
     entryPath: event.path,
@@ -1874,46 +1930,62 @@ async function advanceFunnel(
   reportRollupDeltas: ReportRollupDeltas,
   batchState?: BatchState,
 ): Promise<FunnelReach | undefined> {
-  const cacheKey = JSON.stringify([funnel._id, event.visitorId]);
+  let identities = batchState?.funnelIdentities.get(event.visitorId);
+  if (!identities) {
+    identities = await resolveVisitorIdentities(ctx, siteId, event.visitorId);
+    for (const id of identities)
+      batchState?.funnelIdentities.set(id, identities);
+  }
+  const cacheKey = JSON.stringify([funnel._id, identities[0]]);
   let progressState = batchState?.funnelProgress.get(cacheKey);
-  if (batchState && !progressState) {
-    const current = await ctx.db
-      .query("funnelProgress")
-      .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
-        range
-          .eq("siteId", siteId)
-          .eq("funnelId", funnel._id)
-          .eq("visitorId", event.visitorId),
-      )
-      .unique();
+  if (!progressState) {
+    const candidates = await Promise.all(
+      identities.map((visitorId) =>
+        ctx.db
+          .query("funnelProgress")
+          .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
+            range
+              .eq("siteId", siteId)
+              .eq("funnelId", funnel._id)
+              .eq("visitorId", visitorId),
+          )
+          .unique(),
+      ),
+    );
+    // Reuse the furthest attempt still inside its window. Already-counted
+    // attempts are not replayed or merged into historical aggregates.
+    const current =
+      candidates
+        .filter((row) => row !== null)
+        .sort((a, b) => {
+          const activeA =
+            event.timestamp - a.startedAt <= funnel.conversionWindowMs;
+          const activeB =
+            event.timestamp - b.startedAt <= funnel.conversionWindowMs;
+          return (
+            Number(activeB) - Number(activeA) ||
+            (activeA ? b.currentStep - a.currentStep : 0) ||
+            b.lastStepAt - a.lastStepAt
+          );
+        })[0] ?? null;
     progressState = { current, dirty: false };
-    batchState.funnelProgress.set(cacheKey, progressState);
+    batchState?.funnelProgress.set(cacheKey, progressState);
   }
-  const progress = progressState
-    ? progressState.current
-    : await ctx.db
-        .query("funnelProgress")
-        .withIndex("by_siteId_and_funnelId_and_visitorId", (range) =>
-          range
-            .eq("siteId", siteId)
-            .eq("funnelId", funnel._id)
-            .eq("visitorId", event.visitorId),
-        )
-        .unique();
+  const progress = progressState.current;
   const matchesFirst = stepMatches(funnel.steps[0], event);
   if (!progress) {
     if (!matchesFirst) return;
     const fields: Omit<Doc<"funnelProgress">, "_id" | "_creationTime"> = {
       siteId,
       funnelId: funnel._id,
-      visitorId: event.visitorId,
+      visitorId: identities[0],
       sessionId: event.sessionId,
       currentStep: 1,
       startedAt: event.timestamp,
       lastStepAt: event.timestamp,
     };
     const progressId = await ctx.db.insert("funnelProgress", fields);
-    if (progressState) {
+    if (batchState) {
       progressState.current = {
         ...fields,
         _id: progressId,
@@ -1944,7 +2016,7 @@ async function advanceFunnel(
       lastStepAt: event.timestamp,
     };
     delete updated.completedAt;
-    if (progressState) {
+    if (batchState) {
       progressState.current = updated;
       progressState.dirty = true;
     } else {
@@ -1978,7 +2050,7 @@ async function advanceFunnel(
     lastStepAt: event.timestamp,
     ...(completed ? { completedAt: event.timestamp } : {}),
   };
-  if (progressState) {
+  if (batchState) {
     progressState.current = { ...progress, ...updates };
     progressState.dirty = true;
   } else {

package/src/component/origin.ts

@@ -0,0 +1,273 @@
+import { CLICK_IDS, PLATFORMS, type PlatformRow } from "./platforms.js";
+import { channelValidator, type Channel, type Evidence } from "./validators.js";
+
+/**
+ * Where a session came from, with the rule that proved it. `platform` is an
+ * id from platforms.ts, `unknown` for a referrer or utm_source outside the
+ * table (the raw value stays in the session's `source`), or `none`.
+ */
+export type Origin = { platform: string; channel: Channel; evidence: Evidence };
+
+export type OriginInput = {
+  utmSource?: string;
+  utmMedium?: string;
+  /** Click-ID key names from the landing URL, never their values. */
+  clickIds?: string[];
+  /** An external referrer: callers drop the site's own (externalReferrer). */
+  referrer?: string;
+  /** The platform whose in-app browser opened the page (inAppPlatform). */
+  inApp?: string;
+  /** The landing carried a `ref` that resolved to an affiliate. */
+  affiliate?: boolean;
+};
+
+const MAX_CLICK_IDS = 8;
+const DIRECT: Origin = {
+  platform: "none",
+  channel: "direct",
+  evidence: "none",
+};
+const UNKNOWN_REFERRAL = { platform: "unknown", channel: "referral" } as const;
+// Surfaces that add `fbclid` to their outbound links: a referrer or in-app
+// browser naming one of them says which Meta app the click came from.
+const META_FAMILY = new Set(["facebook", "instagram", "messenger", "threads"]);
+const CHANNELS = new Set<string>(
+  channelValidator.members.map((member) => member.value),
+);
+
+type Hit = { row: PlatformRow; channel: Channel };
+const BY_ID = new Map(PLATFORMS.map((row) => [row.id, row]));
+const BY_HOST = new Map<string, Hit>();
+const BY_ALIAS = new Map<string, Hit>();
+for (const row of PLATFORMS) {
+  const paid = { row, channel: paidChannel(row.channel) ?? row.channel };
+  for (const host of [...(row.domains ?? []), ...(row.packages ?? [])]) {
+    BY_HOST.set(host, { row, channel: row.channel });
+  }
+  for (const host of row.paidDomains ?? []) BY_HOST.set(host, paid);
+  for (const alias of [row.id, ...(row.aliases ?? [])]) {
+    BY_ALIAS.set(alias, { row, channel: row.channel });
+  }
+  for (const alias of row.paidAliases ?? []) BY_ALIAS.set(alias, paid);
+}
+const CLICK_ID_RANK = new Map(CLICK_IDS.map(([key], rank) => [key, rank]));
+
+// utm_medium spellings besides the channel ids, which are accepted as they
+// are (tracked links send them). "paid" and "organic" follow the platform:
+// search or social, whichever it is. A Map, so no utm_medium can reach
+// Object.prototype.
+const MEDIUMS = new Map<string, Channel | "paid" | "organic">([
+  ["cpc", "paid"],
+  ["ppc", "paid"],
+  ["paid", "paid"],
+  ["paidsearch", "paid_search"],
+  ["sem", "paid_search"],
+  ["paidsocial", "paid_social"],
+  ["social", "organic_social"],
+  ["social-media", "organic_social"],
+  ["social_media", "organic_social"],
+  ["e-mail", "email"],
+  ["newsletter", "email"],
+  ["mail", "email"],
+  ["affiliates", "affiliate"],
+  ["partner", "affiliate"],
+  ["banner", "display"],
+  ["video", "display"],
+  ["cpm", "display"],
+  ["whatsapp", "messaging"],
+  ["sms", "messaging"],
+  ["print", "offline"],
+  ["qr", "offline"],
+  ["organic", "organic"],
+]);
+
+/**
+ * Classifies a session's first touch. One rule decides and the next only
+ * runs when it did not: utm_source, then click IDs, then the referrer host,
+ * then the in-app browser; an affiliate `ref` then sets the channel, keeping
+ * the platform and evidence an earlier rule found. Nothing at all is direct,
+ * the only way to be direct. Matching is exact, never by substring.
+ */
+export function classifyOrigin(input: OriginInput): Origin {
+  const referred = referrerPlatform(input.referrer);
+  const inApp = input.inApp === undefined ? undefined : BY_ID.get(input.inApp);
+  const origin =
+    fromUtm(input.utmSource, input.utmMedium) ??
+    fromClickIds(input.clickIds, referred?.platform, inApp?.id) ??
+    (referred && { ...referred, evidence: "referrer" as const }) ??
+    (inApp && {
+      platform: inApp.id,
+      channel: inApp.channel,
+      evidence: "inApp" as const,
+    });
+  if (input.affiliate) {
+    return origin
+      ? { ...origin, channel: "affiliate" }
+      : { platform: "none", channel: "affiliate", evidence: "affiliate" };
+  }
+  return origin ?? DIRECT;
+}
+
+/** The known click-ID keys of a list, deduplicated and bounded. */
+export function knownClickIds(keys: string[]): string[] | undefined {
+  const known = [...new Set(keys.filter((key) => CLICK_ID_RANK.has(key)))];
+  return known.length > 0 ? known.slice(0, MAX_CLICK_IDS) : undefined;
+}
+
+export function isPlatform(id: string): boolean {
+  return BY_ID.has(id);
+}
+
+/**
+ * The platform behind a stored `source` — a `utm_source` alias, a referrer
+ * host, or an Android package, as sessions and the overview keep them.
+ * `unknown` when the table has no row for it, `none` when the session had no
+ * source at all. Matching is `classifyOrigin`'s, never by substring, so a
+ * dashboard can group raw hosts the same way the classifier does.
+ */
+export function platformForSource(source: string): string {
+  const value = source.trim().toLowerCase();
+  if (!value || value === "direct" || value === "internal") return "none";
+  return (BY_ALIAS.get(value) ?? hostHit(value))?.row.id ?? "unknown";
+}
+
+/**
+ * Where a platform's traffic lands when nothing says paid or organic:
+ * `direct` for `none`, `referral` for an id outside the table.
+ */
+export function channelForPlatform(platform: string): Channel {
+  if (platform === "none") return DIRECT.channel;
+  return BY_ID.get(platform)?.channel ?? UNKNOWN_REFERRAL.channel;
+}
+
+/**
+ * The value the `platform` aggregate dimension carries. A session that
+ * arrived by its platform's default channel keeps the bare id; one that
+ * deviated carries `platform@channel` (`google@paid_search` for a `gclid`,
+ * `unknown@email` for a newsletter nobody's table knows), so the aggregates
+ * hold the pair without a dimension of their own. `none` is always `none`:
+ * a session with no platform is the direct row, whatever its channel.
+ */
+export function platformDimensionValue(origin: {
+  platform: string;
+  channel: Channel;
+}): string {
+  if (origin.platform === "none") return "none";
+  return origin.channel === channelForPlatform(origin.platform)
+    ? origin.platform
+    : `${origin.platform}@${origin.channel}`;
+}
+
+/**
+ * The inverse: the platform and the channel behind a `platform` dimension
+ * value. A bare id — every session recorded before the pair was encoded
+ * included — reads as that platform's default channel, which is what it was
+ * classified with.
+ */
+export function splitPlatformDimension(value: string): {
+  platform: string;
+  channel: Channel;
+} {
+  const at = value.indexOf("@");
+  const platform = at < 0 ? value : value.slice(0, at);
+  const channel = at < 0 ? "" : value.slice(at + 1);
+  return {
+    platform,
+    channel: CHANNELS.has(channel)
+      ? (channel as Channel)
+      : channelForPlatform(platform),
+  };
+}
+
+function fromUtm(
+  utmSource: string | undefined,
+  utmMedium: string | undefined,
+): Origin | undefined {
+  const source = utmSource?.trim().toLowerCase();
+  if (!source) return undefined;
+  // A host-shaped value (utm_source=chatgpt.com) matches like a referrer.
+  const hit =
+    BY_ALIAS.get(source) ?? (source.includes(".") ? hostHit(source) : null);
+  const declared = hit?.channel ?? UNKNOWN_REFERRAL.channel;
+  const mediumKey = utmMedium?.trim().toLowerCase() ?? "";
+  const medium = CHANNELS.has(mediumKey)
+    ? (mediumKey as Channel)
+    : MEDIUMS.get(mediumKey);
+  const channel: Channel =
+    medium === "paid"
+      ? (paidChannel(hit?.row.channel) ?? declared)
+      : medium === "organic"
+        ? (hit?.row.channel ?? declared)
+        : (medium ?? declared);
+  return {
+    platform: hit?.row.id ?? UNKNOWN_REFERRAL.platform,
+    channel,
+    evidence: "utm",
+  };
+}
+
+function fromClickIds(
+  keys: string[] | undefined,
+  referred: string | undefined,
+  inApp: string | undefined,
+): Origin | undefined {
+  let best: (typeof CLICK_IDS)[number] | undefined;
+  for (const key of keys ?? []) {
+    const rank = CLICK_ID_RANK.get(key);
+    if (rank !== undefined && (!best || rank < CLICK_ID_RANK.get(best[0])!)) {
+      best = CLICK_IDS[rank];
+    }
+  }
+  if (!best) return undefined;
+  const [, platform, channel] = best;
+  // The click ID names who generated the link. A Meta id defers to the Meta
+  // app the referrer or the in-app browser names, channel included (a
+  // Messenger link is messaging); an e-mail id takes the referrer's platform.
+  if (platform === "meta") {
+    const app = [referred, inApp].find((id) => id && META_FAMILY.has(id));
+    const row = app === undefined ? undefined : BY_ID.get(app);
+    return row
+      ? { platform: row.id, channel: row.channel, evidence: "clickId" }
+      : { platform, channel, evidence: "clickId" };
+  }
+  const resolved =
+    platform === "unknown" && referred && referred !== "unknown"
+      ? referred
+      : platform;
+  return { platform: resolved, channel, evidence: "clickId" };
+}
+
+function referrerPlatform(
+  referrer: string | undefined,
+): { platform: string; channel: Channel } | undefined {
+  if (!referrer) return undefined;
+  let host: string;
+  try {
+    host = new URL(referrer).hostname.toLowerCase().replace(/\.$/, "");
+  } catch {
+    return undefined;
+  }
+  if (!host) return undefined;
+  const hit = hostHit(host);
+  return hit
+    ? { platform: hit.row.id, channel: hit.channel }
+    : UNKNOWN_REFERRAL;
+}
+
+/** The longest listed suffix of a host (a whole label at a time), or null. */
+function hostHit(host: string): Hit | null {
+  for (let suffix = host; suffix;) {
+    const hit = BY_HOST.get(suffix);
+    if (hit) return hit;
+    const dot = suffix.indexOf(".");
+    suffix = dot < 0 ? "" : suffix.slice(dot + 1);
+  }
+  const row = PLATFORMS.find((candidate) => candidate.hostPattern?.test(host));
+  return row ? { row, channel: row.channel } : null;
+}
+
+function paidChannel(channel: Channel | undefined): Channel | undefined {
+  if (channel === "organic_search") return "paid_search";
+  if (channel === "organic_social") return "paid_social";
+  return undefined;
+}