@iann29/rastro 0.4.0 → 0.5.0

package/src/component/origin.ts

@@ -0,0 +1,273 @@
+import { CLICK_IDS, PLATFORMS, type PlatformRow } from "./platforms.js";
+import { channelValidator, type Channel, type Evidence } from "./validators.js";
+
+/**
+ * Where a session came from, with the rule that proved it. `platform` is an
+ * id from platforms.ts, `unknown` for a referrer or utm_source outside the
+ * table (the raw value stays in the session's `source`), or `none`.
+ */
+export type Origin = { platform: string; channel: Channel; evidence: Evidence };
+
+export type OriginInput = {
+  utmSource?: string;
+  utmMedium?: string;
+  /** Click-ID key names from the landing URL, never their values. */
+  clickIds?: string[];
+  /** An external referrer: callers drop the site's own (externalReferrer). */
+  referrer?: string;
+  /** The platform whose in-app browser opened the page (inAppPlatform). */
+  inApp?: string;
+  /** The landing carried a `ref` that resolved to an affiliate. */
+  affiliate?: boolean;
+};
+
+const MAX_CLICK_IDS = 8;
+const DIRECT: Origin = {
+  platform: "none",
+  channel: "direct",
+  evidence: "none",
+};
+const UNKNOWN_REFERRAL = { platform: "unknown", channel: "referral" } as const;
+// Surfaces that add `fbclid` to their outbound links: a referrer or in-app
+// browser naming one of them says which Meta app the click came from.
+const META_FAMILY = new Set(["facebook", "instagram", "messenger", "threads"]);
+const CHANNELS = new Set<string>(
+  channelValidator.members.map((member) => member.value),
+);
+
+type Hit = { row: PlatformRow; channel: Channel };
+const BY_ID = new Map(PLATFORMS.map((row) => [row.id, row]));
+const BY_HOST = new Map<string, Hit>();
+const BY_ALIAS = new Map<string, Hit>();
+for (const row of PLATFORMS) {
+  const paid = { row, channel: paidChannel(row.channel) ?? row.channel };
+  for (const host of [...(row.domains ?? []), ...(row.packages ?? [])]) {
+    BY_HOST.set(host, { row, channel: row.channel });
+  }
+  for (const host of row.paidDomains ?? []) BY_HOST.set(host, paid);
+  for (const alias of [row.id, ...(row.aliases ?? [])]) {
+    BY_ALIAS.set(alias, { row, channel: row.channel });
+  }
+  for (const alias of row.paidAliases ?? []) BY_ALIAS.set(alias, paid);
+}
+const CLICK_ID_RANK = new Map(CLICK_IDS.map(([key], rank) => [key, rank]));
+
+// utm_medium spellings besides the channel ids, which are accepted as they
+// are (tracked links send them). "paid" and "organic" follow the platform:
+// search or social, whichever it is. A Map, so no utm_medium can reach
+// Object.prototype.
+const MEDIUMS = new Map<string, Channel | "paid" | "organic">([
+  ["cpc", "paid"],
+  ["ppc", "paid"],
+  ["paid", "paid"],
+  ["paidsearch", "paid_search"],
+  ["sem", "paid_search"],
+  ["paidsocial", "paid_social"],
+  ["social", "organic_social"],
+  ["social-media", "organic_social"],
+  ["social_media", "organic_social"],
+  ["e-mail", "email"],
+  ["newsletter", "email"],
+  ["mail", "email"],
+  ["affiliates", "affiliate"],
+  ["partner", "affiliate"],
+  ["banner", "display"],
+  ["video", "display"],
+  ["cpm", "display"],
+  ["whatsapp", "messaging"],
+  ["sms", "messaging"],
+  ["print", "offline"],
+  ["qr", "offline"],
+  ["organic", "organic"],
+]);
+
+/**
+ * Classifies a session's first touch. One rule decides and the next only
+ * runs when it did not: utm_source, then click IDs, then the referrer host,
+ * then the in-app browser; an affiliate `ref` then sets the channel, keeping
+ * the platform and evidence an earlier rule found. Nothing at all is direct,
+ * the only way to be direct. Matching is exact, never by substring.
+ */
+export function classifyOrigin(input: OriginInput): Origin {
+  const referred = referrerPlatform(input.referrer);
+  const inApp = input.inApp === undefined ? undefined : BY_ID.get(input.inApp);
+  const origin =
+    fromUtm(input.utmSource, input.utmMedium) ??
+    fromClickIds(input.clickIds, referred?.platform, inApp?.id) ??
+    (referred && { ...referred, evidence: "referrer" as const }) ??
+    (inApp && {
+      platform: inApp.id,
+      channel: inApp.channel,
+      evidence: "inApp" as const,
+    });
+  if (input.affiliate) {
+    return origin
+      ? { ...origin, channel: "affiliate" }
+      : { platform: "none", channel: "affiliate", evidence: "affiliate" };
+  }
+  return origin ?? DIRECT;
+}
+
+/** The known click-ID keys of a list, deduplicated and bounded. */
+export function knownClickIds(keys: string[]): string[] | undefined {
+  const known = [...new Set(keys.filter((key) => CLICK_ID_RANK.has(key)))];
+  return known.length > 0 ? known.slice(0, MAX_CLICK_IDS) : undefined;
+}
+
+export function isPlatform(id: string): boolean {
+  return BY_ID.has(id);
+}
+
+/**
+ * The platform behind a stored `source` — a `utm_source` alias, a referrer
+ * host, or an Android package, as sessions and the overview keep them.
+ * `unknown` when the table has no row for it, `none` when the session had no
+ * source at all. Matching is `classifyOrigin`'s, never by substring, so a
+ * dashboard can group raw hosts the same way the classifier does.
+ */
+export function platformForSource(source: string): string {
+  const value = source.trim().toLowerCase();
+  if (!value || value === "direct" || value === "internal") return "none";
+  return (BY_ALIAS.get(value) ?? hostHit(value))?.row.id ?? "unknown";
+}
+
+/**
+ * Where a platform's traffic lands when nothing says paid or organic:
+ * `direct` for `none`, `referral` for an id outside the table.
+ */
+export function channelForPlatform(platform: string): Channel {
+  if (platform === "none") return DIRECT.channel;
+  return BY_ID.get(platform)?.channel ?? UNKNOWN_REFERRAL.channel;
+}
+
+/**
+ * The value the `platform` aggregate dimension carries. A session that
+ * arrived by its platform's default channel keeps the bare id; one that
+ * deviated carries `platform@channel` (`google@paid_search` for a `gclid`,
+ * `unknown@email` for a newsletter nobody's table knows), so the aggregates
+ * hold the pair without a dimension of their own. `none` is always `none`:
+ * a session with no platform is the direct row, whatever its channel.
+ */
+export function platformDimensionValue(origin: {
+  platform: string;
+  channel: Channel;
+}): string {
+  if (origin.platform === "none") return "none";
+  return origin.channel === channelForPlatform(origin.platform)
+    ? origin.platform
+    : `${origin.platform}@${origin.channel}`;
+}
+
+/**
+ * The inverse: the platform and the channel behind a `platform` dimension
+ * value. A bare id — every session recorded before the pair was encoded
+ * included — reads as that platform's default channel, which is what it was
+ * classified with.
+ */
+export function splitPlatformDimension(value: string): {
+  platform: string;
+  channel: Channel;
+} {
+  const at = value.indexOf("@");
+  const platform = at < 0 ? value : value.slice(0, at);
+  const channel = at < 0 ? "" : value.slice(at + 1);
+  return {
+    platform,
+    channel: CHANNELS.has(channel)
+      ? (channel as Channel)
+      : channelForPlatform(platform),
+  };
+}
+
+function fromUtm(
+  utmSource: string | undefined,
+  utmMedium: string | undefined,
+): Origin | undefined {
+  const source = utmSource?.trim().toLowerCase();
+  if (!source) return undefined;
+  // A host-shaped value (utm_source=chatgpt.com) matches like a referrer.
+  const hit =
+    BY_ALIAS.get(source) ?? (source.includes(".") ? hostHit(source) : null);
+  const declared = hit?.channel ?? UNKNOWN_REFERRAL.channel;
+  const mediumKey = utmMedium?.trim().toLowerCase() ?? "";
+  const medium = CHANNELS.has(mediumKey)
+    ? (mediumKey as Channel)
+    : MEDIUMS.get(mediumKey);
+  const channel: Channel =
+    medium === "paid"
+      ? (paidChannel(hit?.row.channel) ?? declared)
+      : medium === "organic"
+        ? (hit?.row.channel ?? declared)
+        : (medium ?? declared);
+  return {
+    platform: hit?.row.id ?? UNKNOWN_REFERRAL.platform,
+    channel,
+    evidence: "utm",
+  };
+}
+
+function fromClickIds(
+  keys: string[] | undefined,
+  referred: string | undefined,
+  inApp: string | undefined,
+): Origin | undefined {
+  let best: (typeof CLICK_IDS)[number] | undefined;
+  for (const key of keys ?? []) {
+    const rank = CLICK_ID_RANK.get(key);
+    if (rank !== undefined && (!best || rank < CLICK_ID_RANK.get(best[0])!)) {
+      best = CLICK_IDS[rank];
+    }
+  }
+  if (!best) return undefined;
+  const [, platform, channel] = best;
+  // The click ID names who generated the link. A Meta id defers to the Meta
+  // app the referrer or the in-app browser names, channel included (a
+  // Messenger link is messaging); an e-mail id takes the referrer's platform.
+  if (platform === "meta") {
+    const app = [referred, inApp].find((id) => id && META_FAMILY.has(id));
+    const row = app === undefined ? undefined : BY_ID.get(app);
+    return row
+      ? { platform: row.id, channel: row.channel, evidence: "clickId" }
+      : { platform, channel, evidence: "clickId" };
+  }
+  const resolved =
+    platform === "unknown" && referred && referred !== "unknown"
+      ? referred
+      : platform;
+  return { platform: resolved, channel, evidence: "clickId" };
+}
+
+function referrerPlatform(
+  referrer: string | undefined,
+): { platform: string; channel: Channel } | undefined {
+  if (!referrer) return undefined;
+  let host: string;
+  try {
+    host = new URL(referrer).hostname.toLowerCase().replace(/\.$/, "");
+  } catch {
+    return undefined;
+  }
+  if (!host) return undefined;
+  const hit = hostHit(host);
+  return hit
+    ? { platform: hit.row.id, channel: hit.channel }
+    : UNKNOWN_REFERRAL;
+}
+
+/** The longest listed suffix of a host (a whole label at a time), or null. */
+function hostHit(host: string): Hit | null {
+  for (let suffix = host; suffix;) {
+    const hit = BY_HOST.get(suffix);
+    if (hit) return hit;
+    const dot = suffix.indexOf(".");
+    suffix = dot < 0 ? "" : suffix.slice(dot + 1);
+  }
+  const row = PLATFORMS.find((candidate) => candidate.hostPattern?.test(host));
+  return row ? { row, channel: row.channel } : null;
+}
+
+function paidChannel(channel: Channel | undefined): Channel | undefined {
+  if (channel === "organic_search") return "paid_search";
+  if (channel === "organic_social") return "paid_social";
+  return undefined;
+}

package/src/component/platforms.ts

@@ -0,0 +1,359 @@
+import type { Channel } from "./validators.js";
+
+/**
+ * The closed table origin.ts classifies against. Bump the version whenever a
+ * row changes, and add a corpus case (origin.test.ts) for the change.
+ */
+export const PLATFORM_TABLE_VERSION = 1;
+
+export type PlatformRow = {
+  id: string;
+  /** Where the platform's traffic lands when nothing says paid or organic. */
+  channel: Channel;
+  /** Registrable-domain suffixes: a host equal to one or ending in "." + it. */
+  domains?: readonly string[];
+  /** Hosts matched in full, for a family with many country domains. */
+  hostPattern?: RegExp;
+  /** Hosts that only ever send the platform's ads (the paid channel). */
+  paidDomains?: readonly string[];
+  /** Android packages, as an `android-app://<package>` referrer names them. */
+  packages?: readonly string[];
+  /** The platform's in-app browser marker in the User-Agent. */
+  agent?: RegExp;
+  /** utm_source spellings, lowercase; the id itself always matches. */
+  aliases?: readonly string[];
+  /** utm_source spellings that themselves declare the paid channel. */
+  paidAliases?: readonly string[];
+};
+
+// Ordered for `inAppPlatform`: the first row whose marker matches wins, so
+// Messenger (its iOS agent carries `FBAN/`) precedes Facebook and Threads
+// precedes Instagram. Sources per row: "spec" is the approved proposal's
+// table; the rest is each platform's own domains, link shims, and Play Store
+// package ids, and in-app agents as the apps send them.
+export const PLATFORMS: readonly PlatformRow[] = [
+  // Social.
+  {
+    // Threads: spec (threads.net → threads.com); agent is the app's codename.
+    id: "threads",
+    channel: "organic_social",
+    domains: ["threads.net", "threads.com"],
+    packages: ["com.instagram.barcelona"],
+    agent: /Barcelona/,
+  },
+  {
+    // spec.
+    id: "instagram",
+    channel: "organic_social",
+    domains: ["instagram.com"],
+    packages: ["com.instagram.android"],
+    agent: /Instagram/,
+    aliases: ["ig", "insta"],
+  },
+  {
+    // spec; + com.facebook.mlite (Messenger Lite), "msg" is Meta's
+    // {{site_source_name}} value for Messenger placements.
+    id: "messenger",
+    channel: "messaging",
+    domains: ["messenger.com", "m.me"],
+    packages: ["com.facebook.orca", "com.facebook.mlite"],
+    agent: /Orca-Android|MessengerLite|MessengerForiOS/,
+    aliases: ["msg"],
+  },
+  {
+    // spec; + fb.watch (video short links).
+    id: "facebook",
+    channel: "organic_social",
+    domains: ["facebook.com", "fb.com", "fb.me", "fb.watch"],
+    packages: ["com.facebook.katana", "com.facebook.lite"],
+    agent: /FBAN\/|FBAV\/|FB_IAB/,
+    aliases: ["fb", "face"],
+  },
+  {
+    // spec: only an `fbclid` with nothing that tells Facebook from Instagram.
+    id: "meta",
+    channel: "organic_social",
+  },
+  {
+    // spec; + TikTok Lite (com.zhiliaoapp.musically.go).
+    id: "tiktok",
+    channel: "organic_social",
+    domains: ["tiktok.com"],
+    packages: [
+      "com.zhiliaoapp.musically",
+      "com.ss.android.ugc.trill",
+      "com.zhiliaoapp.musically.go",
+    ],
+    agent: /BytedanceWebview|musical_ly|TikTok/,
+    aliases: ["tt"],
+  },
+  {
+    // spec.
+    id: "kwai",
+    channel: "organic_social",
+    domains: ["kwai.com", "kwai-video.com", "kw.ai"],
+    packages: ["com.kwai.video"],
+    agent: /Kwai/,
+  },
+  {
+    // spec.
+    id: "youtube",
+    channel: "organic_social",
+    domains: ["youtube.com", "youtu.be"],
+    packages: ["com.google.android.youtube"],
+    aliases: ["yt"],
+  },
+  {
+    // spec; + TwitterAndroid (the Android app's agent).
+    id: "x",
+    channel: "organic_social",
+    domains: ["x.com", "twitter.com", "t.co"],
+    packages: ["com.twitter.android"],
+    agent: /Twitter for |TwitterAndroid/,
+    aliases: ["twitter"],
+  },
+  {
+    // spec.
+    id: "linkedin",
+    channel: "organic_social",
+    domains: ["linkedin.com", "lnkd.in"],
+    packages: ["com.linkedin.android"],
+    agent: /LinkedInApp/,
+    aliases: ["li"],
+  },
+  {
+    id: "pinterest",
+    channel: "organic_social",
+    domains: ["pinterest.com", "pin.it"],
+    packages: ["com.pinterest"],
+    agent: /Pinterest\//,
+    aliases: ["pin"],
+  },
+  {
+    id: "reddit",
+    channel: "organic_social",
+    domains: ["reddit.com", "redd.it"],
+    packages: ["com.reddit.frontpage"],
+  },
+  {
+    id: "snapchat",
+    channel: "organic_social",
+    domains: ["snapchat.com"],
+    packages: ["com.snapchat.android"],
+    agent: /Snapchat/,
+    aliases: ["snap"],
+  },
+
+  // Messaging.
+  {
+    // spec; + wl.co (WhatsApp's link shim, l.wl.co).
+    id: "whatsapp",
+    channel: "messaging",
+    domains: ["whatsapp.com", "wa.me", "wl.co"],
+    packages: ["com.whatsapp", "com.whatsapp.w4b"],
+    aliases: ["wa", "zap", "whats"],
+  },
+  {
+    // spec; + Telegram X (org.thunderdog.challegram).
+    id: "telegram",
+    channel: "messaging",
+    domains: ["t.me", "telegram.me", "telegram.org"],
+    packages: ["org.telegram.messenger", "org.thunderdog.challegram"],
+    aliases: ["tg"],
+  },
+  {
+    id: "discord",
+    channel: "messaging",
+    domains: ["discord.com", "discordapp.com", "discord.gg"],
+    packages: ["com.discord"],
+  },
+  {
+    // The Android package is `com.Slack`; referrer hosts arrive lowercase.
+    id: "slack",
+    channel: "messaging",
+    domains: ["slack.com"],
+    packages: ["com.slack"],
+  },
+  {
+    id: "line",
+    channel: "messaging",
+    domains: ["line.me"],
+    packages: ["jp.naver.line.android"],
+    agent: / Line\//,
+  },
+
+  // Search. Google's country domains match one pattern, not a suffix, so
+  // mail, docs, drive and every other google.com role host stay out of it.
+  {
+    // spec: google.<tld>, googleadservices.com, the Google app and GSA/.
+    id: "google",
+    channel: "organic_search",
+    hostPattern: /^(?:www\.)?google\.[a-z.]{2,7}$/,
+    domains: ["news.google.com"],
+    paidDomains: ["googleadservices.com"],
+    packages: ["com.google.android.googlequicksearchbox"],
+    agent: /GSA\//,
+    paidAliases: ["gads", "adwords", "google-ads", "google_ads", "googleads"],
+  },
+  {
+    id: "bing",
+    channel: "organic_search",
+    domains: ["bing.com"],
+    packages: ["com.microsoft.bing"],
+  },
+  {
+    id: "duckduckgo",
+    channel: "organic_search",
+    domains: ["duckduckgo.com"],
+    packages: ["com.duckduckgo.mobile.android"],
+    aliases: ["ddg"],
+  },
+  {
+    // Search hosts only: yahoo.com itself is a portal, mail is yahoo_mail.
+    id: "yahoo",
+    channel: "organic_search",
+    domains: ["search.yahoo.com", "search.yahoo.co.jp"],
+  },
+  {
+    id: "ecosia",
+    channel: "organic_search",
+    domains: ["ecosia.org"],
+    packages: ["com.ecosia.android"],
+  },
+  {
+    id: "yandex",
+    channel: "organic_search",
+    domains: ["yandex.ru", "yandex.com", "ya.ru"],
+  },
+  {
+    id: "baidu",
+    channel: "organic_search",
+    domains: ["baidu.com"],
+  },
+
+  // E-mail.
+  {
+    // spec.
+    id: "gmail",
+    channel: "email",
+    domains: ["mail.google.com"],
+    packages: ["com.google.android.gm"],
+  },
+  {
+    id: "outlook",
+    channel: "email",
+    domains: [
+      "outlook.com",
+      "outlook.live.com",
+      "outlook.office.com",
+      "outlook.office365.com",
+    ],
+    packages: ["com.microsoft.office.outlook"],
+    aliases: ["hotmail"],
+  },
+  {
+    id: "yahoo_mail",
+    channel: "email",
+    domains: ["mail.yahoo.com"],
+    packages: ["com.yahoo.mobile.client.android.mail"],
+    aliases: ["yahoomail"],
+  },
+
+  // AI assistants.
+  {
+    // spec. ChatGPT tags the links it cites with utm_source=chatgpt.com,
+    // which the host fallback in origin.ts resolves.
+    id: "chatgpt",
+    channel: "ai",
+    domains: ["chatgpt.com", "chat.openai.com"],
+    packages: ["com.openai.chatgpt"],
+    aliases: ["openai"],
+  },
+  {
+    id: "perplexity",
+    channel: "ai",
+    domains: ["perplexity.ai"],
+    packages: ["ai.perplexity.app.android"],
+  },
+  {
+    // spec: gemini.google.com; bard is the product's former name.
+    id: "gemini",
+    channel: "ai",
+    domains: ["gemini.google.com", "bard.google.com"],
+    packages: ["com.google.android.apps.bard"],
+    aliases: ["bard"],
+  },
+  {
+    id: "copilot",
+    channel: "ai",
+    domains: ["copilot.microsoft.com"],
+    packages: ["com.microsoft.copilot"],
+  },
+  {
+    id: "claude",
+    channel: "ai",
+    domains: ["claude.ai"],
+    packages: ["com.anthropic.claude"],
+    aliases: ["anthropic"],
+  },
+
+  // Display ad servers and a known referral: github.io pages belong to their
+  // authors, so only github.com itself is GitHub.
+  {
+    id: "doubleclick",
+    channel: "display",
+    domains: ["doubleclick.net"],
+  },
+  {
+    id: "taboola",
+    channel: "display",
+    domains: ["taboola.com"],
+  },
+  {
+    id: "outbrain",
+    channel: "display",
+    domains: ["outbrain.com"],
+  },
+  {
+    id: "github",
+    channel: "referral",
+    domains: ["github.com"],
+    packages: ["com.github.android"],
+  },
+
+  // A QR code is a tracked link printed off the web: the scan carries no
+  // referrer and no app, so the link's own declaration is the only evidence.
+  {
+    id: "qr",
+    channel: "offline",
+    aliases: ["qrcode", "qr_code", "qr-code"],
+  },
+];
+
+/**
+ * Click-ID keys the tracker may report by name, in priority order: the ad
+ * networks' own ids first, then the organic share ids, `fbclid` last because
+ * Meta adds it to organic clicks too. `unknown` platforms take the referrer's.
+ */
+export const CLICK_IDS: ReadonlyArray<
+  readonly [key: string, platform: string, channel: Channel]
+> = [
+  ["gclid", "google", "paid_search"],
+  ["gbraid", "google", "paid_search"],
+  ["wbraid", "google", "paid_search"],
+  ["msclkid", "bing", "paid_search"],
+  ["ttclid", "tiktok", "paid_social"],
+  ["twclid", "x", "paid_social"],
+  ["li_fat_id", "linkedin", "paid_social"],
+  ["ScCid", "snapchat", "paid_social"],
+  ["rdt_cid", "reddit", "paid_social"],
+  ["dclid", "doubleclick", "display"],
+  ["epik", "pinterest", "organic_social"],
+  ["srsltid", "google", "organic_search"],
+  ["mc_cid", "unknown", "email"],
+  ["mc_eid", "unknown", "email"],
+  ["_hsenc", "unknown", "email"],
+  ["igsh", "instagram", "organic_social"],
+  ["igshid", "instagram", "organic_social"],
+  ["fbclid", "meta", "organic_social"],
+];