@iann29/rastro 0.4.0 → 0.6.0

package/src/component/people.ts

@@ -0,0 +1,321 @@
+import { v } from "convex/values";
+import { query, mutation, type MutationCtx } from "./_generated/server.js";
+import type { Doc, Id } from "./_generated/dataModel.js";
+import { fail } from "./errors.js";
+import {
+  personFilterValidator,
+  visitorProfileValidator,
+  type PersonAttributes,
+  type PersonFilter,
+} from "./validators.js";
+
+export const personDocumentValidator = visitorProfileValidator.extend({
+  _id: v.id("visitorProfiles"),
+  _creationTime: v.number(),
+  siteId: v.id("sites"),
+});
+
+export function profileSearchFields(name?: string, email?: string) {
+  return {
+    searchName: normalizeSearch(name ?? ""),
+    searchEmail: normalizeSearch(email ?? ""),
+  };
+}
+
+function normalizeSearch(value: string) {
+  return value.normalize("NFKC").trim().toLowerCase();
+}
+
+export function mergePersonAttributes(
+  previous: PersonAttributes = {},
+  patch?: Record<string, string | number | boolean | null>,
+): PersonAttributes {
+  const attributes = { ...previous };
+  for (const [key, value] of Object.entries(patch ?? {})) {
+    validateAttributeKey(key);
+    if (value === null) delete attributes[key];
+    else {
+      validateAttributeValue(value);
+      attributes[key] = value;
+    }
+  }
+  if (
+    Object.keys(attributes).length > 32 ||
+    new TextEncoder().encode(JSON.stringify(attributes)).length > 4096
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attributes allow at most 32 keys and 4096 UTF-8 bytes",
+    );
+  return Object.fromEntries(
+    Object.entries(attributes).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
+  );
+}
+
+function validateAttributeKey(key: string) {
+  if (
+    !/^[A-Za-z][A-Za-z0-9_.-]{0,63}$/.test(key) ||
+    ["__proto__", "constructor", "prototype"].includes(key)
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute keys must start with a letter and contain up to 64 ASCII letters, digits, dots, underscores or hyphens",
+    );
+}
+
+function validateAttributeValue(value: string | number | boolean) {
+  if (
+    (typeof value === "string" && value.length > 512) ||
+    (typeof value === "number" && !Number.isFinite(value))
+  )
+    fail(
+      "INVALID_ARGUMENT",
+      "attribute values require finite numbers or strings up to 512 characters",
+    );
+}
+
+/** Index each current attribute once; ordinary reads never scan all profiles. */
+export async function syncPersonAttributes(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+  attributes: PersonAttributes,
+) {
+  const rows = await ctx.db
+    .query("visitorAttributes")
+    .withIndex("by_siteId_and_visitorId", (q) =>
+      q.eq("siteId", siteId).eq("visitorId", visitorId),
+    )
+    .take(33);
+  const present = new Set<string>();
+  for (const row of rows) {
+    present.add(row.key);
+    if (!Object.hasOwn(attributes, row.key))
+      await ctx.db.delete("visitorAttributes", row._id);
+    else if (attributes[row.key] !== row.value)
+      await ctx.db.patch("visitorAttributes", row._id, {
+        value: attributes[row.key],
+      });
+  }
+  for (const [key, value] of Object.entries(attributes)) {
+    if (!present.has(key))
+      await ctx.db.insert("visitorAttributes", {
+        siteId,
+        visitorId,
+        key,
+        value,
+      });
+  }
+}
+
+export function publicPerson(row: Doc<"visitorProfiles">) {
+  const { searchName: _name, searchEmail: _email, ...person } = row;
+  return person;
+}
+
+/** An optional, bounded upgrade for profiles created before the directory. */
+export const backfillSearch = mutation({
+  args: { siteId: v.id("sites") },
+  returns: v.object({ updated: v.number(), isDone: v.boolean() }),
+  handler: async (ctx, { siteId }) => {
+    if (!(await ctx.db.get("sites", siteId)))
+      fail("NOT_FOUND", "site not found");
+    const rows = await ctx.db
+      .query("visitorProfiles")
+      .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+        q.eq("siteId", siteId).eq("searchName", undefined),
+      )
+      .take(65);
+    for (const row of rows.slice(0, 64)) {
+      await ctx.db.patch(
+        "visitorProfiles",
+        row._id,
+        profileSearchFields(row.name, row.email),
+      );
+      await syncPersonAttributes(
+        ctx,
+        siteId,
+        row.visitorId,
+        row.attributes ?? {},
+      );
+    }
+    return { updated: Math.min(64, rows.length), isDone: rows.length <= 64 };
+  },
+});
+
+/** Exclusive upper bound for a Unicode prefix, including supplementary characters. */
+function afterPrefix(prefix: string) {
+  const points = Array.from(prefix);
+  for (let i = points.length - 1; i >= 0; i--) {
+    const code = points[i].codePointAt(0)!;
+    if (code < 0x10ffff)
+      return (
+        points.slice(0, i).join("") +
+        String.fromCodePoint(code + 1 === 0xd800 ? 0xe000 : code + 1)
+      );
+  }
+  return undefined;
+}
+
+function normalizedFilter(filter?: PersonFilter): PersonFilter | undefined {
+  if (!filter) return undefined;
+  if (filter.field === "attribute") {
+    validateAttributeKey(filter.key);
+    validateAttributeValue(filter.value);
+    return { field: "attribute", key: filter.key, value: filter.value };
+  }
+  if (filter.prefix.length > 320)
+    fail("INVALID_ARGUMENT", "search prefix exceeds 320 characters");
+  const prefix =
+    filter.field === "visitorId"
+      ? filter.prefix.trim()
+      : normalizeSearch(filter.prefix);
+  return prefix ? { field: filter.field, prefix } : undefined;
+}
+
+export const list = query({
+  args: {
+    siteId: v.id("sites"),
+    filter: v.optional(personFilterValidator),
+    paginationOpts: v.object({
+      cursor: v.union(v.string(), v.null()),
+      numItems: v.number(),
+    }),
+  },
+  returns: v.object({
+    page: v.array(personDocumentValidator),
+    isDone: v.boolean(),
+    continueCursor: v.string(),
+  }),
+  handler: async (ctx, args) => {
+    const filter = normalizedFilter(args.filter);
+    const scope = JSON.stringify([args.siteId, filter ?? null]);
+    const limit = args.paginationOpts.numItems;
+    if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50)
+      fail("INVALID_ARGUMENT", "numItems must be an integer between 1 and 50");
+    let after: { value: string; visitorId: string } | undefined;
+    if (args.paginationOpts.cursor !== null) {
+      try {
+        // NFKC may expand one character into several, in both scope and value.
+        if (args.paginationOpts.cursor.length > 16_384) throw new Error();
+        const cursor = JSON.parse(args.paginationOpts.cursor) as Record<
+          string,
+          unknown
+        >;
+        if (
+          cursor.v !== 1 ||
+          cursor.scope !== scope ||
+          typeof cursor.value !== "string" ||
+          typeof cursor.visitorId !== "string"
+        )
+          throw new Error();
+        after = { value: cursor.value, visitorId: cursor.visitorId };
+        if (
+          filter &&
+          filter.field !== "attribute" &&
+          !after.value.startsWith(filter.prefix)
+        )
+          throw new Error();
+      } catch {
+        fail("INVALID_ARGUMENT", "cursor does not belong to this people query");
+      }
+    }
+    let rows: Doc<"visitorProfiles">[];
+    const take = limit + 1;
+    if (filter?.field === "attribute") {
+      const matches = await ctx.db
+        .query("visitorAttributes")
+        .withIndex("by_siteId_and_key_and_value_and_visitorId", (q) =>
+          q
+            .eq("siteId", args.siteId)
+            .eq("key", filter.key)
+            .eq("value", filter.value)
+            .gt("visitorId", after?.visitorId ?? ""),
+        )
+        .take(take);
+      rows = await Promise.all(
+        matches.map(async (match) => {
+          const row = await ctx.db
+            .query("visitorProfiles")
+            .withIndex("by_siteId_and_visitorId", (q) =>
+              q.eq("siteId", args.siteId).eq("visitorId", match.visitorId),
+            )
+            .unique();
+          if (!row) fail("NOT_FOUND", "person attribute index has no profile");
+          return row;
+        }),
+      );
+    } else if (filter && filter.field !== "visitorId") {
+      const pending = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_searchName_and_visitorId", (q) =>
+          q.eq("siteId", args.siteId).eq("searchName", undefined),
+        )
+        .first();
+      if (pending)
+        fail(
+          "CONFLICT",
+          "Run backfillVisitorProfiles for this site before searching existing profiles",
+        );
+      const field = filter.field === "name" ? "searchName" : "searchEmail";
+      const index =
+        filter.field === "name"
+          ? "by_siteId_and_searchName_and_visitorId"
+          : "by_siteId_and_searchEmail_and_visitorId";
+      rows = after
+        ? await ctx.db
+            .query("visitorProfiles")
+            .withIndex(index, (q) =>
+              q
+                .eq("siteId", args.siteId)
+                .eq(field, after.value)
+                .gt("visitorId", after.visitorId),
+            )
+            .take(take)
+        : [];
+      if (rows.length < take) {
+        const upper = afterPrefix(filter.prefix);
+        const rest = await ctx.db
+          .query("visitorProfiles")
+          .withIndex(index, (q) => {
+            const lower = after
+              ? q.eq("siteId", args.siteId).gt(field, after.value)
+              : q.eq("siteId", args.siteId).gte(field, filter.prefix);
+            return upper === undefined ? lower : lower.lt(field, upper);
+          })
+          .take(take - rows.length);
+        rows.push(...rest);
+      }
+    } else {
+      const prefix = filter?.prefix ?? "";
+      const upper = afterPrefix(prefix);
+      rows = await ctx.db
+        .query("visitorProfiles")
+        .withIndex("by_siteId_and_visitorId", (q) => {
+          const lower = after
+            ? q.eq("siteId", args.siteId).gt("visitorId", after.visitorId)
+            : q.eq("siteId", args.siteId).gte("visitorId", prefix);
+          return upper === undefined ? lower : lower.lt("visitorId", upper);
+        })
+        .take(take);
+    }
+    const page = rows.slice(0, limit);
+    const last = page.at(-1);
+    return {
+      page: page.map(publicPerson),
+      isDone: rows.length <= limit,
+      continueCursor: JSON.stringify({
+        v: 1,
+        scope,
+        visitorId: last?.visitorId ?? after?.visitorId ?? "",
+        value: last
+          ? filter?.field === "name"
+            ? last.searchName
+            : filter?.field === "email"
+              ? last.searchEmail
+              : last.visitorId
+          : (after?.value ?? ""),
+      }),
+    };
+  },
+});

package/src/component/platforms.ts

@@ -0,0 +1,359 @@
+import type { Channel } from "./validators.js";
+
+/**
+ * The closed table origin.ts classifies against. Bump the version whenever a
+ * row changes, and add a corpus case (origin.test.ts) for the change.
+ */
+export const PLATFORM_TABLE_VERSION = 1;
+
+export type PlatformRow = {
+  id: string;
+  /** Where the platform's traffic lands when nothing says paid or organic. */
+  channel: Channel;
+  /** Registrable-domain suffixes: a host equal to one or ending in "." + it. */
+  domains?: readonly string[];
+  /** Hosts matched in full, for a family with many country domains. */
+  hostPattern?: RegExp;
+  /** Hosts that only ever send the platform's ads (the paid channel). */
+  paidDomains?: readonly string[];
+  /** Android packages, as an `android-app://<package>` referrer names them. */
+  packages?: readonly string[];
+  /** The platform's in-app browser marker in the User-Agent. */
+  agent?: RegExp;
+  /** utm_source spellings, lowercase; the id itself always matches. */
+  aliases?: readonly string[];
+  /** utm_source spellings that themselves declare the paid channel. */
+  paidAliases?: readonly string[];
+};
+
+// Ordered for `inAppPlatform`: the first row whose marker matches wins, so
+// Messenger (its iOS agent carries `FBAN/`) precedes Facebook and Threads
+// precedes Instagram. Sources per row: "spec" is the approved proposal's
+// table; the rest is each platform's own domains, link shims, and Play Store
+// package ids, and in-app agents as the apps send them.
+export const PLATFORMS: readonly PlatformRow[] = [
+  // Social.
+  {
+    // Threads: spec (threads.net → threads.com); agent is the app's codename.
+    id: "threads",
+    channel: "organic_social",
+    domains: ["threads.net", "threads.com"],
+    packages: ["com.instagram.barcelona"],
+    agent: /Barcelona/,
+  },
+  {
+    // spec.
+    id: "instagram",
+    channel: "organic_social",
+    domains: ["instagram.com"],
+    packages: ["com.instagram.android"],
+    agent: /Instagram/,
+    aliases: ["ig", "insta"],
+  },
+  {
+    // spec; + com.facebook.mlite (Messenger Lite), "msg" is Meta's
+    // {{site_source_name}} value for Messenger placements.
+    id: "messenger",
+    channel: "messaging",
+    domains: ["messenger.com", "m.me"],
+    packages: ["com.facebook.orca", "com.facebook.mlite"],
+    agent: /Orca-Android|MessengerLite|MessengerForiOS/,
+    aliases: ["msg"],
+  },
+  {
+    // spec; + fb.watch (video short links).
+    id: "facebook",
+    channel: "organic_social",
+    domains: ["facebook.com", "fb.com", "fb.me", "fb.watch"],
+    packages: ["com.facebook.katana", "com.facebook.lite"],
+    agent: /FBAN\/|FBAV\/|FB_IAB/,
+    aliases: ["fb", "face"],
+  },
+  {
+    // spec: only an `fbclid` with nothing that tells Facebook from Instagram.
+    id: "meta",
+    channel: "organic_social",
+  },
+  {
+    // spec; + TikTok Lite (com.zhiliaoapp.musically.go).
+    id: "tiktok",
+    channel: "organic_social",
+    domains: ["tiktok.com"],
+    packages: [
+      "com.zhiliaoapp.musically",
+      "com.ss.android.ugc.trill",
+      "com.zhiliaoapp.musically.go",
+    ],
+    agent: /BytedanceWebview|musical_ly|TikTok/,
+    aliases: ["tt"],
+  },
+  {
+    // spec.
+    id: "kwai",
+    channel: "organic_social",
+    domains: ["kwai.com", "kwai-video.com", "kw.ai"],
+    packages: ["com.kwai.video"],
+    agent: /Kwai/,
+  },
+  {
+    // spec.
+    id: "youtube",
+    channel: "organic_social",
+    domains: ["youtube.com", "youtu.be"],
+    packages: ["com.google.android.youtube"],
+    aliases: ["yt"],
+  },
+  {
+    // spec; + TwitterAndroid (the Android app's agent).
+    id: "x",
+    channel: "organic_social",
+    domains: ["x.com", "twitter.com", "t.co"],
+    packages: ["com.twitter.android"],
+    agent: /Twitter for |TwitterAndroid/,
+    aliases: ["twitter"],
+  },
+  {
+    // spec.
+    id: "linkedin",
+    channel: "organic_social",
+    domains: ["linkedin.com", "lnkd.in"],
+    packages: ["com.linkedin.android"],
+    agent: /LinkedInApp/,
+    aliases: ["li"],
+  },
+  {
+    id: "pinterest",
+    channel: "organic_social",
+    domains: ["pinterest.com", "pin.it"],
+    packages: ["com.pinterest"],
+    agent: /Pinterest\//,
+    aliases: ["pin"],
+  },
+  {
+    id: "reddit",
+    channel: "organic_social",
+    domains: ["reddit.com", "redd.it"],
+    packages: ["com.reddit.frontpage"],
+  },
+  {
+    id: "snapchat",
+    channel: "organic_social",
+    domains: ["snapchat.com"],
+    packages: ["com.snapchat.android"],
+    agent: /Snapchat/,
+    aliases: ["snap"],
+  },
+
+  // Messaging.
+  {
+    // spec; + wl.co (WhatsApp's link shim, l.wl.co).
+    id: "whatsapp",
+    channel: "messaging",
+    domains: ["whatsapp.com", "wa.me", "wl.co"],
+    packages: ["com.whatsapp", "com.whatsapp.w4b"],
+    aliases: ["wa", "zap", "whats"],
+  },
+  {
+    // spec; + Telegram X (org.thunderdog.challegram).
+    id: "telegram",
+    channel: "messaging",
+    domains: ["t.me", "telegram.me", "telegram.org"],
+    packages: ["org.telegram.messenger", "org.thunderdog.challegram"],
+    aliases: ["tg"],
+  },
+  {
+    id: "discord",
+    channel: "messaging",
+    domains: ["discord.com", "discordapp.com", "discord.gg"],
+    packages: ["com.discord"],
+  },
+  {
+    // The Android package is `com.Slack`; referrer hosts arrive lowercase.
+    id: "slack",
+    channel: "messaging",
+    domains: ["slack.com"],
+    packages: ["com.slack"],
+  },
+  {
+    id: "line",
+    channel: "messaging",
+    domains: ["line.me"],
+    packages: ["jp.naver.line.android"],
+    agent: / Line\//,
+  },
+
+  // Search. Google's country domains match one pattern, not a suffix, so
+  // mail, docs, drive and every other google.com role host stay out of it.
+  {
+    // spec: google.<tld>, googleadservices.com, the Google app and GSA/.
+    id: "google",
+    channel: "organic_search",
+    hostPattern: /^(?:www\.)?google\.[a-z.]{2,7}$/,
+    domains: ["news.google.com"],
+    paidDomains: ["googleadservices.com"],
+    packages: ["com.google.android.googlequicksearchbox"],
+    agent: /GSA\//,
+    paidAliases: ["gads", "adwords", "google-ads", "google_ads", "googleads"],
+  },
+  {
+    id: "bing",
+    channel: "organic_search",
+    domains: ["bing.com"],
+    packages: ["com.microsoft.bing"],
+  },
+  {
+    id: "duckduckgo",
+    channel: "organic_search",
+    domains: ["duckduckgo.com"],
+    packages: ["com.duckduckgo.mobile.android"],
+    aliases: ["ddg"],
+  },
+  {
+    // Search hosts only: yahoo.com itself is a portal, mail is yahoo_mail.
+    id: "yahoo",
+    channel: "organic_search",
+    domains: ["search.yahoo.com", "search.yahoo.co.jp"],
+  },
+  {
+    id: "ecosia",
+    channel: "organic_search",
+    domains: ["ecosia.org"],
+    packages: ["com.ecosia.android"],
+  },
+  {
+    id: "yandex",
+    channel: "organic_search",
+    domains: ["yandex.ru", "yandex.com", "ya.ru"],
+  },
+  {
+    id: "baidu",
+    channel: "organic_search",
+    domains: ["baidu.com"],
+  },
+
+  // E-mail.
+  {
+    // spec.
+    id: "gmail",
+    channel: "email",
+    domains: ["mail.google.com"],
+    packages: ["com.google.android.gm"],
+  },
+  {
+    id: "outlook",
+    channel: "email",
+    domains: [
+      "outlook.com",
+      "outlook.live.com",
+      "outlook.office.com",
+      "outlook.office365.com",
+    ],
+    packages: ["com.microsoft.office.outlook"],
+    aliases: ["hotmail"],
+  },
+  {
+    id: "yahoo_mail",
+    channel: "email",
+    domains: ["mail.yahoo.com"],
+    packages: ["com.yahoo.mobile.client.android.mail"],
+    aliases: ["yahoomail"],
+  },
+
+  // AI assistants.
+  {
+    // spec. ChatGPT tags the links it cites with utm_source=chatgpt.com,
+    // which the host fallback in origin.ts resolves.
+    id: "chatgpt",
+    channel: "ai",
+    domains: ["chatgpt.com", "chat.openai.com"],
+    packages: ["com.openai.chatgpt"],
+    aliases: ["openai"],
+  },
+  {
+    id: "perplexity",
+    channel: "ai",
+    domains: ["perplexity.ai"],
+    packages: ["ai.perplexity.app.android"],
+  },
+  {
+    // spec: gemini.google.com; bard is the product's former name.
+    id: "gemini",
+    channel: "ai",
+    domains: ["gemini.google.com", "bard.google.com"],
+    packages: ["com.google.android.apps.bard"],
+    aliases: ["bard"],
+  },
+  {
+    id: "copilot",
+    channel: "ai",
+    domains: ["copilot.microsoft.com"],
+    packages: ["com.microsoft.copilot"],
+  },
+  {
+    id: "claude",
+    channel: "ai",
+    domains: ["claude.ai"],
+    packages: ["com.anthropic.claude"],
+    aliases: ["anthropic"],
+  },
+
+  // Display ad servers and a known referral: github.io pages belong to their
+  // authors, so only github.com itself is GitHub.
+  {
+    id: "doubleclick",
+    channel: "display",
+    domains: ["doubleclick.net"],
+  },
+  {
+    id: "taboola",
+    channel: "display",
+    domains: ["taboola.com"],
+  },
+  {
+    id: "outbrain",
+    channel: "display",
+    domains: ["outbrain.com"],
+  },
+  {
+    id: "github",
+    channel: "referral",
+    domains: ["github.com"],
+    packages: ["com.github.android"],
+  },
+
+  // A QR code is a tracked link printed off the web: the scan carries no
+  // referrer and no app, so the link's own declaration is the only evidence.
+  {
+    id: "qr",
+    channel: "offline",
+    aliases: ["qrcode", "qr_code", "qr-code"],
+  },
+];
+
+/**
+ * Click-ID keys the tracker may report by name, in priority order: the ad
+ * networks' own ids first, then the organic share ids, `fbclid` last because
+ * Meta adds it to organic clicks too. `unknown` platforms take the referrer's.
+ */
+export const CLICK_IDS: ReadonlyArray<
+  readonly [key: string, platform: string, channel: Channel]
+> = [
+  ["gclid", "google", "paid_search"],
+  ["gbraid", "google", "paid_search"],
+  ["wbraid", "google", "paid_search"],
+  ["msclkid", "bing", "paid_search"],
+  ["ttclid", "tiktok", "paid_social"],
+  ["twclid", "x", "paid_social"],
+  ["li_fat_id", "linkedin", "paid_social"],
+  ["ScCid", "snapchat", "paid_social"],
+  ["rdt_cid", "reddit", "paid_social"],
+  ["dclid", "doubleclick", "display"],
+  ["epik", "pinterest", "organic_social"],
+  ["srsltid", "google", "organic_search"],
+  ["mc_cid", "unknown", "email"],
+  ["mc_eid", "unknown", "email"],
+  ["_hsenc", "unknown", "email"],
+  ["igsh", "instagram", "organic_social"],
+  ["igshid", "instagram", "organic_social"],
+  ["fbclid", "meta", "organic_social"],
+];