@iann29/rastro 0.4.0 → 0.6.0

package/src/component/trackedLinks.ts

@@ -0,0 +1,384 @@
+import { v, type Infer } from "convex/values";
+import { internal } from "./_generated/api.js";
+import { internalMutation, mutation, query } from "./_generated/server.js";
+import {
+  DAY_MS,
+  MAX_TRACKED_LINKS_PER_SITE,
+  TRACKED_LINK_CLICK_SHARDS,
+} from "./constants.js";
+import { fail } from "./errors.js";
+import { cleanString, originAllowed, sanitizeOpaqueId } from "./sanitize.js";
+import {
+  trackedLinkChannelValidator,
+  trackedLinkFieldsValidator,
+} from "./validators.js";
+
+const SLUG = /^[a-z0-9][a-z0-9-]{1,30}[a-z0-9]$/;
+const PLATFORM = /^[a-z0-9_]{2,24}$/;
+const MAX_DESTINATION_LENGTH = 2048;
+const MAX_CAMPAIGN_LENGTH = 64;
+const RECENT_DAYS = 30;
+// Each shard's all-time counter sits beside the daily ones under this day, so
+// a link's total is TRACKED_LINK_CLICK_SHARDS rows however long it has lived.
+const ALL_TIME = 0;
+const PURGE_BATCH = 500;
+
+type Channel = Infer<typeof trackedLinkChannelValidator>;
+
+const trackedLinkDocumentValidator = trackedLinkFieldsValidator.extend({
+  _id: v.id("trackedLinks"),
+  _creationTime: v.number(),
+});
+
+const redirectValidator = trackedLinkFieldsValidator.pick(
+  "slug",
+  "destination",
+  "platform",
+  "channel",
+  "campaign",
+);
+
+export function isTrackedLinkSlug(value: string): boolean {
+  return SLUG.test(value);
+}
+
+/**
+ * Where a click lands: the destination with the link's utm_* appended. A key
+ * the destination already carries wins, and its own query keeps its exact
+ * encoding because the tags are appended as text.
+ */
+export function trackedLinkLocation(
+  link: Infer<typeof redirectValidator>,
+): string {
+  const url = new URL(link.destination);
+  const utm: Array<[string, string]> = [
+    ["utm_source", link.platform],
+    ["utm_medium", link.channel],
+    ["utm_campaign", link.campaign ?? link.slug],
+    ["utm_content", link.slug],
+  ];
+  const tags = new URLSearchParams(
+    utm.filter(([key]) => !url.searchParams.has(key)),
+  ).toString();
+  if (tags) url.search = url.search ? `${url.search}&${tags}` : `?${tags}`;
+  return url.href;
+}
+
+function sanitizeLinkSlug(value: string): string {
+  const slug = value.trim().toLowerCase();
+  if (!isTrackedLinkSlug(slug)) {
+    fail(
+      "INVALID_ARGUMENT",
+      "slug must be 3 to 32 lowercase letters, digits or hyphens, starting and ending with a letter or digit",
+    );
+  }
+  return slug;
+}
+
+function sanitizePlatform(value: string): string {
+  const platform = value.trim().toLowerCase();
+  if (!PLATFORM.test(platform)) {
+    fail(
+      "INVALID_ARGUMENT",
+      "platform must be 2 to 24 lowercase letters, digits or underscores",
+    );
+  }
+  return platform;
+}
+
+function sanitizeCampaign(value: string | undefined): string | undefined {
+  return value === undefined
+    ? undefined
+    : cleanString(value, MAX_CAMPAIGN_LENGTH) || undefined;
+}
+
+// No open redirect: the destination is an absolute http(s) URL on one of the
+// site's own domains, the same rule the ingestion origin check applies.
+function sanitizeDestination(value: string, domains: string[]): string {
+  let url: URL | undefined;
+  try {
+    if (value.length <= MAX_DESTINATION_LENGTH) {
+      url = new URL(cleanString(value, MAX_DESTINATION_LENGTH));
+    }
+  } catch {
+    url = undefined;
+  }
+  if (
+    !url ||
+    (url.protocol !== "http:" && url.protocol !== "https:") ||
+    url.username ||
+    url.password
+  ) {
+    fail(
+      "INVALID_ARGUMENT",
+      "destination must be an absolute http(s) URL without credentials",
+    );
+  }
+  if (!originAllowed(url.href, domains)) {
+    fail(
+      "INVALID_ARGUMENT",
+      "destination must be on one of the site's domains",
+    );
+  }
+  return url.href;
+}
+
+export const create = mutation({
+  args: {
+    siteId: v.id("sites"),
+    slug: v.string(),
+    destination: v.string(),
+    platform: v.string(),
+    channel: trackedLinkChannelValidator,
+    campaign: v.optional(v.string()),
+    createdBy: v.string(),
+  },
+  returns: v.id("trackedLinks"),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    const slug = sanitizeLinkSlug(args.slug);
+    const destination = sanitizeDestination(args.destination, site.domains);
+    const platform = sanitizePlatform(args.platform);
+    let createdBy: string;
+    try {
+      createdBy = sanitizeOpaqueId(args.createdBy, "createdBy");
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid createdBy",
+      );
+    }
+    const taken = await ctx.db
+      .query("trackedLinks")
+      .withIndex("by_slug", (range) => range.eq("slug", slug))
+      .unique();
+    if (taken) fail("CONFLICT", "another tracked link already uses this slug");
+    const links = await ctx.db
+      .query("trackedLinks")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .take(MAX_TRACKED_LINKS_PER_SITE);
+    if (links.length >= MAX_TRACKED_LINKS_PER_SITE) {
+      fail("LIMIT_EXCEEDED", "tracked link limit reached", {
+        limit: MAX_TRACKED_LINKS_PER_SITE,
+      });
+    }
+    return await ctx.db.insert("trackedLinks", {
+      siteId: args.siteId,
+      slug,
+      destination,
+      platform,
+      channel: args.channel,
+      campaign: sanitizeCampaign(args.campaign),
+      createdAt: Date.now(),
+      createdBy,
+    });
+  },
+});
+
+/** Everything but the slug, which every link already printed carries. */
+export const update = mutation({
+  args: {
+    siteId: v.id("sites"),
+    linkId: v.id("trackedLinks"),
+    destination: v.optional(v.string()),
+    platform: v.optional(v.string()),
+    channel: v.optional(trackedLinkChannelValidator),
+    campaign: v.optional(v.union(v.string(), v.null())),
+    disabled: v.optional(v.boolean()),
+  },
+  returns: v.null(),
+  handler: async (ctx, args) => {
+    const link = await ctx.db.get("trackedLinks", args.linkId);
+    if (!link || link.siteId !== args.siteId) {
+      fail("NOT_FOUND", "tracked link not found");
+    }
+    const patch: {
+      destination?: string;
+      platform?: string;
+      channel?: Channel;
+      campaign?: string;
+      disabledAt?: number;
+    } = {};
+    if (args.destination !== undefined) {
+      const site = await ctx.db.get("sites", args.siteId);
+      if (!site) fail("NOT_FOUND", "site not found");
+      patch.destination = sanitizeDestination(args.destination, site.domains);
+    }
+    if (args.platform !== undefined) {
+      patch.platform = sanitizePlatform(args.platform);
+    }
+    if (args.channel !== undefined) patch.channel = args.channel;
+    if (args.campaign !== undefined) {
+      patch.campaign = sanitizeCampaign(args.campaign ?? undefined);
+    }
+    if (args.disabled !== undefined) {
+      patch.disabledAt = args.disabled
+        ? (link.disabledAt ?? Date.now())
+        : undefined;
+    }
+    await ctx.db.patch("trackedLinks", args.linkId, patch);
+    return null;
+  },
+});
+
+export const remove = mutation({
+  args: { siteId: v.id("sites"), linkId: v.id("trackedLinks") },
+  returns: v.null(),
+  handler: async (ctx, args): Promise<null> => {
+    const link = await ctx.db.get("trackedLinks", args.linkId);
+    if (!link || link.siteId !== args.siteId) {
+      fail("NOT_FOUND", "tracked link not found");
+    }
+    await ctx.db.delete("trackedLinks", args.linkId);
+    await ctx.scheduler.runAfter(0, internal.trackedLinks.purgeClicks, {
+      linkId: args.linkId,
+    });
+    return null;
+  },
+});
+
+/**
+ * The site's links, newest first, with their clicks and bot hits: all time,
+ * and over the 30 UTC days ending on the day of `now`. The caller passes the
+ * clock so the subscription does not go stale.
+ */
+export const list = query({
+  args: { siteId: v.id("sites"), now: v.number() },
+  returns: v.array(
+    trackedLinkDocumentValidator.extend({
+      clicks: v.number(),
+      bots: v.number(),
+      clicksLast30Days: v.number(),
+      botsLast30Days: v.number(),
+    }),
+  ),
+  handler: async (ctx, args) => {
+    if (!Number.isSafeInteger(args.now) || args.now < RECENT_DAYS * DAY_MS) {
+      fail("INVALID_ARGUMENT", "now must be a Unix time in milliseconds");
+    }
+    const since =
+      Math.floor(args.now / DAY_MS) * DAY_MS - (RECENT_DAYS - 1) * DAY_MS;
+    const links = await ctx.db
+      .query("trackedLinks")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .order("desc")
+      .take(MAX_TRACKED_LINKS_PER_SITE);
+    const sum = (rows: Array<{ clicks: number; bots: number }>) =>
+      rows.reduce(
+        (total, row) => ({
+          clicks: total.clicks + row.clicks,
+          bots: total.bots + row.bots,
+        }),
+        { clicks: 0, bots: 0 },
+      );
+    // Up to 4 + 120 counter rows per link, so a full site of 200 links whose
+    // every link fills every shard every day reads about 24,800 rows, inside
+    // the 32,000 documents a query may read. Raising either cap needs a
+    // finished day's shards folded into one row first.
+    return await Promise.all(
+      links.map(async (link) => {
+        const total = sum(
+          await ctx.db
+            .query("trackedLinkClicks")
+            .withIndex("by_linkId_and_dayStart_and_shard", (range) =>
+              range.eq("linkId", link._id).eq("dayStart", ALL_TIME),
+            )
+            .take(TRACKED_LINK_CLICK_SHARDS),
+        );
+        const recent = sum(
+          await ctx.db
+            .query("trackedLinkClicks")
+            .withIndex("by_linkId_and_dayStart_and_shard", (range) =>
+              range
+                .eq("linkId", link._id)
+                .gte("dayStart", since)
+                .lt("dayStart", since + RECENT_DAYS * DAY_MS),
+            )
+            .take(RECENT_DAYS * TRACKED_LINK_CLICK_SHARDS),
+        );
+        return {
+          ...link,
+          clicks: total.clicks,
+          bots: total.bots,
+          clicksLast30Days: recent.clicks,
+          botsLast30Days: recent.bots,
+        };
+      }),
+    );
+  },
+});
+
+/**
+ * Counts one click on a live link and returns where it goes, or null for an
+ * unknown or disabled slug. Bots are counted apart and never as clicks. The
+ * destination is checked against the site's domains again, so a domain the
+ * site has since dropped stops redirecting.
+ */
+export const recordClick = internalMutation({
+  args: { slug: v.string(), bot: v.boolean() },
+  returns: v.union(redirectValidator, v.null()),
+  handler: async (ctx, args) => {
+    const link = await ctx.db
+      .query("trackedLinks")
+      .withIndex("by_slug", (range) => range.eq("slug", args.slug))
+      .unique();
+    if (!link || link.disabledAt !== undefined) return null;
+    const site = await ctx.db.get("sites", link.siteId);
+    if (!site || !originAllowed(link.destination, site.domains)) return null;
+    const now = Date.now();
+    const shard = Math.floor(Math.random() * TRACKED_LINK_CLICK_SHARDS);
+    for (const dayStart of [Math.floor(now / DAY_MS) * DAY_MS, ALL_TIME]) {
+      const row = await ctx.db
+        .query("trackedLinkClicks")
+        .withIndex("by_linkId_and_dayStart_and_shard", (range) =>
+          range
+            .eq("linkId", link._id)
+            .eq("dayStart", dayStart)
+            .eq("shard", shard),
+        )
+        .unique();
+      if (row) {
+        await ctx.db.patch(
+          "trackedLinkClicks",
+          row._id,
+          args.bot ? { bots: row.bots + 1 } : { clicks: row.clicks + 1 },
+        );
+      } else {
+        await ctx.db.insert("trackedLinkClicks", {
+          linkId: link._id,
+          dayStart,
+          shard,
+          clicks: args.bot ? 0 : 1,
+          bots: args.bot ? 1 : 0,
+        });
+      }
+    }
+    return {
+      slug: link.slug,
+      destination: link.destination,
+      platform: link.platform,
+      channel: link.channel,
+      ...(link.campaign === undefined ? {} : { campaign: link.campaign }),
+    };
+  },
+});
+
+/** Deletes a removed link's counters a batch at a time. */
+export const purgeClicks = internalMutation({
+  args: { linkId: v.id("trackedLinks") },
+  returns: v.null(),
+  handler: async (ctx, args): Promise<null> => {
+    const rows = await ctx.db
+      .query("trackedLinkClicks")
+      .withIndex("by_linkId_and_dayStart_and_shard", (range) =>
+        range.eq("linkId", args.linkId),
+      )
+      .take(PURGE_BATCH);
+    for (const row of rows) await ctx.db.delete("trackedLinkClicks", row._id);
+    if (rows.length === PURGE_BATCH) {
+      await ctx.scheduler.runAfter(0, internal.trackedLinks.purgeClicks, args);
+    }
+    return null;
+  },
+});

package/src/component/useragent.ts

@@ -1,3 +1,5 @@
+import { PLATFORMS } from "./platforms.js";
+
 export type ClientClassification = {
   browser: string;
   os: string;
@@ -100,6 +102,15 @@ export function classifyUserAgent(userAgent: string): ClientClassification {
   };
 }
 
+/**
+ * The platform whose in-app browser sent the request (Instagram, Facebook,
+ * TikTok…), from the marker each app appends to the User-Agent. The system
+ * browser a messaging app hands a link to carries none.
+ */
+export function inAppPlatform(userAgent: string): string | undefined {
+  return PLATFORMS.find((row) => row.agent?.test(userAgent))?.id;
+}
+
 function matchAgent(
   table: readonly (readonly [RegExp, string])[],
   userAgent: string,

package/src/component/validators.ts

@@ -1,5 +1,54 @@
 import { v, type Infer } from "convex/values";
 
+export const personAttributeValueValidator = v.union(
+  v.string(),
+  v.number(),
+  v.boolean(),
+);
+export const personAttributesValidator = v.record(
+  v.string(),
+  personAttributeValueValidator,
+);
+export type PersonAttributes = Infer<typeof personAttributesValidator>;
+export const personFilterValidator = v.union(
+  v.object({
+    field: v.union(
+      v.literal("name"),
+      v.literal("email"),
+      v.literal("visitorId"),
+    ),
+    prefix: v.string(),
+  }),
+  v.object({
+    field: v.literal("attribute"),
+    key: v.string(),
+    value: personAttributeValueValidator,
+  }),
+);
+export type PersonFilter = Infer<typeof personFilterValidator>;
+
+/** A profile supplied by the product's trusted backend, shared by its visits. */
+export const visitorProfileValidator = v.object({
+  visitorId: v.string(),
+  name: v.optional(v.string()),
+  email: v.optional(v.string()),
+  attributes: v.optional(personAttributesValidator),
+  identifiedAt: v.number(),
+  updatedAt: v.number(),
+});
+export type VisitorProfile = Infer<typeof visitorProfileValidator>;
+
+export const identifyVisitorFields = {
+  visitorId: v.string(),
+  previousVisitorId: v.optional(v.string()),
+  // Omission preserves the value; null explicitly removes it.
+  name: v.optional(v.union(v.string(), v.null())),
+  email: v.optional(v.union(v.string(), v.null())),
+  attributes: v.optional(
+    v.record(v.string(), v.union(personAttributeValueValidator, v.null())),
+  ),
+};
+
 export const eventTypeValidator = v.union(
   v.literal("pageview"),
   v.literal("click"),
@@ -50,6 +99,9 @@ export const trackerEventValidator = v.object({
   // Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
   // events carry it; sanitization strips it from every other type.
   value: v.optional(v.number()),
+  // The names of the click-ID keys on the landing URL (`fbclid`, `gclid`…),
+  // never their values; sanitization keeps only the known ones (origin.ts).
+  clid: v.optional(v.array(v.string())),
 });
 
 export const ingestContextValidator = v.object({
@@ -61,8 +113,41 @@ export const ingestContextValidator = v.object({
   os: v.optional(v.string()),
   device: v.optional(v.string()),
   visitorKey: v.optional(v.string()),
+  // The platform whose in-app browser sent the batch (useragent.ts), derived
+  // by the HTTP action so the raw User-Agent never reaches a mutation.
+  inApp: v.optional(v.string()),
 });
 
+// How a session arrived (channel) and which rule proved it (evidence); see
+// origin.ts. "legacy" is never stored: reports mark a session recorded before
+// origins existed with it when they classify that session on read.
+export const channelValidator = v.union(
+  v.literal("direct"),
+  v.literal("organic_search"),
+  v.literal("paid_search"),
+  v.literal("organic_social"),
+  v.literal("paid_social"),
+  v.literal("messaging"),
+  v.literal("email"),
+  v.literal("affiliate"),
+  v.literal("ai"),
+  v.literal("display"),
+  v.literal("referral"),
+  // A link printed off the web: a QR code, a flyer, a receipt. Only a tracked
+  // link can say so, since the scan itself carries no referrer.
+  v.literal("offline"),
+);
+
+export const evidenceValidator = v.union(
+  v.literal("utm"),
+  v.literal("clickId"),
+  v.literal("referrer"),
+  v.literal("inApp"),
+  v.literal("affiliate"),
+  v.literal("none"),
+  v.literal("legacy"),
+);
+
 export const batchedEventValidator = trackerEventValidator.extend({
   ...ingestContextValidator.fields,
   aggregateCountry: v.optional(v.string()),
@@ -94,6 +179,10 @@ export const sessionFieldsValidator = v.object({
   exitPath: v.string(),
   referrer: v.optional(v.string()),
   source: v.string(),
+  // The first touch's origin (origin.ts); absent on sessions recorded before.
+  platform: v.optional(v.string()),
+  channel: v.optional(channelValidator),
+  evidence: v.optional(evidenceValidator),
   utmSource: v.optional(v.string()),
   utmMedium: v.optional(v.string()),
   utmCampaign: v.optional(v.string()),
@@ -155,6 +244,10 @@ export const dimensionTypeValidator = v.union(
   // host of an outbound click (never its path).
   v.literal("medium"),
   v.literal("outbound"),
+  // The session's origin platform, channel and evidence (origin.ts).
+  v.literal("platform"),
+  v.literal("channel"),
+  v.literal("evidence"),
 );
 
 export const dimensionSlotValidator = v.object({
@@ -254,6 +347,31 @@ export const affiliateFieldsValidator = v.object({
   updatedAt: v.number(),
 });
 
+// Where a tracked link was published: chosen at creation and carried to the
+// landing page as its utm_medium.
+export const trackedLinkChannelValidator = v.union(
+  v.literal("organic_social"),
+  v.literal("paid_social"),
+  v.literal("messaging"),
+  v.literal("email"),
+  v.literal("affiliate"),
+  v.literal("display"),
+  v.literal("referral"),
+  v.literal("offline"),
+);
+
+export const trackedLinkFieldsValidator = v.object({
+  siteId: v.id("sites"),
+  slug: v.string(),
+  destination: v.string(),
+  platform: v.string(),
+  channel: trackedLinkChannelValidator,
+  campaign: v.optional(v.string()),
+  createdAt: v.number(),
+  createdBy: v.string(),
+  disabledAt: v.optional(v.number()),
+});
+
 export const ingestResultValidator = v.object({
   accepted: v.number(),
   duplicates: v.number(),
@@ -278,3 +396,5 @@ export type IngestContext = Infer<typeof ingestContextValidator>;
 export type EventProperties = Infer<typeof eventPropertiesValidator>;
 export type FunnelStep = Infer<typeof funnelStepValidator>;
 export type EventType = Infer<typeof eventTypeValidator>;
+export type Channel = Infer<typeof channelValidator>;
+export type Evidence = Infer<typeof evidenceValidator>;