@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.5

package/src/component/ingest.ts

@@ -69,6 +69,7 @@ import {
 
 type DimensionType =
   | "source"
+  | "campaign"
   | "page"
   | "country"
   | "device"
@@ -146,6 +147,8 @@ type SessionState = {
   isNewVisitorDay: boolean;
   startedAt: number;
   source: string;
+  visitorKey: string;
+  utmCampaign?: string;
   country?: string;
   city?: string;
   latitude?: number;
@@ -783,8 +786,12 @@ async function accountEvent(
     );
   }
 
+  const campaignDimensions: Dimension[] = session.utmCampaign
+    ? [{ type: "campaign", value: session.utmCampaign }]
+    : [];
   const dimensions: Dimension[] = [
     { type: "source", value: session.source },
+    ...campaignDimensions,
     { type: "page", value: event.path },
     { type: "country", value: session.country ?? "unknown" },
     { type: "device", value: input.context.device ?? "unknown" },
@@ -842,6 +849,8 @@ async function accountTrustedSession(
     isNewVisitorDay: false,
     startedAt: existing.startedAt,
     source: existing.source,
+    utmCampaign: existing.utmCampaign,
+    visitorKey: existing.visitorKey ?? existing.visitorId,
     country: existing.country,
     city: existing.city,
     latitude: existing.latitude,
@@ -865,8 +874,16 @@ async function upsertSession(
   reportRollupDeltas: ReportRollupDeltas,
   batchState?: BatchState,
 ): Promise<SessionState> {
-  const source = sourceFromReferrer(event.referrer);
+  // A host-supplied id is the identity; anonymous sessions count through the
+  // daily key derived by the HTTP action, or once per session without one.
+  const visitorKey =
+    existing?.visitorKey ??
+    (event.visitorId !== event.sessionId
+      ? event.visitorId
+      : (context.visitorKey ?? event.sessionId));
   const utmSource = stringProperty(event.properties, "utm_source");
+  // Campaign traffic names its own source; the referrer host is the fallback.
+  const source = utmSource?.toLowerCase() ?? sourceFromReferrer(event.referrer);
   const utmMedium = stringProperty(event.properties, "utm_medium");
   const utmCampaign = stringProperty(event.properties, "utm_campaign");
   const attribution = await attributionForEvent(
@@ -883,6 +900,7 @@ async function upsertSession(
         siteId,
         event,
         attribution,
+        visitorKey,
       );
     }
     const startedAt = Math.min(existing.startedAt, event.timestamp);
@@ -932,6 +950,8 @@ async function upsertSession(
         Math.floor(event.timestamp / DAY_MS),
       startedAt,
       source: existing.source,
+      utmCampaign: existing.utmCampaign,
+      visitorKey,
       country: context.country ?? existing.country,
       city: context.city ?? existing.city,
       latitude: context.latitude ?? existing.latitude,
@@ -967,6 +987,7 @@ async function upsertSession(
     siteId,
     sessionId: event.sessionId,
     visitorId: event.visitorId,
+    visitorKey,
     startedAt: event.timestamp,
     lastSeenAt: event.timestamp,
     entryPath: event.path,
@@ -1000,6 +1021,7 @@ async function upsertSession(
       siteId,
       event,
       attribution,
+      visitorKey,
     );
   }
   if (batchState) {
@@ -1022,6 +1044,8 @@ async function upsertSession(
         Math.floor(event.timestamp / DAY_MS),
     startedAt: event.timestamp,
     source,
+    utmCampaign,
+    visitorKey,
     country: context.country,
     city: context.city,
     latitude: context.latitude,
@@ -1218,6 +1242,7 @@ async function accountHeartbeat(
       isNewVisitorDay: false,
       startedAt,
       source: session.source,
+      visitorKey: session.visitorKey ?? session.visitorId,
       country: context.country ?? session.country,
       city: context.city ?? session.city,
       latitude: context.latitude ?? session.latitude,
@@ -1855,6 +1880,7 @@ function foldAffiliateVisitRollup(
   siteId: Id<"sites">,
   event: TrackerEvent,
   attribution: AttributionSnapshot,
+  visitorKey: string,
 ) {
   const coordinates = reportRollupCoordinates(
     siteId,
@@ -1875,7 +1901,7 @@ function foldAffiliateVisitRollup(
     commissionCents: 0,
   };
   current.visits += 1;
-  current.visitorSketch = addVisitor(current.visitorSketch, event.visitorId);
+  current.visitorSketch = addVisitor(current.visitorSketch, visitorKey);
   deltas.affiliates.set(coordinates.key, current);
 }
 
@@ -2076,7 +2102,7 @@ function foldAggregates(
         : Number(session.isNewVisitorDay)
       : 0;
     if (recordTelemetry) {
-      delta.visitorSketch = addVisitor(delta.visitorSketch, event.visitorId);
+      delta.visitorSketch = addVisitor(delta.visitorSketch, session.visitorKey);
     }
     delta.conversions += Number(financialConversion);
     delta.revenueCents += revenueCents;

package/src/component/reports.ts

@@ -49,6 +49,7 @@ import {
   goalFieldsValidator,
   sessionFieldsValidator,
 } from "./validators.js";
+import { resolveVisitorIdentities } from "./visitors.js";
 
 const sessionDocumentValidator = sessionFieldsValidator.extend({
   _id: v.id("sessions"),
@@ -155,6 +156,7 @@ export const overview = query({
     totals: metricTotalsValidator,
     timeSeries: v.array(timePointValidator),
     topSources: v.array(topItemValidator),
+    topCampaigns: v.array(topItemValidator),
     topPages: v.array(topItemValidator),
     topCountries: v.array(topItemValidator),
     topDevices: v.array(topItemValidator),
@@ -174,6 +176,7 @@ export const overview = query({
       }),
       breakdowns: v.object({
         topSources: v.literal("eventVolume"),
+        topCampaigns: v.literal("eventVolume"),
         topPages: v.literal("eventVolume"),
         topCountries: v.literal("eventVolume"),
         topDevices: v.literal("eventVolume"),
@@ -187,7 +190,7 @@ export const overview = query({
         sessionReplay: v.literal("unsupported"),
         errorInsights: v.literal("unsupported"),
         outboundLinks: v.literal("eventVolume"),
-        botDetection: v.literal("unsupported"),
+        botDetection: v.literal("userAgent"),
       }),
     }),
   }),
@@ -314,6 +317,7 @@ export const overview = query({
         .sort(([left], [right]) => left - right)
         .map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
       topSources: top("source"),
+      topCampaigns: top("campaign"),
       topPages: top("page"),
       topCountries: top("country"),
       topDevices: top("device"),
@@ -330,6 +334,7 @@ export const overview = query({
         },
         breakdowns: {
           topSources: "eventVolume" as const,
+          topCampaigns: "eventVolume" as const,
           topPages: "eventVolume" as const,
           topCountries: "eventVolume" as const,
           topDevices: "eventVolume" as const,
@@ -343,7 +348,7 @@ export const overview = query({
           sessionReplay: "unsupported" as const,
           errorInsights: "unsupported" as const,
           outboundLinks: "eventVolume" as const,
-          botDetection: "unsupported" as const,
+          botDetection: "userAgent" as const,
         },
       },
     };
@@ -593,18 +598,20 @@ export const visitorJourney = query({
     }
     const rows: RawJourneyEvent[] = [];
     for (const siteId of siteIds) {
-      const cutoff = rows.length >= limit
-        ? rows[limit - 1]!.timestamp
-        : args.to;
-      rows.push(...await loadVisitorJourneyEvents(ctx, {
-        siteId,
-        visitorId: args.visitorId,
-        from: args.from,
-        to: cutoff,
-        resultLimit: limit,
-      }));
-      rows.sort(compareJourneyEvents);
-      if (rows.length > limit) rows.length = limit;
+      for (const visitorId of await resolveVisitorIdentities(ctx, siteId, args.visitorId)) {
+        const cutoff = rows.length >= limit
+          ? rows[limit - 1]!.timestamp
+          : args.to;
+        rows.push(...await loadVisitorJourneyEvents(ctx, {
+          siteId,
+          visitorId,
+          from: args.from,
+          to: cutoff,
+          resultLimit: limit,
+        }));
+        rows.sort(compareJourneyEvents);
+        if (rows.length > limit) rows.length = limit;
+      }
     }
     return hydrateEventContexts(ctx, rows);
   },

package/src/component/sanitize.ts

@@ -147,7 +147,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     sequence: event.sequence,
     target: event.target === undefined
       ? undefined
-      : cleanString(event.target, 160) || undefined,
+      : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
     href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
@@ -236,6 +236,9 @@ export function sanitizeContext(context: IngestContext | undefined): IngestConte
     browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
     os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
     device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
+    visitorKey: context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
+      ? cleanString(context.visitorKey, 128) || undefined
+      : undefined,
   };
 }
 

package/src/component/schema.ts

@@ -283,6 +283,21 @@ export default defineSchema({
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_slug", ["siteId", "slug"]),
 
+  visitorSecrets: defineTable({
+    key: v.literal("current"),
+    value: v.string(),
+    createdAt: v.number(),
+  }).index("by_key", ["key"]),
+
+  visitorAliases: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+    linkedAt: v.number(),
+  })
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
+
   visitorAttributions: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),

package/src/component/useragent.ts

@@ -0,0 +1,171 @@
+export type ClientClassification = {
+  browser: string;
+  os: string;
+  device: string;
+};
+
+// The tracker never reports the client: everything here is derived from headers
+// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
+// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
+// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
+// Firefox and Safari implement no client hints, so the User-Agent stays the
+// fallback for them and for insecure origins, where no hint is sent at all.
+const MAX_HINT_LENGTH = 512;
+const MAX_HINT_BRANDS = 16;
+
+const BRAND_PATTERN =
+  /"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
+const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
+
+// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
+// allowlisted: anything unrecognized (including the GREASE entries Chromium
+// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
+// entering the browser dimension verbatim.
+const HINT_BRANDS = new Map<string, string>([
+  ["brave", "Brave"],
+  ["chromium", "Chromium"],
+  ["google chrome", "Chrome"],
+  ["microsoft edge", "Edge"],
+  ["opera", "Opera"],
+  ["opera gx", "Opera"],
+  ["samsung internet", "Samsung Internet"],
+  ["vivaldi", "Vivaldi"],
+  ["yandex", "Yandex"],
+]);
+
+// Every Chromium brand list contains "Chromium", and Chrome adds "Google
+// Chrome" beside it; both only win when no fork identifies itself.
+const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
+
+const HINT_PLATFORMS = new Map<string, string>([
+  ["android", "Android"],
+  ["chrome os", "ChromeOS"],
+  ["chromium os", "ChromeOS"],
+  ["ios", "iOS"],
+  ["linux", "Linux"],
+  ["macos", "macOS"],
+  ["windows", "Windows"],
+]);
+
+// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
+// browser on iOS carries `Safari/`, so the specific tokens must be tested
+// first. Brave has no token at all and is only detectable through the hints.
+const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
+  [/Edg(?:A|iOS)?\//, "Edge"],
+  [/(?:Firefox|FxiOS)\//, "Firefox"],
+  [/OPR\//, "Opera"],
+  [/SamsungBrowser\//, "Samsung Internet"],
+  [/Vivaldi\//, "Vivaldi"],
+  [/YaBrowser\//, "Yandex"],
+  [/(?:Chrome|CriOS)\//, "Chrome"],
+  [/Safari\//, "Safari"],
+];
+
+// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
+const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
+  [/Android/, "Android"],
+  [/iPhone|iPad|iPod/, "iOS"],
+  [/CrOS/, "ChromeOS"],
+  [/Windows/, "Windows"],
+  [/Mac OS X/, "macOS"],
+  [/Linux/, "Linux"],
+];
+
+// Ordered: tablet user agents that carry `Android` must not be read as mobile.
+const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
+  [/iPad|Tablet/, "tablet"],
+  [/Mobile|Android|iPhone/, "mobile"],
+];
+
+export function classifyClient(
+  userAgent: string,
+  headers: Headers,
+): ClientClassification {
+  const agent = classifyUserAgent(userAgent);
+  const brand = brandFromClientHints(headers.get("sec-ch-ua"));
+  const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
+  const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
+  return {
+    browser: brand ?? agent.browser,
+    os: platform ?? agent.os,
+    device: deviceFromClientHints(mobile, platform) ?? agent.device,
+  };
+}
+
+export function classifyUserAgent(userAgent: string): ClientClassification {
+  return {
+    browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
+    os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
+    device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
+  };
+}
+
+function matchAgent(
+  table: readonly (readonly [RegExp, string])[],
+  userAgent: string,
+): string | undefined {
+  for (const [pattern, value] of table) {
+    if (pattern.test(userAgent)) return value;
+  }
+  return undefined;
+}
+
+function brandFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  let generic: string | undefined;
+  let seen = 0;
+  for (const match of header.matchAll(BRAND_PATTERN)) {
+    if (seen >= MAX_HINT_BRANDS) break;
+    seen += 1;
+    const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
+    if (!brand) continue;
+    if (!GENERIC_BRANDS.has(brand)) return brand;
+    if (brand === "Chrome" || generic === undefined) generic = brand;
+  }
+  return generic;
+}
+
+function platformFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  const match = PLATFORM_PATTERN.exec(header);
+  if (!match) return undefined;
+  return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
+}
+
+function mobileFromClientHints(header: string | null): boolean | undefined {
+  if (header === null) return undefined;
+  const value = header.trim();
+  if (value === "?1") return true;
+  if (value === "?0") return false;
+  return undefined;
+}
+
+function deviceFromClientHints(
+  mobile: boolean | undefined,
+  platform: string | undefined,
+): string | undefined {
+  if (mobile === undefined) return undefined;
+  if (mobile) return "mobile";
+  // Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
+  // `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
+  // `Accept-CH` opt-in and a permissions-policy delegation from every tracked
+  // site; "Android but not mobile" is the only tablet signal available here.
+  if (platform === "Android") return "tablet";
+  return "desktop";
+}
+
+function unquote(value: string): string {
+  return value.replace(/\\(.)/g, "$1");
+}
+
+// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
+// headless browsers. A URL inside the User-Agent is the crawler convention
+// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
+// alone because some runtimes do not expose it at all, and a scripted browser
+// that spoofs a stock User-Agent is indistinguishable from a person here.
+const BOT_PATTERN =
+  /bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
+
+export function isKnownBot(userAgent: string): boolean {
+  return BOT_PATTERN.test(userAgent);
+}

package/src/component/validators.ts

@@ -47,6 +47,7 @@ export const ingestContextValidator = v.object({
   browser: v.optional(v.string()),
   os: v.optional(v.string()),
   device: v.optional(v.string()),
+  visitorKey: v.optional(v.string()),
 });
 
 export const batchedEventValidator = trackerEventValidator.extend(
@@ -72,6 +73,7 @@ export const sessionFieldsValidator = v.object({
   siteId: v.id("sites"),
   sessionId: v.string(),
   visitorId: v.string(),
+  visitorKey: v.optional(v.string()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -128,6 +130,7 @@ export const eventFieldsValidator = v.object({
 
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
+  v.literal("campaign"),
   v.literal("page"),
   v.literal("country"),
   v.literal("device"),

package/src/component/visitors.ts

@@ -0,0 +1,105 @@
+import { v } from "convex/values";
+import { MAX_VISITOR_ALIASES } from "./constants.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+import type { Id } from "./_generated/dataModel.js";
+import { mutation, type QueryCtx } from "./_generated/server.js";
+
+/**
+ * Folds an anonymous visitor id into the pseudonymous identity the host
+ * supplies after signup or sign-in. Links are one level deep: an alias can
+ * belong to one identity, an identity can never become an alias, and an
+ * alias can never own aliases, so journeys stay bounded and unambiguous.
+ */
+export const link = mutation({
+  args: {
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+  },
+  returns: v.object({ linked: v.boolean(), aliasCount: v.number() }),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    let visitorId: string;
+    let previousVisitorId: string;
+    try {
+      visitorId = sanitizeOpaqueId(args.visitorId, "visitorId");
+      previousVisitorId = sanitizeOpaqueId(
+        args.previousVisitorId,
+        "previousVisitorId",
+      );
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid visitor id",
+      );
+    }
+    if (visitorId === previousVisitorId) {
+      fail("INVALID_ARGUMENT", "previousVisitorId must differ from visitorId");
+    }
+    if (await findAlias(ctx, args.siteId, visitorId)) {
+      fail("CONFLICT", "visitorId is already linked to another visitor");
+    }
+    const previousOwnsAliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", previousVisitorId),
+      )
+      .first();
+    if (previousOwnsAliases) {
+      fail("CONFLICT", "previousVisitorId already owns linked visitors");
+    }
+    const aliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", visitorId),
+      )
+      .take(MAX_VISITOR_ALIASES);
+    const existing = await findAlias(ctx, args.siteId, previousVisitorId);
+    if (existing) {
+      if (existing.visitorId !== visitorId) {
+        fail("CONFLICT", "previousVisitorId is already linked to a different visitor");
+      }
+      return { linked: false, aliasCount: aliases.length };
+    }
+    if (aliases.length >= MAX_VISITOR_ALIASES) {
+      fail("LIMIT_EXCEEDED", `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`, {
+        limit: MAX_VISITOR_ALIASES,
+      });
+    }
+    await ctx.db.insert("visitorAliases", {
+      siteId: args.siteId,
+      visitorId,
+      previousVisitorId,
+      linkedAt: Date.now(),
+    });
+    return { linked: true, aliasCount: aliases.length + 1 };
+  },
+});
+
+/** The identity a visitor id resolves to, followed by every alias it owns. */
+export async function resolveVisitorIdentities(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+): Promise<string[]> {
+  const alias = await findAlias(ctx, siteId, visitorId);
+  const identity = alias?.visitorId ?? visitorId;
+  const aliases = await ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_visitorId", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", identity),
+    )
+    .take(MAX_VISITOR_ALIASES);
+  return [identity, ...aliases.map((row) => row.previousVisitorId)];
+}
+
+function findAlias(ctx: QueryCtx, siteId: Id<"sites">, previousVisitorId: string) {
+  return ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_previousVisitorId", (range) =>
+      range.eq("siteId", siteId).eq("previousVisitorId", previousVisitorId),
+    )
+    .unique();
+}