@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/src/component/sanitize.ts

@@ -8,6 +8,7 @@ import type {
   IngestContext,
   TrackerEvent,
 } from "./validators.js";
+import { isValidVitalValue, isVitalMetric } from "./vitals.js";
 
 const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -62,9 +63,10 @@ export function sanitizePath(value: string): string {
   const input = requireString(value, "path", 2048);
   let path: string;
   try {
-    path = input.startsWith("http://") || input.startsWith("https://")
-      ? new URL(input).pathname
-      : input.split(/[?#]/, 1)[0];
+    path =
+      input.startsWith("http://") || input.startsWith("https://")
+        ? new URL(input).pathname
+        : input.split(/[?#]/, 1)[0];
   } catch {
     throw new Error("path is invalid");
   }
@@ -112,14 +114,19 @@ export function sanitizeProperties(
 }
 
 export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
-  if (!Number.isFinite(event.timestamp)) throw new Error("timestamp is invalid");
+  if (!Number.isFinite(event.timestamp))
+    throw new Error("timestamp is invalid");
   if (
     event.timestamp < now - MAX_EVENT_AGE_MS ||
     event.timestamp > now + MAX_FUTURE_SKEW_MS
   ) {
     throw new Error("timestamp is outside the accepted window");
   }
-  if (!Number.isSafeInteger(event.sequence) || event.sequence < 0 || event.sequence > 1_000_000_000) {
+  if (
+    !Number.isSafeInteger(event.sequence) ||
+    event.sequence < 0 ||
+    event.sequence > 1_000_000_000
+  ) {
     throw new Error("sequence is out of range");
   }
   if (event.revenueCents !== undefined && !isValidMoney(event.revenueCents)) {
@@ -130,9 +137,20 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     throw new Error("name is required for custom events");
   }
   if (event.type === "conversion" && !name) name = "conversion";
-  const currency = event.currency === undefined
-    ? event.type === "conversion" ? "USD" : undefined
-    : sanitizeCurrency(event.currency);
+  if (event.type === "vital") {
+    if (!name || !isVitalMetric(name)) {
+      throw new Error("vital name must be a Web Vitals metric");
+    }
+    if (event.value === undefined || !isValidVitalValue(event.value)) {
+      throw new Error("vital value is out of range");
+    }
+  }
+  const currency =
+    event.currency === undefined
+      ? event.type === "conversion"
+        ? "USD"
+        : undefined
+      : sanitizeCurrency(event.currency);
   return {
     eventId: sanitizeOpaqueId(event.eventId, "eventId"),
     sessionId: sanitizeOpaqueId(event.sessionId, "sessionId"),
@@ -140,19 +158,26 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     type: event.type,
     name,
     path: sanitizePath(event.path),
-    referrer: event.referrer === undefined
-      ? undefined
-      : sanitizePublicUrl(event.referrer),
+    referrer:
+      event.referrer === undefined
+        ? undefined
+        : sanitizePublicUrl(event.referrer),
     timestamp: Math.trunc(event.timestamp),
     sequence: event.sequence,
-    target: event.target === undefined
-      ? undefined
-      : cleanString(event.target, 160) || undefined,
+    target:
+      event.target === undefined
+        ? undefined
+        : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
     href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
     currency,
     affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
+    // Only measurements keep a value; a stray number on any other type is noise.
+    value:
+      event.type === "vital" && event.value !== undefined
+        ? Math.round(event.value)
+        : undefined,
   };
 }
 
@@ -167,7 +192,9 @@ function optionalAffiliateSlug(value: string | undefined): string | undefined {
 
 export function normalizeDomains(domains: string[]): string[] {
   if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
-    throw new Error(`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`);
+    throw new Error(
+      `domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`,
+    );
   }
   const normalized = domains.map(normalizeDomainPattern);
   return [...new Set(normalized)].sort();
@@ -185,7 +212,11 @@ export function normalizeDomainPattern(value: string): string {
       throw new Error("domains cannot include paths or query strings");
     }
     hostname = url.hostname;
-  } else if (hostInput.includes("/") || hostInput.includes("?") || hostInput.includes("#")) {
+  } else if (
+    hostInput.includes("/") ||
+    hostInput.includes("?") ||
+    hostInput.includes("#")
+  ) {
     throw new Error("domain is invalid");
   }
   hostname = hostname.replace(/\.$/, "");
@@ -214,7 +245,9 @@ export function originAllowed(origin: string, domains: string[]): boolean {
   });
 }
 
-export function validateTimezone(value: string | undefined): string | undefined {
+export function validateTimezone(
+  value: string | undefined,
+): string | undefined {
   if (value === undefined || value.trim() === "") return undefined;
   const timezone = cleanString(value, 80);
   try {
@@ -225,17 +258,33 @@ export function validateTimezone(value: string | undefined): string | undefined
   }
 }
 
-export function sanitizeContext(context: IngestContext | undefined): IngestContext {
+export function sanitizeContext(
+  context: IngestContext | undefined,
+): IngestContext {
   const latitude = finiteCoordinate(context?.latitude, -90, 90);
   const longitude = finiteCoordinate(context?.longitude, -180, 180);
   return {
-    country: context?.country ? cleanString(context.country.toUpperCase(), 2) || undefined : undefined,
-    city: context?.city ? cleanString(context.city, 80) || undefined : undefined,
-    latitude: latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
-    longitude: longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
-    browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
+    country: context?.country
+      ? cleanString(context.country.toUpperCase(), 2) || undefined
+      : undefined,
+    city: context?.city
+      ? cleanString(context.city, 80) || undefined
+      : undefined,
+    latitude:
+      latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
+    longitude:
+      longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
+    browser: context?.browser
+      ? cleanString(context.browser, 32) || undefined
+      : undefined,
     os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
-    device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
+    device: context?.device
+      ? cleanString(context.device, 24) || undefined
+      : undefined,
+    visitorKey:
+      context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
+        ? cleanString(context.visitorKey, 128) || undefined
+        : undefined,
   };
 }
 
@@ -254,8 +303,8 @@ export function measureBatchBytes(
   events: TrackerEvent[],
 ): { batchBytes: number; eventByteCounts: number[] } {
   const encoder = new TextEncoder();
-  const eventByteCounts = events.map((event) =>
-    encoder.encode(JSON.stringify(event)).byteLength
+  const eventByteCounts = events.map(
+    (event) => encoder.encode(JSON.stringify(event)).byteLength,
   );
   // The empty envelope already includes both array brackets; only commas remain.
   const batchBytes =
@@ -291,6 +340,7 @@ function finiteCoordinate(
   maximum: number,
 ): number | undefined {
   if (value === undefined) return undefined;
-  if (!Number.isFinite(value) || value < minimum || value > maximum) return undefined;
+  if (!Number.isFinite(value) || value < minimum || value > maximum)
+    return undefined;
   return value;
 }

package/src/component/schema.ts

@@ -9,6 +9,7 @@ import {
   sessionFieldsValidator,
   siteFieldsValidator,
   visitorSketchValidator,
+  vitalMetricValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -131,11 +132,13 @@ export default defineSchema({
     bucketStart: v.number(),
     shard: v.number(),
     generation: v.literal(1),
-    steps: v.array(v.object({
-      step: v.number(),
-      entrants: v.number(),
-      completions: v.number(),
-    })),
+    steps: v.array(
+      v.object({
+        step: v.number(),
+        entrants: v.number(),
+        completions: v.number(),
+      }),
+    ),
   })
     .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
       "siteId",
@@ -150,6 +153,29 @@ export default defineSchema({
     ])
     .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
 
+  // Field Web Vitals. One row per site/day/shard and either a page (device
+  // "(all)") or a device class (page "(all)"); the row folds every metric's
+  // histogram together so a report reads five metrics for one document. Pages
+  // are bounded like dimension slots: at most VITAL_PAGE_SLOTS distinct pages
+  // per day, then "(other)".
+  vitalRollups: defineTable({
+    siteId: v.id("sites"),
+    bucketStart: v.number(),
+    device: v.string(),
+    page: v.string(),
+    shard: v.number(),
+    metrics: v.array(
+      v.object({
+        metric: vitalMetricValidator,
+        count: v.number(),
+        sum: v.number(),
+        histogram: v.array(v.number()),
+      }),
+    ),
+  })
+    .index("by_key", ["siteId", "bucketStart", "device", "page", "shard"])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
   affiliateDailyRollups: defineTable({
     siteId: v.id("sites"),
     affiliateId: v.id("affiliates"),
@@ -283,6 +309,21 @@ export default defineSchema({
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_slug", ["siteId", "slug"]),
 
+  visitorSecrets: defineTable({
+    key: v.literal("current"),
+    value: v.string(),
+    createdAt: v.number(),
+  }).index("by_key", ["key"]),
+
+  visitorAliases: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+    linkedAt: v.number(),
+  })
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
+
   visitorAttributions: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),

package/src/component/sites.ts

@@ -32,7 +32,8 @@ export const create = mutation({
   handler: async (ctx, args) => {
     const ownerId = cleanString(args.ownerId, 128);
     const name = cleanString(args.name, 120);
-    if (!ownerId || !name) fail("INVALID_SITE", "ownerId and name are required");
+    if (!ownerId || !name)
+      fail("INVALID_SITE", "ownerId and name are required");
 
     let domains: string[];
     let timezone: string | undefined;
@@ -46,7 +47,10 @@ export const create = mutation({
         : undefined;
       currency = sanitizeCurrency(args.currency ?? "USD");
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     const cookieless = args.cookieless ?? true;
 
@@ -65,7 +69,10 @@ export const create = mutation({
         (existing.currency ?? "USD") !== currency ||
         existing.cookieless !== cookieless
       ) {
-        fail("CONFLICT", "site name is already associated with different settings");
+        fail(
+          "CONFLICT",
+          "site name is already associated with different settings",
+        );
       }
       await ensureAnalyticsControl(ctx, existing._id, Date.now());
       return existing._id;
@@ -137,22 +144,26 @@ export const update = mutation({
       patch.name = name;
     }
     try {
-      if (args.domains !== undefined) patch.domains = normalizeDomains(args.domains);
+      if (args.domains !== undefined)
+        patch.domains = normalizeDomains(args.domains);
       if (args.timezone !== undefined) {
-        patch.timezone = args.timezone === null
-          ? undefined
-          : validateTimezone(args.timezone);
+        patch.timezone =
+          args.timezone === null ? undefined : validateTimezone(args.timezone);
       }
       if (args.networkId !== undefined) {
-        patch.networkId = args.networkId === null
-          ? undefined
-          : sanitizeOpaqueId(args.networkId, "networkId");
+        patch.networkId =
+          args.networkId === null
+            ? undefined
+            : sanitizeOpaqueId(args.networkId, "networkId");
       }
       if (args.currency !== undefined) {
         patch.currency = sanitizeCurrency(args.currency);
       }
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     if (
       patch.currency !== undefined &&

package/src/component/useragent.ts

@@ -0,0 +1,171 @@
+export type ClientClassification = {
+  browser: string;
+  os: string;
+  device: string;
+};
+
+// The tracker never reports the client: everything here is derived from headers
+// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
+// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
+// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
+// Firefox and Safari implement no client hints, so the User-Agent stays the
+// fallback for them and for insecure origins, where no hint is sent at all.
+const MAX_HINT_LENGTH = 512;
+const MAX_HINT_BRANDS = 16;
+
+const BRAND_PATTERN =
+  /"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
+const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
+
+// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
+// allowlisted: anything unrecognized (including the GREASE entries Chromium
+// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
+// entering the browser dimension verbatim.
+const HINT_BRANDS = new Map<string, string>([
+  ["brave", "Brave"],
+  ["chromium", "Chromium"],
+  ["google chrome", "Chrome"],
+  ["microsoft edge", "Edge"],
+  ["opera", "Opera"],
+  ["opera gx", "Opera"],
+  ["samsung internet", "Samsung Internet"],
+  ["vivaldi", "Vivaldi"],
+  ["yandex", "Yandex"],
+]);
+
+// Every Chromium brand list contains "Chromium", and Chrome adds "Google
+// Chrome" beside it; both only win when no fork identifies itself.
+const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
+
+const HINT_PLATFORMS = new Map<string, string>([
+  ["android", "Android"],
+  ["chrome os", "ChromeOS"],
+  ["chromium os", "ChromeOS"],
+  ["ios", "iOS"],
+  ["linux", "Linux"],
+  ["macos", "macOS"],
+  ["windows", "Windows"],
+]);
+
+// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
+// browser on iOS carries `Safari/`, so the specific tokens must be tested
+// first. Brave has no token at all and is only detectable through the hints.
+const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
+  [/Edg(?:A|iOS)?\//, "Edge"],
+  [/(?:Firefox|FxiOS)\//, "Firefox"],
+  [/OPR\//, "Opera"],
+  [/SamsungBrowser\//, "Samsung Internet"],
+  [/Vivaldi\//, "Vivaldi"],
+  [/YaBrowser\//, "Yandex"],
+  [/(?:Chrome|CriOS)\//, "Chrome"],
+  [/Safari\//, "Safari"],
+];
+
+// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
+const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
+  [/Android/, "Android"],
+  [/iPhone|iPad|iPod/, "iOS"],
+  [/CrOS/, "ChromeOS"],
+  [/Windows/, "Windows"],
+  [/Mac OS X/, "macOS"],
+  [/Linux/, "Linux"],
+];
+
+// Ordered: tablet user agents that carry `Android` must not be read as mobile.
+const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
+  [/iPad|Tablet/, "tablet"],
+  [/Mobile|Android|iPhone/, "mobile"],
+];
+
+export function classifyClient(
+  userAgent: string,
+  headers: Headers,
+): ClientClassification {
+  const agent = classifyUserAgent(userAgent);
+  const brand = brandFromClientHints(headers.get("sec-ch-ua"));
+  const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
+  const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
+  return {
+    browser: brand ?? agent.browser,
+    os: platform ?? agent.os,
+    device: deviceFromClientHints(mobile, platform) ?? agent.device,
+  };
+}
+
+export function classifyUserAgent(userAgent: string): ClientClassification {
+  return {
+    browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
+    os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
+    device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
+  };
+}
+
+function matchAgent(
+  table: readonly (readonly [RegExp, string])[],
+  userAgent: string,
+): string | undefined {
+  for (const [pattern, value] of table) {
+    if (pattern.test(userAgent)) return value;
+  }
+  return undefined;
+}
+
+function brandFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  let generic: string | undefined;
+  let seen = 0;
+  for (const match of header.matchAll(BRAND_PATTERN)) {
+    if (seen >= MAX_HINT_BRANDS) break;
+    seen += 1;
+    const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
+    if (!brand) continue;
+    if (!GENERIC_BRANDS.has(brand)) return brand;
+    if (brand === "Chrome" || generic === undefined) generic = brand;
+  }
+  return generic;
+}
+
+function platformFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  const match = PLATFORM_PATTERN.exec(header);
+  if (!match) return undefined;
+  return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
+}
+
+function mobileFromClientHints(header: string | null): boolean | undefined {
+  if (header === null) return undefined;
+  const value = header.trim();
+  if (value === "?1") return true;
+  if (value === "?0") return false;
+  return undefined;
+}
+
+function deviceFromClientHints(
+  mobile: boolean | undefined,
+  platform: string | undefined,
+): string | undefined {
+  if (mobile === undefined) return undefined;
+  if (mobile) return "mobile";
+  // Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
+  // `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
+  // `Accept-CH` opt-in and a permissions-policy delegation from every tracked
+  // site; "Android but not mobile" is the only tablet signal available here.
+  if (platform === "Android") return "tablet";
+  return "desktop";
+}
+
+function unquote(value: string): string {
+  return value.replace(/\\(.)/g, "$1");
+}
+
+// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
+// headless browsers. A URL inside the User-Agent is the crawler convention
+// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
+// alone because some runtimes do not expose it at all, and a scripted browser
+// that spoofs a stock User-Agent is indistinguishable from a person here.
+const BOT_PATTERN =
+  /bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
+
+export function isKnownBot(userAgent: string): boolean {
+  return BOT_PATTERN.test(userAgent);
+}

package/src/component/validators.ts

@@ -7,6 +7,15 @@ export const eventTypeValidator = v.union(
   v.literal("conversion"),
   v.literal("heartbeat"),
   v.literal("outbound"),
+  v.literal("vital"),
+);
+
+export const vitalMetricValidator = v.union(
+  v.literal("LCP"),
+  v.literal("CLS"),
+  v.literal("INP"),
+  v.literal("FCP"),
+  v.literal("TTFB"),
 );
 
 export const propertyValueValidator = v.union(
@@ -37,6 +46,9 @@ export const trackerEventValidator = v.object({
   revenueCents: v.optional(v.number()),
   currency: v.optional(v.string()),
   affiliateSlug: v.optional(v.string()),
+  // Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
+  // events carry it; sanitization strips it from every other type.
+  value: v.optional(v.number()),
 });
 
 export const ingestContextValidator = v.object({
@@ -47,14 +59,13 @@ export const ingestContextValidator = v.object({
   browser: v.optional(v.string()),
   os: v.optional(v.string()),
   device: v.optional(v.string()),
+  visitorKey: v.optional(v.string()),
 });
 
-export const batchedEventValidator = trackerEventValidator.extend(
-  {
-    ...ingestContextValidator.fields,
-    aggregateCountry: v.optional(v.string()),
-  },
-);
+export const batchedEventValidator = trackerEventValidator.extend({
+  ...ingestContextValidator.fields,
+  aggregateCountry: v.optional(v.string()),
+});
 
 export const siteFieldsValidator = v.object({
   ownerId: v.string(),
@@ -72,6 +83,7 @@ export const sessionFieldsValidator = v.object({
   siteId: v.id("sites"),
   sessionId: v.string(),
   visitorId: v.string(),
+  visitorKey: v.optional(v.string()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -128,6 +140,7 @@ export const eventFieldsValidator = v.object({
 
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
+  v.literal("campaign"),
   v.literal("page"),
   v.literal("country"),
   v.literal("device"),
@@ -233,7 +246,6 @@ export const trustedConversionValidator = v.object({
   properties: v.optional(eventPropertiesValidator),
 });
 
-
 export type TrackerEvent = Infer<typeof trackerEventValidator>;
 export type IngestContext = Infer<typeof ingestContextValidator>;
 export type EventProperties = Infer<typeof eventPropertiesValidator>;