@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.10

package/src/component/useragent.ts

@@ -0,0 +1,171 @@
+export type ClientClassification = {
+  browser: string;
+  os: string;
+  device: string;
+};
+
+// The tracker never reports the client: everything here is derived from headers
+// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
+// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
+// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
+// Firefox and Safari implement no client hints, so the User-Agent stays the
+// fallback for them and for insecure origins, where no hint is sent at all.
+const MAX_HINT_LENGTH = 512;
+const MAX_HINT_BRANDS = 16;
+
+const BRAND_PATTERN =
+  /"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
+const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
+
+// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
+// allowlisted: anything unrecognized (including the GREASE entries Chromium
+// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
+// entering the browser dimension verbatim.
+const HINT_BRANDS = new Map<string, string>([
+  ["brave", "Brave"],
+  ["chromium", "Chromium"],
+  ["google chrome", "Chrome"],
+  ["microsoft edge", "Edge"],
+  ["opera", "Opera"],
+  ["opera gx", "Opera"],
+  ["samsung internet", "Samsung Internet"],
+  ["vivaldi", "Vivaldi"],
+  ["yandex", "Yandex"],
+]);
+
+// Every Chromium brand list contains "Chromium", and Chrome adds "Google
+// Chrome" beside it; both only win when no fork identifies itself.
+const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
+
+const HINT_PLATFORMS = new Map<string, string>([
+  ["android", "Android"],
+  ["chrome os", "ChromeOS"],
+  ["chromium os", "ChromeOS"],
+  ["ios", "iOS"],
+  ["linux", "Linux"],
+  ["macos", "macOS"],
+  ["windows", "Windows"],
+]);
+
+// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
+// browser on iOS carries `Safari/`, so the specific tokens must be tested
+// first. Brave has no token at all and is only detectable through the hints.
+const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
+  [/Edg(?:A|iOS)?\//, "Edge"],
+  [/(?:Firefox|FxiOS)\//, "Firefox"],
+  [/OPR\//, "Opera"],
+  [/SamsungBrowser\//, "Samsung Internet"],
+  [/Vivaldi\//, "Vivaldi"],
+  [/YaBrowser\//, "Yandex"],
+  [/(?:Chrome|CriOS)\//, "Chrome"],
+  [/Safari\//, "Safari"],
+];
+
+// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
+const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
+  [/Android/, "Android"],
+  [/iPhone|iPad|iPod/, "iOS"],
+  [/CrOS/, "ChromeOS"],
+  [/Windows/, "Windows"],
+  [/Mac OS X/, "macOS"],
+  [/Linux/, "Linux"],
+];
+
+// Ordered: tablet user agents that carry `Android` must not be read as mobile.
+const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
+  [/iPad|Tablet/, "tablet"],
+  [/Mobile|Android|iPhone/, "mobile"],
+];
+
+export function classifyClient(
+  userAgent: string,
+  headers: Headers,
+): ClientClassification {
+  const agent = classifyUserAgent(userAgent);
+  const brand = brandFromClientHints(headers.get("sec-ch-ua"));
+  const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
+  const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
+  return {
+    browser: brand ?? agent.browser,
+    os: platform ?? agent.os,
+    device: deviceFromClientHints(mobile, platform) ?? agent.device,
+  };
+}
+
+export function classifyUserAgent(userAgent: string): ClientClassification {
+  return {
+    browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
+    os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
+    device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
+  };
+}
+
+function matchAgent(
+  table: readonly (readonly [RegExp, string])[],
+  userAgent: string,
+): string | undefined {
+  for (const [pattern, value] of table) {
+    if (pattern.test(userAgent)) return value;
+  }
+  return undefined;
+}
+
+function brandFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  let generic: string | undefined;
+  let seen = 0;
+  for (const match of header.matchAll(BRAND_PATTERN)) {
+    if (seen >= MAX_HINT_BRANDS) break;
+    seen += 1;
+    const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
+    if (!brand) continue;
+    if (!GENERIC_BRANDS.has(brand)) return brand;
+    if (brand === "Chrome" || generic === undefined) generic = brand;
+  }
+  return generic;
+}
+
+function platformFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  const match = PLATFORM_PATTERN.exec(header);
+  if (!match) return undefined;
+  return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
+}
+
+function mobileFromClientHints(header: string | null): boolean | undefined {
+  if (header === null) return undefined;
+  const value = header.trim();
+  if (value === "?1") return true;
+  if (value === "?0") return false;
+  return undefined;
+}
+
+function deviceFromClientHints(
+  mobile: boolean | undefined,
+  platform: string | undefined,
+): string | undefined {
+  if (mobile === undefined) return undefined;
+  if (mobile) return "mobile";
+  // Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
+  // `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
+  // `Accept-CH` opt-in and a permissions-policy delegation from every tracked
+  // site; "Android but not mobile" is the only tablet signal available here.
+  if (platform === "Android") return "tablet";
+  return "desktop";
+}
+
+function unquote(value: string): string {
+  return value.replace(/\\(.)/g, "$1");
+}
+
+// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
+// headless browsers. A URL inside the User-Agent is the crawler convention
+// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
+// alone because some runtimes do not expose it at all, and a scripted browser
+// that spoofs a stock User-Agent is indistinguishable from a person here.
+const BOT_PATTERN =
+  /bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
+
+export function isKnownBot(userAgent: string): boolean {
+  return BOT_PATTERN.test(userAgent);
+}

package/src/component/validators.ts

@@ -6,7 +6,17 @@ export const eventTypeValidator = v.union(
   v.literal("custom"),
   v.literal("conversion"),
   v.literal("heartbeat"),
+  v.literal("leave"),
   v.literal("outbound"),
+  v.literal("vital"),
+);
+
+export const vitalMetricValidator = v.union(
+  v.literal("LCP"),
+  v.literal("CLS"),
+  v.literal("INP"),
+  v.literal("FCP"),
+  v.literal("TTFB"),
 );
 
 export const propertyValueValidator = v.union(
@@ -37,6 +47,9 @@ export const trackerEventValidator = v.object({
   revenueCents: v.optional(v.number()),
   currency: v.optional(v.string()),
   affiliateSlug: v.optional(v.string()),
+  // Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
+  // events carry it; sanitization strips it from every other type.
+  value: v.optional(v.number()),
 });
 
 export const ingestContextValidator = v.object({
@@ -47,6 +60,12 @@ export const ingestContextValidator = v.object({
   browser: v.optional(v.string()),
   os: v.optional(v.string()),
   device: v.optional(v.string()),
+  visitorKey: v.optional(v.string()),
+});
+
+export const batchedEventValidator = trackerEventValidator.extend({
+  ...ingestContextValidator.fields,
+  aggregateCountry: v.optional(v.string()),
 });
 
 export const siteFieldsValidator = v.object({
@@ -65,6 +84,7 @@ export const sessionFieldsValidator = v.object({
   siteId: v.id("sites"),
   sessionId: v.string(),
   visitorId: v.string(),
+  visitorKey: v.optional(v.string()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -119,20 +139,9 @@ export const eventFieldsValidator = v.object({
   device: v.string(),
 });
 
-export const storedEventFieldsValidator = eventFieldsValidator
-  .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
-  .extend({
-    country: v.optional(v.string()),
-    city: v.optional(v.string()),
-    latitude: v.optional(v.number()),
-    longitude: v.optional(v.number()),
-    browser: v.optional(v.string()),
-    os: v.optional(v.string()),
-    device: v.optional(v.string()),
-  });
-
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
+  v.literal("campaign"),
   v.literal("page"),
   v.literal("country"),
   v.literal("device"),
@@ -149,6 +158,12 @@ export const dimensionSlotValidator = v.object({
   revenueCents: v.number(),
 });
 
+export const visitorSketchValidator = v.object({
+  version: v.literal(1),
+  precision: v.number(),
+  registers: v.bytes(),
+});
+
 export const aggregateFieldsValidator = v.object({
   siteId: v.id("sites"),
   granularity: v.union(v.literal("hour"), v.literal("day")),
@@ -161,6 +176,8 @@ export const aggregateFieldsValidator = v.object({
   outboundClicks: v.number(),
   sessions: v.number(),
   visitors: v.number(),
+  visitorSketch: v.optional(visitorSketchValidator),
+  visitorSketchComplete: v.optional(v.boolean()),
   conversions: v.number(),
   revenueCents: v.number(),
   dimensions: v.array(dimensionSlotValidator),
@@ -230,7 +247,6 @@ export const trustedConversionValidator = v.object({
   properties: v.optional(eventPropertiesValidator),
 });
 
-
 export type TrackerEvent = Infer<typeof trackerEventValidator>;
 export type IngestContext = Infer<typeof ingestContextValidator>;
 export type EventProperties = Infer<typeof eventPropertiesValidator>;

package/src/component/visitors.ts

@@ -0,0 +1,116 @@
+import { v } from "convex/values";
+import { MAX_VISITOR_ALIASES } from "./constants.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+import type { Id } from "./_generated/dataModel.js";
+import { mutation, type QueryCtx } from "./_generated/server.js";
+
+/**
+ * Folds an anonymous visitor id into the pseudonymous identity the host
+ * supplies after signup or sign-in. Links are one level deep: an alias can
+ * belong to one identity, an identity can never become an alias, and an
+ * alias can never own aliases, so journeys stay bounded and unambiguous.
+ */
+export const link = mutation({
+  args: {
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+  },
+  returns: v.object({ linked: v.boolean(), aliasCount: v.number() }),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    let visitorId: string;
+    let previousVisitorId: string;
+    try {
+      visitorId = sanitizeOpaqueId(args.visitorId, "visitorId");
+      previousVisitorId = sanitizeOpaqueId(
+        args.previousVisitorId,
+        "previousVisitorId",
+      );
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid visitor id",
+      );
+    }
+    if (visitorId === previousVisitorId) {
+      fail("INVALID_ARGUMENT", "previousVisitorId must differ from visitorId");
+    }
+    if (await findAlias(ctx, args.siteId, visitorId)) {
+      fail("CONFLICT", "visitorId is already linked to another visitor");
+    }
+    const previousOwnsAliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", previousVisitorId),
+      )
+      .first();
+    if (previousOwnsAliases) {
+      fail("CONFLICT", "previousVisitorId already owns linked visitors");
+    }
+    const aliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", visitorId),
+      )
+      .take(MAX_VISITOR_ALIASES);
+    const existing = await findAlias(ctx, args.siteId, previousVisitorId);
+    if (existing) {
+      if (existing.visitorId !== visitorId) {
+        fail(
+          "CONFLICT",
+          "previousVisitorId is already linked to a different visitor",
+        );
+      }
+      return { linked: false, aliasCount: aliases.length };
+    }
+    if (aliases.length >= MAX_VISITOR_ALIASES) {
+      fail(
+        "LIMIT_EXCEEDED",
+        `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`,
+        {
+          limit: MAX_VISITOR_ALIASES,
+        },
+      );
+    }
+    await ctx.db.insert("visitorAliases", {
+      siteId: args.siteId,
+      visitorId,
+      previousVisitorId,
+      linkedAt: Date.now(),
+    });
+    return { linked: true, aliasCount: aliases.length + 1 };
+  },
+});
+
+/** The identity a visitor id resolves to, followed by every alias it owns. */
+export async function resolveVisitorIdentities(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+): Promise<string[]> {
+  const alias = await findAlias(ctx, siteId, visitorId);
+  const identity = alias?.visitorId ?? visitorId;
+  const aliases = await ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_visitorId", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", identity),
+    )
+    .take(MAX_VISITOR_ALIASES);
+  return [identity, ...aliases.map((row) => row.previousVisitorId)];
+}
+
+function findAlias(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  previousVisitorId: string,
+) {
+  return ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_previousVisitorId", (range) =>
+      range.eq("siteId", siteId).eq("previousVisitorId", previousVisitorId),
+    )
+    .unique();
+}

package/src/component/vitals.ts

@@ -0,0 +1,146 @@
+import { VITAL_HISTOGRAM_EDGES, MAX_VITAL_VALUE } from "./constants.js";
+
+/**
+ * Field-measured Web Vitals. Values are integers: milliseconds for the time
+ * metrics, and CLS scaled by 1000 so one shared histogram covers every metric.
+ */
+export const VITAL_METRICS = ["LCP", "CLS", "INP", "FCP", "TTFB"] as const;
+
+export type VitalMetric = (typeof VITAL_METRICS)[number];
+
+/**
+ * The Google-published rating thresholds. Every threshold is also a histogram
+ * edge, so good/needs-improvement/poor counts are exact, never interpolated.
+ */
+export const VITAL_THRESHOLDS: Record<
+  VitalMetric,
+  { good: number; poor: number }
+> = {
+  LCP: { good: 2_500, poor: 4_000 },
+  CLS: { good: 100, poor: 250 },
+  INP: { good: 200, poor: 500 },
+  FCP: { good: 1_800, poor: 3_000 },
+  TTFB: { good: 800, poor: 1_800 },
+};
+
+/** Sentinel row keys; sanitized paths always start with "/" so neither collides. */
+export const VITAL_ALL = "(all)";
+export const VITAL_OTHER_PAGES = "(other)";
+
+/** Bounded device classes; anything else folds into "unknown". */
+export const VITAL_DEVICES = ["desktop", "mobile", "tablet"] as const;
+
+export function vitalDevice(device: string | undefined): string {
+  return device && (VITAL_DEVICES as readonly string[]).includes(device)
+    ? device
+    : "unknown";
+}
+
+export const VITAL_HISTOGRAM_BUCKETS = VITAL_HISTOGRAM_EDGES.length + 1;
+
+export function isVitalMetric(value: string): value is VitalMetric {
+  return (VITAL_METRICS as readonly string[]).includes(value);
+}
+
+export function isValidVitalValue(value: number): boolean {
+  return Number.isFinite(value) && value >= 0 && value <= MAX_VITAL_VALUE;
+}
+
+export function emptyVitalHistogram(): number[] {
+  return Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
+}
+
+/** Bucket i holds values in (edge[i-1], edge[i]]; the last bucket is open. */
+export function vitalHistogramBucket(value: number): number {
+  for (let index = 0; index < VITAL_HISTOGRAM_EDGES.length; index += 1) {
+    if (value <= VITAL_HISTOGRAM_EDGES[index]) return index;
+  }
+  return VITAL_HISTOGRAM_EDGES.length;
+}
+
+export function addToVitalHistogram(
+  histogram: number[],
+  value: number,
+): number[] {
+  const next = normalizedVitalHistogram(histogram);
+  next[vitalHistogramBucket(value)] += 1;
+  return next;
+}
+
+export function mergeVitalHistograms(
+  left: number[],
+  right: number[],
+): number[] {
+  const result = normalizedVitalHistogram(left);
+  const addition = normalizedVitalHistogram(right);
+  for (let index = 0; index < result.length; index += 1) {
+    result[index] += addition[index];
+  }
+  return result;
+}
+
+/**
+ * Percentile estimated by linear interpolation inside the winning bucket. The
+ * top bucket is open-ended, so estimates saturate at the highest edge; a
+ * reported p75 equal to that edge means "at least this much".
+ */
+export function vitalHistogramPercentile(
+  histogram: number[],
+  fraction: number,
+): number {
+  const buckets = normalizedVitalHistogram(histogram);
+  const total = buckets.reduce((sum, count) => sum + count, 0);
+  if (total === 0) return 0;
+  const target = Math.ceil(total * fraction);
+  let cumulative = 0;
+  for (let index = 0; index < buckets.length; index += 1) {
+    const count = buckets[index];
+    if (count === 0) continue;
+    if (cumulative + count >= target) {
+      const lower = index === 0 ? 0 : VITAL_HISTOGRAM_EDGES[index - 1];
+      const upper =
+        index < VITAL_HISTOGRAM_EDGES.length
+          ? VITAL_HISTOGRAM_EDGES[index]
+          : VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
+      const position = (target - cumulative) / count;
+      return Math.round(Math.min(upper, lower + position * (upper - lower)));
+    }
+    cumulative += count;
+  }
+  return VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
+}
+
+/** Exact rating counts; every threshold is a bucket edge by construction. */
+export function vitalRatingCounts(
+  histogram: number[],
+  metric: VitalMetric,
+): { good: number; needsImprovement: number; poor: number } {
+  const buckets = normalizedVitalHistogram(histogram);
+  const thresholds = VITAL_THRESHOLDS[metric];
+  let good = 0;
+  let needsImprovement = 0;
+  let poor = 0;
+  for (let index = 0; index < buckets.length; index += 1) {
+    const upper =
+      index < VITAL_HISTOGRAM_EDGES.length
+        ? VITAL_HISTOGRAM_EDGES[index]
+        : Number.POSITIVE_INFINITY;
+    if (upper <= thresholds.good) good += buckets[index];
+    else if (upper <= thresholds.poor) needsImprovement += buckets[index];
+    else poor += buckets[index];
+  }
+  return { good, needsImprovement, poor };
+}
+
+function normalizedVitalHistogram(histogram: number[]): number[] {
+  const result = Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
+  for (
+    let index = 0;
+    index < Math.min(histogram.length, result.length);
+    index += 1
+  ) {
+    const count = histogram[index];
+    result[index] = Number.isSafeInteger(count) && count > 0 ? count : 0;
+  }
+  return result;
+}