@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11

package/src/component/sites.ts

@@ -1,7 +1,10 @@
 import { v } from "convex/values";
 import { internalQuery, mutation, query } from "./_generated/server.js";
 import { MAX_SITES_PER_OWNER } from "./constants.js";
+import { ensureAnalyticsControl } from "./coverage.js";
 import { fail } from "./errors.js";
+import { resetLocalDayCoverage } from "./coverage.js";
+import { localDayTimezone } from "./localTime.js";
 import {
   cleanString,
   normalizeDomains,
@@ -31,7 +34,8 @@ export const create = mutation({
   handler: async (ctx, args) => {
     const ownerId = cleanString(args.ownerId, 128);
     const name = cleanString(args.name, 120);
-    if (!ownerId || !name) fail("INVALID_SITE", "ownerId and name are required");
+    if (!ownerId || !name)
+      fail("INVALID_SITE", "ownerId and name are required");
 
     let domains: string[];
     let timezone: string | undefined;
@@ -45,7 +49,10 @@ export const create = mutation({
         : undefined;
       currency = sanitizeCurrency(args.currency ?? "USD");
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     const cookieless = args.cookieless ?? true;
 
@@ -64,8 +71,12 @@ export const create = mutation({
         (existing.currency ?? "USD") !== currency ||
         existing.cookieless !== cookieless
       ) {
-        fail("CONFLICT", "site name is already associated with different settings");
+        fail(
+          "CONFLICT",
+          "site name is already associated with different settings",
+        );
       }
+      await ensureAnalyticsControl(ctx, existing._id, Date.now());
       return existing._id;
     }
 
@@ -80,7 +91,7 @@ export const create = mutation({
     }
 
     const now = Date.now();
-    return await ctx.db.insert("sites", {
+    const siteId = await ctx.db.insert("sites", {
       ownerId,
       name,
       domains,
@@ -91,6 +102,8 @@ export const create = mutation({
       createdAt: now,
       updatedAt: now,
     });
+    await ensureAnalyticsControl(ctx, siteId, now);
+    return siteId;
   },
 });
 
@@ -133,22 +146,26 @@ export const update = mutation({
       patch.name = name;
     }
     try {
-      if (args.domains !== undefined) patch.domains = normalizeDomains(args.domains);
+      if (args.domains !== undefined)
+        patch.domains = normalizeDomains(args.domains);
       if (args.timezone !== undefined) {
-        patch.timezone = args.timezone === null
-          ? undefined
-          : validateTimezone(args.timezone);
+        patch.timezone =
+          args.timezone === null ? undefined : validateTimezone(args.timezone);
       }
       if (args.networkId !== undefined) {
-        patch.networkId = args.networkId === null
-          ? undefined
-          : sanitizeOpaqueId(args.networkId, "networkId");
+        patch.networkId =
+          args.networkId === null
+            ? undefined
+            : sanitizeOpaqueId(args.networkId, "networkId");
       }
       if (args.currency !== undefined) {
         patch.currency = sanitizeCurrency(args.currency);
       }
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     if (
       patch.currency !== undefined &&
@@ -158,6 +175,9 @@ export const update = mutation({
     }
     if (args.cookieless !== undefined) patch.cookieless = args.cookieless;
     await ctx.db.patch("sites", args.siteId, patch);
+    if (args.timezone !== undefined && site.timezone !== patch.timezone) {
+      await resetLocalDayCoverage(ctx, args.siteId, patch.updatedAt);
+    }
     return null;
   },
 });
@@ -170,6 +190,33 @@ export const get = query({
   },
 });
 
+/**
+ * The calendar days a site's daily overview can answer in: the timezone its
+ * local-day buckets follow and the instant from which they are complete, or
+ * null when the site keeps none (no timezone, or a zone that never leaves
+ * UTC, whose days are the UTC days already kept).
+ */
+export const localDays = query({
+  args: { siteId: v.id("sites") },
+  returns: v.union(
+    v.object({ timezone: v.string(), since: v.number() }),
+    v.null(),
+  ),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    const timezone = localDayTimezone(site?.timezone);
+    if (!timezone) return null;
+    const control = await ctx.db
+      .query("analyticsCoverage")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    // A site that has not ingested since the upgrade has no stamp yet; its
+    // days begin with its first batch, which is later than now.
+    if (control?.localDaySince === undefined) return null;
+    return { timezone, since: control.localDaySince };
+  },
+});
+
 export const listByOwner = query({
   args: {
     ownerId: v.string(),

package/src/component/useragent.ts

@@ -0,0 +1,171 @@
+export type ClientClassification = {
+  browser: string;
+  os: string;
+  device: string;
+};
+
+// The tracker never reports the client: everything here is derived from headers
+// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
+// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
+// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
+// Firefox and Safari implement no client hints, so the User-Agent stays the
+// fallback for them and for insecure origins, where no hint is sent at all.
+const MAX_HINT_LENGTH = 512;
+const MAX_HINT_BRANDS = 16;
+
+const BRAND_PATTERN =
+  /"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
+const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
+
+// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
+// allowlisted: anything unrecognized (including the GREASE entries Chromium
+// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
+// entering the browser dimension verbatim.
+const HINT_BRANDS = new Map<string, string>([
+  ["brave", "Brave"],
+  ["chromium", "Chromium"],
+  ["google chrome", "Chrome"],
+  ["microsoft edge", "Edge"],
+  ["opera", "Opera"],
+  ["opera gx", "Opera"],
+  ["samsung internet", "Samsung Internet"],
+  ["vivaldi", "Vivaldi"],
+  ["yandex", "Yandex"],
+]);
+
+// Every Chromium brand list contains "Chromium", and Chrome adds "Google
+// Chrome" beside it; both only win when no fork identifies itself.
+const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
+
+const HINT_PLATFORMS = new Map<string, string>([
+  ["android", "Android"],
+  ["chrome os", "ChromeOS"],
+  ["chromium os", "ChromeOS"],
+  ["ios", "iOS"],
+  ["linux", "Linux"],
+  ["macos", "macOS"],
+  ["windows", "Windows"],
+]);
+
+// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
+// browser on iOS carries `Safari/`, so the specific tokens must be tested
+// first. Brave has no token at all and is only detectable through the hints.
+const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
+  [/Edg(?:A|iOS)?\//, "Edge"],
+  [/(?:Firefox|FxiOS)\//, "Firefox"],
+  [/OPR\//, "Opera"],
+  [/SamsungBrowser\//, "Samsung Internet"],
+  [/Vivaldi\//, "Vivaldi"],
+  [/YaBrowser\//, "Yandex"],
+  [/(?:Chrome|CriOS)\//, "Chrome"],
+  [/Safari\//, "Safari"],
+];
+
+// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
+const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
+  [/Android/, "Android"],
+  [/iPhone|iPad|iPod/, "iOS"],
+  [/CrOS/, "ChromeOS"],
+  [/Windows/, "Windows"],
+  [/Mac OS X/, "macOS"],
+  [/Linux/, "Linux"],
+];
+
+// Ordered: tablet user agents that carry `Android` must not be read as mobile.
+const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
+  [/iPad|Tablet/, "tablet"],
+  [/Mobile|Android|iPhone/, "mobile"],
+];
+
+export function classifyClient(
+  userAgent: string,
+  headers: Headers,
+): ClientClassification {
+  const agent = classifyUserAgent(userAgent);
+  const brand = brandFromClientHints(headers.get("sec-ch-ua"));
+  const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
+  const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
+  return {
+    browser: brand ?? agent.browser,
+    os: platform ?? agent.os,
+    device: deviceFromClientHints(mobile, platform) ?? agent.device,
+  };
+}
+
+export function classifyUserAgent(userAgent: string): ClientClassification {
+  return {
+    browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
+    os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
+    device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
+  };
+}
+
+function matchAgent(
+  table: readonly (readonly [RegExp, string])[],
+  userAgent: string,
+): string | undefined {
+  for (const [pattern, value] of table) {
+    if (pattern.test(userAgent)) return value;
+  }
+  return undefined;
+}
+
+function brandFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  let generic: string | undefined;
+  let seen = 0;
+  for (const match of header.matchAll(BRAND_PATTERN)) {
+    if (seen >= MAX_HINT_BRANDS) break;
+    seen += 1;
+    const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
+    if (!brand) continue;
+    if (!GENERIC_BRANDS.has(brand)) return brand;
+    if (brand === "Chrome" || generic === undefined) generic = brand;
+  }
+  return generic;
+}
+
+function platformFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  const match = PLATFORM_PATTERN.exec(header);
+  if (!match) return undefined;
+  return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
+}
+
+function mobileFromClientHints(header: string | null): boolean | undefined {
+  if (header === null) return undefined;
+  const value = header.trim();
+  if (value === "?1") return true;
+  if (value === "?0") return false;
+  return undefined;
+}
+
+function deviceFromClientHints(
+  mobile: boolean | undefined,
+  platform: string | undefined,
+): string | undefined {
+  if (mobile === undefined) return undefined;
+  if (mobile) return "mobile";
+  // Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
+  // `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
+  // `Accept-CH` opt-in and a permissions-policy delegation from every tracked
+  // site; "Android but not mobile" is the only tablet signal available here.
+  if (platform === "Android") return "tablet";
+  return "desktop";
+}
+
+function unquote(value: string): string {
+  return value.replace(/\\(.)/g, "$1");
+}
+
+// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
+// headless browsers. A URL inside the User-Agent is the crawler convention
+// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
+// alone because some runtimes do not expose it at all, and a scripted browser
+// that spoofs a stock User-Agent is indistinguishable from a person here.
+const BOT_PATTERN =
+  /bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
+
+export function isKnownBot(userAgent: string): boolean {
+  return BOT_PATTERN.test(userAgent);
+}

package/src/component/validators.ts

@@ -6,7 +6,17 @@ export const eventTypeValidator = v.union(
   v.literal("custom"),
   v.literal("conversion"),
   v.literal("heartbeat"),
+  v.literal("leave"),
   v.literal("outbound"),
+  v.literal("vital"),
+);
+
+export const vitalMetricValidator = v.union(
+  v.literal("LCP"),
+  v.literal("CLS"),
+  v.literal("INP"),
+  v.literal("FCP"),
+  v.literal("TTFB"),
 );
 
 export const propertyValueValidator = v.union(
@@ -37,6 +47,9 @@ export const trackerEventValidator = v.object({
   revenueCents: v.optional(v.number()),
   currency: v.optional(v.string()),
   affiliateSlug: v.optional(v.string()),
+  // Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
+  // events carry it; sanitization strips it from every other type.
+  value: v.optional(v.number()),
 });
 
 export const ingestContextValidator = v.object({
@@ -47,6 +60,12 @@ export const ingestContextValidator = v.object({
   browser: v.optional(v.string()),
   os: v.optional(v.string()),
   device: v.optional(v.string()),
+  visitorKey: v.optional(v.string()),
+});
+
+export const batchedEventValidator = trackerEventValidator.extend({
+  ...ingestContextValidator.fields,
+  aggregateCountry: v.optional(v.string()),
 });
 
 export const siteFieldsValidator = v.object({
@@ -65,6 +84,7 @@ export const sessionFieldsValidator = v.object({
   siteId: v.id("sites"),
   sessionId: v.string(),
   visitorId: v.string(),
+  visitorKey: v.optional(v.string()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -119,20 +139,9 @@ export const eventFieldsValidator = v.object({
   device: v.string(),
 });
 
-export const storedEventFieldsValidator = eventFieldsValidator
-  .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
-  .extend({
-    country: v.optional(v.string()),
-    city: v.optional(v.string()),
-    latitude: v.optional(v.number()),
-    longitude: v.optional(v.number()),
-    browser: v.optional(v.string()),
-    os: v.optional(v.string()),
-    device: v.optional(v.string()),
-  });
-
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
+  v.literal("campaign"),
   v.literal("page"),
   v.literal("country"),
   v.literal("device"),
@@ -147,11 +156,30 @@ export const dimensionSlotValidator = v.object({
   value: v.string(),
   count: v.number(),
   revenueCents: v.number(),
+  // Slots written since alpha.11 also count the pageviews and the sessions
+  // (a session's first event) they saw, so pages rank by views and every
+  // other dimension by sessions instead of by raw event volume.
+  pageviews: v.optional(v.number()),
+  sessions: v.optional(v.number()),
 });
 
+export const visitorSketchValidator = v.object({
+  version: v.literal(1),
+  precision: v.number(),
+  registers: v.bytes(),
+});
+
+export const aggregateGranularityValidator = v.union(
+  v.literal("hour"),
+  v.literal("day"),
+  // One bucket per calendar day in the site's timezone, written alongside
+  // the UTC day for sites with a timezone (see localTime.ts).
+  v.literal("localDay"),
+);
+
 export const aggregateFieldsValidator = v.object({
   siteId: v.id("sites"),
-  granularity: v.union(v.literal("hour"), v.literal("day")),
+  granularity: aggregateGranularityValidator,
   bucketStart: v.number(),
   shard: v.number(),
   events: v.number(),
@@ -161,9 +189,17 @@ export const aggregateFieldsValidator = v.object({
   outboundClicks: v.number(),
   sessions: v.number(),
   visitors: v.number(),
+  visitorSketch: v.optional(visitorSketchValidator),
+  visitorSketchComplete: v.optional(v.boolean()),
   conversions: v.number(),
   revenueCents: v.number(),
   dimensions: v.array(dimensionSlotValidator),
+  // Engagement counters recorded since alpha.11: how much session duration
+  // grew from the activity that landed in this bucket, sessions that reached
+  // a second pageview here, and sessions of visitors never seen before.
+  durationMs: v.optional(v.number()),
+  continuedSessions: v.optional(v.number()),
+  newVisitors: v.optional(v.number()),
 });
 
 export const goalMatchTypeValidator = v.union(
@@ -230,7 +266,6 @@ export const trustedConversionValidator = v.object({
   properties: v.optional(eventPropertiesValidator),
 });
 
-
 export type TrackerEvent = Infer<typeof trackerEventValidator>;
 export type IngestContext = Infer<typeof ingestContextValidator>;
 export type EventProperties = Infer<typeof eventPropertiesValidator>;

package/src/component/visitors.ts

@@ -0,0 +1,116 @@
+import { v } from "convex/values";
+import { MAX_VISITOR_ALIASES } from "./constants.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+import type { Id } from "./_generated/dataModel.js";
+import { mutation, type QueryCtx } from "./_generated/server.js";
+
+/**
+ * Folds an anonymous visitor id into the pseudonymous identity the host
+ * supplies after signup or sign-in. Links are one level deep: an alias can
+ * belong to one identity, an identity can never become an alias, and an
+ * alias can never own aliases, so journeys stay bounded and unambiguous.
+ */
+export const link = mutation({
+  args: {
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+  },
+  returns: v.object({ linked: v.boolean(), aliasCount: v.number() }),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    let visitorId: string;
+    let previousVisitorId: string;
+    try {
+      visitorId = sanitizeOpaqueId(args.visitorId, "visitorId");
+      previousVisitorId = sanitizeOpaqueId(
+        args.previousVisitorId,
+        "previousVisitorId",
+      );
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid visitor id",
+      );
+    }
+    if (visitorId === previousVisitorId) {
+      fail("INVALID_ARGUMENT", "previousVisitorId must differ from visitorId");
+    }
+    if (await findAlias(ctx, args.siteId, visitorId)) {
+      fail("CONFLICT", "visitorId is already linked to another visitor");
+    }
+    const previousOwnsAliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", previousVisitorId),
+      )
+      .first();
+    if (previousOwnsAliases) {
+      fail("CONFLICT", "previousVisitorId already owns linked visitors");
+    }
+    const aliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", visitorId),
+      )
+      .take(MAX_VISITOR_ALIASES);
+    const existing = await findAlias(ctx, args.siteId, previousVisitorId);
+    if (existing) {
+      if (existing.visitorId !== visitorId) {
+        fail(
+          "CONFLICT",
+          "previousVisitorId is already linked to a different visitor",
+        );
+      }
+      return { linked: false, aliasCount: aliases.length };
+    }
+    if (aliases.length >= MAX_VISITOR_ALIASES) {
+      fail(
+        "LIMIT_EXCEEDED",
+        `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`,
+        {
+          limit: MAX_VISITOR_ALIASES,
+        },
+      );
+    }
+    await ctx.db.insert("visitorAliases", {
+      siteId: args.siteId,
+      visitorId,
+      previousVisitorId,
+      linkedAt: Date.now(),
+    });
+    return { linked: true, aliasCount: aliases.length + 1 };
+  },
+});
+
+/** The identity a visitor id resolves to, followed by every alias it owns. */
+export async function resolveVisitorIdentities(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+): Promise<string[]> {
+  const alias = await findAlias(ctx, siteId, visitorId);
+  const identity = alias?.visitorId ?? visitorId;
+  const aliases = await ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_visitorId", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", identity),
+    )
+    .take(MAX_VISITOR_ALIASES);
+  return [identity, ...aliases.map((row) => row.previousVisitorId)];
+}
+
+function findAlias(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  previousVisitorId: string,
+) {
+  return ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_previousVisitorId", (range) =>
+      range.eq("siteId", siteId).eq("previousVisitorId", previousVisitorId),
+    )
+    .unique();
+}