@iann29/rastro 0.1.0-alpha.3 → 0.1.0-alpha.5

package/src/component/sanitize.ts

@@ -147,7 +147,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     sequence: event.sequence,
     target: event.target === undefined
       ? undefined
-      : cleanString(event.target, 160) || undefined,
+      : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
     href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
@@ -236,6 +236,9 @@ export function sanitizeContext(context: IngestContext | undefined): IngestConte
     browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
     os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
     device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
+    visitorKey: context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
+      ? cleanString(context.visitorKey, 128) || undefined
+      : undefined,
   };
 }
 
@@ -249,10 +252,21 @@ export function isValidMoney(value: number): boolean {
   return Number.isSafeInteger(value) && value >= 0 && value <= 1_000_000_000;
 }
 
-export function ensureBatchBytes(value: unknown): number {
-  const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
-  if (bytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
-  return bytes;
+export function measureBatchBytes(
+  siteId: string,
+  events: TrackerEvent[],
+): { batchBytes: number; eventByteCounts: number[] } {
+  const encoder = new TextEncoder();
+  const eventByteCounts = events.map((event) =>
+    encoder.encode(JSON.stringify(event)).byteLength
+  );
+  // The empty envelope already includes both array brackets; only commas remain.
+  const batchBytes =
+    encoder.encode(JSON.stringify({ siteId, events: [] })).byteLength +
+    eventByteCounts.reduce((total, byteCount) => total + byteCount, 0) +
+    Math.max(0, events.length - 1);
+  if (batchBytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
+  return { batchBytes, eventByteCounts };
 }
 
 export function sourceFromReferrer(referrer: string | undefined): string {

package/src/component/schema.ts

@@ -8,6 +8,7 @@ import {
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
+  visitorSketchValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -101,6 +102,115 @@ export default defineSchema({
       "bucketStart",
     ]),
 
+  goalDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    goalId: v.id("goals"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    goalKey: v.string(),
+    completions: v.number(),
+    valueCents: v.number(),
+  })
+    .index("by_siteId_and_goalId_and_bucketStart_and_shard", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_goalId_and_bucketStart", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  funnelDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    funnelId: v.id("funnels"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    steps: v.array(v.object({
+      step: v.number(),
+      entrants: v.number(),
+      completions: v.number(),
+    })),
+  })
+    .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_funnelId_and_bucketStart", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  affiliateDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    affiliateId: v.id("affiliates"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    affiliateSlug: v.string(),
+    visits: v.number(),
+    visitorSketch: visitorSketchValidator,
+    conversions: v.number(),
+    revenueCents: v.number(),
+    commissionCents: v.number(),
+  })
+    .index("by_siteId_and_affiliateId_and_bucketStart_and_shard", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_affiliateId_and_bucketStart", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  analyticsCoverage: defineTable({
+    siteId: v.id("sites"),
+    generation: v.literal(1),
+    rollupCompleteFrom: v.number(),
+    retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionPolicies: defineTable({
+    siteId: v.id("sites"),
+    eventsDays: v.number(),
+    sessionsDays: v.number(),
+    conversionsDays: v.number(),
+    aggregatesDays: v.number(),
+    reportsDays: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionStates: defineTable({
+    siteId: v.id("sites"),
+    kind: v.string(),
+    before: v.number(),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("complete"),
+    ),
+    generation: v.number(),
+    attempt: v.number(),
+    leaseExpiresAt: v.optional(v.number()),
+    scheduledAt: v.optional(v.number()),
+    deleted: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId_and_kind", ["siteId", "kind"]),
+
   goals: defineTable(goalFieldsValidator)
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_key", ["siteId", "key"]),
@@ -173,6 +283,21 @@ export default defineSchema({
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_slug", ["siteId", "slug"]),
 
+  visitorSecrets: defineTable({
+    key: v.literal("current"),
+    value: v.string(),
+    createdAt: v.number(),
+  }).index("by_key", ["key"]),
+
+  visitorAliases: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+    linkedAt: v.number(),
+  })
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
+
   visitorAttributions: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),

package/src/component/sites.ts

@@ -1,6 +1,7 @@
 import { v } from "convex/values";
 import { internalQuery, mutation, query } from "./_generated/server.js";
 import { MAX_SITES_PER_OWNER } from "./constants.js";
+import { ensureAnalyticsControl } from "./coverage.js";
 import { fail } from "./errors.js";
 import {
   cleanString,
@@ -66,6 +67,7 @@ export const create = mutation({
       ) {
         fail("CONFLICT", "site name is already associated with different settings");
       }
+      await ensureAnalyticsControl(ctx, existing._id, Date.now());
       return existing._id;
     }
 
@@ -80,7 +82,7 @@ export const create = mutation({
     }
 
     const now = Date.now();
-    return await ctx.db.insert("sites", {
+    const siteId = await ctx.db.insert("sites", {
       ownerId,
       name,
       domains,
@@ -91,6 +93,8 @@ export const create = mutation({
       createdAt: now,
       updatedAt: now,
     });
+    await ensureAnalyticsControl(ctx, siteId, now);
+    return siteId;
   },
 });
 

package/src/component/useragent.ts

@@ -0,0 +1,171 @@
+export type ClientClassification = {
+  browser: string;
+  os: string;
+  device: string;
+};
+
+// The tracker never reports the client: everything here is derived from headers
+// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
+// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
+// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
+// Firefox and Safari implement no client hints, so the User-Agent stays the
+// fallback for them and for insecure origins, where no hint is sent at all.
+const MAX_HINT_LENGTH = 512;
+const MAX_HINT_BRANDS = 16;
+
+const BRAND_PATTERN =
+  /"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
+const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
+
+// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
+// allowlisted: anything unrecognized (including the GREASE entries Chromium
+// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
+// entering the browser dimension verbatim.
+const HINT_BRANDS = new Map<string, string>([
+  ["brave", "Brave"],
+  ["chromium", "Chromium"],
+  ["google chrome", "Chrome"],
+  ["microsoft edge", "Edge"],
+  ["opera", "Opera"],
+  ["opera gx", "Opera"],
+  ["samsung internet", "Samsung Internet"],
+  ["vivaldi", "Vivaldi"],
+  ["yandex", "Yandex"],
+]);
+
+// Every Chromium brand list contains "Chromium", and Chrome adds "Google
+// Chrome" beside it; both only win when no fork identifies itself.
+const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
+
+const HINT_PLATFORMS = new Map<string, string>([
+  ["android", "Android"],
+  ["chrome os", "ChromeOS"],
+  ["chromium os", "ChromeOS"],
+  ["ios", "iOS"],
+  ["linux", "Linux"],
+  ["macos", "macOS"],
+  ["windows", "Windows"],
+]);
+
+// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
+// browser on iOS carries `Safari/`, so the specific tokens must be tested
+// first. Brave has no token at all and is only detectable through the hints.
+const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
+  [/Edg(?:A|iOS)?\//, "Edge"],
+  [/(?:Firefox|FxiOS)\//, "Firefox"],
+  [/OPR\//, "Opera"],
+  [/SamsungBrowser\//, "Samsung Internet"],
+  [/Vivaldi\//, "Vivaldi"],
+  [/YaBrowser\//, "Yandex"],
+  [/(?:Chrome|CriOS)\//, "Chrome"],
+  [/Safari\//, "Safari"],
+];
+
+// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
+const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
+  [/Android/, "Android"],
+  [/iPhone|iPad|iPod/, "iOS"],
+  [/CrOS/, "ChromeOS"],
+  [/Windows/, "Windows"],
+  [/Mac OS X/, "macOS"],
+  [/Linux/, "Linux"],
+];
+
+// Ordered: tablet user agents that carry `Android` must not be read as mobile.
+const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
+  [/iPad|Tablet/, "tablet"],
+  [/Mobile|Android|iPhone/, "mobile"],
+];
+
+export function classifyClient(
+  userAgent: string,
+  headers: Headers,
+): ClientClassification {
+  const agent = classifyUserAgent(userAgent);
+  const brand = brandFromClientHints(headers.get("sec-ch-ua"));
+  const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
+  const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
+  return {
+    browser: brand ?? agent.browser,
+    os: platform ?? agent.os,
+    device: deviceFromClientHints(mobile, platform) ?? agent.device,
+  };
+}
+
+export function classifyUserAgent(userAgent: string): ClientClassification {
+  return {
+    browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
+    os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
+    device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
+  };
+}
+
+function matchAgent(
+  table: readonly (readonly [RegExp, string])[],
+  userAgent: string,
+): string | undefined {
+  for (const [pattern, value] of table) {
+    if (pattern.test(userAgent)) return value;
+  }
+  return undefined;
+}
+
+function brandFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  let generic: string | undefined;
+  let seen = 0;
+  for (const match of header.matchAll(BRAND_PATTERN)) {
+    if (seen >= MAX_HINT_BRANDS) break;
+    seen += 1;
+    const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
+    if (!brand) continue;
+    if (!GENERIC_BRANDS.has(brand)) return brand;
+    if (brand === "Chrome" || generic === undefined) generic = brand;
+  }
+  return generic;
+}
+
+function platformFromClientHints(header: string | null): string | undefined {
+  if (!header || header.length > MAX_HINT_LENGTH) return undefined;
+  const match = PLATFORM_PATTERN.exec(header);
+  if (!match) return undefined;
+  return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
+}
+
+function mobileFromClientHints(header: string | null): boolean | undefined {
+  if (header === null) return undefined;
+  const value = header.trim();
+  if (value === "?1") return true;
+  if (value === "?0") return false;
+  return undefined;
+}
+
+function deviceFromClientHints(
+  mobile: boolean | undefined,
+  platform: string | undefined,
+): string | undefined {
+  if (mobile === undefined) return undefined;
+  if (mobile) return "mobile";
+  // Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
+  // `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
+  // `Accept-CH` opt-in and a permissions-policy delegation from every tracked
+  // site; "Android but not mobile" is the only tablet signal available here.
+  if (platform === "Android") return "tablet";
+  return "desktop";
+}
+
+function unquote(value: string): string {
+  return value.replace(/\\(.)/g, "$1");
+}
+
+// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
+// headless browsers. A URL inside the User-Agent is the crawler convention
+// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
+// alone because some runtimes do not expose it at all, and a scripted browser
+// that spoofs a stock User-Agent is indistinguishable from a person here.
+const BOT_PATTERN =
+  /bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
+
+export function isKnownBot(userAgent: string): boolean {
+  return BOT_PATTERN.test(userAgent);
+}

package/src/component/validators.ts

@@ -47,6 +47,7 @@ export const ingestContextValidator = v.object({
   browser: v.optional(v.string()),
   os: v.optional(v.string()),
   device: v.optional(v.string()),
+  visitorKey: v.optional(v.string()),
 });
 
 export const batchedEventValidator = trackerEventValidator.extend(
@@ -72,6 +73,7 @@ export const sessionFieldsValidator = v.object({
   siteId: v.id("sites"),
   sessionId: v.string(),
   visitorId: v.string(),
+  visitorKey: v.optional(v.string()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -128,6 +130,7 @@ export const eventFieldsValidator = v.object({
 
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
+  v.literal("campaign"),
   v.literal("page"),
   v.literal("country"),
   v.literal("device"),
@@ -144,6 +147,12 @@ export const dimensionSlotValidator = v.object({
   revenueCents: v.number(),
 });
 
+export const visitorSketchValidator = v.object({
+  version: v.literal(1),
+  precision: v.number(),
+  registers: v.bytes(),
+});
+
 export const aggregateFieldsValidator = v.object({
   siteId: v.id("sites"),
   granularity: v.union(v.literal("hour"), v.literal("day")),
@@ -156,6 +165,8 @@ export const aggregateFieldsValidator = v.object({
   outboundClicks: v.number(),
   sessions: v.number(),
   visitors: v.number(),
+  visitorSketch: v.optional(visitorSketchValidator),
+  visitorSketchComplete: v.optional(v.boolean()),
   conversions: v.number(),
   revenueCents: v.number(),
   dimensions: v.array(dimensionSlotValidator),

package/src/component/visitors.ts

@@ -0,0 +1,105 @@
+import { v } from "convex/values";
+import { MAX_VISITOR_ALIASES } from "./constants.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+import type { Id } from "./_generated/dataModel.js";
+import { mutation, type QueryCtx } from "./_generated/server.js";
+
+/**
+ * Folds an anonymous visitor id into the pseudonymous identity the host
+ * supplies after signup or sign-in. Links are one level deep: an alias can
+ * belong to one identity, an identity can never become an alias, and an
+ * alias can never own aliases, so journeys stay bounded and unambiguous.
+ */
+export const link = mutation({
+  args: {
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+  },
+  returns: v.object({ linked: v.boolean(), aliasCount: v.number() }),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    if (!site) fail("NOT_FOUND", "site not found");
+    let visitorId: string;
+    let previousVisitorId: string;
+    try {
+      visitorId = sanitizeOpaqueId(args.visitorId, "visitorId");
+      previousVisitorId = sanitizeOpaqueId(
+        args.previousVisitorId,
+        "previousVisitorId",
+      );
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid visitor id",
+      );
+    }
+    if (visitorId === previousVisitorId) {
+      fail("INVALID_ARGUMENT", "previousVisitorId must differ from visitorId");
+    }
+    if (await findAlias(ctx, args.siteId, visitorId)) {
+      fail("CONFLICT", "visitorId is already linked to another visitor");
+    }
+    const previousOwnsAliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", previousVisitorId),
+      )
+      .first();
+    if (previousOwnsAliases) {
+      fail("CONFLICT", "previousVisitorId already owns linked visitors");
+    }
+    const aliases = await ctx.db
+      .query("visitorAliases")
+      .withIndex("by_siteId_and_visitorId", (range) =>
+        range.eq("siteId", args.siteId).eq("visitorId", visitorId),
+      )
+      .take(MAX_VISITOR_ALIASES);
+    const existing = await findAlias(ctx, args.siteId, previousVisitorId);
+    if (existing) {
+      if (existing.visitorId !== visitorId) {
+        fail("CONFLICT", "previousVisitorId is already linked to a different visitor");
+      }
+      return { linked: false, aliasCount: aliases.length };
+    }
+    if (aliases.length >= MAX_VISITOR_ALIASES) {
+      fail("LIMIT_EXCEEDED", `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`, {
+        limit: MAX_VISITOR_ALIASES,
+      });
+    }
+    await ctx.db.insert("visitorAliases", {
+      siteId: args.siteId,
+      visitorId,
+      previousVisitorId,
+      linkedAt: Date.now(),
+    });
+    return { linked: true, aliasCount: aliases.length + 1 };
+  },
+});
+
+/** The identity a visitor id resolves to, followed by every alias it owns. */
+export async function resolveVisitorIdentities(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  visitorId: string,
+): Promise<string[]> {
+  const alias = await findAlias(ctx, siteId, visitorId);
+  const identity = alias?.visitorId ?? visitorId;
+  const aliases = await ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_visitorId", (range) =>
+      range.eq("siteId", siteId).eq("visitorId", identity),
+    )
+    .take(MAX_VISITOR_ALIASES);
+  return [identity, ...aliases.map((row) => row.previousVisitorId)];
+}
+
+function findAlias(ctx: QueryCtx, siteId: Id<"sites">, previousVisitorId: string) {
+  return ctx.db
+    .query("visitorAliases")
+    .withIndex("by_siteId_and_previousVisitorId", (range) =>
+      range.eq("siteId", siteId).eq("previousVisitorId", previousVisitorId),
+    )
+    .unique();
+}