@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/src/component/sanitize.ts

@@ -1,4 +1,5 @@
 import {
+  CLOCK_SKEW_TOLERANCE_MS,
   MAX_BATCH_BYTES,
   MAX_PROPERTIES,
   MAX_SITE_DOMAINS,
@@ -8,6 +9,7 @@ import type {
   IngestContext,
   TrackerEvent,
 } from "./validators.js";
+import { isValidVitalValue, isVitalMetric } from "./vitals.js";
 
 const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -62,9 +64,10 @@ export function sanitizePath(value: string): string {
   const input = requireString(value, "path", 2048);
   let path: string;
   try {
-    path = input.startsWith("http://") || input.startsWith("https://")
-      ? new URL(input).pathname
-      : input.split(/[?#]/, 1)[0];
+    path =
+      input.startsWith("http://") || input.startsWith("https://")
+        ? new URL(input).pathname
+        : input.split(/[?#]/, 1)[0];
   } catch {
     throw new Error("path is invalid");
   }
@@ -73,6 +76,37 @@ export function sanitizePath(value: string): string {
   return cleaned || "/";
 }
 
+const ROUTE_ID = ":id";
+const NUMERIC_SEGMENT = /^\d+$/;
+const UUID_SEGMENT =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
+
+/**
+ * Collapses the identifiers that unmistakably vary per record — numbers,
+ * UUIDs, and long hexadecimal tokens — into `:id`, so `/blog/42` and
+ * `/blog/43` roll up as one route in the site map. Slugs are pages of their
+ * own and stay as they are. A trailing slash is dropped because both
+ * spellings serve the same page. Only the map uses this; every other report
+ * keeps the sanitized path exactly as the tracker sent it.
+ */
+export function normalizeRoute(path: string): string {
+  const trimmed =
+    path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
+  const segments = trimmed.split("/");
+  for (let index = 1; index < segments.length; index += 1) {
+    const segment = segments[index] ?? "";
+    if (
+      NUMERIC_SEGMENT.test(segment) ||
+      UUID_SEGMENT.test(segment) ||
+      HEX_SEGMENT.test(segment)
+    ) {
+      segments[index] = ROUTE_ID;
+    }
+  }
+  return segments.join("/") || "/";
+}
+
 export function sanitizePublicUrl(value: string): string | undefined {
   const input = cleanString(value, 2048);
   if (!input) return undefined;
@@ -111,15 +145,42 @@ export function sanitizeProperties(
   return Object.keys(result).length > 0 ? result : undefined;
 }
 
+/**
+ * Re-anchors a batch whose sender's clock disagrees with ours. `sentAt` is the
+ * sender's clock at flush time, so `now - sentAt` is the clock offset alone: an
+ * event queued long before the flush keeps its age, and nothing is inferred
+ * from the events themselves. Offsets inside the tolerance are latency or a
+ * slightly drifted clock and leave the batch untouched, so a well-set client's
+ * timestamps stay exactly as sent.
+ */
+export function alignClock<T extends { timestamp: number }>(
+  events: T[],
+  sentAt: number | undefined,
+  now: number,
+): T[] {
+  if (sentAt === undefined || !Number.isFinite(sentAt)) return events;
+  const offset = now - sentAt;
+  if (Math.abs(offset) <= CLOCK_SKEW_TOLERANCE_MS) return events;
+  return events.map((event) => ({
+    ...event,
+    timestamp: event.timestamp + offset,
+  }));
+}
+
 export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
-  if (!Number.isFinite(event.timestamp)) throw new Error("timestamp is invalid");
+  if (!Number.isFinite(event.timestamp))
+    throw new Error("timestamp is invalid");
   if (
     event.timestamp < now - MAX_EVENT_AGE_MS ||
     event.timestamp > now + MAX_FUTURE_SKEW_MS
   ) {
     throw new Error("timestamp is outside the accepted window");
   }
-  if (!Number.isSafeInteger(event.sequence) || event.sequence < 0 || event.sequence > 1_000_000_000) {
+  if (
+    !Number.isSafeInteger(event.sequence) ||
+    event.sequence < 0 ||
+    event.sequence > 1_000_000_000
+  ) {
     throw new Error("sequence is out of range");
   }
   if (event.revenueCents !== undefined && !isValidMoney(event.revenueCents)) {
@@ -130,9 +191,20 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     throw new Error("name is required for custom events");
   }
   if (event.type === "conversion" && !name) name = "conversion";
-  const currency = event.currency === undefined
-    ? event.type === "conversion" ? "USD" : undefined
-    : sanitizeCurrency(event.currency);
+  if (event.type === "vital") {
+    if (!name || !isVitalMetric(name)) {
+      throw new Error("vital name must be a Web Vitals metric");
+    }
+    if (event.value === undefined || !isValidVitalValue(event.value)) {
+      throw new Error("vital value is out of range");
+    }
+  }
+  const currency =
+    event.currency === undefined
+      ? event.type === "conversion"
+        ? "USD"
+        : undefined
+      : sanitizeCurrency(event.currency);
   return {
     eventId: sanitizeOpaqueId(event.eventId, "eventId"),
     sessionId: sanitizeOpaqueId(event.sessionId, "sessionId"),
@@ -140,27 +212,43 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     type: event.type,
     name,
     path: sanitizePath(event.path),
-    referrer: event.referrer === undefined
-      ? undefined
-      : sanitizePublicUrl(event.referrer),
+    referrer:
+      event.referrer === undefined
+        ? undefined
+        : sanitizePublicUrl(event.referrer),
     timestamp: Math.trunc(event.timestamp),
     sequence: event.sequence,
-    target: event.target === undefined
-      ? undefined
-      : cleanString(event.target, 160) || undefined,
+    target:
+      event.target === undefined
+        ? undefined
+        : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
     href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
     currency,
-    affiliateSlug: event.affiliateSlug === undefined
-      ? undefined
-      : sanitizeSlug(event.affiliateSlug, "affiliateSlug"),
+    affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
+    // Only measurements keep a value; a stray number on any other type is noise.
+    value:
+      event.type === "vital" && event.value !== undefined
+        ? Math.round(event.value)
+        : undefined,
   };
 }
 
+function optionalAffiliateSlug(value: string | undefined): string | undefined {
+  if (value === undefined) return undefined;
+  try {
+    return sanitizeSlug(value, "affiliateSlug");
+  } catch {
+    return undefined;
+  }
+}
+
 export function normalizeDomains(domains: string[]): string[] {
   if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
-    throw new Error(`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`);
+    throw new Error(
+      `domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`,
+    );
   }
   const normalized = domains.map(normalizeDomainPattern);
   return [...new Set(normalized)].sort();
@@ -178,7 +266,11 @@ export function normalizeDomainPattern(value: string): string {
       throw new Error("domains cannot include paths or query strings");
     }
     hostname = url.hostname;
-  } else if (hostInput.includes("/") || hostInput.includes("?") || hostInput.includes("#")) {
+  } else if (
+    hostInput.includes("/") ||
+    hostInput.includes("?") ||
+    hostInput.includes("#")
+  ) {
     throw new Error("domain is invalid");
   }
   hostname = hostname.replace(/\.$/, "");
@@ -207,7 +299,9 @@ export function originAllowed(origin: string, domains: string[]): boolean {
   });
 }
 
-export function validateTimezone(value: string | undefined): string | undefined {
+export function validateTimezone(
+  value: string | undefined,
+): string | undefined {
   if (value === undefined || value.trim() === "") return undefined;
   const timezone = cleanString(value, 80);
   try {
@@ -218,17 +312,33 @@ export function validateTimezone(value: string | undefined): string | undefined
   }
 }
 
-export function sanitizeContext(context: IngestContext | undefined): IngestContext {
+export function sanitizeContext(
+  context: IngestContext | undefined,
+): IngestContext {
   const latitude = finiteCoordinate(context?.latitude, -90, 90);
   const longitude = finiteCoordinate(context?.longitude, -180, 180);
   return {
-    country: context?.country ? cleanString(context.country.toUpperCase(), 2) || undefined : undefined,
-    city: context?.city ? cleanString(context.city, 80) || undefined : undefined,
-    latitude: latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
-    longitude: longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
-    browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
+    country: context?.country
+      ? cleanString(context.country.toUpperCase(), 2) || undefined
+      : undefined,
+    city: context?.city
+      ? cleanString(context.city, 80) || undefined
+      : undefined,
+    latitude:
+      latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
+    longitude:
+      longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
+    browser: context?.browser
+      ? cleanString(context.browser, 32) || undefined
+      : undefined,
     os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
-    device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
+    device: context?.device
+      ? cleanString(context.device, 24) || undefined
+      : undefined,
+    visitorKey:
+      context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
+        ? cleanString(context.visitorKey, 128) || undefined
+        : undefined,
   };
 }
 
@@ -242,10 +352,21 @@ export function isValidMoney(value: number): boolean {
   return Number.isSafeInteger(value) && value >= 0 && value <= 1_000_000_000;
 }
 
-export function ensureBatchBytes(value: unknown): number {
-  const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
-  if (bytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
-  return bytes;
+export function measureBatchBytes(
+  siteId: string,
+  events: TrackerEvent[],
+): { batchBytes: number; eventByteCounts: number[] } {
+  const encoder = new TextEncoder();
+  const eventByteCounts = events.map(
+    (event) => encoder.encode(JSON.stringify(event)).byteLength,
+  );
+  // The empty envelope already includes both array brackets; only commas remain.
+  const batchBytes =
+    encoder.encode(JSON.stringify({ siteId, events: [] })).byteLength +
+    eventByteCounts.reduce((total, byteCount) => total + byteCount, 0) +
+    Math.max(0, events.length - 1);
+  if (batchBytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
+  return { batchBytes, eventByteCounts };
 }
 
 export function sourceFromReferrer(referrer: string | undefined): string {
@@ -273,6 +394,7 @@ function finiteCoordinate(
   maximum: number,
 ): number | undefined {
   if (value === undefined) return undefined;
-  if (!Number.isFinite(value) || value < minimum || value > maximum) return undefined;
+  if (!Number.isFinite(value) || value < minimum || value > maximum)
+    return undefined;
   return value;
 }

package/src/component/schema.ts

@@ -3,11 +3,13 @@ import { v } from "convex/values";
 import {
   affiliateFieldsValidator,
   aggregateFieldsValidator,
-  eventFieldsValidator,
+  batchedEventValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
+  visitorSketchValidator,
+  vitalMetricValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -16,7 +18,12 @@ export default defineSchema({
     .index("by_ownerId_and_name", ["ownerId", "name"])
     .index("by_networkId", ["networkId"]),
 
-  sessions: defineTable(sessionFieldsValidator)
+  sessions: defineTable(
+    sessionFieldsValidator.extend({
+      geoLookupAttemptedAt: v.optional(v.number()),
+      lastPageviewAt: v.optional(v.number()),
+    }),
+  )
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
     .index("by_siteId_and_startedAt", ["siteId", "startedAt"])
@@ -45,26 +52,46 @@ export default defineSchema({
     device: v.string(),
     pageviewCount: v.number(),
     lastHeartbeatEventId: v.optional(v.string()),
+    recentHeartbeatEventIds: v.optional(v.array(v.string())),
     eventCount: v.number(),
+    previousPath: v.optional(v.string()),
+    movedAt: v.optional(v.number()),
   })
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
-    .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"]),
+    .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
+    .index("by_expiresAt", { fields: ["expiresAt"], staged: true }),
 
-  events: defineTable(eventFieldsValidator)
-    .index("by_siteId_and_eventId", ["siteId", "eventId"])
-    .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
-    .index("by_siteId_and_visitorId_and_timestamp", [
-      "siteId",
-      "visitorId",
-      "timestamp",
-    ])
-    .index("by_siteId_and_sessionId_and_timestamp", [
+  liveSweepStates: defineTable({
+    siteId: v.id("sites"),
+    scheduledAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  geoLookupBudgets: defineTable({
+    siteId: v.id("sites"),
+    dayStart: v.number(),
+    count: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  eventBatches: defineTable({
+    siteId: v.id("sites"),
+    sessionId: v.string(),
+    visitorId: v.string(),
+    bucketStart: v.number(),
+    events: v.array(batchedEventValidator),
+    migrationVersion: v.optional(v.literal(1)),
+  })
+    .index("by_siteId_and_sessionId_and_bucketStart", [
       "siteId",
       "sessionId",
-      "timestamp",
+      "bucketStart",
     ])
-    .index("by_siteId_and_type_and_timestamp", ["siteId", "type", "timestamp"]),
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
+    .index("by_siteId_and_visitorId_and_bucketStart", [
+      "siteId",
+      "visitorId",
+      "bucketStart",
+    ]),
 
   aggregateBuckets: defineTable(aggregateFieldsValidator)
     .index("by_siteId_and_granularity_and_bucketStart_and_shard", [
@@ -79,6 +106,163 @@ export default defineSchema({
       "bucketStart",
     ]),
 
+  routeDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    route: v.string(),
+    pageviews: v.number(),
+    entries: v.number(),
+    continued: v.number(),
+    dwellMs: v.number(),
+    dwellSamples: v.number(),
+    visitorSketch: visitorSketchValidator,
+    outgoing: v.array(v.object({ route: v.string(), count: v.number() })),
+  })
+    .index("by_siteId_and_route_and_bucketStart_and_shard", [
+      "siteId",
+      "route",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  goalDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    goalId: v.id("goals"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    goalKey: v.string(),
+    completions: v.number(),
+    valueCents: v.number(),
+  })
+    .index("by_siteId_and_goalId_and_bucketStart_and_shard", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_goalId_and_bucketStart", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  funnelDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    funnelId: v.id("funnels"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    steps: v.array(
+      v.object({
+        step: v.number(),
+        entrants: v.number(),
+        completions: v.number(),
+      }),
+    ),
+  })
+    .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_funnelId_and_bucketStart", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  // Field Web Vitals. One row per site/day/shard and either a page (device
+  // "(all)") or a device class (page "(all)"); the row folds every metric's
+  // histogram together so a report reads five metrics for one document. Pages
+  // are bounded like dimension slots: at most VITAL_PAGE_SLOTS distinct pages
+  // per day, then "(other)".
+  vitalRollups: defineTable({
+    siteId: v.id("sites"),
+    bucketStart: v.number(),
+    device: v.string(),
+    page: v.string(),
+    shard: v.number(),
+    metrics: v.array(
+      v.object({
+        metric: vitalMetricValidator,
+        count: v.number(),
+        sum: v.number(),
+        histogram: v.array(v.number()),
+      }),
+    ),
+  })
+    .index("by_key", ["siteId", "bucketStart", "device", "page", "shard"])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  affiliateDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    affiliateId: v.id("affiliates"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    affiliateSlug: v.string(),
+    visits: v.number(),
+    visitorSketch: visitorSketchValidator,
+    conversions: v.number(),
+    revenueCents: v.number(),
+    commissionCents: v.number(),
+  })
+    .index("by_siteId_and_affiliateId_and_bucketStart_and_shard", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_affiliateId_and_bucketStart", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  analyticsCoverage: defineTable({
+    siteId: v.id("sites"),
+    generation: v.literal(1),
+    rollupCompleteFrom: v.number(),
+    siteMapSince: v.optional(v.number()),
+    retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionPolicies: defineTable({
+    siteId: v.id("sites"),
+    eventsDays: v.number(),
+    sessionsDays: v.number(),
+    conversionsDays: v.number(),
+    aggregatesDays: v.number(),
+    reportsDays: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionStates: defineTable({
+    siteId: v.id("sites"),
+    kind: v.string(),
+    before: v.number(),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("complete"),
+    ),
+    generation: v.number(),
+    attempt: v.number(),
+    leaseExpiresAt: v.optional(v.number()),
+    scheduledAt: v.optional(v.number()),
+    deleted: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId_and_kind", ["siteId", "kind"]),
+
   goals: defineTable(goalFieldsValidator)
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_key", ["siteId", "key"]),
@@ -88,13 +272,18 @@ export default defineSchema({
     goalId: v.id("goals"),
     goalKey: v.string(),
     eventId: v.string(),
+    idempotencyVersion: v.optional(v.literal(2)),
     sessionId: v.string(),
     visitorId: v.string(),
     timestamp: v.number(),
     valueCents: v.number(),
   })
     .index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
-    .index("by_siteId_and_goalId_and_timestamp", ["siteId", "goalId", "timestamp"])
+    .index("by_siteId_and_goalId_and_timestamp", [
+      "siteId",
+      "goalId",
+      "timestamp",
+    ])
     .index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
 
   funnels: defineTable(funnelFieldsValidator)
@@ -146,6 +335,21 @@ export default defineSchema({
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_slug", ["siteId", "slug"]),
 
+  visitorSecrets: defineTable({
+    key: v.literal("current"),
+    value: v.string(),
+    createdAt: v.number(),
+  }).index("by_key", ["key"]),
+
+  visitorAliases: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    previousVisitorId: v.string(),
+    linkedAt: v.number(),
+  })
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
+
   visitorAttributions: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),