@iann29/rastro 0.1.0-alpha.5 → 0.1.0-alpha.7

package/src/component/sanitize.ts

@@ -1,4 +1,5 @@
 import {
+  CLOCK_SKEW_TOLERANCE_MS,
   MAX_BATCH_BYTES,
   MAX_PROPERTIES,
   MAX_SITE_DOMAINS,
@@ -8,6 +9,7 @@ import type {
   IngestContext,
   TrackerEvent,
 } from "./validators.js";
+import { isValidVitalValue, isVitalMetric } from "./vitals.js";
 
 const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -62,9 +64,10 @@ export function sanitizePath(value: string): string {
   const input = requireString(value, "path", 2048);
   let path: string;
   try {
-    path = input.startsWith("http://") || input.startsWith("https://")
-      ? new URL(input).pathname
-      : input.split(/[?#]/, 1)[0];
+    path =
+      input.startsWith("http://") || input.startsWith("https://")
+        ? new URL(input).pathname
+        : input.split(/[?#]/, 1)[0];
   } catch {
     throw new Error("path is invalid");
   }
@@ -111,15 +114,42 @@ export function sanitizeProperties(
   return Object.keys(result).length > 0 ? result : undefined;
 }
 
+/**
+ * Re-anchors a batch whose sender's clock disagrees with ours. `sentAt` is the
+ * sender's clock at flush time, so `now - sentAt` is the clock offset alone: an
+ * event queued long before the flush keeps its age, and nothing is inferred
+ * from the events themselves. Offsets inside the tolerance are latency or a
+ * slightly drifted clock and leave the batch untouched, so a well-set client's
+ * timestamps stay exactly as sent.
+ */
+export function alignClock<T extends { timestamp: number }>(
+  events: T[],
+  sentAt: number | undefined,
+  now: number,
+): T[] {
+  if (sentAt === undefined || !Number.isFinite(sentAt)) return events;
+  const offset = now - sentAt;
+  if (Math.abs(offset) <= CLOCK_SKEW_TOLERANCE_MS) return events;
+  return events.map((event) => ({
+    ...event,
+    timestamp: event.timestamp + offset,
+  }));
+}
+
 export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
-  if (!Number.isFinite(event.timestamp)) throw new Error("timestamp is invalid");
+  if (!Number.isFinite(event.timestamp))
+    throw new Error("timestamp is invalid");
   if (
     event.timestamp < now - MAX_EVENT_AGE_MS ||
     event.timestamp > now + MAX_FUTURE_SKEW_MS
   ) {
     throw new Error("timestamp is outside the accepted window");
   }
-  if (!Number.isSafeInteger(event.sequence) || event.sequence < 0 || event.sequence > 1_000_000_000) {
+  if (
+    !Number.isSafeInteger(event.sequence) ||
+    event.sequence < 0 ||
+    event.sequence > 1_000_000_000
+  ) {
     throw new Error("sequence is out of range");
   }
   if (event.revenueCents !== undefined && !isValidMoney(event.revenueCents)) {
@@ -130,9 +160,20 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     throw new Error("name is required for custom events");
   }
   if (event.type === "conversion" && !name) name = "conversion";
-  const currency = event.currency === undefined
-    ? event.type === "conversion" ? "USD" : undefined
-    : sanitizeCurrency(event.currency);
+  if (event.type === "vital") {
+    if (!name || !isVitalMetric(name)) {
+      throw new Error("vital name must be a Web Vitals metric");
+    }
+    if (event.value === undefined || !isValidVitalValue(event.value)) {
+      throw new Error("vital value is out of range");
+    }
+  }
+  const currency =
+    event.currency === undefined
+      ? event.type === "conversion"
+        ? "USD"
+        : undefined
+      : sanitizeCurrency(event.currency);
   return {
     eventId: sanitizeOpaqueId(event.eventId, "eventId"),
     sessionId: sanitizeOpaqueId(event.sessionId, "sessionId"),
@@ -140,19 +181,26 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     type: event.type,
     name,
     path: sanitizePath(event.path),
-    referrer: event.referrer === undefined
-      ? undefined
-      : sanitizePublicUrl(event.referrer),
+    referrer:
+      event.referrer === undefined
+        ? undefined
+        : sanitizePublicUrl(event.referrer),
     timestamp: Math.trunc(event.timestamp),
     sequence: event.sequence,
-    target: event.target === undefined
-      ? undefined
-      : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
+    target:
+      event.target === undefined
+        ? undefined
+        : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
     href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
     currency,
     affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
+    // Only measurements keep a value; a stray number on any other type is noise.
+    value:
+      event.type === "vital" && event.value !== undefined
+        ? Math.round(event.value)
+        : undefined,
   };
 }
 
@@ -167,7 +215,9 @@ function optionalAffiliateSlug(value: string | undefined): string | undefined {
 
 export function normalizeDomains(domains: string[]): string[] {
   if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
-    throw new Error(`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`);
+    throw new Error(
+      `domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`,
+    );
   }
   const normalized = domains.map(normalizeDomainPattern);
   return [...new Set(normalized)].sort();
@@ -185,7 +235,11 @@ export function normalizeDomainPattern(value: string): string {
       throw new Error("domains cannot include paths or query strings");
     }
     hostname = url.hostname;
-  } else if (hostInput.includes("/") || hostInput.includes("?") || hostInput.includes("#")) {
+  } else if (
+    hostInput.includes("/") ||
+    hostInput.includes("?") ||
+    hostInput.includes("#")
+  ) {
     throw new Error("domain is invalid");
   }
   hostname = hostname.replace(/\.$/, "");
@@ -214,7 +268,9 @@ export function originAllowed(origin: string, domains: string[]): boolean {
   });
 }
 
-export function validateTimezone(value: string | undefined): string | undefined {
+export function validateTimezone(
+  value: string | undefined,
+): string | undefined {
   if (value === undefined || value.trim() === "") return undefined;
   const timezone = cleanString(value, 80);
   try {
@@ -225,20 +281,33 @@ export function validateTimezone(value: string | undefined): string | undefined
   }
 }
 
-export function sanitizeContext(context: IngestContext | undefined): IngestContext {
+export function sanitizeContext(
+  context: IngestContext | undefined,
+): IngestContext {
   const latitude = finiteCoordinate(context?.latitude, -90, 90);
   const longitude = finiteCoordinate(context?.longitude, -180, 180);
   return {
-    country: context?.country ? cleanString(context.country.toUpperCase(), 2) || undefined : undefined,
-    city: context?.city ? cleanString(context.city, 80) || undefined : undefined,
-    latitude: latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
-    longitude: longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
-    browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
+    country: context?.country
+      ? cleanString(context.country.toUpperCase(), 2) || undefined
+      : undefined,
+    city: context?.city
+      ? cleanString(context.city, 80) || undefined
+      : undefined,
+    latitude:
+      latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
+    longitude:
+      longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
+    browser: context?.browser
+      ? cleanString(context.browser, 32) || undefined
+      : undefined,
     os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
-    device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
-    visitorKey: context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
-      ? cleanString(context.visitorKey, 128) || undefined
+    device: context?.device
+      ? cleanString(context.device, 24) || undefined
       : undefined,
+    visitorKey:
+      context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
+        ? cleanString(context.visitorKey, 128) || undefined
+        : undefined,
   };
 }
 
@@ -257,8 +326,8 @@ export function measureBatchBytes(
   events: TrackerEvent[],
 ): { batchBytes: number; eventByteCounts: number[] } {
   const encoder = new TextEncoder();
-  const eventByteCounts = events.map((event) =>
-    encoder.encode(JSON.stringify(event)).byteLength
+  const eventByteCounts = events.map(
+    (event) => encoder.encode(JSON.stringify(event)).byteLength,
   );
   // The empty envelope already includes both array brackets; only commas remain.
   const batchBytes =
@@ -294,6 +363,7 @@ function finiteCoordinate(
   maximum: number,
 ): number | undefined {
   if (value === undefined) return undefined;
-  if (!Number.isFinite(value) || value < minimum || value > maximum) return undefined;
+  if (!Number.isFinite(value) || value < minimum || value > maximum)
+    return undefined;
   return value;
 }

package/src/component/schema.ts

@@ -9,6 +9,7 @@ import {
   sessionFieldsValidator,
   siteFieldsValidator,
   visitorSketchValidator,
+  vitalMetricValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -131,11 +132,13 @@ export default defineSchema({
     bucketStart: v.number(),
     shard: v.number(),
     generation: v.literal(1),
-    steps: v.array(v.object({
-      step: v.number(),
-      entrants: v.number(),
-      completions: v.number(),
-    })),
+    steps: v.array(
+      v.object({
+        step: v.number(),
+        entrants: v.number(),
+        completions: v.number(),
+      }),
+    ),
   })
     .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
       "siteId",
@@ -150,6 +153,29 @@ export default defineSchema({
     ])
     .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
 
+  // Field Web Vitals. One row per site/day/shard and either a page (device
+  // "(all)") or a device class (page "(all)"); the row folds every metric's
+  // histogram together so a report reads five metrics for one document. Pages
+  // are bounded like dimension slots: at most VITAL_PAGE_SLOTS distinct pages
+  // per day, then "(other)".
+  vitalRollups: defineTable({
+    siteId: v.id("sites"),
+    bucketStart: v.number(),
+    device: v.string(),
+    page: v.string(),
+    shard: v.number(),
+    metrics: v.array(
+      v.object({
+        metric: vitalMetricValidator,
+        count: v.number(),
+        sum: v.number(),
+        histogram: v.array(v.number()),
+      }),
+    ),
+  })
+    .index("by_key", ["siteId", "bucketStart", "device", "page", "shard"])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
   affiliateDailyRollups: defineTable({
     siteId: v.id("sites"),
     affiliateId: v.id("affiliates"),

package/src/component/sites.ts

@@ -32,7 +32,8 @@ export const create = mutation({
   handler: async (ctx, args) => {
     const ownerId = cleanString(args.ownerId, 128);
     const name = cleanString(args.name, 120);
-    if (!ownerId || !name) fail("INVALID_SITE", "ownerId and name are required");
+    if (!ownerId || !name)
+      fail("INVALID_SITE", "ownerId and name are required");
 
     let domains: string[];
     let timezone: string | undefined;
@@ -46,7 +47,10 @@ export const create = mutation({
         : undefined;
       currency = sanitizeCurrency(args.currency ?? "USD");
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     const cookieless = args.cookieless ?? true;
 
@@ -65,7 +69,10 @@ export const create = mutation({
         (existing.currency ?? "USD") !== currency ||
         existing.cookieless !== cookieless
       ) {
-        fail("CONFLICT", "site name is already associated with different settings");
+        fail(
+          "CONFLICT",
+          "site name is already associated with different settings",
+        );
       }
       await ensureAnalyticsControl(ctx, existing._id, Date.now());
       return existing._id;
@@ -137,22 +144,26 @@ export const update = mutation({
       patch.name = name;
     }
     try {
-      if (args.domains !== undefined) patch.domains = normalizeDomains(args.domains);
+      if (args.domains !== undefined)
+        patch.domains = normalizeDomains(args.domains);
       if (args.timezone !== undefined) {
-        patch.timezone = args.timezone === null
-          ? undefined
-          : validateTimezone(args.timezone);
+        patch.timezone =
+          args.timezone === null ? undefined : validateTimezone(args.timezone);
       }
       if (args.networkId !== undefined) {
-        patch.networkId = args.networkId === null
-          ? undefined
-          : sanitizeOpaqueId(args.networkId, "networkId");
+        patch.networkId =
+          args.networkId === null
+            ? undefined
+            : sanitizeOpaqueId(args.networkId, "networkId");
       }
       if (args.currency !== undefined) {
         patch.currency = sanitizeCurrency(args.currency);
       }
     } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+      fail(
+        "INVALID_SITE",
+        error instanceof Error ? error.message : "invalid site",
+      );
     }
     if (
       patch.currency !== undefined &&

package/src/component/validators.ts

@@ -7,6 +7,15 @@ export const eventTypeValidator = v.union(
   v.literal("conversion"),
   v.literal("heartbeat"),
   v.literal("outbound"),
+  v.literal("vital"),
+);
+
+export const vitalMetricValidator = v.union(
+  v.literal("LCP"),
+  v.literal("CLS"),
+  v.literal("INP"),
+  v.literal("FCP"),
+  v.literal("TTFB"),
 );
 
 export const propertyValueValidator = v.union(
@@ -37,6 +46,9 @@ export const trackerEventValidator = v.object({
   revenueCents: v.optional(v.number()),
   currency: v.optional(v.string()),
   affiliateSlug: v.optional(v.string()),
+  // Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
+  // events carry it; sanitization strips it from every other type.
+  value: v.optional(v.number()),
 });
 
 export const ingestContextValidator = v.object({
@@ -50,12 +62,10 @@ export const ingestContextValidator = v.object({
   visitorKey: v.optional(v.string()),
 });
 
-export const batchedEventValidator = trackerEventValidator.extend(
-  {
-    ...ingestContextValidator.fields,
-    aggregateCountry: v.optional(v.string()),
-  },
-);
+export const batchedEventValidator = trackerEventValidator.extend({
+  ...ingestContextValidator.fields,
+  aggregateCountry: v.optional(v.string()),
+});
 
 export const siteFieldsValidator = v.object({
   ownerId: v.string(),
@@ -236,7 +246,6 @@ export const trustedConversionValidator = v.object({
   properties: v.optional(eventPropertiesValidator),
 });
 
-
 export type TrackerEvent = Infer<typeof trackerEventValidator>;
 export type IngestContext = Infer<typeof ingestContextValidator>;
 export type EventProperties = Infer<typeof eventPropertiesValidator>;

package/src/component/visitors.ts

@@ -59,14 +59,21 @@ export const link = mutation({
     const existing = await findAlias(ctx, args.siteId, previousVisitorId);
     if (existing) {
       if (existing.visitorId !== visitorId) {
-        fail("CONFLICT", "previousVisitorId is already linked to a different visitor");
+        fail(
+          "CONFLICT",
+          "previousVisitorId is already linked to a different visitor",
+        );
       }
       return { linked: false, aliasCount: aliases.length };
     }
     if (aliases.length >= MAX_VISITOR_ALIASES) {
-      fail("LIMIT_EXCEEDED", `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`, {
-        limit: MAX_VISITOR_ALIASES,
-      });
+      fail(
+        "LIMIT_EXCEEDED",
+        `a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`,
+        {
+          limit: MAX_VISITOR_ALIASES,
+        },
+      );
     }
     await ctx.db.insert("visitorAliases", {
       siteId: args.siteId,
@@ -95,7 +102,11 @@ export async function resolveVisitorIdentities(
   return [identity, ...aliases.map((row) => row.previousVisitorId)];
 }
 
-function findAlias(ctx: QueryCtx, siteId: Id<"sites">, previousVisitorId: string) {
+function findAlias(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  previousVisitorId: string,
+) {
   return ctx.db
     .query("visitorAliases")
     .withIndex("by_siteId_and_previousVisitorId", (range) =>

package/src/component/vitals.ts

@@ -0,0 +1,146 @@
+import { VITAL_HISTOGRAM_EDGES, MAX_VITAL_VALUE } from "./constants.js";
+
+/**
+ * Field-measured Web Vitals. Values are integers: milliseconds for the time
+ * metrics, and CLS scaled by 1000 so one shared histogram covers every metric.
+ */
+export const VITAL_METRICS = ["LCP", "CLS", "INP", "FCP", "TTFB"] as const;
+
+export type VitalMetric = (typeof VITAL_METRICS)[number];
+
+/**
+ * The Google-published rating thresholds. Every threshold is also a histogram
+ * edge, so good/needs-improvement/poor counts are exact, never interpolated.
+ */
+export const VITAL_THRESHOLDS: Record<
+  VitalMetric,
+  { good: number; poor: number }
+> = {
+  LCP: { good: 2_500, poor: 4_000 },
+  CLS: { good: 100, poor: 250 },
+  INP: { good: 200, poor: 500 },
+  FCP: { good: 1_800, poor: 3_000 },
+  TTFB: { good: 800, poor: 1_800 },
+};
+
+/** Sentinel row keys; sanitized paths always start with "/" so neither collides. */
+export const VITAL_ALL = "(all)";
+export const VITAL_OTHER_PAGES = "(other)";
+
+/** Bounded device classes; anything else folds into "unknown". */
+export const VITAL_DEVICES = ["desktop", "mobile", "tablet"] as const;
+
+export function vitalDevice(device: string | undefined): string {
+  return device && (VITAL_DEVICES as readonly string[]).includes(device)
+    ? device
+    : "unknown";
+}
+
+export const VITAL_HISTOGRAM_BUCKETS = VITAL_HISTOGRAM_EDGES.length + 1;
+
+export function isVitalMetric(value: string): value is VitalMetric {
+  return (VITAL_METRICS as readonly string[]).includes(value);
+}
+
+export function isValidVitalValue(value: number): boolean {
+  return Number.isFinite(value) && value >= 0 && value <= MAX_VITAL_VALUE;
+}
+
+export function emptyVitalHistogram(): number[] {
+  return Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
+}
+
+/** Bucket i holds values in (edge[i-1], edge[i]]; the last bucket is open. */
+export function vitalHistogramBucket(value: number): number {
+  for (let index = 0; index < VITAL_HISTOGRAM_EDGES.length; index += 1) {
+    if (value <= VITAL_HISTOGRAM_EDGES[index]) return index;
+  }
+  return VITAL_HISTOGRAM_EDGES.length;
+}
+
+export function addToVitalHistogram(
+  histogram: number[],
+  value: number,
+): number[] {
+  const next = normalizedVitalHistogram(histogram);
+  next[vitalHistogramBucket(value)] += 1;
+  return next;
+}
+
+export function mergeVitalHistograms(
+  left: number[],
+  right: number[],
+): number[] {
+  const result = normalizedVitalHistogram(left);
+  const addition = normalizedVitalHistogram(right);
+  for (let index = 0; index < result.length; index += 1) {
+    result[index] += addition[index];
+  }
+  return result;
+}
+
+/**
+ * Percentile estimated by linear interpolation inside the winning bucket. The
+ * top bucket is open-ended, so estimates saturate at the highest edge; a
+ * reported p75 equal to that edge means "at least this much".
+ */
+export function vitalHistogramPercentile(
+  histogram: number[],
+  fraction: number,
+): number {
+  const buckets = normalizedVitalHistogram(histogram);
+  const total = buckets.reduce((sum, count) => sum + count, 0);
+  if (total === 0) return 0;
+  const target = Math.ceil(total * fraction);
+  let cumulative = 0;
+  for (let index = 0; index < buckets.length; index += 1) {
+    const count = buckets[index];
+    if (count === 0) continue;
+    if (cumulative + count >= target) {
+      const lower = index === 0 ? 0 : VITAL_HISTOGRAM_EDGES[index - 1];
+      const upper =
+        index < VITAL_HISTOGRAM_EDGES.length
+          ? VITAL_HISTOGRAM_EDGES[index]
+          : VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
+      const position = (target - cumulative) / count;
+      return Math.round(Math.min(upper, lower + position * (upper - lower)));
+    }
+    cumulative += count;
+  }
+  return VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
+}
+
+/** Exact rating counts; every threshold is a bucket edge by construction. */
+export function vitalRatingCounts(
+  histogram: number[],
+  metric: VitalMetric,
+): { good: number; needsImprovement: number; poor: number } {
+  const buckets = normalizedVitalHistogram(histogram);
+  const thresholds = VITAL_THRESHOLDS[metric];
+  let good = 0;
+  let needsImprovement = 0;
+  let poor = 0;
+  for (let index = 0; index < buckets.length; index += 1) {
+    const upper =
+      index < VITAL_HISTOGRAM_EDGES.length
+        ? VITAL_HISTOGRAM_EDGES[index]
+        : Number.POSITIVE_INFINITY;
+    if (upper <= thresholds.good) good += buckets[index];
+    else if (upper <= thresholds.poor) needsImprovement += buckets[index];
+    else poor += buckets[index];
+  }
+  return { good, needsImprovement, poor };
+}
+
+function normalizedVitalHistogram(histogram: number[]): number[] {
+  const result = Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
+  for (
+    let index = 0;
+    index < Math.min(histogram.length, result.length);
+    index += 1
+  ) {
+    const count = histogram[index];
+    result[index] = Number.isSafeInteger(count) && count > 0 ? count : 0;
+  }
+  return result;
+}