@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/src/component/_generated/component.ts

@@ -185,6 +185,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             latitude?: number;
             longitude?: number;
             os?: string;
+            visitorKey?: string;
           };
           events: Array<{
             affiliateSlug?: string;
@@ -206,7 +207,9 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "custom"
               | "conversion"
               | "heartbeat"
-              | "outbound";
+              | "outbound"
+              | "vital";
+            value?: number;
             visitorId: string;
           }>;
           origin: string;
@@ -302,7 +305,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "conversions"
               | "goals"
               | "funnels"
-              | "affiliates";
+              | "affiliates"
+              | "vitals";
           }>;
           from: number;
           siteId: string;
@@ -488,6 +492,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             utmMedium?: string;
             utmSource?: string;
             visitorId: string;
+            visitorKey?: string;
           }>;
           pageStatus?: "SplitRecommended" | "SplitRequired" | null;
           splitCursor?: string | null;
@@ -537,6 +542,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           metadata: {
             breakdowns: {
               topBrowsers: "eventVolume";
+              topCampaigns: "eventVolume";
               topCountries: "eventVolume";
               topDevices: "eventVolume";
               topEvents: "eventVolume";
@@ -552,7 +558,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               state: "complete" | "partial" | "retained" | "unavailable";
             };
             features: {
-              botDetection: "unsupported";
+              botDetection: "userAgent";
               errorInsights: "unsupported";
               outboundLinks: "eventVolume";
               sessionReplay: "unsupported";
@@ -589,6 +595,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topCampaigns: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topCountries: Array<{
             count: number;
             revenueCents: number;
@@ -683,7 +694,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "custom"
               | "conversion"
               | "heartbeat"
-              | "outbound";
+              | "outbound"
+              | "vital";
             visitorId: string;
           }>;
           pageStatus?: "SplitRecommended" | "SplitRequired" | null;
@@ -731,11 +743,69 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "custom"
             | "conversion"
             | "heartbeat"
-            | "outbound";
+            | "outbound"
+            | "vital";
           visitorId: string;
         }>,
         Name
       >;
+      vitalsReport: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteId: string; to: number },
+        {
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
+          metrics: Array<{
+            average: number;
+            count: number;
+            devices: Array<{
+              average: number;
+              count: number;
+              device: string;
+              good: number;
+              needsImprovement: number;
+              p75: number;
+              poor: number;
+            }>;
+            good: number;
+            metric: "LCP" | "CLS" | "INP" | "FCP" | "TTFB";
+            needsImprovement: number;
+            p75: number;
+            pages: Array<{
+              average: number;
+              count: number;
+              good: number;
+              needsImprovement: number;
+              p75: number;
+              page: string;
+              poor: number;
+            }>;
+            poor: number;
+            timeSeries: Array<{
+              count: number;
+              p75: number;
+              timestamp: number;
+            }>;
+          }>;
+          truncated: boolean;
+        },
+        Name
+      >;
     };
     retention: {
       cleanup: FunctionReference<
@@ -757,6 +827,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "goalRollups"
             | "funnelRollups"
             | "affiliateRollups"
+            | "vitalRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
@@ -791,6 +862,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "goalRollups"
             | "funnelRollups"
             | "affiliateRollups"
+            | "vitalRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
@@ -836,6 +908,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "goalRollups"
               | "funnelRollups"
               | "affiliateRollups"
+              | "vitalRollups"
               | "attributions"
               | "liveSessions"
               | "ingestWindows"
@@ -926,4 +999,13 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    visitors: {
+      link: FunctionReference<
+        "mutation",
+        "internal",
+        { previousVisitorId: string; siteId: string; visitorId: string },
+        { aliasCount: number; linked: boolean },
+        Name
+      >;
+    };
   };

package/src/component/affiliates.ts

@@ -22,16 +22,27 @@ export const upsert = mutation({
   },
   returns: v.id("affiliates"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
-    if (!Number.isSafeInteger(args.commissionBps) || args.commissionBps < 0 || args.commissionBps > 10_000) {
-      fail("INVALID_ARGUMENT", "commissionBps must be an integer between 0 and 10000");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
+    if (
+      !Number.isSafeInteger(args.commissionBps) ||
+      args.commissionBps < 0 ||
+      args.commissionBps > 10_000
+    ) {
+      fail(
+        "INVALID_ARGUMENT",
+        "commissionBps must be an integer between 0 and 10000",
+      );
     }
     if (
       !Number.isSafeInteger(args.attributionWindowDays) ||
       args.attributionWindowDays < 1 ||
       args.attributionWindowDays > 365
     ) {
-      fail("INVALID_ARGUMENT", "attributionWindowDays must be an integer between 1 and 365");
+      fail(
+        "INVALID_ARGUMENT",
+        "attributionWindowDays must be an integer between 1 and 365",
+      );
     }
     let slug: string;
     try {
@@ -52,7 +63,11 @@ export const upsert = mutation({
       )
       .unique();
     const affiliateId = args.affiliateId ?? existingBySlug?._id;
-    if (args.affiliateId && existingBySlug && existingBySlug._id !== args.affiliateId) {
+    if (
+      args.affiliateId &&
+      existingBySlug &&
+      existingBySlug._id !== args.affiliateId
+    ) {
       fail("CONFLICT", "another affiliate already uses this slug");
     }
     const now = Date.now();

package/src/component/cardinality.ts

@@ -60,13 +60,14 @@ export function unionVisitorSketches(
 export function estimateVisitors(sketch: VisitorSketch): number {
   const registers = checkedRegisters(sketch);
   const count = registers.length;
-  const alpha = count === 16
-    ? 0.673
-    : count === 32
-      ? 0.697
-      : count === 64
-        ? 0.709
-        : 0.7213 / (1 + 1.079 / count);
+  const alpha =
+    count === 16
+      ? 0.673
+      : count === 32
+        ? 0.697
+        : count === 64
+          ? 0.709
+          : 0.7213 / (1 + 1.079 / count);
   let denominator = 0;
   let empty = 0;
   for (const register of registers) {

package/src/component/constants.ts

@@ -36,8 +36,20 @@ export const FEATURE_REPORT_DOCUMENT_RESERVE = 500;
 export const REPORT_ROLLUP_SHARDS = 2;
 export const ROLLUP_SAFETY_DELAY_MS = 5 * 60_000;
 export const RETENTION_LEASE_MS = 60_000;
+// Shared Web Vitals histogram edges. Every Google rating threshold (LCP
+// 2500/4000, CLS×1000 100/250, INP 200/500, FCP 1800/3000, TTFB 800/1800) is
+// an edge, so rating counts stay exact while one layout serves all metrics.
+export const VITAL_HISTOGRAM_EDGES = [
+  100, 200, 250, 300, 500, 800, 1_000, 1_200, 1_800, 2_500, 3_000, 4_000, 5_000,
+  7_500, 10_000,
+];
+export const MAX_VITAL_VALUE = 600_000;
+export const VITAL_PAGE_SLOTS = 48;
+export const MAX_VITALS_REPORT_ROWS = 5_000;
+export const VITALS_REPORT_PAGE_LIMIT = 12;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
 export const MINUTE_MS = 60_000;
 export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;
+export const MAX_VISITOR_ALIASES = 10;

package/src/component/diagnostics.ts

@@ -29,7 +29,8 @@ export const sessionEventTotals = query({
     try {
       sessionIds = args.sessionIds.map((sessionId) => {
         const sanitized = sanitizeOpaqueId(sessionId, "sessionId");
-        if (sanitized !== sessionId) throw new Error("sessionId is not canonical");
+        if (sanitized !== sessionId)
+          throw new Error("sessionId is not canonical");
         return sanitized;
       });
     } catch (error) {
@@ -49,7 +50,7 @@ export const sessionEventTotals = query({
           .withIndex("by_siteId_and_sessionId", (range) =>
             range.eq("siteId", args.siteId).eq("sessionId", sessionId),
           )
-          .unique()
+          .unique(),
       ),
     );
     return {

package/src/component/eventStore.ts

@@ -90,8 +90,8 @@ export async function findTelemetry(
   let batch: Doc<"eventBatches"> | undefined;
   let nestedEvent: Doc<"eventBatches">["events"][number] | undefined;
   for (const candidate of batches) {
-    nestedEvent = candidate.events.find((nested) =>
-      nested.eventId === event.eventId
+    nestedEvent = candidate.events.find(
+      (nested) => nested.eventId === event.eventId,
     );
     if (nestedEvent) {
       batch = candidate;
@@ -99,13 +99,14 @@ export async function findTelemetry(
     }
   }
   return {
-    match: batch && nestedEvent
-      ? {
-        sessionId: batch.sessionId,
-        visitorId: batch.visitorId,
-        ...eventContext(nestedEvent),
-      }
-      : null,
+    match:
+      batch && nestedEvent
+        ? {
+            sessionId: batch.sessionId,
+            visitorId: batch.visitorId,
+            ...eventContext(nestedEvent),
+          }
+        : null,
     saturated,
   };
 }
@@ -190,17 +191,23 @@ export async function loadSessionJourneyEvents(
       )
       .order("asc");
     for await (const batch of query) {
-      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+      if (
+        completedBucket !== undefined &&
+        batch.bucketStart !== completedBucket
+      ) {
         break;
       }
       batches.push(batch);
       qualifyingEvents += batch.events.reduce((count, _event, index) => {
         const event = eventFromBatch(batch, index);
-        return count + Number(
-          event.timestamp >= input.from &&
-            event.timestamp <= input.to &&
-            (!input.cursor ||
-              compareJourneyEventToCursor(event, input.cursor) > 0),
+        return (
+          count +
+          Number(
+            event.timestamp >= input.from &&
+              event.timestamp <= input.to &&
+              (!input.cursor ||
+                compareJourneyEventToCursor(event, input.cursor) > 0),
+          )
         );
       }, 0);
       if (qualifyingEvents >= input.resultLimit) {
@@ -213,10 +220,11 @@ export async function loadSessionJourneyEvents(
   const rows: RawJourneyEvent[] = [];
   appendBatchEvents(rows, batches);
   return rows
-    .filter((event) =>
-      event.timestamp >= input.from &&
-      event.timestamp <= input.to &&
-      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+    .filter(
+      (event) =>
+        event.timestamp >= input.from &&
+        event.timestamp <= input.to &&
+        (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0),
     )
     .sort(compareJourneyEvents)
     .slice(0, input.resultLimit);
@@ -249,13 +257,16 @@ export async function loadVisitorJourneyEvents(
       )
       .order("asc");
     for await (const batch of query) {
-      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+      if (
+        completedBucket !== undefined &&
+        batch.bucketStart !== completedBucket
+      ) {
         break;
       }
       batches.push(batch);
       await requireJourneyReadHeadroom(ctx);
-      qualifyingEvents += batch.events.filter((event) =>
-        event.timestamp >= input.from && event.timestamp <= input.to
+      qualifyingEvents += batch.events.filter(
+        (event) => event.timestamp >= input.from && event.timestamp <= input.to,
       ).length;
       if (qualifyingEvents >= input.resultLimit) {
         completedBucket = batch.bucketStart;
@@ -267,8 +278,8 @@ export async function loadVisitorJourneyEvents(
   const rows: RawJourneyEvent[] = [];
   appendBatchEvents(rows, batches);
   return rows
-    .filter((event) =>
-      event.timestamp >= input.from && event.timestamp <= input.to
+    .filter(
+      (event) => event.timestamp >= input.from && event.timestamp <= input.to,
     )
     .sort(compareJourneyEvents)
     .slice(0, input.resultLimit);
@@ -278,9 +289,11 @@ export function compareJourneyEvents(
   left: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   right: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
 ): number {
-  return left.timestamp - right.timestamp ||
+  return (
+    left.timestamp - right.timestamp ||
     left._creationTime - right._creationTime ||
-    left._id.localeCompare(right._id);
+    left._id.localeCompare(right._id)
+  );
 }
 
 export async function hydrateEventContexts(
@@ -304,8 +317,8 @@ export async function hydrateEventContexts(
   }
   const sessions = new Map(
     await Promise.all(
-      [...sessionPromises].map(async ([key, session]) =>
-        [key, await session] as const
+      [...sessionPromises].map(
+        async ([key, session]) => [key, await session] as const,
       ),
     ),
   );
@@ -373,13 +386,8 @@ async function loadTelemetryMinute(
     .take(MAX_EVENTS_PER_SESSION_WINDOW + 1);
 }
 
-function telemetryMinuteCount(
-  batches: Array<{ events: TrackerEvent[] }>,
-) {
-  return batches.reduce(
-    (total, batch) => total + batch.events.length,
-    0,
-  );
+function telemetryMinuteCount(batches: Array<{ events: TrackerEvent[] }>) {
+  return batches.reduce((total, batch) => total + batch.events.length, 0);
 }
 
 function eventForBatch(
@@ -423,7 +431,8 @@ function eventFromBatch(
   batch: Doc<"eventBatches">,
   index: number,
 ): RawJourneyEvent {
-  const { aggregateCountry: _aggregateCountry, ...event } = batch.events[index]!;
+  const { aggregateCountry: _aggregateCountry, ...event } =
+    batch.events[index]!;
   void _aggregateCountry;
   return {
     ...event,
@@ -461,14 +470,10 @@ async function requireJourneyReadHeadroom(ctx: QueryCtx) {
     metrics.bytesRead.remaining < JOURNEY_READ_BYTE_RESERVE ||
     metrics.documentsRead.remaining < JOURNEY_READ_DOCUMENT_RESERVE
   ) {
-    fail(
-      "LIMIT_EXCEEDED",
-      "visitor journey exceeds the safe read budget",
-      {
-        bytesRead: metrics.bytesRead.used,
-        documentsRead: metrics.documentsRead.used,
-      },
-    );
+    fail("LIMIT_EXCEEDED", "visitor journey exceeds the safe read budget", {
+      bytesRead: metrics.bytesRead.used,
+      documentsRead: metrics.documentsRead.used,
+    });
   }
 }
 
@@ -476,7 +481,9 @@ function compareJourneyEventToCursor(
   event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
   cursor: JourneyCursor,
 ) {
-  return event.timestamp - cursor.timestamp ||
+  return (
+    event.timestamp - cursor.timestamp ||
     event._creationTime - cursor.creationTime ||
-    event._id.localeCompare(cursor.id);
+    event._id.localeCompare(cursor.id)
+  );
 }

package/src/component/funnels.ts

@@ -1,15 +1,9 @@
 import { v } from "convex/values";
 import { mutation, query } from "./_generated/server.js";
-import {
-  MAX_FUNNELS_PER_SITE,
-  MAX_FUNNEL_STEPS,
-} from "./constants.js";
+import { MAX_FUNNELS_PER_SITE, MAX_FUNNEL_STEPS } from "./constants.js";
 import { fail } from "./errors.js";
 import { cleanString, sanitizePath, sanitizeSlug } from "./sanitize.js";
-import {
-  funnelFieldsValidator,
-  funnelStepValidator,
-} from "./validators.js";
+import { funnelFieldsValidator, funnelStepValidator } from "./validators.js";
 
 const funnelDocumentValidator = funnelFieldsValidator.extend({
   _id: v.id("funnels"),
@@ -28,7 +22,8 @@ export const upsert = mutation({
   },
   returns: v.id("funnels"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
     if (args.steps.length < 2 || args.steps.length > MAX_FUNNEL_STEPS) {
       fail("INVALID_ARGUMENT", `funnels require 2-${MAX_FUNNEL_STEPS} steps`);
     }
@@ -37,7 +32,10 @@ export const upsert = mutation({
       args.conversionWindowMs < 60_000 ||
       args.conversionWindowMs > 90 * 86_400_000
     ) {
-      fail("INVALID_ARGUMENT", "conversionWindowMs must be between one minute and 90 days");
+      fail(
+        "INVALID_ARGUMENT",
+        "conversionWindowMs must be between one minute and 90 days",
+      );
     }
     let key: string;
     let steps: Array<{
@@ -49,9 +47,10 @@ export const upsert = mutation({
       key = sanitizeSlug(args.key, "key");
       steps = args.steps.map((step) => {
         const stepName = cleanString(step.name, 80);
-        const value = step.type === "pageview"
-          ? sanitizePath(step.value)
-          : cleanString(step.value, 80);
+        const value =
+          step.type === "pageview"
+            ? sanitizePath(step.value)
+            : cleanString(step.value, 80);
         if (!stepName || !value) {
           throw new Error("funnel steps require name and value");
         }
@@ -79,7 +78,8 @@ export const upsert = mutation({
     const now = Date.now();
     if (funnelId) {
       const funnel = await ctx.db.get("funnels", funnelId);
-      if (!funnel || funnel.siteId !== args.siteId) fail("NOT_FOUND", "funnel not found");
+      if (!funnel || funnel.siteId !== args.siteId)
+        fail("NOT_FOUND", "funnel not found");
       await ctx.db.patch("funnels", funnelId, {
         key,
         name,
@@ -96,7 +96,9 @@ export const upsert = mutation({
       .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
       .take(MAX_FUNNELS_PER_SITE + 1);
     if (funnels.length >= MAX_FUNNELS_PER_SITE) {
-      fail("LIMIT_EXCEEDED", "funnel limit reached", { limit: MAX_FUNNELS_PER_SITE });
+      fail("LIMIT_EXCEEDED", "funnel limit reached", {
+        limit: MAX_FUNNELS_PER_SITE,
+      });
     }
     return await ctx.db.insert("funnels", {
       siteId: args.siteId,
@@ -127,7 +129,8 @@ export const remove = mutation({
   returns: v.null(),
   handler: async (ctx, args) => {
     const funnel = await ctx.db.get("funnels", args.funnelId);
-    if (!funnel || funnel.siteId !== args.siteId) fail("NOT_FOUND", "funnel not found");
+    if (!funnel || funnel.siteId !== args.siteId)
+      fail("NOT_FOUND", "funnel not found");
     await ctx.db.delete("funnels", args.funnelId);
     return null;
   },

package/src/component/geo.ts

@@ -15,10 +15,7 @@ import {
 import { eventBatchKey, eventBucketStart } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
 import { sanitizeContext, stableHash } from "./sanitize.js";
-import {
-  trackerEventValidator,
-  type IngestContext,
-} from "./validators.js";
+import { trackerEventValidator, type IngestContext } from "./validators.js";
 
 export type GeoIpProvider = "ipinfo" | "ipwhois";
 export type GeoIpStatus = "disabled" | "ready" | "missingToken";
@@ -153,10 +150,13 @@ export const claimEnrichment = internalMutation({
             batches.flatMap((batch) =>
               batch.events
                 .filter((event) => group.eventIds.has(event.eventId))
-                .map((event) => [
-                  event.eventId,
-                  event.aggregateCountry ?? event.country,
-                ] as const),
+                .map(
+                  (event) =>
+                    [
+                      event.eventId,
+                      event.aggregateCountry ?? event.country,
+                    ] as const,
+                ),
             ),
           ),
         );
@@ -237,16 +237,16 @@ export const claimEnrichment = internalMutation({
         seenEvents.has(key) ||
         !claimed.has(event.sessionId) ||
         !persistedEvents
-          .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
+          .get(
+            eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)),
+          )
           ?.has(event.eventId)
       ) {
         continue;
       }
       seenEvents.add(key);
       const priorCountry = persistedEvents
-        .get(
-          eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)),
-        )
+        .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
         ?.get(event.eventId);
       if (priorCountry === undefined) continue;
       correctionEvents.push({
@@ -469,7 +469,8 @@ function classifyIpv6(ip: string): Exclude<IpKind, "unavailable"> {
     (groups[0]! & 0xffc0) === 0xfe80 ||
     (groups[0]! & 0xffc0) === 0xfec0 ||
     (groups[0]! & 0xff00) === 0xff00 ||
-    (groups[0] === 0x0100 && groups.slice(1, 4).every((group) => group === 0)) ||
+    (groups[0] === 0x0100 &&
+      groups.slice(1, 4).every((group) => group === 0)) ||
     (groups[0] === 0x0064 && groups[1] === 0xff9b && groups[2] === 1) ||
     (groups[0] === 0x2001 && (groups[1]! & 0xfe00) === 0) ||
     (groups[0] === 0x2001 && groups[1] === 0x0db8) ||
@@ -563,7 +564,8 @@ async function correctCountryAggregates(
     for (const granularity of ["hour", "day"] as const) {
       const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
       const bucketStart = Math.floor(event.timestamp / interval) * interval;
-      const shard = stableHash(event.sessionId) %
+      const shard =
+        stableHash(event.sessionId) %
         (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
       const key = `${granularity}:${bucketStart}:${shard}`;
       const correction = corrections.get(key) ?? {