@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/src/component/_generated/component.ts

@@ -65,6 +65,19 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    diagnostics: {
+      sessionEventTotals: FunctionReference<
+        "query",
+        "internal",
+        { sessionIds: Array<string>; siteId: string },
+        {
+          foundSessions: number;
+          requestedSessions: number;
+          storedEvents: number;
+        },
+        Name
+      >;
+    };
     funnels: {
       list: FunctionReference<
         "query",
@@ -172,6 +185,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             latitude?: number;
             longitude?: number;
             os?: string;
+            visitorKey?: string;
           };
           events: Array<{
             affiliateSlug?: string;
@@ -193,7 +207,10 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "custom"
               | "conversion"
               | "heartbeat"
-              | "outbound";
+              | "leave"
+              | "outbound"
+              | "vital";
+            value?: number;
             visitorId: string;
           }>;
           origin: string;
@@ -244,10 +261,61 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             visits: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            uniqueVisitors: {
+              basis: "estimatedUnique" | "exactLegacy";
+              exact: boolean;
+            };
+          };
           truncated: boolean;
         },
         Name
       >;
+      dataCoverage: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteId: string; to: number },
+        {
+          datasets: Array<{
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            dataset:
+              | "overviewHour"
+              | "overviewDay"
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goals"
+              | "funnels"
+              | "affiliates"
+              | "vitals"
+              | "siteMap";
+          }>;
+          from: number;
+          siteId: string;
+          to: number;
+        },
+        Name
+      >;
       funnelsReport: FunctionReference<
         "query",
         "internal",
@@ -259,6 +327,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           conversionWindowMs: number;
           createdAt: number;
           key: string;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           name: string;
           siteId: string;
           steps: Array<{
@@ -296,6 +380,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             valueCents: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           truncated: boolean;
         },
         Name
@@ -394,6 +494,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             utmMedium?: string;
             utmSource?: string;
             visitorId: string;
+            visitorKey?: string;
           }>;
           pageStatus?: "SplitRecommended" | "SplitRequired" | null;
           splitCursor?: string | null;
@@ -416,9 +517,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           lastSeenAt: number;
           latitude?: number;
           longitude?: number;
+          movedAt?: number;
           os: string;
           pageviewCount: number;
           path: string;
+          previousPath?: string;
           sessionId: string;
           siteId: string;
           source: string;
@@ -440,6 +543,44 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           currency: string;
           from: number;
           interval: "hour" | "day";
+          metadata: {
+            breakdowns: {
+              topBrowsers: "eventVolume";
+              topCampaigns: "eventVolume";
+              topCountries: "eventVolume";
+              topDevices: "eventVolume";
+              topEvents: "eventVolume";
+              topGoals: "goalCompletionVolume";
+              topPages: "eventVolume";
+              topSources: "eventVolume";
+            };
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            features: {
+              botDetection: "userAgent";
+              errorInsights: "unsupported";
+              outboundLinks: "eventVolume";
+              sessionReplay: "unsupported";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            visitors: {
+              basis: "estimatedUnique" | "bucketOccurrences";
+              exact: boolean;
+              legacyIncomplete: boolean;
+              precision: number | null;
+              sketchVersion: number | null;
+            };
+          };
           timeSeries: Array<{
             clicks: number;
             conversions: number;
@@ -458,6 +599,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topCampaigns: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topCountries: Array<{
             count: number;
             revenueCents: number;
@@ -552,7 +698,9 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "custom"
               | "conversion"
               | "heartbeat"
-              | "outbound";
+              | "leave"
+              | "outbound"
+              | "vital";
             visitorId: string;
           }>;
           pageStatus?: "SplitRecommended" | "SplitRequired" | null;
@@ -560,6 +708,54 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         },
         Name
       >;
+      siteMap: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteIds: Array<string>; to: number },
+        {
+          from: number;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            limits: {
+              routesPerDay: number;
+              rows: number;
+              transitionSlots: number;
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            visitors: { basis: "estimatedUnique"; exact: boolean };
+          };
+          routes: Array<{
+            bounces: number;
+            daily: Array<{ bucketStart: number; pageviews: number }>;
+            dwellMs: number;
+            dwellSamples: number;
+            entries: number;
+            exits: number;
+            pageviews: number;
+            route: string;
+            transitionsIn: number;
+            transitionsOut: number;
+            visitors: number;
+          }>;
+          to: number;
+          transitions: Array<{ count: number; from: string; to: string }>;
+          truncated: { routes: boolean; transitions: boolean };
+          window: { from: number; to: number } | null;
+        },
+        Name
+      >;
       visitorJourney: FunctionReference<
         "query",
         "internal",
@@ -600,11 +796,70 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "custom"
             | "conversion"
             | "heartbeat"
-            | "outbound";
+            | "leave"
+            | "outbound"
+            | "vital";
           visitorId: string;
         }>,
         Name
       >;
+      vitalsReport: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteId: string; to: number },
+        {
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
+          metrics: Array<{
+            average: number;
+            count: number;
+            devices: Array<{
+              average: number;
+              count: number;
+              device: string;
+              good: number;
+              needsImprovement: number;
+              p75: number;
+              poor: number;
+            }>;
+            good: number;
+            metric: "LCP" | "CLS" | "INP" | "FCP" | "TTFB";
+            needsImprovement: number;
+            p75: number;
+            pages: Array<{
+              average: number;
+              count: number;
+              good: number;
+              needsImprovement: number;
+              p75: number;
+              page: string;
+              poor: number;
+            }>;
+            poor: number;
+            timeSeries: Array<{
+              count: number;
+              p75: number;
+              timestamp: number;
+            }>;
+          }>;
+          truncated: boolean;
+        },
+        Name
+      >;
     };
     retention: {
       cleanup: FunctionReference<
@@ -623,13 +878,111 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "funnelProgress"
             | "funnelSteps"
             | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
+            | "vitalRollups"
+            | "siteMapRollups"
+            | "attributions"
+            | "liveSessions"
+            | "ingestWindows"
+            | "siteIngestWindows";
+          siteId: string;
+        },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      disablePolicy: FunctionReference<
+        "mutation",
+        "internal",
+        { siteId: string },
+        null,
+        Name
+      >;
+      runPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          batchSize?: number;
+          kind:
+            | "events"
+            | "sessions"
+            | "conversions"
+            | "goalCompletions"
+            | "aggregateHour"
+            | "aggregateDay"
+            | "funnelProgress"
+            | "funnelSteps"
+            | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
+            | "vitalRollups"
+            | "siteMapRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
             | "siteIngestWindows";
           siteId: string;
         },
-        { deleted: number; hasMore: boolean },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      setPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          aggregatesDays: number;
+          conversionsDays: number;
+          eventsDays: number;
+          reportsDays: number;
+          sessionsDays: number;
+          siteId: string;
+        },
+        null,
+        Name
+      >;
+      status: FunctionReference<
+        "query",
+        "internal",
+        { siteId: string },
+        {
+          backlog: number;
+          jobs: Array<{
+            before: number;
+            deleted: number;
+            kind:
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goalCompletions"
+              | "aggregateHour"
+              | "aggregateDay"
+              | "funnelProgress"
+              | "funnelSteps"
+              | "affiliateStats"
+              | "goalRollups"
+              | "funnelRollups"
+              | "affiliateRollups"
+              | "vitalRollups"
+              | "siteMapRollups"
+              | "attributions"
+              | "liveSessions"
+              | "ingestWindows"
+              | "siteIngestWindows";
+            scheduledAt: number | null;
+            status: "pending" | "running" | "complete";
+            updatedAt: number;
+          }>;
+          policy: {
+            aggregatesDays: number;
+            conversionsDays: number;
+            eventsDays: number;
+            reportsDays: number;
+            sessionsDays: number;
+            updatedAt: number;
+          } | null;
+        },
         Name
       >;
     };
@@ -703,4 +1056,13 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    visitors: {
+      link: FunctionReference<
+        "mutation",
+        "internal",
+        { previousVisitorId: string; siteId: string; visitorId: string },
+        { aliasCount: number; linked: boolean },
+        Name
+      >;
+    };
   };

package/src/component/_generated/server.ts

@@ -39,6 +39,10 @@ import type { DataModel } from "./dataModel.js";
 type Env = {
   readonly CONVEX_CLOUD_URL: string;
   readonly CONVEX_SITE_URL: string;
+  readonly RASTRO_GEOIP_DAILY_LIMIT: string | undefined;
+  readonly RASTRO_GEOIP_PROVIDER: "ipinfo" | "ipwhois" | undefined;
+  readonly RASTRO_GEOIP_TOKEN: string | undefined;
+  readonly RASTRO_TRUST_PROXY: "true" | undefined;
 };
 
 /**

package/src/component/affiliates.ts

@@ -22,16 +22,27 @@ export const upsert = mutation({
   },
   returns: v.id("affiliates"),
   handler: async (ctx, args) => {
-    if (!(await ctx.db.get("sites", args.siteId))) fail("NOT_FOUND", "site not found");
-    if (!Number.isSafeInteger(args.commissionBps) || args.commissionBps < 0 || args.commissionBps > 10_000) {
-      fail("INVALID_ARGUMENT", "commissionBps must be an integer between 0 and 10000");
+    if (!(await ctx.db.get("sites", args.siteId)))
+      fail("NOT_FOUND", "site not found");
+    if (
+      !Number.isSafeInteger(args.commissionBps) ||
+      args.commissionBps < 0 ||
+      args.commissionBps > 10_000
+    ) {
+      fail(
+        "INVALID_ARGUMENT",
+        "commissionBps must be an integer between 0 and 10000",
+      );
     }
     if (
       !Number.isSafeInteger(args.attributionWindowDays) ||
       args.attributionWindowDays < 1 ||
       args.attributionWindowDays > 365
     ) {
-      fail("INVALID_ARGUMENT", "attributionWindowDays must be an integer between 1 and 365");
+      fail(
+        "INVALID_ARGUMENT",
+        "attributionWindowDays must be an integer between 1 and 365",
+      );
     }
     let slug: string;
     try {
@@ -52,7 +63,11 @@ export const upsert = mutation({
       )
       .unique();
     const affiliateId = args.affiliateId ?? existingBySlug?._id;
-    if (args.affiliateId && existingBySlug && existingBySlug._id !== args.affiliateId) {
+    if (
+      args.affiliateId &&
+      existingBySlug &&
+      existingBySlug._id !== args.affiliateId
+    ) {
       fail("CONFLICT", "another affiliate already uses this slug");
     }
     const now = Date.now();

package/src/component/cardinality.ts

@@ -0,0 +1,117 @@
+const SKETCH_VERSION = 1 as const;
+const DEFAULT_PRECISION = 6;
+
+export type VisitorSketch = {
+  version: typeof SKETCH_VERSION;
+  precision: number;
+  registers: ArrayBuffer;
+};
+
+export function createVisitorSketch(
+  precision = DEFAULT_PRECISION,
+): VisitorSketch {
+  if (!Number.isSafeInteger(precision) || precision < 4 || precision > 10) {
+    throw new Error("visitor sketch precision must be between 4 and 10");
+  }
+  return {
+    version: SKETCH_VERSION,
+    precision,
+    registers: new Uint8Array(1 << precision).buffer,
+  };
+}
+
+export function addVisitor(
+  sketch: VisitorSketch,
+  visitorId: string,
+): VisitorSketch {
+  const registers = checkedRegisters(sketch).slice();
+  const hash = fnv1a64(visitorId);
+  const remainingBits = 64 - sketch.precision;
+  const index = Number(hash >> BigInt(remainingBits));
+  const remaining = hash & ((1n << BigInt(remainingBits)) - 1n);
+  const rank = leadingZeroRank(remaining, remainingBits);
+  registers[index] = Math.max(registers[index] ?? 0, rank);
+  return { ...sketch, registers: registers.buffer };
+}
+
+export function unionVisitorSketches(
+  left: VisitorSketch,
+  right: VisitorSketch,
+): VisitorSketch {
+  const leftRegisters = checkedRegisters(left);
+  const rightRegisters = checkedRegisters(right);
+  if (
+    left.version !== right.version ||
+    left.precision !== right.precision ||
+    leftRegisters.length !== rightRegisters.length
+  ) {
+    throw new Error("visitor sketches are incompatible");
+  }
+  const registers = leftRegisters.slice();
+  for (let index = 0; index < registers.length; index += 1) {
+    registers[index] = Math.max(
+      registers[index] ?? 0,
+      rightRegisters[index] ?? 0,
+    );
+  }
+  return { ...left, registers: registers.buffer };
+}
+
+export function estimateVisitors(sketch: VisitorSketch): number {
+  const registers = checkedRegisters(sketch);
+  const count = registers.length;
+  const alpha =
+    count === 16
+      ? 0.673
+      : count === 32
+        ? 0.697
+        : count === 64
+          ? 0.709
+          : 0.7213 / (1 + 1.079 / count);
+  let denominator = 0;
+  let empty = 0;
+  for (const register of registers) {
+    denominator += 2 ** -register;
+    if (register === 0) empty += 1;
+  }
+  let estimate = (alpha * count * count) / denominator;
+  if (estimate <= 2.5 * count && empty > 0) {
+    estimate = count * Math.log(count / empty);
+  }
+  return Math.max(0, Math.round(estimate));
+}
+
+function checkedRegisters(sketch: VisitorSketch): Uint8Array<ArrayBuffer> {
+  if (sketch.version !== SKETCH_VERSION) {
+    throw new Error("unsupported visitor sketch version");
+  }
+  const registers = new Uint8Array(sketch.registers);
+  if (registers.length !== 1 << sketch.precision) {
+    throw new Error("visitor sketch has invalid register storage");
+  }
+  return registers;
+}
+
+function leadingZeroRank(value: bigint, bits: number): number {
+  if (value === 0n) return bits + 1;
+  let rank = 1;
+  for (let bit = bits - 1; bit >= 0; bit -= 1) {
+    if ((value & (1n << BigInt(bit))) !== 0n) break;
+    rank += 1;
+  }
+  return rank;
+}
+
+function fnv1a64(value: string): bigint {
+  let hash = 0xcbf29ce484222325n;
+  for (const byte of new TextEncoder().encode(value)) {
+    hash ^= BigInt(byte);
+    hash = BigInt.asUintN(64, hash * 0x100000001b3n);
+  }
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xff51afd7ed558ccdn);
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xc4ceb9fe1a85ec53n);
+  hash ^= hash >> 33n;
+  return hash;
+}