@iann29/rastro 0.1.0-alpha.3 → 0.1.0-alpha.5

package/src/component/_generated/component.ts

@@ -185,6 +185,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             latitude?: number;
             longitude?: number;
             os?: string;
+            visitorKey?: string;
           };
           events: Array<{
             affiliateSlug?: string;
@@ -257,10 +258,59 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             visits: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            uniqueVisitors: {
+              basis: "estimatedUnique" | "exactLegacy";
+              exact: boolean;
+            };
+          };
           truncated: boolean;
         },
         Name
       >;
+      dataCoverage: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteId: string; to: number },
+        {
+          datasets: Array<{
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            dataset:
+              | "overviewHour"
+              | "overviewDay"
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goals"
+              | "funnels"
+              | "affiliates";
+          }>;
+          from: number;
+          siteId: string;
+          to: number;
+        },
+        Name
+      >;
       funnelsReport: FunctionReference<
         "query",
         "internal",
@@ -272,6 +322,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           conversionWindowMs: number;
           createdAt: number;
           key: string;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           name: string;
           siteId: string;
           steps: Array<{
@@ -309,6 +375,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             valueCents: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           truncated: boolean;
         },
         Name
@@ -407,6 +489,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             utmMedium?: string;
             utmSource?: string;
             visitorId: string;
+            visitorKey?: string;
           }>;
           pageStatus?: "SplitRecommended" | "SplitRequired" | null;
           splitCursor?: string | null;
@@ -453,6 +536,44 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           currency: string;
           from: number;
           interval: "hour" | "day";
+          metadata: {
+            breakdowns: {
+              topBrowsers: "eventVolume";
+              topCampaigns: "eventVolume";
+              topCountries: "eventVolume";
+              topDevices: "eventVolume";
+              topEvents: "eventVolume";
+              topGoals: "goalCompletionVolume";
+              topPages: "eventVolume";
+              topSources: "eventVolume";
+            };
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            features: {
+              botDetection: "userAgent";
+              errorInsights: "unsupported";
+              outboundLinks: "eventVolume";
+              sessionReplay: "unsupported";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            visitors: {
+              basis: "estimatedUnique" | "bucketOccurrences";
+              exact: boolean;
+              legacyIncomplete: boolean;
+              precision: number | null;
+              sketchVersion: number | null;
+            };
+          };
           timeSeries: Array<{
             clicks: number;
             conversions: number;
@@ -471,6 +592,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topCampaigns: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topCountries: Array<{
             count: number;
             revenueCents: number;
@@ -636,13 +762,105 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "funnelProgress"
             | "funnelSteps"
             | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
+            | "attributions"
+            | "liveSessions"
+            | "ingestWindows"
+            | "siteIngestWindows";
+          siteId: string;
+        },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      disablePolicy: FunctionReference<
+        "mutation",
+        "internal",
+        { siteId: string },
+        null,
+        Name
+      >;
+      runPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          batchSize?: number;
+          kind:
+            | "events"
+            | "sessions"
+            | "conversions"
+            | "goalCompletions"
+            | "aggregateHour"
+            | "aggregateDay"
+            | "funnelProgress"
+            | "funnelSteps"
+            | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
             | "siteIngestWindows";
           siteId: string;
         },
-        { deleted: number; hasMore: boolean },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      setPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          aggregatesDays: number;
+          conversionsDays: number;
+          eventsDays: number;
+          reportsDays: number;
+          sessionsDays: number;
+          siteId: string;
+        },
+        null,
+        Name
+      >;
+      status: FunctionReference<
+        "query",
+        "internal",
+        { siteId: string },
+        {
+          backlog: number;
+          jobs: Array<{
+            before: number;
+            deleted: number;
+            kind:
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goalCompletions"
+              | "aggregateHour"
+              | "aggregateDay"
+              | "funnelProgress"
+              | "funnelSteps"
+              | "affiliateStats"
+              | "goalRollups"
+              | "funnelRollups"
+              | "affiliateRollups"
+              | "attributions"
+              | "liveSessions"
+              | "ingestWindows"
+              | "siteIngestWindows";
+            scheduledAt: number | null;
+            status: "pending" | "running" | "complete";
+            updatedAt: number;
+          }>;
+          policy: {
+            aggregatesDays: number;
+            conversionsDays: number;
+            eventsDays: number;
+            reportsDays: number;
+            sessionsDays: number;
+            updatedAt: number;
+          } | null;
+        },
         Name
       >;
     };
@@ -716,4 +934,13 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    visitors: {
+      link: FunctionReference<
+        "mutation",
+        "internal",
+        { previousVisitorId: string; siteId: string; visitorId: string },
+        { aliasCount: number; linked: boolean },
+        Name
+      >;
+    };
   };

package/src/component/cardinality.ts

@@ -0,0 +1,116 @@
+const SKETCH_VERSION = 1 as const;
+const DEFAULT_PRECISION = 6;
+
+export type VisitorSketch = {
+  version: typeof SKETCH_VERSION;
+  precision: number;
+  registers: ArrayBuffer;
+};
+
+export function createVisitorSketch(
+  precision = DEFAULT_PRECISION,
+): VisitorSketch {
+  if (!Number.isSafeInteger(precision) || precision < 4 || precision > 10) {
+    throw new Error("visitor sketch precision must be between 4 and 10");
+  }
+  return {
+    version: SKETCH_VERSION,
+    precision,
+    registers: new Uint8Array(1 << precision).buffer,
+  };
+}
+
+export function addVisitor(
+  sketch: VisitorSketch,
+  visitorId: string,
+): VisitorSketch {
+  const registers = checkedRegisters(sketch).slice();
+  const hash = fnv1a64(visitorId);
+  const remainingBits = 64 - sketch.precision;
+  const index = Number(hash >> BigInt(remainingBits));
+  const remaining = hash & ((1n << BigInt(remainingBits)) - 1n);
+  const rank = leadingZeroRank(remaining, remainingBits);
+  registers[index] = Math.max(registers[index] ?? 0, rank);
+  return { ...sketch, registers: registers.buffer };
+}
+
+export function unionVisitorSketches(
+  left: VisitorSketch,
+  right: VisitorSketch,
+): VisitorSketch {
+  const leftRegisters = checkedRegisters(left);
+  const rightRegisters = checkedRegisters(right);
+  if (
+    left.version !== right.version ||
+    left.precision !== right.precision ||
+    leftRegisters.length !== rightRegisters.length
+  ) {
+    throw new Error("visitor sketches are incompatible");
+  }
+  const registers = leftRegisters.slice();
+  for (let index = 0; index < registers.length; index += 1) {
+    registers[index] = Math.max(
+      registers[index] ?? 0,
+      rightRegisters[index] ?? 0,
+    );
+  }
+  return { ...left, registers: registers.buffer };
+}
+
+export function estimateVisitors(sketch: VisitorSketch): number {
+  const registers = checkedRegisters(sketch);
+  const count = registers.length;
+  const alpha = count === 16
+    ? 0.673
+    : count === 32
+      ? 0.697
+      : count === 64
+        ? 0.709
+        : 0.7213 / (1 + 1.079 / count);
+  let denominator = 0;
+  let empty = 0;
+  for (const register of registers) {
+    denominator += 2 ** -register;
+    if (register === 0) empty += 1;
+  }
+  let estimate = (alpha * count * count) / denominator;
+  if (estimate <= 2.5 * count && empty > 0) {
+    estimate = count * Math.log(count / empty);
+  }
+  return Math.max(0, Math.round(estimate));
+}
+
+function checkedRegisters(sketch: VisitorSketch): Uint8Array<ArrayBuffer> {
+  if (sketch.version !== SKETCH_VERSION) {
+    throw new Error("unsupported visitor sketch version");
+  }
+  const registers = new Uint8Array(sketch.registers);
+  if (registers.length !== 1 << sketch.precision) {
+    throw new Error("visitor sketch has invalid register storage");
+  }
+  return registers;
+}
+
+function leadingZeroRank(value: bigint, bits: number): number {
+  if (value === 0n) return bits + 1;
+  let rank = 1;
+  for (let bit = bits - 1; bit >= 0; bit -= 1) {
+    if ((value & (1n << BigInt(bit))) !== 0n) break;
+    rank += 1;
+  }
+  return rank;
+}
+
+function fnv1a64(value: string): bigint {
+  let hash = 0xcbf29ce484222325n;
+  for (const byte of new TextEncoder().encode(value)) {
+    hash ^= BigInt(byte);
+    hash = BigInt.asUintN(64, hash * 0x100000001b3n);
+  }
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xff51afd7ed558ccdn);
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xc4ceb9fe1a85ec53n);
+  hash ^= hash >> 33n;
+  return hash;
+}

package/src/component/constants.ts

@@ -28,6 +28,17 @@ export const JOURNEY_READ_BYTE_RESERVE = 4 * 1024 * 1024;
 export const JOURNEY_READ_DOCUMENT_RESERVE = 1_000;
 export const MAX_RETENTION_BATCH = 500;
 export const MAX_OVERVIEW_AGGREGATE_ROWS = 13_000;
+export const MAX_REPORT_BUCKETS = 5_000;
+export const MAX_FEATURE_ROLLUP_DAYS = 90;
+export const MAX_FEATURE_ROLLUP_ROWS = 5_000;
+export const FEATURE_REPORT_BYTE_RESERVE = 2 * 1024 * 1024;
+export const FEATURE_REPORT_DOCUMENT_RESERVE = 500;
+export const REPORT_ROLLUP_SHARDS = 2;
+export const ROLLUP_SAFETY_DELAY_MS = 5 * 60_000;
+export const RETENTION_LEASE_MS = 60_000;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
+export const MINUTE_MS = 60_000;
+export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;
+export const MAX_VISITOR_ALIASES = 10;

package/src/component/coverage.ts

@@ -0,0 +1,25 @@
+import type { Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
+import { ROLLUP_SAFETY_DELAY_MS } from "./constants.js";
+
+export async function ensureAnalyticsControl(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+  retained: Array<{ dataset: string; before: number }> = [],
+) {
+  const existing = await ctx.db
+    .query("analyticsCoverage")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (existing) return existing;
+  const fields = {
+    siteId,
+    generation: 1 as const,
+    rollupCompleteFrom: now + ROLLUP_SAFETY_DELAY_MS,
+    retained,
+    updatedAt: now,
+  };
+  const _id = await ctx.db.insert("analyticsCoverage", fields);
+  return { ...fields, _id, _creationTime: now };
+}

package/src/component/errors.ts

@@ -9,7 +9,8 @@ export type RastroErrorCode =
   | "LIMIT_EXCEEDED"
   | "CONFLICT"
   | "CURRENCY_MISMATCH"
-  | "RATE_LIMITED";
+  | "RATE_LIMITED"
+  | "REPORT_INCOMPLETE";
 
 export function fail(
   code: RastroErrorCode,