@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.4

package/src/component/_generated/component.ts

@@ -65,6 +65,19 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    diagnostics: {
+      sessionEventTotals: FunctionReference<
+        "query",
+        "internal",
+        { sessionIds: Array<string>; siteId: string },
+        {
+          foundSessions: number;
+          requestedSessions: number;
+          storedEvents: number;
+        },
+        Name
+      >;
+    };
     funnels: {
       list: FunctionReference<
         "query",
@@ -244,10 +257,59 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             visits: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            uniqueVisitors: {
+              basis: "estimatedUnique" | "exactLegacy";
+              exact: boolean;
+            };
+          };
           truncated: boolean;
         },
         Name
       >;
+      dataCoverage: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteId: string; to: number },
+        {
+          datasets: Array<{
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            dataset:
+              | "overviewHour"
+              | "overviewDay"
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goals"
+              | "funnels"
+              | "affiliates";
+          }>;
+          from: number;
+          siteId: string;
+          to: number;
+        },
+        Name
+      >;
       funnelsReport: FunctionReference<
         "query",
         "internal",
@@ -259,6 +321,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           conversionWindowMs: number;
           createdAt: number;
           key: string;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           name: string;
           siteId: string;
           steps: Array<{
@@ -296,6 +374,22 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             updatedAt: number;
             valueCents: number;
           }>;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+          };
           truncated: boolean;
         },
         Name
@@ -440,6 +534,43 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           currency: string;
           from: number;
           interval: "hour" | "day";
+          metadata: {
+            breakdowns: {
+              topBrowsers: "eventVolume";
+              topCountries: "eventVolume";
+              topDevices: "eventVolume";
+              topEvents: "eventVolume";
+              topGoals: "goalCompletionVolume";
+              topPages: "eventVolume";
+              topSources: "eventVolume";
+            };
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            features: {
+              botDetection: "unsupported";
+              errorInsights: "unsupported";
+              outboundLinks: "eventVolume";
+              sessionReplay: "unsupported";
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            visitors: {
+              basis: "estimatedUnique" | "bucketOccurrences";
+              exact: boolean;
+              legacyIncomplete: boolean;
+              precision: number | null;
+              sketchVersion: number | null;
+            };
+          };
           timeSeries: Array<{
             clicks: number;
             conversions: number;
@@ -623,13 +754,105 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "funnelProgress"
             | "funnelSteps"
             | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
+            | "attributions"
+            | "liveSessions"
+            | "ingestWindows"
+            | "siteIngestWindows";
+          siteId: string;
+        },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      disablePolicy: FunctionReference<
+        "mutation",
+        "internal",
+        { siteId: string },
+        null,
+        Name
+      >;
+      runPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          batchSize?: number;
+          kind:
+            | "events"
+            | "sessions"
+            | "conversions"
+            | "goalCompletions"
+            | "aggregateHour"
+            | "aggregateDay"
+            | "funnelProgress"
+            | "funnelSteps"
+            | "affiliateStats"
+            | "goalRollups"
+            | "funnelRollups"
+            | "affiliateRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
             | "siteIngestWindows";
           siteId: string;
         },
-        { deleted: number; hasMore: boolean },
+        { before: number; deleted: number; hasMore: boolean },
+        Name
+      >;
+      setPolicy: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          aggregatesDays: number;
+          conversionsDays: number;
+          eventsDays: number;
+          reportsDays: number;
+          sessionsDays: number;
+          siteId: string;
+        },
+        null,
+        Name
+      >;
+      status: FunctionReference<
+        "query",
+        "internal",
+        { siteId: string },
+        {
+          backlog: number;
+          jobs: Array<{
+            before: number;
+            deleted: number;
+            kind:
+              | "events"
+              | "sessions"
+              | "conversions"
+              | "goalCompletions"
+              | "aggregateHour"
+              | "aggregateDay"
+              | "funnelProgress"
+              | "funnelSteps"
+              | "affiliateStats"
+              | "goalRollups"
+              | "funnelRollups"
+              | "affiliateRollups"
+              | "attributions"
+              | "liveSessions"
+              | "ingestWindows"
+              | "siteIngestWindows";
+            scheduledAt: number | null;
+            status: "pending" | "running" | "complete";
+            updatedAt: number;
+          }>;
+          policy: {
+            aggregatesDays: number;
+            conversionsDays: number;
+            eventsDays: number;
+            reportsDays: number;
+            sessionsDays: number;
+            updatedAt: number;
+          } | null;
+        },
         Name
       >;
     };

package/src/component/_generated/server.ts

@@ -39,6 +39,10 @@ import type { DataModel } from "./dataModel.js";
 type Env = {
   readonly CONVEX_CLOUD_URL: string;
   readonly CONVEX_SITE_URL: string;
+  readonly RASTRO_GEOIP_DAILY_LIMIT: string | undefined;
+  readonly RASTRO_GEOIP_PROVIDER: "ipinfo" | "ipwhois" | undefined;
+  readonly RASTRO_GEOIP_TOKEN: string | undefined;
+  readonly RASTRO_TRUST_PROXY: "true" | undefined;
 };
 
 /**

package/src/component/cardinality.ts

@@ -0,0 +1,116 @@
+const SKETCH_VERSION = 1 as const;
+const DEFAULT_PRECISION = 6;
+
+export type VisitorSketch = {
+  version: typeof SKETCH_VERSION;
+  precision: number;
+  registers: ArrayBuffer;
+};
+
+export function createVisitorSketch(
+  precision = DEFAULT_PRECISION,
+): VisitorSketch {
+  if (!Number.isSafeInteger(precision) || precision < 4 || precision > 10) {
+    throw new Error("visitor sketch precision must be between 4 and 10");
+  }
+  return {
+    version: SKETCH_VERSION,
+    precision,
+    registers: new Uint8Array(1 << precision).buffer,
+  };
+}
+
+export function addVisitor(
+  sketch: VisitorSketch,
+  visitorId: string,
+): VisitorSketch {
+  const registers = checkedRegisters(sketch).slice();
+  const hash = fnv1a64(visitorId);
+  const remainingBits = 64 - sketch.precision;
+  const index = Number(hash >> BigInt(remainingBits));
+  const remaining = hash & ((1n << BigInt(remainingBits)) - 1n);
+  const rank = leadingZeroRank(remaining, remainingBits);
+  registers[index] = Math.max(registers[index] ?? 0, rank);
+  return { ...sketch, registers: registers.buffer };
+}
+
+export function unionVisitorSketches(
+  left: VisitorSketch,
+  right: VisitorSketch,
+): VisitorSketch {
+  const leftRegisters = checkedRegisters(left);
+  const rightRegisters = checkedRegisters(right);
+  if (
+    left.version !== right.version ||
+    left.precision !== right.precision ||
+    leftRegisters.length !== rightRegisters.length
+  ) {
+    throw new Error("visitor sketches are incompatible");
+  }
+  const registers = leftRegisters.slice();
+  for (let index = 0; index < registers.length; index += 1) {
+    registers[index] = Math.max(
+      registers[index] ?? 0,
+      rightRegisters[index] ?? 0,
+    );
+  }
+  return { ...left, registers: registers.buffer };
+}
+
+export function estimateVisitors(sketch: VisitorSketch): number {
+  const registers = checkedRegisters(sketch);
+  const count = registers.length;
+  const alpha = count === 16
+    ? 0.673
+    : count === 32
+      ? 0.697
+      : count === 64
+        ? 0.709
+        : 0.7213 / (1 + 1.079 / count);
+  let denominator = 0;
+  let empty = 0;
+  for (const register of registers) {
+    denominator += 2 ** -register;
+    if (register === 0) empty += 1;
+  }
+  let estimate = (alpha * count * count) / denominator;
+  if (estimate <= 2.5 * count && empty > 0) {
+    estimate = count * Math.log(count / empty);
+  }
+  return Math.max(0, Math.round(estimate));
+}
+
+function checkedRegisters(sketch: VisitorSketch): Uint8Array<ArrayBuffer> {
+  if (sketch.version !== SKETCH_VERSION) {
+    throw new Error("unsupported visitor sketch version");
+  }
+  const registers = new Uint8Array(sketch.registers);
+  if (registers.length !== 1 << sketch.precision) {
+    throw new Error("visitor sketch has invalid register storage");
+  }
+  return registers;
+}
+
+function leadingZeroRank(value: bigint, bits: number): number {
+  if (value === 0n) return bits + 1;
+  let rank = 1;
+  for (let bit = bits - 1; bit >= 0; bit -= 1) {
+    if ((value & (1n << BigInt(bit))) !== 0n) break;
+    rank += 1;
+  }
+  return rank;
+}
+
+function fnv1a64(value: string): bigint {
+  let hash = 0xcbf29ce484222325n;
+  for (const byte of new TextEncoder().encode(value)) {
+    hash ^= BigInt(byte);
+    hash = BigInt.asUintN(64, hash * 0x100000001b3n);
+  }
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xff51afd7ed558ccdn);
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xc4ceb9fe1a85ec53n);
+  hash ^= hash >> 33n;
+  return hash;
+}

package/src/component/constants.ts

@@ -28,6 +28,16 @@ export const JOURNEY_READ_BYTE_RESERVE = 4 * 1024 * 1024;
 export const JOURNEY_READ_DOCUMENT_RESERVE = 1_000;
 export const MAX_RETENTION_BATCH = 500;
 export const MAX_OVERVIEW_AGGREGATE_ROWS = 13_000;
+export const MAX_REPORT_BUCKETS = 5_000;
+export const MAX_FEATURE_ROLLUP_DAYS = 90;
+export const MAX_FEATURE_ROLLUP_ROWS = 5_000;
+export const FEATURE_REPORT_BYTE_RESERVE = 2 * 1024 * 1024;
+export const FEATURE_REPORT_DOCUMENT_RESERVE = 500;
+export const REPORT_ROLLUP_SHARDS = 2;
+export const ROLLUP_SAFETY_DELAY_MS = 5 * 60_000;
+export const RETENTION_LEASE_MS = 60_000;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
+export const MINUTE_MS = 60_000;
+export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;

package/src/component/convex.config.ts

@@ -1,7 +1,13 @@
-import migrations from "@convex-dev/migrations/convex.config.js";
 import { defineComponent } from "convex/server";
+import { v } from "convex/values";
 
-const component = defineComponent("rastroAnalytics");
-component.use(migrations);
-
-export default component;
+export default defineComponent("rastroAnalytics", {
+  env: {
+    RASTRO_GEOIP_PROVIDER: v.optional(
+      v.union(v.literal("ipinfo"), v.literal("ipwhois")),
+    ),
+    RASTRO_GEOIP_TOKEN: v.optional(v.string()),
+    RASTRO_GEOIP_DAILY_LIMIT: v.optional(v.string()),
+    RASTRO_TRUST_PROXY: v.optional(v.literal("true")),
+  },
+});

package/src/component/coverage.ts

@@ -0,0 +1,25 @@
+import type { Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
+import { ROLLUP_SAFETY_DELAY_MS } from "./constants.js";
+
+export async function ensureAnalyticsControl(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+  retained: Array<{ dataset: string; before: number }> = [],
+) {
+  const existing = await ctx.db
+    .query("analyticsCoverage")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (existing) return existing;
+  const fields = {
+    siteId,
+    generation: 1 as const,
+    rollupCompleteFrom: now + ROLLUP_SAFETY_DELAY_MS,
+    retained,
+    updatedAt: now,
+  };
+  const _id = await ctx.db.insert("analyticsCoverage", fields);
+  return { ...fields, _id, _creationTime: now };
+}

package/src/component/diagnostics.ts

@@ -0,0 +1,64 @@
+import { v } from "convex/values";
+import { query } from "./_generated/server.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+
+const MAX_SESSION_TOTAL_IDS = 100;
+
+export const sessionEventTotals = query({
+  args: {
+    siteId: v.id("sites"),
+    sessionIds: v.array(v.string()),
+  },
+  returns: v.object({
+    requestedSessions: v.number(),
+    foundSessions: v.number(),
+    storedEvents: v.number(),
+  }),
+  handler: async (ctx, args) => {
+    if (
+      args.sessionIds.length < 1 ||
+      args.sessionIds.length > MAX_SESSION_TOTAL_IDS
+    ) {
+      fail(
+        "INVALID_ARGUMENT",
+        `sessionIds must contain between 1 and ${MAX_SESSION_TOTAL_IDS} IDs`,
+      );
+    }
+    let sessionIds: string[];
+    try {
+      sessionIds = args.sessionIds.map((sessionId) => {
+        const sanitized = sanitizeOpaqueId(sessionId, "sessionId");
+        if (sanitized !== sessionId) throw new Error("sessionId is not canonical");
+        return sanitized;
+      });
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid sessionId",
+      );
+    }
+    if (new Set(sessionIds).size !== sessionIds.length) {
+      fail("INVALID_ARGUMENT", "sessionIds must be unique");
+    }
+
+    const sessions = await Promise.all(
+      sessionIds.map((sessionId) =>
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", args.siteId).eq("sessionId", sessionId),
+          )
+          .unique()
+      ),
+    );
+    return {
+      requestedSessions: sessionIds.length,
+      foundSessions: sessions.filter((session) => session !== null).length,
+      storedEvents: sessions.reduce(
+        (total, session) => total + (session?.eventCount ?? 0),
+        0,
+      ),
+    };
+  },
+});

package/src/component/errors.ts

@@ -9,7 +9,8 @@ export type RastroErrorCode =
   | "LIMIT_EXCEEDED"
   | "CONFLICT"
   | "CURRENCY_MISMATCH"
-  | "RATE_LIMITED";
+  | "RATE_LIMITED"
+  | "REPORT_INCOMPLETE";
 
 export function fail(
   code: RastroErrorCode,