@iann29/rastro 0.1.0-alpha.3 → 0.1.0-alpha.4

package/src/component/cardinality.ts

@@ -0,0 +1,116 @@
+const SKETCH_VERSION = 1 as const;
+const DEFAULT_PRECISION = 6;
+
+export type VisitorSketch = {
+  version: typeof SKETCH_VERSION;
+  precision: number;
+  registers: ArrayBuffer;
+};
+
+export function createVisitorSketch(
+  precision = DEFAULT_PRECISION,
+): VisitorSketch {
+  if (!Number.isSafeInteger(precision) || precision < 4 || precision > 10) {
+    throw new Error("visitor sketch precision must be between 4 and 10");
+  }
+  return {
+    version: SKETCH_VERSION,
+    precision,
+    registers: new Uint8Array(1 << precision).buffer,
+  };
+}
+
+export function addVisitor(
+  sketch: VisitorSketch,
+  visitorId: string,
+): VisitorSketch {
+  const registers = checkedRegisters(sketch).slice();
+  const hash = fnv1a64(visitorId);
+  const remainingBits = 64 - sketch.precision;
+  const index = Number(hash >> BigInt(remainingBits));
+  const remaining = hash & ((1n << BigInt(remainingBits)) - 1n);
+  const rank = leadingZeroRank(remaining, remainingBits);
+  registers[index] = Math.max(registers[index] ?? 0, rank);
+  return { ...sketch, registers: registers.buffer };
+}
+
+export function unionVisitorSketches(
+  left: VisitorSketch,
+  right: VisitorSketch,
+): VisitorSketch {
+  const leftRegisters = checkedRegisters(left);
+  const rightRegisters = checkedRegisters(right);
+  if (
+    left.version !== right.version ||
+    left.precision !== right.precision ||
+    leftRegisters.length !== rightRegisters.length
+  ) {
+    throw new Error("visitor sketches are incompatible");
+  }
+  const registers = leftRegisters.slice();
+  for (let index = 0; index < registers.length; index += 1) {
+    registers[index] = Math.max(
+      registers[index] ?? 0,
+      rightRegisters[index] ?? 0,
+    );
+  }
+  return { ...left, registers: registers.buffer };
+}
+
+export function estimateVisitors(sketch: VisitorSketch): number {
+  const registers = checkedRegisters(sketch);
+  const count = registers.length;
+  const alpha = count === 16
+    ? 0.673
+    : count === 32
+      ? 0.697
+      : count === 64
+        ? 0.709
+        : 0.7213 / (1 + 1.079 / count);
+  let denominator = 0;
+  let empty = 0;
+  for (const register of registers) {
+    denominator += 2 ** -register;
+    if (register === 0) empty += 1;
+  }
+  let estimate = (alpha * count * count) / denominator;
+  if (estimate <= 2.5 * count && empty > 0) {
+    estimate = count * Math.log(count / empty);
+  }
+  return Math.max(0, Math.round(estimate));
+}
+
+function checkedRegisters(sketch: VisitorSketch): Uint8Array<ArrayBuffer> {
+  if (sketch.version !== SKETCH_VERSION) {
+    throw new Error("unsupported visitor sketch version");
+  }
+  const registers = new Uint8Array(sketch.registers);
+  if (registers.length !== 1 << sketch.precision) {
+    throw new Error("visitor sketch has invalid register storage");
+  }
+  return registers;
+}
+
+function leadingZeroRank(value: bigint, bits: number): number {
+  if (value === 0n) return bits + 1;
+  let rank = 1;
+  for (let bit = bits - 1; bit >= 0; bit -= 1) {
+    if ((value & (1n << BigInt(bit))) !== 0n) break;
+    rank += 1;
+  }
+  return rank;
+}
+
+function fnv1a64(value: string): bigint {
+  let hash = 0xcbf29ce484222325n;
+  for (const byte of new TextEncoder().encode(value)) {
+    hash ^= BigInt(byte);
+    hash = BigInt.asUintN(64, hash * 0x100000001b3n);
+  }
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xff51afd7ed558ccdn);
+  hash ^= hash >> 33n;
+  hash = BigInt.asUintN(64, hash * 0xc4ceb9fe1a85ec53n);
+  hash ^= hash >> 33n;
+  return hash;
+}

package/src/component/constants.ts

@@ -28,6 +28,16 @@ export const JOURNEY_READ_BYTE_RESERVE = 4 * 1024 * 1024;
 export const JOURNEY_READ_DOCUMENT_RESERVE = 1_000;
 export const MAX_RETENTION_BATCH = 500;
 export const MAX_OVERVIEW_AGGREGATE_ROWS = 13_000;
+export const MAX_REPORT_BUCKETS = 5_000;
+export const MAX_FEATURE_ROLLUP_DAYS = 90;
+export const MAX_FEATURE_ROLLUP_ROWS = 5_000;
+export const FEATURE_REPORT_BYTE_RESERVE = 2 * 1024 * 1024;
+export const FEATURE_REPORT_DOCUMENT_RESERVE = 500;
+export const REPORT_ROLLUP_SHARDS = 2;
+export const ROLLUP_SAFETY_DELAY_MS = 5 * 60_000;
+export const RETENTION_LEASE_MS = 60_000;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
+export const MINUTE_MS = 60_000;
+export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;

package/src/component/coverage.ts

@@ -0,0 +1,25 @@
+import type { Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
+import { ROLLUP_SAFETY_DELAY_MS } from "./constants.js";
+
+export async function ensureAnalyticsControl(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+  retained: Array<{ dataset: string; before: number }> = [],
+) {
+  const existing = await ctx.db
+    .query("analyticsCoverage")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (existing) return existing;
+  const fields = {
+    siteId,
+    generation: 1 as const,
+    rollupCompleteFrom: now + ROLLUP_SAFETY_DELAY_MS,
+    retained,
+    updatedAt: now,
+  };
+  const _id = await ctx.db.insert("analyticsCoverage", fields);
+  return { ...fields, _id, _creationTime: now };
+}

package/src/component/errors.ts

@@ -9,7 +9,8 @@ export type RastroErrorCode =
   | "LIMIT_EXCEEDED"
   | "CONFLICT"
   | "CURRENCY_MISMATCH"
-  | "RATE_LIMITED";
+  | "RATE_LIMITED"
+  | "REPORT_INCOMPLETE";
 
 export function fail(
   code: RastroErrorCode,

package/src/component/http.ts

@@ -7,6 +7,7 @@ import {
   MAX_BATCH_BYTES,
   MAX_BATCH_EVENT_GROUPS,
   MAX_BATCH_EVENTS,
+  RATE_LIMIT_WINDOW_MS,
 } from "./constants.js";
 import {
   classifyIp,
@@ -35,9 +36,16 @@ http.route({
   path: "/events",
   method: "POST",
   handler: httpAction(async (ctx, request) => {
+    const requestId = crypto.randomUUID();
     const origin = request.headers.get("origin");
-    if (!origin) return json({ error: { code: "ORIGIN_REQUIRED" } }, 400);
-    const headers = corsHeaders(origin);
+    if (!origin) {
+      return json(
+        { error: { code: "ORIGIN_REQUIRED" } },
+        400,
+        requestIdHeaders(requestId),
+      );
+    }
+    const headers = corsHeaders(origin, requestId);
     const contentLength = Number(request.headers.get("content-length"));
     if (Number.isFinite(contentLength) && contentLength > MAX_BATCH_BYTES) {
       return json(
@@ -100,6 +108,7 @@ http.route({
       ) {
         const events = enrichmentCandidateEvents(payload.events);
         if (events) {
+          let stage: GeoIpFailureStage = "claim";
           try {
             const claim = await ctx.runMutation(
               internal.geo.claimEnrichment,
@@ -110,22 +119,34 @@ http.route({
               },
             );
             if (claim.sessionIds.length > 0) {
+              stage = "resolve";
               const providerGeo = await resolveGeoIp(
                 clientIp.ip,
                 env.RASTRO_GEOIP_PROVIDER!,
                 env.RASTRO_GEOIP_TOKEN,
               );
               if (providerGeo) {
+                stage = "apply";
                 await ctx.runMutation(internal.geo.applyEnrichment, {
                   siteId: payload.siteId as Id<"sites">,
                   sessionIds: claim.sessionIds,
                   events: claim.events,
                   context: providerGeo,
                 });
+              } else {
+                logGeoIpFailure(
+                  requestId,
+                  stage,
+                  env.RASTRO_GEOIP_PROVIDER!,
+                );
               }
             }
           } catch {
-            // Analytics ingestion remains available if optional enrichment fails.
+            logGeoIpFailure(
+              requestId,
+              stage,
+              env.RASTRO_GEOIP_PROVIDER!,
+            );
           }
         }
       }
@@ -139,9 +160,25 @@ http.route({
             : data.code === "RATE_LIMITED"
               ? 429
               : 400;
-        return json({ error: data }, status, headers);
+        return json(
+          { error: data },
+          status,
+          status === 429
+            ? {
+                ...headers,
+                "Retry-After": String(
+                  Math.ceil(RATE_LIMIT_WINDOW_MS / 1_000),
+                ),
+              }
+            : headers,
+        );
       }
-      return json({ error: { code: "INVALID_REQUEST" } }, 400, headers);
+      logInternalHttpError(requestId);
+      return json(
+        { error: { code: "INTERNAL_ERROR", requestId } },
+        500,
+        headers,
+      );
     }
   }),
 });
@@ -491,12 +528,45 @@ function classifyUserAgent(userAgent: string): {
   return { browser, os, device };
 }
 
-function corsHeaders(origin: string): HeadersInit {
+type GeoIpFailureStage = "claim" | "resolve" | "apply";
+
+function logInternalHttpError(requestId: string): void {
+  console.error(JSON.stringify({
+    level: "error",
+    event: "rastro.http.internal_error",
+    requestId,
+    route: "/events",
+    method: "POST",
+  }));
+}
+
+function logGeoIpFailure(
+  requestId: string,
+  stage: GeoIpFailureStage,
+  provider: "ipinfo" | "ipwhois",
+): void {
+  console.warn(JSON.stringify({
+    level: "warn",
+    event: "rastro.geoip.failure",
+    requestId,
+    route: "/events",
+    stage,
+    provider,
+  }));
+}
+
+function requestIdHeaders(requestId: string): Record<string, string> {
+  return { "X-Request-ID": requestId };
+}
+
+function corsHeaders(origin: string, requestId?: string): Record<string, string> {
   return {
     "Access-Control-Allow-Origin": origin,
     "Access-Control-Allow-Methods": "POST, OPTIONS",
     "Access-Control-Allow-Headers": "Content-Type",
+    "Access-Control-Expose-Headers": "X-Request-ID, Retry-After",
     "Access-Control-Max-Age": "86400",
+    ...(requestId ? requestIdHeaders(requestId) : {}),
     Vary: "Origin",
   };
 }