@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11

package/src/component/constants.ts

@@ -7,8 +7,10 @@ export const MAX_BYTES_PER_SESSION_WINDOW = 256 * 1024;
 export const SITE_INGEST_SHARDS = 4_096;
 export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 300;
 export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 512 * 1024;
-export const LIVE_SESSION_TTL_MS = 60_000;
+export const LIVE_SESSION_TTL_MS = 90_000;
+export const LIVE_LEAVE_GRACE_MS = 10_000;
 export const LIVE_SWEEP_INTERVAL_MS = 5_000;
+export const CLOCK_SKEW_TOLERANCE_MS = 30_000;
 export const AGGREGATE_SHARDS = 128;
 export const HOURLY_AGGREGATE_SHARDS = 256;
 export const SECONDARY_AGGREGATE_SHARDS = 16;
@@ -24,8 +26,49 @@ export const MAX_REPORT_SITES = 10;
 export const MAX_REPORT_RANGE_DAYS = 366;
 export const MAX_LIVE_VISITORS = 500;
 export const MAX_JOURNEY_EVENTS = 500;
+export const JOURNEY_READ_BYTE_RESERVE = 4 * 1024 * 1024;
+export const JOURNEY_READ_DOCUMENT_RESERVE = 1_000;
 export const MAX_RETENTION_BATCH = 500;
 export const MAX_OVERVIEW_AGGREGATE_ROWS = 13_000;
+export const MAX_REPORT_BUCKETS = 5_000;
+export const MAX_FEATURE_ROLLUP_DAYS = 90;
+export const MAX_FEATURE_ROLLUP_ROWS = 5_000;
+export const FEATURE_REPORT_BYTE_RESERVE = 2 * 1024 * 1024;
+export const FEATURE_REPORT_DOCUMENT_RESERVE = 500;
+export const REPORT_ROLLUP_SHARDS = 2;
+export const ROLLUP_SAFETY_DELAY_MS = 5 * 60_000;
+export const RETENTION_LEASE_MS = 60_000;
+// Report rollups are folded by one leased job per site (see rollupStore.ts).
+// A run takes at most this many queued batches, stops early once the merged
+// deltas would touch this many rows, and holds its lease for this long.
+export const ROLLUP_FOLD_BATCH_DOCS = 200;
+export const ROLLUP_FOLD_MAX_ROWS = 1_500;
+export const ROLLUP_FOLD_LEASE_MS = 60_000;
+// Shared Web Vitals histogram edges. Every Google rating threshold (LCP
+// 2500/4000, CLS×1000 100/250, INP 200/500, FCP 1800/3000, TTFB 800/1800) is
+// an edge, so rating counts stay exact while one layout serves all metrics.
+export const VITAL_HISTOGRAM_EDGES = [
+  100, 200, 250, 300, 500, 800, 1_000, 1_200, 1_800, 2_500, 3_000, 4_000, 5_000,
+  7_500, 10_000,
+];
+export const MAX_VITAL_VALUE = 600_000;
+export const VITAL_PAGE_SLOTS = 48;
+export const MAX_VITALS_REPORT_ROWS = 5_000;
+export const VITALS_REPORT_PAGE_LIMIT = 12;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
+export const MINUTE_MS = 60_000;
+export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;
+export const MAX_VISITOR_ALIASES = 10;
+// Site map rollups: one row per (site, route, UTC day, shard). A day keeps at
+// most this many route rows per shard before new routes fold into "(other)";
+// each row keeps this many outgoing targets before they fold the same way. A
+// report reads at most MAX_SITE_MAP_ROWS rows, newest day first — about
+// 5 MiB of rows, under the 8 MiB a function may read once the feature-report
+// byte reserve is kept — and declares the older days it could not read as
+// partial coverage: 20 days at the daily cap, 40 for a 100-route site.
+export const MAX_SITE_MAP_ROUTES_PER_DAY = 200;
+export const ROUTE_TRANSITION_SLOTS = 24;
+export const MAX_SITE_MAP_ROWS = 8_000;
+export const SITE_MAP_OTHER_ROUTE = "(other)";

package/src/component/convex.config.ts

@@ -1,3 +1,13 @@
 import { defineComponent } from "convex/server";
+import { v } from "convex/values";
 
-export default defineComponent("rastroAnalytics");
+export default defineComponent("rastroAnalytics", {
+  env: {
+    RASTRO_GEOIP_PROVIDER: v.optional(
+      v.union(v.literal("ipinfo"), v.literal("ipwhois")),
+    ),
+    RASTRO_GEOIP_TOKEN: v.optional(v.string()),
+    RASTRO_GEOIP_DAILY_LIMIT: v.optional(v.string()),
+    RASTRO_TRUST_PROXY: v.optional(v.literal("true")),
+  },
+});

package/src/component/coverage.ts

@@ -0,0 +1,71 @@
+import type { Id } from "./_generated/dataModel.js";
+import type { MutationCtx } from "./_generated/server.js";
+import { ROLLUP_SAFETY_DELAY_MS } from "./constants.js";
+
+export async function ensureAnalyticsControl(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+  retained: Array<{ dataset: string; before: number }> = [],
+) {
+  const existing = await ctx.db
+    .query("analyticsCoverage")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  if (existing) {
+    // A site that predates a feature starts that feature's coverage at the
+    // first ingest after the upgrade, with the same safety delay as rollups:
+    // the site map rows, and the alpha.11 counters (pageviews and sessions
+    // per dimension slot; duration, continued sessions and new visitors per
+    // bucket).
+    const stamps: {
+      siteMapSince?: number;
+      countersSince?: number;
+      localDaySince?: number;
+    } = {};
+    if (existing.siteMapSince === undefined) {
+      stamps.siteMapSince = now + ROLLUP_SAFETY_DELAY_MS;
+    }
+    if (existing.countersSince === undefined) {
+      stamps.countersSince = now + ROLLUP_SAFETY_DELAY_MS;
+    }
+    if (existing.localDaySince === undefined) {
+      stamps.localDaySince = now + ROLLUP_SAFETY_DELAY_MS;
+    }
+    if (Object.keys(stamps).length === 0) return existing;
+    await ctx.db.patch("analyticsCoverage", existing._id, {
+      ...stamps,
+      updatedAt: now,
+    });
+    return { ...existing, ...stamps, updatedAt: now };
+  }
+  const fields = {
+    siteId,
+    generation: 1 as const,
+    rollupCompleteFrom: now + ROLLUP_SAFETY_DELAY_MS,
+    siteMapSince: now + ROLLUP_SAFETY_DELAY_MS,
+    countersSince: now + ROLLUP_SAFETY_DELAY_MS,
+    localDaySince: now + ROLLUP_SAFETY_DELAY_MS,
+    retained,
+    updatedAt: now,
+  };
+  const _id = await ctx.db.insert("analyticsCoverage", fields);
+  return { ...fields, _id, _creationTime: now };
+}
+
+/**
+ * A site's local-day buckets are computed in its timezone at ingest, so a
+ * changed timezone restarts their coverage: rows before the change belong to
+ * the old zone's days and must not answer for the new one.
+ */
+export async function resetLocalDayCoverage(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  now: number,
+) {
+  const control = await ensureAnalyticsControl(ctx, siteId, now);
+  await ctx.db.patch("analyticsCoverage", control._id, {
+    localDaySince: now + ROLLUP_SAFETY_DELAY_MS,
+    updatedAt: now,
+  });
+}

package/src/component/diagnostics.ts

@@ -0,0 +1,65 @@
+import { v } from "convex/values";
+import { query } from "./_generated/server.js";
+import { fail } from "./errors.js";
+import { sanitizeOpaqueId } from "./sanitize.js";
+
+const MAX_SESSION_TOTAL_IDS = 100;
+
+export const sessionEventTotals = query({
+  args: {
+    siteId: v.id("sites"),
+    sessionIds: v.array(v.string()),
+  },
+  returns: v.object({
+    requestedSessions: v.number(),
+    foundSessions: v.number(),
+    storedEvents: v.number(),
+  }),
+  handler: async (ctx, args) => {
+    if (
+      args.sessionIds.length < 1 ||
+      args.sessionIds.length > MAX_SESSION_TOTAL_IDS
+    ) {
+      fail(
+        "INVALID_ARGUMENT",
+        `sessionIds must contain between 1 and ${MAX_SESSION_TOTAL_IDS} IDs`,
+      );
+    }
+    let sessionIds: string[];
+    try {
+      sessionIds = args.sessionIds.map((sessionId) => {
+        const sanitized = sanitizeOpaqueId(sessionId, "sessionId");
+        if (sanitized !== sessionId)
+          throw new Error("sessionId is not canonical");
+        return sanitized;
+      });
+    } catch (error) {
+      fail(
+        "INVALID_ARGUMENT",
+        error instanceof Error ? error.message : "invalid sessionId",
+      );
+    }
+    if (new Set(sessionIds).size !== sessionIds.length) {
+      fail("INVALID_ARGUMENT", "sessionIds must be unique");
+    }
+
+    const sessions = await Promise.all(
+      sessionIds.map((sessionId) =>
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", args.siteId).eq("sessionId", sessionId),
+          )
+          .unique(),
+      ),
+    );
+    return {
+      requestedSessions: sessionIds.length,
+      foundSessions: sessions.filter((session) => session !== null).length,
+      storedEvents: sessions.reduce(
+        (total, session) => total + (session?.eventCount ?? 0),
+        0,
+      ),
+    };
+  },
+});

package/src/component/errors.ts

@@ -9,7 +9,8 @@ export type RastroErrorCode =
   | "LIMIT_EXCEEDED"
   | "CONFLICT"
   | "CURRENCY_MISMATCH"
-  | "RATE_LIMITED";
+  | "RATE_LIMITED"
+  | "REPORT_INCOMPLETE";
 
 export function fail(
   code: RastroErrorCode,