@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.2

package/scripts/benchmark-ingest.mjs

@@ -0,0 +1,499 @@
+import process from "node:process";
+import { Buffer } from "node:buffer";
+import { performance } from "node:perf_hooks";
+import { clearInterval, setInterval } from "node:timers";
+import { ConvexHttpClient } from "convex/browser";
+import { makeFunctionReference } from "convex/server";
+
+const overviewQuery = makeFunctionReference("example:overview");
+
+const PROFILE_DEFAULTS = {
+  lean: { batchSize: 50, sessions: 0 },
+  realistic: { batchSize: 20, sessions: 2_048 },
+  heavy: { batchSize: 20, sessions: 2_048 },
+  feature: { batchSize: 15, sessions: 2_048 },
+};
+const OPTION_NAMES = new Set([
+  "url",
+  "convex-url",
+  "site-id",
+  "origin",
+  "label",
+  "profile",
+  "duration",
+  "requests",
+  "concurrency",
+  "batch-size",
+  "sessions",
+  "progress",
+  "dry-run",
+  "certify",
+]);
+
+const LOCATIONS = [
+  ["BR", "Sao Paulo", -23.55, -46.63],
+  ["US", "New York", 40.71, -74.01],
+  ["CA", "Toronto", 43.65, -79.38],
+  ["GB", "London", 51.51, -0.13],
+  ["DE", "Berlin", 52.52, 13.41],
+  ["FR", "Paris", 48.86, 2.35],
+  ["SE", "Stockholm", 59.33, 18.07],
+  ["IN", "Bengaluru", 12.97, 77.59],
+  ["SG", "Singapore", 1.35, 103.82],
+  ["JP", "Tokyo", 35.68, 139.69],
+  ["AU", "Sydney", -33.87, 151.21],
+  ["ZA", "Cape Town", -33.92, 18.42],
+];
+
+const USER_AGENTS = [
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/127.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 Version/17.6 Safari/605.1.15",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6 like Mac OS X) AppleWebKit/605.1.15 Version/17.6 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
+];
+
+const REALISTIC_PATHS = [
+  "/products",
+  "/docs",
+  "/blog/scale",
+  "/account",
+  "/checkout",
+];
+
+function parseOptions(argv) {
+  const values = new Map();
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (!argument.startsWith("--")) throw new Error(`unexpected argument: ${argument}`);
+    const [rawName, inlineValue] = argument.slice(2).split("=", 2);
+    if (!OPTION_NAMES.has(rawName)) throw new Error(`unknown option: --${rawName}`);
+    if (values.has(rawName)) throw new Error(`duplicate option: --${rawName}`);
+    if (rawName === "dry-run" || rawName === "certify") {
+      values.set(rawName, "true");
+      continue;
+    }
+    const value = inlineValue ?? argv[++index];
+    if (!value || value.startsWith("--")) throw new Error(`missing value for --${rawName}`);
+    values.set(rawName, value);
+  }
+  const profile = values.get("profile") ?? "realistic";
+  if (!(profile in PROFILE_DEFAULTS)) {
+    throw new Error("--profile must be lean, realistic, heavy, or feature");
+  }
+  const defaults = PROFILE_DEFAULTS[profile];
+  return {
+    url: values.get("url"),
+    convexUrl: values.get("convex-url"),
+    siteId: values.get("site-id"),
+    origin: values.get("origin") ?? "https://demo.rastro.local",
+    label: values.get("label") ?? `${profile}-${new Date().toISOString()}`,
+    profile,
+    durationSeconds: positiveNumber(values.get("duration") ?? "60", "duration"),
+    maximumRequests: values.has("requests")
+      ? positiveInteger(values.get("requests"), "requests")
+      : Number.POSITIVE_INFINITY,
+    concurrency: positiveInteger(values.get("concurrency") ?? "100", "concurrency"),
+    batchSize: positiveInteger(
+      values.get("batch-size") ?? String(defaults.batchSize),
+      "batch-size",
+    ),
+    sessionCount: positiveInteger(
+      values.get("sessions") ?? String(defaults.sessions || 1),
+      "sessions",
+    ),
+    progressSeconds: positiveNumber(values.get("progress") ?? "10", "progress"),
+    dryRun: values.get("dry-run") === "true",
+    certify: values.get("certify") === "true",
+  };
+}
+
+function positiveNumber(value, name) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0) throw new Error(`--${name} must be positive`);
+  return parsed;
+}
+
+function positiveInteger(value, name) {
+  const parsed = positiveNumber(value, name);
+  if (!Number.isSafeInteger(parsed)) throw new Error(`--${name} must be an integer`);
+  return parsed;
+}
+
+function eventFor(profile, session, sequence, timestamp) {
+  const path = profile === "lean"
+    ? `/benchmark/lean/${sequence % 4}`
+    : REALISTIC_PATHS[sequence % REALISTIC_PATHS.length];
+  const base = {
+    eventId: `${session.id}.${sequence.toString(36)}`,
+    sessionId: session.id,
+    visitorId: session.visitorId,
+    path,
+    timestamp,
+    sequence,
+  };
+  if (profile === "lean") return { ...base, type: "pageview" };
+  if (profile === "heavy") {
+    return {
+      ...base,
+      type: "custom",
+      name: "payload-sample",
+      properties: Object.fromEntries(
+        Array.from({ length: 16 }, (_, index) => [
+          `property_${index}`,
+          `${index.toString(36)}-${"x".repeat(157)}`,
+        ]),
+      ),
+    };
+  }
+  if (profile === "feature") {
+    const step = sequence % 3;
+    if (step === 0) {
+      return { ...base, type: "pageview", path: "/", affiliateSlug: "creator-club" };
+    }
+    if (step === 1) {
+      return { ...base, type: "pageview", path: "/pricing", affiliateSlug: "creator-club" };
+    }
+    return {
+      ...base,
+      type: "custom",
+      name: "signup",
+      path: "/signup",
+      affiliateSlug: "creator-club",
+      properties: { plan: sequence % 2 === 0 ? "pro" : "starter" },
+    };
+  }
+
+  const kind = sequence === 0 ? 0 : sequence % 20;
+  if (kind === 19) return { ...base, type: "heartbeat" };
+  if (kind < 6) {
+    return {
+      ...base,
+      type: "pageview",
+      referrer: sequence % 2 === 0 ? "https://www.google.com/search?q=rastro" : undefined,
+    };
+  }
+  if (kind < 13) return { ...base, type: "click", target: "Get started" };
+  if (kind < 17) {
+    return {
+      ...base,
+      type: "custom",
+      name: "cta-engaged",
+      properties: { placement: "hero", experiment: `variant-${sequence % 3}` },
+    };
+  }
+  return { ...base, type: "outbound", href: "https://github.com/amageweb/amage-rastro" };
+}
+
+function requestContext(session) {
+  const [country, city, latitude, longitude] = LOCATIONS[session.locationIndex];
+  return {
+    "User-Agent": USER_AGENTS[session.userAgentIndex],
+    "x-vercel-ip-country": country,
+    "x-vercel-ip-city": encodeURIComponent(city),
+    "x-vercel-ip-latitude": String(latitude),
+    "x-vercel-ip-longitude": String(longitude),
+  };
+}
+
+function percentile(sorted, fraction) {
+  if (sorted.length === 0) return 0;
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)];
+}
+
+async function main() {
+  const options = parseOptions(process.argv.slice(2));
+  if (options.batchSize > 50) throw new Error("--batch-size cannot exceed 50");
+  if (options.certify && options.durationSeconds < 600) {
+    throw new Error("--certify requires --duration of at least 600 seconds");
+  }
+  if (options.certify && !options.convexUrl) {
+    throw new Error("--certify requires --convex-url reconciliation");
+  }
+  if (options.certify && Number.isFinite(options.maximumRequests)) {
+    throw new Error("--certify cannot be combined with --requests");
+  }
+  const runId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
+  const sessions = Array.from({ length: options.sessionCount }, (_, index) => ({
+    id: `bench-${runId}-${index}`,
+    visitorId: `visitor-${runId}-${index}`,
+    sequence: 0,
+    locationIndex: index % LOCATIONS.length,
+    userAgentIndex: index % USER_AGENTS.length,
+  }));
+  let nextRequest = 0;
+  let stopping = false;
+  let terminationReason = null;
+  let accepted = 0;
+  let duplicates = 0;
+  let rejected = 0;
+  let failures = 0;
+  let failedEvents = 0;
+  let completedRequests = 0;
+  let sentEvents = 0;
+  let sentHeartbeats = 0;
+  let sentBytes = 0;
+  let acceptedHeartbeats = 0;
+  let reconciliationReliable = true;
+  const errors = new Map();
+  const durations = [];
+
+  const buildRequest = (requestIndex) => {
+    const session = options.profile === "lean"
+      ? {
+          id: `bench-${runId}-${requestIndex}`,
+          visitorId: `visitor-${runId}-${requestIndex}`,
+          sequence: 0,
+          locationIndex: requestIndex % LOCATIONS.length,
+          userAgentIndex: requestIndex % USER_AGENTS.length,
+        }
+      : sessions[requestIndex % sessions.length];
+    const sequenceStart = session.sequence;
+    session.sequence += options.batchSize;
+    const now = Date.now();
+    const events = Array.from({ length: options.batchSize }, (_, index) =>
+      eventFor(options.profile, session, sequenceStart + index, now + index),
+    );
+    return {
+      body: JSON.stringify({ siteId: options.siteId, events }),
+      context: requestContext(session),
+      heartbeats: events.filter((event) => event.type === "heartbeat").length,
+    };
+  };
+
+  const dryRequest = buildRequest(0);
+  if (Buffer.byteLength(dryRequest.body) > 64 * 1024) {
+    throw new Error("generated request exceeds the 64 KiB HTTP limit");
+  }
+  if (options.dryRun) {
+    process.stdout.write(`${JSON.stringify({
+      profile: options.profile,
+      batchSize: options.batchSize,
+      requestBytes: Buffer.byteLength(dryRequest.body),
+      heartbeats: dryRequest.heartbeats,
+      location: requestContext(sessions[0]),
+    }, null, 2)}\n`);
+    return;
+  }
+  if (options.profile !== "lean") sessions[0].sequence = 0;
+  if (!options.url || !options.siteId) {
+    throw new Error("--url and --site-id are required unless --dry-run is used");
+  }
+
+  const stopForSignal = (signal) => {
+    stopping = true;
+    terminationReason = signal;
+  };
+  const stopForInterrupt = () => stopForSignal("SIGINT");
+  const stopForTermination = () => stopForSignal("SIGTERM");
+  process.once("SIGINT", stopForInterrupt);
+  process.once("SIGTERM", stopForTermination);
+
+  const wallClockStartedAt = Date.now();
+  const reconciliationFrom = Math.floor(wallClockStartedAt / 3_600_000) * 3_600_000;
+  const reconciliationTo = Math.ceil(
+    (wallClockStartedAt + options.durationSeconds * 1_000 + 60_000) / 3_600_000,
+  ) * 3_600_000 - 1;
+  const convex = options.convexUrl ? new ConvexHttpClient(options.convexUrl) : null;
+  const aggregateBefore = convex
+    ? await convex.query(overviewQuery, {
+        siteIds: [options.siteId],
+        from: reconciliationFrom,
+        to: reconciliationTo,
+        interval: "hour",
+      })
+    : null;
+  const startedAt = performance.now();
+  const deadline = startedAt + options.durationSeconds * 1_000;
+  const recordError = (key) => errors.set(key, (errors.get(key) ?? 0) + 1);
+  const worker = async () => {
+    while (!stopping) {
+      if (nextRequest >= options.maximumRequests) {
+        terminationReason ??= "request-limit";
+        return;
+      }
+      if (performance.now() >= deadline) {
+        terminationReason ??= "duration";
+        return;
+      }
+      const requestIndex = nextRequest++;
+      const request = buildRequest(requestIndex);
+      const requestBytes = Buffer.byteLength(request.body);
+      if (requestBytes > 64 * 1024) {
+        throw new Error(`generated request ${requestIndex} exceeds the 64 KiB HTTP limit`);
+      }
+      sentBytes += requestBytes;
+      sentEvents += options.batchSize;
+      sentHeartbeats += request.heartbeats;
+      const requestStartedAt = performance.now();
+      try {
+        const response = await globalThis.fetch(options.url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Origin: options.origin,
+            ...request.context,
+          },
+          body: request.body,
+          signal: globalThis.AbortSignal.timeout(30_000),
+        });
+        if (!response.ok) {
+          failures += 1;
+          failedEvents += options.batchSize;
+          const body = await response.text();
+          let code = body.slice(0, 160);
+          try {
+            const parsed = JSON.parse(body);
+            code = parsed?.error?.code ?? parsed?.error?.message ?? code;
+          } catch {
+            // Preserve the bounded response text.
+          }
+          recordError(`${response.status}:${code}`);
+        } else {
+          const result = await response.json();
+          if (
+            ![result.accepted, result.duplicates, result.rejected].every(
+              (value) => Number.isSafeInteger(value) && value >= 0,
+            ) ||
+            result.accepted + result.duplicates + result.rejected !== options.batchSize
+          ) {
+            throw new Error("response has invalid event accounting");
+          }
+          accepted += result.accepted;
+          duplicates += result.duplicates;
+          rejected += result.rejected;
+          if (result.accepted === options.batchSize) {
+            acceptedHeartbeats += request.heartbeats;
+          } else {
+            reconciliationReliable = false;
+          }
+        }
+      } catch (error) {
+        failures += 1;
+        failedEvents += options.batchSize;
+        recordError(`fetch:${error instanceof Error ? error.message : "unknown"}`);
+      } finally {
+        durations.push(performance.now() - requestStartedAt);
+        completedRequests += 1;
+      }
+    }
+  };
+
+  const progress = setInterval(() => {
+    const elapsedSeconds = Math.max(0.001, (performance.now() - startedAt) / 1_000);
+    process.stderr.write(
+      `[${options.label}] ${Math.round(elapsedSeconds)}s ` +
+      `${accepted} accepted ${Math.round(accepted / elapsedSeconds)} events/s ` +
+      `${failures} failures\n`,
+    );
+  }, options.progressSeconds * 1_000);
+  progress.unref();
+  await Promise.all(Array.from({ length: options.concurrency }, worker));
+  clearInterval(progress);
+
+  const elapsedMs = performance.now() - startedAt;
+  terminationReason ??= performance.now() >= deadline ? "duration" : "completed";
+  const aggregateAfter = convex
+    ? await convex.query(overviewQuery, {
+        siteIds: [options.siteId],
+        from: reconciliationFrom,
+        to: reconciliationTo,
+        interval: "hour",
+      })
+    : null;
+  process.removeListener("SIGINT", stopForInterrupt);
+  process.removeListener("SIGTERM", stopForTermination);
+  const durationCompleted = terminationReason === "duration";
+  durations.sort((left, right) => left - right);
+  const storedEvents = accepted - acceptedHeartbeats;
+  const reconciliation = aggregateBefore && aggregateAfter
+    ? {
+        scope: "site-wide-delta",
+        reliable: reconciliationReliable && failures === 0 && rejected === 0,
+        expectedStoredEvents: storedEvents,
+        aggregateEventsDelta:
+          aggregateAfter.totals.events - aggregateBefore.totals.events,
+        matches:
+          reconciliationReliable &&
+          failures === 0 &&
+          rejected === 0 &&
+          aggregateAfter.totals.events - aggregateBefore.totals.events === storedEvents,
+      }
+    : null;
+  const cleanRun =
+    failures === 0 && duplicates === 0 && rejected === 0 && durationCompleted;
+  const reconciliationPassed = Boolean(
+    reconciliation?.reliable && reconciliation.matches,
+  );
+  const certificationPassed =
+    options.certify && cleanRun && reconciliationPassed;
+  const result = {
+    label: options.label,
+    runId,
+    profile: options.profile,
+    targetDurationSeconds: options.durationSeconds,
+    elapsedMs: Math.round(elapsedMs),
+    terminationReason,
+    durationCompleted,
+    certification: {
+      requested: options.certify,
+      passed: certificationPassed,
+      minimumDurationSeconds: 600,
+    },
+    concurrency: options.concurrency,
+    batchSize: options.batchSize,
+    sessionPool: options.profile === "lean" ? "unique-per-request" : options.sessionCount,
+    maximumRequests: Number.isFinite(options.maximumRequests)
+      ? options.maximumRequests
+      : null,
+    requests: { started: nextRequest, completed: completedRequests, failures },
+    events: {
+      sent: sentEvents,
+      heartbeats: sentHeartbeats,
+      accepted,
+      stored: storedEvents,
+      duplicates,
+      rejected,
+      failed: failedEvents,
+    },
+    errors: Object.fromEntries(errors),
+    throughput: {
+      eventsPerSecond: Math.round((accepted * 1_000) / elapsedMs),
+      storedEventsPerSecond: reconciliationReliable
+        ? Math.round((storedEvents * 1_000) / elapsedMs)
+        : null,
+      estimatedEventsPerDay: durationCompleted
+        ? Math.round((accepted * 86_400_000) / elapsedMs)
+        : null,
+      estimatedStoredEventsPerDay: durationCompleted && reconciliationReliable
+        ? Math.round((storedEvents * 86_400_000) / elapsedMs)
+        : null,
+      certifiedEventsPerDay: certificationPassed
+        ? Math.round((accepted * 86_400_000) / elapsedMs)
+        : null,
+      certifiedStoredEventsPerDay: certificationPassed
+        ? Math.round((storedEvents * 86_400_000) / elapsedMs)
+        : null,
+      requestBytesPerSecond: Math.round((sentBytes * 1_000) / elapsedMs),
+    },
+    latencyMs: {
+      p50: Math.round(percentile(durations, 0.5)),
+      p95: Math.round(percentile(durations, 0.95)),
+      p99: Math.round(percentile(durations, 0.99)),
+      max: Math.round(durations.at(-1) ?? 0),
+    },
+    reconciliation,
+  };
+  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+  if (
+    failures > 0 ||
+    duplicates > 0 ||
+    rejected > 0 ||
+    !durationCompleted ||
+    (options.convexUrl && !reconciliationPassed) ||
+    (options.certify && !certificationPassed)
+  ) {
+    process.exitCode = 1;
+  }
+}
+
+await main();

package/src/component/_generated/api.ts

@@ -11,12 +11,14 @@
 import type * as affiliates from "../affiliates.js";
 import type * as constants from "../constants.js";
 import type * as errors from "../errors.js";
+import type * as eventStore from "../eventStore.js";
 import type * as funnels from "../funnels.js";
 import type * as goals from "../goals.js";
 import type * as guards from "../guards.js";
 import type * as http from "../http.js";
 import type * as ingest from "../ingest.js";
 import type * as live from "../live.js";
+import type * as migrations from "../migrations.js";
 import type * as reports from "../reports.js";
 import type * as retention from "../retention.js";
 import type * as sanitize from "../sanitize.js";
@@ -34,12 +36,14 @@ const fullApi: ApiFromModules<{
   affiliates: typeof affiliates;
   constants: typeof constants;
   errors: typeof errors;
+  eventStore: typeof eventStore;
   funnels: typeof funnels;
   goals: typeof goals;
   guards: typeof guards;
   http: typeof http;
   ingest: typeof ingest;
   live: typeof live;
+  migrations: typeof migrations;
   reports: typeof reports;
   retention: typeof retention;
   sanitize: typeof sanitize;
@@ -73,4 +77,6 @@ export const internal: FilterApi<
   FunctionReference<any, "internal">
 > = anyApi as any;
 
-export const components = componentsGeneric() as unknown as {};
+export const components = componentsGeneric() as unknown as {
+  migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">;
+};

package/src/component/constants.ts

@@ -1,13 +1,17 @@
-export const MAX_BATCH_EVENTS = 20;
+export const MAX_BATCH_EVENTS = 50;
+export const MAX_BATCH_EVENT_GROUPS = 4;
 export const MAX_BATCH_BYTES = 64 * 1024;
 export const MAX_EVENTS_PER_SESSION_WINDOW = 120;
 export const RATE_LIMIT_WINDOW_MS = 60_000;
 export const MAX_BYTES_PER_SESSION_WINDOW = 256 * 1024;
-export const SITE_INGEST_SHARDS = 8;
-export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 120;
+export const SITE_INGEST_SHARDS = 4_096;
+export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 300;
 export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 512 * 1024;
 export const LIVE_SESSION_TTL_MS = 60_000;
-export const AGGREGATE_SHARDS = 16;
+export const LIVE_SWEEP_INTERVAL_MS = 5_000;
+export const AGGREGATE_SHARDS = 128;
+export const HOURLY_AGGREGATE_SHARDS = 256;
+export const SECONDARY_AGGREGATE_SHARDS = 16;
 export const DIMENSION_SLOTS = 16;
 export const MAX_SITE_DOMAINS = 32;
 export const MAX_SITES_PER_OWNER = 100;
@@ -20,9 +24,10 @@ export const MAX_REPORT_SITES = 10;
 export const MAX_REPORT_RANGE_DAYS = 366;
 export const MAX_LIVE_VISITORS = 500;
 export const MAX_JOURNEY_EVENTS = 500;
+export const JOURNEY_READ_BYTE_RESERVE = 4 * 1024 * 1024;
+export const JOURNEY_READ_DOCUMENT_RESERVE = 1_000;
 export const MAX_RETENTION_BATCH = 500;
-export const MAX_OVERVIEW_AGGREGATE_ROWS = 12_000;
+export const MAX_OVERVIEW_AGGREGATE_ROWS = 13_000;
 
 export const DAY_MS = 86_400_000;
 export const HOUR_MS = 3_600_000;
-export const MAX_MANUAL_PAGINATION_OFFSET = 1_000;

package/src/component/convex.config.ts

@@ -1,3 +1,7 @@
+import migrations from "@convex-dev/migrations/convex.config.js";
 import { defineComponent } from "convex/server";
 
-export default defineComponent("rastroAnalytics");
+const component = defineComponent("rastroAnalytics");
+component.use(migrations);
+
+export default component;