@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/scripts/benchmark-ingest.mjs

@@ -0,0 +1,601 @@
+import process from "node:process";
+import { Buffer } from "node:buffer";
+import { randomUUID } from "node:crypto";
+import { performance } from "node:perf_hooks";
+import { clearInterval, setInterval } from "node:timers";
+import { ConvexHttpClient } from "convex/browser";
+import { makeFunctionReference } from "convex/server";
+
+const reconcileSessionsQuery = makeFunctionReference(
+  "benchmark:reconcileSessionsPage",
+);
+
+const PROFILE_DEFAULTS = {
+  lean: { batchSize: 50, sessions: 0 },
+  realistic: { batchSize: 20, sessions: 2_048 },
+  heavy: { batchSize: 20, sessions: 2_048 },
+  feature: { batchSize: 15, sessions: 2_048 },
+};
+const OPTION_NAMES = new Set([
+  "url",
+  "convex-url",
+  "site-id",
+  "origin",
+  "label",
+  "profile",
+  "duration",
+  "requests",
+  "concurrency",
+  "batch-size",
+  "sessions",
+  "progress",
+  "dry-run",
+  "certify",
+]);
+
+const LOCATIONS = [
+  ["BR", "Sao Paulo", -23.55, -46.63],
+  ["US", "New York", 40.71, -74.01],
+  ["CA", "Toronto", 43.65, -79.38],
+  ["GB", "London", 51.51, -0.13],
+  ["DE", "Berlin", 52.52, 13.41],
+  ["FR", "Paris", 48.86, 2.35],
+  ["SE", "Stockholm", 59.33, 18.07],
+  ["IN", "Bengaluru", 12.97, 77.59],
+  ["SG", "Singapore", 1.35, 103.82],
+  ["JP", "Tokyo", 35.68, 139.69],
+  ["AU", "Sydney", -33.87, 151.21],
+  ["ZA", "Cape Town", -33.92, 18.42],
+];
+
+const USER_AGENTS = [
+  "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/127.0 Safari/537.36",
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 Version/17.6 Safari/605.1.15",
+  "Mozilla/5.0 (iPhone; CPU iPhone OS 17_6 like Mac OS X) AppleWebKit/605.1.15 Version/17.6 Mobile/15E148 Safari/604.1",
+  "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
+];
+
+const REALISTIC_PATHS = [
+  "/products",
+  "/docs",
+  "/blog/scale",
+  "/account",
+  "/checkout",
+];
+
+function parseOptions(argv) {
+  const values = new Map();
+  for (let index = 0; index < argv.length; index += 1) {
+    const argument = argv[index];
+    if (!argument.startsWith("--"))
+      throw new Error(`unexpected argument: ${argument}`);
+    const [rawName, inlineValue] = argument.slice(2).split("=", 2);
+    if (!OPTION_NAMES.has(rawName))
+      throw new Error(`unknown option: --${rawName}`);
+    if (values.has(rawName)) throw new Error(`duplicate option: --${rawName}`);
+    if (rawName === "dry-run" || rawName === "certify") {
+      values.set(rawName, "true");
+      continue;
+    }
+    const value = inlineValue ?? argv[++index];
+    if (!value || value.startsWith("--"))
+      throw new Error(`missing value for --${rawName}`);
+    values.set(rawName, value);
+  }
+  const profile = values.get("profile") ?? "realistic";
+  if (!(profile in PROFILE_DEFAULTS)) {
+    throw new Error("--profile must be lean, realistic, heavy, or feature");
+  }
+  const defaults = PROFILE_DEFAULTS[profile];
+  return {
+    url: values.get("url"),
+    convexUrl: values.get("convex-url"),
+    siteId: values.get("site-id"),
+    origin: values.get("origin") ?? "https://demo.rastro.local",
+    label: values.get("label") ?? `${profile}-${new Date().toISOString()}`,
+    profile,
+    durationSeconds: positiveNumber(values.get("duration") ?? "60", "duration"),
+    maximumRequests: values.has("requests")
+      ? positiveInteger(values.get("requests"), "requests")
+      : Number.POSITIVE_INFINITY,
+    concurrency: positiveInteger(
+      values.get("concurrency") ?? "100",
+      "concurrency",
+    ),
+    batchSize: positiveInteger(
+      values.get("batch-size") ?? String(defaults.batchSize),
+      "batch-size",
+    ),
+    sessionCount: positiveInteger(
+      values.get("sessions") ?? String(defaults.sessions || 1),
+      "sessions",
+    ),
+    progressSeconds: positiveNumber(values.get("progress") ?? "10", "progress"),
+    dryRun: values.get("dry-run") === "true",
+    certify: values.get("certify") === "true",
+  };
+}
+
+function positiveNumber(value, name) {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed) || parsed <= 0)
+    throw new Error(`--${name} must be positive`);
+  return parsed;
+}
+
+function positiveInteger(value, name) {
+  const parsed = positiveNumber(value, name);
+  if (!Number.isSafeInteger(parsed))
+    throw new Error(`--${name} must be an integer`);
+  return parsed;
+}
+
+function eventFor(profile, session, sequence, timestamp) {
+  const path =
+    profile === "lean"
+      ? `/benchmark/lean/${sequence % 4}`
+      : REALISTIC_PATHS[sequence % REALISTIC_PATHS.length];
+  const base = {
+    eventId: `${session.id}.${sequence.toString(36)}`,
+    sessionId: session.id,
+    visitorId: session.visitorId,
+    path,
+    timestamp,
+    sequence,
+  };
+  if (profile === "lean") return { ...base, type: "pageview" };
+  if (profile === "heavy") {
+    return {
+      ...base,
+      type: "custom",
+      name: "payload-sample",
+      properties: Object.fromEntries(
+        Array.from({ length: 16 }, (_, index) => [
+          `property_${index}`,
+          `${index.toString(36)}-${"x".repeat(157)}`,
+        ]),
+      ),
+    };
+  }
+  if (profile === "feature") {
+    const step = sequence % 3;
+    if (step === 0) {
+      return {
+        ...base,
+        type: "pageview",
+        path: "/",
+        affiliateSlug: "creator-club",
+      };
+    }
+    if (step === 1) {
+      return {
+        ...base,
+        type: "pageview",
+        path: "/pricing",
+        affiliateSlug: "creator-club",
+      };
+    }
+    return {
+      ...base,
+      type: "custom",
+      name: "signup",
+      path: "/signup",
+      affiliateSlug: "creator-club",
+      properties: { plan: sequence % 2 === 0 ? "pro" : "starter" },
+    };
+  }
+
+  const kind = sequence === 0 ? 0 : sequence % 20;
+  if (kind === 19) return { ...base, type: "heartbeat" };
+  if (kind < 6) {
+    return {
+      ...base,
+      type: "pageview",
+      referrer:
+        sequence % 2 === 0
+          ? "https://www.google.com/search?q=rastro"
+          : undefined,
+    };
+  }
+  if (kind < 13) return { ...base, type: "click", target: "Get started" };
+  if (kind < 17) {
+    return {
+      ...base,
+      type: "custom",
+      name: "cta-engaged",
+      properties: { placement: "hero", experiment: `variant-${sequence % 3}` },
+    };
+  }
+  return {
+    ...base,
+    type: "outbound",
+    href: "https://github.com/amageweb/amage-rastro",
+  };
+}
+
+function requestContext(session) {
+  const [country, city, latitude, longitude] = LOCATIONS[session.locationIndex];
+  return {
+    "User-Agent": USER_AGENTS[session.userAgentIndex],
+    "x-vercel-ip-country": country,
+    "x-vercel-ip-city": encodeURIComponent(city),
+    "x-vercel-ip-latitude": String(latitude),
+    "x-vercel-ip-longitude": String(longitude),
+  };
+}
+
+function percentile(sorted, fraction) {
+  if (sorted.length === 0) return 0;
+  return sorted[
+    Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)
+  ];
+}
+
+function hasValidSessionTotals(value) {
+  return (
+    value !== null &&
+    typeof value === "object" &&
+    [value.requestedSessions, value.foundSessions, value.storedEvents].every(
+      (total) => Number.isSafeInteger(total) && total >= 0,
+    )
+  );
+}
+
+async function main() {
+  const options = parseOptions(process.argv.slice(2));
+  if (options.batchSize > 50) throw new Error("--batch-size cannot exceed 50");
+  if (options.certify && options.durationSeconds < 600) {
+    throw new Error("--certify requires --duration of at least 600 seconds");
+  }
+  if (options.certify && !options.convexUrl) {
+    throw new Error("--certify requires --convex-url reconciliation");
+  }
+  if (options.certify && Number.isFinite(options.maximumRequests)) {
+    throw new Error("--certify cannot be combined with --requests");
+  }
+  const runId = randomUUID();
+  const sessions = Array.from({ length: options.sessionCount }, (_, index) => ({
+    id: `bench-${runId}-${index}`,
+    visitorId: `visitor-${runId}-${index}`,
+    sequence: 0,
+    locationIndex: index % LOCATIONS.length,
+    userAgentIndex: index % USER_AGENTS.length,
+  }));
+  let nextRequest = 0;
+  let stopping = false;
+  let terminationReason = null;
+  let accepted = 0;
+  let duplicates = 0;
+  let rejected = 0;
+  let failures = 0;
+  let failedEvents = 0;
+  let completedRequests = 0;
+  let sentEvents = 0;
+  let sentHeartbeats = 0;
+  let sentBytes = 0;
+  let acceptedHeartbeats = 0;
+  let reconciliationReliable = true;
+  const errors = new Map();
+  const durations = [];
+
+  const buildRequest = (requestIndex) => {
+    const session =
+      options.profile === "lean"
+        ? {
+            id: `bench-${runId}-${requestIndex}`,
+            visitorId: `visitor-${runId}-${requestIndex}`,
+            sequence: 0,
+            locationIndex: requestIndex % LOCATIONS.length,
+            userAgentIndex: requestIndex % USER_AGENTS.length,
+          }
+        : sessions[requestIndex % sessions.length];
+    const sequenceStart = session.sequence;
+    session.sequence += options.batchSize;
+    const now = Date.now();
+    const events = Array.from({ length: options.batchSize }, (_, index) =>
+      eventFor(options.profile, session, sequenceStart + index, now + index),
+    );
+    return {
+      body: JSON.stringify({ siteId: options.siteId, events }),
+      context: requestContext(session),
+      heartbeats: events.filter((event) => event.type === "heartbeat").length,
+    };
+  };
+
+  const dryRequest = buildRequest(0);
+  if (Buffer.byteLength(dryRequest.body) > 64 * 1024) {
+    throw new Error("generated request exceeds the 64 KiB HTTP limit");
+  }
+  if (options.dryRun) {
+    process.stdout.write(
+      `${JSON.stringify(
+        {
+          profile: options.profile,
+          batchSize: options.batchSize,
+          requestBytes: Buffer.byteLength(dryRequest.body),
+          heartbeats: dryRequest.heartbeats,
+          location: requestContext(sessions[0]),
+        },
+        null,
+        2,
+      )}\n`,
+    );
+    return;
+  }
+  if (options.profile !== "lean") sessions[0].sequence = 0;
+  if (!options.url || !options.siteId) {
+    throw new Error(
+      "--url and --site-id are required unless --dry-run is used",
+    );
+  }
+
+  const stopForSignal = (signal) => {
+    stopping = true;
+    terminationReason = signal;
+  };
+  const stopForInterrupt = () => stopForSignal("SIGINT");
+  const stopForTermination = () => stopForSignal("SIGTERM");
+  process.once("SIGINT", stopForInterrupt);
+  process.once("SIGTERM", stopForTermination);
+
+  const convex = options.convexUrl
+    ? new ConvexHttpClient(options.convexUrl)
+    : null;
+  if (convex) {
+    const preflight = await convex.query(reconcileSessionsQuery, {
+      siteId: options.siteId,
+      runId,
+      start: 0,
+      count: 1,
+    });
+    if (
+      !hasValidSessionTotals(preflight) ||
+      preflight.requestedSessions !== 1 ||
+      preflight.foundSessions !== 0 ||
+      preflight.storedEvents !== 0
+    ) {
+      throw new Error("reconciliation preflight returned unexpected totals");
+    }
+  }
+  const startedAt = performance.now();
+  const deadline = startedAt + options.durationSeconds * 1_000;
+  const recordError = (key) => errors.set(key, (errors.get(key) ?? 0) + 1);
+  const worker = async () => {
+    while (!stopping) {
+      if (nextRequest >= options.maximumRequests) {
+        terminationReason ??= "request-limit";
+        return;
+      }
+      if (performance.now() >= deadline) {
+        terminationReason ??= "duration";
+        return;
+      }
+      const requestIndex = nextRequest++;
+      const request = buildRequest(requestIndex);
+      const requestBytes = Buffer.byteLength(request.body);
+      if (requestBytes > 64 * 1024) {
+        throw new Error(
+          `generated request ${requestIndex} exceeds the 64 KiB HTTP limit`,
+        );
+      }
+      sentBytes += requestBytes;
+      sentEvents += options.batchSize;
+      sentHeartbeats += request.heartbeats;
+      const requestStartedAt = performance.now();
+      try {
+        const response = await globalThis.fetch(options.url, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Origin: options.origin,
+            ...request.context,
+          },
+          body: request.body,
+          signal: globalThis.AbortSignal.timeout(30_000),
+        });
+        if (!response.ok) {
+          failures += 1;
+          failedEvents += options.batchSize;
+          const body = await response.text();
+          let code = body.slice(0, 160);
+          try {
+            const parsed = JSON.parse(body);
+            code = parsed?.error?.code ?? parsed?.error?.message ?? code;
+          } catch {
+            // Preserve the bounded response text.
+          }
+          recordError(`${response.status}:${code}`);
+        } else {
+          const result = await response.json();
+          if (
+            ![result.accepted, result.duplicates, result.rejected].every(
+              (value) => Number.isSafeInteger(value) && value >= 0,
+            ) ||
+            result.accepted + result.duplicates + result.rejected !==
+              options.batchSize
+          ) {
+            throw new Error("response has invalid event accounting");
+          }
+          accepted += result.accepted;
+          duplicates += result.duplicates;
+          rejected += result.rejected;
+          if (result.accepted === options.batchSize) {
+            acceptedHeartbeats += request.heartbeats;
+          } else {
+            reconciliationReliable = false;
+          }
+        }
+      } catch (error) {
+        failures += 1;
+        failedEvents += options.batchSize;
+        recordError(
+          `fetch:${error instanceof Error ? error.message : "unknown"}`,
+        );
+      } finally {
+        durations.push(performance.now() - requestStartedAt);
+        completedRequests += 1;
+      }
+    }
+  };
+
+  const progress = setInterval(() => {
+    const elapsedSeconds = Math.max(
+      0.001,
+      (performance.now() - startedAt) / 1_000,
+    );
+    process.stderr.write(
+      `[${options.label}] ${Math.round(elapsedSeconds)}s ` +
+        `${accepted} accepted ${Math.round(accepted / elapsedSeconds)} events/s ` +
+        `${failures} failures\n`,
+    );
+  }, options.progressSeconds * 1_000);
+  progress.unref();
+  await Promise.all(Array.from({ length: options.concurrency }, worker));
+  clearInterval(progress);
+
+  const elapsedMs = performance.now() - startedAt;
+  terminationReason ??=
+    performance.now() >= deadline ? "duration" : "completed";
+  process.removeListener("SIGINT", stopForInterrupt);
+  process.removeListener("SIGTERM", stopForTermination);
+  const durationCompleted = terminationReason === "duration";
+  durations.sort((left, right) => left - right);
+  const expectedSessions =
+    options.profile === "lean"
+      ? nextRequest
+      : Math.min(nextRequest, options.sessionCount);
+  let sessionTotals = null;
+  if (convex && expectedSessions > 0) {
+    try {
+      sessionTotals = {
+        requestedSessions: 0,
+        foundSessions: 0,
+        storedEvents: 0,
+      };
+      for (let start = 0; start < expectedSessions; start += 100) {
+        const page = await convex.query(reconcileSessionsQuery, {
+          siteId: options.siteId,
+          runId,
+          start,
+          count: Math.min(100, expectedSessions - start),
+        });
+        if (!hasValidSessionTotals(page)) {
+          throw new Error("reconciliation response has invalid totals");
+        }
+        sessionTotals.requestedSessions += page.requestedSessions;
+        sessionTotals.foundSessions += page.foundSessions;
+        sessionTotals.storedEvents += page.storedEvents;
+      }
+    } catch (error) {
+      reconciliationReliable = false;
+      sessionTotals = null;
+      recordError(
+        `reconciliation:${error instanceof Error ? error.message : "unknown"}`,
+      );
+    }
+  }
+  const expectedStoredEvents = reconciliationReliable
+    ? accepted - acceptedHeartbeats
+    : null;
+  const storedEvents = sessionTotals?.storedEvents ?? expectedStoredEvents;
+  const reconciliation = sessionTotals
+    ? {
+        scope: "run-session-ids",
+        reliable: reconciliationReliable && failures === 0 && rejected === 0,
+        expectedSessions,
+        requestedSessions: sessionTotals.requestedSessions,
+        foundSessions: sessionTotals.foundSessions,
+        acceptedEvents: accepted,
+        acceptedHeartbeats: reconciliationReliable ? acceptedHeartbeats : null,
+        expectedStoredEvents,
+        storedEvents: sessionTotals.storedEvents,
+        matches:
+          reconciliationReliable &&
+          failures === 0 &&
+          rejected === 0 &&
+          expectedStoredEvents !== null &&
+          sessionTotals.requestedSessions === expectedSessions &&
+          sessionTotals.foundSessions === expectedSessions &&
+          sessionTotals.storedEvents === expectedStoredEvents,
+      }
+    : null;
+  const cleanRun =
+    failures === 0 && duplicates === 0 && rejected === 0 && durationCompleted;
+  const reconciliationPassed = Boolean(
+    reconciliation?.reliable && reconciliation.matches,
+  );
+  const certificationPassed =
+    options.certify && cleanRun && reconciliationPassed;
+  const result = {
+    label: options.label,
+    runId,
+    profile: options.profile,
+    targetDurationSeconds: options.durationSeconds,
+    elapsedMs: Math.round(elapsedMs),
+    terminationReason,
+    durationCompleted,
+    certification: {
+      requested: options.certify,
+      passed: certificationPassed,
+      minimumDurationSeconds: 600,
+    },
+    concurrency: options.concurrency,
+    batchSize: options.batchSize,
+    sessionPool:
+      options.profile === "lean" ? "unique-per-request" : options.sessionCount,
+    maximumRequests: Number.isFinite(options.maximumRequests)
+      ? options.maximumRequests
+      : null,
+    requests: { started: nextRequest, completed: completedRequests, failures },
+    events: {
+      sent: sentEvents,
+      heartbeats: sentHeartbeats,
+      accepted,
+      stored: storedEvents,
+      duplicates,
+      rejected,
+      failed: failedEvents,
+    },
+    errors: Object.fromEntries(errors),
+    throughput: {
+      eventsPerSecond: Math.round((accepted * 1_000) / elapsedMs),
+      storedEventsPerSecond: reconciliationPassed
+        ? Math.round((storedEvents * 1_000) / elapsedMs)
+        : null,
+      estimatedEventsPerDay: durationCompleted
+        ? Math.round((accepted * 86_400_000) / elapsedMs)
+        : null,
+      estimatedStoredEventsPerDay:
+        durationCompleted && reconciliationPassed
+          ? Math.round((storedEvents * 86_400_000) / elapsedMs)
+          : null,
+      certifiedEventsPerDay: certificationPassed
+        ? Math.round((accepted * 86_400_000) / elapsedMs)
+        : null,
+      certifiedStoredEventsPerDay: certificationPassed
+        ? Math.round((storedEvents * 86_400_000) / elapsedMs)
+        : null,
+      requestBytesPerSecond: Math.round((sentBytes * 1_000) / elapsedMs),
+    },
+    latencyMs: {
+      p50: Math.round(percentile(durations, 0.5)),
+      p95: Math.round(percentile(durations, 0.95)),
+      p99: Math.round(percentile(durations, 0.99)),
+      max: Math.round(durations.at(-1) ?? 0),
+    },
+    reconciliation,
+  };
+  process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
+  if (
+    failures > 0 ||
+    duplicates > 0 ||
+    rejected > 0 ||
+    !durationCompleted ||
+    (options.convexUrl && !reconciliationPassed) ||
+    (options.certify && !certificationPassed)
+  ) {
+    process.exitCode = 1;
+  }
+}
+
+await main();

package/scripts/generate-federation-keys.mjs

@@ -0,0 +1,20 @@
+import { randomUUID } from "node:crypto";
+import process from "node:process";
+import { exportJWK, exportPKCS8, generateKeyPair } from "jose";
+
+const { privateKey, publicKey } = await generateKeyPair("RS256", {
+  extractable: true,
+});
+const publicJwk = await exportJWK(publicKey);
+
+process.stdout.write(
+  JSON.stringify(
+    {
+      keyId: `rastro-${randomUUID()}`,
+      privateKeyPem: await exportPKCS8(privateKey),
+      publicJwk,
+    },
+    null,
+    2,
+  ) + "\n",
+);

package/src/component/_generated/api.ts

@@ -9,19 +9,28 @@
  */
 
 import type * as affiliates from "../affiliates.js";
+import type * as cardinality from "../cardinality.js";
 import type * as constants from "../constants.js";
+import type * as coverage from "../coverage.js";
+import type * as diagnostics from "../diagnostics.js";
 import type * as errors from "../errors.js";
+import type * as eventStore from "../eventStore.js";
 import type * as funnels from "../funnels.js";
+import type * as geo from "../geo.js";
 import type * as goals from "../goals.js";
 import type * as guards from "../guards.js";
 import type * as http from "../http.js";
+import type * as identity from "../identity.js";
 import type * as ingest from "../ingest.js";
 import type * as live from "../live.js";
 import type * as reports from "../reports.js";
 import type * as retention from "../retention.js";
 import type * as sanitize from "../sanitize.js";
 import type * as sites from "../sites.js";
+import type * as useragent from "../useragent.js";
 import type * as validators from "../validators.js";
+import type * as visitors from "../visitors.js";
+import type * as vitals from "../vitals.js";
 
 import type {
   ApiFromModules,
@@ -32,19 +41,28 @@ import { anyApi, componentsGeneric } from "convex/server";
 
 const fullApi: ApiFromModules<{
   affiliates: typeof affiliates;
+  cardinality: typeof cardinality;
   constants: typeof constants;
+  coverage: typeof coverage;
+  diagnostics: typeof diagnostics;
   errors: typeof errors;
+  eventStore: typeof eventStore;
   funnels: typeof funnels;
+  geo: typeof geo;
   goals: typeof goals;
   guards: typeof guards;
   http: typeof http;
+  identity: typeof identity;
   ingest: typeof ingest;
   live: typeof live;
   reports: typeof reports;
   retention: typeof retention;
   sanitize: typeof sanitize;
   sites: typeof sites;
+  useragent: typeof useragent;
   validators: typeof validators;
+  visitors: typeof visitors;
+  vitals: typeof vitals;
 }> = anyApi as any;
 
 /**