@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.1

package/src/component/live.ts

@@ -1,6 +1,9 @@
 import { v } from "convex/values";
 import { internal } from "./_generated/api.js";
 import { internalMutation } from "./_generated/server.js";
+import { LIVE_SWEEP_INTERVAL_MS } from "./constants.js";
+
+const SWEEP_BATCH_SIZE = 500;
 
 export const expire = internalMutation({
   args: {
@@ -21,16 +24,88 @@ export const expire = internalMutation({
       return { expired: false, rescheduled: false, stale: true };
     }
     if (liveSession.expiresAt > Date.now()) {
-      await ctx.db.patch("liveSessions", liveSession._id, {
-        expiryChainDeadline: liveSession.expiresAt,
-      });
-      await ctx.scheduler.runAt(liveSession.expiresAt, internal.live.expire, {
-        liveSessionId: liveSession._id,
-        scheduledDeadline: liveSession.expiresAt,
-      });
-      return { expired: false, rescheduled: true, stale: false };
+      const state = await ctx.db
+        .query("liveSweepStates")
+        .withIndex("by_siteId", (range) => range.eq("siteId", liveSession.siteId))
+        .unique();
+      const scheduledAt = Math.min(
+        liveSession.expiresAt,
+        Date.now() + LIVE_SWEEP_INTERVAL_MS,
+      );
+      if (!state || state.scheduledAt > scheduledAt) {
+        if (state) {
+          await ctx.db.patch("liveSweepStates", state._id, {
+            scheduledAt,
+          });
+        } else {
+          await ctx.db.insert("liveSweepStates", {
+            siteId: liveSession.siteId,
+            scheduledAt,
+          });
+        }
+        await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
+          siteId: liveSession.siteId,
+          scheduledAt,
+        });
+        return { expired: false, rescheduled: true, stale: false };
+      }
+      return { expired: false, rescheduled: false, stale: false };
     }
     await ctx.db.delete("liveSessions", liveSession._id);
     return { expired: true, rescheduled: false, stale: false };
   },
 });
+
+export const sweep = internalMutation({
+  args: {
+    siteId: v.id("sites"),
+    scheduledAt: v.number(),
+  },
+  returns: v.object({
+    expired: v.number(),
+    rescheduled: v.boolean(),
+    stale: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    const state = await ctx.db
+      .query("liveSweepStates")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    if (!state || state.scheduledAt !== args.scheduledAt) {
+      return { expired: 0, rescheduled: false, stale: true };
+    }
+
+    const now = Date.now();
+    const expired = await ctx.db
+      .query("liveSessions")
+      .withIndex("by_siteId_and_expiresAt", (range) =>
+        range.eq("siteId", args.siteId).lte("expiresAt", now),
+      )
+      .take(SWEEP_BATCH_SIZE);
+    for (const session of expired) {
+      await ctx.db.delete("liveSessions", session._id);
+    }
+
+    const next = await ctx.db
+      .query("liveSessions")
+      .withIndex("by_siteId_and_expiresAt", (range) =>
+        range.eq("siteId", args.siteId),
+      )
+      .order("asc")
+      .first();
+    if (!next) {
+      await ctx.db.delete("liveSweepStates", state._id);
+      return { expired: expired.length, rescheduled: false, stale: false };
+    }
+
+    const scheduledAt = expired.length === SWEEP_BATCH_SIZE
+      ? now
+      : now + LIVE_SWEEP_INTERVAL_MS;
+    await ctx.db.patch("liveSweepStates", state._id, { scheduledAt });
+    await ctx.scheduler.runAt(scheduledAt, internal.live.sweep, {
+      siteId: args.siteId,
+      scheduledAt,
+    });
+    return { expired: expired.length, rescheduled: true, stale: false };
+  },
+});

package/src/component/reports.ts

@@ -3,22 +3,30 @@ import {
   paginationResultValidator,
 } from "convex/server";
 import { v } from "convex/values";
+import type { Doc } from "./_generated/dataModel.js";
 import { query } from "./_generated/server.js";
 import {
   AGGREGATE_SHARDS,
   DAY_MS,
   HOUR_MS,
+  HOURLY_AGGREGATE_SHARDS,
   MAX_AFFILIATES_PER_SITE,
   MAX_FUNNELS_PER_SITE,
   MAX_GOALS_PER_SITE,
   MAX_JOURNEY_EVENTS,
   MAX_LIVE_VISITORS,
-  MAX_MANUAL_PAGINATION_OFFSET,
   MAX_OVERVIEW_AGGREGATE_ROWS,
   MAX_REPORT_RANGE_DAYS,
   MAX_REPORT_SITES,
 } from "./constants.js";
 import { fail } from "./errors.js";
+import {
+  compareJourneyEvents,
+  hydrateEventContexts,
+  type JourneyCursor,
+  loadSessionJourneyEvents,
+  type RawJourneyEvent,
+} from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
 import {
   affiliateFieldsValidator,
@@ -33,7 +41,7 @@ const sessionDocumentValidator = sessionFieldsValidator.extend({
   _creationTime: v.number(),
 });
 const eventDocumentValidator = eventFieldsValidator.extend({
-  _id: v.id("events"),
+  _id: v.string(),
   _creationTime: v.number(),
 });
 const liveSessionDocumentValidator = v.object({
@@ -131,15 +139,16 @@ export const overview = query({
     }
     const currency = [...currencies][0];
     const { from, to } = validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-    const interval = args.interval ?? (to - from <= 7 * DAY_MS ? "hour" : "day");
-    if (interval === "hour" && to - from > 7 * DAY_MS) {
-      fail("INVALID_ARGUMENT", "hourly reports are limited to seven days");
+    const interval = args.interval ?? (to - from <= DAY_MS ? "hour" : "day");
+    if (interval === "hour" && to - from > DAY_MS) {
+      fail("INVALID_ARGUMENT", "hourly reports are limited to 24 hours");
     }
     const intervalMs = interval === "hour" ? HOUR_MS : DAY_MS;
     const firstBucket = Math.floor(from / intervalMs) * intervalMs;
     const lastBucket = Math.floor(to / intervalMs) * intervalMs;
     const bucketCount = Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
-    const perSiteLimit = bucketCount * AGGREGATE_SHARDS;
+    const perSiteLimit = bucketCount *
+      (interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
     if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
       fail(
         "LIMIT_EXCEEDED",
@@ -272,18 +281,59 @@ export const listSessions = query({
   returns: paginationResultValidator(sessionDocumentValidator),
   handler: async (ctx, args) => {
     validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-    const pagination = parseManualPagination(args.paginationOpts);
-    const rows = await ctx.db
-      .query("sessions")
-      .withIndex("by_siteId_and_lastSeenAt", (range) =>
-        range
-          .eq("siteId", args.siteId)
-          .gte("lastSeenAt", args.from)
-          .lte("lastSeenAt", args.to),
-      )
-      .order("desc")
-      .take(pagination.offset + pagination.numItems + 1);
-    return manualPaginationResult(rows, pagination);
+    const pagination = parseKeysetPagination(args.paginationOpts, {
+      kind: "sessions",
+      siteId: String(args.siteId),
+      from: args.from,
+      to: args.to,
+    });
+    const rows: Doc<"sessions">[] = [];
+    const take = pagination.numItems + 1;
+    if (pagination.cursor) {
+      rows.push(
+        ...(await ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_lastSeenAt", (range) =>
+            range
+              .eq("siteId", args.siteId)
+              .eq("lastSeenAt", pagination.cursor!.timestamp)
+              .lt("_creationTime", pagination.cursor!.creationTime),
+          )
+          .order("desc")
+          .take(take)),
+      );
+      if (rows.length < take) {
+        rows.push(
+          ...(await ctx.db
+            .query("sessions")
+            .withIndex("by_siteId_and_lastSeenAt", (range) =>
+              range
+                .eq("siteId", args.siteId)
+                .gte("lastSeenAt", args.from)
+                .lt("lastSeenAt", pagination.cursor!.timestamp),
+            )
+            .order("desc")
+            .take(take - rows.length)),
+        );
+      }
+    } else {
+      rows.push(
+        ...(await ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_lastSeenAt", (range) =>
+            range
+              .eq("siteId", args.siteId)
+              .gte("lastSeenAt", args.from)
+              .lte("lastSeenAt", args.to),
+          )
+          .order("desc")
+          .take(take)),
+      );
+    }
+    return keysetPaginationResult(rows, pagination, (row) => ({
+      timestamp: row.lastSeenAt,
+      creationTime: row._creationTime,
+    }));
   },
 });
 
@@ -298,19 +348,23 @@ export const sessionJourney = query({
   returns: paginationResultValidator(eventDocumentValidator),
   handler: async (ctx, args) => {
     validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-    const pagination = parseManualPagination(args.paginationOpts);
-    const rows = await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
-        range
-          .eq("siteId", args.siteId)
-          .eq("sessionId", args.sessionId)
-          .gte("timestamp", args.from)
-          .lte("timestamp", args.to),
-      )
-      .order("asc")
-      .take(pagination.offset + pagination.numItems + 1);
-    return manualPaginationResult(rows, pagination);
+    const pagination = parseJourneyPagination(args.paginationOpts, {
+      kind: "events",
+      siteId: String(args.siteId),
+      sessionId: args.sessionId,
+      from: args.from,
+      to: args.to,
+    });
+    const rows = await loadSessionJourneyEvents(ctx, {
+      siteId: args.siteId,
+      sessionId: args.sessionId,
+      from: args.from,
+      to: args.to,
+      cursor: pagination.cursor,
+      resultLimit: pagination.numItems + 1,
+    });
+    const hydratedRows = await hydrateEventContexts(ctx, rows);
+    return journeyPaginationResult(hydratedRows, pagination);
   },
 });
 
@@ -324,18 +378,59 @@ export const listConversions = query({
   returns: paginationResultValidator(conversionDocumentValidator),
   handler: async (ctx, args) => {
     validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-    const pagination = parseManualPagination(args.paginationOpts);
-    const rows = await ctx.db
-      .query("conversions")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range
-          .eq("siteId", args.siteId)
-          .gte("timestamp", args.from)
-          .lte("timestamp", args.to),
-      )
-      .order("desc")
-      .take(pagination.offset + pagination.numItems + 1);
-    return manualPaginationResult(rows, pagination);
+    const pagination = parseKeysetPagination(args.paginationOpts, {
+      kind: "conversions",
+      siteId: String(args.siteId),
+      from: args.from,
+      to: args.to,
+    });
+    const rows: Doc<"conversions">[] = [];
+    const take = pagination.numItems + 1;
+    if (pagination.cursor) {
+      rows.push(
+        ...(await ctx.db
+          .query("conversions")
+          .withIndex("by_siteId_and_timestamp", (range) =>
+            range
+              .eq("siteId", args.siteId)
+              .eq("timestamp", pagination.cursor!.timestamp)
+              .lt("_creationTime", pagination.cursor!.creationTime),
+          )
+          .order("desc")
+          .take(take)),
+      );
+      if (rows.length < take) {
+        rows.push(
+          ...(await ctx.db
+            .query("conversions")
+            .withIndex("by_siteId_and_timestamp", (range) =>
+              range
+                .eq("siteId", args.siteId)
+                .gte("timestamp", args.from)
+                .lt("timestamp", pagination.cursor!.timestamp),
+            )
+            .order("desc")
+            .take(take - rows.length)),
+        );
+      }
+    } else {
+      rows.push(
+        ...(await ctx.db
+          .query("conversions")
+          .withIndex("by_siteId_and_timestamp", (range) =>
+            range
+              .eq("siteId", args.siteId)
+              .gte("timestamp", args.from)
+              .lte("timestamp", args.to),
+          )
+          .order("desc")
+          .take(take)),
+      );
+    }
+    return keysetPaginationResult(rows, pagination, (row) => ({
+      timestamp: row.timestamp,
+      creationTime: row._creationTime,
+    }));
   },
 });
 
@@ -355,25 +450,52 @@ export const visitorJourney = query({
     if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
       fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
     }
-    const rows = [];
-    for (const siteId of siteIds) {
-      rows.push(
-        ...(await ctx.db
-          .query("events")
-          .withIndex("by_siteId_and_visitorId_and_timestamp", (range) =>
+    const sessionGroups = await Promise.all(
+      siteIds.map((siteId) =>
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
             range
               .eq("siteId", siteId)
               .eq("visitorId", args.visitorId)
-              .gte("timestamp", args.from)
-              .lte("timestamp", args.to),
+              .gte("lastSeenAt", args.from),
           )
           .order("asc")
-          .take(limit)),
-      );
+          .take(MAX_JOURNEY_EVENTS + 1),
+      ),
+    );
+    const sessions = sessionGroups
+      .flat()
+      .filter((session) => session.startedAt <= args.to)
+      .sort((left, right) => left.startedAt - right.startedAt)
+      .slice(0, MAX_JOURNEY_EVENTS);
+    const rows: RawJourneyEvent[] = [];
+    let overlappingSessionsRead = 0;
+    for (const session of sessions) {
+      const resultIsFull = rows.length >= limit;
+      const cutoff = resultIsFull
+        ? rows[limit - 1]!.timestamp
+        : args.to;
+      if (session.startedAt > cutoff) break;
+      if (resultIsFull && ++overlappingSessionsRead > 8) {
+        fail(
+          "LIMIT_EXCEEDED",
+          "visitor journey has too many overlapping sessions",
+          { maximumOverlappingSessions: 8 },
+        );
+      }
+      rows.push(...await loadSessionJourneyEvents(ctx, {
+        siteId: session.siteId,
+        sessionId: session.sessionId,
+        from: args.from,
+        to: Math.min(args.to, cutoff),
+        cursor: null,
+        resultLimit: resultIsFull ? limit : limit - rows.length,
+      }));
+      rows.sort(compareJourneyEvents);
+      if (rows.length > limit) rows.length = limit;
     }
-    return rows
-      .sort((left, right) => left.timestamp - right.timestamp || left.sequence - right.sequence)
-      .slice(0, limit);
+    return hydrateEventContexts(ctx, rows);
   },
 });
 
@@ -633,24 +755,45 @@ function validateRange(from: number, to: number, maxDays: number) {
   return { from: Math.trunc(from), to: Math.trunc(to) };
 }
 
-type ManualPaginationInput = {
+type KeysetPaginationInput = {
   cursor: string | null;
   numItems: number;
   maximumRowsRead?: number;
   maximumBytesRead?: number;
 };
 
-type ManualPagination = {
-  offset: number;
+type PaginationScope = {
+  kind: "sessions" | "events" | "conversions";
+  siteId: string;
+  sessionId?: string;
+  from: number;
+  to: number;
+};
+
+type KeysetCursor = {
+  timestamp: number;
+  creationTime: number;
+};
+
+type KeysetPagination = {
+  numItems: number;
+  scope: PaginationScope;
+  cursor: KeysetCursor | null;
+};
+
+type JourneyPagination = {
   numItems: number;
+  scope: PaginationScope & { kind: "events"; sessionId: string };
+  cursor: JourneyCursor | null;
 };
 
-function parseManualPagination(
-  options: ManualPaginationInput,
-): ManualPagination {
+function parseKeysetPagination(
+  options: KeysetPaginationInput,
+  scope: PaginationScope,
+): KeysetPagination {
   validatePagination(options);
   if (options.cursor === null) {
-    return { offset: 0, numItems: options.numItems };
+    return { numItems: options.numItems, scope, cursor: null };
   }
   let decoded: unknown;
   try {
@@ -660,35 +803,130 @@ function parseManualPagination(
   }
   if (
     !isPlainRecord(decoded) ||
-    decoded.v !== 1 ||
-    typeof decoded.o !== "number" ||
-    !Number.isSafeInteger(decoded.o) ||
-    decoded.o < 0 ||
-    decoded.o > MAX_MANUAL_PAGINATION_OFFSET
+    decoded.v !== 2 ||
+    decoded.k !== scope.kind ||
+    decoded.s !== scope.siteId ||
+    decoded.n !== (scope.sessionId ?? null) ||
+    decoded.f !== scope.from ||
+    decoded.t !== scope.to ||
+    typeof decoded.x !== "number" ||
+    !Number.isFinite(decoded.x) ||
+    decoded.x < scope.from ||
+    decoded.x > scope.to ||
+    typeof decoded.c !== "number" ||
+    !Number.isFinite(decoded.c) ||
+    decoded.c < 0
   ) {
-    fail("INVALID_ARGUMENT", "pagination cursor is invalid or exceeds the offset limit");
+    fail("INVALID_ARGUMENT", "pagination cursor is invalid for this query");
   }
-  return { offset: decoded.o, numItems: options.numItems };
+  return {
+    numItems: options.numItems,
+    scope,
+    cursor: {
+      timestamp: decoded.x,
+      creationTime: decoded.c,
+    },
+  };
 }
 
-function manualPaginationResult<T>(
+function parseJourneyPagination(
+  options: KeysetPaginationInput,
+  scope: PaginationScope & { kind: "events"; sessionId: string },
+): JourneyPagination {
+  validatePagination(options);
+  if (options.cursor === null) {
+    return { numItems: options.numItems, scope, cursor: null };
+  }
+  let decoded: unknown;
+  try {
+    decoded = JSON.parse(atob(options.cursor));
+  } catch {
+    fail("INVALID_ARGUMENT", "pagination cursor is malformed");
+  }
+  if (
+    !isPlainRecord(decoded) ||
+    decoded.v !== 4 ||
+    decoded.k !== scope.kind ||
+    decoded.s !== scope.siteId ||
+    decoded.n !== scope.sessionId ||
+    decoded.f !== scope.from ||
+    decoded.t !== scope.to ||
+    typeof decoded.x !== "number" ||
+    !Number.isFinite(decoded.x) ||
+    decoded.x < scope.from ||
+    decoded.x > scope.to ||
+    typeof decoded.c !== "number" ||
+    !Number.isFinite(decoded.c) ||
+    decoded.c < 0 ||
+    typeof decoded.i !== "string" ||
+    decoded.i.length === 0
+  ) {
+    fail("INVALID_ARGUMENT", "pagination cursor is invalid for this query");
+  }
+  return {
+    numItems: options.numItems,
+    scope,
+    cursor: {
+      timestamp: decoded.x,
+      creationTime: decoded.c,
+      id: decoded.i,
+    },
+  };
+}
+
+function keysetPaginationResult<T>(
   rows: T[],
-  pagination: ManualPagination,
+  pagination: KeysetPagination,
+  cursorFor: (row: T) => KeysetCursor,
 ) {
-  const page = rows.slice(
-    pagination.offset,
-    pagination.offset + pagination.numItems,
-  );
-  const nextOffset = pagination.offset + page.length;
-  const isDone =
-    rows.length <= pagination.offset + pagination.numItems ||
-    nextOffset >= MAX_MANUAL_PAGINATION_OFFSET;
+  const page = rows.slice(0, pagination.numItems);
+  const isDone = rows.length <= pagination.numItems;
+  const next = page.length > 0
+    ? cursorFor(page[page.length - 1])
+    : pagination.cursor;
+  return {
+    page,
+    isDone,
+    continueCursor: btoa(JSON.stringify({
+      v: 2,
+      k: pagination.scope.kind,
+      s: pagination.scope.siteId,
+      n: pagination.scope.sessionId ?? null,
+      f: pagination.scope.from,
+      t: pagination.scope.to,
+      x: next?.timestamp ?? pagination.scope.to,
+      c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
+    })),
+  };
+}
+
+function journeyPaginationResult(
+  rows: Awaited<ReturnType<typeof hydrateEventContexts>>,
+  pagination: JourneyPagination,
+) {
+  const page = rows.slice(0, pagination.numItems);
+  const isDone = rows.length <= pagination.numItems;
+  const last = page.at(-1);
+  const next = last
+      ? {
+          timestamp: last.timestamp,
+          creationTime: last._creationTime,
+          id: last._id,
+      }
+    : pagination.cursor;
   return {
     page,
     isDone,
     continueCursor: btoa(JSON.stringify({
-      v: 1,
-      o: Math.min(nextOffset, MAX_MANUAL_PAGINATION_OFFSET),
+      v: 4,
+      k: pagination.scope.kind,
+      s: pagination.scope.siteId,
+      n: pagination.scope.sessionId,
+      f: pagination.scope.from,
+      t: pagination.scope.to,
+      x: next?.timestamp ?? pagination.scope.to,
+      c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
+      i: next?.id ?? "~",
     })),
   };
 }

package/src/component/retention.ts

@@ -4,6 +4,7 @@ import type { MutationCtx } from "./_generated/server.js";
 import { mutation } from "./_generated/server.js";
 import { MAX_RETENTION_BATCH } from "./constants.js";
 import { fail } from "./errors.js";
+import { deleteTelemetryBefore } from "./eventStore.js";
 
 const cleanupKindValidator = v.union(
   v.literal("events"),
@@ -37,19 +38,29 @@ export const cleanup = mutation({
     if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > MAX_RETENTION_BATCH) {
       fail("INVALID_ARGUMENT", `batchSize must be an integer between 1 and ${MAX_RETENTION_BATCH}`);
     }
+    const effectiveBatchSize = args.kind === "events"
+      ? Math.min(batchSize, 32)
+      : batchSize;
+    if (args.kind === "events") {
+      return await deleteTelemetryBefore(ctx, {
+        siteId: args.siteId,
+        before: args.before,
+        batchSize: effectiveBatchSize,
+      });
+    }
     const deletions = await deletionsForCleanup(
       ctx,
       args.siteId,
       args.kind,
       args.before,
-      batchSize + 1,
+      effectiveBatchSize + 1,
     );
-    for (const remove of deletions.slice(0, batchSize)) {
+    for (const remove of deletions.slice(0, effectiveBatchSize)) {
       await remove();
     }
     return {
-      deleted: Math.min(deletions.length, batchSize),
-      hasMore: deletions.length > batchSize,
+      deleted: Math.min(deletions.length, effectiveBatchSize),
+      hasMore: deletions.length > effectiveBatchSize,
     };
   },
 });
@@ -78,14 +89,6 @@ async function deletionsForCleanup(
   before: number,
   limit: number,
 ): Promise<CleanupDelete[]> {
-  if (kind === "events") {
-    return (await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range.eq("siteId", siteId).lt("timestamp", before),
-      )
-      .take(limit)).map((row) => () => ctx.db.delete("events", row._id));
-  }
   if (kind === "sessions") {
     return (await ctx.db
       .query("sessions")