@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/src/component/retention.ts

@@ -29,6 +29,7 @@ export const cleanupKindValidator = v.union(
   v.literal("goalRollups"),
   v.literal("funnelRollups"),
   v.literal("affiliateRollups"),
+  v.literal("vitalRollups"),
   v.literal("attributions"),
   v.literal("liveSessions"),
   v.literal("ingestWindows"),
@@ -110,7 +111,8 @@ export const setPolicy = mutation({
         updatedAt: now,
       };
       const stateId = existing
-        ? (await ctx.db.replace("retentionStates", existing._id, fields), existing._id)
+        ? (await ctx.db.replace("retentionStates", existing._id, fields),
+          existing._id)
         : await ctx.db.insert("retentionStates", fields);
       await ctx.scheduler.runAfter(0, internal.retention.runScheduled, {
         stateId,
@@ -133,7 +135,9 @@ export const disablePolicy = mutation({
     const now = Date.now();
     const states = await ctx.db
       .query("retentionStates")
-      .withIndex("by_siteId_and_kind", (range) => range.eq("siteId", args.siteId))
+      .withIndex("by_siteId_and_kind", (range) =>
+        range.eq("siteId", args.siteId),
+      )
       .take(POLICY_CLEANUP_KINDS.length + 1);
     for (const state of states) {
       await ctx.db.patch("retentionStates", state._id, {
@@ -170,29 +174,33 @@ export const runPolicy = mutation({
     ) {
       return { deleted: 0, hasMore: true, before: existing.before };
     }
-    const before = existing && existing.status !== "complete"
-      ? existing.before
-      : safeCleanupBoundary(
-          args.kind,
-          now - retentionDaysForKind(policy, args.kind) * DAY_MS,
-          now,
-        );
+    const before =
+      existing && existing.status !== "complete"
+        ? existing.before
+        : safeCleanupBoundary(
+            args.kind,
+            now - retentionDaysForKind(policy, args.kind) * DAY_MS,
+            now,
+          );
     const result = await executeCleanupBatch(ctx, { ...args, before });
     const generation = existing?.generation ?? 1;
     const fields = {
       siteId: args.siteId,
       kind: args.kind,
       before,
-      status: result.hasMore ? "pending" as const : "complete" as const,
+      status: result.hasMore ? ("pending" as const) : ("complete" as const),
       generation,
       attempt: existing?.attempt ?? 0,
       scheduledAt: result.hasMore ? now : now + RETENTION_POLICY_INTERVAL_MS,
       deleted:
-        existing?.before === before ? existing.deleted + result.deleted : result.deleted,
+        existing?.before === before
+          ? existing.deleted + result.deleted
+          : result.deleted,
       updatedAt: now,
     };
     const stateId = existing
-      ? (await ctx.db.replace("retentionStates", existing._id, fields), existing._id)
+      ? (await ctx.db.replace("retentionStates", existing._id, fields),
+        existing._id)
       : await ctx.db.insert("retentionStates", fields);
     await recordRetentionBoundary(ctx, args.siteId, args.kind, before);
     if (result.hasMore) {
@@ -295,7 +303,12 @@ export const executeScheduled = internalMutation({
       deleted: state.deleted + result.deleted,
       updatedAt: now,
     });
-    await recordRetentionBoundary(ctx, state.siteId, state.kind as CleanupKind, state.before);
+    await recordRetentionBoundary(
+      ctx,
+      state.siteId,
+      state.kind as CleanupKind,
+      state.before,
+    );
     if (result.hasMore) {
       await ctx.scheduler.runAfter(0, internal.retention.runScheduled, {
         stateId: state._id,
@@ -361,19 +374,24 @@ export const startScheduledCycle = internalMutation({
 export const status = query({
   args: { siteId: v.id("sites") },
   returns: v.object({
-    policy: v.union(policyValidator.extend({ updatedAt: v.number() }), v.null()),
-    jobs: v.array(v.object({
-      kind: cleanupKindValidator,
-      before: v.number(),
-      status: v.union(
-        v.literal("pending"),
-        v.literal("running"),
-        v.literal("complete"),
-      ),
-      deleted: v.number(),
-      scheduledAt: v.union(v.number(), v.null()),
-      updatedAt: v.number(),
-    })),
+    policy: v.union(
+      policyValidator.extend({ updatedAt: v.number() }),
+      v.null(),
+    ),
+    jobs: v.array(
+      v.object({
+        kind: cleanupKindValidator,
+        before: v.number(),
+        status: v.union(
+          v.literal("pending"),
+          v.literal("running"),
+          v.literal("complete"),
+        ),
+        deleted: v.number(),
+        scheduledAt: v.union(v.number(), v.null()),
+        updatedAt: v.number(),
+      }),
+    ),
     backlog: v.number(),
   }),
   handler: async (ctx, args) => {
@@ -383,23 +401,20 @@ export const status = query({
       .unique();
     const jobs = await ctx.db
       .query("retentionStates")
-      .withIndex("by_siteId_and_kind", (range) => range.eq("siteId", args.siteId))
+      .withIndex("by_siteId_and_kind", (range) =>
+        range.eq("siteId", args.siteId),
+      )
       .take(POLICY_CLEANUP_KINDS.length + 1);
-    const publicJobs = jobs.map(({
-      kind,
-      before,
-      status,
-      deleted,
-      scheduledAt,
-      updatedAt,
-    }) => ({
-      kind: kind as CleanupKind,
-      before,
-      status,
-      deleted,
-      scheduledAt: scheduledAt ?? null,
-      updatedAt,
-    }));
+    const publicJobs = jobs.map(
+      ({ kind, before, status, deleted, scheduledAt, updatedAt }) => ({
+        kind: kind as CleanupKind,
+        before,
+        status,
+        deleted,
+        scheduledAt: scheduledAt ?? null,
+        updatedAt,
+      }),
+    );
     return {
       policy: policy
         ? {
@@ -430,6 +445,7 @@ type CleanupKind =
   | "goalRollups"
   | "funnelRollups"
   | "affiliateRollups"
+  | "vitalRollups"
   | "attributions"
   | "liveSessions"
   | "ingestWindows"
@@ -448,6 +464,7 @@ const POLICY_CLEANUP_KINDS: CleanupKind[] = [
   "goalRollups",
   "funnelRollups",
   "affiliateRollups",
+  "vitalRollups",
   "attributions",
   "ingestWindows",
   "siteIngestWindows",
@@ -487,19 +504,25 @@ function retentionDaysForKind(
   return policy.reportsDays;
 }
 
-function safeCleanupBoundary(kind: CleanupKind, requested: number, now: number) {
+function safeCleanupBoundary(
+  kind: CleanupKind,
+  requested: number,
+  now: number,
+) {
   if (!Number.isFinite(requested) || requested < 0) {
     fail("INVALID_ARGUMENT", "before must be a positive timestamp");
   }
   if (kind === "events") return safeBucketBoundary(requested, now, MINUTE_MS);
-  if (kind === "aggregateHour") return safeBucketBoundary(requested, now, HOUR_MS);
+  if (kind === "aggregateHour")
+    return safeBucketBoundary(requested, now, HOUR_MS);
   if (
     kind === "aggregateDay" ||
     kind === "funnelSteps" ||
     kind === "affiliateStats" ||
     kind === "goalRollups" ||
     kind === "funnelRollups" ||
-    kind === "affiliateRollups"
+    kind === "affiliateRollups" ||
+    kind === "vitalRollups"
   ) {
     return safeBucketBoundary(requested, now, DAY_MS);
   }
@@ -512,14 +535,22 @@ function safeCleanupBoundary(kind: CleanupKind, requested: number, now: number)
   return Math.trunc(requested);
 }
 
-function safeBucketBoundary(requested: number, now: number, intervalMs: number) {
+function safeBucketBoundary(
+  requested: number,
+  now: number,
+  intervalMs: number,
+) {
   return Math.min(
     Math.floor(requested / intervalMs) * intervalMs,
     Math.floor(now / intervalMs) * intervalMs,
   );
 }
 
-async function loadState(ctx: MutationCtx, siteId: Id<"sites">, kind: CleanupKind) {
+async function loadState(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  kind: CleanupKind,
+) {
   return ctx.db
     .query("retentionStates")
     .withIndex("by_siteId_and_kind", (range) =>
@@ -539,11 +570,13 @@ async function persistDirectState(
     siteId: args.siteId,
     kind: args.kind,
     before: args.before,
-    status: result.hasMore ? "pending" as const : "complete" as const,
+    status: result.hasMore ? ("pending" as const) : ("complete" as const),
     generation: (existing?.generation ?? 0) + 1,
     attempt: 0,
     deleted:
-      existing?.before === args.before ? existing.deleted + result.deleted : result.deleted,
+      existing?.before === args.before
+        ? existing.deleted + result.deleted
+        : result.deleted,
     updatedAt: now,
   };
   if (existing) {
@@ -573,7 +606,8 @@ async function recordRetentionBoundary(
   );
   let changed = retained.length !== coverage.retained.length;
   for (const dataset of datasets) {
-    const prior = coverage.retained.find((item) => item.dataset === dataset)?.before ?? 0;
+    const prior =
+      coverage.retained.find((item) => item.dataset === dataset)?.before ?? 0;
     const next = Math.max(prior, before);
     retained.push({ dataset, before: next });
     changed ||= next !== prior;
@@ -598,6 +632,7 @@ function coverageDatasetsForKind(kind: CleanupKind): string[] {
   if (kind === "goalRollups") return ["goalRollups"];
   if (kind === "funnelRollups") return ["funnelRollups"];
   if (kind === "affiliateRollups") return ["affiliateRollups"];
+  if (kind === "vitalRollups") return ["vitalRollups"];
   return [];
 }
 
@@ -611,10 +646,18 @@ async function executeCleanupBatch(
   },
 ) {
   const batchSize = args.batchSize ?? 100;
-  if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > MAX_RETENTION_BATCH) {
-    fail("INVALID_ARGUMENT", `batchSize must be an integer between 1 and ${MAX_RETENTION_BATCH}`);
+  if (
+    !Number.isSafeInteger(batchSize) ||
+    batchSize < 1 ||
+    batchSize > MAX_RETENTION_BATCH
+  ) {
+    fail(
+      "INVALID_ARGUMENT",
+      `batchSize must be an integer between 1 and ${MAX_RETENTION_BATCH}`,
+    );
   }
-  const effectiveBatchSize = args.kind === "events" ? Math.min(batchSize, 32) : batchSize;
+  const effectiveBatchSize =
+    args.kind === "events" ? Math.min(batchSize, 32) : batchSize;
   if (args.kind === "events") {
     return deleteTelemetryBefore(ctx, {
       siteId: args.siteId,
@@ -646,74 +689,154 @@ async function deletionsForCleanup(
   limit: number,
 ): Promise<CleanupDelete[]> {
   if (kind === "sessions") {
-    return (await ctx.db.query("sessions")
-      .withIndex("by_siteId_and_lastSeenAt", (q) => q.eq("siteId", siteId).lt("lastSeenAt", before))
-      .take(limit)).map((row) => () => ctx.db.delete("sessions", row._id));
+    return (
+      await ctx.db
+        .query("sessions")
+        .withIndex("by_siteId_and_lastSeenAt", (q) =>
+          q.eq("siteId", siteId).lt("lastSeenAt", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("sessions", row._id));
   }
   if (kind === "conversions") {
-    return (await ctx.db.query("conversions")
-      .withIndex("by_siteId_and_timestamp", (q) => q.eq("siteId", siteId).lt("timestamp", before))
-      .take(limit)).map((row) => () => ctx.db.delete("conversions", row._id));
+    return (
+      await ctx.db
+        .query("conversions")
+        .withIndex("by_siteId_and_timestamp", (q) =>
+          q.eq("siteId", siteId).lt("timestamp", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("conversions", row._id));
   }
   if (kind === "goalCompletions") {
-    return (await ctx.db.query("goalCompletions")
-      .withIndex("by_siteId_and_timestamp", (q) => q.eq("siteId", siteId).lt("timestamp", before))
-      .take(limit)).map((row) => () => ctx.db.delete("goalCompletions", row._id));
+    return (
+      await ctx.db
+        .query("goalCompletions")
+        .withIndex("by_siteId_and_timestamp", (q) =>
+          q.eq("siteId", siteId).lt("timestamp", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("goalCompletions", row._id));
   }
   if (kind === "aggregateHour" || kind === "aggregateDay") {
-    return (await ctx.db.query("aggregateBuckets")
-      .withIndex("by_siteId_and_granularity_and_bucketStart", (q) => q
-        .eq("siteId", siteId)
-        .eq("granularity", kind === "aggregateHour" ? "hour" : "day")
-        .lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("aggregateBuckets", row._id));
+    return (
+      await ctx.db
+        .query("aggregateBuckets")
+        .withIndex("by_siteId_and_granularity_and_bucketStart", (q) =>
+          q
+            .eq("siteId", siteId)
+            .eq("granularity", kind === "aggregateHour" ? "hour" : "day")
+            .lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("aggregateBuckets", row._id));
   }
   if (kind === "funnelProgress") {
-    return (await ctx.db.query("funnelProgress")
-      .withIndex("by_siteId_and_lastStepAt", (q) => q.eq("siteId", siteId).lt("lastStepAt", before))
-      .take(limit)).map((row) => () => ctx.db.delete("funnelProgress", row._id));
+    return (
+      await ctx.db
+        .query("funnelProgress")
+        .withIndex("by_siteId_and_lastStepAt", (q) =>
+          q.eq("siteId", siteId).lt("lastStepAt", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("funnelProgress", row._id));
   }
   if (kind === "funnelSteps") {
-    return (await ctx.db.query("funnelStepBuckets")
-      .withIndex("by_siteId_and_bucketStart", (q) => q.eq("siteId", siteId).lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("funnelStepBuckets", row._id));
+    return (
+      await ctx.db
+        .query("funnelStepBuckets")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("funnelStepBuckets", row._id));
   }
   if (kind === "affiliateStats") {
-    return (await ctx.db.query("affiliateDailyStats")
-      .withIndex("by_siteId_and_bucketStart", (q) => q.eq("siteId", siteId).lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("affiliateDailyStats", row._id));
+    return (
+      await ctx.db
+        .query("affiliateDailyStats")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("affiliateDailyStats", row._id));
   }
   if (kind === "goalRollups") {
-    return (await ctx.db.query("goalDailyRollups")
-      .withIndex("by_siteId_and_bucketStart", (q) => q.eq("siteId", siteId).lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("goalDailyRollups", row._id));
+    return (
+      await ctx.db
+        .query("goalDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("goalDailyRollups", row._id));
   }
   if (kind === "funnelRollups") {
-    return (await ctx.db.query("funnelDailyRollups")
-      .withIndex("by_siteId_and_bucketStart", (q) => q.eq("siteId", siteId).lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("funnelDailyRollups", row._id));
+    return (
+      await ctx.db
+        .query("funnelDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("funnelDailyRollups", row._id));
   }
   if (kind === "affiliateRollups") {
-    return (await ctx.db.query("affiliateDailyRollups")
-      .withIndex("by_siteId_and_bucketStart", (q) => q.eq("siteId", siteId).lt("bucketStart", before))
-      .take(limit)).map((row) => () => ctx.db.delete("affiliateDailyRollups", row._id));
+    return (
+      await ctx.db
+        .query("affiliateDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("affiliateDailyRollups", row._id));
+  }
+  if (kind === "vitalRollups") {
+    return (
+      await ctx.db
+        .query("vitalRollups")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("vitalRollups", row._id));
   }
   if (kind === "attributions") {
-    return (await ctx.db.query("visitorAttributions")
-      .withIndex("by_siteId_and_expiresAt", (q) => q.eq("siteId", siteId).lt("expiresAt", before))
-      .take(limit)).map((row) => () => ctx.db.delete("visitorAttributions", row._id));
+    return (
+      await ctx.db
+        .query("visitorAttributions")
+        .withIndex("by_siteId_and_expiresAt", (q) =>
+          q.eq("siteId", siteId).lt("expiresAt", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("visitorAttributions", row._id));
   }
   if (kind === "liveSessions") {
-    return (await ctx.db.query("liveSessions")
-      .withIndex("by_siteId_and_expiresAt", (q) => q.eq("siteId", siteId).lt("expiresAt", before))
-      .take(limit)).map((row) => () => ctx.db.delete("liveSessions", row._id));
+    return (
+      await ctx.db
+        .query("liveSessions")
+        .withIndex("by_siteId_and_expiresAt", (q) =>
+          q.eq("siteId", siteId).lt("expiresAt", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("liveSessions", row._id));
   }
   if (kind === "siteIngestWindows") {
-    return (await ctx.db.query("siteIngestWindows")
-      .withIndex("by_siteId_and_updatedAt", (q) => q.eq("siteId", siteId).lt("updatedAt", before))
-      .take(limit)).map((row) => () => ctx.db.delete("siteIngestWindows", row._id));
+    return (
+      await ctx.db
+        .query("siteIngestWindows")
+        .withIndex("by_siteId_and_updatedAt", (q) =>
+          q.eq("siteId", siteId).lt("updatedAt", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("siteIngestWindows", row._id));
   }
-  return (await ctx.db.query("ingestWindows")
-    .withIndex("by_siteId_and_updatedAt", (q) => q.eq("siteId", siteId).lt("updatedAt", before))
-    .take(limit)).map((row) => () => ctx.db.delete("ingestWindows", row._id));
+  return (
+    await ctx.db
+      .query("ingestWindows")
+      .withIndex("by_siteId_and_updatedAt", (q) =>
+        q.eq("siteId", siteId).lt("updatedAt", before),
+      )
+      .take(limit)
+  ).map((row) => () => ctx.db.delete("ingestWindows", row._id));
 }