@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.2

package/src/component/retention.ts

@@ -4,6 +4,7 @@ import type { MutationCtx } from "./_generated/server.js";
 import { mutation } from "./_generated/server.js";
 import { MAX_RETENTION_BATCH } from "./constants.js";
 import { fail } from "./errors.js";
+import { deleteTelemetryBefore } from "./eventStore.js";
 
 const cleanupKindValidator = v.union(
   v.literal("events"),
@@ -37,19 +38,29 @@ export const cleanup = mutation({
     if (!Number.isSafeInteger(batchSize) || batchSize < 1 || batchSize > MAX_RETENTION_BATCH) {
       fail("INVALID_ARGUMENT", `batchSize must be an integer between 1 and ${MAX_RETENTION_BATCH}`);
     }
+    const effectiveBatchSize = args.kind === "events"
+      ? Math.min(batchSize, 32)
+      : batchSize;
+    if (args.kind === "events") {
+      return await deleteTelemetryBefore(ctx, {
+        siteId: args.siteId,
+        before: args.before,
+        batchSize: effectiveBatchSize,
+      });
+    }
     const deletions = await deletionsForCleanup(
       ctx,
       args.siteId,
       args.kind,
       args.before,
-      batchSize + 1,
+      effectiveBatchSize + 1,
     );
-    for (const remove of deletions.slice(0, batchSize)) {
+    for (const remove of deletions.slice(0, effectiveBatchSize)) {
       await remove();
     }
     return {
-      deleted: Math.min(deletions.length, batchSize),
-      hasMore: deletions.length > batchSize,
+      deleted: Math.min(deletions.length, effectiveBatchSize),
+      hasMore: deletions.length > effectiveBatchSize,
     };
   },
 });
@@ -78,14 +89,6 @@ async function deletionsForCleanup(
   before: number,
   limit: number,
 ): Promise<CleanupDelete[]> {
-  if (kind === "events") {
-    return (await ctx.db
-      .query("events")
-      .withIndex("by_siteId_and_timestamp", (range) =>
-        range.eq("siteId", siteId).lt("timestamp", before),
-      )
-      .take(limit)).map((row) => () => ctx.db.delete("events", row._id));
-  }
   if (kind === "sessions") {
     return (await ctx.db
       .query("sessions")

package/src/component/sanitize.ts

@@ -152,12 +152,19 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     properties: sanitizeProperties(event.properties),
     revenueCents: event.revenueCents,
     currency,
-    affiliateSlug: event.affiliateSlug === undefined
-      ? undefined
-      : sanitizeSlug(event.affiliateSlug, "affiliateSlug"),
+    affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
   };
 }
 
+function optionalAffiliateSlug(value: string | undefined): string | undefined {
+  if (value === undefined) return undefined;
+  try {
+    return sanitizeSlug(value, "affiliateSlug");
+  } catch {
+    return undefined;
+  }
+}
+
 export function normalizeDomains(domains: string[]): string[] {
   if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
     throw new Error(`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`);

package/src/component/schema.ts

@@ -3,7 +3,8 @@ import { v } from "convex/values";
 import {
   affiliateFieldsValidator,
   aggregateFieldsValidator,
-  eventFieldsValidator,
+  batchedEventValidator,
+  storedEventFieldsValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
   sessionFieldsValidator,
@@ -45,26 +46,52 @@ export default defineSchema({
     device: v.string(),
     pageviewCount: v.number(),
     lastHeartbeatEventId: v.optional(v.string()),
+    recentHeartbeatEventIds: v.optional(v.array(v.string())),
     eventCount: v.number(),
   })
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
-    .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"]),
+    .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
+    .index("by_expiresAt", { fields: ["expiresAt"], staged: true }),
+
+  liveSweepStates: defineTable({
+    siteId: v.id("sites"),
+    scheduledAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
 
-  events: defineTable(eventFieldsValidator)
+  events: defineTable(storedEventFieldsValidator)
     .index("by_siteId_and_eventId", ["siteId", "eventId"])
     .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
+    .index("by_siteId_and_sessionId_and_timestamp", [
+      "siteId",
+      "sessionId",
+      "timestamp",
+    ])
     .index("by_siteId_and_visitorId_and_timestamp", [
       "siteId",
       "visitorId",
       "timestamp",
-    ])
-    .index("by_siteId_and_sessionId_and_timestamp", [
+    ]),
+
+  eventBatches: defineTable({
+    siteId: v.id("sites"),
+    sessionId: v.string(),
+    visitorId: v.string(),
+    bucketStart: v.number(),
+    events: v.array(batchedEventValidator),
+    migrationVersion: v.optional(v.literal(1)),
+  })
+    .index("by_siteId_and_sessionId_and_bucketStart", [
       "siteId",
       "sessionId",
-      "timestamp",
+      "bucketStart",
     ])
-    .index("by_siteId_and_type_and_timestamp", ["siteId", "type", "timestamp"]),
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
+    .index("by_siteId_and_visitorId_and_bucketStart", [
+      "siteId",
+      "visitorId",
+      "bucketStart",
+    ]),
 
   aggregateBuckets: defineTable(aggregateFieldsValidator)
     .index("by_siteId_and_granularity_and_bucketStart_and_shard", [
@@ -88,6 +115,7 @@ export default defineSchema({
     goalId: v.id("goals"),
     goalKey: v.string(),
     eventId: v.string(),
+    idempotencyVersion: v.optional(v.literal(2)),
     sessionId: v.string(),
     visitorId: v.string(),
     timestamp: v.number(),

package/src/component/sites.ts

@@ -33,13 +33,41 @@ export const create = mutation({
     const name = cleanString(args.name, 120);
     if (!ownerId || !name) fail("INVALID_SITE", "ownerId and name are required");
 
+    let domains: string[];
+    let timezone: string | undefined;
+    let networkId: string | undefined;
+    let currency: string;
+    try {
+      domains = normalizeDomains(args.domains);
+      timezone = validateTimezone(args.timezone);
+      networkId = args.networkId
+        ? sanitizeOpaqueId(args.networkId, "networkId")
+        : undefined;
+      currency = sanitizeCurrency(args.currency ?? "USD");
+    } catch (error) {
+      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
+    }
+    const cookieless = args.cookieless ?? true;
+
     const existing = await ctx.db
       .query("sites")
       .withIndex("by_ownerId_and_name", (range) =>
         range.eq("ownerId", ownerId).eq("name", name),
       )
       .unique();
-    if (existing) return existing._id;
+    if (existing) {
+      if (
+        existing.domains.length !== domains.length ||
+        existing.domains.some((domain, index) => domain !== domains[index]) ||
+        existing.networkId !== networkId ||
+        existing.timezone !== timezone ||
+        (existing.currency ?? "USD") !== currency ||
+        existing.cookieless !== cookieless
+      ) {
+        fail("CONFLICT", "site name is already associated with different settings");
+      }
+      return existing._id;
+    }
 
     const sites = await ctx.db
       .query("sites")
@@ -51,20 +79,6 @@ export const create = mutation({
       });
     }
 
-    let domains: string[];
-    let timezone: string | undefined;
-    let networkId: string | undefined;
-    let currency: string;
-    try {
-      domains = normalizeDomains(args.domains);
-      timezone = validateTimezone(args.timezone);
-      networkId = args.networkId
-        ? sanitizeOpaqueId(args.networkId, "networkId")
-        : undefined;
-      currency = sanitizeCurrency(args.currency ?? "USD");
-    } catch (error) {
-      fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
-    }
     const now = Date.now();
     return await ctx.db.insert("sites", {
       ownerId,
@@ -73,7 +87,7 @@ export const create = mutation({
       networkId,
       timezone,
       currency,
-      cookieless: args.cookieless ?? true,
+      cookieless,
       createdAt: now,
       updatedAt: now,
     });
@@ -138,8 +152,7 @@ export const update = mutation({
     }
     if (
       patch.currency !== undefined &&
-      site.currency !== undefined &&
-      site.currency !== patch.currency
+      (site.currency ?? "USD") !== patch.currency
     ) {
       fail("CONFLICT", "site currency cannot be changed");
     }

package/src/component/validators.ts

@@ -49,6 +49,10 @@ export const ingestContextValidator = v.object({
   device: v.optional(v.string()),
 });
 
+export const batchedEventValidator = trackerEventValidator.extend(
+  ingestContextValidator.fields,
+);
+
 export const siteFieldsValidator = v.object({
   ownerId: v.string(),
   name: v.string(),
@@ -119,6 +123,18 @@ export const eventFieldsValidator = v.object({
   device: v.string(),
 });
 
+export const storedEventFieldsValidator = eventFieldsValidator
+  .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
+  .extend({
+    country: v.optional(v.string()),
+    city: v.optional(v.string()),
+    latitude: v.optional(v.number()),
+    longitude: v.optional(v.number()),
+    browser: v.optional(v.string()),
+    os: v.optional(v.string()),
+    device: v.optional(v.string()),
+  });
+
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
   v.literal("page"),

package/src/test.ts

@@ -1,4 +1,5 @@
 /// <reference types="vite/client" />
+import migrationsComponent from "@convex-dev/migrations/test";
 import type { TestConvex } from "convex-test";
 import type { GenericSchema, SchemaDefinition } from "convex/server";
 import schema from "./component/schema.js";
@@ -15,5 +16,6 @@ export function register(
   name: string = "rastroAnalytics",
 ) {
   t.registerComponent(name, schema, modules);
+  migrationsComponent.register(t, `${name}/migrations`);
 }
 export default { register, schema, modules };