@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.4

package/src/component/sanitize.ts

@@ -249,10 +249,21 @@ export function isValidMoney(value: number): boolean {
   return Number.isSafeInteger(value) && value >= 0 && value <= 1_000_000_000;
 }
 
-export function ensureBatchBytes(value: unknown): number {
-  const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
-  if (bytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
-  return bytes;
+export function measureBatchBytes(
+  siteId: string,
+  events: TrackerEvent[],
+): { batchBytes: number; eventByteCounts: number[] } {
+  const encoder = new TextEncoder();
+  const eventByteCounts = events.map((event) =>
+    encoder.encode(JSON.stringify(event)).byteLength
+  );
+  // The empty envelope already includes both array brackets; only commas remain.
+  const batchBytes =
+    encoder.encode(JSON.stringify({ siteId, events: [] })).byteLength +
+    eventByteCounts.reduce((total, byteCount) => total + byteCount, 0) +
+    Math.max(0, events.length - 1);
+  if (batchBytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
+  return { batchBytes, eventByteCounts };
 }
 
 export function sourceFromReferrer(referrer: string | undefined): string {

package/src/component/schema.ts

@@ -4,11 +4,11 @@ import {
   affiliateFieldsValidator,
   aggregateFieldsValidator,
   batchedEventValidator,
-  storedEventFieldsValidator,
   funnelFieldsValidator,
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
+  visitorSketchValidator,
 } from "./validators.js";
 
 export default defineSchema({
@@ -17,7 +17,11 @@ export default defineSchema({
     .index("by_ownerId_and_name", ["ownerId", "name"])
     .index("by_networkId", ["networkId"]),
 
-  sessions: defineTable(sessionFieldsValidator)
+  sessions: defineTable(
+    sessionFieldsValidator.extend({
+      geoLookupAttemptedAt: v.optional(v.number()),
+    }),
+  )
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
     .index("by_siteId_and_startedAt", ["siteId", "startedAt"])
@@ -59,19 +63,11 @@ export default defineSchema({
     scheduledAt: v.number(),
   }).index("by_siteId", ["siteId"]),
 
-  events: defineTable(storedEventFieldsValidator)
-    .index("by_siteId_and_eventId", ["siteId", "eventId"])
-    .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
-    .index("by_siteId_and_sessionId_and_timestamp", [
-      "siteId",
-      "sessionId",
-      "timestamp",
-    ])
-    .index("by_siteId_and_visitorId_and_timestamp", [
-      "siteId",
-      "visitorId",
-      "timestamp",
-    ]),
+  geoLookupBudgets: defineTable({
+    siteId: v.id("sites"),
+    dayStart: v.number(),
+    count: v.number(),
+  }).index("by_siteId", ["siteId"]),
 
   eventBatches: defineTable({
     siteId: v.id("sites"),
@@ -106,6 +102,115 @@ export default defineSchema({
       "bucketStart",
     ]),
 
+  goalDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    goalId: v.id("goals"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    goalKey: v.string(),
+    completions: v.number(),
+    valueCents: v.number(),
+  })
+    .index("by_siteId_and_goalId_and_bucketStart_and_shard", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_goalId_and_bucketStart", [
+      "siteId",
+      "goalId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  funnelDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    funnelId: v.id("funnels"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    steps: v.array(v.object({
+      step: v.number(),
+      entrants: v.number(),
+      completions: v.number(),
+    })),
+  })
+    .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_funnelId_and_bucketStart", [
+      "siteId",
+      "funnelId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  affiliateDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    affiliateId: v.id("affiliates"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    affiliateSlug: v.string(),
+    visits: v.number(),
+    visitorSketch: visitorSketchValidator,
+    conversions: v.number(),
+    revenueCents: v.number(),
+    commissionCents: v.number(),
+  })
+    .index("by_siteId_and_affiliateId_and_bucketStart_and_shard", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_affiliateId_and_bucketStart", [
+      "siteId",
+      "affiliateId",
+      "bucketStart",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
+  analyticsCoverage: defineTable({
+    siteId: v.id("sites"),
+    generation: v.literal(1),
+    rollupCompleteFrom: v.number(),
+    retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionPolicies: defineTable({
+    siteId: v.id("sites"),
+    eventsDays: v.number(),
+    sessionsDays: v.number(),
+    conversionsDays: v.number(),
+    aggregatesDays: v.number(),
+    reportsDays: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId", ["siteId"]),
+
+  retentionStates: defineTable({
+    siteId: v.id("sites"),
+    kind: v.string(),
+    before: v.number(),
+    status: v.union(
+      v.literal("pending"),
+      v.literal("running"),
+      v.literal("complete"),
+    ),
+    generation: v.number(),
+    attempt: v.number(),
+    leaseExpiresAt: v.optional(v.number()),
+    scheduledAt: v.optional(v.number()),
+    deleted: v.number(),
+    updatedAt: v.number(),
+  }).index("by_siteId_and_kind", ["siteId", "kind"]),
+
   goals: defineTable(goalFieldsValidator)
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_key", ["siteId", "key"]),
@@ -122,7 +227,11 @@ export default defineSchema({
     valueCents: v.number(),
   })
     .index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
-    .index("by_siteId_and_goalId_and_timestamp", ["siteId", "goalId", "timestamp"])
+    .index("by_siteId_and_goalId_and_timestamp", [
+      "siteId",
+      "goalId",
+      "timestamp",
+    ])
     .index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
 
   funnels: defineTable(funnelFieldsValidator)

package/src/component/sites.ts

@@ -1,6 +1,7 @@
 import { v } from "convex/values";
 import { internalQuery, mutation, query } from "./_generated/server.js";
 import { MAX_SITES_PER_OWNER } from "./constants.js";
+import { ensureAnalyticsControl } from "./coverage.js";
 import { fail } from "./errors.js";
 import {
   cleanString,
@@ -66,6 +67,7 @@ export const create = mutation({
       ) {
         fail("CONFLICT", "site name is already associated with different settings");
       }
+      await ensureAnalyticsControl(ctx, existing._id, Date.now());
       return existing._id;
     }
 
@@ -80,7 +82,7 @@ export const create = mutation({
     }
 
     const now = Date.now();
-    return await ctx.db.insert("sites", {
+    const siteId = await ctx.db.insert("sites", {
       ownerId,
       name,
       domains,
@@ -91,6 +93,8 @@ export const create = mutation({
       createdAt: now,
       updatedAt: now,
     });
+    await ensureAnalyticsControl(ctx, siteId, now);
+    return siteId;
   },
 });
 

package/src/component/validators.ts

@@ -50,7 +50,10 @@ export const ingestContextValidator = v.object({
 });
 
 export const batchedEventValidator = trackerEventValidator.extend(
-  ingestContextValidator.fields,
+  {
+    ...ingestContextValidator.fields,
+    aggregateCountry: v.optional(v.string()),
+  },
 );
 
 export const siteFieldsValidator = v.object({
@@ -123,18 +126,6 @@ export const eventFieldsValidator = v.object({
   device: v.string(),
 });
 
-export const storedEventFieldsValidator = eventFieldsValidator
-  .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
-  .extend({
-    country: v.optional(v.string()),
-    city: v.optional(v.string()),
-    latitude: v.optional(v.number()),
-    longitude: v.optional(v.number()),
-    browser: v.optional(v.string()),
-    os: v.optional(v.string()),
-    device: v.optional(v.string()),
-  });
-
 export const dimensionTypeValidator = v.union(
   v.literal("source"),
   v.literal("page"),
@@ -153,6 +144,12 @@ export const dimensionSlotValidator = v.object({
   revenueCents: v.number(),
 });
 
+export const visitorSketchValidator = v.object({
+  version: v.literal(1),
+  precision: v.number(),
+  registers: v.bytes(),
+});
+
 export const aggregateFieldsValidator = v.object({
   siteId: v.id("sites"),
   granularity: v.union(v.literal("hour"), v.literal("day")),
@@ -165,6 +162,8 @@ export const aggregateFieldsValidator = v.object({
   outboundClicks: v.number(),
   sessions: v.number(),
   visitors: v.number(),
+  visitorSketch: v.optional(visitorSketchValidator),
+  visitorSketchComplete: v.optional(v.boolean()),
   conversions: v.number(),
   revenueCents: v.number(),
   dimensions: v.array(dimensionSlotValidator),

package/src/test.ts

@@ -1,5 +1,4 @@
 /// <reference types="vite/client" />
-import migrationsComponent from "@convex-dev/migrations/test";
 import type { TestConvex } from "convex-test";
 import type { GenericSchema, SchemaDefinition } from "convex/server";
 import schema from "./component/schema.js";
@@ -16,6 +15,5 @@ export function register(
   name: string = "rastroAnalytics",
 ) {
   t.registerComponent(name, schema, modules);
-  migrationsComponent.register(t, `${name}/migrations`);
 }
 export default { register, schema, modules };

package/dist/component/migrations.d.ts

@@ -1,17 +0,0 @@
-export declare const migrateLegacyEvents: import("convex/server").RegisteredMutation<"internal", {
-    fn?: string | undefined;
-    cursor?: string | null | undefined;
-    batchSize?: number | undefined;
-    dryRun?: boolean | undefined;
-    next?: string[] | undefined;
-    reset?: boolean | undefined;
-    oneBatchOnly?: boolean | undefined;
-}, Promise<any>>;
-export declare const legacyEventsStatus: import("convex/server").RegisteredQuery<"internal", {}, Promise<{
-    complete: boolean;
-    sourceEmpty: boolean;
-    migrationState: "canceled" | "failed" | "inProgress" | "success" | "unknown";
-    processed: number;
-    sampleRemainingId: import("convex/values").GenericId<"events"> | null;
-}>>;
-//# sourceMappingURL=migrations.d.ts.map

package/dist/component/migrations.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,mBAAmB;;;;;;;;gBAI9B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;GAiC7B,CAAC"}

package/dist/component/migrations.js

@@ -1,44 +0,0 @@
-import { Migrations } from "@convex-dev/migrations";
-import { v } from "convex/values";
-import { components } from "./_generated/api.js";
-import { internalMutation, internalQuery } from "./_generated/server.js";
-import { migrateLegacyEventToBatch } from "./eventStore.js";
-import schema from "./schema.js";
-const migrations = new Migrations(components.migrations, {
-    internalMutation,
-    schema,
-});
-export const migrateLegacyEvents = migrations.define({
-    table: "events",
-    batchSize: 16,
-    migrateOne: migrateLegacyEventToBatch,
-});
-export const legacyEventsStatus = internalQuery({
-    args: {},
-    returns: v.object({
-        complete: v.boolean(),
-        sourceEmpty: v.boolean(),
-        migrationState: v.union(v.literal("inProgress"), v.literal("success"), v.literal("failed"), v.literal("canceled"), v.literal("unknown")),
-        processed: v.number(),
-        sampleRemainingId: v.union(v.id("events"), v.null()),
-    }),
-    handler: async (ctx) => {
-        const [remaining, statuses] = await Promise.all([
-            ctx.db.query("events").first(),
-            migrations.getStatus(ctx, {
-                migrations: ["migrations:migrateLegacyEvents"],
-            }),
-        ]);
-        const migrationStatus = statuses[0];
-        const migrationState = migrationStatus?.state ?? "unknown";
-        const sourceEmpty = remaining === null;
-        return {
-            complete: sourceEmpty && migrationState === "success",
-            sourceEmpty,
-            migrationState,
-            processed: migrationStatus?.processed ?? 0,
-            sampleRemainingId: remaining?._id ?? null,
-        };
-    },
-});
-//# sourceMappingURL=migrations.js.map

package/dist/component/migrations.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,EAAE;IACvD,gBAAgB;IAChB,MAAM;CACP,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC,MAAM,CAAC;IACnD,KAAK,EAAE,QAAQ;IACf,SAAS,EAAE,EAAE;IACb,UAAU,EAAE,yBAAyB;CACtC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAC9C,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE;QACrB,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE;QACxB,cAAc,EAAE,CAAC,CAAC,KAAK,CACrB,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EACvB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EACnB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EACrB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CACrB;QACD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;KACrD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE;YAC9B,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE;gBACxB,UAAU,EAAE,CAAC,gCAAgC,CAAC;aAC/C,CAAC;SACH,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,eAAe,EAAE,KAAK,IAAI,SAAS,CAAC;QAC3D,MAAM,WAAW,GAAG,SAAS,KAAK,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,WAAW,IAAI,cAAc,KAAK,SAAS;YACrD,WAAW;YACX,cAAc;YACd,SAAS,EAAE,eAAe,EAAE,SAAS,IAAI,CAAC;YAC1C,iBAAiB,EAAE,SAAS,EAAE,GAAG,IAAI,IAAI;SAC1C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}

package/src/component/migrations.ts

@@ -1,52 +0,0 @@
-import { Migrations } from "@convex-dev/migrations";
-import { v } from "convex/values";
-import { components } from "./_generated/api.js";
-import { internalMutation, internalQuery } from "./_generated/server.js";
-import { migrateLegacyEventToBatch } from "./eventStore.js";
-import schema from "./schema.js";
-
-const migrations = new Migrations(components.migrations, {
-  internalMutation,
-  schema,
-});
-
-export const migrateLegacyEvents = migrations.define({
-  table: "events",
-  batchSize: 16,
-  migrateOne: migrateLegacyEventToBatch,
-});
-
-export const legacyEventsStatus = internalQuery({
-  args: {},
-  returns: v.object({
-    complete: v.boolean(),
-    sourceEmpty: v.boolean(),
-    migrationState: v.union(
-      v.literal("inProgress"),
-      v.literal("success"),
-      v.literal("failed"),
-      v.literal("canceled"),
-      v.literal("unknown"),
-    ),
-    processed: v.number(),
-    sampleRemainingId: v.union(v.id("events"), v.null()),
-  }),
-  handler: async (ctx) => {
-    const [remaining, statuses] = await Promise.all([
-      ctx.db.query("events").first(),
-      migrations.getStatus(ctx, {
-        migrations: ["migrations:migrateLegacyEvents"],
-      }),
-    ]);
-    const migrationStatus = statuses[0];
-    const migrationState = migrationStatus?.state ?? "unknown";
-    const sourceEmpty = remaining === null;
-    return {
-      complete: sourceEmpty && migrationState === "success",
-      sourceEmpty,
-      migrationState,
-      processed: migrationStatus?.processed ?? 0,
-      sampleRemainingId: remaining?._id ?? null,
-    };
-  },
-});