@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.1

package/src/component/eventStore.ts

@@ -0,0 +1,465 @@
+import type { Doc, Id } from "./_generated/dataModel.js";
+import type { MutationCtx, QueryCtx } from "./_generated/server.js";
+import {
+  MAX_EVENTS_PER_SESSION_WINDOW,
+  RATE_LIMIT_WINDOW_MS,
+} from "./constants.js";
+import type { TrackerEvent } from "./validators.js";
+
+export type TelemetryMatch = {
+  sessionId: string;
+  visitorId: string;
+  country?: string;
+  city?: string;
+  latitude?: number;
+  longitude?: number;
+  browser?: string;
+  os?: string;
+  device?: string;
+};
+
+type BufferedEventBatch = {
+  siteId: Id<"sites">;
+  sessionId: string;
+  visitorId: string;
+  bucketStart: number;
+  events: TrackerEvent[];
+};
+
+export type EventBatchBuffer = Map<string, BufferedEventBatch>;
+
+export type JourneyCursor = {
+  timestamp: number;
+  creationTime: number;
+  id: string;
+};
+
+export type RawJourneyEvent = Omit<Doc<"events">, "_id"> & { _id: string };
+
+export function eventBucketStart(timestamp: number): number {
+  return Math.floor(timestamp / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
+}
+
+export function eventBatchKey(sessionId: string, bucketStart: number): string {
+  return JSON.stringify([sessionId, bucketStart]);
+}
+
+export async function inspectTelemetryMinute(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+    candidateEventIds: Set<string>;
+  },
+) {
+  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, input);
+  const eventIds = new Set([
+    ...legacyEvents.map((event) => event.eventId),
+    ...batches.flatMap((batch) =>
+      batch.events.map((event) => event.eventId),
+    ),
+  ]);
+  if (legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW) {
+    await Promise.all(
+      [...input.candidateEventIds]
+        .filter((eventId) => !eventIds.has(eventId))
+        .map(async (eventId) => {
+          const legacy = await findLegacyTelemetry(ctx, {
+            ...input,
+            eventId,
+          });
+          if (legacy) eventIds.add(eventId);
+        }),
+    );
+  }
+  return {
+    eventIds,
+    persistedCount: telemetryMinuteCount(legacyEvents, batches),
+  };
+}
+
+export async function findTelemetry(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+): Promise<{ match: TelemetryMatch | null; saturated: boolean }> {
+  const bucketStart = eventBucketStart(event.timestamp);
+  const [legacyEvents, batches] = await loadTelemetryMinute(ctx, {
+    siteId,
+    sessionId: event.sessionId,
+    bucketStart,
+  });
+  const persistedCount = telemetryMinuteCount(legacyEvents, batches);
+  let legacyEvent = legacyEvents.find((candidate) =>
+    candidate.eventId === event.eventId
+  );
+  if (
+    !legacyEvent &&
+    legacyEvents.length > MAX_EVENTS_PER_SESSION_WINDOW
+  ) {
+    legacyEvent = await findLegacyTelemetry(ctx, {
+      siteId,
+      sessionId: event.sessionId,
+      bucketStart,
+      eventId: event.eventId,
+    }) ?? undefined;
+  }
+  const saturated = persistedCount >= MAX_EVENTS_PER_SESSION_WINDOW;
+  if (legacyEvent) return { match: legacyEvent, saturated };
+  const batch = batches.find((candidate) =>
+    candidate.events.some((nested) => nested.eventId === event.eventId)
+  );
+  return {
+    match: batch
+      ? { sessionId: batch.sessionId, visitorId: batch.visitorId }
+      : null,
+    saturated,
+  };
+}
+
+export function bufferEventBatch(
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+  buffer: EventBatchBuffer,
+) {
+  const bucketStart = eventBucketStart(event.timestamp);
+  const key = eventBatchKey(event.sessionId, bucketStart);
+  const existing = buffer.get(key);
+  if (existing) {
+    if (existing.visitorId !== event.visitorId) {
+      throw new Error("event batch visitor does not match its session");
+    }
+    existing.events.push(eventForBatch(event));
+    return;
+  }
+  buffer.set(key, {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    bucketStart,
+    events: [eventForBatch(event)],
+  });
+}
+
+export async function insertEventBatch(
+  ctx: MutationCtx,
+  siteId: Id<"sites">,
+  event: TrackerEvent,
+) {
+  await ctx.db.insert("eventBatches", {
+    siteId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    bucketStart: eventBucketStart(event.timestamp),
+    events: [eventForBatch(event)],
+  });
+}
+
+export async function flushEventBatches(
+  ctx: MutationCtx,
+  buffer: EventBatchBuffer,
+) {
+  for (const batch of buffer.values()) {
+    await ctx.db.insert("eventBatches", batch);
+  }
+}
+
+export async function loadSessionJourneyEvents(
+  ctx: QueryCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    from: number;
+    to: number;
+    cursor: JourneyCursor | null;
+    resultLimit: number;
+  },
+): Promise<RawJourneyEvent[]> {
+  const startTimestamp = input.cursor?.timestamp ?? input.from;
+  const firstBucket = eventBucketStart(startTimestamp);
+  const lastBucket = eventBucketStart(input.to);
+  // Nested event tiebreakers are not index fields, so the boundary minute must
+  // be read completely before applying the cursor.
+  const loadBatches = async () => {
+    const batches: Doc<"eventBatches">[] = [];
+    let eventCount = 0;
+    const requiredEvents = input.resultLimit + MAX_EVENTS_PER_SESSION_WINDOW;
+    let completedBucket: number | undefined;
+    const query = ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .gte("bucketStart", firstBucket)
+          .lte("bucketStart", lastBucket),
+      )
+      .order("asc");
+    for await (const batch of query) {
+      if (completedBucket !== undefined && batch.bucketStart !== completedBucket) {
+        break;
+      }
+      batches.push(batch);
+      eventCount += batch.events.length;
+      if (eventCount >= requiredEvents) completedBucket = batch.bucketStart;
+    }
+    return batches;
+  };
+  const loadLegacyEvents = async () => {
+    const take = input.resultLimit + 1;
+    if (!input.cursor) {
+      return ctx.db
+        .query("events")
+        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+          range
+            .eq("siteId", input.siteId)
+            .eq("sessionId", input.sessionId)
+            .gte("timestamp", input.from)
+            .lte("timestamp", input.to),
+        )
+        .order("asc")
+        .take(take);
+    }
+    const rows = await ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .eq("timestamp", input.cursor!.timestamp)
+          .gt("_creationTime", input.cursor!.creationTime),
+      )
+      .order("asc")
+      .take(take);
+    if (rows.length < take) {
+      rows.push(...await ctx.db
+        .query("events")
+        .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+          range
+            .eq("siteId", input.siteId)
+            .eq("sessionId", input.sessionId)
+            .gt("timestamp", input.cursor!.timestamp)
+            .lte("timestamp", input.to),
+        )
+        .order("asc")
+        .take(take - rows.length));
+    }
+    return rows;
+  };
+  const [legacyEvents, batches] = await Promise.all([
+    loadLegacyEvents(),
+    loadBatches(),
+  ]);
+  const rows: RawJourneyEvent[] = legacyEvents.map((event) => ({
+    ...event,
+    _id: String(event._id),
+  }));
+  for (const batch of batches) {
+    for (let index = 0; index < batch.events.length; index += 1) {
+      const event = batch.events[index];
+      rows.push({
+        ...event,
+        _id: `eventBatch:${batch._id}:${index.toString().padStart(3, "0")}`,
+        _creationTime: batch._creationTime,
+        siteId: batch.siteId,
+        sessionId: batch.sessionId,
+        visitorId: batch.visitorId,
+      });
+    }
+  }
+  return rows
+    .filter((event) =>
+      event.timestamp >= input.from &&
+      event.timestamp <= input.to &&
+      (!input.cursor || compareJourneyEventToCursor(event, input.cursor) > 0)
+    )
+    .sort(compareJourneyEvents)
+    .slice(0, input.resultLimit);
+}
+
+export function compareJourneyEvents(
+  left: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+  right: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+): number {
+  return left.timestamp - right.timestamp ||
+    left._creationTime - right._creationTime ||
+    left._id.localeCompare(right._id);
+}
+
+export async function hydrateEventContexts(
+  ctx: QueryCtx,
+  rows: RawJourneyEvent[],
+) {
+  const sessionPromises = new Map<string, Promise<Doc<"sessions"> | null>>();
+  for (const row of rows) {
+    const key = `${row.siteId}:${row.sessionId}`;
+    if (!sessionPromises.has(key)) {
+      sessionPromises.set(
+        key,
+        ctx.db
+          .query("sessions")
+          .withIndex("by_siteId_and_sessionId", (range) =>
+            range.eq("siteId", row.siteId).eq("sessionId", row.sessionId),
+          )
+          .unique(),
+      );
+    }
+  }
+  const sessions = new Map(
+    await Promise.all(
+      [...sessionPromises].map(async ([key, session]) =>
+        [key, await session] as const
+      ),
+    ),
+  );
+  return rows.map((row) => {
+    const session = sessions.get(`${row.siteId}:${row.sessionId}`);
+    const country = row.country ?? session?.country;
+    const city = row.city ?? session?.city;
+    const latitude = row.latitude ?? session?.latitude;
+    const longitude = row.longitude ?? session?.longitude;
+    return {
+      ...row,
+      ...(country ? { country } : {}),
+      ...(city ? { city } : {}),
+      ...(latitude !== undefined ? { latitude } : {}),
+      ...(longitude !== undefined ? { longitude } : {}),
+      browser: row.browser ?? session?.browser ?? "Other",
+      os: row.os ?? session?.os ?? "Other",
+      device: row.device ?? session?.device ?? "unknown",
+    };
+  });
+}
+
+export async function deleteTelemetryBefore(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    before: number;
+    batchSize: number;
+  },
+) {
+  const batchCutoff = eventBucketStart(input.before);
+  const limit = input.batchSize + 1;
+  const [legacyEvents, batches] = await Promise.all([
+    ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_timestamp", (range) =>
+        range.eq("siteId", input.siteId).lt("timestamp", input.before),
+      )
+      .take(limit),
+    ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_bucketStart", (range) =>
+        range.eq("siteId", input.siteId).lt("bucketStart", batchCutoff),
+      )
+      .take(limit),
+  ]);
+  const rows = [...legacyEvents, ...batches].slice(0, input.batchSize);
+  for (const row of rows) {
+    if ("bucketStart" in row) {
+      await ctx.db.delete("eventBatches", row._id);
+    } else {
+      await ctx.db.delete("events", row._id);
+    }
+  }
+  return {
+    deleted: rows.length,
+    hasMore:
+      legacyEvents.length + batches.length > input.batchSize,
+  };
+}
+
+async function loadTelemetryMinute(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+  },
+) {
+  return Promise.all([
+    ctx.db
+      .query("events")
+      .withIndex("by_siteId_and_sessionId_and_timestamp", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .gte("timestamp", input.bucketStart)
+          .lt("timestamp", input.bucketStart + RATE_LIMIT_WINDOW_MS),
+      )
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
+    ctx.db
+      .query("eventBatches")
+      .withIndex("by_siteId_and_sessionId_and_bucketStart", (range) =>
+        range
+          .eq("siteId", input.siteId)
+          .eq("sessionId", input.sessionId)
+          .eq("bucketStart", input.bucketStart),
+      )
+      .take(MAX_EVENTS_PER_SESSION_WINDOW + 1),
+  ]);
+}
+
+async function findLegacyTelemetry(
+  ctx: MutationCtx,
+  input: {
+    siteId: Id<"sites">;
+    sessionId: string;
+    bucketStart: number;
+    eventId: string;
+  },
+) {
+  const candidates = await ctx.db
+    .query("events")
+    .withIndex("by_siteId_and_eventId", (range) =>
+      range.eq("siteId", input.siteId).eq("eventId", input.eventId),
+    )
+    .take(2);
+  return candidates.find((candidate) =>
+    candidate.sessionId === input.sessionId &&
+    eventBucketStart(candidate.timestamp) === input.bucketStart
+  ) ?? null;
+}
+
+function telemetryMinuteCount(
+  legacyEvents: Array<{ eventId: string }>,
+  batches: Array<{ events: TrackerEvent[] }>,
+) {
+  return legacyEvents.length + batches.reduce(
+    (total, batch) => total + batch.events.length,
+    0,
+  );
+}
+
+function eventForBatch(event: TrackerEvent): TrackerEvent {
+  return {
+    eventId: event.eventId,
+    sessionId: event.sessionId,
+    visitorId: event.visitorId,
+    type: event.type,
+    ...(event.name ? { name: event.name } : {}),
+    path: event.path,
+    ...(event.referrer ? { referrer: event.referrer } : {}),
+    timestamp: event.timestamp,
+    sequence: event.sequence,
+    ...(event.target ? { target: event.target } : {}),
+    ...(event.href ? { href: event.href } : {}),
+    ...(event.properties ? { properties: event.properties } : {}),
+    ...(event.revenueCents !== undefined
+      ? { revenueCents: event.revenueCents }
+      : {}),
+    ...(event.currency ? { currency: event.currency } : {}),
+    ...(event.affiliateSlug ? { affiliateSlug: event.affiliateSlug } : {}),
+  };
+}
+
+function compareJourneyEventToCursor(
+  event: Pick<RawJourneyEvent, "timestamp" | "_creationTime" | "_id">,
+  cursor: JourneyCursor,
+) {
+  return event.timestamp - cursor.timestamp ||
+    event._creationTime - cursor.creationTime ||
+    event._id.localeCompare(cursor.id);
+}

package/src/component/goals.ts

@@ -62,10 +62,25 @@ export const upsert = mutation({
     if (args.goalId && existingByKey && existingByKey._id !== args.goalId) {
       fail("CONFLICT", "another goal already uses this key");
     }
+    const goals = await ctx.db
+      .query("goals")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .take(MAX_GOALS_PER_SITE + 1);
     const now = Date.now();
     if (goalId) {
       const goal = await ctx.db.get("goals", goalId);
       if (!goal || goal.siteId !== args.siteId) fail("NOT_FOUND", "goal not found");
+      if (
+        (args.active ?? goal.active) &&
+        goals.some((candidate) =>
+          candidate._id !== goalId &&
+          candidate.active &&
+          candidate.matchType === args.matchType &&
+          candidate.matchValue === matchValue
+        )
+      ) {
+        fail("CONFLICT", "another active goal already uses this matcher");
+      }
       await ctx.db.patch("goals", goalId, {
         key,
         name,
@@ -78,10 +93,16 @@ export const upsert = mutation({
       return goalId;
     }
 
-    const goals = await ctx.db
-      .query("goals")
-      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
-      .take(MAX_GOALS_PER_SITE + 1);
+    if (
+      (args.active ?? true) &&
+      goals.some((candidate) =>
+        candidate.active &&
+        candidate.matchType === args.matchType &&
+        candidate.matchValue === matchValue
+      )
+    ) {
+      fail("CONFLICT", "another active goal already uses this matcher");
+    }
     if (goals.length >= MAX_GOALS_PER_SITE) {
       fail("LIMIT_EXCEEDED", "goal limit reached", { limit: MAX_GOALS_PER_SITE });
     }

package/src/component/http.ts

@@ -24,16 +24,29 @@ http.route({
   handler: httpAction(async (ctx, request) => {
     const origin = request.headers.get("origin");
     if (!origin) return json({ error: { code: "ORIGIN_REQUIRED" } }, 400);
-    const bytes = await request.arrayBuffer();
-    if (bytes.byteLength > MAX_BATCH_BYTES) {
-      return json({ error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } }, 413);
+    const headers = corsHeaders(origin);
+    const contentLength = Number(request.headers.get("content-length"));
+    if (Number.isFinite(contentLength) && contentLength > MAX_BATCH_BYTES) {
+      return json(
+        { error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } },
+        413,
+        headers,
+      );
     }
 
     let payload: { siteId: string; events: TrackerEvent[] };
     try {
+      const bytes = await readLimitedBody(request, MAX_BATCH_BYTES);
       const parsed: unknown = JSON.parse(new TextDecoder().decode(bytes));
       payload = parsePayload(parsed);
     } catch (error) {
+      if (error instanceof PayloadTooLargeError) {
+        return json(
+          { error: { code: "BATCH_TOO_LARGE", limitBytes: MAX_BATCH_BYTES } },
+          413,
+          headers,
+        );
+      }
       return json(
         {
           error: {
@@ -42,6 +55,7 @@ http.route({
           },
         },
         400,
+        headers,
       );
     }
 
@@ -61,7 +75,7 @@ http.route({
         events: payload.events,
         context,
       });
-      return json(result, 200, corsHeaders(origin));
+      return json(result, 200, headers);
     } catch (error) {
       if (error instanceof ConvexError) {
         const data = error.data as { code?: string; message?: string };
@@ -71,9 +85,9 @@ http.route({
             : data.code === "RATE_LIMITED"
               ? 429
               : 400;
-        return json({ error: data }, status);
+        return json({ error: data }, status, headers);
       }
-      return json({ error: { code: "INVALID_REQUEST" } }, 400);
+      return json({ error: { code: "INVALID_REQUEST" } }, 400, headers);
     }
   }),
 });
@@ -136,6 +150,35 @@ function parsePayload(value: unknown): { siteId: string; events: TrackerEvent[]
   return { siteId: value.siteId, events: value.events.map(parseEvent) };
 }
 
+class PayloadTooLargeError extends Error {}
+
+async function readLimitedBody(
+  request: Request,
+  maximumBytes: number,
+): Promise<Uint8Array> {
+  if (!request.body) return new Uint8Array();
+  const reader = request.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let length = 0;
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    length += value.byteLength;
+    if (length > maximumBytes) {
+      await reader.cancel();
+      throw new PayloadTooLargeError();
+    }
+    chunks.push(value);
+  }
+  const result = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return result;
+}
+
 function parseEvent(value: unknown): TrackerEvent {
   if (!isPlainRecord(value)) throw new Error("each event must be an object");
   const type = value.type;