@iann29/rastro 0.1.0-alpha.6 → 0.1.0-alpha.8

package/src/component/reports.ts

@@ -24,8 +24,12 @@ import {
   MAX_REPORT_BUCKETS,
   MAX_REPORT_RANGE_DAYS,
   MAX_REPORT_SITES,
+  MAX_SITE_MAP_ROUTES_PER_DAY,
+  MAX_SITE_MAP_ROWS,
   MAX_VITALS_REPORT_ROWS,
   REPORT_ROLLUP_SHARDS,
+  ROUTE_TRANSITION_SLOTS,
+  SITE_MAP_OTHER_ROUTE,
   VITALS_REPORT_PAGE_LIMIT,
 } from "./constants.js";
 import {
@@ -90,6 +94,8 @@ const liveSessionDocumentValidator = v.object({
   device: v.string(),
   pageviewCount: v.number(),
   eventCount: v.number(),
+  previousPath: v.optional(v.string()),
+  movedAt: v.optional(v.number()),
 });
 const conversionDocumentValidator = v.object({
   _id: v.id("conversions"),
@@ -439,10 +445,311 @@ export const liveVisitors = query({
         device: row.device,
         pageviewCount: row.pageviewCount,
         eventCount: row.eventCount,
+        ...(row.previousPath ? { previousPath: row.previousPath } : {}),
+        ...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
       }));
   },
 });
 
+const siteMapRouteValidator = v.object({
+  route: v.string(),
+  pageviews: v.number(),
+  visitors: v.number(),
+  entries: v.number(),
+  exits: v.number(),
+  bounces: v.number(),
+  transitionsIn: v.number(),
+  transitionsOut: v.number(),
+  dwellMs: v.number(),
+  dwellSamples: v.number(),
+  daily: v.array(v.object({ bucketStart: v.number(), pageviews: v.number() })),
+});
+const siteMapTransitionValidator = v.object({
+  from: v.string(),
+  to: v.string(),
+  count: v.number(),
+});
+const siteMapCoverageDataset = "siteMapRollups";
+
+/**
+ * Every route the sites served in the range and the transitions between
+ * them, read from the daily route rollups newest day first. Exits and
+ * bounces are derived rather than counted: a session enters once, leaves
+ * once, and every later pageview is a transition, so per route
+ * `exits = entries + transitionsIn - transitionsOut` and
+ * `bounces = entries - continued` hold without a session-end hook. The
+ * report never falls back to raw telemetry; days it could not cover are
+ * declared through `coverage` and `window`.
+ */
+export const siteMap = query({
+  args: {
+    siteIds: v.array(v.id("sites")),
+    from: v.number(),
+    to: v.number(),
+  },
+  returns: v.object({
+    from: v.number(),
+    to: v.number(),
+    window: v.union(v.object({ from: v.number(), to: v.number() }), v.null()),
+    routes: v.array(siteMapRouteValidator),
+    transitions: v.array(siteMapTransitionValidator),
+    truncated: v.object({ routes: v.boolean(), transitions: v.boolean() }),
+    metadata: reportMetadataValidator.extend({
+      visitors: v.object({
+        basis: v.literal("estimatedUnique"),
+        exact: v.boolean(),
+      }),
+      limits: v.object({
+        routesPerDay: v.number(),
+        transitionSlots: v.number(),
+        rows: v.number(),
+      }),
+    }),
+  }),
+  handler: async (ctx, args) => {
+    const siteIds = validateSiteIds(args.siteIds);
+    validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
+    requireAlignedRange(args.from, args.to, DAY_MS, "site map");
+    const controls = await Promise.all(
+      siteIds.map((siteId) =>
+        ctx.db
+          .query("analyticsCoverage")
+          .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+          .unique(),
+      ),
+    );
+    const since = controls.every(
+      (control) => control?.siteMapSince !== undefined,
+    )
+      ? Math.max(...controls.map((control) => control!.siteMapSince!))
+      : null;
+    const retainedBefore = controls.reduce<number | null>((latest, control) => {
+      const before = control?.retained.find(
+        (item) => item.dataset === siteMapCoverageDataset,
+      )?.before;
+      return before === undefined ? latest : Math.max(latest ?? 0, before);
+    }, null);
+    const metadata = (state: "complete" | "partial" | "unavailable") => ({
+      basis: "dailyRollup" as const,
+      coverage: {
+        state,
+        generation: 1 as const,
+        availableFrom: since,
+        availableThrough: since === null ? null : args.to,
+        retainedBefore,
+      },
+      range: rangeSemantics(args.from, args.to),
+      visitors: { basis: "estimatedUnique" as const, exact: false },
+      limits: {
+        routesPerDay: MAX_SITE_MAP_ROUTES_PER_DAY,
+        transitionSlots: ROUTE_TRANSITION_SLOTS,
+        rows: MAX_SITE_MAP_ROWS,
+      },
+    });
+    if (
+      since === null ||
+      args.to < since ||
+      (retainedBefore !== null && args.to < retainedBefore)
+    ) {
+      return {
+        from: args.from,
+        to: args.to,
+        window: null,
+        routes: [],
+        transitions: [],
+        truncated: { routes: false, transitions: false },
+        metadata: metadata("unavailable"),
+      };
+    }
+    const readFrom = Math.max(
+      args.from,
+      Math.floor(since / DAY_MS) * DAY_MS,
+      retainedBefore ?? 0,
+    );
+    let budget = MAX_SITE_MAP_ROWS;
+    let completeFrom = readFrom;
+    const rows: Doc<"routeDailyRollups">[] = [];
+    for (const siteId of siteIds) {
+      const read = await readRouteRollups(
+        ctx,
+        siteId,
+        readFrom,
+        args.to,
+        budget,
+      );
+      budget -= read.rows.length;
+      completeFrom = Math.max(completeFrom, read.completeFrom);
+      rows.push(...read.rows);
+    }
+    const windowFrom = Math.max(
+      args.from,
+      completeFrom,
+      since,
+      retainedBefore ?? 0,
+    );
+
+    type RouteTotals = {
+      pageviews: number;
+      entries: number;
+      continued: number;
+      dwellMs: number;
+      dwellSamples: number;
+      transitionsIn: number;
+      transitionsOut: number;
+      sketch: VisitorSketch;
+      daily: Map<number, number>;
+    };
+    const routes = new Map<string, RouteTotals>();
+    const totalsFor = (route: string) => {
+      let current = routes.get(route);
+      if (!current) {
+        current = {
+          pageviews: 0,
+          entries: 0,
+          continued: 0,
+          dwellMs: 0,
+          dwellSamples: 0,
+          transitionsIn: 0,
+          transitionsOut: 0,
+          sketch: createVisitorSketch(),
+          daily: new Map<number, number>(),
+        };
+        routes.set(route, current);
+      }
+      return current;
+    };
+    const edges = new Map<
+      string,
+      { from: string; to: string; count: number }
+    >();
+    for (const row of rows) {
+      if (row.bucketStart < completeFrom) continue;
+      const current = totalsFor(row.route);
+      current.pageviews += row.pageviews;
+      current.entries += row.entries;
+      current.continued += row.continued;
+      current.dwellMs += row.dwellMs;
+      current.dwellSamples += row.dwellSamples;
+      current.sketch = unionVisitorSketches(current.sketch, row.visitorSketch);
+      current.daily.set(
+        row.bucketStart,
+        (current.daily.get(row.bucketStart) ?? 0) + row.pageviews,
+      );
+      for (const target of row.outgoing) {
+        const key = `${row.route}\u0000${target.route}`;
+        const edge = edges.get(key) ?? {
+          from: row.route,
+          to: target.route,
+          count: 0,
+        };
+        edge.count += target.count;
+        edges.set(key, edge);
+        current.transitionsOut += target.count;
+      }
+    }
+    // A target without a row of its own in this window — folded past the
+    // daily cap, or already retained — counts under "(other)".
+    const transitions = new Map<
+      string,
+      { from: string; to: string; count: number }
+    >();
+    for (const edge of edges.values()) {
+      const to = routes.has(edge.to) ? edge.to : SITE_MAP_OTHER_ROUTE;
+      const key = `${edge.from}\u0000${to}`;
+      const merged = transitions.get(key) ?? { from: edge.from, to, count: 0 };
+      merged.count += edge.count;
+      transitions.set(key, merged);
+      totalsFor(to).transitionsIn += edge.count;
+    }
+    const routeRows = [...routes.entries()]
+      .map(([route, totals]) => ({
+        route,
+        pageviews: totals.pageviews,
+        visitors: estimateVisitors(totals.sketch),
+        entries: totals.entries,
+        exits: Math.max(
+          0,
+          totals.entries + totals.transitionsIn - totals.transitionsOut,
+        ),
+        bounces: Math.max(0, totals.entries - totals.continued),
+        transitionsIn: totals.transitionsIn,
+        transitionsOut: totals.transitionsOut,
+        dwellMs: totals.dwellMs,
+        dwellSamples: totals.dwellSamples,
+        daily: [...totals.daily.entries()]
+          .sort(([left], [right]) => left - right)
+          .map(([bucketStart, pageviews]) => ({ bucketStart, pageviews })),
+      }))
+      .sort(
+        (left, right) =>
+          right.pageviews - left.pageviews ||
+          left.route.localeCompare(right.route),
+      );
+    const transitionRows = [...transitions.values()].sort(
+      (left, right) =>
+        right.count - left.count ||
+        left.from.localeCompare(right.from) ||
+        left.to.localeCompare(right.to),
+    );
+    return {
+      from: args.from,
+      to: args.to,
+      window: completeFrom > args.to ? null : { from: windowFrom, to: args.to },
+      routes: routeRows,
+      transitions: transitionRows,
+      truncated: {
+        routes: routes.has(SITE_MAP_OTHER_ROUTE),
+        transitions: transitionRows.some(
+          (transition) =>
+            transition.from === SITE_MAP_OTHER_ROUTE ||
+            transition.to === SITE_MAP_OTHER_ROUTE,
+        ),
+      },
+      metadata: metadata(windowFrom > args.from ? "partial" : "complete"),
+    };
+  },
+});
+
+/**
+ * Reads a site's route rollups newest day first until the row budget or the
+ * transaction's read headroom runs out, then drops the day it could only
+ * read in part so every day it returns is complete.
+ */
+async function readRouteRollups(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  from: number,
+  to: number,
+  budget: number,
+) {
+  const rows: Doc<"routeDailyRollups">[] = [];
+  let cutAt: number | null = null;
+  const query = ctx.db
+    .query("routeDailyRollups")
+    .withIndex("by_siteId_and_bucketStart", (range) =>
+      range
+        .eq("siteId", siteId)
+        .gte("bucketStart", from)
+        .lte("bucketStart", to),
+    )
+    .order("desc");
+  for await (const row of query) {
+    if (
+      rows.length >= budget ||
+      (rows.length % 256 === 255 && !(await hasFeatureReadHeadroom(ctx)))
+    ) {
+      cutAt = row.bucketStart;
+      break;
+    }
+    rows.push(row);
+  }
+  if (cutAt === null) return { rows, completeFrom: from };
+  while (rows.length > 0 && rows[rows.length - 1]!.bucketStart === cutAt) {
+    rows.pop();
+  }
+  return { rows, completeFrom: cutAt + DAY_MS };
+}
+
 export const listSessions = query({
   args: {
     siteId: v.id("sites"),
@@ -504,7 +811,11 @@ export const listSessions = query({
       );
     }
     const publicRows = rows.map(
-      ({ geoLookupAttemptedAt: _attemptedAt, ...row }) => row,
+      ({
+        geoLookupAttemptedAt: _attemptedAt,
+        lastPageviewAt: _lastPageviewAt,
+        ...row
+      }) => row,
     );
     return keysetPaginationResult(publicRows, pagination, (row) => ({
       timestamp: row.lastSeenAt,
@@ -1219,6 +1530,7 @@ const coverageDatasetValidator = v.union(
   v.literal("funnels"),
   v.literal("affiliates"),
   v.literal("vitals"),
+  v.literal("siteMap"),
 );
 
 export const dataCoverage = query({
@@ -1250,6 +1562,7 @@ export const dataCoverage = query({
       "funnels",
       "affiliates",
       "vitals",
+      "siteMap",
     ] as const;
     return {
       siteId: args.siteId,
@@ -1402,6 +1715,14 @@ function requireFeatureRollupBudget(
   }
 }
 
+async function hasFeatureReadHeadroom(ctx: QueryCtx) {
+  const metrics = await ctx.meta.getTransactionMetrics();
+  return (
+    metrics.bytesRead.remaining >= FEATURE_REPORT_BYTE_RESERVE &&
+    metrics.documentsRead.remaining >= FEATURE_REPORT_DOCUMENT_RESERVE
+  );
+}
+
 async function requireFeatureReadHeadroom(ctx: QueryCtx, dataset: string) {
   const metrics = await ctx.meta.getTransactionMetrics();
   if (
@@ -1545,6 +1866,9 @@ async function datasetCoverageForRange(
   from: number,
   to: number,
 ) {
+  if (dataset === "siteMap") {
+    return siteMapDatasetCoverage(ctx, siteId, from, to);
+  }
   // Vitals have no legacy fallback source; the rollup either covers the range
   // or the coverage states say how much of it exists.
   if (dataset === "vitals") {
@@ -1584,6 +1908,43 @@ async function datasetCoverageForRange(
   )[0]!;
 }
 
+/**
+ * The site map's coverage without reading its rows: it starts at the site's
+ * `siteMapSince` and ends where retention cut it. The report itself may still
+ * narrow the window when the range exceeds its row budget.
+ */
+async function siteMapDatasetCoverage(
+  ctx: QueryCtx,
+  siteId: Id<"sites">,
+  from: number,
+  to: number,
+) {
+  const control = await ctx.db
+    .query("analyticsCoverage")
+    .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+    .unique();
+  const since = control?.siteMapSince ?? null;
+  const retainedBefore =
+    control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
+      ?.before ?? null;
+  const unavailable =
+    since === null ||
+    to < since ||
+    (retainedBefore !== null && to < retainedBefore);
+  const state = unavailable
+    ? ("unavailable" as const)
+    : from < Math.max(since, retainedBefore ?? 0)
+      ? ("partial" as const)
+      : ("complete" as const);
+  return {
+    state,
+    generation: 1 as const,
+    availableFrom: since,
+    availableThrough: since === null ? null : to,
+    retainedBefore,
+  };
+}
+
 type PublicCoverageDataset =
   | "overviewHour"
   | "overviewDay"
@@ -1593,7 +1954,8 @@ type PublicCoverageDataset =
   | "goals"
   | "funnels"
   | "affiliates"
-  | "vitals";
+  | "vitals"
+  | "siteMap";
 
 type CoverageSource =
   | "overviewHour"

package/src/component/retention.ts

@@ -30,6 +30,7 @@ export const cleanupKindValidator = v.union(
   v.literal("funnelRollups"),
   v.literal("affiliateRollups"),
   v.literal("vitalRollups"),
+  v.literal("siteMapRollups"),
   v.literal("attributions"),
   v.literal("liveSessions"),
   v.literal("ingestWindows"),
@@ -446,6 +447,7 @@ type CleanupKind =
   | "funnelRollups"
   | "affiliateRollups"
   | "vitalRollups"
+  | "siteMapRollups"
   | "attributions"
   | "liveSessions"
   | "ingestWindows"
@@ -465,6 +467,7 @@ const POLICY_CLEANUP_KINDS: CleanupKind[] = [
   "funnelRollups",
   "affiliateRollups",
   "vitalRollups",
+  "siteMapRollups",
   "attributions",
   "ingestWindows",
   "siteIngestWindows",
@@ -522,7 +525,8 @@ function safeCleanupBoundary(
     kind === "goalRollups" ||
     kind === "funnelRollups" ||
     kind === "affiliateRollups" ||
-    kind === "vitalRollups"
+    kind === "vitalRollups" ||
+    kind === "siteMapRollups"
   ) {
     return safeBucketBoundary(requested, now, DAY_MS);
   }
@@ -633,6 +637,7 @@ function coverageDatasetsForKind(kind: CleanupKind): string[] {
   if (kind === "funnelRollups") return ["funnelRollups"];
   if (kind === "affiliateRollups") return ["affiliateRollups"];
   if (kind === "vitalRollups") return ["vitalRollups"];
+  if (kind === "siteMapRollups") return ["siteMapRollups"];
   return [];
 }
 
@@ -801,6 +806,16 @@ async function deletionsForCleanup(
         .take(limit)
     ).map((row) => () => ctx.db.delete("vitalRollups", row._id));
   }
+  if (kind === "siteMapRollups") {
+    return (
+      await ctx.db
+        .query("routeDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (q) =>
+          q.eq("siteId", siteId).lt("bucketStart", before),
+        )
+        .take(limit)
+    ).map((row) => () => ctx.db.delete("routeDailyRollups", row._id));
+  }
   if (kind === "attributions") {
     return (
       await ctx.db

package/src/component/sanitize.ts

@@ -1,4 +1,5 @@
 import {
+  CLOCK_SKEW_TOLERANCE_MS,
   MAX_BATCH_BYTES,
   MAX_PROPERTIES,
   MAX_SITE_DOMAINS,
@@ -75,6 +76,37 @@ export function sanitizePath(value: string): string {
   return cleaned || "/";
 }
 
+const ROUTE_ID = ":id";
+const NUMERIC_SEGMENT = /^\d+$/;
+const UUID_SEGMENT =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
+
+/**
+ * Collapses the identifiers that unmistakably vary per record — numbers,
+ * UUIDs, and long hexadecimal tokens — into `:id`, so `/blog/42` and
+ * `/blog/43` roll up as one route in the site map. Slugs are pages of their
+ * own and stay as they are. A trailing slash is dropped because both
+ * spellings serve the same page. Only the map uses this; every other report
+ * keeps the sanitized path exactly as the tracker sent it.
+ */
+export function normalizeRoute(path: string): string {
+  const trimmed =
+    path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
+  const segments = trimmed.split("/");
+  for (let index = 1; index < segments.length; index += 1) {
+    const segment = segments[index] ?? "";
+    if (
+      NUMERIC_SEGMENT.test(segment) ||
+      UUID_SEGMENT.test(segment) ||
+      HEX_SEGMENT.test(segment)
+    ) {
+      segments[index] = ROUTE_ID;
+    }
+  }
+  return segments.join("/") || "/";
+}
+
 export function sanitizePublicUrl(value: string): string | undefined {
   const input = cleanString(value, 2048);
   if (!input) return undefined;
@@ -113,6 +145,28 @@ export function sanitizeProperties(
   return Object.keys(result).length > 0 ? result : undefined;
 }
 
+/**
+ * Re-anchors a batch whose sender's clock disagrees with ours. `sentAt` is the
+ * sender's clock at flush time, so `now - sentAt` is the clock offset alone: an
+ * event queued long before the flush keeps its age, and nothing is inferred
+ * from the events themselves. Offsets inside the tolerance are latency or a
+ * slightly drifted clock and leave the batch untouched, so a well-set client's
+ * timestamps stay exactly as sent.
+ */
+export function alignClock<T extends { timestamp: number }>(
+  events: T[],
+  sentAt: number | undefined,
+  now: number,
+): T[] {
+  if (sentAt === undefined || !Number.isFinite(sentAt)) return events;
+  const offset = now - sentAt;
+  if (Math.abs(offset) <= CLOCK_SKEW_TOLERANCE_MS) return events;
+  return events.map((event) => ({
+    ...event,
+    timestamp: event.timestamp + offset,
+  }));
+}
+
 export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
   if (!Number.isFinite(event.timestamp))
     throw new Error("timestamp is invalid");

package/src/component/schema.ts

@@ -21,6 +21,7 @@ export default defineSchema({
   sessions: defineTable(
     sessionFieldsValidator.extend({
       geoLookupAttemptedAt: v.optional(v.number()),
+      lastPageviewAt: v.optional(v.number()),
     }),
   )
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
@@ -53,6 +54,8 @@ export default defineSchema({
     lastHeartbeatEventId: v.optional(v.string()),
     recentHeartbeatEventIds: v.optional(v.array(v.string())),
     eventCount: v.number(),
+    previousPath: v.optional(v.string()),
+    movedAt: v.optional(v.number()),
   })
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
@@ -103,6 +106,28 @@ export default defineSchema({
       "bucketStart",
     ]),
 
+  routeDailyRollups: defineTable({
+    siteId: v.id("sites"),
+    bucketStart: v.number(),
+    shard: v.number(),
+    generation: v.literal(1),
+    route: v.string(),
+    pageviews: v.number(),
+    entries: v.number(),
+    continued: v.number(),
+    dwellMs: v.number(),
+    dwellSamples: v.number(),
+    visitorSketch: visitorSketchValidator,
+    outgoing: v.array(v.object({ route: v.string(), count: v.number() })),
+  })
+    .index("by_siteId_and_route_and_bucketStart_and_shard", [
+      "siteId",
+      "route",
+      "bucketStart",
+      "shard",
+    ])
+    .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
+
   goalDailyRollups: defineTable({
     siteId: v.id("sites"),
     goalId: v.id("goals"),
@@ -206,6 +231,7 @@ export default defineSchema({
     siteId: v.id("sites"),
     generation: v.literal(1),
     rollupCompleteFrom: v.number(),
+    siteMapSince: v.optional(v.number()),
     retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
     updatedAt: v.number(),
   }).index("by_siteId", ["siteId"]),