@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10

package/dist/component/reports.js

@@ -1,16 +1,20 @@
 import { paginationOptsValidator, paginationResultValidator, } from "convex/server";
 import { v } from "convex/values";
 import { query } from "./_generated/server.js";
-import { AGGREGATE_SHARDS, DAY_MS, HOUR_MS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_MANUAL_PAGINATION_OFFSET, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, } from "./constants.js";
+import { AGGREGATE_SHARDS, DAY_MS, FEATURE_REPORT_BYTE_RESERVE, FEATURE_REPORT_DOCUMENT_RESERVE, HOUR_MS, HOURLY_AGGREGATE_SHARDS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_FEATURE_ROLLUP_DAYS, MAX_FEATURE_ROLLUP_ROWS, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_BUCKETS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, MAX_SITE_MAP_ROUTES_PER_DAY, MAX_SITE_MAP_ROWS, MAX_VITALS_REPORT_ROWS, REPORT_ROLLUP_SHARDS, ROUTE_TRANSITION_SLOTS, SITE_MAP_OTHER_ROUTE, VITALS_REPORT_PAGE_LIMIT, } from "./constants.js";
+import { createVisitorSketch, estimateVisitors, unionVisitorSketches, } from "./cardinality.js";
 import { fail } from "./errors.js";
+import { compareJourneyEvents, hydrateEventContexts, loadSessionJourneyEvents, loadVisitorJourneyEvents, } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
-import { affiliateFieldsValidator, eventFieldsValidator, funnelFieldsValidator, goalFieldsValidator, sessionFieldsValidator, } from "./validators.js";
+import { affiliateFieldsValidator, eventFieldsValidator, funnelFieldsValidator, goalFieldsValidator, sessionFieldsValidator, vitalMetricValidator, } from "./validators.js";
+import { resolveVisitorIdentities } from "./visitors.js";
+import { mergeVitalHistograms, VITAL_ALL, VITAL_METRICS, vitalHistogramPercentile, vitalRatingCounts, } from "./vitals.js";
 const sessionDocumentValidator = sessionFieldsValidator.extend({
     _id: v.id("sessions"),
     _creationTime: v.number(),
 });
 const eventDocumentValidator = eventFieldsValidator.extend({
-    _id: v.id("events"),
+    _id: v.string(),
     _creationTime: v.number(),
 });
 const liveSessionDocumentValidator = v.object({
@@ -33,6 +37,8 @@ const liveSessionDocumentValidator = v.object({
     device: v.string(),
     pageviewCount: v.number(),
     eventCount: v.number(),
+    previousPath: v.optional(v.string()),
+    movedAt: v.optional(v.number()),
 });
 const conversionDocumentValidator = v.object({
     _id: v.id("conversions"),
@@ -64,12 +70,33 @@ const metricTotalsValidator = v.object({
     conversions: v.number(),
     revenueCents: v.number(),
 });
-const timePointValidator = metricTotalsValidator.extend({ timestamp: v.number() });
+const timePointValidator = metricTotalsValidator.extend({
+    timestamp: v.number(),
+});
 const topItemValidator = v.object({
     value: v.string(),
     count: v.number(),
     revenueCents: v.number(),
 });
+const coverageStateValidator = v.union(v.literal("complete"), v.literal("partial"), v.literal("retained"), v.literal("unavailable"));
+const resultCoverageValidator = v.object({
+    state: coverageStateValidator,
+    generation: v.union(v.number(), v.null()),
+    availableFrom: v.union(v.number(), v.null()),
+    availableThrough: v.union(v.number(), v.null()),
+    retainedBefore: v.union(v.number(), v.null()),
+});
+const rangeSemanticsValidator = v.object({
+    from: v.number(),
+    to: v.number(),
+    boundary: v.literal("inclusive"),
+    timezone: v.literal("UTC"),
+});
+const reportMetadataValidator = v.object({
+    basis: v.union(v.literal("dailyRollup"), v.literal("boundedLegacy")),
+    coverage: resultCoverageValidator,
+    range: rangeSemanticsValidator,
+});
 export const overview = query({
     args: {
         siteIds: v.array(v.id("sites")),
@@ -85,12 +112,40 @@ export const overview = query({
         totals: metricTotalsValidator,
         timeSeries: v.array(timePointValidator),
         topSources: v.array(topItemValidator),
+        topCampaigns: v.array(topItemValidator),
         topPages: v.array(topItemValidator),
         topCountries: v.array(topItemValidator),
         topDevices: v.array(topItemValidator),
         topBrowsers: v.array(topItemValidator),
         topEvents: v.array(topItemValidator),
         topGoals: v.array(topItemValidator),
+        metadata: v.object({
+            visitors: v.object({
+                basis: v.union(v.literal("estimatedUnique"), v.literal("bucketOccurrences")),
+                exact: v.boolean(),
+                legacyIncomplete: v.boolean(),
+                sketchVersion: v.union(v.number(), v.null()),
+                precision: v.union(v.number(), v.null()),
+            }),
+            breakdowns: v.object({
+                topSources: v.literal("eventVolume"),
+                topCampaigns: v.literal("eventVolume"),
+                topPages: v.literal("eventVolume"),
+                topCountries: v.literal("eventVolume"),
+                topDevices: v.literal("eventVolume"),
+                topBrowsers: v.literal("eventVolume"),
+                topEvents: v.literal("eventVolume"),
+                topGoals: v.literal("goalCompletionVolume"),
+            }),
+            coverage: resultCoverageValidator,
+            range: rangeSemanticsValidator,
+            features: v.object({
+                sessionReplay: v.literal("unsupported"),
+                errorInsights: v.literal("unsupported"),
+                outboundLinks: v.literal("eventVolume"),
+                botDetection: v.literal("userAgent"),
+            }),
+        }),
     }),
     handler: async (ctx, args) => {
         const siteIds = validateSiteIds(args.siteIds);
@@ -108,21 +163,31 @@ export const overview = query({
         }
         const currency = [...currencies][0];
         const { from, to } = validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-        const interval = args.interval ?? (to - from <= 7 * DAY_MS ? "hour" : "day");
-        if (interval === "hour" && to - from > 7 * DAY_MS) {
-            fail("INVALID_ARGUMENT", "hourly reports are limited to seven days");
+        const interval = args.interval ?? (to - from <= DAY_MS ? "hour" : "day");
+        if (interval === "hour" && to - from > DAY_MS) {
+            fail("INVALID_ARGUMENT", "hourly reports are limited to 24 hours");
         }
         const intervalMs = interval === "hour" ? HOUR_MS : DAY_MS;
+        requireAlignedRange(from, to, intervalMs, `${interval} overview`);
         const firstBucket = Math.floor(from / intervalMs) * intervalMs;
         const lastBucket = Math.floor(to / intervalMs) * intervalMs;
         const bucketCount = Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
-        const perSiteLimit = bucketCount * AGGREGATE_SHARDS;
+        const perSiteLimit = bucketCount *
+            (interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
         if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
             fail("LIMIT_EXCEEDED", "overview range is too wide for the requested number of sites", { maximumAggregateRows: MAX_OVERVIEW_AGGREGATE_ROWS });
         }
         const totals = emptyTotals();
         const points = new Map();
+        const visitorSketch = createVisitorSketch();
+        let mergedVisitorSketch = visitorSketch;
+        const pointVisitorSketches = new Map();
+        let visitorsComplete = true;
         const dimensions = new Map();
+        const overviewDataset = interval === "hour"
+            ? "overviewHour"
+            : "overviewDay";
+        await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
         for (const siteId of siteIds) {
             const buckets = await ctx.db
                 .query("aggregateBuckets")
@@ -131,12 +196,21 @@ export const overview = query({
                 .eq("granularity", interval)
                 .gte("bucketStart", firstBucket)
                 .lte("bucketStart", lastBucket))
-                .take(perSiteLimit);
+                .take(perSiteLimit + 1);
+            failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
             for (const bucket of buckets) {
                 addMetrics(totals, bucket);
                 const point = points.get(bucket.bucketStart) ?? emptyTotals();
                 addMetrics(point, bucket);
                 points.set(bucket.bucketStart, point);
+                if (bucket.visitorSketch && bucket.visitorSketchComplete === true) {
+                    mergedVisitorSketch = unionVisitorSketches(mergedVisitorSketch, bucket.visitorSketch);
+                    pointVisitorSketches.set(bucket.bucketStart, unionVisitorSketches(pointVisitorSketches.get(bucket.bucketStart) ??
+                        createVisitorSketch(bucket.visitorSketch.precision), bucket.visitorSketch));
+                }
+                else if (bucket.events > 0) {
+                    visitorsComplete = false;
+                }
                 for (const dimension of bucket.dimensions) {
                     const key = `${dimension.type}:${dimension.value}`;
                     const current = dimensions.get(key) ?? {
@@ -155,6 +229,17 @@ export const overview = query({
             .map(([, value]) => value)
             .sort((left, right) => right.count - left.count || left.value.localeCompare(right.value))
             .slice(0, 10);
+        if (visitorsComplete) {
+            totals.visitors = estimateVisitors(mergedVisitorSketch);
+            for (const [timestamp, point] of points) {
+                point.visitors = estimateVisitors(pointVisitorSketches.get(timestamp) ?? createVisitorSketch());
+            }
+        }
+        const coverage = await coverageForRange(ctx, siteIds, from, to, {
+            datasets: [overviewDataset],
+            complete: visitorsComplete,
+        });
+        const visitorBasis = visitorsComplete ? "estimatedUnique" : "bucketOccurrences";
         return {
             from,
             to,
@@ -165,12 +250,40 @@ export const overview = query({
                 .sort(([left], [right]) => left - right)
                 .map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
             topSources: top("source"),
+            topCampaigns: top("campaign"),
             topPages: top("page"),
             topCountries: top("country"),
             topDevices: top("device"),
             topBrowsers: top("browser"),
             topEvents: top("event"),
             topGoals: top("goal"),
+            metadata: {
+                visitors: {
+                    basis: visitorBasis,
+                    exact: false,
+                    legacyIncomplete: !visitorsComplete,
+                    sketchVersion: visitorsComplete ? 1 : null,
+                    precision: visitorsComplete ? 6 : null,
+                },
+                breakdowns: {
+                    topSources: "eventVolume",
+                    topCampaigns: "eventVolume",
+                    topPages: "eventVolume",
+                    topCountries: "eventVolume",
+                    topDevices: "eventVolume",
+                    topBrowsers: "eventVolume",
+                    topEvents: "eventVolume",
+                    topGoals: "goalCompletionVolume",
+                },
+                coverage,
+                range: rangeSemantics(from, to),
+                features: {
+                    sessionReplay: "unsupported",
+                    errorInsights: "unsupported",
+                    outboundLinks: "eventVolume",
+                    botDetection: "userAgent",
+                },
+            },
         };
     },
 });
@@ -184,7 +297,9 @@ export const liveVisitors = query({
     handler: async (ctx, args) => {
         const siteIds = validateSiteIds(args.siteIds);
         const limit = args.limit ?? 100;
-        if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIVE_VISITORS) {
+        if (!Number.isSafeInteger(limit) ||
+            limit < 1 ||
+            limit > MAX_LIVE_VISITORS) {
             fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_LIVE_VISITORS}`);
         }
         if (!Number.isFinite(args.now) || args.now < 0) {
@@ -221,9 +336,241 @@ export const liveVisitors = query({
             device: row.device,
             pageviewCount: row.pageviewCount,
             eventCount: row.eventCount,
+            ...(row.previousPath ? { previousPath: row.previousPath } : {}),
+            ...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
         }));
     },
 });
+const siteMapRouteValidator = v.object({
+    route: v.string(),
+    pageviews: v.number(),
+    visitors: v.number(),
+    entries: v.number(),
+    exits: v.number(),
+    bounces: v.number(),
+    transitionsIn: v.number(),
+    transitionsOut: v.number(),
+    dwellMs: v.number(),
+    dwellSamples: v.number(),
+    daily: v.array(v.object({ bucketStart: v.number(), pageviews: v.number() })),
+});
+const siteMapTransitionValidator = v.object({
+    from: v.string(),
+    to: v.string(),
+    count: v.number(),
+});
+const siteMapCoverageDataset = "siteMapRollups";
+/**
+ * Every route the sites served in the range and the transitions between
+ * them, read from the daily route rollups newest day first. Exits and
+ * bounces are derived rather than counted: a session enters once, leaves
+ * once, and every later pageview is a transition, so per route
+ * `exits = entries + transitionsIn - transitionsOut` and
+ * `bounces = entries - continued` hold without a session-end hook. The
+ * report never falls back to raw telemetry; days it could not cover are
+ * declared through `coverage` and `window`.
+ */
+export const siteMap = query({
+    args: {
+        siteIds: v.array(v.id("sites")),
+        from: v.number(),
+        to: v.number(),
+    },
+    returns: v.object({
+        from: v.number(),
+        to: v.number(),
+        window: v.union(v.object({ from: v.number(), to: v.number() }), v.null()),
+        routes: v.array(siteMapRouteValidator),
+        transitions: v.array(siteMapTransitionValidator),
+        truncated: v.object({ routes: v.boolean(), transitions: v.boolean() }),
+        metadata: reportMetadataValidator.extend({
+            visitors: v.object({
+                basis: v.literal("estimatedUnique"),
+                exact: v.boolean(),
+            }),
+            limits: v.object({
+                routesPerDay: v.number(),
+                transitionSlots: v.number(),
+                rows: v.number(),
+            }),
+        }),
+    }),
+    handler: async (ctx, args) => {
+        const siteIds = validateSiteIds(args.siteIds);
+        validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
+        requireAlignedRange(args.from, args.to, DAY_MS, "site map");
+        const controls = await Promise.all(siteIds.map((siteId) => ctx.db
+            .query("analyticsCoverage")
+            .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+            .unique()));
+        const since = controls.every((control) => control?.siteMapSince !== undefined)
+            ? Math.max(...controls.map((control) => control.siteMapSince))
+            : null;
+        const retainedBefore = controls.reduce((latest, control) => {
+            const before = control?.retained.find((item) => item.dataset === siteMapCoverageDataset)?.before;
+            return before === undefined ? latest : Math.max(latest ?? 0, before);
+        }, null);
+        const metadata = (state) => ({
+            basis: "dailyRollup",
+            coverage: {
+                state,
+                generation: 1,
+                availableFrom: since,
+                availableThrough: since === null ? null : args.to,
+                retainedBefore,
+            },
+            range: rangeSemantics(args.from, args.to),
+            visitors: { basis: "estimatedUnique", exact: false },
+            limits: {
+                routesPerDay: MAX_SITE_MAP_ROUTES_PER_DAY,
+                transitionSlots: ROUTE_TRANSITION_SLOTS,
+                rows: MAX_SITE_MAP_ROWS,
+            },
+        });
+        if (since === null ||
+            args.to < since ||
+            (retainedBefore !== null && args.to < retainedBefore)) {
+            return {
+                from: args.from,
+                to: args.to,
+                window: null,
+                routes: [],
+                transitions: [],
+                truncated: { routes: false, transitions: false },
+                metadata: metadata("unavailable"),
+            };
+        }
+        const readFrom = Math.max(args.from, Math.floor(since / DAY_MS) * DAY_MS, retainedBefore ?? 0);
+        let budget = MAX_SITE_MAP_ROWS;
+        let completeFrom = readFrom;
+        const rows = [];
+        for (const siteId of siteIds) {
+            const read = await readRouteRollups(ctx, siteId, readFrom, args.to, budget);
+            budget -= read.rows.length;
+            completeFrom = Math.max(completeFrom, read.completeFrom);
+            rows.push(...read.rows);
+        }
+        const windowFrom = Math.max(args.from, completeFrom, since, retainedBefore ?? 0);
+        const routes = new Map();
+        const totalsFor = (route) => {
+            let current = routes.get(route);
+            if (!current) {
+                current = {
+                    pageviews: 0,
+                    entries: 0,
+                    continued: 0,
+                    dwellMs: 0,
+                    dwellSamples: 0,
+                    transitionsIn: 0,
+                    transitionsOut: 0,
+                    sketch: createVisitorSketch(),
+                    daily: new Map(),
+                };
+                routes.set(route, current);
+            }
+            return current;
+        };
+        const edges = new Map();
+        for (const row of rows) {
+            if (row.bucketStart < completeFrom)
+                continue;
+            const current = totalsFor(row.route);
+            current.pageviews += row.pageviews;
+            current.entries += row.entries;
+            current.continued += row.continued;
+            current.dwellMs += row.dwellMs;
+            current.dwellSamples += row.dwellSamples;
+            current.sketch = unionVisitorSketches(current.sketch, row.visitorSketch);
+            current.daily.set(row.bucketStart, (current.daily.get(row.bucketStart) ?? 0) + row.pageviews);
+            for (const target of row.outgoing) {
+                const key = `${row.route}\u0000${target.route}`;
+                const edge = edges.get(key) ?? {
+                    from: row.route,
+                    to: target.route,
+                    count: 0,
+                };
+                edge.count += target.count;
+                edges.set(key, edge);
+                current.transitionsOut += target.count;
+            }
+        }
+        // A target without a row of its own in this window — folded past the
+        // daily cap, or already retained — counts under "(other)".
+        const transitions = new Map();
+        for (const edge of edges.values()) {
+            const to = routes.has(edge.to) ? edge.to : SITE_MAP_OTHER_ROUTE;
+            const key = `${edge.from}\u0000${to}`;
+            const merged = transitions.get(key) ?? { from: edge.from, to, count: 0 };
+            merged.count += edge.count;
+            transitions.set(key, merged);
+            totalsFor(to).transitionsIn += edge.count;
+        }
+        const routeRows = [...routes.entries()]
+            .map(([route, totals]) => ({
+            route,
+            pageviews: totals.pageviews,
+            visitors: estimateVisitors(totals.sketch),
+            entries: totals.entries,
+            exits: Math.max(0, totals.entries + totals.transitionsIn - totals.transitionsOut),
+            bounces: Math.max(0, totals.entries - totals.continued),
+            transitionsIn: totals.transitionsIn,
+            transitionsOut: totals.transitionsOut,
+            dwellMs: totals.dwellMs,
+            dwellSamples: totals.dwellSamples,
+            daily: [...totals.daily.entries()]
+                .sort(([left], [right]) => left - right)
+                .map(([bucketStart, pageviews]) => ({ bucketStart, pageviews })),
+        }))
+            .sort((left, right) => right.pageviews - left.pageviews ||
+            left.route.localeCompare(right.route));
+        const transitionRows = [...transitions.values()].sort((left, right) => right.count - left.count ||
+            left.from.localeCompare(right.from) ||
+            left.to.localeCompare(right.to));
+        return {
+            from: args.from,
+            to: args.to,
+            window: completeFrom > args.to ? null : { from: windowFrom, to: args.to },
+            routes: routeRows,
+            transitions: transitionRows,
+            truncated: {
+                routes: routes.has(SITE_MAP_OTHER_ROUTE),
+                transitions: transitionRows.some((transition) => transition.from === SITE_MAP_OTHER_ROUTE ||
+                    transition.to === SITE_MAP_OTHER_ROUTE),
+            },
+            metadata: metadata(windowFrom > args.from ? "partial" : "complete"),
+        };
+    },
+});
+/**
+ * Reads a site's route rollups newest day first until the row budget or the
+ * transaction's read headroom runs out, then drops the day it could only
+ * read in part so every day it returns is complete.
+ */
+async function readRouteRollups(ctx, siteId, from, to, budget) {
+    const rows = [];
+    let cutAt = null;
+    const query = ctx.db
+        .query("routeDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (range) => range
+        .eq("siteId", siteId)
+        .gte("bucketStart", from)
+        .lte("bucketStart", to))
+        .order("desc");
+    for await (const row of query) {
+        if (rows.length >= budget ||
+            (rows.length % 256 === 255 && !(await hasFeatureReadHeadroom(ctx)))) {
+            cutAt = row.bucketStart;
+            break;
+        }
+        rows.push(row);
+    }
+    if (cutAt === null)
+        return { rows, completeFrom: from };
+    while (rows.length > 0 && rows[rows.length - 1].bucketStart === cutAt) {
+        rows.pop();
+    }
+    return { rows, completeFrom: cutAt + DAY_MS };
+}
 export const listSessions = query({
     args: {
         siteId: v.id("sites"),
@@ -234,16 +581,50 @@ export const listSessions = query({
     returns: paginationResultValidator(sessionDocumentValidator),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-        const pagination = parseManualPagination(args.paginationOpts);
-        const rows = await ctx.db
-            .query("sessions")
-            .withIndex("by_siteId_and_lastSeenAt", (range) => range
-            .eq("siteId", args.siteId)
-            .gte("lastSeenAt", args.from)
-            .lte("lastSeenAt", args.to))
-            .order("desc")
-            .take(pagination.offset + pagination.numItems + 1);
-        return manualPaginationResult(rows, pagination);
+        await assertSourceAvailable(ctx, [args.siteId], ["sessions"], args.from);
+        const pagination = parseKeysetPagination(args.paginationOpts, {
+            kind: "sessions",
+            siteId: String(args.siteId),
+            from: args.from,
+            to: args.to,
+        });
+        const rows = [];
+        const take = pagination.numItems + 1;
+        if (pagination.cursor) {
+            rows.push(...(await ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_lastSeenAt", (range) => range
+                .eq("siteId", args.siteId)
+                .eq("lastSeenAt", pagination.cursor.timestamp)
+                .lt("_creationTime", pagination.cursor.creationTime))
+                .order("desc")
+                .take(take)));
+            if (rows.length < take) {
+                rows.push(...(await ctx.db
+                    .query("sessions")
+                    .withIndex("by_siteId_and_lastSeenAt", (range) => range
+                    .eq("siteId", args.siteId)
+                    .gte("lastSeenAt", args.from)
+                    .lt("lastSeenAt", pagination.cursor.timestamp))
+                    .order("desc")
+                    .take(take - rows.length)));
+            }
+        }
+        else {
+            rows.push(...(await ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_lastSeenAt", (range) => range
+                .eq("siteId", args.siteId)
+                .gte("lastSeenAt", args.from)
+                .lte("lastSeenAt", args.to))
+                .order("desc")
+                .take(take)));
+        }
+        const publicRows = rows.map(({ geoLookupAttemptedAt: _attemptedAt, lastPageviewAt: _lastPageviewAt, ...row }) => row);
+        return keysetPaginationResult(publicRows, pagination, (row) => ({
+            timestamp: row.lastSeenAt,
+            creationTime: row._creationTime,
+        }));
     },
 });
 export const sessionJourney = query({
@@ -257,17 +638,24 @@ export const sessionJourney = query({
     returns: paginationResultValidator(eventDocumentValidator),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-        const pagination = parseManualPagination(args.paginationOpts);
-        const rows = await ctx.db
-            .query("events")
-            .withIndex("by_siteId_and_sessionId_and_timestamp", (range) => range
-            .eq("siteId", args.siteId)
-            .eq("sessionId", args.sessionId)
-            .gte("timestamp", args.from)
-            .lte("timestamp", args.to))
-            .order("asc")
-            .take(pagination.offset + pagination.numItems + 1);
-        return manualPaginationResult(rows, pagination);
+        await assertSourceAvailable(ctx, [args.siteId], ["events"], args.from);
+        const pagination = parseJourneyPagination(args.paginationOpts, {
+            kind: "events",
+            siteId: String(args.siteId),
+            sessionId: args.sessionId,
+            from: args.from,
+            to: args.to,
+        });
+        const rows = await loadSessionJourneyEvents(ctx, {
+            siteId: args.siteId,
+            sessionId: args.sessionId,
+            from: args.from,
+            to: args.to,
+            cursor: pagination.cursor,
+            resultLimit: pagination.numItems + 1,
+        });
+        const hydratedRows = await hydrateEventContexts(ctx, rows);
+        return journeyPaginationResult(hydratedRows, pagination);
     },
 });
 export const listConversions = query({
@@ -280,16 +668,49 @@ export const listConversions = query({
     returns: paginationResultValidator(conversionDocumentValidator),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-        const pagination = parseManualPagination(args.paginationOpts);
-        const rows = await ctx.db
-            .query("conversions")
-            .withIndex("by_siteId_and_timestamp", (range) => range
-            .eq("siteId", args.siteId)
-            .gte("timestamp", args.from)
-            .lte("timestamp", args.to))
-            .order("desc")
-            .take(pagination.offset + pagination.numItems + 1);
-        return manualPaginationResult(rows, pagination);
+        await assertSourceAvailable(ctx, [args.siteId], ["conversions"], args.from);
+        const pagination = parseKeysetPagination(args.paginationOpts, {
+            kind: "conversions",
+            siteId: String(args.siteId),
+            from: args.from,
+            to: args.to,
+        });
+        const rows = [];
+        const take = pagination.numItems + 1;
+        if (pagination.cursor) {
+            rows.push(...(await ctx.db
+                .query("conversions")
+                .withIndex("by_siteId_and_timestamp", (range) => range
+                .eq("siteId", args.siteId)
+                .eq("timestamp", pagination.cursor.timestamp)
+                .lt("_creationTime", pagination.cursor.creationTime))
+                .order("desc")
+                .take(take)));
+            if (rows.length < take) {
+                rows.push(...(await ctx.db
+                    .query("conversions")
+                    .withIndex("by_siteId_and_timestamp", (range) => range
+                    .eq("siteId", args.siteId)
+                    .gte("timestamp", args.from)
+                    .lt("timestamp", pagination.cursor.timestamp))
+                    .order("desc")
+                    .take(take - rows.length)));
+            }
+        }
+        else {
+            rows.push(...(await ctx.db
+                .query("conversions")
+                .withIndex("by_siteId_and_timestamp", (range) => range
+                .eq("siteId", args.siteId)
+                .gte("timestamp", args.from)
+                .lte("timestamp", args.to))
+                .order("desc")
+                .take(take)));
+        }
+        return keysetPaginationResult(rows, pagination, (row) => ({
+            timestamp: row.timestamp,
+            creationTime: row._creationTime,
+        }));
     },
 });
 export const visitorJourney = query({
@@ -304,25 +725,30 @@ export const visitorJourney = query({
     handler: async (ctx, args) => {
         const siteIds = validateSiteIds(args.siteIds);
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
+        await assertSourceAvailable(ctx, siteIds, ["events"], args.from);
         const limit = args.limit ?? 200;
-        if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
+        if (!Number.isSafeInteger(limit) ||
+            limit < 1 ||
+            limit > MAX_JOURNEY_EVENTS) {
             fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
         }
         const rows = [];
         for (const siteId of siteIds) {
-            rows.push(...(await ctx.db
-                .query("events")
-                .withIndex("by_siteId_and_visitorId_and_timestamp", (range) => range
-                .eq("siteId", siteId)
-                .eq("visitorId", args.visitorId)
-                .gte("timestamp", args.from)
-                .lte("timestamp", args.to))
-                .order("asc")
-                .take(limit)));
+            for (const visitorId of await resolveVisitorIdentities(ctx, siteId, args.visitorId)) {
+                const cutoff = rows.length >= limit ? rows[limit - 1].timestamp : args.to;
+                rows.push(...(await loadVisitorJourneyEvents(ctx, {
+                    siteId,
+                    visitorId,
+                    from: args.from,
+                    to: cutoff,
+                    resultLimit: limit,
+                })));
+                rows.sort(compareJourneyEvents);
+                if (rows.length > limit)
+                    rows.length = limit;
+            }
         }
-        return rows
-            .sort((left, right) => left.timestamp - right.timestamp || left.sequence - right.sequence)
-            .slice(0, limit);
+        return hydrateEventContexts(ctx, rows);
     },
 });
 const goalReportItemValidator = goalFieldsValidator.extend({
@@ -336,69 +762,126 @@ export const goalsReport = query({
     returns: v.object({
         goals: v.array(goalReportItemValidator),
         truncated: v.boolean(),
+        metadata: reportMetadataValidator,
     }),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
-        const goals = await ctx.db
+        const goalRows = await ctx.db
             .query("goals")
             .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
-            .take(MAX_GOALS_PER_SITE);
-        const completions = await ctx.db
-            .query("goalCompletions")
-            .withIndex("by_siteId_and_timestamp", (range) => range
-            .eq("siteId", args.siteId)
-            .gte("timestamp", args.from)
-            .lte("timestamp", args.to))
-            .take(5_001);
+            .take(MAX_GOALS_PER_SITE + 1);
+        failIfIncomplete(goalRows, MAX_GOALS_PER_SITE, "goal definitions");
+        const goals = goalRows;
         const totals = new Map();
-        for (const completion of completions.slice(0, 5_000)) {
-            const key = String(completion.goalId);
-            const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
-            total.completions += 1;
-            total.valueCents += completion.valueCents;
-            totals.set(key, total);
+        const useRollups = await dailyRollupsAvailable(ctx, args.siteId, args.from, args.to, "goalRollups");
+        const rollups = useRollups
+            ? await loadGoalDailyRollups(ctx, args.siteId, goals.map((goal) => goal._id), args.from, args.to)
+            : null;
+        if (rollups) {
+            for (const row of rollups) {
+                const key = String(row.goalId);
+                const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
+                total.completions += row.completions;
+                total.valueCents += row.valueCents;
+                totals.set(key, total);
+            }
+        }
+        else {
+            await assertSourceAvailable(ctx, [args.siteId], ["goalCompletions"], args.from);
+            const completions = await ctx.db
+                .query("goalCompletions")
+                .withIndex("by_siteId_and_timestamp", (range) => range
+                .eq("siteId", args.siteId)
+                .gte("timestamp", args.from)
+                .lte("timestamp", args.to))
+                .take(MAX_REPORT_BUCKETS + 1);
+            failIfIncomplete(completions, MAX_REPORT_BUCKETS, "goal completions");
+            for (const completion of completions) {
+                const key = String(completion.goalId);
+                const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
+                total.completions += 1;
+                total.valueCents += completion.valueCents;
+                totals.set(key, total);
+            }
         }
         return {
             goals: goals.map((goal) => ({
                 ...goal,
                 ...(totals.get(String(goal._id)) ?? { completions: 0, valueCents: 0 }),
             })),
-            truncated: completions.length > 5_000,
+            truncated: false,
+            metadata: {
+                basis: rollups ? "dailyRollup" : "boundedLegacy",
+                coverage: await coverageForRange(ctx, [args.siteId], args.from, args.to, {
+                    datasets: [rollups ? "goalRollups" : "goalCompletions"],
+                    complete: true,
+                    generation: rollups ? 1 : null,
+                }),
+                range: rangeSemantics(args.from, args.to),
+            },
         };
     },
 });
 const funnelReportItemValidator = funnelFieldsValidator.extend({
     _id: v.id("funnels"),
     _creationTime: v.number(),
-    stepsReport: v.array(v.object({ step: v.number(), name: v.string(), entrants: v.number(), completions: v.number() })),
+    stepsReport: v.array(v.object({
+        step: v.number(),
+        name: v.string(),
+        entrants: v.number(),
+        completions: v.number(),
+    })),
     truncated: v.boolean(),
+    metadata: reportMetadataValidator,
 });
 export const funnelsReport = query({
     args: { siteId: v.id("sites"), from: v.number(), to: v.number() },
     returns: v.array(funnelReportItemValidator),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, 90);
-        const funnels = await ctx.db
+        requireAlignedRange(args.from, args.to, DAY_MS, "funnel report");
+        const funnelRows = await ctx.db
             .query("funnels")
             .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
-            .take(MAX_FUNNELS_PER_SITE);
+            .take(MAX_FUNNELS_PER_SITE + 1);
+        failIfIncomplete(funnelRows, MAX_FUNNELS_PER_SITE, "funnel definitions");
+        const funnels = funnelRows;
+        const useRollups = await dailyRollupsAvailable(ctx, args.siteId, args.from, args.to, "funnelRollups");
+        const rollups = useRollups
+            ? await loadFunnelDailyRollups(ctx, args.siteId, funnels.map((funnel) => funnel._id), args.from, args.to)
+            : null;
+        if (!rollups) {
+            await assertSourceAvailable(ctx, [args.siteId], ["funnelSteps"], args.from);
+        }
+        const reportCoverage = await coverageForRange(ctx, [args.siteId], args.from, args.to, {
+            datasets: [rollups ? "funnelRollups" : "funnelSteps"],
+            complete: true,
+            generation: rollups ? 1 : null,
+        });
         const reports = [];
         for (const funnel of funnels) {
-            const buckets = await ctx.db
-                .query("funnelStepBuckets")
-                .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) => range
-                .eq("siteId", args.siteId)
-                .eq("funnelId", funnel._id)
-                .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
-                .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS))
-                .take(601);
             const stepsReport = funnel.steps.map((step, index) => ({
                 step: index,
                 name: step.name,
                 entrants: 0,
                 completions: 0,
             }));
-            for (const bucket of buckets.slice(0, 600)) {
+            const buckets = rollups
+                ? rollups
+                    .filter((row) => row.funnelId === funnel._id)
+                    .flatMap((row) => row.steps)
+                : await ctx.db
+                    .query("funnelStepBuckets")
+                    .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) => range
+                    .eq("siteId", args.siteId)
+                    .eq("funnelId", funnel._id)
+                    .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
+                    .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS))
+                    .take(MAX_REPORT_BUCKETS + 1);
+            if (!rollups) {
+                failIfIncomplete(buckets, MAX_REPORT_BUCKETS, "funnel step buckets");
+            }
+            for (const bucket of buckets) {
                 const step = stepsReport[bucket.step];
                 if (step) {
                     step.entrants += bucket.entrants;
@@ -408,7 +891,14 @@ export const funnelsReport = query({
             reports.push({
                 ...funnel,
                 stepsReport,
-                truncated: buckets.length > 600,
+                truncated: false,
+                metadata: {
+                    basis: rollups
+                        ? "dailyRollup"
+                        : "boundedLegacy",
+                    coverage: reportCoverage,
+                    range: rangeSemantics(args.from, args.to),
+                },
             });
         }
         return reports;
@@ -428,54 +918,94 @@ export const affiliatesReport = query({
     returns: v.object({
         affiliates: v.array(affiliateReportItemValidator),
         truncated: v.boolean(),
+        metadata: reportMetadataValidator.extend({
+            uniqueVisitors: v.object({
+                basis: v.union(v.literal("estimatedUnique"), v.literal("exactLegacy")),
+                exact: v.boolean(),
+            }),
+        }),
     }),
     handler: async (ctx, args) => {
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
+        requireAlignedRange(args.from, args.to, DAY_MS, "affiliate report");
         const affiliateRows = await ctx.db
             .query("affiliates")
             .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
             .order("desc")
             .take(MAX_AFFILIATES_PER_SITE + 1);
-        const affiliates = affiliateRows.slice(0, MAX_AFFILIATES_PER_SITE);
-        const stats = await ctx.db
-            .query("affiliateDailyStats")
-            .withIndex("by_siteId_and_bucketStart", (range) => range
-            .eq("siteId", args.siteId)
-            .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
-            .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS))
-            .take(5_001);
-        const sessions = await ctx.db
-            .query("sessions")
-            .withIndex("by_siteId_and_startedAt", (range) => range
-            .eq("siteId", args.siteId)
-            .gte("startedAt", args.from)
-            .lte("startedAt", args.to))
-            .take(5_001);
+        failIfIncomplete(affiliateRows, MAX_AFFILIATES_PER_SITE, "affiliate definitions");
+        const affiliates = affiliateRows;
+        const useRollups = await dailyRollupsAvailable(ctx, args.siteId, args.from, args.to, "affiliateRollups");
+        const rollups = useRollups
+            ? await loadAffiliateDailyRollups(ctx, args.siteId, affiliates.map((affiliate) => affiliate._id), args.from, args.to)
+            : null;
         const visits = new Map();
-        for (const session of sessions.slice(0, 5_000)) {
-            if (!session.affiliateId)
-                continue;
-            const key = String(session.affiliateId);
-            const total = visits.get(key) ?? {
-                visits: 0,
-                visitors: new Set(),
-            };
-            total.visits += 1;
-            total.visitors.add(session.visitorId);
-            visits.set(key, total);
-        }
         const totals = new Map();
-        for (const row of stats.slice(0, 5_000)) {
-            const key = String(row.affiliateId);
-            const total = totals.get(key) ?? {
-                conversions: 0,
-                revenueCents: 0,
-                commissionCents: 0,
-            };
-            total.conversions += row.conversions;
-            total.revenueCents += row.revenueCents;
-            total.commissionCents += row.commissionCents;
-            totals.set(key, total);
+        if (rollups) {
+            for (const row of rollups) {
+                const key = String(row.affiliateId);
+                const traffic = visits.get(key) ?? {
+                    visits: 0,
+                    visitors: new Set(),
+                    sketch: createVisitorSketch(),
+                };
+                traffic.visits += row.visits;
+                traffic.sketch = unionVisitorSketches(traffic.sketch, row.visitorSketch);
+                visits.set(key, traffic);
+                const total = totals.get(key) ?? {
+                    conversions: 0,
+                    revenueCents: 0,
+                    commissionCents: 0,
+                };
+                total.conversions += row.conversions;
+                total.revenueCents += row.revenueCents;
+                total.commissionCents += row.commissionCents;
+                totals.set(key, total);
+            }
+        }
+        else {
+            await assertSourceAvailable(ctx, [args.siteId], ["affiliateStats", "sessions"], args.from);
+            const stats = await ctx.db
+                .query("affiliateDailyStats")
+                .withIndex("by_siteId_and_bucketStart", (range) => range
+                .eq("siteId", args.siteId)
+                .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
+                .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS))
+                .take(MAX_REPORT_BUCKETS + 1);
+            const sessions = await ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_startedAt", (range) => range
+                .eq("siteId", args.siteId)
+                .gte("startedAt", args.from)
+                .lte("startedAt", args.to))
+                .take(MAX_REPORT_BUCKETS + 1);
+            failIfIncomplete(stats, MAX_REPORT_BUCKETS, "affiliate stat buckets");
+            failIfIncomplete(sessions, MAX_REPORT_BUCKETS, "affiliate sessions");
+            for (const session of sessions) {
+                if (!session.affiliateId)
+                    continue;
+                const key = String(session.affiliateId);
+                const total = visits.get(key) ?? {
+                    visits: 0,
+                    visitors: new Set(),
+                    sketch: createVisitorSketch(),
+                };
+                total.visits += 1;
+                total.visitors.add(session.visitorId);
+                visits.set(key, total);
+            }
+            for (const row of stats) {
+                const key = String(row.affiliateId);
+                const total = totals.get(key) ?? {
+                    conversions: 0,
+                    revenueCents: 0,
+                    commissionCents: 0,
+                };
+                total.conversions += row.conversions;
+                total.revenueCents += row.revenueCents;
+                total.commissionCents += row.commissionCents;
+                totals.set(key, total);
+            }
         }
         return {
             affiliates: affiliates.map((affiliate) => {
@@ -489,15 +1019,548 @@ export const affiliatesReport = query({
                     ...affiliate,
                     ...performance,
                     visits: traffic?.visits ?? 0,
-                    uniqueVisitors: traffic?.visitors.size ?? 0,
+                    uniqueVisitors: rollups
+                        ? estimateVisitors(traffic?.sketch ?? createVisitorSketch())
+                        : (traffic?.visitors.size ?? 0),
                 };
             }),
-            truncated: affiliateRows.length > MAX_AFFILIATES_PER_SITE ||
-                stats.length > 5_000 ||
-                sessions.length > 5_000,
+            truncated: false,
+            metadata: {
+                basis: rollups ? "dailyRollup" : "boundedLegacy",
+                coverage: await coverageForRange(ctx, [args.siteId], args.from, args.to, {
+                    datasets: rollups
+                        ? ["affiliateRollups"]
+                        : ["affiliateStats", "sessions"],
+                    complete: true,
+                    generation: rollups ? 1 : null,
+                }),
+                range: rangeSemantics(args.from, args.to),
+                uniqueVisitors: {
+                    basis: rollups
+                        ? "estimatedUnique"
+                        : "exactLegacy",
+                    exact: !rollups,
+                },
+            },
+        };
+    },
+});
+const vitalStatsValidator = v.object({
+    count: v.number(),
+    p75: v.number(),
+    average: v.number(),
+    good: v.number(),
+    needsImprovement: v.number(),
+    poor: v.number(),
+});
+export const vitalsReport = query({
+    args: { siteId: v.id("sites"), from: v.number(), to: v.number() },
+    returns: v.object({
+        metrics: v.array(vitalStatsValidator.extend({
+            metric: vitalMetricValidator,
+            devices: v.array(vitalStatsValidator.extend({ device: v.string() })),
+            pages: v.array(vitalStatsValidator.extend({ page: v.string() })),
+            timeSeries: v.array(v.object({
+                timestamp: v.number(),
+                count: v.number(),
+                p75: v.number(),
+            })),
+        })),
+        truncated: v.boolean(),
+        metadata: reportMetadataValidator,
+    }),
+    handler: async (ctx, args) => {
+        validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
+        requireAlignedRange(args.from, args.to, DAY_MS, "vitals report");
+        await assertSourceAvailable(ctx, [args.siteId], ["vitalRollups"], args.from);
+        const rows = await ctx.db
+            .query("vitalRollups")
+            .withIndex("by_siteId_and_bucketStart", (range) => range
+            .eq("siteId", args.siteId)
+            .gte("bucketStart", args.from)
+            .lte("bucketStart", args.to))
+            .take(MAX_VITALS_REPORT_ROWS + 1);
+        failIfIncomplete(rows, MAX_VITALS_REPORT_ROWS, "vital rollups");
+        const fold = (groups, metric, key, entry) => {
+            const group = groups.get(metric) ?? new Map();
+            const current = group.get(key) ?? { count: 0, sum: 0, histogram: [] };
+            current.count += entry.count;
+            current.sum += entry.sum;
+            current.histogram = mergeVitalHistograms(current.histogram, entry.histogram);
+            group.set(key, current);
+            groups.set(metric, group);
+        };
+        const byDevice = new Map();
+        const byPage = new Map();
+        const byDay = new Map();
+        for (const row of rows) {
+            for (const entry of row.metrics) {
+                if (row.page === VITAL_ALL) {
+                    // A device row carries the complete signal once, so overall totals
+                    // and the time series derive from this family alone.
+                    fold(byDevice, entry.metric, row.device, entry);
+                    fold(byDay, entry.metric, row.bucketStart, entry);
+                }
+                else {
+                    fold(byPage, entry.metric, row.page, entry);
+                }
+            }
+        }
+        const stats = (metric, accumulated) => ({
+            count: accumulated.count,
+            p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
+            average: accumulated.count > 0
+                ? Math.round(accumulated.sum / accumulated.count)
+                : 0,
+            ...vitalRatingCounts(accumulated.histogram, metric),
+        });
+        const metrics = VITAL_METRICS.map((metric) => {
+            const overall = { count: 0, sum: 0, histogram: [] };
+            for (const accumulated of byDevice.get(metric)?.values() ?? []) {
+                overall.count += accumulated.count;
+                overall.sum += accumulated.sum;
+                overall.histogram = mergeVitalHistograms(overall.histogram, accumulated.histogram);
+            }
+            return {
+                metric,
+                ...stats(metric, overall),
+                devices: [...(byDevice.get(metric) ?? [])]
+                    .map(([device, accumulated]) => ({
+                    device,
+                    ...stats(metric, accumulated),
+                }))
+                    .sort((left, right) => right.count - left.count),
+                pages: [...(byPage.get(metric) ?? [])]
+                    .map(([page, accumulated]) => ({
+                    page,
+                    ...stats(metric, accumulated),
+                }))
+                    .sort((left, right) => right.count - left.count || left.page.localeCompare(right.page))
+                    .slice(0, VITALS_REPORT_PAGE_LIMIT),
+                timeSeries: [...(byDay.get(metric) ?? [])]
+                    .sort(([left], [right]) => left - right)
+                    .map(([timestamp, accumulated]) => ({
+                    timestamp,
+                    count: accumulated.count,
+                    p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
+                })),
+            };
+        });
+        return {
+            metrics,
+            truncated: false,
+            metadata: {
+                basis: "dailyRollup",
+                coverage: await coverageForRange(ctx, [args.siteId], args.from, args.to, { datasets: ["vitalRollups"], complete: true }),
+                range: rangeSemantics(args.from, args.to),
+            },
+        };
+    },
+});
+const coverageDatasetValidator = v.union(v.literal("overviewHour"), v.literal("overviewDay"), v.literal("events"), v.literal("sessions"), v.literal("conversions"), v.literal("goals"), v.literal("funnels"), v.literal("affiliates"), v.literal("vitals"), v.literal("siteMap"));
+export const dataCoverage = query({
+    args: {
+        siteId: v.id("sites"),
+        from: v.number(),
+        to: v.number(),
+    },
+    returns: v.object({
+        siteId: v.id("sites"),
+        from: v.number(),
+        to: v.number(),
+        datasets: v.array(v.object({
+            dataset: coverageDatasetValidator,
+            coverage: resultCoverageValidator,
+        })),
+    }),
+    handler: async (ctx, args) => {
+        validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
+        const datasets = [
+            "overviewHour",
+            "overviewDay",
+            "events",
+            "sessions",
+            "conversions",
+            "goals",
+            "funnels",
+            "affiliates",
+            "vitals",
+            "siteMap",
+        ];
+        return {
+            siteId: args.siteId,
+            from: args.from,
+            to: args.to,
+            datasets: await Promise.all(datasets.map(async (dataset) => ({
+                dataset,
+                coverage: await datasetCoverageForRange(ctx, args.siteId, dataset, args.from, args.to),
+            }))),
         };
     },
 });
+async function dailyRollupsAvailable(ctx, siteId, from, to, dataset) {
+    if (!isAlignedRange(from, to, DAY_MS))
+        return false;
+    const coverage = await ctx.db
+        .query("analyticsCoverage")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    if (!coverage || coverage.rollupCompleteFrom > from) {
+        return false;
+    }
+    const retainedBefore = coverage.retained.find((item) => item.dataset === dataset)?.before;
+    return retainedBefore === undefined || from >= retainedBefore;
+}
+async function loadGoalDailyRollups(ctx, siteId, goalIds, from, to) {
+    const days = Math.floor((to - from) / DAY_MS) + 1;
+    requireFeatureRollupBudget(goalIds.length, days, "goal daily rollups");
+    const rows = [];
+    for (const goalId of goalIds) {
+        const perGoalLimit = days * REPORT_ROLLUP_SHARDS;
+        const goalRows = await ctx.db
+            .query("goalDailyRollups")
+            .withIndex("by_siteId_and_goalId_and_bucketStart", (range) => range
+            .eq("siteId", siteId)
+            .eq("goalId", goalId)
+            .gte("bucketStart", from)
+            .lte("bucketStart", to))
+            .take(perGoalLimit + 1);
+        failIfIncomplete(goalRows, perGoalLimit, "goal daily rollups");
+        rows.push(...goalRows);
+        await requireFeatureReadHeadroom(ctx, "goal daily rollups");
+    }
+    return rows;
+}
+async function loadFunnelDailyRollups(ctx, siteId, funnelIds, from, to) {
+    const days = Math.floor((to - from) / DAY_MS) + 1;
+    requireFeatureRollupBudget(funnelIds.length, days, "funnel daily rollups");
+    const rows = [];
+    for (const funnelId of funnelIds) {
+        const perFunnelLimit = days * REPORT_ROLLUP_SHARDS;
+        const funnelRows = await ctx.db
+            .query("funnelDailyRollups")
+            .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) => range
+            .eq("siteId", siteId)
+            .eq("funnelId", funnelId)
+            .gte("bucketStart", from)
+            .lte("bucketStart", to))
+            .take(perFunnelLimit + 1);
+        failIfIncomplete(funnelRows, perFunnelLimit, "funnel daily rollups");
+        rows.push(...funnelRows);
+        await requireFeatureReadHeadroom(ctx, "funnel daily rollups");
+    }
+    return rows;
+}
+async function loadAffiliateDailyRollups(ctx, siteId, affiliateIds, from, to) {
+    const days = Math.floor((to - from) / DAY_MS) + 1;
+    requireFeatureRollupBudget(affiliateIds.length, days, "affiliate daily rollups");
+    const rows = [];
+    for (const affiliateId of affiliateIds) {
+        const perAffiliateLimit = days * REPORT_ROLLUP_SHARDS;
+        const affiliateRows = await ctx.db
+            .query("affiliateDailyRollups")
+            .withIndex("by_siteId_and_affiliateId_and_bucketStart", (range) => range
+            .eq("siteId", siteId)
+            .eq("affiliateId", affiliateId)
+            .gte("bucketStart", from)
+            .lte("bucketStart", to))
+            .take(perAffiliateLimit + 1);
+        failIfIncomplete(affiliateRows, perAffiliateLimit, "affiliate daily rollups");
+        rows.push(...affiliateRows);
+        await requireFeatureReadHeadroom(ctx, "affiliate daily rollups");
+    }
+    return rows;
+}
+function requireFeatureRollupBudget(entities, days, dataset) {
+    const maximumRows = entities * days * REPORT_ROLLUP_SHARDS;
+    if (days > MAX_FEATURE_ROLLUP_DAYS || maximumRows > MAX_FEATURE_ROLLUP_ROWS) {
+        fail("REPORT_INCOMPLETE", `${dataset} exceeds its complete read budget`, {
+            dataset,
+            maximumDays: MAX_FEATURE_ROLLUP_DAYS,
+            maximumRows: MAX_FEATURE_ROLLUP_ROWS,
+        });
+    }
+}
+async function hasFeatureReadHeadroom(ctx) {
+    const metrics = await ctx.meta.getTransactionMetrics();
+    return (metrics.bytesRead.remaining >= FEATURE_REPORT_BYTE_RESERVE &&
+        metrics.documentsRead.remaining >= FEATURE_REPORT_DOCUMENT_RESERVE);
+}
+async function requireFeatureReadHeadroom(ctx, dataset) {
+    const metrics = await ctx.meta.getTransactionMetrics();
+    if (metrics.bytesRead.remaining < FEATURE_REPORT_BYTE_RESERVE ||
+        metrics.documentsRead.remaining < FEATURE_REPORT_DOCUMENT_RESERVE) {
+        fail("REPORT_INCOMPLETE", `${dataset} exceeds its byte or document budget`, {
+            dataset,
+            bytesRead: metrics.bytesRead.used,
+            documentsRead: metrics.documentsRead.used,
+        });
+    }
+}
+function failIfIncomplete(rows, maximum, dataset) {
+    if (rows.length > maximum) {
+        fail("REPORT_INCOMPLETE", `${dataset} exceeds the complete report read budget`, {
+            maximumRows: maximum,
+            dataset,
+        });
+    }
+}
+async function coverageForRange(ctx, siteIds, from, to, options) {
+    const controls = await Promise.all(siteIds.map((siteId) => ctx.db
+        .query("analyticsCoverage")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique()));
+    const sourceRanges = await Promise.all(siteIds.map((siteId) => sourceSetRange(ctx, siteId, options.datasets)));
+    const retainedBefore = controls.reduce((latest, control) => {
+        const before = control?.retained
+            .filter((item) => options.datasets.some((dataset) => dataset === item.dataset))
+            .reduce((latest, item) => Math.max(latest ?? 0, item.before), undefined);
+        return before === undefined ? latest : Math.max(latest ?? 0, before);
+    }, null);
+    const availableFrom = options.generation === 1
+        ? controls.length > 0 && controls.every(Boolean)
+            ? Math.max(...controls.map((control) => control.rollupCompleteFrom))
+            : null
+        : sourceRanges.length > 0 && sourceRanges.every(Boolean)
+            ? Math.max(...sourceRanges.map((range) => range.from))
+            : null;
+    const availableThrough = sourceRanges.length > 0 && sourceRanges.every(Boolean)
+        ? Math.min(...sourceRanges.map((range) => range.through))
+        : null;
+    let state;
+    if (retainedBefore !== null && to < retainedBefore) {
+        state = "unavailable";
+    }
+    else if (retainedBefore !== null && from < retainedBefore) {
+        state = "partial";
+    }
+    else if (options.complete ||
+        (availableFrom !== null &&
+            availableThrough !== null &&
+            from >= availableFrom &&
+            to <= availableThrough)) {
+        state = "complete";
+    }
+    else if (sourceRanges.some(Boolean)) {
+        state = "retained";
+    }
+    else {
+        state = "unavailable";
+    }
+    return {
+        state,
+        generation: options.generation ?? null,
+        availableFrom,
+        availableThrough,
+        retainedBefore,
+    };
+}
+async function assertSourceAvailable(ctx, siteIds, sources, from) {
+    const rows = await Promise.all(siteIds.map((siteId) => ctx.db
+        .query("analyticsCoverage")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique()));
+    const cutoff = rows.reduce((latest, row) => {
+        for (const retained of row?.retained ?? []) {
+            if (sources.some((source) => source === retained.dataset)) {
+                latest = Math.max(latest ?? 0, retained.before);
+            }
+        }
+        return latest;
+    }, null);
+    if (cutoff !== null && from < cutoff) {
+        fail("REPORT_INCOMPLETE", "requested report range intersects retained data", {
+            retainedBefore: cutoff,
+        });
+    }
+}
+async function datasetCoverageForRange(ctx, siteId, dataset, from, to) {
+    if (dataset === "siteMap") {
+        return siteMapDatasetCoverage(ctx, siteId, from, to);
+    }
+    // Vitals have no legacy fallback source; the rollup either covers the range
+    // or the coverage states say how much of it exists.
+    if (dataset === "vitals") {
+        return coverageForRange(ctx, [siteId], from, to, {
+            datasets: ["vitalRollups"],
+            complete: false,
+        });
+    }
+    const sourceOptions = dataset === "goals"
+        ? [["goalRollups"], ["goalCompletions"]]
+        : dataset === "funnels"
+            ? [["funnelRollups"], ["funnelSteps"]]
+            : dataset === "affiliates"
+                ? [["affiliateRollups"], ["affiliateStats", "sessions"]]
+                : [[dataset]];
+    const options = (await Promise.all(sourceOptions.map(async (sources) => {
+        const rollupDataset = sources[0];
+        const isRollup = rollupDataset.endsWith("Rollups");
+        const complete = isRollup
+            ? await dailyRollupsAvailable(ctx, siteId, from, to, rollupDataset)
+            : false;
+        if (isRollup && !complete)
+            return null;
+        return coverageForRange(ctx, [siteId], from, to, {
+            datasets: sources,
+            complete,
+            generation: complete ? 1 : null,
+        });
+    }))).filter((option) => option !== null);
+    return options.sort((left, right) => coverageRank(right.state) - coverageRank(left.state))[0];
+}
+/**
+ * The site map's coverage without reading its rows: it starts at the site's
+ * `siteMapSince` and ends where retention cut it. The report itself may still
+ * narrow the window when the range exceeds its row budget.
+ */
+async function siteMapDatasetCoverage(ctx, siteId, from, to) {
+    const control = await ctx.db
+        .query("analyticsCoverage")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    const since = control?.siteMapSince ?? null;
+    const retainedBefore = control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
+        ?.before ?? null;
+    const unavailable = since === null ||
+        to < since ||
+        (retainedBefore !== null && to < retainedBefore);
+    const state = unavailable
+        ? "unavailable"
+        : from < Math.max(since, retainedBefore ?? 0)
+            ? "partial"
+            : "complete";
+    return {
+        state,
+        generation: 1,
+        availableFrom: since,
+        availableThrough: since === null ? null : to,
+        retainedBefore,
+    };
+}
+async function sourceSetRange(ctx, siteId, sources) {
+    const ranges = (await Promise.all(sources.map((source) => sourceRange(ctx, siteId, source)))).filter((range) => range !== null);
+    if (ranges.length === 0)
+        return null;
+    return {
+        from: Math.min(...ranges.map((range) => range.from)),
+        through: Math.max(...ranges.map((range) => range.through)),
+    };
+}
+async function sourceRange(ctx, siteId, source) {
+    if (source === "overviewHour" || source === "overviewDay") {
+        const granularity = source === "overviewHour" ? "hour" : "day";
+        const [first, last] = await Promise.all([
+            ctx.db
+                .query("aggregateBuckets")
+                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("aggregateBuckets")
+                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity))
+                .order("desc")
+                .take(1),
+        ]);
+        return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
+    }
+    if (source === "events") {
+        const [first, last] = await Promise.all([
+            ctx.db
+                .query("eventBatches")
+                .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("eventBatches")
+                .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
+                .order("desc")
+                .take(1),
+        ]);
+        return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
+    }
+    if (source === "sessions") {
+        const [first, last] = await Promise.all([
+            ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_startedAt", (range) => range.eq("siteId", siteId))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("sessions")
+                .withIndex("by_siteId_and_lastSeenAt", (range) => range.eq("siteId", siteId))
+                .order("desc")
+                .take(1),
+        ]);
+        return bounds(first[0]?.startedAt, last[0]?.lastSeenAt);
+    }
+    if (source === "conversions") {
+        const [first, last] = await Promise.all([
+            ctx.db
+                .query("conversions")
+                .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("conversions")
+                .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
+                .order("desc")
+                .take(1),
+        ]);
+        return bounds(first[0]?.timestamp, last[0]?.timestamp);
+    }
+    if (source === "goalCompletions") {
+        const [first, last] = await Promise.all([
+            ctx.db
+                .query("goalCompletions")
+                .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("goalCompletions")
+                .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
+                .order("desc")
+                .take(1),
+        ]);
+        return bounds(first[0]?.timestamp, last[0]?.timestamp);
+    }
+    const table = source === "funnelSteps"
+        ? "funnelStepBuckets"
+        : source === "affiliateStats"
+            ? "affiliateDailyStats"
+            : source === "goalRollups"
+                ? "goalDailyRollups"
+                : source === "funnelRollups"
+                    ? "funnelDailyRollups"
+                    : source === "vitalRollups"
+                        ? "vitalRollups"
+                        : "affiliateDailyRollups";
+    const [first, last] = await Promise.all([
+        ctx.db
+            .query(table)
+            .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
+            .order("asc")
+            .take(1),
+        ctx.db
+            .query(table)
+            .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
+            .order("desc")
+            .take(1),
+    ]);
+    return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
+}
+function bounds(from, through) {
+    return from === undefined || through === undefined ? null : { from, through };
+}
+function coverageRank(state) {
+    if (state === "complete")
+        return 3;
+    if (state === "retained")
+        return 2;
+    if (state === "partial")
+        return 1;
+    return 0;
+}
 function emptyTotals() {
     return {
         events: 0,
@@ -524,25 +1587,43 @@ function addMetrics(target, source) {
 }
 function validateSiteIds(siteIds) {
     const unique = [...new Set(siteIds)];
-    if (unique.length < 1 || unique.length > MAX_REPORT_SITES || unique.length !== siteIds.length) {
+    if (unique.length < 1 ||
+        unique.length > MAX_REPORT_SITES ||
+        unique.length !== siteIds.length) {
         fail("INVALID_ARGUMENT", `siteIds must contain 1-${MAX_REPORT_SITES} unique IDs`);
     }
     return unique;
 }
 function validateRange(from, to, maxDays) {
-    if (!Number.isFinite(from) ||
-        !Number.isFinite(to) ||
+    if (!Number.isSafeInteger(from) ||
+        !Number.isSafeInteger(to) ||
         from < 0 ||
-        to <= from ||
-        to - from > maxDays * DAY_MS) {
-        fail("INVALID_ARGUMENT", `range must be positive, ordered, and at most ${maxDays} days`);
+        to < from ||
+        to - from >= maxDays * DAY_MS) {
+        fail("INVALID_ARGUMENT", `range must use inclusive integer milliseconds and span at most ${maxDays} UTC days`);
     }
-    return { from: Math.trunc(from), to: Math.trunc(to) };
+    return { from, to };
+}
+function isAlignedRange(from, to, intervalMs) {
+    return from % intervalMs === 0 && (to + 1) % intervalMs === 0;
 }
-function parseManualPagination(options) {
+function requireAlignedRange(from, to, intervalMs, report) {
+    if (!isAlignedRange(from, to, intervalMs)) {
+        fail("REPORT_INCOMPLETE", `${report} requires complete inclusive UTC bucket boundaries`, { from, to, intervalMs, timezone: "UTC" });
+    }
+}
+function rangeSemantics(from, to) {
+    return {
+        from,
+        to,
+        boundary: "inclusive",
+        timezone: "UTC",
+    };
+}
+function parseKeysetPagination(options, scope) {
     validatePagination(options);
     if (options.cursor === null) {
-        return { offset: 0, numItems: options.numItems };
+        return { numItems: options.numItems, scope, cursor: null };
     }
     let decoded;
     try {
@@ -552,31 +1633,120 @@ function parseManualPagination(options) {
         fail("INVALID_ARGUMENT", "pagination cursor is malformed");
     }
     if (!isPlainRecord(decoded) ||
-        decoded.v !== 1 ||
-        typeof decoded.o !== "number" ||
-        !Number.isSafeInteger(decoded.o) ||
-        decoded.o < 0 ||
-        decoded.o > MAX_MANUAL_PAGINATION_OFFSET) {
-        fail("INVALID_ARGUMENT", "pagination cursor is invalid or exceeds the offset limit");
-    }
-    return { offset: decoded.o, numItems: options.numItems };
+        decoded.v !== 2 ||
+        decoded.k !== scope.kind ||
+        decoded.s !== scope.siteId ||
+        decoded.n !== (scope.sessionId ?? null) ||
+        decoded.f !== scope.from ||
+        decoded.t !== scope.to ||
+        typeof decoded.x !== "number" ||
+        !Number.isFinite(decoded.x) ||
+        decoded.x < scope.from ||
+        decoded.x > scope.to ||
+        typeof decoded.c !== "number" ||
+        !Number.isFinite(decoded.c) ||
+        decoded.c < 0) {
+        fail("INVALID_ARGUMENT", "pagination cursor is invalid for this query");
+    }
+    return {
+        numItems: options.numItems,
+        scope,
+        cursor: {
+            timestamp: decoded.x,
+            creationTime: decoded.c,
+        },
+    };
 }
-function manualPaginationResult(rows, pagination) {
-    const page = rows.slice(pagination.offset, pagination.offset + pagination.numItems);
-    const nextOffset = pagination.offset + page.length;
-    const isDone = rows.length <= pagination.offset + pagination.numItems ||
-        nextOffset >= MAX_MANUAL_PAGINATION_OFFSET;
+function parseJourneyPagination(options, scope) {
+    validatePagination(options);
+    if (options.cursor === null) {
+        return { numItems: options.numItems, scope, cursor: null };
+    }
+    let decoded;
+    try {
+        decoded = JSON.parse(atob(options.cursor));
+    }
+    catch {
+        fail("INVALID_ARGUMENT", "pagination cursor is malformed");
+    }
+    if (!isPlainRecord(decoded) ||
+        decoded.v !== 4 ||
+        decoded.k !== scope.kind ||
+        decoded.s !== scope.siteId ||
+        decoded.n !== scope.sessionId ||
+        decoded.f !== scope.from ||
+        decoded.t !== scope.to ||
+        typeof decoded.x !== "number" ||
+        !Number.isFinite(decoded.x) ||
+        decoded.x < scope.from ||
+        decoded.x > scope.to ||
+        typeof decoded.c !== "number" ||
+        !Number.isFinite(decoded.c) ||
+        decoded.c < 0 ||
+        typeof decoded.i !== "string" ||
+        decoded.i.length === 0) {
+        fail("INVALID_ARGUMENT", "pagination cursor is invalid for this query");
+    }
+    return {
+        numItems: options.numItems,
+        scope,
+        cursor: {
+            timestamp: decoded.x,
+            creationTime: decoded.c,
+            id: decoded.i,
+        },
+    };
+}
+function keysetPaginationResult(rows, pagination, cursorFor) {
+    const page = rows.slice(0, pagination.numItems);
+    const isDone = rows.length <= pagination.numItems;
+    const next = page.length > 0 ? cursorFor(page[page.length - 1]) : pagination.cursor;
+    return {
+        page,
+        isDone,
+        continueCursor: btoa(JSON.stringify({
+            v: 2,
+            k: pagination.scope.kind,
+            s: pagination.scope.siteId,
+            n: pagination.scope.sessionId ?? null,
+            f: pagination.scope.from,
+            t: pagination.scope.to,
+            x: next?.timestamp ?? pagination.scope.to,
+            c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
+        })),
+    };
+}
+function journeyPaginationResult(rows, pagination) {
+    const page = rows.slice(0, pagination.numItems);
+    const isDone = rows.length <= pagination.numItems;
+    const last = page.at(-1);
+    const next = last
+        ? {
+            timestamp: last.timestamp,
+            creationTime: last._creationTime,
+            id: last._id,
+        }
+        : pagination.cursor;
     return {
         page,
         isDone,
         continueCursor: btoa(JSON.stringify({
-            v: 1,
-            o: Math.min(nextOffset, MAX_MANUAL_PAGINATION_OFFSET),
+            v: 4,
+            k: pagination.scope.kind,
+            s: pagination.scope.siteId,
+            n: pagination.scope.sessionId,
+            f: pagination.scope.from,
+            t: pagination.scope.to,
+            x: next?.timestamp ?? pagination.scope.to,
+            c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
+            i: next?.id ?? "~",
         })),
     };
 }
 function validatePagination(options) {
-    if (!Number.isSafeInteger(options.numItems) || options.numItems < 1 || options.numItems > 100) {
+    if (!Number.isSafeInteger(options.numItems) ||
+        options.numItems < 1 ||
+        options.numItems > 100) {
         fail("INVALID_ARGUMENT", "pagination numItems must be an integer between 1 and 100");
     }
     if (options.maximumRowsRead !== undefined &&