@iann29/rastro 0.1.0-alpha.7 → 0.1.0-alpha.8

package/dist/component/reports.js

@@ -1,7 +1,7 @@
 import { paginationOptsValidator, paginationResultValidator, } from "convex/server";
 import { v } from "convex/values";
 import { query } from "./_generated/server.js";
-import { AGGREGATE_SHARDS, DAY_MS, FEATURE_REPORT_BYTE_RESERVE, FEATURE_REPORT_DOCUMENT_RESERVE, HOUR_MS, HOURLY_AGGREGATE_SHARDS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_FEATURE_ROLLUP_DAYS, MAX_FEATURE_ROLLUP_ROWS, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_BUCKETS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, MAX_VITALS_REPORT_ROWS, REPORT_ROLLUP_SHARDS, VITALS_REPORT_PAGE_LIMIT, } from "./constants.js";
+import { AGGREGATE_SHARDS, DAY_MS, FEATURE_REPORT_BYTE_RESERVE, FEATURE_REPORT_DOCUMENT_RESERVE, HOUR_MS, HOURLY_AGGREGATE_SHARDS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_FEATURE_ROLLUP_DAYS, MAX_FEATURE_ROLLUP_ROWS, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_BUCKETS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, MAX_SITE_MAP_ROUTES_PER_DAY, MAX_SITE_MAP_ROWS, MAX_VITALS_REPORT_ROWS, REPORT_ROLLUP_SHARDS, ROUTE_TRANSITION_SLOTS, SITE_MAP_OTHER_ROUTE, VITALS_REPORT_PAGE_LIMIT, } from "./constants.js";
 import { createVisitorSketch, estimateVisitors, unionVisitorSketches, } from "./cardinality.js";
 import { fail } from "./errors.js";
 import { compareJourneyEvents, hydrateEventContexts, loadSessionJourneyEvents, loadVisitorJourneyEvents, } from "./eventStore.js";
@@ -37,6 +37,8 @@ const liveSessionDocumentValidator = v.object({
     device: v.string(),
     pageviewCount: v.number(),
     eventCount: v.number(),
+    previousPath: v.optional(v.string()),
+    movedAt: v.optional(v.number()),
 });
 const conversionDocumentValidator = v.object({
     _id: v.id("conversions"),
@@ -334,9 +336,241 @@ export const liveVisitors = query({
             device: row.device,
             pageviewCount: row.pageviewCount,
             eventCount: row.eventCount,
+            ...(row.previousPath ? { previousPath: row.previousPath } : {}),
+            ...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
         }));
     },
 });
+const siteMapRouteValidator = v.object({
+    route: v.string(),
+    pageviews: v.number(),
+    visitors: v.number(),
+    entries: v.number(),
+    exits: v.number(),
+    bounces: v.number(),
+    transitionsIn: v.number(),
+    transitionsOut: v.number(),
+    dwellMs: v.number(),
+    dwellSamples: v.number(),
+    daily: v.array(v.object({ bucketStart: v.number(), pageviews: v.number() })),
+});
+const siteMapTransitionValidator = v.object({
+    from: v.string(),
+    to: v.string(),
+    count: v.number(),
+});
+const siteMapCoverageDataset = "siteMapRollups";
+/**
+ * Every route the sites served in the range and the transitions between
+ * them, read from the daily route rollups newest day first. Exits and
+ * bounces are derived rather than counted: a session enters once, leaves
+ * once, and every later pageview is a transition, so per route
+ * `exits = entries + transitionsIn - transitionsOut` and
+ * `bounces = entries - continued` hold without a session-end hook. The
+ * report never falls back to raw telemetry; days it could not cover are
+ * declared through `coverage` and `window`.
+ */
+export const siteMap = query({
+    args: {
+        siteIds: v.array(v.id("sites")),
+        from: v.number(),
+        to: v.number(),
+    },
+    returns: v.object({
+        from: v.number(),
+        to: v.number(),
+        window: v.union(v.object({ from: v.number(), to: v.number() }), v.null()),
+        routes: v.array(siteMapRouteValidator),
+        transitions: v.array(siteMapTransitionValidator),
+        truncated: v.object({ routes: v.boolean(), transitions: v.boolean() }),
+        metadata: reportMetadataValidator.extend({
+            visitors: v.object({
+                basis: v.literal("estimatedUnique"),
+                exact: v.boolean(),
+            }),
+            limits: v.object({
+                routesPerDay: v.number(),
+                transitionSlots: v.number(),
+                rows: v.number(),
+            }),
+        }),
+    }),
+    handler: async (ctx, args) => {
+        const siteIds = validateSiteIds(args.siteIds);
+        validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
+        requireAlignedRange(args.from, args.to, DAY_MS, "site map");
+        const controls = await Promise.all(siteIds.map((siteId) => ctx.db
+            .query("analyticsCoverage")
+            .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+            .unique()));
+        const since = controls.every((control) => control?.siteMapSince !== undefined)
+            ? Math.max(...controls.map((control) => control.siteMapSince))
+            : null;
+        const retainedBefore = controls.reduce((latest, control) => {
+            const before = control?.retained.find((item) => item.dataset === siteMapCoverageDataset)?.before;
+            return before === undefined ? latest : Math.max(latest ?? 0, before);
+        }, null);
+        const metadata = (state) => ({
+            basis: "dailyRollup",
+            coverage: {
+                state,
+                generation: 1,
+                availableFrom: since,
+                availableThrough: since === null ? null : args.to,
+                retainedBefore,
+            },
+            range: rangeSemantics(args.from, args.to),
+            visitors: { basis: "estimatedUnique", exact: false },
+            limits: {
+                routesPerDay: MAX_SITE_MAP_ROUTES_PER_DAY,
+                transitionSlots: ROUTE_TRANSITION_SLOTS,
+                rows: MAX_SITE_MAP_ROWS,
+            },
+        });
+        if (since === null ||
+            args.to < since ||
+            (retainedBefore !== null && args.to < retainedBefore)) {
+            return {
+                from: args.from,
+                to: args.to,
+                window: null,
+                routes: [],
+                transitions: [],
+                truncated: { routes: false, transitions: false },
+                metadata: metadata("unavailable"),
+            };
+        }
+        const readFrom = Math.max(args.from, Math.floor(since / DAY_MS) * DAY_MS, retainedBefore ?? 0);
+        let budget = MAX_SITE_MAP_ROWS;
+        let completeFrom = readFrom;
+        const rows = [];
+        for (const siteId of siteIds) {
+            const read = await readRouteRollups(ctx, siteId, readFrom, args.to, budget);
+            budget -= read.rows.length;
+            completeFrom = Math.max(completeFrom, read.completeFrom);
+            rows.push(...read.rows);
+        }
+        const windowFrom = Math.max(args.from, completeFrom, since, retainedBefore ?? 0);
+        const routes = new Map();
+        const totalsFor = (route) => {
+            let current = routes.get(route);
+            if (!current) {
+                current = {
+                    pageviews: 0,
+                    entries: 0,
+                    continued: 0,
+                    dwellMs: 0,
+                    dwellSamples: 0,
+                    transitionsIn: 0,
+                    transitionsOut: 0,
+                    sketch: createVisitorSketch(),
+                    daily: new Map(),
+                };
+                routes.set(route, current);
+            }
+            return current;
+        };
+        const edges = new Map();
+        for (const row of rows) {
+            if (row.bucketStart < completeFrom)
+                continue;
+            const current = totalsFor(row.route);
+            current.pageviews += row.pageviews;
+            current.entries += row.entries;
+            current.continued += row.continued;
+            current.dwellMs += row.dwellMs;
+            current.dwellSamples += row.dwellSamples;
+            current.sketch = unionVisitorSketches(current.sketch, row.visitorSketch);
+            current.daily.set(row.bucketStart, (current.daily.get(row.bucketStart) ?? 0) + row.pageviews);
+            for (const target of row.outgoing) {
+                const key = `${row.route}\u0000${target.route}`;
+                const edge = edges.get(key) ?? {
+                    from: row.route,
+                    to: target.route,
+                    count: 0,
+                };
+                edge.count += target.count;
+                edges.set(key, edge);
+                current.transitionsOut += target.count;
+            }
+        }
+        // A target without a row of its own in this window — folded past the
+        // daily cap, or already retained — counts under "(other)".
+        const transitions = new Map();
+        for (const edge of edges.values()) {
+            const to = routes.has(edge.to) ? edge.to : SITE_MAP_OTHER_ROUTE;
+            const key = `${edge.from}\u0000${to}`;
+            const merged = transitions.get(key) ?? { from: edge.from, to, count: 0 };
+            merged.count += edge.count;
+            transitions.set(key, merged);
+            totalsFor(to).transitionsIn += edge.count;
+        }
+        const routeRows = [...routes.entries()]
+            .map(([route, totals]) => ({
+            route,
+            pageviews: totals.pageviews,
+            visitors: estimateVisitors(totals.sketch),
+            entries: totals.entries,
+            exits: Math.max(0, totals.entries + totals.transitionsIn - totals.transitionsOut),
+            bounces: Math.max(0, totals.entries - totals.continued),
+            transitionsIn: totals.transitionsIn,
+            transitionsOut: totals.transitionsOut,
+            dwellMs: totals.dwellMs,
+            dwellSamples: totals.dwellSamples,
+            daily: [...totals.daily.entries()]
+                .sort(([left], [right]) => left - right)
+                .map(([bucketStart, pageviews]) => ({ bucketStart, pageviews })),
+        }))
+            .sort((left, right) => right.pageviews - left.pageviews ||
+            left.route.localeCompare(right.route));
+        const transitionRows = [...transitions.values()].sort((left, right) => right.count - left.count ||
+            left.from.localeCompare(right.from) ||
+            left.to.localeCompare(right.to));
+        return {
+            from: args.from,
+            to: args.to,
+            window: completeFrom > args.to ? null : { from: windowFrom, to: args.to },
+            routes: routeRows,
+            transitions: transitionRows,
+            truncated: {
+                routes: routes.has(SITE_MAP_OTHER_ROUTE),
+                transitions: transitionRows.some((transition) => transition.from === SITE_MAP_OTHER_ROUTE ||
+                    transition.to === SITE_MAP_OTHER_ROUTE),
+            },
+            metadata: metadata(windowFrom > args.from ? "partial" : "complete"),
+        };
+    },
+});
+/**
+ * Reads a site's route rollups newest day first until the row budget or the
+ * transaction's read headroom runs out, then drops the day it could only
+ * read in part so every day it returns is complete.
+ */
+async function readRouteRollups(ctx, siteId, from, to, budget) {
+    const rows = [];
+    let cutAt = null;
+    const query = ctx.db
+        .query("routeDailyRollups")
+        .withIndex("by_siteId_and_bucketStart", (range) => range
+        .eq("siteId", siteId)
+        .gte("bucketStart", from)
+        .lte("bucketStart", to))
+        .order("desc");
+    for await (const row of query) {
+        if (rows.length >= budget ||
+            (rows.length % 256 === 255 && !(await hasFeatureReadHeadroom(ctx)))) {
+            cutAt = row.bucketStart;
+            break;
+        }
+        rows.push(row);
+    }
+    if (cutAt === null)
+        return { rows, completeFrom: from };
+    while (rows.length > 0 && rows[rows.length - 1].bucketStart === cutAt) {
+        rows.pop();
+    }
+    return { rows, completeFrom: cutAt + DAY_MS };
+}
 export const listSessions = query({
     args: {
         siteId: v.id("sites"),
@@ -386,7 +620,7 @@ export const listSessions = query({
                 .order("desc")
                 .take(take)));
         }
-        const publicRows = rows.map(({ geoLookupAttemptedAt: _attemptedAt, ...row }) => row);
+        const publicRows = rows.map(({ geoLookupAttemptedAt: _attemptedAt, lastPageviewAt: _lastPageviewAt, ...row }) => row);
         return keysetPaginationResult(publicRows, pagination, (row) => ({
             timestamp: row.lastSeenAt,
             creationTime: row._creationTime,
@@ -923,7 +1157,7 @@ export const vitalsReport = query({
         };
     },
 });
-const coverageDatasetValidator = v.union(v.literal("overviewHour"), v.literal("overviewDay"), v.literal("events"), v.literal("sessions"), v.literal("conversions"), v.literal("goals"), v.literal("funnels"), v.literal("affiliates"), v.literal("vitals"));
+const coverageDatasetValidator = v.union(v.literal("overviewHour"), v.literal("overviewDay"), v.literal("events"), v.literal("sessions"), v.literal("conversions"), v.literal("goals"), v.literal("funnels"), v.literal("affiliates"), v.literal("vitals"), v.literal("siteMap"));
 export const dataCoverage = query({
     args: {
         siteId: v.id("sites"),
@@ -951,6 +1185,7 @@ export const dataCoverage = query({
             "funnels",
             "affiliates",
             "vitals",
+            "siteMap",
         ];
         return {
             siteId: args.siteId,
@@ -1046,6 +1281,11 @@ function requireFeatureRollupBudget(entities, days, dataset) {
         });
     }
 }
+async function hasFeatureReadHeadroom(ctx) {
+    const metrics = await ctx.meta.getTransactionMetrics();
+    return (metrics.bytesRead.remaining >= FEATURE_REPORT_BYTE_RESERVE &&
+        metrics.documentsRead.remaining >= FEATURE_REPORT_DOCUMENT_RESERVE);
+}
 async function requireFeatureReadHeadroom(ctx, dataset) {
     const metrics = await ctx.meta.getTransactionMetrics();
     if (metrics.bytesRead.remaining < FEATURE_REPORT_BYTE_RESERVE ||
@@ -1135,6 +1375,9 @@ async function assertSourceAvailable(ctx, siteIds, sources, from) {
     }
 }
 async function datasetCoverageForRange(ctx, siteId, dataset, from, to) {
+    if (dataset === "siteMap") {
+        return siteMapDatasetCoverage(ctx, siteId, from, to);
+    }
     // Vitals have no legacy fallback source; the rollup either covers the range
     // or the coverage states say how much of it exists.
     if (dataset === "vitals") {
@@ -1166,6 +1409,35 @@ async function datasetCoverageForRange(ctx, siteId, dataset, from, to) {
     }))).filter((option) => option !== null);
     return options.sort((left, right) => coverageRank(right.state) - coverageRank(left.state))[0];
 }
+/**
+ * The site map's coverage without reading its rows: it starts at the site's
+ * `siteMapSince` and ends where retention cut it. The report itself may still
+ * narrow the window when the range exceeds its row budget.
+ */
+async function siteMapDatasetCoverage(ctx, siteId, from, to) {
+    const control = await ctx.db
+        .query("analyticsCoverage")
+        .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
+        .unique();
+    const since = control?.siteMapSince ?? null;
+    const retainedBefore = control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
+        ?.before ?? null;
+    const unavailable = since === null ||
+        to < since ||
+        (retainedBefore !== null && to < retainedBefore);
+    const state = unavailable
+        ? "unavailable"
+        : from < Math.max(since, retainedBefore ?? 0)
+            ? "partial"
+            : "complete";
+    return {
+        state,
+        generation: 1,
+        availableFrom: since,
+        availableThrough: since === null ? null : to,
+        retainedBefore,
+    };
+}
 async function sourceSetRange(ctx, siteId, sources) {
     const ranges = (await Promise.all(sources.map((source) => sourceRange(ctx, siteId, source)))).filter((range) => range !== null);
     if (ranges.length === 0)