@iann29/rastro 0.1.0-alpha.4 → 0.1.0-alpha.6

package/dist/component/reports.js

@@ -1,12 +1,14 @@
 import { paginationOptsValidator, paginationResultValidator, } from "convex/server";
 import { v } from "convex/values";
 import { query } from "./_generated/server.js";
-import { AGGREGATE_SHARDS, DAY_MS, FEATURE_REPORT_BYTE_RESERVE, FEATURE_REPORT_DOCUMENT_RESERVE, HOUR_MS, HOURLY_AGGREGATE_SHARDS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_FEATURE_ROLLUP_DAYS, MAX_FEATURE_ROLLUP_ROWS, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_BUCKETS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, REPORT_ROLLUP_SHARDS, } from "./constants.js";
+import { AGGREGATE_SHARDS, DAY_MS, FEATURE_REPORT_BYTE_RESERVE, FEATURE_REPORT_DOCUMENT_RESERVE, HOUR_MS, HOURLY_AGGREGATE_SHARDS, MAX_AFFILIATES_PER_SITE, MAX_FUNNELS_PER_SITE, MAX_FEATURE_ROLLUP_DAYS, MAX_FEATURE_ROLLUP_ROWS, MAX_GOALS_PER_SITE, MAX_JOURNEY_EVENTS, MAX_LIVE_VISITORS, MAX_OVERVIEW_AGGREGATE_ROWS, MAX_REPORT_BUCKETS, MAX_REPORT_RANGE_DAYS, MAX_REPORT_SITES, MAX_VITALS_REPORT_ROWS, REPORT_ROLLUP_SHARDS, VITALS_REPORT_PAGE_LIMIT, } from "./constants.js";
 import { createVisitorSketch, estimateVisitors, unionVisitorSketches, } from "./cardinality.js";
 import { fail } from "./errors.js";
 import { compareJourneyEvents, hydrateEventContexts, loadSessionJourneyEvents, loadVisitorJourneyEvents, } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
-import { affiliateFieldsValidator, eventFieldsValidator, funnelFieldsValidator, goalFieldsValidator, sessionFieldsValidator, } from "./validators.js";
+import { affiliateFieldsValidator, eventFieldsValidator, funnelFieldsValidator, goalFieldsValidator, sessionFieldsValidator, vitalMetricValidator, } from "./validators.js";
+import { resolveVisitorIdentities } from "./visitors.js";
+import { mergeVitalHistograms, VITAL_ALL, VITAL_METRICS, vitalHistogramPercentile, vitalRatingCounts, } from "./vitals.js";
 const sessionDocumentValidator = sessionFieldsValidator.extend({
     _id: v.id("sessions"),
     _creationTime: v.number(),
@@ -66,7 +68,9 @@ const metricTotalsValidator = v.object({
     conversions: v.number(),
     revenueCents: v.number(),
 });
-const timePointValidator = metricTotalsValidator.extend({ timestamp: v.number() });
+const timePointValidator = metricTotalsValidator.extend({
+    timestamp: v.number(),
+});
 const topItemValidator = v.object({
     value: v.string(),
     count: v.number(),
@@ -106,6 +110,7 @@ export const overview = query({
         totals: metricTotalsValidator,
         timeSeries: v.array(timePointValidator),
         topSources: v.array(topItemValidator),
+        topCampaigns: v.array(topItemValidator),
         topPages: v.array(topItemValidator),
         topCountries: v.array(topItemValidator),
         topDevices: v.array(topItemValidator),
@@ -122,6 +127,7 @@ export const overview = query({
             }),
             breakdowns: v.object({
                 topSources: v.literal("eventVolume"),
+                topCampaigns: v.literal("eventVolume"),
                 topPages: v.literal("eventVolume"),
                 topCountries: v.literal("eventVolume"),
                 topDevices: v.literal("eventVolume"),
@@ -135,7 +141,7 @@ export const overview = query({
                 sessionReplay: v.literal("unsupported"),
                 errorInsights: v.literal("unsupported"),
                 outboundLinks: v.literal("eventVolume"),
-                botDetection: v.literal("unsupported"),
+                botDetection: v.literal("userAgent"),
             }),
         }),
     }),
@@ -242,6 +248,7 @@ export const overview = query({
                 .sort(([left], [right]) => left - right)
                 .map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
             topSources: top("source"),
+            topCampaigns: top("campaign"),
             topPages: top("page"),
             topCountries: top("country"),
             topDevices: top("device"),
@@ -258,6 +265,7 @@ export const overview = query({
                 },
                 breakdowns: {
                     topSources: "eventVolume",
+                    topCampaigns: "eventVolume",
                     topPages: "eventVolume",
                     topCountries: "eventVolume",
                     topDevices: "eventVolume",
@@ -271,7 +279,7 @@ export const overview = query({
                     sessionReplay: "unsupported",
                     errorInsights: "unsupported",
                     outboundLinks: "eventVolume",
-                    botDetection: "unsupported",
+                    botDetection: "userAgent",
                 },
             },
         };
@@ -287,7 +295,9 @@ export const liveVisitors = query({
     handler: async (ctx, args) => {
         const siteIds = validateSiteIds(args.siteIds);
         const limit = args.limit ?? 100;
-        if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIVE_VISITORS) {
+        if (!Number.isSafeInteger(limit) ||
+            limit < 1 ||
+            limit > MAX_LIVE_VISITORS) {
             fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_LIVE_VISITORS}`);
         }
         if (!Number.isFinite(args.now) || args.now < 0) {
@@ -483,24 +493,26 @@ export const visitorJourney = query({
         validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
         await assertSourceAvailable(ctx, siteIds, ["events"], args.from);
         const limit = args.limit ?? 200;
-        if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
+        if (!Number.isSafeInteger(limit) ||
+            limit < 1 ||
+            limit > MAX_JOURNEY_EVENTS) {
             fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
         }
         const rows = [];
         for (const siteId of siteIds) {
-            const cutoff = rows.length >= limit
-                ? rows[limit - 1].timestamp
-                : args.to;
-            rows.push(...await loadVisitorJourneyEvents(ctx, {
-                siteId,
-                visitorId: args.visitorId,
-                from: args.from,
-                to: cutoff,
-                resultLimit: limit,
-            }));
-            rows.sort(compareJourneyEvents);
-            if (rows.length > limit)
-                rows.length = limit;
+            for (const visitorId of await resolveVisitorIdentities(ctx, siteId, args.visitorId)) {
+                const cutoff = rows.length >= limit ? rows[limit - 1].timestamp : args.to;
+                rows.push(...(await loadVisitorJourneyEvents(ctx, {
+                    siteId,
+                    visitorId,
+                    from: args.from,
+                    to: cutoff,
+                    resultLimit: limit,
+                })));
+                rows.sort(compareJourneyEvents);
+                if (rows.length > limit)
+                    rows.length = limit;
+            }
         }
         return hydrateEventContexts(ctx, rows);
     },
@@ -579,7 +591,12 @@ export const goalsReport = query({
 const funnelReportItemValidator = funnelFieldsValidator.extend({
     _id: v.id("funnels"),
     _creationTime: v.number(),
-    stepsReport: v.array(v.object({ step: v.number(), name: v.string(), entrants: v.number(), completions: v.number() })),
+    stepsReport: v.array(v.object({
+        step: v.number(),
+        name: v.string(),
+        entrants: v.number(),
+        completions: v.number(),
+    })),
     truncated: v.boolean(),
     metadata: reportMetadataValidator,
 });
@@ -642,7 +659,9 @@ export const funnelsReport = query({
                 stepsReport,
                 truncated: false,
                 metadata: {
-                    basis: rollups ? "dailyRollup" : "boundedLegacy",
+                    basis: rollups
+                        ? "dailyRollup"
+                        : "boundedLegacy",
                     coverage: reportCoverage,
                     range: rangeSemantics(args.from, args.to),
                 },
@@ -768,7 +787,7 @@ export const affiliatesReport = query({
                     visits: traffic?.visits ?? 0,
                     uniqueVisitors: rollups
                         ? estimateVisitors(traffic?.sketch ?? createVisitorSketch())
-                        : traffic?.visitors.size ?? 0,
+                        : (traffic?.visitors.size ?? 0),
                 };
             }),
             truncated: false,
@@ -783,14 +802,128 @@ export const affiliatesReport = query({
                 }),
                 range: rangeSemantics(args.from, args.to),
                 uniqueVisitors: {
-                    basis: rollups ? "estimatedUnique" : "exactLegacy",
+                    basis: rollups
+                        ? "estimatedUnique"
+                        : "exactLegacy",
                     exact: !rollups,
                 },
             },
         };
     },
 });
-const coverageDatasetValidator = v.union(v.literal("overviewHour"), v.literal("overviewDay"), v.literal("events"), v.literal("sessions"), v.literal("conversions"), v.literal("goals"), v.literal("funnels"), v.literal("affiliates"));
+const vitalStatsValidator = v.object({
+    count: v.number(),
+    p75: v.number(),
+    average: v.number(),
+    good: v.number(),
+    needsImprovement: v.number(),
+    poor: v.number(),
+});
+export const vitalsReport = query({
+    args: { siteId: v.id("sites"), from: v.number(), to: v.number() },
+    returns: v.object({
+        metrics: v.array(vitalStatsValidator.extend({
+            metric: vitalMetricValidator,
+            devices: v.array(vitalStatsValidator.extend({ device: v.string() })),
+            pages: v.array(vitalStatsValidator.extend({ page: v.string() })),
+            timeSeries: v.array(v.object({
+                timestamp: v.number(),
+                count: v.number(),
+                p75: v.number(),
+            })),
+        })),
+        truncated: v.boolean(),
+        metadata: reportMetadataValidator,
+    }),
+    handler: async (ctx, args) => {
+        validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
+        requireAlignedRange(args.from, args.to, DAY_MS, "vitals report");
+        await assertSourceAvailable(ctx, [args.siteId], ["vitalRollups"], args.from);
+        const rows = await ctx.db
+            .query("vitalRollups")
+            .withIndex("by_siteId_and_bucketStart", (range) => range
+            .eq("siteId", args.siteId)
+            .gte("bucketStart", args.from)
+            .lte("bucketStart", args.to))
+            .take(MAX_VITALS_REPORT_ROWS + 1);
+        failIfIncomplete(rows, MAX_VITALS_REPORT_ROWS, "vital rollups");
+        const fold = (groups, metric, key, entry) => {
+            const group = groups.get(metric) ?? new Map();
+            const current = group.get(key) ?? { count: 0, sum: 0, histogram: [] };
+            current.count += entry.count;
+            current.sum += entry.sum;
+            current.histogram = mergeVitalHistograms(current.histogram, entry.histogram);
+            group.set(key, current);
+            groups.set(metric, group);
+        };
+        const byDevice = new Map();
+        const byPage = new Map();
+        const byDay = new Map();
+        for (const row of rows) {
+            for (const entry of row.metrics) {
+                if (row.page === VITAL_ALL) {
+                    // A device row carries the complete signal once, so overall totals
+                    // and the time series derive from this family alone.
+                    fold(byDevice, entry.metric, row.device, entry);
+                    fold(byDay, entry.metric, row.bucketStart, entry);
+                }
+                else {
+                    fold(byPage, entry.metric, row.page, entry);
+                }
+            }
+        }
+        const stats = (metric, accumulated) => ({
+            count: accumulated.count,
+            p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
+            average: accumulated.count > 0
+                ? Math.round(accumulated.sum / accumulated.count)
+                : 0,
+            ...vitalRatingCounts(accumulated.histogram, metric),
+        });
+        const metrics = VITAL_METRICS.map((metric) => {
+            const overall = { count: 0, sum: 0, histogram: [] };
+            for (const accumulated of byDevice.get(metric)?.values() ?? []) {
+                overall.count += accumulated.count;
+                overall.sum += accumulated.sum;
+                overall.histogram = mergeVitalHistograms(overall.histogram, accumulated.histogram);
+            }
+            return {
+                metric,
+                ...stats(metric, overall),
+                devices: [...(byDevice.get(metric) ?? [])]
+                    .map(([device, accumulated]) => ({
+                    device,
+                    ...stats(metric, accumulated),
+                }))
+                    .sort((left, right) => right.count - left.count),
+                pages: [...(byPage.get(metric) ?? [])]
+                    .map(([page, accumulated]) => ({
+                    page,
+                    ...stats(metric, accumulated),
+                }))
+                    .sort((left, right) => right.count - left.count || left.page.localeCompare(right.page))
+                    .slice(0, VITALS_REPORT_PAGE_LIMIT),
+                timeSeries: [...(byDay.get(metric) ?? [])]
+                    .sort(([left], [right]) => left - right)
+                    .map(([timestamp, accumulated]) => ({
+                    timestamp,
+                    count: accumulated.count,
+                    p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
+                })),
+            };
+        });
+        return {
+            metrics,
+            truncated: false,
+            metadata: {
+                basis: "dailyRollup",
+                coverage: await coverageForRange(ctx, [args.siteId], args.from, args.to, { datasets: ["vitalRollups"], complete: true }),
+                range: rangeSemantics(args.from, args.to),
+            },
+        };
+    },
+});
+const coverageDatasetValidator = v.union(v.literal("overviewHour"), v.literal("overviewDay"), v.literal("events"), v.literal("sessions"), v.literal("conversions"), v.literal("goals"), v.literal("funnels"), v.literal("affiliates"), v.literal("vitals"));
 export const dataCoverage = query({
     args: {
         siteId: v.id("sites"),
@@ -817,6 +950,7 @@ export const dataCoverage = query({
             "goals",
             "funnels",
             "affiliates",
+            "vitals",
         ];
         return {
             siteId: args.siteId,
@@ -960,10 +1094,11 @@ async function coverageForRange(ctx, siteIds, from, to, options) {
     else if (retainedBefore !== null && from < retainedBefore) {
         state = "partial";
     }
-    else if (options.complete || (availableFrom !== null &&
-        availableThrough !== null &&
-        from >= availableFrom &&
-        to <= availableThrough)) {
+    else if (options.complete ||
+        (availableFrom !== null &&
+            availableThrough !== null &&
+            from >= availableFrom &&
+            to <= availableThrough)) {
         state = "complete";
     }
     else if (sourceRanges.some(Boolean)) {
@@ -1000,6 +1135,14 @@ async function assertSourceAvailable(ctx, siteIds, sources, from) {
     }
 }
 async function datasetCoverageForRange(ctx, siteId, dataset, from, to) {
+    // Vitals have no legacy fallback source; the rollup either covers the range
+    // or the coverage states say how much of it exists.
+    if (dataset === "vitals") {
+        return coverageForRange(ctx, [siteId], from, to, {
+            datasets: ["vitalRollups"],
+            complete: false,
+        });
+    }
     const sourceOptions = dataset === "goals"
         ? [["goalRollups"], ["goalCompletions"]]
         : dataset === "funnels"
@@ -1036,54 +1179,76 @@ async function sourceRange(ctx, siteId, source) {
     if (source === "overviewHour" || source === "overviewDay") {
         const granularity = source === "overviewHour" ? "hour" : "day";
         const [first, last] = await Promise.all([
-            ctx.db.query("aggregateBuckets")
-                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity)).order("asc").take(1),
-            ctx.db.query("aggregateBuckets")
-                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity)).order("desc").take(1),
+            ctx.db
+                .query("aggregateBuckets")
+                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity))
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("aggregateBuckets")
+                .withIndex("by_siteId_and_granularity_and_bucketStart", (range) => range.eq("siteId", siteId).eq("granularity", granularity))
+                .order("desc")
+                .take(1),
         ]);
         return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
     }
     if (source === "events") {
         const [first, last] = await Promise.all([
-            ctx.db.query("eventBatches")
+            ctx.db
+                .query("eventBatches")
                 .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
-                .order("asc").take(1),
-            ctx.db.query("eventBatches")
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("eventBatches")
                 .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
-                .order("desc").take(1),
+                .order("desc")
+                .take(1),
         ]);
         return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
     }
     if (source === "sessions") {
         const [first, last] = await Promise.all([
-            ctx.db.query("sessions")
+            ctx.db
+                .query("sessions")
                 .withIndex("by_siteId_and_startedAt", (range) => range.eq("siteId", siteId))
-                .order("asc").take(1),
-            ctx.db.query("sessions")
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("sessions")
                 .withIndex("by_siteId_and_lastSeenAt", (range) => range.eq("siteId", siteId))
-                .order("desc").take(1),
+                .order("desc")
+                .take(1),
         ]);
         return bounds(first[0]?.startedAt, last[0]?.lastSeenAt);
     }
     if (source === "conversions") {
         const [first, last] = await Promise.all([
-            ctx.db.query("conversions")
+            ctx.db
+                .query("conversions")
                 .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
-                .order("asc").take(1),
-            ctx.db.query("conversions")
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("conversions")
                 .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
-                .order("desc").take(1),
+                .order("desc")
+                .take(1),
         ]);
         return bounds(first[0]?.timestamp, last[0]?.timestamp);
     }
     if (source === "goalCompletions") {
         const [first, last] = await Promise.all([
-            ctx.db.query("goalCompletions")
+            ctx.db
+                .query("goalCompletions")
                 .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
-                .order("asc").take(1),
-            ctx.db.query("goalCompletions")
+                .order("asc")
+                .take(1),
+            ctx.db
+                .query("goalCompletions")
                 .withIndex("by_siteId_and_timestamp", (range) => range.eq("siteId", siteId))
-                .order("desc").take(1),
+                .order("desc")
+                .take(1),
         ]);
         return bounds(first[0]?.timestamp, last[0]?.timestamp);
     }
@@ -1095,14 +1260,20 @@ async function sourceRange(ctx, siteId, source) {
                 ? "goalDailyRollups"
                 : source === "funnelRollups"
                     ? "funnelDailyRollups"
-                    : "affiliateDailyRollups";
+                    : source === "vitalRollups"
+                        ? "vitalRollups"
+                        : "affiliateDailyRollups";
     const [first, last] = await Promise.all([
-        ctx.db.query(table)
+        ctx.db
+            .query(table)
             .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
-            .order("asc").take(1),
-        ctx.db.query(table)
+            .order("asc")
+            .take(1),
+        ctx.db
+            .query(table)
             .withIndex("by_siteId_and_bucketStart", (range) => range.eq("siteId", siteId))
-            .order("desc").take(1),
+            .order("desc")
+            .take(1),
     ]);
     return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
 }
@@ -1144,7 +1315,9 @@ function addMetrics(target, source) {
 }
 function validateSiteIds(siteIds) {
     const unique = [...new Set(siteIds)];
-    if (unique.length < 1 || unique.length > MAX_REPORT_SITES || unique.length !== siteIds.length) {
+    if (unique.length < 1 ||
+        unique.length > MAX_REPORT_SITES ||
+        unique.length !== siteIds.length) {
         fail("INVALID_ARGUMENT", `siteIds must contain 1-${MAX_REPORT_SITES} unique IDs`);
     }
     return unique;
@@ -1255,9 +1428,7 @@ function parseJourneyPagination(options, scope) {
 function keysetPaginationResult(rows, pagination, cursorFor) {
     const page = rows.slice(0, pagination.numItems);
     const isDone = rows.length <= pagination.numItems;
-    const next = page.length > 0
-        ? cursorFor(page[page.length - 1])
-        : pagination.cursor;
+    const next = page.length > 0 ? cursorFor(page[page.length - 1]) : pagination.cursor;
     return {
         page,
         isDone,
@@ -1301,7 +1472,9 @@ function journeyPaginationResult(rows, pagination) {
     };
 }
 function validatePagination(options) {
-    if (!Number.isSafeInteger(options.numItems) || options.numItems < 1 || options.numItems > 100) {
+    if (!Number.isSafeInteger(options.numItems) ||
+        options.numItems < 1 ||
+        options.numItems > 100) {
         fail("INVALID_ARGUMENT", "pagination numItems must be an integer between 1 and 100");
     }
     if (options.maximumRowsRead !== undefined &&