@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (179) hide show
  1. package/README.md +693 -69
  2. package/agent/integration.md +801 -0
  3. package/agent/manifest.json +205 -0
  4. package/agent/manifest.schema.json +444 -0
  5. package/dist/client/federation.d.ts +381 -0
  6. package/dist/client/federation.d.ts.map +1 -0
  7. package/dist/client/federation.js +274 -0
  8. package/dist/client/federation.js.map +1 -0
  9. package/dist/client/index.d.ts +2673 -15
  10. package/dist/client/index.d.ts.map +1 -1
  11. package/dist/client/index.js +565 -33
  12. package/dist/client/index.js.map +1 -1
  13. package/dist/component/_generated/api.d.ts +22 -0
  14. package/dist/component/_generated/api.d.ts.map +1 -1
  15. package/dist/component/_generated/api.js.map +1 -1
  16. package/dist/component/_generated/component.d.ts +315 -4
  17. package/dist/component/_generated/component.d.ts.map +1 -1
  18. package/dist/component/_generated/server.d.ts +4 -0
  19. package/dist/component/_generated/server.d.ts.map +1 -1
  20. package/dist/component/_generated/server.js.map +1 -1
  21. package/dist/component/affiliates.d.ts.map +1 -1
  22. package/dist/component/affiliates.js +6 -2
  23. package/dist/component/affiliates.js.map +1 -1
  24. package/dist/component/cardinality.d.ts +12 -0
  25. package/dist/component/cardinality.d.ts.map +1 -0
  26. package/dist/component/cardinality.js +94 -0
  27. package/dist/component/cardinality.js.map +1 -0
  28. package/dist/component/constants.d.ts +28 -1
  29. package/dist/component/constants.d.ts.map +1 -1
  30. package/dist/component/constants.js +44 -1
  31. package/dist/component/constants.js.map +1 -1
  32. package/dist/component/convex.config.d.ts +6 -1
  33. package/dist/component/convex.config.js +9 -1
  34. package/dist/component/convex.config.js.map +1 -1
  35. package/dist/component/coverage.d.ts +27 -0
  36. package/dist/component/coverage.d.ts.map +1 -0
  37. package/dist/component/coverage.js +56 -0
  38. package/dist/component/coverage.js.map +1 -0
  39. package/dist/component/diagnostics.d.ts +9 -0
  40. package/dist/component/diagnostics.d.ts.map +1 -0
  41. package/dist/component/diagnostics.js +47 -0
  42. package/dist/component/diagnostics.js.map +1 -0
  43. package/dist/component/errors.d.ts +1 -1
  44. package/dist/component/errors.d.ts.map +1 -1
  45. package/dist/component/errors.js.map +1 -1
  46. package/dist/component/eventStore.d.ts +21 -9
  47. package/dist/component/eventStore.d.ts.map +1 -1
  48. package/dist/component/eventStore.js +142 -152
  49. package/dist/component/eventStore.js.map +1 -1
  50. package/dist/component/funnels.d.ts.map +1 -1
  51. package/dist/component/funnels.js +5 -3
  52. package/dist/component/funnels.js.map +1 -1
  53. package/dist/component/geo.d.ts +73 -0
  54. package/dist/component/geo.d.ts.map +1 -0
  55. package/dist/component/geo.js +648 -0
  56. package/dist/component/geo.js.map +1 -0
  57. package/dist/component/goals.d.ts.map +1 -1
  58. package/dist/component/goals.js +8 -5
  59. package/dist/component/goals.js.map +1 -1
  60. package/dist/component/guards.d.ts.map +1 -1
  61. package/dist/component/guards.js.map +1 -1
  62. package/dist/component/http.d.ts.map +1 -1
  63. package/dist/component/http.js +281 -62
  64. package/dist/component/http.js.map +1 -1
  65. package/dist/component/identity.d.ts +13 -0
  66. package/dist/component/identity.d.ts.map +1 -0
  67. package/dist/component/identity.js +58 -0
  68. package/dist/component/identity.js.map +1 -0
  69. package/dist/component/ingest.d.ts +3 -1
  70. package/dist/component/ingest.d.ts.map +1 -1
  71. package/dist/component/ingest.js +498 -95
  72. package/dist/component/ingest.js.map +1 -1
  73. package/dist/component/live.d.ts.map +1 -1
  74. package/dist/component/live.js +7 -6
  75. package/dist/component/live.js.map +1 -1
  76. package/dist/component/localTime.d.ts +25 -0
  77. package/dist/component/localTime.d.ts.map +1 -0
  78. package/dist/component/localTime.js +126 -0
  79. package/dist/component/localTime.js.map +1 -0
  80. package/dist/component/reports.d.ts +266 -10
  81. package/dist/component/reports.d.ts.map +1 -1
  82. package/dist/component/reports.js +1243 -137
  83. package/dist/component/reports.js.map +1 -1
  84. package/dist/component/retention.d.ts +75 -1
  85. package/dist/component/retention.d.ts.map +1 -1
  86. package/dist/component/retention.js +561 -38
  87. package/dist/component/retention.js.map +1 -1
  88. package/dist/component/rollupStore.d.ts +320 -0
  89. package/dist/component/rollupStore.d.ts.map +1 -0
  90. package/dist/component/rollupStore.js +596 -0
  91. package/dist/component/rollupStore.js.map +1 -0
  92. package/dist/component/rollups.d.ts +20 -0
  93. package/dist/component/rollups.d.ts.map +1 -0
  94. package/dist/component/rollups.js +73 -0
  95. package/dist/component/rollups.js.map +1 -0
  96. package/dist/component/sanitize.d.ts +24 -1
  97. package/dist/component/sanitize.d.ts.map +1 -1
  98. package/dist/component/sanitize.js +96 -16
  99. package/dist/component/sanitize.js.map +1 -1
  100. package/dist/component/schema.d.ts +687 -65
  101. package/dist/component/schema.js +187 -20
  102. package/dist/component/schema.js.map +1 -1
  103. package/dist/component/sites.d.ts +12 -0
  104. package/dist/component/sites.d.ts.map +1 -1
  105. package/dist/component/sites.js +41 -7
  106. package/dist/component/sites.js.map +1 -1
  107. package/dist/component/useragent.d.ts +9 -0
  108. package/dist/component/useragent.d.ts.map +1 -0
  109. package/dist/component/useragent.js +152 -0
  110. package/dist/component/useragent.js.map +1 -0
  111. package/dist/component/validators.d.ts +124 -69
  112. package/dist/component/validators.d.ts.map +1 -1
  113. package/dist/component/validators.js +35 -14
  114. package/dist/component/validators.js.map +1 -1
  115. package/dist/component/visitors.d.ts +19 -0
  116. package/dist/component/visitors.d.ts.map +1 -0
  117. package/dist/component/visitors.js +86 -0
  118. package/dist/component/visitors.js.map +1 -0
  119. package/dist/component/vitals.d.ts +41 -0
  120. package/dist/component/vitals.d.ts.map +1 -0
  121. package/dist/component/vitals.js +115 -0
  122. package/dist/component/vitals.js.map +1 -0
  123. package/dist/react/index.d.ts.map +1 -1
  124. package/dist/react/index.js.map +1 -1
  125. package/dist/tracker/generated.d.ts +11 -4
  126. package/dist/tracker/generated.d.ts.map +1 -1
  127. package/dist/tracker/generated.js +11 -4
  128. package/dist/tracker/generated.js.map +1 -1
  129. package/dist/tracker/tracker.d.ts +1 -1
  130. package/dist/tracker/tracker.d.ts.map +1 -1
  131. package/dist/tracker/tracker.js +57 -20
  132. package/dist/tracker/tracker.js.map +1 -1
  133. package/dist/tracker/vitals.d.ts +10 -0
  134. package/dist/tracker/vitals.d.ts.map +1 -0
  135. package/dist/tracker/vitals.js +140 -0
  136. package/dist/tracker/vitals.js.map +1 -0
  137. package/dist/tracker.min.js +1 -1
  138. package/dist/vitals.min.js +1 -0
  139. package/docs/benchmarks/2026-08-20-realistic.md +76 -76
  140. package/docs/benchmarks/2026-08-21-formal-certification.md +353 -0
  141. package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
  142. package/docs/federation-setup.md +464 -0
  143. package/docs/federation.md +352 -0
  144. package/docs/upgrading.md +344 -0
  145. package/llms.txt +72 -0
  146. package/package.json +55 -11
  147. package/scripts/benchmark-ingest.mjs +175 -73
  148. package/scripts/generate-federation-keys.mjs +20 -0
  149. package/src/component/_generated/api.ts +22 -0
  150. package/src/component/_generated/component.ts +381 -4
  151. package/src/component/_generated/server.ts +4 -0
  152. package/src/component/affiliates.ts +20 -5
  153. package/src/component/cardinality.ts +117 -0
  154. package/src/component/constants.ts +44 -1
  155. package/src/component/convex.config.ts +11 -1
  156. package/src/component/coverage.ts +71 -0
  157. package/src/component/diagnostics.ts +65 -0
  158. package/src/component/errors.ts +2 -1
  159. package/src/component/eventStore.ts +217 -193
  160. package/src/component/funnels.ts +19 -16
  161. package/src/component/geo.ts +835 -0
  162. package/src/component/goals.ts +29 -21
  163. package/src/component/guards.ts +3 -1
  164. package/src/component/http.ts +404 -70
  165. package/src/component/identity.ts +74 -0
  166. package/src/component/ingest.ts +894 -188
  167. package/src/component/live.ts +13 -7
  168. package/src/component/localTime.ts +167 -0
  169. package/src/component/reports.ts +1872 -197
  170. package/src/component/retention.ts +788 -96
  171. package/src/component/rollupStore.ts +799 -0
  172. package/src/component/rollups.ts +82 -0
  173. package/src/component/sanitize.ts +144 -29
  174. package/src/component/schema.ts +217 -21
  175. package/src/component/sites.ts +59 -12
  176. package/src/component/useragent.ts +171 -0
  177. package/src/component/validators.ts +49 -14
  178. package/src/component/visitors.ts +116 -0
  179. package/src/component/vitals.ts +146 -0
@@ -3,28 +3,55 @@ import {
3
3
  paginationResultValidator,
4
4
  } from "convex/server";
5
5
  import { v } from "convex/values";
6
- import type { Doc } from "./_generated/dataModel.js";
6
+ import type { Doc, Id } from "./_generated/dataModel.js";
7
+ import type { QueryCtx } from "./_generated/server.js";
7
8
  import { query } from "./_generated/server.js";
8
9
  import {
9
10
  AGGREGATE_SHARDS,
10
11
  DAY_MS,
12
+ FEATURE_REPORT_BYTE_RESERVE,
13
+ FEATURE_REPORT_DOCUMENT_RESERVE,
11
14
  HOUR_MS,
12
15
  HOURLY_AGGREGATE_SHARDS,
13
16
  MAX_AFFILIATES_PER_SITE,
14
17
  MAX_FUNNELS_PER_SITE,
18
+ MAX_FEATURE_ROLLUP_DAYS,
19
+ MAX_FEATURE_ROLLUP_ROWS,
15
20
  MAX_GOALS_PER_SITE,
16
21
  MAX_JOURNEY_EVENTS,
17
22
  MAX_LIVE_VISITORS,
18
23
  MAX_OVERVIEW_AGGREGATE_ROWS,
24
+ MAX_REPORT_BUCKETS,
19
25
  MAX_REPORT_RANGE_DAYS,
20
26
  MAX_REPORT_SITES,
27
+ MAX_SITE_MAP_ROUTES_PER_DAY,
28
+ MAX_SITE_MAP_ROWS,
29
+ MAX_VITALS_REPORT_ROWS,
30
+ REPORT_ROLLUP_SHARDS,
31
+ ROUTE_TRANSITION_SLOTS,
32
+ SITE_MAP_OTHER_ROUTE,
33
+ VITALS_REPORT_PAGE_LIMIT,
21
34
  } from "./constants.js";
35
+ import {
36
+ createVisitorSketch,
37
+ estimateVisitors,
38
+ unionVisitorSketches,
39
+ type VisitorSketch,
40
+ } from "./cardinality.js";
22
41
  import { fail } from "./errors.js";
42
+ import {
43
+ isLocalDayAligned,
44
+ localDayStart,
45
+ localDayStarts,
46
+ localDayTimezone,
47
+ sharedLocalTimezone,
48
+ } from "./localTime.js";
23
49
  import {
24
50
  compareJourneyEvents,
25
51
  hydrateEventContexts,
26
52
  type JourneyCursor,
27
53
  loadSessionJourneyEvents,
54
+ loadVisitorJourneyEvents,
28
55
  type RawJourneyEvent,
29
56
  } from "./eventStore.js";
30
57
  import { isPlainRecord } from "./guards.js";
@@ -34,7 +61,17 @@ import {
34
61
  funnelFieldsValidator,
35
62
  goalFieldsValidator,
36
63
  sessionFieldsValidator,
64
+ vitalMetricValidator,
37
65
  } from "./validators.js";
66
+ import { resolveVisitorIdentities } from "./visitors.js";
67
+ import {
68
+ mergeVitalHistograms,
69
+ VITAL_ALL,
70
+ VITAL_METRICS,
71
+ vitalHistogramPercentile,
72
+ vitalRatingCounts,
73
+ type VitalMetric,
74
+ } from "./vitals.js";
38
75
 
39
76
  const sessionDocumentValidator = sessionFieldsValidator.extend({
40
77
  _id: v.id("sessions"),
@@ -64,6 +101,8 @@ const liveSessionDocumentValidator = v.object({
64
101
  device: v.string(),
65
102
  pageviewCount: v.number(),
66
103
  eventCount: v.number(),
104
+ previousPath: v.optional(v.string()),
105
+ movedAt: v.optional(v.number()),
67
106
  });
68
107
  const conversionDocumentValidator = v.object({
69
108
  _id: v.id("conversions"),
@@ -94,13 +133,54 @@ const metricTotalsValidator = v.object({
94
133
  visitors: v.number(),
95
134
  conversions: v.number(),
96
135
  revenueCents: v.number(),
136
+ // Engagement: summed session duration, sessions that reached a second
137
+ // pageview, and first-ever sessions of a visitor. Zero for buckets older
138
+ // than the counters; metadata.engagement says whether the range is covered.
139
+ durationMs: v.number(),
140
+ continuedSessions: v.number(),
141
+ newVisitors: v.number(),
142
+ });
143
+ const timePointValidator = metricTotalsValidator.extend({
144
+ timestamp: v.number(),
97
145
  });
98
- const timePointValidator = metricTotalsValidator.extend({ timestamp: v.number() });
99
146
  const topItemValidator = v.object({
100
147
  value: v.string(),
101
148
  count: v.number(),
102
149
  revenueCents: v.number(),
103
150
  });
151
+ // What a breakdown's `count` is: pageviews for pages and sessions for the
152
+ // other dimensions once every bucket in the range carries those counters,
153
+ // raw event volume for ranges that reach back before they existed.
154
+ const breakdownBasisValidator = v.union(
155
+ v.literal("pageviews"),
156
+ v.literal("sessions"),
157
+ v.literal("eventVolume"),
158
+ );
159
+ const coverageStateValidator = v.union(
160
+ v.literal("complete"),
161
+ v.literal("partial"),
162
+ v.literal("retained"),
163
+ v.literal("unavailable"),
164
+ );
165
+ const resultCoverageValidator = v.object({
166
+ state: coverageStateValidator,
167
+ generation: v.union(v.number(), v.null()),
168
+ availableFrom: v.union(v.number(), v.null()),
169
+ availableThrough: v.union(v.number(), v.null()),
170
+ retainedBefore: v.union(v.number(), v.null()),
171
+ });
172
+ const rangeSemanticsValidator = v.object({
173
+ from: v.number(),
174
+ to: v.number(),
175
+ boundary: v.literal("inclusive"),
176
+ // "UTC", or the site timezone whose calendar days a daily overview used.
177
+ timezone: v.string(),
178
+ });
179
+ const reportMetadataValidator = v.object({
180
+ basis: v.union(v.literal("dailyRollup"), v.literal("boundedLegacy")),
181
+ coverage: resultCoverageValidator,
182
+ range: rangeSemanticsValidator,
183
+ });
104
184
 
105
185
  export const overview = query({
106
186
  args: {
@@ -117,20 +197,58 @@ export const overview = query({
117
197
  totals: metricTotalsValidator,
118
198
  timeSeries: v.array(timePointValidator),
119
199
  topSources: v.array(topItemValidator),
200
+ topCampaigns: v.array(topItemValidator),
120
201
  topPages: v.array(topItemValidator),
121
202
  topCountries: v.array(topItemValidator),
122
203
  topDevices: v.array(topItemValidator),
123
204
  topBrowsers: v.array(topItemValidator),
124
205
  topEvents: v.array(topItemValidator),
125
206
  topGoals: v.array(topItemValidator),
207
+ metadata: v.object({
208
+ visitors: v.object({
209
+ basis: v.union(
210
+ v.literal("estimatedUnique"),
211
+ v.literal("bucketOccurrences"),
212
+ ),
213
+ exact: v.boolean(),
214
+ legacyIncomplete: v.boolean(),
215
+ sketchVersion: v.union(v.number(), v.null()),
216
+ precision: v.union(v.number(), v.null()),
217
+ }),
218
+ breakdowns: v.object({
219
+ topSources: breakdownBasisValidator,
220
+ topCampaigns: breakdownBasisValidator,
221
+ topPages: breakdownBasisValidator,
222
+ topCountries: breakdownBasisValidator,
223
+ topDevices: breakdownBasisValidator,
224
+ topBrowsers: breakdownBasisValidator,
225
+ topEvents: v.literal("eventVolume"),
226
+ topGoals: v.literal("goalCompletionVolume"),
227
+ }),
228
+ // "counted" when every bucket in the range carries the engagement
229
+ // counters, so bounce rate, duration, pages per session and new
230
+ // visitors can be derived from the totals; "unavailable" otherwise.
231
+ engagement: v.union(v.literal("counted"), v.literal("unavailable")),
232
+ coverage: resultCoverageValidator,
233
+ range: rangeSemanticsValidator,
234
+ features: v.object({
235
+ sessionReplay: v.literal("unsupported"),
236
+ errorInsights: v.literal("unsupported"),
237
+ outboundLinks: v.literal("eventVolume"),
238
+ botDetection: v.literal("userAgent"),
239
+ }),
240
+ }),
126
241
  }),
127
242
  handler: async (ctx, args) => {
128
243
  const siteIds = validateSiteIds(args.siteIds);
129
244
  const currencies = new Set<string>();
245
+ const siteTimezones: Array<string | undefined> = [];
130
246
  for (const siteId of siteIds) {
131
247
  const site = await ctx.db.get("sites", siteId);
132
- if (!site) fail("NOT_FOUND", "site not found", { siteId: String(siteId) });
248
+ if (!site)
249
+ fail("NOT_FOUND", "site not found", { siteId: String(siteId) });
133
250
  currencies.add(site.currency ?? "USD");
251
+ siteTimezones.push(localDayTimezone(site.timezone));
134
252
  }
135
253
  if (currencies.size !== 1) {
136
254
  fail("CURRENCY_MISMATCH", "overview sites must use the same currency", {
@@ -138,16 +256,76 @@ export const overview = query({
138
256
  });
139
257
  }
140
258
  const currency = [...currencies][0];
141
- const { from, to } = validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
259
+ const { from, to } = validateRange(
260
+ args.from,
261
+ args.to,
262
+ MAX_REPORT_RANGE_DAYS,
263
+ );
142
264
  const interval = args.interval ?? (to - from <= DAY_MS ? "hour" : "day");
143
265
  if (interval === "hour" && to - from > DAY_MS) {
144
266
  fail("INVALID_ARGUMENT", "hourly reports are limited to 24 hours");
145
267
  }
146
268
  const intervalMs = interval === "hour" ? HOUR_MS : DAY_MS;
147
- const firstBucket = Math.floor(from / intervalMs) * intervalMs;
148
- const lastBucket = Math.floor(to / intervalMs) * intervalMs;
149
- const bucketCount = Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
150
- const perSiteLimit = bucketCount *
269
+ // Sites that share a timezone answer a daily range that covers whole
270
+ // calendar days in that zone from their local-day buckets; any other
271
+ // daily range must cover whole UTC days, as before.
272
+ const siteTimezone =
273
+ interval === "day" ? sharedLocalTimezone(siteTimezones) : undefined;
274
+ const localDays =
275
+ siteTimezone !== undefined && isLocalDayAligned(from, to, siteTimezone);
276
+ if (!localDays) {
277
+ requireAlignedRange(
278
+ from,
279
+ to,
280
+ intervalMs,
281
+ `${interval} overview`,
282
+ siteTimezone,
283
+ );
284
+ }
285
+ const granularity: "hour" | "day" | "localDay" = localDays
286
+ ? "localDay"
287
+ : interval;
288
+ const controls = await Promise.all(
289
+ siteIds.map((siteId) =>
290
+ ctx.db
291
+ .query("analyticsCoverage")
292
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
293
+ .unique(),
294
+ ),
295
+ );
296
+ // Local-day buckets are complete from each site's stamp (its first ingest
297
+ // after the upgrade or after a timezone change, plus the safety delay).
298
+ // Buckets before it belong to another zone's days or to no complete day
299
+ // and are left out; the coverage says since when the days are whole.
300
+ let localDaysFrom = Number.POSITIVE_INFINITY;
301
+ if (localDays) {
302
+ localDaysFrom = controls.reduce(
303
+ (latest, control) =>
304
+ control?.localDaySince === undefined
305
+ ? Number.POSITIVE_INFINITY
306
+ : Math.max(latest, control.localDaySince),
307
+ 0,
308
+ );
309
+ }
310
+ const bucketStarts = localDays
311
+ ? localDayStarts(from, to, siteTimezone!)
312
+ : [];
313
+ const bucketStartSet = new Set(bucketStarts);
314
+ const localDaysFirstBucket =
315
+ localDays && Number.isFinite(localDaysFrom)
316
+ ? localDayStart(localDaysFrom, siteTimezone!)
317
+ : Number.POSITIVE_INFINITY;
318
+ const firstBucket = localDays
319
+ ? bucketStarts[0]!
320
+ : Math.floor(from / intervalMs) * intervalMs;
321
+ const lastBucket = localDays
322
+ ? bucketStarts[bucketStarts.length - 1]!
323
+ : Math.floor(to / intervalMs) * intervalMs;
324
+ const bucketCount = localDays
325
+ ? bucketStarts.length
326
+ : Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
327
+ const perSiteLimit =
328
+ bucketCount *
151
329
  (interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
152
330
  if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
153
331
  fail(
@@ -158,7 +336,28 @@ export const overview = query({
158
336
  }
159
337
  const totals = emptyTotals();
160
338
  const points = new Map<number, MetricTotals>();
161
- const dimensions = new Map<string, { value: string; count: number; revenueCents: number }>();
339
+ const visitorSketch = createVisitorSketch();
340
+ let mergedVisitorSketch = visitorSketch;
341
+ const pointVisitorSketches = new Map<number, VisitorSketch>();
342
+ let visitorsComplete = true;
343
+ const dimensions = new Map<
344
+ string,
345
+ {
346
+ value: string;
347
+ count: number;
348
+ revenueCents: number;
349
+ pageviews: number;
350
+ sessions: number;
351
+ }
352
+ >();
353
+ const overviewDataset =
354
+ granularity === "hour"
355
+ ? ("overviewHour" as const)
356
+ : granularity === "day"
357
+ ? ("overviewDay" as const)
358
+ : ("overviewLocalDay" as const);
359
+
360
+ await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
162
361
 
163
362
  for (const siteId of siteIds) {
164
363
  const buckets = await ctx.db
@@ -166,36 +365,120 @@ export const overview = query({
166
365
  .withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
167
366
  range
168
367
  .eq("siteId", siteId)
169
- .eq("granularity", interval)
368
+ .eq("granularity", granularity)
170
369
  .gte("bucketStart", firstBucket)
171
370
  .lte("bucketStart", lastBucket),
172
371
  )
173
- .take(perSiteLimit);
372
+ .take(perSiteLimit + 1);
373
+ failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
174
374
  for (const bucket of buckets) {
375
+ if (
376
+ localDays &&
377
+ (!bucketStartSet.has(bucket.bucketStart) ||
378
+ bucket.bucketStart < localDaysFirstBucket)
379
+ ) {
380
+ continue;
381
+ }
175
382
  addMetrics(totals, bucket);
176
383
  const point = points.get(bucket.bucketStart) ?? emptyTotals();
177
384
  addMetrics(point, bucket);
178
385
  points.set(bucket.bucketStart, point);
386
+ if (bucket.visitorSketch && bucket.visitorSketchComplete === true) {
387
+ mergedVisitorSketch = unionVisitorSketches(
388
+ mergedVisitorSketch,
389
+ bucket.visitorSketch,
390
+ );
391
+ pointVisitorSketches.set(
392
+ bucket.bucketStart,
393
+ unionVisitorSketches(
394
+ pointVisitorSketches.get(bucket.bucketStart) ??
395
+ createVisitorSketch(bucket.visitorSketch.precision),
396
+ bucket.visitorSketch,
397
+ ),
398
+ );
399
+ } else if (bucket.events > 0) {
400
+ visitorsComplete = false;
401
+ }
179
402
  for (const dimension of bucket.dimensions) {
180
403
  const key = `${dimension.type}:${dimension.value}`;
181
404
  const current = dimensions.get(key) ?? {
182
405
  value: dimension.value,
183
406
  count: 0,
184
407
  revenueCents: 0,
408
+ pageviews: 0,
409
+ sessions: 0,
185
410
  };
186
411
  current.count += dimension.count;
187
412
  current.revenueCents += dimension.revenueCents;
413
+ current.pageviews += dimension.pageviews ?? 0;
414
+ current.sessions += dimension.sessions ?? 0;
188
415
  dimensions.set(key, current);
189
416
  }
190
417
  }
191
418
  }
192
419
 
193
- const top = (type: string) =>
420
+ // Pages rank by pageviews and the other dimensions by sessions once every
421
+ // site's counters are proven for the whole range; before that stamp a
422
+ // bucket may hold slots without them, so the ranking stays event volume.
423
+ const countersProven = controls.every(
424
+ (control) =>
425
+ control?.countersSince !== undefined && from >= control.countersSince,
426
+ );
427
+ const pageBasis = countersProven
428
+ ? ("pageviews" as const)
429
+ : ("eventVolume" as const);
430
+ const sessionBasis = countersProven
431
+ ? ("sessions" as const)
432
+ : ("eventVolume" as const);
433
+ const top = (
434
+ type: string,
435
+ basis: "pageviews" | "sessions" | "eventVolume",
436
+ ) =>
194
437
  [...dimensions.entries()]
195
438
  .filter(([key]) => key.startsWith(`${type}:`))
196
- .map(([, value]) => value)
197
- .sort((left, right) => right.count - left.count || left.value.localeCompare(right.value))
439
+ .map(([, item]) => ({
440
+ value: item.value,
441
+ count:
442
+ basis === "pageviews"
443
+ ? item.pageviews
444
+ : basis === "sessions"
445
+ ? item.sessions
446
+ : item.count,
447
+ revenueCents: item.revenueCents,
448
+ }))
449
+ .filter((item) => item.count > 0 || item.revenueCents > 0)
450
+ .sort(
451
+ (left, right) =>
452
+ right.count - left.count || left.value.localeCompare(right.value),
453
+ )
198
454
  .slice(0, 10);
455
+ if (visitorsComplete) {
456
+ totals.visitors = estimateVisitors(mergedVisitorSketch);
457
+ for (const [timestamp, point] of points) {
458
+ point.visitors = estimateVisitors(
459
+ pointVisitorSketches.get(timestamp) ?? createVisitorSketch(),
460
+ );
461
+ }
462
+ }
463
+ const coverage = await coverageForRange(ctx, siteIds, from, to, {
464
+ datasets: [overviewDataset],
465
+ complete: visitorsComplete,
466
+ });
467
+ if (localDays) {
468
+ // The stamp, not the oldest row, says where the site's days begin.
469
+ const availableFrom = Number.isFinite(localDaysFrom)
470
+ ? Math.max(coverage.availableFrom ?? 0, localDaysFrom)
471
+ : null;
472
+ coverage.availableFrom = availableFrom;
473
+ if (
474
+ coverage.state === "complete" &&
475
+ (availableFrom === null || from < availableFrom)
476
+ ) {
477
+ coverage.state = availableFrom === null ? "unavailable" : "partial";
478
+ }
479
+ }
480
+ const visitorBasis: "estimatedUnique" | "bucketOccurrences" =
481
+ visitorsComplete ? "estimatedUnique" : "bucketOccurrences";
199
482
  return {
200
483
  from,
201
484
  to,
@@ -205,13 +488,44 @@ export const overview = query({
205
488
  timeSeries: [...points.entries()]
206
489
  .sort(([left], [right]) => left - right)
207
490
  .map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
208
- topSources: top("source"),
209
- topPages: top("page"),
210
- topCountries: top("country"),
211
- topDevices: top("device"),
212
- topBrowsers: top("browser"),
213
- topEvents: top("event"),
214
- topGoals: top("goal"),
491
+ topSources: top("source", sessionBasis),
492
+ topCampaigns: top("campaign", sessionBasis),
493
+ topPages: top("page", pageBasis),
494
+ topCountries: top("country", sessionBasis),
495
+ topDevices: top("device", sessionBasis),
496
+ topBrowsers: top("browser", sessionBasis),
497
+ topEvents: top("event", "eventVolume"),
498
+ topGoals: top("goal", "eventVolume"),
499
+ metadata: {
500
+ visitors: {
501
+ basis: visitorBasis,
502
+ exact: false,
503
+ legacyIncomplete: !visitorsComplete,
504
+ sketchVersion: visitorsComplete ? 1 : null,
505
+ precision: visitorsComplete ? 6 : null,
506
+ },
507
+ breakdowns: {
508
+ topSources: sessionBasis,
509
+ topCampaigns: sessionBasis,
510
+ topPages: pageBasis,
511
+ topCountries: sessionBasis,
512
+ topDevices: sessionBasis,
513
+ topBrowsers: sessionBasis,
514
+ topEvents: "eventVolume" as const,
515
+ topGoals: "goalCompletionVolume" as const,
516
+ },
517
+ engagement: countersProven
518
+ ? ("counted" as const)
519
+ : ("unavailable" as const),
520
+ coverage,
521
+ range: rangeSemantics(from, to, localDays ? siteTimezone : "UTC"),
522
+ features: {
523
+ sessionReplay: "unsupported" as const,
524
+ errorInsights: "unsupported" as const,
525
+ outboundLinks: "eventVolume" as const,
526
+ botDetection: "userAgent" as const,
527
+ },
528
+ },
215
529
  };
216
530
  },
217
531
  });
@@ -226,8 +540,15 @@ export const liveVisitors = query({
226
540
  handler: async (ctx, args) => {
227
541
  const siteIds = validateSiteIds(args.siteIds);
228
542
  const limit = args.limit ?? 100;
229
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_LIVE_VISITORS) {
230
- fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_LIVE_VISITORS}`);
543
+ if (
544
+ !Number.isSafeInteger(limit) ||
545
+ limit < 1 ||
546
+ limit > MAX_LIVE_VISITORS
547
+ ) {
548
+ fail(
549
+ "INVALID_ARGUMENT",
550
+ `limit must be an integer between 1 and ${MAX_LIVE_VISITORS}`,
551
+ );
231
552
  }
232
553
  if (!Number.isFinite(args.now) || args.now < 0) {
233
554
  fail("INVALID_ARGUMENT", "now must be a positive timestamp");
@@ -267,10 +588,311 @@ export const liveVisitors = query({
267
588
  device: row.device,
268
589
  pageviewCount: row.pageviewCount,
269
590
  eventCount: row.eventCount,
591
+ ...(row.previousPath ? { previousPath: row.previousPath } : {}),
592
+ ...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
270
593
  }));
271
594
  },
272
595
  });
273
596
 
597
+ const siteMapRouteValidator = v.object({
598
+ route: v.string(),
599
+ pageviews: v.number(),
600
+ visitors: v.number(),
601
+ entries: v.number(),
602
+ exits: v.number(),
603
+ bounces: v.number(),
604
+ transitionsIn: v.number(),
605
+ transitionsOut: v.number(),
606
+ dwellMs: v.number(),
607
+ dwellSamples: v.number(),
608
+ daily: v.array(v.object({ bucketStart: v.number(), pageviews: v.number() })),
609
+ });
610
+ const siteMapTransitionValidator = v.object({
611
+ from: v.string(),
612
+ to: v.string(),
613
+ count: v.number(),
614
+ });
615
+ const siteMapCoverageDataset = "siteMapRollups";
616
+
617
+ /**
618
+ * Every route the sites served in the range and the transitions between
619
+ * them, read from the daily route rollups newest day first. Exits and
620
+ * bounces are derived rather than counted: a session enters once, leaves
621
+ * once, and every later pageview is a transition, so per route
622
+ * `exits = entries + transitionsIn - transitionsOut` and
623
+ * `bounces = entries - continued` hold without a session-end hook. The
624
+ * report never falls back to raw telemetry; days it could not cover are
625
+ * declared through `coverage` and `window`.
626
+ */
627
+ export const siteMap = query({
628
+ args: {
629
+ siteIds: v.array(v.id("sites")),
630
+ from: v.number(),
631
+ to: v.number(),
632
+ },
633
+ returns: v.object({
634
+ from: v.number(),
635
+ to: v.number(),
636
+ window: v.union(v.object({ from: v.number(), to: v.number() }), v.null()),
637
+ routes: v.array(siteMapRouteValidator),
638
+ transitions: v.array(siteMapTransitionValidator),
639
+ truncated: v.object({ routes: v.boolean(), transitions: v.boolean() }),
640
+ metadata: reportMetadataValidator.extend({
641
+ visitors: v.object({
642
+ basis: v.literal("estimatedUnique"),
643
+ exact: v.boolean(),
644
+ }),
645
+ limits: v.object({
646
+ routesPerDay: v.number(),
647
+ transitionSlots: v.number(),
648
+ rows: v.number(),
649
+ }),
650
+ }),
651
+ }),
652
+ handler: async (ctx, args) => {
653
+ const siteIds = validateSiteIds(args.siteIds);
654
+ validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
655
+ requireAlignedRange(args.from, args.to, DAY_MS, "site map");
656
+ const controls = await Promise.all(
657
+ siteIds.map((siteId) =>
658
+ ctx.db
659
+ .query("analyticsCoverage")
660
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
661
+ .unique(),
662
+ ),
663
+ );
664
+ const since = controls.every(
665
+ (control) => control?.siteMapSince !== undefined,
666
+ )
667
+ ? Math.max(...controls.map((control) => control!.siteMapSince!))
668
+ : null;
669
+ const retainedBefore = controls.reduce<number | null>((latest, control) => {
670
+ const before = control?.retained.find(
671
+ (item) => item.dataset === siteMapCoverageDataset,
672
+ )?.before;
673
+ return before === undefined ? latest : Math.max(latest ?? 0, before);
674
+ }, null);
675
+ const metadata = (state: "complete" | "partial" | "unavailable") => ({
676
+ basis: "dailyRollup" as const,
677
+ coverage: {
678
+ state,
679
+ generation: 1 as const,
680
+ availableFrom: since,
681
+ availableThrough: since === null ? null : args.to,
682
+ retainedBefore,
683
+ },
684
+ range: rangeSemantics(args.from, args.to),
685
+ visitors: { basis: "estimatedUnique" as const, exact: false },
686
+ limits: {
687
+ routesPerDay: MAX_SITE_MAP_ROUTES_PER_DAY,
688
+ transitionSlots: ROUTE_TRANSITION_SLOTS,
689
+ rows: MAX_SITE_MAP_ROWS,
690
+ },
691
+ });
692
+ if (
693
+ since === null ||
694
+ args.to < since ||
695
+ (retainedBefore !== null && args.to < retainedBefore)
696
+ ) {
697
+ return {
698
+ from: args.from,
699
+ to: args.to,
700
+ window: null,
701
+ routes: [],
702
+ transitions: [],
703
+ truncated: { routes: false, transitions: false },
704
+ metadata: metadata("unavailable"),
705
+ };
706
+ }
707
+ const readFrom = Math.max(
708
+ args.from,
709
+ Math.floor(since / DAY_MS) * DAY_MS,
710
+ retainedBefore ?? 0,
711
+ );
712
+ let budget = MAX_SITE_MAP_ROWS;
713
+ let completeFrom = readFrom;
714
+ const rows: Doc<"routeDailyRollups">[] = [];
715
+ for (const siteId of siteIds) {
716
+ const read = await readRouteRollups(
717
+ ctx,
718
+ siteId,
719
+ readFrom,
720
+ args.to,
721
+ budget,
722
+ );
723
+ budget -= read.rows.length;
724
+ completeFrom = Math.max(completeFrom, read.completeFrom);
725
+ rows.push(...read.rows);
726
+ }
727
+ const windowFrom = Math.max(
728
+ args.from,
729
+ completeFrom,
730
+ since,
731
+ retainedBefore ?? 0,
732
+ );
733
+
734
+ type RouteTotals = {
735
+ pageviews: number;
736
+ entries: number;
737
+ continued: number;
738
+ dwellMs: number;
739
+ dwellSamples: number;
740
+ transitionsIn: number;
741
+ transitionsOut: number;
742
+ sketch: VisitorSketch;
743
+ daily: Map<number, number>;
744
+ };
745
+ const routes = new Map<string, RouteTotals>();
746
+ const totalsFor = (route: string) => {
747
+ let current = routes.get(route);
748
+ if (!current) {
749
+ current = {
750
+ pageviews: 0,
751
+ entries: 0,
752
+ continued: 0,
753
+ dwellMs: 0,
754
+ dwellSamples: 0,
755
+ transitionsIn: 0,
756
+ transitionsOut: 0,
757
+ sketch: createVisitorSketch(),
758
+ daily: new Map<number, number>(),
759
+ };
760
+ routes.set(route, current);
761
+ }
762
+ return current;
763
+ };
764
+ const edges = new Map<
765
+ string,
766
+ { from: string; to: string; count: number }
767
+ >();
768
+ for (const row of rows) {
769
+ if (row.bucketStart < completeFrom) continue;
770
+ const current = totalsFor(row.route);
771
+ current.pageviews += row.pageviews;
772
+ current.entries += row.entries;
773
+ current.continued += row.continued;
774
+ current.dwellMs += row.dwellMs;
775
+ current.dwellSamples += row.dwellSamples;
776
+ current.sketch = unionVisitorSketches(current.sketch, row.visitorSketch);
777
+ current.daily.set(
778
+ row.bucketStart,
779
+ (current.daily.get(row.bucketStart) ?? 0) + row.pageviews,
780
+ );
781
+ for (const target of row.outgoing) {
782
+ const key = `${row.route}\u0000${target.route}`;
783
+ const edge = edges.get(key) ?? {
784
+ from: row.route,
785
+ to: target.route,
786
+ count: 0,
787
+ };
788
+ edge.count += target.count;
789
+ edges.set(key, edge);
790
+ current.transitionsOut += target.count;
791
+ }
792
+ }
793
+ // A target without a row of its own in this window — folded past the
794
+ // daily cap, or already retained — counts under "(other)".
795
+ const transitions = new Map<
796
+ string,
797
+ { from: string; to: string; count: number }
798
+ >();
799
+ for (const edge of edges.values()) {
800
+ const to = routes.has(edge.to) ? edge.to : SITE_MAP_OTHER_ROUTE;
801
+ const key = `${edge.from}\u0000${to}`;
802
+ const merged = transitions.get(key) ?? { from: edge.from, to, count: 0 };
803
+ merged.count += edge.count;
804
+ transitions.set(key, merged);
805
+ totalsFor(to).transitionsIn += edge.count;
806
+ }
807
+ const routeRows = [...routes.entries()]
808
+ .map(([route, totals]) => ({
809
+ route,
810
+ pageviews: totals.pageviews,
811
+ visitors: estimateVisitors(totals.sketch),
812
+ entries: totals.entries,
813
+ exits: Math.max(
814
+ 0,
815
+ totals.entries + totals.transitionsIn - totals.transitionsOut,
816
+ ),
817
+ bounces: Math.max(0, totals.entries - totals.continued),
818
+ transitionsIn: totals.transitionsIn,
819
+ transitionsOut: totals.transitionsOut,
820
+ dwellMs: totals.dwellMs,
821
+ dwellSamples: totals.dwellSamples,
822
+ daily: [...totals.daily.entries()]
823
+ .sort(([left], [right]) => left - right)
824
+ .map(([bucketStart, pageviews]) => ({ bucketStart, pageviews })),
825
+ }))
826
+ .sort(
827
+ (left, right) =>
828
+ right.pageviews - left.pageviews ||
829
+ left.route.localeCompare(right.route),
830
+ );
831
+ const transitionRows = [...transitions.values()].sort(
832
+ (left, right) =>
833
+ right.count - left.count ||
834
+ left.from.localeCompare(right.from) ||
835
+ left.to.localeCompare(right.to),
836
+ );
837
+ return {
838
+ from: args.from,
839
+ to: args.to,
840
+ window: completeFrom > args.to ? null : { from: windowFrom, to: args.to },
841
+ routes: routeRows,
842
+ transitions: transitionRows,
843
+ truncated: {
844
+ routes: routes.has(SITE_MAP_OTHER_ROUTE),
845
+ transitions: transitionRows.some(
846
+ (transition) =>
847
+ transition.from === SITE_MAP_OTHER_ROUTE ||
848
+ transition.to === SITE_MAP_OTHER_ROUTE,
849
+ ),
850
+ },
851
+ metadata: metadata(windowFrom > args.from ? "partial" : "complete"),
852
+ };
853
+ },
854
+ });
855
+
856
+ /**
857
+ * Reads a site's route rollups newest day first until the row budget or the
858
+ * transaction's read headroom runs out, then drops the day it could only
859
+ * read in part so every day it returns is complete.
860
+ */
861
+ async function readRouteRollups(
862
+ ctx: QueryCtx,
863
+ siteId: Id<"sites">,
864
+ from: number,
865
+ to: number,
866
+ budget: number,
867
+ ) {
868
+ const rows: Doc<"routeDailyRollups">[] = [];
869
+ let cutAt: number | null = null;
870
+ const query = ctx.db
871
+ .query("routeDailyRollups")
872
+ .withIndex("by_siteId_and_bucketStart", (range) =>
873
+ range
874
+ .eq("siteId", siteId)
875
+ .gte("bucketStart", from)
876
+ .lte("bucketStart", to),
877
+ )
878
+ .order("desc");
879
+ for await (const row of query) {
880
+ if (
881
+ rows.length >= budget ||
882
+ (rows.length % 256 === 255 && !(await hasFeatureReadHeadroom(ctx)))
883
+ ) {
884
+ cutAt = row.bucketStart;
885
+ break;
886
+ }
887
+ rows.push(row);
888
+ }
889
+ if (cutAt === null) return { rows, completeFrom: from };
890
+ while (rows.length > 0 && rows[rows.length - 1]!.bucketStart === cutAt) {
891
+ rows.pop();
892
+ }
893
+ return { rows, completeFrom: cutAt + DAY_MS };
894
+ }
895
+
274
896
  export const listSessions = query({
275
897
  args: {
276
898
  siteId: v.id("sites"),
@@ -281,6 +903,7 @@ export const listSessions = query({
281
903
  returns: paginationResultValidator(sessionDocumentValidator),
282
904
  handler: async (ctx, args) => {
283
905
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
906
+ await assertSourceAvailable(ctx, [args.siteId], ["sessions"], args.from);
284
907
  const pagination = parseKeysetPagination(args.paginationOpts, {
285
908
  kind: "sessions",
286
909
  siteId: String(args.siteId),
@@ -330,7 +953,14 @@ export const listSessions = query({
330
953
  .take(take)),
331
954
  );
332
955
  }
333
- return keysetPaginationResult(rows, pagination, (row) => ({
956
+ const publicRows = rows.map(
957
+ ({
958
+ geoLookupAttemptedAt: _attemptedAt,
959
+ lastPageviewAt: _lastPageviewAt,
960
+ ...row
961
+ }) => row,
962
+ );
963
+ return keysetPaginationResult(publicRows, pagination, (row) => ({
334
964
  timestamp: row.lastSeenAt,
335
965
  creationTime: row._creationTime,
336
966
  }));
@@ -348,6 +978,7 @@ export const sessionJourney = query({
348
978
  returns: paginationResultValidator(eventDocumentValidator),
349
979
  handler: async (ctx, args) => {
350
980
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
981
+ await assertSourceAvailable(ctx, [args.siteId], ["events"], args.from);
351
982
  const pagination = parseJourneyPagination(args.paginationOpts, {
352
983
  kind: "events",
353
984
  siteId: String(args.siteId),
@@ -378,6 +1009,7 @@ export const listConversions = query({
378
1009
  returns: paginationResultValidator(conversionDocumentValidator),
379
1010
  handler: async (ctx, args) => {
380
1011
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
1012
+ await assertSourceAvailable(ctx, [args.siteId], ["conversions"], args.from);
381
1013
  const pagination = parseKeysetPagination(args.paginationOpts, {
382
1014
  kind: "conversions",
383
1015
  siteId: String(args.siteId),
@@ -446,54 +1078,39 @@ export const visitorJourney = query({
446
1078
  handler: async (ctx, args) => {
447
1079
  const siteIds = validateSiteIds(args.siteIds);
448
1080
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
1081
+ await assertSourceAvailable(ctx, siteIds, ["events"], args.from);
449
1082
  const limit = args.limit ?? 200;
450
- if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_JOURNEY_EVENTS) {
451
- fail("INVALID_ARGUMENT", `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`);
1083
+ if (
1084
+ !Number.isSafeInteger(limit) ||
1085
+ limit < 1 ||
1086
+ limit > MAX_JOURNEY_EVENTS
1087
+ ) {
1088
+ fail(
1089
+ "INVALID_ARGUMENT",
1090
+ `limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`,
1091
+ );
452
1092
  }
453
- const sessionGroups = await Promise.all(
454
- siteIds.map((siteId) =>
455
- ctx.db
456
- .query("sessions")
457
- .withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
458
- range
459
- .eq("siteId", siteId)
460
- .eq("visitorId", args.visitorId)
461
- .gte("lastSeenAt", args.from),
462
- )
463
- .order("asc")
464
- .take(MAX_JOURNEY_EVENTS + 1),
465
- ),
466
- );
467
- const sessions = sessionGroups
468
- .flat()
469
- .filter((session) => session.startedAt <= args.to)
470
- .sort((left, right) => left.startedAt - right.startedAt)
471
- .slice(0, MAX_JOURNEY_EVENTS);
472
1093
  const rows: RawJourneyEvent[] = [];
473
- let overlappingSessionsRead = 0;
474
- for (const session of sessions) {
475
- const resultIsFull = rows.length >= limit;
476
- const cutoff = resultIsFull
477
- ? rows[limit - 1]!.timestamp
478
- : args.to;
479
- if (session.startedAt > cutoff) break;
480
- if (resultIsFull && ++overlappingSessionsRead > 8) {
481
- fail(
482
- "LIMIT_EXCEEDED",
483
- "visitor journey has too many overlapping sessions",
484
- { maximumOverlappingSessions: 8 },
1094
+ for (const siteId of siteIds) {
1095
+ for (const visitorId of await resolveVisitorIdentities(
1096
+ ctx,
1097
+ siteId,
1098
+ args.visitorId,
1099
+ )) {
1100
+ const cutoff =
1101
+ rows.length >= limit ? rows[limit - 1]!.timestamp : args.to;
1102
+ rows.push(
1103
+ ...(await loadVisitorJourneyEvents(ctx, {
1104
+ siteId,
1105
+ visitorId,
1106
+ from: args.from,
1107
+ to: cutoff,
1108
+ resultLimit: limit,
1109
+ })),
485
1110
  );
1111
+ rows.sort(compareJourneyEvents);
1112
+ if (rows.length > limit) rows.length = limit;
486
1113
  }
487
- rows.push(...await loadSessionJourneyEvents(ctx, {
488
- siteId: session.siteId,
489
- sessionId: session.sessionId,
490
- from: args.from,
491
- to: Math.min(args.to, cutoff),
492
- cursor: null,
493
- resultLimit: resultIsFull ? limit : limit - rows.length,
494
- }));
495
- rows.sort(compareJourneyEvents);
496
- if (rows.length > limit) rows.length = limit;
497
1114
  }
498
1115
  return hydrateEventContexts(ctx, rows);
499
1116
  },
@@ -511,36 +1128,90 @@ export const goalsReport = query({
511
1128
  returns: v.object({
512
1129
  goals: v.array(goalReportItemValidator),
513
1130
  truncated: v.boolean(),
1131
+ metadata: reportMetadataValidator,
514
1132
  }),
515
1133
  handler: async (ctx, args) => {
516
1134
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
517
- const goals = await ctx.db
1135
+ const goalRows = await ctx.db
518
1136
  .query("goals")
519
1137
  .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
520
- .take(MAX_GOALS_PER_SITE);
521
- const completions = await ctx.db
522
- .query("goalCompletions")
523
- .withIndex("by_siteId_and_timestamp", (range) =>
524
- range
525
- .eq("siteId", args.siteId)
526
- .gte("timestamp", args.from)
527
- .lte("timestamp", args.to),
528
- )
529
- .take(5_001);
530
- const totals = new Map<string, { completions: number; valueCents: number }>();
531
- for (const completion of completions.slice(0, 5_000)) {
532
- const key = String(completion.goalId);
533
- const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
534
- total.completions += 1;
535
- total.valueCents += completion.valueCents;
536
- totals.set(key, total);
1138
+ .take(MAX_GOALS_PER_SITE + 1);
1139
+ failIfIncomplete(goalRows, MAX_GOALS_PER_SITE, "goal definitions");
1140
+ const goals = goalRows;
1141
+ const totals = new Map<
1142
+ string,
1143
+ { completions: number; valueCents: number }
1144
+ >();
1145
+ const useRollups = await dailyRollupsAvailable(
1146
+ ctx,
1147
+ args.siteId,
1148
+ args.from,
1149
+ args.to,
1150
+ "goalRollups",
1151
+ );
1152
+ const rollups = useRollups
1153
+ ? await loadGoalDailyRollups(
1154
+ ctx,
1155
+ args.siteId,
1156
+ goals.map((goal) => goal._id),
1157
+ args.from,
1158
+ args.to,
1159
+ )
1160
+ : null;
1161
+ if (rollups) {
1162
+ for (const row of rollups) {
1163
+ const key = String(row.goalId);
1164
+ const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
1165
+ total.completions += row.completions;
1166
+ total.valueCents += row.valueCents;
1167
+ totals.set(key, total);
1168
+ }
1169
+ } else {
1170
+ await assertSourceAvailable(
1171
+ ctx,
1172
+ [args.siteId],
1173
+ ["goalCompletions"],
1174
+ args.from,
1175
+ );
1176
+ const completions = await ctx.db
1177
+ .query("goalCompletions")
1178
+ .withIndex("by_siteId_and_timestamp", (range) =>
1179
+ range
1180
+ .eq("siteId", args.siteId)
1181
+ .gte("timestamp", args.from)
1182
+ .lte("timestamp", args.to),
1183
+ )
1184
+ .take(MAX_REPORT_BUCKETS + 1);
1185
+ failIfIncomplete(completions, MAX_REPORT_BUCKETS, "goal completions");
1186
+ for (const completion of completions) {
1187
+ const key = String(completion.goalId);
1188
+ const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
1189
+ total.completions += 1;
1190
+ total.valueCents += completion.valueCents;
1191
+ totals.set(key, total);
1192
+ }
537
1193
  }
538
1194
  return {
539
1195
  goals: goals.map((goal) => ({
540
1196
  ...goal,
541
1197
  ...(totals.get(String(goal._id)) ?? { completions: 0, valueCents: 0 }),
542
1198
  })),
543
- truncated: completions.length > 5_000,
1199
+ truncated: false,
1200
+ metadata: {
1201
+ basis: rollups ? ("dailyRollup" as const) : ("boundedLegacy" as const),
1202
+ coverage: await coverageForRange(
1203
+ ctx,
1204
+ [args.siteId],
1205
+ args.from,
1206
+ args.to,
1207
+ {
1208
+ datasets: [rollups ? "goalRollups" : "goalCompletions"],
1209
+ complete: true,
1210
+ generation: rollups ? 1 : null,
1211
+ },
1212
+ ),
1213
+ range: rangeSemantics(args.from, args.to),
1214
+ },
544
1215
  };
545
1216
  },
546
1217
  });
@@ -549,9 +1220,15 @@ const funnelReportItemValidator = funnelFieldsValidator.extend({
549
1220
  _id: v.id("funnels"),
550
1221
  _creationTime: v.number(),
551
1222
  stepsReport: v.array(
552
- v.object({ step: v.number(), name: v.string(), entrants: v.number(), completions: v.number() }),
1223
+ v.object({
1224
+ step: v.number(),
1225
+ name: v.string(),
1226
+ entrants: v.number(),
1227
+ completions: v.number(),
1228
+ }),
553
1229
  ),
554
1230
  truncated: v.boolean(),
1231
+ metadata: reportMetadataValidator,
555
1232
  });
556
1233
 
557
1234
  export const funnelsReport = query({
@@ -559,29 +1236,74 @@ export const funnelsReport = query({
559
1236
  returns: v.array(funnelReportItemValidator),
560
1237
  handler: async (ctx, args) => {
561
1238
  validateRange(args.from, args.to, 90);
562
- const funnels = await ctx.db
1239
+ requireAlignedRange(args.from, args.to, DAY_MS, "funnel report");
1240
+ const funnelRows = await ctx.db
563
1241
  .query("funnels")
564
1242
  .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
565
- .take(MAX_FUNNELS_PER_SITE);
1243
+ .take(MAX_FUNNELS_PER_SITE + 1);
1244
+ failIfIncomplete(funnelRows, MAX_FUNNELS_PER_SITE, "funnel definitions");
1245
+ const funnels = funnelRows;
1246
+ const useRollups = await dailyRollupsAvailable(
1247
+ ctx,
1248
+ args.siteId,
1249
+ args.from,
1250
+ args.to,
1251
+ "funnelRollups",
1252
+ );
1253
+ const rollups = useRollups
1254
+ ? await loadFunnelDailyRollups(
1255
+ ctx,
1256
+ args.siteId,
1257
+ funnels.map((funnel) => funnel._id),
1258
+ args.from,
1259
+ args.to,
1260
+ )
1261
+ : null;
1262
+ if (!rollups) {
1263
+ await assertSourceAvailable(
1264
+ ctx,
1265
+ [args.siteId],
1266
+ ["funnelSteps"],
1267
+ args.from,
1268
+ );
1269
+ }
1270
+ const reportCoverage = await coverageForRange(
1271
+ ctx,
1272
+ [args.siteId],
1273
+ args.from,
1274
+ args.to,
1275
+ {
1276
+ datasets: [rollups ? "funnelRollups" : "funnelSteps"],
1277
+ complete: true,
1278
+ generation: rollups ? 1 : null,
1279
+ },
1280
+ );
566
1281
  const reports = [];
567
1282
  for (const funnel of funnels) {
568
- const buckets = await ctx.db
569
- .query("funnelStepBuckets")
570
- .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
571
- range
572
- .eq("siteId", args.siteId)
573
- .eq("funnelId", funnel._id)
574
- .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
575
- .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
576
- )
577
- .take(601);
578
1283
  const stepsReport = funnel.steps.map((step, index) => ({
579
1284
  step: index,
580
1285
  name: step.name,
581
1286
  entrants: 0,
582
1287
  completions: 0,
583
1288
  }));
584
- for (const bucket of buckets.slice(0, 600)) {
1289
+ const buckets = rollups
1290
+ ? rollups
1291
+ .filter((row) => row.funnelId === funnel._id)
1292
+ .flatMap((row) => row.steps)
1293
+ : await ctx.db
1294
+ .query("funnelStepBuckets")
1295
+ .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
1296
+ range
1297
+ .eq("siteId", args.siteId)
1298
+ .eq("funnelId", funnel._id)
1299
+ .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
1300
+ .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
1301
+ )
1302
+ .take(MAX_REPORT_BUCKETS + 1);
1303
+ if (!rollups) {
1304
+ failIfIncomplete(buckets, MAX_REPORT_BUCKETS, "funnel step buckets");
1305
+ }
1306
+ for (const bucket of buckets) {
585
1307
  const step = stepsReport[bucket.step];
586
1308
  if (step) {
587
1309
  step.entrants += bucket.entrants;
@@ -591,7 +1313,14 @@ export const funnelsReport = query({
591
1313
  reports.push({
592
1314
  ...funnel,
593
1315
  stepsReport,
594
- truncated: buckets.length > 600,
1316
+ truncated: false,
1317
+ metadata: {
1318
+ basis: rollups
1319
+ ? ("dailyRollup" as const)
1320
+ : ("boundedLegacy" as const),
1321
+ coverage: reportCoverage,
1322
+ range: rangeSemantics(args.from, args.to),
1323
+ },
595
1324
  });
596
1325
  }
597
1326
  return reports;
@@ -613,64 +1342,134 @@ export const affiliatesReport = query({
613
1342
  returns: v.object({
614
1343
  affiliates: v.array(affiliateReportItemValidator),
615
1344
  truncated: v.boolean(),
1345
+ metadata: reportMetadataValidator.extend({
1346
+ uniqueVisitors: v.object({
1347
+ basis: v.union(v.literal("estimatedUnique"), v.literal("exactLegacy")),
1348
+ exact: v.boolean(),
1349
+ }),
1350
+ }),
616
1351
  }),
617
1352
  handler: async (ctx, args) => {
618
1353
  validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
1354
+ requireAlignedRange(args.from, args.to, DAY_MS, "affiliate report");
619
1355
  const affiliateRows = await ctx.db
620
1356
  .query("affiliates")
621
1357
  .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
622
1358
  .order("desc")
623
1359
  .take(MAX_AFFILIATES_PER_SITE + 1);
624
- const affiliates = affiliateRows.slice(0, MAX_AFFILIATES_PER_SITE);
625
- const stats = await ctx.db
626
- .query("affiliateDailyStats")
627
- .withIndex("by_siteId_and_bucketStart", (range) =>
628
- range
629
- .eq("siteId", args.siteId)
630
- .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
631
- .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
632
- )
633
- .take(5_001);
634
- const sessions = await ctx.db
635
- .query("sessions")
636
- .withIndex("by_siteId_and_startedAt", (range) =>
637
- range
638
- .eq("siteId", args.siteId)
639
- .gte("startedAt", args.from)
640
- .lte("startedAt", args.to),
641
- )
642
- .take(5_001);
643
- const visits = new Map<string, {
644
- visits: number;
645
- visitors: Set<string>;
646
- }>();
647
- for (const session of sessions.slice(0, 5_000)) {
648
- if (!session.affiliateId) continue;
649
- const key = String(session.affiliateId);
650
- const total = visits.get(key) ?? {
651
- visits: 0,
652
- visitors: new Set<string>(),
653
- };
654
- total.visits += 1;
655
- total.visitors.add(session.visitorId);
656
- visits.set(key, total);
657
- }
658
- const totals = new Map<string, {
659
- conversions: number;
660
- revenueCents: number;
661
- commissionCents: number;
662
- }>();
663
- for (const row of stats.slice(0, 5_000)) {
664
- const key = String(row.affiliateId);
665
- const total = totals.get(key) ?? {
666
- conversions: 0,
667
- revenueCents: 0,
668
- commissionCents: 0,
669
- };
670
- total.conversions += row.conversions;
671
- total.revenueCents += row.revenueCents;
672
- total.commissionCents += row.commissionCents;
673
- totals.set(key, total);
1360
+ failIfIncomplete(
1361
+ affiliateRows,
1362
+ MAX_AFFILIATES_PER_SITE,
1363
+ "affiliate definitions",
1364
+ );
1365
+ const affiliates = affiliateRows;
1366
+ const useRollups = await dailyRollupsAvailable(
1367
+ ctx,
1368
+ args.siteId,
1369
+ args.from,
1370
+ args.to,
1371
+ "affiliateRollups",
1372
+ );
1373
+ const rollups = useRollups
1374
+ ? await loadAffiliateDailyRollups(
1375
+ ctx,
1376
+ args.siteId,
1377
+ affiliates.map((affiliate) => affiliate._id),
1378
+ args.from,
1379
+ args.to,
1380
+ )
1381
+ : null;
1382
+ const visits = new Map<
1383
+ string,
1384
+ {
1385
+ visits: number;
1386
+ visitors: Set<string>;
1387
+ sketch: VisitorSketch;
1388
+ }
1389
+ >();
1390
+ const totals = new Map<
1391
+ string,
1392
+ {
1393
+ conversions: number;
1394
+ revenueCents: number;
1395
+ commissionCents: number;
1396
+ }
1397
+ >();
1398
+ if (rollups) {
1399
+ for (const row of rollups) {
1400
+ const key = String(row.affiliateId);
1401
+ const traffic = visits.get(key) ?? {
1402
+ visits: 0,
1403
+ visitors: new Set<string>(),
1404
+ sketch: createVisitorSketch(),
1405
+ };
1406
+ traffic.visits += row.visits;
1407
+ traffic.sketch = unionVisitorSketches(
1408
+ traffic.sketch,
1409
+ row.visitorSketch,
1410
+ );
1411
+ visits.set(key, traffic);
1412
+ const total = totals.get(key) ?? {
1413
+ conversions: 0,
1414
+ revenueCents: 0,
1415
+ commissionCents: 0,
1416
+ };
1417
+ total.conversions += row.conversions;
1418
+ total.revenueCents += row.revenueCents;
1419
+ total.commissionCents += row.commissionCents;
1420
+ totals.set(key, total);
1421
+ }
1422
+ } else {
1423
+ await assertSourceAvailable(
1424
+ ctx,
1425
+ [args.siteId],
1426
+ ["affiliateStats", "sessions"],
1427
+ args.from,
1428
+ );
1429
+ const stats = await ctx.db
1430
+ .query("affiliateDailyStats")
1431
+ .withIndex("by_siteId_and_bucketStart", (range) =>
1432
+ range
1433
+ .eq("siteId", args.siteId)
1434
+ .gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
1435
+ .lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
1436
+ )
1437
+ .take(MAX_REPORT_BUCKETS + 1);
1438
+ const sessions = await ctx.db
1439
+ .query("sessions")
1440
+ .withIndex("by_siteId_and_startedAt", (range) =>
1441
+ range
1442
+ .eq("siteId", args.siteId)
1443
+ .gte("startedAt", args.from)
1444
+ .lte("startedAt", args.to),
1445
+ )
1446
+ .take(MAX_REPORT_BUCKETS + 1);
1447
+ failIfIncomplete(stats, MAX_REPORT_BUCKETS, "affiliate stat buckets");
1448
+ failIfIncomplete(sessions, MAX_REPORT_BUCKETS, "affiliate sessions");
1449
+ for (const session of sessions) {
1450
+ if (!session.affiliateId) continue;
1451
+ const key = String(session.affiliateId);
1452
+ const total = visits.get(key) ?? {
1453
+ visits: 0,
1454
+ visitors: new Set<string>(),
1455
+ sketch: createVisitorSketch(),
1456
+ };
1457
+ total.visits += 1;
1458
+ total.visitors.add(session.visitorId);
1459
+ visits.set(key, total);
1460
+ }
1461
+ for (const row of stats) {
1462
+ const key = String(row.affiliateId);
1463
+ const total = totals.get(key) ?? {
1464
+ conversions: 0,
1465
+ revenueCents: 0,
1466
+ commissionCents: 0,
1467
+ };
1468
+ total.conversions += row.conversions;
1469
+ total.revenueCents += row.revenueCents;
1470
+ total.commissionCents += row.commissionCents;
1471
+ totals.set(key, total);
1472
+ }
674
1473
  }
675
1474
  return {
676
1475
  affiliates: affiliates.map((affiliate) => {
@@ -684,17 +1483,811 @@ export const affiliatesReport = query({
684
1483
  ...affiliate,
685
1484
  ...performance,
686
1485
  visits: traffic?.visits ?? 0,
687
- uniqueVisitors: traffic?.visitors.size ?? 0,
1486
+ uniqueVisitors: rollups
1487
+ ? estimateVisitors(traffic?.sketch ?? createVisitorSketch())
1488
+ : (traffic?.visitors.size ?? 0),
688
1489
  };
689
1490
  }),
690
- truncated:
691
- affiliateRows.length > MAX_AFFILIATES_PER_SITE ||
692
- stats.length > 5_000 ||
693
- sessions.length > 5_000,
1491
+ truncated: false,
1492
+ metadata: {
1493
+ basis: rollups ? ("dailyRollup" as const) : ("boundedLegacy" as const),
1494
+ coverage: await coverageForRange(
1495
+ ctx,
1496
+ [args.siteId],
1497
+ args.from,
1498
+ args.to,
1499
+ {
1500
+ datasets: rollups
1501
+ ? ["affiliateRollups"]
1502
+ : ["affiliateStats", "sessions"],
1503
+ complete: true,
1504
+ generation: rollups ? 1 : null,
1505
+ },
1506
+ ),
1507
+ range: rangeSemantics(args.from, args.to),
1508
+ uniqueVisitors: {
1509
+ basis: rollups
1510
+ ? ("estimatedUnique" as const)
1511
+ : ("exactLegacy" as const),
1512
+ exact: !rollups,
1513
+ },
1514
+ },
1515
+ };
1516
+ },
1517
+ });
1518
+
1519
+ const vitalStatsValidator = v.object({
1520
+ count: v.number(),
1521
+ p75: v.number(),
1522
+ average: v.number(),
1523
+ good: v.number(),
1524
+ needsImprovement: v.number(),
1525
+ poor: v.number(),
1526
+ });
1527
+
1528
+ export const vitalsReport = query({
1529
+ args: { siteId: v.id("sites"), from: v.number(), to: v.number() },
1530
+ returns: v.object({
1531
+ metrics: v.array(
1532
+ vitalStatsValidator.extend({
1533
+ metric: vitalMetricValidator,
1534
+ devices: v.array(vitalStatsValidator.extend({ device: v.string() })),
1535
+ pages: v.array(vitalStatsValidator.extend({ page: v.string() })),
1536
+ timeSeries: v.array(
1537
+ v.object({
1538
+ timestamp: v.number(),
1539
+ count: v.number(),
1540
+ p75: v.number(),
1541
+ }),
1542
+ ),
1543
+ }),
1544
+ ),
1545
+ truncated: v.boolean(),
1546
+ metadata: reportMetadataValidator,
1547
+ }),
1548
+ handler: async (ctx, args) => {
1549
+ validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
1550
+ requireAlignedRange(args.from, args.to, DAY_MS, "vitals report");
1551
+ await assertSourceAvailable(
1552
+ ctx,
1553
+ [args.siteId],
1554
+ ["vitalRollups"],
1555
+ args.from,
1556
+ );
1557
+ const rows = await ctx.db
1558
+ .query("vitalRollups")
1559
+ .withIndex("by_siteId_and_bucketStart", (range) =>
1560
+ range
1561
+ .eq("siteId", args.siteId)
1562
+ .gte("bucketStart", args.from)
1563
+ .lte("bucketStart", args.to),
1564
+ )
1565
+ .take(MAX_VITALS_REPORT_ROWS + 1);
1566
+ failIfIncomplete(rows, MAX_VITALS_REPORT_ROWS, "vital rollups");
1567
+
1568
+ type Accumulated = { count: number; sum: number; histogram: number[] };
1569
+ const fold = <Key>(
1570
+ groups: Map<VitalMetric, Map<Key, Accumulated>>,
1571
+ metric: VitalMetric,
1572
+ key: Key,
1573
+ entry: { count: number; sum: number; histogram: number[] },
1574
+ ) => {
1575
+ const group = groups.get(metric) ?? new Map<Key, Accumulated>();
1576
+ const current = group.get(key) ?? { count: 0, sum: 0, histogram: [] };
1577
+ current.count += entry.count;
1578
+ current.sum += entry.sum;
1579
+ current.histogram = mergeVitalHistograms(
1580
+ current.histogram,
1581
+ entry.histogram,
1582
+ );
1583
+ group.set(key, current);
1584
+ groups.set(metric, group);
1585
+ };
1586
+ const byDevice = new Map<VitalMetric, Map<string, Accumulated>>();
1587
+ const byPage = new Map<VitalMetric, Map<string, Accumulated>>();
1588
+ const byDay = new Map<VitalMetric, Map<number, Accumulated>>();
1589
+ for (const row of rows) {
1590
+ for (const entry of row.metrics) {
1591
+ if (row.page === VITAL_ALL) {
1592
+ // A device row carries the complete signal once, so overall totals
1593
+ // and the time series derive from this family alone.
1594
+ fold(byDevice, entry.metric, row.device, entry);
1595
+ fold(byDay, entry.metric, row.bucketStart, entry);
1596
+ } else {
1597
+ fold(byPage, entry.metric, row.page, entry);
1598
+ }
1599
+ }
1600
+ }
1601
+ const stats = (metric: VitalMetric, accumulated: Accumulated) => ({
1602
+ count: accumulated.count,
1603
+ p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
1604
+ average:
1605
+ accumulated.count > 0
1606
+ ? Math.round(accumulated.sum / accumulated.count)
1607
+ : 0,
1608
+ ...vitalRatingCounts(accumulated.histogram, metric),
1609
+ });
1610
+ const metrics = VITAL_METRICS.map((metric) => {
1611
+ const overall: Accumulated = { count: 0, sum: 0, histogram: [] };
1612
+ for (const accumulated of byDevice.get(metric)?.values() ?? []) {
1613
+ overall.count += accumulated.count;
1614
+ overall.sum += accumulated.sum;
1615
+ overall.histogram = mergeVitalHistograms(
1616
+ overall.histogram,
1617
+ accumulated.histogram,
1618
+ );
1619
+ }
1620
+ return {
1621
+ metric,
1622
+ ...stats(metric, overall),
1623
+ devices: [...(byDevice.get(metric) ?? [])]
1624
+ .map(([device, accumulated]) => ({
1625
+ device,
1626
+ ...stats(metric, accumulated),
1627
+ }))
1628
+ .sort((left, right) => right.count - left.count),
1629
+ pages: [...(byPage.get(metric) ?? [])]
1630
+ .map(([page, accumulated]) => ({
1631
+ page,
1632
+ ...stats(metric, accumulated),
1633
+ }))
1634
+ .sort(
1635
+ (left, right) =>
1636
+ right.count - left.count || left.page.localeCompare(right.page),
1637
+ )
1638
+ .slice(0, VITALS_REPORT_PAGE_LIMIT),
1639
+ timeSeries: [...(byDay.get(metric) ?? [])]
1640
+ .sort(([left], [right]) => left - right)
1641
+ .map(([timestamp, accumulated]) => ({
1642
+ timestamp,
1643
+ count: accumulated.count,
1644
+ p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
1645
+ })),
1646
+ };
1647
+ });
1648
+ return {
1649
+ metrics,
1650
+ truncated: false,
1651
+ metadata: {
1652
+ basis: "dailyRollup" as const,
1653
+ coverage: await coverageForRange(
1654
+ ctx,
1655
+ [args.siteId],
1656
+ args.from,
1657
+ args.to,
1658
+ { datasets: ["vitalRollups"], complete: true },
1659
+ ),
1660
+ range: rangeSemantics(args.from, args.to),
1661
+ },
694
1662
  };
695
1663
  },
696
1664
  });
697
1665
 
1666
+ const coverageDatasetValidator = v.union(
1667
+ v.literal("overviewHour"),
1668
+ v.literal("overviewDay"),
1669
+ v.literal("overviewLocalDay"),
1670
+ v.literal("events"),
1671
+ v.literal("sessions"),
1672
+ v.literal("conversions"),
1673
+ v.literal("goals"),
1674
+ v.literal("funnels"),
1675
+ v.literal("affiliates"),
1676
+ v.literal("vitals"),
1677
+ v.literal("siteMap"),
1678
+ );
1679
+
1680
+ export const dataCoverage = query({
1681
+ args: {
1682
+ siteId: v.id("sites"),
1683
+ from: v.number(),
1684
+ to: v.number(),
1685
+ },
1686
+ returns: v.object({
1687
+ siteId: v.id("sites"),
1688
+ from: v.number(),
1689
+ to: v.number(),
1690
+ datasets: v.array(
1691
+ v.object({
1692
+ dataset: coverageDatasetValidator,
1693
+ coverage: resultCoverageValidator,
1694
+ }),
1695
+ ),
1696
+ }),
1697
+ handler: async (ctx, args) => {
1698
+ validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
1699
+ const datasets = [
1700
+ "overviewHour",
1701
+ "overviewDay",
1702
+ "overviewLocalDay",
1703
+ "events",
1704
+ "sessions",
1705
+ "conversions",
1706
+ "goals",
1707
+ "funnels",
1708
+ "affiliates",
1709
+ "vitals",
1710
+ "siteMap",
1711
+ ] as const;
1712
+ return {
1713
+ siteId: args.siteId,
1714
+ from: args.from,
1715
+ to: args.to,
1716
+ datasets: await Promise.all(
1717
+ datasets.map(async (dataset) => ({
1718
+ dataset,
1719
+ coverage: await datasetCoverageForRange(
1720
+ ctx,
1721
+ args.siteId,
1722
+ dataset,
1723
+ args.from,
1724
+ args.to,
1725
+ ),
1726
+ })),
1727
+ ),
1728
+ };
1729
+ },
1730
+ });
1731
+
1732
+ async function dailyRollupsAvailable(
1733
+ ctx: QueryCtx,
1734
+ siteId: Id<"sites">,
1735
+ from: number,
1736
+ to: number,
1737
+ dataset: "goalRollups" | "funnelRollups" | "affiliateRollups",
1738
+ ) {
1739
+ if (!isAlignedRange(from, to, DAY_MS)) return false;
1740
+ const coverage = await ctx.db
1741
+ .query("analyticsCoverage")
1742
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
1743
+ .unique();
1744
+ if (!coverage || coverage.rollupCompleteFrom > from) {
1745
+ return false;
1746
+ }
1747
+ const retainedBefore = coverage.retained.find(
1748
+ (item) => item.dataset === dataset,
1749
+ )?.before;
1750
+ return retainedBefore === undefined || from >= retainedBefore;
1751
+ }
1752
+
1753
+ async function loadGoalDailyRollups(
1754
+ ctx: QueryCtx,
1755
+ siteId: Id<"sites">,
1756
+ goalIds: Id<"goals">[],
1757
+ from: number,
1758
+ to: number,
1759
+ ) {
1760
+ const days = Math.floor((to - from) / DAY_MS) + 1;
1761
+ requireFeatureRollupBudget(goalIds.length, days, "goal daily rollups");
1762
+ const rows: Doc<"goalDailyRollups">[] = [];
1763
+ for (const goalId of goalIds) {
1764
+ const perGoalLimit = days * REPORT_ROLLUP_SHARDS;
1765
+ const goalRows = await ctx.db
1766
+ .query("goalDailyRollups")
1767
+ .withIndex("by_siteId_and_goalId_and_bucketStart", (range) =>
1768
+ range
1769
+ .eq("siteId", siteId)
1770
+ .eq("goalId", goalId)
1771
+ .gte("bucketStart", from)
1772
+ .lte("bucketStart", to),
1773
+ )
1774
+ .take(perGoalLimit + 1);
1775
+ failIfIncomplete(goalRows, perGoalLimit, "goal daily rollups");
1776
+ rows.push(...goalRows);
1777
+ await requireFeatureReadHeadroom(ctx, "goal daily rollups");
1778
+ }
1779
+ return rows;
1780
+ }
1781
+
1782
+ async function loadFunnelDailyRollups(
1783
+ ctx: QueryCtx,
1784
+ siteId: Id<"sites">,
1785
+ funnelIds: Id<"funnels">[],
1786
+ from: number,
1787
+ to: number,
1788
+ ) {
1789
+ const days = Math.floor((to - from) / DAY_MS) + 1;
1790
+ requireFeatureRollupBudget(funnelIds.length, days, "funnel daily rollups");
1791
+ const rows: Doc<"funnelDailyRollups">[] = [];
1792
+ for (const funnelId of funnelIds) {
1793
+ const perFunnelLimit = days * REPORT_ROLLUP_SHARDS;
1794
+ const funnelRows = await ctx.db
1795
+ .query("funnelDailyRollups")
1796
+ .withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
1797
+ range
1798
+ .eq("siteId", siteId)
1799
+ .eq("funnelId", funnelId)
1800
+ .gte("bucketStart", from)
1801
+ .lte("bucketStart", to),
1802
+ )
1803
+ .take(perFunnelLimit + 1);
1804
+ failIfIncomplete(funnelRows, perFunnelLimit, "funnel daily rollups");
1805
+ rows.push(...funnelRows);
1806
+ await requireFeatureReadHeadroom(ctx, "funnel daily rollups");
1807
+ }
1808
+ return rows;
1809
+ }
1810
+
1811
+ async function loadAffiliateDailyRollups(
1812
+ ctx: QueryCtx,
1813
+ siteId: Id<"sites">,
1814
+ affiliateIds: Id<"affiliates">[],
1815
+ from: number,
1816
+ to: number,
1817
+ ) {
1818
+ const days = Math.floor((to - from) / DAY_MS) + 1;
1819
+ requireFeatureRollupBudget(
1820
+ affiliateIds.length,
1821
+ days,
1822
+ "affiliate daily rollups",
1823
+ );
1824
+ const rows: Doc<"affiliateDailyRollups">[] = [];
1825
+ for (const affiliateId of affiliateIds) {
1826
+ const perAffiliateLimit = days * REPORT_ROLLUP_SHARDS;
1827
+ const affiliateRows = await ctx.db
1828
+ .query("affiliateDailyRollups")
1829
+ .withIndex("by_siteId_and_affiliateId_and_bucketStart", (range) =>
1830
+ range
1831
+ .eq("siteId", siteId)
1832
+ .eq("affiliateId", affiliateId)
1833
+ .gte("bucketStart", from)
1834
+ .lte("bucketStart", to),
1835
+ )
1836
+ .take(perAffiliateLimit + 1);
1837
+ failIfIncomplete(
1838
+ affiliateRows,
1839
+ perAffiliateLimit,
1840
+ "affiliate daily rollups",
1841
+ );
1842
+ rows.push(...affiliateRows);
1843
+ await requireFeatureReadHeadroom(ctx, "affiliate daily rollups");
1844
+ }
1845
+ return rows;
1846
+ }
1847
+
1848
+ function requireFeatureRollupBudget(
1849
+ entities: number,
1850
+ days: number,
1851
+ dataset: string,
1852
+ ) {
1853
+ const maximumRows = entities * days * REPORT_ROLLUP_SHARDS;
1854
+ if (days > MAX_FEATURE_ROLLUP_DAYS || maximumRows > MAX_FEATURE_ROLLUP_ROWS) {
1855
+ fail("REPORT_INCOMPLETE", `${dataset} exceeds its complete read budget`, {
1856
+ dataset,
1857
+ maximumDays: MAX_FEATURE_ROLLUP_DAYS,
1858
+ maximumRows: MAX_FEATURE_ROLLUP_ROWS,
1859
+ });
1860
+ }
1861
+ }
1862
+
1863
+ async function hasFeatureReadHeadroom(ctx: QueryCtx) {
1864
+ const metrics = await ctx.meta.getTransactionMetrics();
1865
+ return (
1866
+ metrics.bytesRead.remaining >= FEATURE_REPORT_BYTE_RESERVE &&
1867
+ metrics.documentsRead.remaining >= FEATURE_REPORT_DOCUMENT_RESERVE
1868
+ );
1869
+ }
1870
+
1871
+ async function requireFeatureReadHeadroom(ctx: QueryCtx, dataset: string) {
1872
+ const metrics = await ctx.meta.getTransactionMetrics();
1873
+ if (
1874
+ metrics.bytesRead.remaining < FEATURE_REPORT_BYTE_RESERVE ||
1875
+ metrics.documentsRead.remaining < FEATURE_REPORT_DOCUMENT_RESERVE
1876
+ ) {
1877
+ fail(
1878
+ "REPORT_INCOMPLETE",
1879
+ `${dataset} exceeds its byte or document budget`,
1880
+ {
1881
+ dataset,
1882
+ bytesRead: metrics.bytesRead.used,
1883
+ documentsRead: metrics.documentsRead.used,
1884
+ },
1885
+ );
1886
+ }
1887
+ }
1888
+
1889
+ function failIfIncomplete<T>(rows: T[], maximum: number, dataset: string) {
1890
+ if (rows.length > maximum) {
1891
+ fail(
1892
+ "REPORT_INCOMPLETE",
1893
+ `${dataset} exceeds the complete report read budget`,
1894
+ {
1895
+ maximumRows: maximum,
1896
+ dataset,
1897
+ },
1898
+ );
1899
+ }
1900
+ }
1901
+
1902
+ async function coverageForRange(
1903
+ ctx: QueryCtx,
1904
+ siteIds: Id<"sites">[],
1905
+ from: number,
1906
+ to: number,
1907
+ options: {
1908
+ datasets: CoverageSource[];
1909
+ complete?: boolean;
1910
+ generation?: 1 | null;
1911
+ },
1912
+ ) {
1913
+ const controls = await Promise.all(
1914
+ siteIds.map((siteId) =>
1915
+ ctx.db
1916
+ .query("analyticsCoverage")
1917
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
1918
+ .unique(),
1919
+ ),
1920
+ );
1921
+ const sourceRanges = await Promise.all(
1922
+ siteIds.map((siteId) => sourceSetRange(ctx, siteId, options.datasets)),
1923
+ );
1924
+ const retainedBefore = controls.reduce<number | null>((latest, control) => {
1925
+ const before = control?.retained
1926
+ .filter((item) =>
1927
+ options.datasets.some((dataset) => dataset === item.dataset),
1928
+ )
1929
+ .reduce<number | undefined>(
1930
+ (latest, item) => Math.max(latest ?? 0, item.before),
1931
+ undefined,
1932
+ );
1933
+ return before === undefined ? latest : Math.max(latest ?? 0, before);
1934
+ }, null);
1935
+ const availableFrom =
1936
+ options.generation === 1
1937
+ ? controls.length > 0 && controls.every(Boolean)
1938
+ ? Math.max(...controls.map((control) => control!.rollupCompleteFrom))
1939
+ : null
1940
+ : sourceRanges.length > 0 && sourceRanges.every(Boolean)
1941
+ ? Math.max(...sourceRanges.map((range) => range!.from))
1942
+ : null;
1943
+ const availableThrough =
1944
+ sourceRanges.length > 0 && sourceRanges.every(Boolean)
1945
+ ? Math.min(...sourceRanges.map((range) => range!.through))
1946
+ : null;
1947
+ let state: "complete" | "partial" | "retained" | "unavailable";
1948
+ if (retainedBefore !== null && to < retainedBefore) {
1949
+ state = "unavailable";
1950
+ } else if (retainedBefore !== null && from < retainedBefore) {
1951
+ state = "partial";
1952
+ } else if (
1953
+ options.complete ||
1954
+ (availableFrom !== null &&
1955
+ availableThrough !== null &&
1956
+ from >= availableFrom &&
1957
+ to <= availableThrough)
1958
+ ) {
1959
+ state = "complete";
1960
+ } else if (sourceRanges.some(Boolean)) {
1961
+ state = "retained";
1962
+ } else {
1963
+ state = "unavailable";
1964
+ }
1965
+ return {
1966
+ state,
1967
+ generation: options.generation ?? null,
1968
+ availableFrom,
1969
+ availableThrough,
1970
+ retainedBefore,
1971
+ };
1972
+ }
1973
+
1974
+ async function assertSourceAvailable(
1975
+ ctx: QueryCtx,
1976
+ siteIds: Id<"sites">[],
1977
+ sources: CoverageSource[],
1978
+ from: number,
1979
+ ) {
1980
+ const rows = await Promise.all(
1981
+ siteIds.map((siteId) =>
1982
+ ctx.db
1983
+ .query("analyticsCoverage")
1984
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
1985
+ .unique(),
1986
+ ),
1987
+ );
1988
+ const cutoff = rows.reduce<number | null>((latest, row) => {
1989
+ for (const retained of row?.retained ?? []) {
1990
+ if (sources.some((source) => source === retained.dataset)) {
1991
+ latest = Math.max(latest ?? 0, retained.before);
1992
+ }
1993
+ }
1994
+ return latest;
1995
+ }, null);
1996
+ if (cutoff !== null && from < cutoff) {
1997
+ fail(
1998
+ "REPORT_INCOMPLETE",
1999
+ "requested report range intersects retained data",
2000
+ {
2001
+ retainedBefore: cutoff,
2002
+ },
2003
+ );
2004
+ }
2005
+ }
2006
+
2007
+ async function datasetCoverageForRange(
2008
+ ctx: QueryCtx,
2009
+ siteId: Id<"sites">,
2010
+ dataset: PublicCoverageDataset,
2011
+ from: number,
2012
+ to: number,
2013
+ ) {
2014
+ if (dataset === "siteMap") {
2015
+ return siteMapDatasetCoverage(ctx, siteId, from, to);
2016
+ }
2017
+ // Vitals have no legacy fallback source; the rollup either covers the range
2018
+ // or the coverage states say how much of it exists.
2019
+ if (dataset === "vitals") {
2020
+ return coverageForRange(ctx, [siteId], from, to, {
2021
+ datasets: ["vitalRollups"],
2022
+ complete: false,
2023
+ });
2024
+ }
2025
+ const sourceOptions: CoverageSource[][] =
2026
+ dataset === "goals"
2027
+ ? [["goalRollups"], ["goalCompletions"]]
2028
+ : dataset === "funnels"
2029
+ ? [["funnelRollups"], ["funnelSteps"]]
2030
+ : dataset === "affiliates"
2031
+ ? [["affiliateRollups"], ["affiliateStats", "sessions"]]
2032
+ : [[dataset]];
2033
+ const options = (
2034
+ await Promise.all(
2035
+ sourceOptions.map(async (sources) => {
2036
+ const rollupDataset = sources[0] as
2037
+ "goalRollups" | "funnelRollups" | "affiliateRollups";
2038
+ const isRollup = rollupDataset.endsWith("Rollups");
2039
+ const complete = isRollup
2040
+ ? await dailyRollupsAvailable(ctx, siteId, from, to, rollupDataset)
2041
+ : false;
2042
+ if (isRollup && !complete) return null;
2043
+ return coverageForRange(ctx, [siteId], from, to, {
2044
+ datasets: sources,
2045
+ complete,
2046
+ generation: complete ? 1 : null,
2047
+ });
2048
+ }),
2049
+ )
2050
+ ).filter((option): option is NonNullable<typeof option> => option !== null);
2051
+ return options.sort(
2052
+ (left, right) => coverageRank(right.state) - coverageRank(left.state),
2053
+ )[0]!;
2054
+ }
2055
+
2056
+ /**
2057
+ * The site map's coverage without reading its rows: it starts at the site's
2058
+ * `siteMapSince` and ends where retention cut it. The report itself may still
2059
+ * narrow the window when the range exceeds its row budget.
2060
+ */
2061
+ async function siteMapDatasetCoverage(
2062
+ ctx: QueryCtx,
2063
+ siteId: Id<"sites">,
2064
+ from: number,
2065
+ to: number,
2066
+ ) {
2067
+ const control = await ctx.db
2068
+ .query("analyticsCoverage")
2069
+ .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
2070
+ .unique();
2071
+ const since = control?.siteMapSince ?? null;
2072
+ const retainedBefore =
2073
+ control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
2074
+ ?.before ?? null;
2075
+ const unavailable =
2076
+ since === null ||
2077
+ to < since ||
2078
+ (retainedBefore !== null && to < retainedBefore);
2079
+ const state = unavailable
2080
+ ? ("unavailable" as const)
2081
+ : from < Math.max(since, retainedBefore ?? 0)
2082
+ ? ("partial" as const)
2083
+ : ("complete" as const);
2084
+ return {
2085
+ state,
2086
+ generation: 1 as const,
2087
+ availableFrom: since,
2088
+ availableThrough: since === null ? null : to,
2089
+ retainedBefore,
2090
+ };
2091
+ }
2092
+
2093
+ type PublicCoverageDataset =
2094
+ | "overviewHour"
2095
+ | "overviewDay"
2096
+ | "overviewLocalDay"
2097
+ | "events"
2098
+ | "sessions"
2099
+ | "conversions"
2100
+ | "goals"
2101
+ | "funnels"
2102
+ | "affiliates"
2103
+ | "vitals"
2104
+ | "siteMap";
2105
+
2106
+ type CoverageSource =
2107
+ | "overviewHour"
2108
+ | "overviewDay"
2109
+ | "overviewLocalDay"
2110
+ | "events"
2111
+ | "sessions"
2112
+ | "conversions"
2113
+ | "goalCompletions"
2114
+ | "funnelSteps"
2115
+ | "affiliateStats"
2116
+ | "goalRollups"
2117
+ | "funnelRollups"
2118
+ | "affiliateRollups"
2119
+ | "vitalRollups";
2120
+
2121
+ type SourceRange = { from: number; through: number };
2122
+
2123
+ async function sourceSetRange(
2124
+ ctx: QueryCtx,
2125
+ siteId: Id<"sites">,
2126
+ sources: CoverageSource[],
2127
+ ): Promise<SourceRange | null> {
2128
+ const ranges = (
2129
+ await Promise.all(sources.map((source) => sourceRange(ctx, siteId, source)))
2130
+ ).filter((range): range is SourceRange => range !== null);
2131
+ if (ranges.length === 0) return null;
2132
+ return {
2133
+ from: Math.min(...ranges.map((range) => range.from)),
2134
+ through: Math.max(...ranges.map((range) => range.through)),
2135
+ };
2136
+ }
2137
+
2138
+ async function sourceRange(
2139
+ ctx: QueryCtx,
2140
+ siteId: Id<"sites">,
2141
+ source: CoverageSource,
2142
+ ): Promise<SourceRange | null> {
2143
+ if (
2144
+ source === "overviewHour" ||
2145
+ source === "overviewDay" ||
2146
+ source === "overviewLocalDay"
2147
+ ) {
2148
+ const granularity =
2149
+ source === "overviewHour"
2150
+ ? ("hour" as const)
2151
+ : source === "overviewDay"
2152
+ ? ("day" as const)
2153
+ : ("localDay" as const);
2154
+ const [first, last] = await Promise.all([
2155
+ ctx.db
2156
+ .query("aggregateBuckets")
2157
+ .withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
2158
+ range.eq("siteId", siteId).eq("granularity", granularity),
2159
+ )
2160
+ .order("asc")
2161
+ .take(1),
2162
+ ctx.db
2163
+ .query("aggregateBuckets")
2164
+ .withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
2165
+ range.eq("siteId", siteId).eq("granularity", granularity),
2166
+ )
2167
+ .order("desc")
2168
+ .take(1),
2169
+ ]);
2170
+ return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
2171
+ }
2172
+ if (source === "events") {
2173
+ const [first, last] = await Promise.all([
2174
+ ctx.db
2175
+ .query("eventBatches")
2176
+ .withIndex("by_siteId_and_bucketStart", (range) =>
2177
+ range.eq("siteId", siteId),
2178
+ )
2179
+ .order("asc")
2180
+ .take(1),
2181
+ ctx.db
2182
+ .query("eventBatches")
2183
+ .withIndex("by_siteId_and_bucketStart", (range) =>
2184
+ range.eq("siteId", siteId),
2185
+ )
2186
+ .order("desc")
2187
+ .take(1),
2188
+ ]);
2189
+ return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
2190
+ }
2191
+ if (source === "sessions") {
2192
+ const [first, last] = await Promise.all([
2193
+ ctx.db
2194
+ .query("sessions")
2195
+ .withIndex("by_siteId_and_startedAt", (range) =>
2196
+ range.eq("siteId", siteId),
2197
+ )
2198
+ .order("asc")
2199
+ .take(1),
2200
+ ctx.db
2201
+ .query("sessions")
2202
+ .withIndex("by_siteId_and_lastSeenAt", (range) =>
2203
+ range.eq("siteId", siteId),
2204
+ )
2205
+ .order("desc")
2206
+ .take(1),
2207
+ ]);
2208
+ return bounds(first[0]?.startedAt, last[0]?.lastSeenAt);
2209
+ }
2210
+ if (source === "conversions") {
2211
+ const [first, last] = await Promise.all([
2212
+ ctx.db
2213
+ .query("conversions")
2214
+ .withIndex("by_siteId_and_timestamp", (range) =>
2215
+ range.eq("siteId", siteId),
2216
+ )
2217
+ .order("asc")
2218
+ .take(1),
2219
+ ctx.db
2220
+ .query("conversions")
2221
+ .withIndex("by_siteId_and_timestamp", (range) =>
2222
+ range.eq("siteId", siteId),
2223
+ )
2224
+ .order("desc")
2225
+ .take(1),
2226
+ ]);
2227
+ return bounds(first[0]?.timestamp, last[0]?.timestamp);
2228
+ }
2229
+ if (source === "goalCompletions") {
2230
+ const [first, last] = await Promise.all([
2231
+ ctx.db
2232
+ .query("goalCompletions")
2233
+ .withIndex("by_siteId_and_timestamp", (range) =>
2234
+ range.eq("siteId", siteId),
2235
+ )
2236
+ .order("asc")
2237
+ .take(1),
2238
+ ctx.db
2239
+ .query("goalCompletions")
2240
+ .withIndex("by_siteId_and_timestamp", (range) =>
2241
+ range.eq("siteId", siteId),
2242
+ )
2243
+ .order("desc")
2244
+ .take(1),
2245
+ ]);
2246
+ return bounds(first[0]?.timestamp, last[0]?.timestamp);
2247
+ }
2248
+ const table =
2249
+ source === "funnelSteps"
2250
+ ? ("funnelStepBuckets" as const)
2251
+ : source === "affiliateStats"
2252
+ ? ("affiliateDailyStats" as const)
2253
+ : source === "goalRollups"
2254
+ ? ("goalDailyRollups" as const)
2255
+ : source === "funnelRollups"
2256
+ ? ("funnelDailyRollups" as const)
2257
+ : source === "vitalRollups"
2258
+ ? ("vitalRollups" as const)
2259
+ : ("affiliateDailyRollups" as const);
2260
+ const [first, last] = await Promise.all([
2261
+ ctx.db
2262
+ .query(table)
2263
+ .withIndex("by_siteId_and_bucketStart", (range) =>
2264
+ range.eq("siteId", siteId),
2265
+ )
2266
+ .order("asc")
2267
+ .take(1),
2268
+ ctx.db
2269
+ .query(table)
2270
+ .withIndex("by_siteId_and_bucketStart", (range) =>
2271
+ range.eq("siteId", siteId),
2272
+ )
2273
+ .order("desc")
2274
+ .take(1),
2275
+ ]);
2276
+ return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
2277
+ }
2278
+
2279
+ function bounds(from: number | undefined, through: number | undefined) {
2280
+ return from === undefined || through === undefined ? null : { from, through };
2281
+ }
2282
+
2283
+ function coverageRank(
2284
+ state: "complete" | "partial" | "retained" | "unavailable",
2285
+ ) {
2286
+ if (state === "complete") return 3;
2287
+ if (state === "retained") return 2;
2288
+ if (state === "partial") return 1;
2289
+ return 0;
2290
+ }
698
2291
 
699
2292
  type MetricTotals = {
700
2293
  events: number;
@@ -706,6 +2299,9 @@ type MetricTotals = {
706
2299
  visitors: number;
707
2300
  conversions: number;
708
2301
  revenueCents: number;
2302
+ durationMs: number;
2303
+ continuedSessions: number;
2304
+ newVisitors: number;
709
2305
  };
710
2306
 
711
2307
  function emptyTotals(): MetricTotals {
@@ -719,10 +2315,23 @@ function emptyTotals(): MetricTotals {
719
2315
  visitors: 0,
720
2316
  conversions: 0,
721
2317
  revenueCents: 0,
2318
+ durationMs: 0,
2319
+ continuedSessions: 0,
2320
+ newVisitors: 0,
722
2321
  };
723
2322
  }
724
2323
 
725
- function addMetrics(target: MetricTotals, source: MetricTotals) {
2324
+ function addMetrics(
2325
+ target: MetricTotals,
2326
+ source: Omit<
2327
+ MetricTotals,
2328
+ "durationMs" | "continuedSessions" | "newVisitors"
2329
+ > & {
2330
+ durationMs?: number;
2331
+ continuedSessions?: number;
2332
+ newVisitors?: number;
2333
+ },
2334
+ ) {
726
2335
  target.events += source.events;
727
2336
  target.pageviews += source.pageviews;
728
2337
  target.clicks += source.clicks;
@@ -732,27 +2341,77 @@ function addMetrics(target: MetricTotals, source: MetricTotals) {
732
2341
  target.visitors += source.visitors;
733
2342
  target.conversions += source.conversions;
734
2343
  target.revenueCents += source.revenueCents;
2344
+ target.durationMs += source.durationMs ?? 0;
2345
+ target.continuedSessions += source.continuedSessions ?? 0;
2346
+ target.newVisitors += source.newVisitors ?? 0;
735
2347
  }
736
2348
 
737
2349
  function validateSiteIds<T>(siteIds: T[]): T[] {
738
2350
  const unique = [...new Set(siteIds)];
739
- if (unique.length < 1 || unique.length > MAX_REPORT_SITES || unique.length !== siteIds.length) {
740
- fail("INVALID_ARGUMENT", `siteIds must contain 1-${MAX_REPORT_SITES} unique IDs`);
2351
+ if (
2352
+ unique.length < 1 ||
2353
+ unique.length > MAX_REPORT_SITES ||
2354
+ unique.length !== siteIds.length
2355
+ ) {
2356
+ fail(
2357
+ "INVALID_ARGUMENT",
2358
+ `siteIds must contain 1-${MAX_REPORT_SITES} unique IDs`,
2359
+ );
741
2360
  }
742
2361
  return unique;
743
2362
  }
744
2363
 
745
2364
  function validateRange(from: number, to: number, maxDays: number) {
746
2365
  if (
747
- !Number.isFinite(from) ||
748
- !Number.isFinite(to) ||
2366
+ !Number.isSafeInteger(from) ||
2367
+ !Number.isSafeInteger(to) ||
749
2368
  from < 0 ||
750
- to <= from ||
751
- to - from > maxDays * DAY_MS
2369
+ to < from ||
2370
+ to - from >= maxDays * DAY_MS
752
2371
  ) {
753
- fail("INVALID_ARGUMENT", `range must be positive, ordered, and at most ${maxDays} days`);
2372
+ fail(
2373
+ "INVALID_ARGUMENT",
2374
+ `range must use inclusive integer milliseconds and span at most ${maxDays} UTC days`,
2375
+ );
754
2376
  }
755
- return { from: Math.trunc(from), to: Math.trunc(to) };
2377
+ return { from, to };
2378
+ }
2379
+
2380
+ function isAlignedRange(from: number, to: number, intervalMs: number) {
2381
+ return from % intervalMs === 0 && (to + 1) % intervalMs === 0;
2382
+ }
2383
+
2384
+ function requireAlignedRange(
2385
+ from: number,
2386
+ to: number,
2387
+ intervalMs: number,
2388
+ report: string,
2389
+ siteTimezone?: string,
2390
+ ) {
2391
+ if (!isAlignedRange(from, to, intervalMs)) {
2392
+ fail(
2393
+ "REPORT_INCOMPLETE",
2394
+ siteTimezone
2395
+ ? `${report} requires complete inclusive UTC bucket boundaries, or whole calendar days in ${siteTimezone}`
2396
+ : `${report} requires complete inclusive UTC bucket boundaries`,
2397
+ {
2398
+ from,
2399
+ to,
2400
+ intervalMs,
2401
+ timezone: "UTC",
2402
+ siteTimezone: siteTimezone ?? null,
2403
+ },
2404
+ );
2405
+ }
2406
+ }
2407
+
2408
+ function rangeSemantics(from: number, to: number, timezone = "UTC") {
2409
+ return {
2410
+ from,
2411
+ to,
2412
+ boundary: "inclusive" as const,
2413
+ timezone,
2414
+ };
756
2415
  }
757
2416
 
758
2417
  type KeysetPaginationInput = {
@@ -881,22 +2540,23 @@ function keysetPaginationResult<T>(
881
2540
  ) {
882
2541
  const page = rows.slice(0, pagination.numItems);
883
2542
  const isDone = rows.length <= pagination.numItems;
884
- const next = page.length > 0
885
- ? cursorFor(page[page.length - 1])
886
- : pagination.cursor;
2543
+ const next =
2544
+ page.length > 0 ? cursorFor(page[page.length - 1]) : pagination.cursor;
887
2545
  return {
888
2546
  page,
889
2547
  isDone,
890
- continueCursor: btoa(JSON.stringify({
891
- v: 2,
892
- k: pagination.scope.kind,
893
- s: pagination.scope.siteId,
894
- n: pagination.scope.sessionId ?? null,
895
- f: pagination.scope.from,
896
- t: pagination.scope.to,
897
- x: next?.timestamp ?? pagination.scope.to,
898
- c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
899
- })),
2548
+ continueCursor: btoa(
2549
+ JSON.stringify({
2550
+ v: 2,
2551
+ k: pagination.scope.kind,
2552
+ s: pagination.scope.siteId,
2553
+ n: pagination.scope.sessionId ?? null,
2554
+ f: pagination.scope.from,
2555
+ t: pagination.scope.to,
2556
+ x: next?.timestamp ?? pagination.scope.to,
2557
+ c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
2558
+ }),
2559
+ ),
900
2560
  };
901
2561
  }
902
2562
 
@@ -908,26 +2568,28 @@ function journeyPaginationResult(
908
2568
  const isDone = rows.length <= pagination.numItems;
909
2569
  const last = page.at(-1);
910
2570
  const next = last
911
- ? {
912
- timestamp: last.timestamp,
913
- creationTime: last._creationTime,
914
- id: last._id,
2571
+ ? {
2572
+ timestamp: last.timestamp,
2573
+ creationTime: last._creationTime,
2574
+ id: last._id,
915
2575
  }
916
2576
  : pagination.cursor;
917
2577
  return {
918
2578
  page,
919
2579
  isDone,
920
- continueCursor: btoa(JSON.stringify({
921
- v: 4,
922
- k: pagination.scope.kind,
923
- s: pagination.scope.siteId,
924
- n: pagination.scope.sessionId,
925
- f: pagination.scope.from,
926
- t: pagination.scope.to,
927
- x: next?.timestamp ?? pagination.scope.to,
928
- c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
929
- i: next?.id ?? "~",
930
- })),
2580
+ continueCursor: btoa(
2581
+ JSON.stringify({
2582
+ v: 4,
2583
+ k: pagination.scope.kind,
2584
+ s: pagination.scope.siteId,
2585
+ n: pagination.scope.sessionId,
2586
+ f: pagination.scope.from,
2587
+ t: pagination.scope.to,
2588
+ x: next?.timestamp ?? pagination.scope.to,
2589
+ c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
2590
+ i: next?.id ?? "~",
2591
+ }),
2592
+ ),
931
2593
  };
932
2594
  }
933
2595
 
@@ -936,8 +2598,15 @@ function validatePagination(options: {
936
2598
  maximumRowsRead?: number;
937
2599
  maximumBytesRead?: number;
938
2600
  }) {
939
- if (!Number.isSafeInteger(options.numItems) || options.numItems < 1 || options.numItems > 100) {
940
- fail("INVALID_ARGUMENT", "pagination numItems must be an integer between 1 and 100");
2601
+ if (
2602
+ !Number.isSafeInteger(options.numItems) ||
2603
+ options.numItems < 1 ||
2604
+ options.numItems > 100
2605
+ ) {
2606
+ fail(
2607
+ "INVALID_ARGUMENT",
2608
+ "pagination numItems must be an integer between 1 and 100",
2609
+ );
941
2610
  }
942
2611
  if (
943
2612
  options.maximumRowsRead !== undefined &&
@@ -945,7 +2614,10 @@ function validatePagination(options: {
945
2614
  options.maximumRowsRead < 1 ||
946
2615
  options.maximumRowsRead > 1_000)
947
2616
  ) {
948
- fail("INVALID_ARGUMENT", "maximumRowsRead must be an integer between 1 and 1000");
2617
+ fail(
2618
+ "INVALID_ARGUMENT",
2619
+ "maximumRowsRead must be an integer between 1 and 1000",
2620
+ );
949
2621
  }
950
2622
  if (
951
2623
  options.maximumBytesRead !== undefined &&
@@ -953,6 +2625,9 @@ function validatePagination(options: {
953
2625
  options.maximumBytesRead < 1 ||
954
2626
  options.maximumBytesRead > 8_000_000)
955
2627
  ) {
956
- fail("INVALID_ARGUMENT", "maximumBytesRead must be an integer between 1 and 8000000");
2628
+ fail(
2629
+ "INVALID_ARGUMENT",
2630
+ "maximumBytesRead must be an integer between 1 and 8000000",
2631
+ );
957
2632
  }
958
2633
  }