@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +693 -69
- package/agent/integration.md +801 -0
- package/agent/manifest.json +205 -0
- package/agent/manifest.schema.json +444 -0
- package/dist/client/federation.d.ts +381 -0
- package/dist/client/federation.d.ts.map +1 -0
- package/dist/client/federation.js +274 -0
- package/dist/client/federation.js.map +1 -0
- package/dist/client/index.d.ts +2673 -15
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +565 -33
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +22 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +315 -4
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/server.d.ts +4 -0
- package/dist/component/_generated/server.d.ts.map +1 -1
- package/dist/component/_generated/server.js.map +1 -1
- package/dist/component/affiliates.d.ts.map +1 -1
- package/dist/component/affiliates.js +6 -2
- package/dist/component/affiliates.js.map +1 -1
- package/dist/component/cardinality.d.ts +12 -0
- package/dist/component/cardinality.d.ts.map +1 -0
- package/dist/component/cardinality.js +94 -0
- package/dist/component/cardinality.js.map +1 -0
- package/dist/component/constants.d.ts +28 -1
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +44 -1
- package/dist/component/constants.js.map +1 -1
- package/dist/component/convex.config.d.ts +6 -1
- package/dist/component/convex.config.js +9 -1
- package/dist/component/convex.config.js.map +1 -1
- package/dist/component/coverage.d.ts +27 -0
- package/dist/component/coverage.d.ts.map +1 -0
- package/dist/component/coverage.js +56 -0
- package/dist/component/coverage.js.map +1 -0
- package/dist/component/diagnostics.d.ts +9 -0
- package/dist/component/diagnostics.d.ts.map +1 -0
- package/dist/component/diagnostics.js +47 -0
- package/dist/component/diagnostics.js.map +1 -0
- package/dist/component/errors.d.ts +1 -1
- package/dist/component/errors.d.ts.map +1 -1
- package/dist/component/errors.js.map +1 -1
- package/dist/component/eventStore.d.ts +21 -9
- package/dist/component/eventStore.d.ts.map +1 -1
- package/dist/component/eventStore.js +142 -152
- package/dist/component/eventStore.js.map +1 -1
- package/dist/component/funnels.d.ts.map +1 -1
- package/dist/component/funnels.js +5 -3
- package/dist/component/funnels.js.map +1 -1
- package/dist/component/geo.d.ts +73 -0
- package/dist/component/geo.d.ts.map +1 -0
- package/dist/component/geo.js +648 -0
- package/dist/component/geo.js.map +1 -0
- package/dist/component/goals.d.ts.map +1 -1
- package/dist/component/goals.js +8 -5
- package/dist/component/goals.js.map +1 -1
- package/dist/component/guards.d.ts.map +1 -1
- package/dist/component/guards.js.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +281 -62
- package/dist/component/http.js.map +1 -1
- package/dist/component/identity.d.ts +13 -0
- package/dist/component/identity.d.ts.map +1 -0
- package/dist/component/identity.js +58 -0
- package/dist/component/identity.js.map +1 -0
- package/dist/component/ingest.d.ts +3 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +498 -95
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/live.d.ts.map +1 -1
- package/dist/component/live.js +7 -6
- package/dist/component/live.js.map +1 -1
- package/dist/component/localTime.d.ts +25 -0
- package/dist/component/localTime.d.ts.map +1 -0
- package/dist/component/localTime.js +126 -0
- package/dist/component/localTime.js.map +1 -0
- package/dist/component/reports.d.ts +266 -10
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +1243 -137
- package/dist/component/reports.js.map +1 -1
- package/dist/component/retention.d.ts +75 -1
- package/dist/component/retention.d.ts.map +1 -1
- package/dist/component/retention.js +561 -38
- package/dist/component/retention.js.map +1 -1
- package/dist/component/rollupStore.d.ts +320 -0
- package/dist/component/rollupStore.d.ts.map +1 -0
- package/dist/component/rollupStore.js +596 -0
- package/dist/component/rollupStore.js.map +1 -0
- package/dist/component/rollups.d.ts +20 -0
- package/dist/component/rollups.d.ts.map +1 -0
- package/dist/component/rollups.js +73 -0
- package/dist/component/rollups.js.map +1 -0
- package/dist/component/sanitize.d.ts +24 -1
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +96 -16
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +687 -65
- package/dist/component/schema.js +187 -20
- package/dist/component/schema.js.map +1 -1
- package/dist/component/sites.d.ts +12 -0
- package/dist/component/sites.d.ts.map +1 -1
- package/dist/component/sites.js +41 -7
- package/dist/component/sites.js.map +1 -1
- package/dist/component/useragent.d.ts +9 -0
- package/dist/component/useragent.d.ts.map +1 -0
- package/dist/component/useragent.js +152 -0
- package/dist/component/useragent.js.map +1 -0
- package/dist/component/validators.d.ts +124 -69
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +35 -14
- package/dist/component/validators.js.map +1 -1
- package/dist/component/visitors.d.ts +19 -0
- package/dist/component/visitors.d.ts.map +1 -0
- package/dist/component/visitors.js +86 -0
- package/dist/component/visitors.js.map +1 -0
- package/dist/component/vitals.d.ts +41 -0
- package/dist/component/vitals.d.ts.map +1 -0
- package/dist/component/vitals.js +115 -0
- package/dist/component/vitals.js.map +1 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js.map +1 -1
- package/dist/tracker/generated.d.ts +11 -4
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +11 -4
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.d.ts +1 -1
- package/dist/tracker/tracker.d.ts.map +1 -1
- package/dist/tracker/tracker.js +57 -20
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker/vitals.d.ts +10 -0
- package/dist/tracker/vitals.d.ts.map +1 -0
- package/dist/tracker/vitals.js +140 -0
- package/dist/tracker/vitals.js.map +1 -0
- package/dist/tracker.min.js +1 -1
- package/dist/vitals.min.js +1 -0
- package/docs/benchmarks/2026-08-20-realistic.md +76 -76
- package/docs/benchmarks/2026-08-21-formal-certification.md +353 -0
- package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
- package/docs/federation-setup.md +464 -0
- package/docs/federation.md +352 -0
- package/docs/upgrading.md +344 -0
- package/llms.txt +72 -0
- package/package.json +55 -11
- package/scripts/benchmark-ingest.mjs +175 -73
- package/scripts/generate-federation-keys.mjs +20 -0
- package/src/component/_generated/api.ts +22 -0
- package/src/component/_generated/component.ts +381 -4
- package/src/component/_generated/server.ts +4 -0
- package/src/component/affiliates.ts +20 -5
- package/src/component/cardinality.ts +117 -0
- package/src/component/constants.ts +44 -1
- package/src/component/convex.config.ts +11 -1
- package/src/component/coverage.ts +71 -0
- package/src/component/diagnostics.ts +65 -0
- package/src/component/errors.ts +2 -1
- package/src/component/eventStore.ts +217 -193
- package/src/component/funnels.ts +19 -16
- package/src/component/geo.ts +835 -0
- package/src/component/goals.ts +29 -21
- package/src/component/guards.ts +3 -1
- package/src/component/http.ts +404 -70
- package/src/component/identity.ts +74 -0
- package/src/component/ingest.ts +894 -188
- package/src/component/live.ts +13 -7
- package/src/component/localTime.ts +167 -0
- package/src/component/reports.ts +1872 -197
- package/src/component/retention.ts +788 -96
- package/src/component/rollupStore.ts +799 -0
- package/src/component/rollups.ts +82 -0
- package/src/component/sanitize.ts +144 -29
- package/src/component/schema.ts +217 -21
- package/src/component/sites.ts +59 -12
- package/src/component/useragent.ts +171 -0
- package/src/component/validators.ts +49 -14
- package/src/component/visitors.ts +116 -0
- package/src/component/vitals.ts +146 -0
package/src/component/reports.ts
CHANGED
|
@@ -3,28 +3,55 @@ import {
|
|
|
3
3
|
paginationResultValidator,
|
|
4
4
|
} from "convex/server";
|
|
5
5
|
import { v } from "convex/values";
|
|
6
|
-
import type { Doc } from "./_generated/dataModel.js";
|
|
6
|
+
import type { Doc, Id } from "./_generated/dataModel.js";
|
|
7
|
+
import type { QueryCtx } from "./_generated/server.js";
|
|
7
8
|
import { query } from "./_generated/server.js";
|
|
8
9
|
import {
|
|
9
10
|
AGGREGATE_SHARDS,
|
|
10
11
|
DAY_MS,
|
|
12
|
+
FEATURE_REPORT_BYTE_RESERVE,
|
|
13
|
+
FEATURE_REPORT_DOCUMENT_RESERVE,
|
|
11
14
|
HOUR_MS,
|
|
12
15
|
HOURLY_AGGREGATE_SHARDS,
|
|
13
16
|
MAX_AFFILIATES_PER_SITE,
|
|
14
17
|
MAX_FUNNELS_PER_SITE,
|
|
18
|
+
MAX_FEATURE_ROLLUP_DAYS,
|
|
19
|
+
MAX_FEATURE_ROLLUP_ROWS,
|
|
15
20
|
MAX_GOALS_PER_SITE,
|
|
16
21
|
MAX_JOURNEY_EVENTS,
|
|
17
22
|
MAX_LIVE_VISITORS,
|
|
18
23
|
MAX_OVERVIEW_AGGREGATE_ROWS,
|
|
24
|
+
MAX_REPORT_BUCKETS,
|
|
19
25
|
MAX_REPORT_RANGE_DAYS,
|
|
20
26
|
MAX_REPORT_SITES,
|
|
27
|
+
MAX_SITE_MAP_ROUTES_PER_DAY,
|
|
28
|
+
MAX_SITE_MAP_ROWS,
|
|
29
|
+
MAX_VITALS_REPORT_ROWS,
|
|
30
|
+
REPORT_ROLLUP_SHARDS,
|
|
31
|
+
ROUTE_TRANSITION_SLOTS,
|
|
32
|
+
SITE_MAP_OTHER_ROUTE,
|
|
33
|
+
VITALS_REPORT_PAGE_LIMIT,
|
|
21
34
|
} from "./constants.js";
|
|
35
|
+
import {
|
|
36
|
+
createVisitorSketch,
|
|
37
|
+
estimateVisitors,
|
|
38
|
+
unionVisitorSketches,
|
|
39
|
+
type VisitorSketch,
|
|
40
|
+
} from "./cardinality.js";
|
|
22
41
|
import { fail } from "./errors.js";
|
|
42
|
+
import {
|
|
43
|
+
isLocalDayAligned,
|
|
44
|
+
localDayStart,
|
|
45
|
+
localDayStarts,
|
|
46
|
+
localDayTimezone,
|
|
47
|
+
sharedLocalTimezone,
|
|
48
|
+
} from "./localTime.js";
|
|
23
49
|
import {
|
|
24
50
|
compareJourneyEvents,
|
|
25
51
|
hydrateEventContexts,
|
|
26
52
|
type JourneyCursor,
|
|
27
53
|
loadSessionJourneyEvents,
|
|
54
|
+
loadVisitorJourneyEvents,
|
|
28
55
|
type RawJourneyEvent,
|
|
29
56
|
} from "./eventStore.js";
|
|
30
57
|
import { isPlainRecord } from "./guards.js";
|
|
@@ -34,7 +61,17 @@ import {
|
|
|
34
61
|
funnelFieldsValidator,
|
|
35
62
|
goalFieldsValidator,
|
|
36
63
|
sessionFieldsValidator,
|
|
64
|
+
vitalMetricValidator,
|
|
37
65
|
} from "./validators.js";
|
|
66
|
+
import { resolveVisitorIdentities } from "./visitors.js";
|
|
67
|
+
import {
|
|
68
|
+
mergeVitalHistograms,
|
|
69
|
+
VITAL_ALL,
|
|
70
|
+
VITAL_METRICS,
|
|
71
|
+
vitalHistogramPercentile,
|
|
72
|
+
vitalRatingCounts,
|
|
73
|
+
type VitalMetric,
|
|
74
|
+
} from "./vitals.js";
|
|
38
75
|
|
|
39
76
|
const sessionDocumentValidator = sessionFieldsValidator.extend({
|
|
40
77
|
_id: v.id("sessions"),
|
|
@@ -64,6 +101,8 @@ const liveSessionDocumentValidator = v.object({
|
|
|
64
101
|
device: v.string(),
|
|
65
102
|
pageviewCount: v.number(),
|
|
66
103
|
eventCount: v.number(),
|
|
104
|
+
previousPath: v.optional(v.string()),
|
|
105
|
+
movedAt: v.optional(v.number()),
|
|
67
106
|
});
|
|
68
107
|
const conversionDocumentValidator = v.object({
|
|
69
108
|
_id: v.id("conversions"),
|
|
@@ -94,13 +133,54 @@ const metricTotalsValidator = v.object({
|
|
|
94
133
|
visitors: v.number(),
|
|
95
134
|
conversions: v.number(),
|
|
96
135
|
revenueCents: v.number(),
|
|
136
|
+
// Engagement: summed session duration, sessions that reached a second
|
|
137
|
+
// pageview, and first-ever sessions of a visitor. Zero for buckets older
|
|
138
|
+
// than the counters; metadata.engagement says whether the range is covered.
|
|
139
|
+
durationMs: v.number(),
|
|
140
|
+
continuedSessions: v.number(),
|
|
141
|
+
newVisitors: v.number(),
|
|
142
|
+
});
|
|
143
|
+
const timePointValidator = metricTotalsValidator.extend({
|
|
144
|
+
timestamp: v.number(),
|
|
97
145
|
});
|
|
98
|
-
const timePointValidator = metricTotalsValidator.extend({ timestamp: v.number() });
|
|
99
146
|
const topItemValidator = v.object({
|
|
100
147
|
value: v.string(),
|
|
101
148
|
count: v.number(),
|
|
102
149
|
revenueCents: v.number(),
|
|
103
150
|
});
|
|
151
|
+
// What a breakdown's `count` is: pageviews for pages and sessions for the
|
|
152
|
+
// other dimensions once every bucket in the range carries those counters,
|
|
153
|
+
// raw event volume for ranges that reach back before they existed.
|
|
154
|
+
const breakdownBasisValidator = v.union(
|
|
155
|
+
v.literal("pageviews"),
|
|
156
|
+
v.literal("sessions"),
|
|
157
|
+
v.literal("eventVolume"),
|
|
158
|
+
);
|
|
159
|
+
const coverageStateValidator = v.union(
|
|
160
|
+
v.literal("complete"),
|
|
161
|
+
v.literal("partial"),
|
|
162
|
+
v.literal("retained"),
|
|
163
|
+
v.literal("unavailable"),
|
|
164
|
+
);
|
|
165
|
+
const resultCoverageValidator = v.object({
|
|
166
|
+
state: coverageStateValidator,
|
|
167
|
+
generation: v.union(v.number(), v.null()),
|
|
168
|
+
availableFrom: v.union(v.number(), v.null()),
|
|
169
|
+
availableThrough: v.union(v.number(), v.null()),
|
|
170
|
+
retainedBefore: v.union(v.number(), v.null()),
|
|
171
|
+
});
|
|
172
|
+
const rangeSemanticsValidator = v.object({
|
|
173
|
+
from: v.number(),
|
|
174
|
+
to: v.number(),
|
|
175
|
+
boundary: v.literal("inclusive"),
|
|
176
|
+
// "UTC", or the site timezone whose calendar days a daily overview used.
|
|
177
|
+
timezone: v.string(),
|
|
178
|
+
});
|
|
179
|
+
const reportMetadataValidator = v.object({
|
|
180
|
+
basis: v.union(v.literal("dailyRollup"), v.literal("boundedLegacy")),
|
|
181
|
+
coverage: resultCoverageValidator,
|
|
182
|
+
range: rangeSemanticsValidator,
|
|
183
|
+
});
|
|
104
184
|
|
|
105
185
|
export const overview = query({
|
|
106
186
|
args: {
|
|
@@ -117,20 +197,58 @@ export const overview = query({
|
|
|
117
197
|
totals: metricTotalsValidator,
|
|
118
198
|
timeSeries: v.array(timePointValidator),
|
|
119
199
|
topSources: v.array(topItemValidator),
|
|
200
|
+
topCampaigns: v.array(topItemValidator),
|
|
120
201
|
topPages: v.array(topItemValidator),
|
|
121
202
|
topCountries: v.array(topItemValidator),
|
|
122
203
|
topDevices: v.array(topItemValidator),
|
|
123
204
|
topBrowsers: v.array(topItemValidator),
|
|
124
205
|
topEvents: v.array(topItemValidator),
|
|
125
206
|
topGoals: v.array(topItemValidator),
|
|
207
|
+
metadata: v.object({
|
|
208
|
+
visitors: v.object({
|
|
209
|
+
basis: v.union(
|
|
210
|
+
v.literal("estimatedUnique"),
|
|
211
|
+
v.literal("bucketOccurrences"),
|
|
212
|
+
),
|
|
213
|
+
exact: v.boolean(),
|
|
214
|
+
legacyIncomplete: v.boolean(),
|
|
215
|
+
sketchVersion: v.union(v.number(), v.null()),
|
|
216
|
+
precision: v.union(v.number(), v.null()),
|
|
217
|
+
}),
|
|
218
|
+
breakdowns: v.object({
|
|
219
|
+
topSources: breakdownBasisValidator,
|
|
220
|
+
topCampaigns: breakdownBasisValidator,
|
|
221
|
+
topPages: breakdownBasisValidator,
|
|
222
|
+
topCountries: breakdownBasisValidator,
|
|
223
|
+
topDevices: breakdownBasisValidator,
|
|
224
|
+
topBrowsers: breakdownBasisValidator,
|
|
225
|
+
topEvents: v.literal("eventVolume"),
|
|
226
|
+
topGoals: v.literal("goalCompletionVolume"),
|
|
227
|
+
}),
|
|
228
|
+
// "counted" when every bucket in the range carries the engagement
|
|
229
|
+
// counters, so bounce rate, duration, pages per session and new
|
|
230
|
+
// visitors can be derived from the totals; "unavailable" otherwise.
|
|
231
|
+
engagement: v.union(v.literal("counted"), v.literal("unavailable")),
|
|
232
|
+
coverage: resultCoverageValidator,
|
|
233
|
+
range: rangeSemanticsValidator,
|
|
234
|
+
features: v.object({
|
|
235
|
+
sessionReplay: v.literal("unsupported"),
|
|
236
|
+
errorInsights: v.literal("unsupported"),
|
|
237
|
+
outboundLinks: v.literal("eventVolume"),
|
|
238
|
+
botDetection: v.literal("userAgent"),
|
|
239
|
+
}),
|
|
240
|
+
}),
|
|
126
241
|
}),
|
|
127
242
|
handler: async (ctx, args) => {
|
|
128
243
|
const siteIds = validateSiteIds(args.siteIds);
|
|
129
244
|
const currencies = new Set<string>();
|
|
245
|
+
const siteTimezones: Array<string | undefined> = [];
|
|
130
246
|
for (const siteId of siteIds) {
|
|
131
247
|
const site = await ctx.db.get("sites", siteId);
|
|
132
|
-
if (!site)
|
|
248
|
+
if (!site)
|
|
249
|
+
fail("NOT_FOUND", "site not found", { siteId: String(siteId) });
|
|
133
250
|
currencies.add(site.currency ?? "USD");
|
|
251
|
+
siteTimezones.push(localDayTimezone(site.timezone));
|
|
134
252
|
}
|
|
135
253
|
if (currencies.size !== 1) {
|
|
136
254
|
fail("CURRENCY_MISMATCH", "overview sites must use the same currency", {
|
|
@@ -138,16 +256,76 @@ export const overview = query({
|
|
|
138
256
|
});
|
|
139
257
|
}
|
|
140
258
|
const currency = [...currencies][0];
|
|
141
|
-
const { from, to } = validateRange(
|
|
259
|
+
const { from, to } = validateRange(
|
|
260
|
+
args.from,
|
|
261
|
+
args.to,
|
|
262
|
+
MAX_REPORT_RANGE_DAYS,
|
|
263
|
+
);
|
|
142
264
|
const interval = args.interval ?? (to - from <= DAY_MS ? "hour" : "day");
|
|
143
265
|
if (interval === "hour" && to - from > DAY_MS) {
|
|
144
266
|
fail("INVALID_ARGUMENT", "hourly reports are limited to 24 hours");
|
|
145
267
|
}
|
|
146
268
|
const intervalMs = interval === "hour" ? HOUR_MS : DAY_MS;
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
const
|
|
269
|
+
// Sites that share a timezone answer a daily range that covers whole
|
|
270
|
+
// calendar days in that zone from their local-day buckets; any other
|
|
271
|
+
// daily range must cover whole UTC days, as before.
|
|
272
|
+
const siteTimezone =
|
|
273
|
+
interval === "day" ? sharedLocalTimezone(siteTimezones) : undefined;
|
|
274
|
+
const localDays =
|
|
275
|
+
siteTimezone !== undefined && isLocalDayAligned(from, to, siteTimezone);
|
|
276
|
+
if (!localDays) {
|
|
277
|
+
requireAlignedRange(
|
|
278
|
+
from,
|
|
279
|
+
to,
|
|
280
|
+
intervalMs,
|
|
281
|
+
`${interval} overview`,
|
|
282
|
+
siteTimezone,
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
const granularity: "hour" | "day" | "localDay" = localDays
|
|
286
|
+
? "localDay"
|
|
287
|
+
: interval;
|
|
288
|
+
const controls = await Promise.all(
|
|
289
|
+
siteIds.map((siteId) =>
|
|
290
|
+
ctx.db
|
|
291
|
+
.query("analyticsCoverage")
|
|
292
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
293
|
+
.unique(),
|
|
294
|
+
),
|
|
295
|
+
);
|
|
296
|
+
// Local-day buckets are complete from each site's stamp (its first ingest
|
|
297
|
+
// after the upgrade or after a timezone change, plus the safety delay).
|
|
298
|
+
// Buckets before it belong to another zone's days or to no complete day
|
|
299
|
+
// and are left out; the coverage says since when the days are whole.
|
|
300
|
+
let localDaysFrom = Number.POSITIVE_INFINITY;
|
|
301
|
+
if (localDays) {
|
|
302
|
+
localDaysFrom = controls.reduce(
|
|
303
|
+
(latest, control) =>
|
|
304
|
+
control?.localDaySince === undefined
|
|
305
|
+
? Number.POSITIVE_INFINITY
|
|
306
|
+
: Math.max(latest, control.localDaySince),
|
|
307
|
+
0,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
const bucketStarts = localDays
|
|
311
|
+
? localDayStarts(from, to, siteTimezone!)
|
|
312
|
+
: [];
|
|
313
|
+
const bucketStartSet = new Set(bucketStarts);
|
|
314
|
+
const localDaysFirstBucket =
|
|
315
|
+
localDays && Number.isFinite(localDaysFrom)
|
|
316
|
+
? localDayStart(localDaysFrom, siteTimezone!)
|
|
317
|
+
: Number.POSITIVE_INFINITY;
|
|
318
|
+
const firstBucket = localDays
|
|
319
|
+
? bucketStarts[0]!
|
|
320
|
+
: Math.floor(from / intervalMs) * intervalMs;
|
|
321
|
+
const lastBucket = localDays
|
|
322
|
+
? bucketStarts[bucketStarts.length - 1]!
|
|
323
|
+
: Math.floor(to / intervalMs) * intervalMs;
|
|
324
|
+
const bucketCount = localDays
|
|
325
|
+
? bucketStarts.length
|
|
326
|
+
: Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
|
|
327
|
+
const perSiteLimit =
|
|
328
|
+
bucketCount *
|
|
151
329
|
(interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
|
|
152
330
|
if (perSiteLimit * siteIds.length > MAX_OVERVIEW_AGGREGATE_ROWS) {
|
|
153
331
|
fail(
|
|
@@ -158,7 +336,28 @@ export const overview = query({
|
|
|
158
336
|
}
|
|
159
337
|
const totals = emptyTotals();
|
|
160
338
|
const points = new Map<number, MetricTotals>();
|
|
161
|
-
const
|
|
339
|
+
const visitorSketch = createVisitorSketch();
|
|
340
|
+
let mergedVisitorSketch = visitorSketch;
|
|
341
|
+
const pointVisitorSketches = new Map<number, VisitorSketch>();
|
|
342
|
+
let visitorsComplete = true;
|
|
343
|
+
const dimensions = new Map<
|
|
344
|
+
string,
|
|
345
|
+
{
|
|
346
|
+
value: string;
|
|
347
|
+
count: number;
|
|
348
|
+
revenueCents: number;
|
|
349
|
+
pageviews: number;
|
|
350
|
+
sessions: number;
|
|
351
|
+
}
|
|
352
|
+
>();
|
|
353
|
+
const overviewDataset =
|
|
354
|
+
granularity === "hour"
|
|
355
|
+
? ("overviewHour" as const)
|
|
356
|
+
: granularity === "day"
|
|
357
|
+
? ("overviewDay" as const)
|
|
358
|
+
: ("overviewLocalDay" as const);
|
|
359
|
+
|
|
360
|
+
await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
|
|
162
361
|
|
|
163
362
|
for (const siteId of siteIds) {
|
|
164
363
|
const buckets = await ctx.db
|
|
@@ -166,36 +365,120 @@ export const overview = query({
|
|
|
166
365
|
.withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
|
|
167
366
|
range
|
|
168
367
|
.eq("siteId", siteId)
|
|
169
|
-
.eq("granularity",
|
|
368
|
+
.eq("granularity", granularity)
|
|
170
369
|
.gte("bucketStart", firstBucket)
|
|
171
370
|
.lte("bucketStart", lastBucket),
|
|
172
371
|
)
|
|
173
|
-
.take(perSiteLimit);
|
|
372
|
+
.take(perSiteLimit + 1);
|
|
373
|
+
failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
|
|
174
374
|
for (const bucket of buckets) {
|
|
375
|
+
if (
|
|
376
|
+
localDays &&
|
|
377
|
+
(!bucketStartSet.has(bucket.bucketStart) ||
|
|
378
|
+
bucket.bucketStart < localDaysFirstBucket)
|
|
379
|
+
) {
|
|
380
|
+
continue;
|
|
381
|
+
}
|
|
175
382
|
addMetrics(totals, bucket);
|
|
176
383
|
const point = points.get(bucket.bucketStart) ?? emptyTotals();
|
|
177
384
|
addMetrics(point, bucket);
|
|
178
385
|
points.set(bucket.bucketStart, point);
|
|
386
|
+
if (bucket.visitorSketch && bucket.visitorSketchComplete === true) {
|
|
387
|
+
mergedVisitorSketch = unionVisitorSketches(
|
|
388
|
+
mergedVisitorSketch,
|
|
389
|
+
bucket.visitorSketch,
|
|
390
|
+
);
|
|
391
|
+
pointVisitorSketches.set(
|
|
392
|
+
bucket.bucketStart,
|
|
393
|
+
unionVisitorSketches(
|
|
394
|
+
pointVisitorSketches.get(bucket.bucketStart) ??
|
|
395
|
+
createVisitorSketch(bucket.visitorSketch.precision),
|
|
396
|
+
bucket.visitorSketch,
|
|
397
|
+
),
|
|
398
|
+
);
|
|
399
|
+
} else if (bucket.events > 0) {
|
|
400
|
+
visitorsComplete = false;
|
|
401
|
+
}
|
|
179
402
|
for (const dimension of bucket.dimensions) {
|
|
180
403
|
const key = `${dimension.type}:${dimension.value}`;
|
|
181
404
|
const current = dimensions.get(key) ?? {
|
|
182
405
|
value: dimension.value,
|
|
183
406
|
count: 0,
|
|
184
407
|
revenueCents: 0,
|
|
408
|
+
pageviews: 0,
|
|
409
|
+
sessions: 0,
|
|
185
410
|
};
|
|
186
411
|
current.count += dimension.count;
|
|
187
412
|
current.revenueCents += dimension.revenueCents;
|
|
413
|
+
current.pageviews += dimension.pageviews ?? 0;
|
|
414
|
+
current.sessions += dimension.sessions ?? 0;
|
|
188
415
|
dimensions.set(key, current);
|
|
189
416
|
}
|
|
190
417
|
}
|
|
191
418
|
}
|
|
192
419
|
|
|
193
|
-
|
|
420
|
+
// Pages rank by pageviews and the other dimensions by sessions once every
|
|
421
|
+
// site's counters are proven for the whole range; before that stamp a
|
|
422
|
+
// bucket may hold slots without them, so the ranking stays event volume.
|
|
423
|
+
const countersProven = controls.every(
|
|
424
|
+
(control) =>
|
|
425
|
+
control?.countersSince !== undefined && from >= control.countersSince,
|
|
426
|
+
);
|
|
427
|
+
const pageBasis = countersProven
|
|
428
|
+
? ("pageviews" as const)
|
|
429
|
+
: ("eventVolume" as const);
|
|
430
|
+
const sessionBasis = countersProven
|
|
431
|
+
? ("sessions" as const)
|
|
432
|
+
: ("eventVolume" as const);
|
|
433
|
+
const top = (
|
|
434
|
+
type: string,
|
|
435
|
+
basis: "pageviews" | "sessions" | "eventVolume",
|
|
436
|
+
) =>
|
|
194
437
|
[...dimensions.entries()]
|
|
195
438
|
.filter(([key]) => key.startsWith(`${type}:`))
|
|
196
|
-
.map(([,
|
|
197
|
-
|
|
439
|
+
.map(([, item]) => ({
|
|
440
|
+
value: item.value,
|
|
441
|
+
count:
|
|
442
|
+
basis === "pageviews"
|
|
443
|
+
? item.pageviews
|
|
444
|
+
: basis === "sessions"
|
|
445
|
+
? item.sessions
|
|
446
|
+
: item.count,
|
|
447
|
+
revenueCents: item.revenueCents,
|
|
448
|
+
}))
|
|
449
|
+
.filter((item) => item.count > 0 || item.revenueCents > 0)
|
|
450
|
+
.sort(
|
|
451
|
+
(left, right) =>
|
|
452
|
+
right.count - left.count || left.value.localeCompare(right.value),
|
|
453
|
+
)
|
|
198
454
|
.slice(0, 10);
|
|
455
|
+
if (visitorsComplete) {
|
|
456
|
+
totals.visitors = estimateVisitors(mergedVisitorSketch);
|
|
457
|
+
for (const [timestamp, point] of points) {
|
|
458
|
+
point.visitors = estimateVisitors(
|
|
459
|
+
pointVisitorSketches.get(timestamp) ?? createVisitorSketch(),
|
|
460
|
+
);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
const coverage = await coverageForRange(ctx, siteIds, from, to, {
|
|
464
|
+
datasets: [overviewDataset],
|
|
465
|
+
complete: visitorsComplete,
|
|
466
|
+
});
|
|
467
|
+
if (localDays) {
|
|
468
|
+
// The stamp, not the oldest row, says where the site's days begin.
|
|
469
|
+
const availableFrom = Number.isFinite(localDaysFrom)
|
|
470
|
+
? Math.max(coverage.availableFrom ?? 0, localDaysFrom)
|
|
471
|
+
: null;
|
|
472
|
+
coverage.availableFrom = availableFrom;
|
|
473
|
+
if (
|
|
474
|
+
coverage.state === "complete" &&
|
|
475
|
+
(availableFrom === null || from < availableFrom)
|
|
476
|
+
) {
|
|
477
|
+
coverage.state = availableFrom === null ? "unavailable" : "partial";
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
const visitorBasis: "estimatedUnique" | "bucketOccurrences" =
|
|
481
|
+
visitorsComplete ? "estimatedUnique" : "bucketOccurrences";
|
|
199
482
|
return {
|
|
200
483
|
from,
|
|
201
484
|
to,
|
|
@@ -205,13 +488,44 @@ export const overview = query({
|
|
|
205
488
|
timeSeries: [...points.entries()]
|
|
206
489
|
.sort(([left], [right]) => left - right)
|
|
207
490
|
.map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
|
|
208
|
-
topSources: top("source"),
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
491
|
+
topSources: top("source", sessionBasis),
|
|
492
|
+
topCampaigns: top("campaign", sessionBasis),
|
|
493
|
+
topPages: top("page", pageBasis),
|
|
494
|
+
topCountries: top("country", sessionBasis),
|
|
495
|
+
topDevices: top("device", sessionBasis),
|
|
496
|
+
topBrowsers: top("browser", sessionBasis),
|
|
497
|
+
topEvents: top("event", "eventVolume"),
|
|
498
|
+
topGoals: top("goal", "eventVolume"),
|
|
499
|
+
metadata: {
|
|
500
|
+
visitors: {
|
|
501
|
+
basis: visitorBasis,
|
|
502
|
+
exact: false,
|
|
503
|
+
legacyIncomplete: !visitorsComplete,
|
|
504
|
+
sketchVersion: visitorsComplete ? 1 : null,
|
|
505
|
+
precision: visitorsComplete ? 6 : null,
|
|
506
|
+
},
|
|
507
|
+
breakdowns: {
|
|
508
|
+
topSources: sessionBasis,
|
|
509
|
+
topCampaigns: sessionBasis,
|
|
510
|
+
topPages: pageBasis,
|
|
511
|
+
topCountries: sessionBasis,
|
|
512
|
+
topDevices: sessionBasis,
|
|
513
|
+
topBrowsers: sessionBasis,
|
|
514
|
+
topEvents: "eventVolume" as const,
|
|
515
|
+
topGoals: "goalCompletionVolume" as const,
|
|
516
|
+
},
|
|
517
|
+
engagement: countersProven
|
|
518
|
+
? ("counted" as const)
|
|
519
|
+
: ("unavailable" as const),
|
|
520
|
+
coverage,
|
|
521
|
+
range: rangeSemantics(from, to, localDays ? siteTimezone : "UTC"),
|
|
522
|
+
features: {
|
|
523
|
+
sessionReplay: "unsupported" as const,
|
|
524
|
+
errorInsights: "unsupported" as const,
|
|
525
|
+
outboundLinks: "eventVolume" as const,
|
|
526
|
+
botDetection: "userAgent" as const,
|
|
527
|
+
},
|
|
528
|
+
},
|
|
215
529
|
};
|
|
216
530
|
},
|
|
217
531
|
});
|
|
@@ -226,8 +540,15 @@ export const liveVisitors = query({
|
|
|
226
540
|
handler: async (ctx, args) => {
|
|
227
541
|
const siteIds = validateSiteIds(args.siteIds);
|
|
228
542
|
const limit = args.limit ?? 100;
|
|
229
|
-
if (
|
|
230
|
-
|
|
543
|
+
if (
|
|
544
|
+
!Number.isSafeInteger(limit) ||
|
|
545
|
+
limit < 1 ||
|
|
546
|
+
limit > MAX_LIVE_VISITORS
|
|
547
|
+
) {
|
|
548
|
+
fail(
|
|
549
|
+
"INVALID_ARGUMENT",
|
|
550
|
+
`limit must be an integer between 1 and ${MAX_LIVE_VISITORS}`,
|
|
551
|
+
);
|
|
231
552
|
}
|
|
232
553
|
if (!Number.isFinite(args.now) || args.now < 0) {
|
|
233
554
|
fail("INVALID_ARGUMENT", "now must be a positive timestamp");
|
|
@@ -267,10 +588,311 @@ export const liveVisitors = query({
|
|
|
267
588
|
device: row.device,
|
|
268
589
|
pageviewCount: row.pageviewCount,
|
|
269
590
|
eventCount: row.eventCount,
|
|
591
|
+
...(row.previousPath ? { previousPath: row.previousPath } : {}),
|
|
592
|
+
...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
|
|
270
593
|
}));
|
|
271
594
|
},
|
|
272
595
|
});
|
|
273
596
|
|
|
597
|
+
const siteMapRouteValidator = v.object({
|
|
598
|
+
route: v.string(),
|
|
599
|
+
pageviews: v.number(),
|
|
600
|
+
visitors: v.number(),
|
|
601
|
+
entries: v.number(),
|
|
602
|
+
exits: v.number(),
|
|
603
|
+
bounces: v.number(),
|
|
604
|
+
transitionsIn: v.number(),
|
|
605
|
+
transitionsOut: v.number(),
|
|
606
|
+
dwellMs: v.number(),
|
|
607
|
+
dwellSamples: v.number(),
|
|
608
|
+
daily: v.array(v.object({ bucketStart: v.number(), pageviews: v.number() })),
|
|
609
|
+
});
|
|
610
|
+
const siteMapTransitionValidator = v.object({
|
|
611
|
+
from: v.string(),
|
|
612
|
+
to: v.string(),
|
|
613
|
+
count: v.number(),
|
|
614
|
+
});
|
|
615
|
+
const siteMapCoverageDataset = "siteMapRollups";
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Every route the sites served in the range and the transitions between
|
|
619
|
+
* them, read from the daily route rollups newest day first. Exits and
|
|
620
|
+
* bounces are derived rather than counted: a session enters once, leaves
|
|
621
|
+
* once, and every later pageview is a transition, so per route
|
|
622
|
+
* `exits = entries + transitionsIn - transitionsOut` and
|
|
623
|
+
* `bounces = entries - continued` hold without a session-end hook. The
|
|
624
|
+
* report never falls back to raw telemetry; days it could not cover are
|
|
625
|
+
* declared through `coverage` and `window`.
|
|
626
|
+
*/
|
|
627
|
+
export const siteMap = query({
|
|
628
|
+
args: {
|
|
629
|
+
siteIds: v.array(v.id("sites")),
|
|
630
|
+
from: v.number(),
|
|
631
|
+
to: v.number(),
|
|
632
|
+
},
|
|
633
|
+
returns: v.object({
|
|
634
|
+
from: v.number(),
|
|
635
|
+
to: v.number(),
|
|
636
|
+
window: v.union(v.object({ from: v.number(), to: v.number() }), v.null()),
|
|
637
|
+
routes: v.array(siteMapRouteValidator),
|
|
638
|
+
transitions: v.array(siteMapTransitionValidator),
|
|
639
|
+
truncated: v.object({ routes: v.boolean(), transitions: v.boolean() }),
|
|
640
|
+
metadata: reportMetadataValidator.extend({
|
|
641
|
+
visitors: v.object({
|
|
642
|
+
basis: v.literal("estimatedUnique"),
|
|
643
|
+
exact: v.boolean(),
|
|
644
|
+
}),
|
|
645
|
+
limits: v.object({
|
|
646
|
+
routesPerDay: v.number(),
|
|
647
|
+
transitionSlots: v.number(),
|
|
648
|
+
rows: v.number(),
|
|
649
|
+
}),
|
|
650
|
+
}),
|
|
651
|
+
}),
|
|
652
|
+
handler: async (ctx, args) => {
|
|
653
|
+
const siteIds = validateSiteIds(args.siteIds);
|
|
654
|
+
validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
|
|
655
|
+
requireAlignedRange(args.from, args.to, DAY_MS, "site map");
|
|
656
|
+
const controls = await Promise.all(
|
|
657
|
+
siteIds.map((siteId) =>
|
|
658
|
+
ctx.db
|
|
659
|
+
.query("analyticsCoverage")
|
|
660
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
661
|
+
.unique(),
|
|
662
|
+
),
|
|
663
|
+
);
|
|
664
|
+
const since = controls.every(
|
|
665
|
+
(control) => control?.siteMapSince !== undefined,
|
|
666
|
+
)
|
|
667
|
+
? Math.max(...controls.map((control) => control!.siteMapSince!))
|
|
668
|
+
: null;
|
|
669
|
+
const retainedBefore = controls.reduce<number | null>((latest, control) => {
|
|
670
|
+
const before = control?.retained.find(
|
|
671
|
+
(item) => item.dataset === siteMapCoverageDataset,
|
|
672
|
+
)?.before;
|
|
673
|
+
return before === undefined ? latest : Math.max(latest ?? 0, before);
|
|
674
|
+
}, null);
|
|
675
|
+
const metadata = (state: "complete" | "partial" | "unavailable") => ({
|
|
676
|
+
basis: "dailyRollup" as const,
|
|
677
|
+
coverage: {
|
|
678
|
+
state,
|
|
679
|
+
generation: 1 as const,
|
|
680
|
+
availableFrom: since,
|
|
681
|
+
availableThrough: since === null ? null : args.to,
|
|
682
|
+
retainedBefore,
|
|
683
|
+
},
|
|
684
|
+
range: rangeSemantics(args.from, args.to),
|
|
685
|
+
visitors: { basis: "estimatedUnique" as const, exact: false },
|
|
686
|
+
limits: {
|
|
687
|
+
routesPerDay: MAX_SITE_MAP_ROUTES_PER_DAY,
|
|
688
|
+
transitionSlots: ROUTE_TRANSITION_SLOTS,
|
|
689
|
+
rows: MAX_SITE_MAP_ROWS,
|
|
690
|
+
},
|
|
691
|
+
});
|
|
692
|
+
if (
|
|
693
|
+
since === null ||
|
|
694
|
+
args.to < since ||
|
|
695
|
+
(retainedBefore !== null && args.to < retainedBefore)
|
|
696
|
+
) {
|
|
697
|
+
return {
|
|
698
|
+
from: args.from,
|
|
699
|
+
to: args.to,
|
|
700
|
+
window: null,
|
|
701
|
+
routes: [],
|
|
702
|
+
transitions: [],
|
|
703
|
+
truncated: { routes: false, transitions: false },
|
|
704
|
+
metadata: metadata("unavailable"),
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
const readFrom = Math.max(
|
|
708
|
+
args.from,
|
|
709
|
+
Math.floor(since / DAY_MS) * DAY_MS,
|
|
710
|
+
retainedBefore ?? 0,
|
|
711
|
+
);
|
|
712
|
+
let budget = MAX_SITE_MAP_ROWS;
|
|
713
|
+
let completeFrom = readFrom;
|
|
714
|
+
const rows: Doc<"routeDailyRollups">[] = [];
|
|
715
|
+
for (const siteId of siteIds) {
|
|
716
|
+
const read = await readRouteRollups(
|
|
717
|
+
ctx,
|
|
718
|
+
siteId,
|
|
719
|
+
readFrom,
|
|
720
|
+
args.to,
|
|
721
|
+
budget,
|
|
722
|
+
);
|
|
723
|
+
budget -= read.rows.length;
|
|
724
|
+
completeFrom = Math.max(completeFrom, read.completeFrom);
|
|
725
|
+
rows.push(...read.rows);
|
|
726
|
+
}
|
|
727
|
+
const windowFrom = Math.max(
|
|
728
|
+
args.from,
|
|
729
|
+
completeFrom,
|
|
730
|
+
since,
|
|
731
|
+
retainedBefore ?? 0,
|
|
732
|
+
);
|
|
733
|
+
|
|
734
|
+
type RouteTotals = {
|
|
735
|
+
pageviews: number;
|
|
736
|
+
entries: number;
|
|
737
|
+
continued: number;
|
|
738
|
+
dwellMs: number;
|
|
739
|
+
dwellSamples: number;
|
|
740
|
+
transitionsIn: number;
|
|
741
|
+
transitionsOut: number;
|
|
742
|
+
sketch: VisitorSketch;
|
|
743
|
+
daily: Map<number, number>;
|
|
744
|
+
};
|
|
745
|
+
const routes = new Map<string, RouteTotals>();
|
|
746
|
+
const totalsFor = (route: string) => {
|
|
747
|
+
let current = routes.get(route);
|
|
748
|
+
if (!current) {
|
|
749
|
+
current = {
|
|
750
|
+
pageviews: 0,
|
|
751
|
+
entries: 0,
|
|
752
|
+
continued: 0,
|
|
753
|
+
dwellMs: 0,
|
|
754
|
+
dwellSamples: 0,
|
|
755
|
+
transitionsIn: 0,
|
|
756
|
+
transitionsOut: 0,
|
|
757
|
+
sketch: createVisitorSketch(),
|
|
758
|
+
daily: new Map<number, number>(),
|
|
759
|
+
};
|
|
760
|
+
routes.set(route, current);
|
|
761
|
+
}
|
|
762
|
+
return current;
|
|
763
|
+
};
|
|
764
|
+
const edges = new Map<
|
|
765
|
+
string,
|
|
766
|
+
{ from: string; to: string; count: number }
|
|
767
|
+
>();
|
|
768
|
+
for (const row of rows) {
|
|
769
|
+
if (row.bucketStart < completeFrom) continue;
|
|
770
|
+
const current = totalsFor(row.route);
|
|
771
|
+
current.pageviews += row.pageviews;
|
|
772
|
+
current.entries += row.entries;
|
|
773
|
+
current.continued += row.continued;
|
|
774
|
+
current.dwellMs += row.dwellMs;
|
|
775
|
+
current.dwellSamples += row.dwellSamples;
|
|
776
|
+
current.sketch = unionVisitorSketches(current.sketch, row.visitorSketch);
|
|
777
|
+
current.daily.set(
|
|
778
|
+
row.bucketStart,
|
|
779
|
+
(current.daily.get(row.bucketStart) ?? 0) + row.pageviews,
|
|
780
|
+
);
|
|
781
|
+
for (const target of row.outgoing) {
|
|
782
|
+
const key = `${row.route}\u0000${target.route}`;
|
|
783
|
+
const edge = edges.get(key) ?? {
|
|
784
|
+
from: row.route,
|
|
785
|
+
to: target.route,
|
|
786
|
+
count: 0,
|
|
787
|
+
};
|
|
788
|
+
edge.count += target.count;
|
|
789
|
+
edges.set(key, edge);
|
|
790
|
+
current.transitionsOut += target.count;
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
// A target without a row of its own in this window — folded past the
|
|
794
|
+
// daily cap, or already retained — counts under "(other)".
|
|
795
|
+
const transitions = new Map<
|
|
796
|
+
string,
|
|
797
|
+
{ from: string; to: string; count: number }
|
|
798
|
+
>();
|
|
799
|
+
for (const edge of edges.values()) {
|
|
800
|
+
const to = routes.has(edge.to) ? edge.to : SITE_MAP_OTHER_ROUTE;
|
|
801
|
+
const key = `${edge.from}\u0000${to}`;
|
|
802
|
+
const merged = transitions.get(key) ?? { from: edge.from, to, count: 0 };
|
|
803
|
+
merged.count += edge.count;
|
|
804
|
+
transitions.set(key, merged);
|
|
805
|
+
totalsFor(to).transitionsIn += edge.count;
|
|
806
|
+
}
|
|
807
|
+
const routeRows = [...routes.entries()]
|
|
808
|
+
.map(([route, totals]) => ({
|
|
809
|
+
route,
|
|
810
|
+
pageviews: totals.pageviews,
|
|
811
|
+
visitors: estimateVisitors(totals.sketch),
|
|
812
|
+
entries: totals.entries,
|
|
813
|
+
exits: Math.max(
|
|
814
|
+
0,
|
|
815
|
+
totals.entries + totals.transitionsIn - totals.transitionsOut,
|
|
816
|
+
),
|
|
817
|
+
bounces: Math.max(0, totals.entries - totals.continued),
|
|
818
|
+
transitionsIn: totals.transitionsIn,
|
|
819
|
+
transitionsOut: totals.transitionsOut,
|
|
820
|
+
dwellMs: totals.dwellMs,
|
|
821
|
+
dwellSamples: totals.dwellSamples,
|
|
822
|
+
daily: [...totals.daily.entries()]
|
|
823
|
+
.sort(([left], [right]) => left - right)
|
|
824
|
+
.map(([bucketStart, pageviews]) => ({ bucketStart, pageviews })),
|
|
825
|
+
}))
|
|
826
|
+
.sort(
|
|
827
|
+
(left, right) =>
|
|
828
|
+
right.pageviews - left.pageviews ||
|
|
829
|
+
left.route.localeCompare(right.route),
|
|
830
|
+
);
|
|
831
|
+
const transitionRows = [...transitions.values()].sort(
|
|
832
|
+
(left, right) =>
|
|
833
|
+
right.count - left.count ||
|
|
834
|
+
left.from.localeCompare(right.from) ||
|
|
835
|
+
left.to.localeCompare(right.to),
|
|
836
|
+
);
|
|
837
|
+
return {
|
|
838
|
+
from: args.from,
|
|
839
|
+
to: args.to,
|
|
840
|
+
window: completeFrom > args.to ? null : { from: windowFrom, to: args.to },
|
|
841
|
+
routes: routeRows,
|
|
842
|
+
transitions: transitionRows,
|
|
843
|
+
truncated: {
|
|
844
|
+
routes: routes.has(SITE_MAP_OTHER_ROUTE),
|
|
845
|
+
transitions: transitionRows.some(
|
|
846
|
+
(transition) =>
|
|
847
|
+
transition.from === SITE_MAP_OTHER_ROUTE ||
|
|
848
|
+
transition.to === SITE_MAP_OTHER_ROUTE,
|
|
849
|
+
),
|
|
850
|
+
},
|
|
851
|
+
metadata: metadata(windowFrom > args.from ? "partial" : "complete"),
|
|
852
|
+
};
|
|
853
|
+
},
|
|
854
|
+
});
|
|
855
|
+
|
|
856
|
+
/**
|
|
857
|
+
* Reads a site's route rollups newest day first until the row budget or the
|
|
858
|
+
* transaction's read headroom runs out, then drops the day it could only
|
|
859
|
+
* read in part so every day it returns is complete.
|
|
860
|
+
*/
|
|
861
|
+
async function readRouteRollups(
|
|
862
|
+
ctx: QueryCtx,
|
|
863
|
+
siteId: Id<"sites">,
|
|
864
|
+
from: number,
|
|
865
|
+
to: number,
|
|
866
|
+
budget: number,
|
|
867
|
+
) {
|
|
868
|
+
const rows: Doc<"routeDailyRollups">[] = [];
|
|
869
|
+
let cutAt: number | null = null;
|
|
870
|
+
const query = ctx.db
|
|
871
|
+
.query("routeDailyRollups")
|
|
872
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
873
|
+
range
|
|
874
|
+
.eq("siteId", siteId)
|
|
875
|
+
.gte("bucketStart", from)
|
|
876
|
+
.lte("bucketStart", to),
|
|
877
|
+
)
|
|
878
|
+
.order("desc");
|
|
879
|
+
for await (const row of query) {
|
|
880
|
+
if (
|
|
881
|
+
rows.length >= budget ||
|
|
882
|
+
(rows.length % 256 === 255 && !(await hasFeatureReadHeadroom(ctx)))
|
|
883
|
+
) {
|
|
884
|
+
cutAt = row.bucketStart;
|
|
885
|
+
break;
|
|
886
|
+
}
|
|
887
|
+
rows.push(row);
|
|
888
|
+
}
|
|
889
|
+
if (cutAt === null) return { rows, completeFrom: from };
|
|
890
|
+
while (rows.length > 0 && rows[rows.length - 1]!.bucketStart === cutAt) {
|
|
891
|
+
rows.pop();
|
|
892
|
+
}
|
|
893
|
+
return { rows, completeFrom: cutAt + DAY_MS };
|
|
894
|
+
}
|
|
895
|
+
|
|
274
896
|
export const listSessions = query({
|
|
275
897
|
args: {
|
|
276
898
|
siteId: v.id("sites"),
|
|
@@ -281,6 +903,7 @@ export const listSessions = query({
|
|
|
281
903
|
returns: paginationResultValidator(sessionDocumentValidator),
|
|
282
904
|
handler: async (ctx, args) => {
|
|
283
905
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
906
|
+
await assertSourceAvailable(ctx, [args.siteId], ["sessions"], args.from);
|
|
284
907
|
const pagination = parseKeysetPagination(args.paginationOpts, {
|
|
285
908
|
kind: "sessions",
|
|
286
909
|
siteId: String(args.siteId),
|
|
@@ -330,7 +953,14 @@ export const listSessions = query({
|
|
|
330
953
|
.take(take)),
|
|
331
954
|
);
|
|
332
955
|
}
|
|
333
|
-
|
|
956
|
+
const publicRows = rows.map(
|
|
957
|
+
({
|
|
958
|
+
geoLookupAttemptedAt: _attemptedAt,
|
|
959
|
+
lastPageviewAt: _lastPageviewAt,
|
|
960
|
+
...row
|
|
961
|
+
}) => row,
|
|
962
|
+
);
|
|
963
|
+
return keysetPaginationResult(publicRows, pagination, (row) => ({
|
|
334
964
|
timestamp: row.lastSeenAt,
|
|
335
965
|
creationTime: row._creationTime,
|
|
336
966
|
}));
|
|
@@ -348,6 +978,7 @@ export const sessionJourney = query({
|
|
|
348
978
|
returns: paginationResultValidator(eventDocumentValidator),
|
|
349
979
|
handler: async (ctx, args) => {
|
|
350
980
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
981
|
+
await assertSourceAvailable(ctx, [args.siteId], ["events"], args.from);
|
|
351
982
|
const pagination = parseJourneyPagination(args.paginationOpts, {
|
|
352
983
|
kind: "events",
|
|
353
984
|
siteId: String(args.siteId),
|
|
@@ -378,6 +1009,7 @@ export const listConversions = query({
|
|
|
378
1009
|
returns: paginationResultValidator(conversionDocumentValidator),
|
|
379
1010
|
handler: async (ctx, args) => {
|
|
380
1011
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
1012
|
+
await assertSourceAvailable(ctx, [args.siteId], ["conversions"], args.from);
|
|
381
1013
|
const pagination = parseKeysetPagination(args.paginationOpts, {
|
|
382
1014
|
kind: "conversions",
|
|
383
1015
|
siteId: String(args.siteId),
|
|
@@ -446,54 +1078,39 @@ export const visitorJourney = query({
|
|
|
446
1078
|
handler: async (ctx, args) => {
|
|
447
1079
|
const siteIds = validateSiteIds(args.siteIds);
|
|
448
1080
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
1081
|
+
await assertSourceAvailable(ctx, siteIds, ["events"], args.from);
|
|
449
1082
|
const limit = args.limit ?? 200;
|
|
450
|
-
if (
|
|
451
|
-
|
|
1083
|
+
if (
|
|
1084
|
+
!Number.isSafeInteger(limit) ||
|
|
1085
|
+
limit < 1 ||
|
|
1086
|
+
limit > MAX_JOURNEY_EVENTS
|
|
1087
|
+
) {
|
|
1088
|
+
fail(
|
|
1089
|
+
"INVALID_ARGUMENT",
|
|
1090
|
+
`limit must be an integer between 1 and ${MAX_JOURNEY_EVENTS}`,
|
|
1091
|
+
);
|
|
452
1092
|
}
|
|
453
|
-
const sessionGroups = await Promise.all(
|
|
454
|
-
siteIds.map((siteId) =>
|
|
455
|
-
ctx.db
|
|
456
|
-
.query("sessions")
|
|
457
|
-
.withIndex("by_siteId_and_visitorId_and_lastSeenAt", (range) =>
|
|
458
|
-
range
|
|
459
|
-
.eq("siteId", siteId)
|
|
460
|
-
.eq("visitorId", args.visitorId)
|
|
461
|
-
.gte("lastSeenAt", args.from),
|
|
462
|
-
)
|
|
463
|
-
.order("asc")
|
|
464
|
-
.take(MAX_JOURNEY_EVENTS + 1),
|
|
465
|
-
),
|
|
466
|
-
);
|
|
467
|
-
const sessions = sessionGroups
|
|
468
|
-
.flat()
|
|
469
|
-
.filter((session) => session.startedAt <= args.to)
|
|
470
|
-
.sort((left, right) => left.startedAt - right.startedAt)
|
|
471
|
-
.slice(0, MAX_JOURNEY_EVENTS);
|
|
472
1093
|
const rows: RawJourneyEvent[] = [];
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
1094
|
+
for (const siteId of siteIds) {
|
|
1095
|
+
for (const visitorId of await resolveVisitorIdentities(
|
|
1096
|
+
ctx,
|
|
1097
|
+
siteId,
|
|
1098
|
+
args.visitorId,
|
|
1099
|
+
)) {
|
|
1100
|
+
const cutoff =
|
|
1101
|
+
rows.length >= limit ? rows[limit - 1]!.timestamp : args.to;
|
|
1102
|
+
rows.push(
|
|
1103
|
+
...(await loadVisitorJourneyEvents(ctx, {
|
|
1104
|
+
siteId,
|
|
1105
|
+
visitorId,
|
|
1106
|
+
from: args.from,
|
|
1107
|
+
to: cutoff,
|
|
1108
|
+
resultLimit: limit,
|
|
1109
|
+
})),
|
|
485
1110
|
);
|
|
1111
|
+
rows.sort(compareJourneyEvents);
|
|
1112
|
+
if (rows.length > limit) rows.length = limit;
|
|
486
1113
|
}
|
|
487
|
-
rows.push(...await loadSessionJourneyEvents(ctx, {
|
|
488
|
-
siteId: session.siteId,
|
|
489
|
-
sessionId: session.sessionId,
|
|
490
|
-
from: args.from,
|
|
491
|
-
to: Math.min(args.to, cutoff),
|
|
492
|
-
cursor: null,
|
|
493
|
-
resultLimit: resultIsFull ? limit : limit - rows.length,
|
|
494
|
-
}));
|
|
495
|
-
rows.sort(compareJourneyEvents);
|
|
496
|
-
if (rows.length > limit) rows.length = limit;
|
|
497
1114
|
}
|
|
498
1115
|
return hydrateEventContexts(ctx, rows);
|
|
499
1116
|
},
|
|
@@ -511,36 +1128,90 @@ export const goalsReport = query({
|
|
|
511
1128
|
returns: v.object({
|
|
512
1129
|
goals: v.array(goalReportItemValidator),
|
|
513
1130
|
truncated: v.boolean(),
|
|
1131
|
+
metadata: reportMetadataValidator,
|
|
514
1132
|
}),
|
|
515
1133
|
handler: async (ctx, args) => {
|
|
516
1134
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
517
|
-
const
|
|
1135
|
+
const goalRows = await ctx.db
|
|
518
1136
|
.query("goals")
|
|
519
1137
|
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
520
|
-
.take(MAX_GOALS_PER_SITE);
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
.
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
1138
|
+
.take(MAX_GOALS_PER_SITE + 1);
|
|
1139
|
+
failIfIncomplete(goalRows, MAX_GOALS_PER_SITE, "goal definitions");
|
|
1140
|
+
const goals = goalRows;
|
|
1141
|
+
const totals = new Map<
|
|
1142
|
+
string,
|
|
1143
|
+
{ completions: number; valueCents: number }
|
|
1144
|
+
>();
|
|
1145
|
+
const useRollups = await dailyRollupsAvailable(
|
|
1146
|
+
ctx,
|
|
1147
|
+
args.siteId,
|
|
1148
|
+
args.from,
|
|
1149
|
+
args.to,
|
|
1150
|
+
"goalRollups",
|
|
1151
|
+
);
|
|
1152
|
+
const rollups = useRollups
|
|
1153
|
+
? await loadGoalDailyRollups(
|
|
1154
|
+
ctx,
|
|
1155
|
+
args.siteId,
|
|
1156
|
+
goals.map((goal) => goal._id),
|
|
1157
|
+
args.from,
|
|
1158
|
+
args.to,
|
|
1159
|
+
)
|
|
1160
|
+
: null;
|
|
1161
|
+
if (rollups) {
|
|
1162
|
+
for (const row of rollups) {
|
|
1163
|
+
const key = String(row.goalId);
|
|
1164
|
+
const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
|
|
1165
|
+
total.completions += row.completions;
|
|
1166
|
+
total.valueCents += row.valueCents;
|
|
1167
|
+
totals.set(key, total);
|
|
1168
|
+
}
|
|
1169
|
+
} else {
|
|
1170
|
+
await assertSourceAvailable(
|
|
1171
|
+
ctx,
|
|
1172
|
+
[args.siteId],
|
|
1173
|
+
["goalCompletions"],
|
|
1174
|
+
args.from,
|
|
1175
|
+
);
|
|
1176
|
+
const completions = await ctx.db
|
|
1177
|
+
.query("goalCompletions")
|
|
1178
|
+
.withIndex("by_siteId_and_timestamp", (range) =>
|
|
1179
|
+
range
|
|
1180
|
+
.eq("siteId", args.siteId)
|
|
1181
|
+
.gte("timestamp", args.from)
|
|
1182
|
+
.lte("timestamp", args.to),
|
|
1183
|
+
)
|
|
1184
|
+
.take(MAX_REPORT_BUCKETS + 1);
|
|
1185
|
+
failIfIncomplete(completions, MAX_REPORT_BUCKETS, "goal completions");
|
|
1186
|
+
for (const completion of completions) {
|
|
1187
|
+
const key = String(completion.goalId);
|
|
1188
|
+
const total = totals.get(key) ?? { completions: 0, valueCents: 0 };
|
|
1189
|
+
total.completions += 1;
|
|
1190
|
+
total.valueCents += completion.valueCents;
|
|
1191
|
+
totals.set(key, total);
|
|
1192
|
+
}
|
|
537
1193
|
}
|
|
538
1194
|
return {
|
|
539
1195
|
goals: goals.map((goal) => ({
|
|
540
1196
|
...goal,
|
|
541
1197
|
...(totals.get(String(goal._id)) ?? { completions: 0, valueCents: 0 }),
|
|
542
1198
|
})),
|
|
543
|
-
truncated:
|
|
1199
|
+
truncated: false,
|
|
1200
|
+
metadata: {
|
|
1201
|
+
basis: rollups ? ("dailyRollup" as const) : ("boundedLegacy" as const),
|
|
1202
|
+
coverage: await coverageForRange(
|
|
1203
|
+
ctx,
|
|
1204
|
+
[args.siteId],
|
|
1205
|
+
args.from,
|
|
1206
|
+
args.to,
|
|
1207
|
+
{
|
|
1208
|
+
datasets: [rollups ? "goalRollups" : "goalCompletions"],
|
|
1209
|
+
complete: true,
|
|
1210
|
+
generation: rollups ? 1 : null,
|
|
1211
|
+
},
|
|
1212
|
+
),
|
|
1213
|
+
range: rangeSemantics(args.from, args.to),
|
|
1214
|
+
},
|
|
544
1215
|
};
|
|
545
1216
|
},
|
|
546
1217
|
});
|
|
@@ -549,9 +1220,15 @@ const funnelReportItemValidator = funnelFieldsValidator.extend({
|
|
|
549
1220
|
_id: v.id("funnels"),
|
|
550
1221
|
_creationTime: v.number(),
|
|
551
1222
|
stepsReport: v.array(
|
|
552
|
-
v.object({
|
|
1223
|
+
v.object({
|
|
1224
|
+
step: v.number(),
|
|
1225
|
+
name: v.string(),
|
|
1226
|
+
entrants: v.number(),
|
|
1227
|
+
completions: v.number(),
|
|
1228
|
+
}),
|
|
553
1229
|
),
|
|
554
1230
|
truncated: v.boolean(),
|
|
1231
|
+
metadata: reportMetadataValidator,
|
|
555
1232
|
});
|
|
556
1233
|
|
|
557
1234
|
export const funnelsReport = query({
|
|
@@ -559,29 +1236,74 @@ export const funnelsReport = query({
|
|
|
559
1236
|
returns: v.array(funnelReportItemValidator),
|
|
560
1237
|
handler: async (ctx, args) => {
|
|
561
1238
|
validateRange(args.from, args.to, 90);
|
|
562
|
-
|
|
1239
|
+
requireAlignedRange(args.from, args.to, DAY_MS, "funnel report");
|
|
1240
|
+
const funnelRows = await ctx.db
|
|
563
1241
|
.query("funnels")
|
|
564
1242
|
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
565
|
-
.take(MAX_FUNNELS_PER_SITE);
|
|
1243
|
+
.take(MAX_FUNNELS_PER_SITE + 1);
|
|
1244
|
+
failIfIncomplete(funnelRows, MAX_FUNNELS_PER_SITE, "funnel definitions");
|
|
1245
|
+
const funnels = funnelRows;
|
|
1246
|
+
const useRollups = await dailyRollupsAvailable(
|
|
1247
|
+
ctx,
|
|
1248
|
+
args.siteId,
|
|
1249
|
+
args.from,
|
|
1250
|
+
args.to,
|
|
1251
|
+
"funnelRollups",
|
|
1252
|
+
);
|
|
1253
|
+
const rollups = useRollups
|
|
1254
|
+
? await loadFunnelDailyRollups(
|
|
1255
|
+
ctx,
|
|
1256
|
+
args.siteId,
|
|
1257
|
+
funnels.map((funnel) => funnel._id),
|
|
1258
|
+
args.from,
|
|
1259
|
+
args.to,
|
|
1260
|
+
)
|
|
1261
|
+
: null;
|
|
1262
|
+
if (!rollups) {
|
|
1263
|
+
await assertSourceAvailable(
|
|
1264
|
+
ctx,
|
|
1265
|
+
[args.siteId],
|
|
1266
|
+
["funnelSteps"],
|
|
1267
|
+
args.from,
|
|
1268
|
+
);
|
|
1269
|
+
}
|
|
1270
|
+
const reportCoverage = await coverageForRange(
|
|
1271
|
+
ctx,
|
|
1272
|
+
[args.siteId],
|
|
1273
|
+
args.from,
|
|
1274
|
+
args.to,
|
|
1275
|
+
{
|
|
1276
|
+
datasets: [rollups ? "funnelRollups" : "funnelSteps"],
|
|
1277
|
+
complete: true,
|
|
1278
|
+
generation: rollups ? 1 : null,
|
|
1279
|
+
},
|
|
1280
|
+
);
|
|
566
1281
|
const reports = [];
|
|
567
1282
|
for (const funnel of funnels) {
|
|
568
|
-
const buckets = await ctx.db
|
|
569
|
-
.query("funnelStepBuckets")
|
|
570
|
-
.withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
|
|
571
|
-
range
|
|
572
|
-
.eq("siteId", args.siteId)
|
|
573
|
-
.eq("funnelId", funnel._id)
|
|
574
|
-
.gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
|
|
575
|
-
.lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
|
|
576
|
-
)
|
|
577
|
-
.take(601);
|
|
578
1283
|
const stepsReport = funnel.steps.map((step, index) => ({
|
|
579
1284
|
step: index,
|
|
580
1285
|
name: step.name,
|
|
581
1286
|
entrants: 0,
|
|
582
1287
|
completions: 0,
|
|
583
1288
|
}));
|
|
584
|
-
|
|
1289
|
+
const buckets = rollups
|
|
1290
|
+
? rollups
|
|
1291
|
+
.filter((row) => row.funnelId === funnel._id)
|
|
1292
|
+
.flatMap((row) => row.steps)
|
|
1293
|
+
: await ctx.db
|
|
1294
|
+
.query("funnelStepBuckets")
|
|
1295
|
+
.withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
|
|
1296
|
+
range
|
|
1297
|
+
.eq("siteId", args.siteId)
|
|
1298
|
+
.eq("funnelId", funnel._id)
|
|
1299
|
+
.gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
|
|
1300
|
+
.lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
|
|
1301
|
+
)
|
|
1302
|
+
.take(MAX_REPORT_BUCKETS + 1);
|
|
1303
|
+
if (!rollups) {
|
|
1304
|
+
failIfIncomplete(buckets, MAX_REPORT_BUCKETS, "funnel step buckets");
|
|
1305
|
+
}
|
|
1306
|
+
for (const bucket of buckets) {
|
|
585
1307
|
const step = stepsReport[bucket.step];
|
|
586
1308
|
if (step) {
|
|
587
1309
|
step.entrants += bucket.entrants;
|
|
@@ -591,7 +1313,14 @@ export const funnelsReport = query({
|
|
|
591
1313
|
reports.push({
|
|
592
1314
|
...funnel,
|
|
593
1315
|
stepsReport,
|
|
594
|
-
truncated:
|
|
1316
|
+
truncated: false,
|
|
1317
|
+
metadata: {
|
|
1318
|
+
basis: rollups
|
|
1319
|
+
? ("dailyRollup" as const)
|
|
1320
|
+
: ("boundedLegacy" as const),
|
|
1321
|
+
coverage: reportCoverage,
|
|
1322
|
+
range: rangeSemantics(args.from, args.to),
|
|
1323
|
+
},
|
|
595
1324
|
});
|
|
596
1325
|
}
|
|
597
1326
|
return reports;
|
|
@@ -613,64 +1342,134 @@ export const affiliatesReport = query({
|
|
|
613
1342
|
returns: v.object({
|
|
614
1343
|
affiliates: v.array(affiliateReportItemValidator),
|
|
615
1344
|
truncated: v.boolean(),
|
|
1345
|
+
metadata: reportMetadataValidator.extend({
|
|
1346
|
+
uniqueVisitors: v.object({
|
|
1347
|
+
basis: v.union(v.literal("estimatedUnique"), v.literal("exactLegacy")),
|
|
1348
|
+
exact: v.boolean(),
|
|
1349
|
+
}),
|
|
1350
|
+
}),
|
|
616
1351
|
}),
|
|
617
1352
|
handler: async (ctx, args) => {
|
|
618
1353
|
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
1354
|
+
requireAlignedRange(args.from, args.to, DAY_MS, "affiliate report");
|
|
619
1355
|
const affiliateRows = await ctx.db
|
|
620
1356
|
.query("affiliates")
|
|
621
1357
|
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
622
1358
|
.order("desc")
|
|
623
1359
|
.take(MAX_AFFILIATES_PER_SITE + 1);
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
.
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
1360
|
+
failIfIncomplete(
|
|
1361
|
+
affiliateRows,
|
|
1362
|
+
MAX_AFFILIATES_PER_SITE,
|
|
1363
|
+
"affiliate definitions",
|
|
1364
|
+
);
|
|
1365
|
+
const affiliates = affiliateRows;
|
|
1366
|
+
const useRollups = await dailyRollupsAvailable(
|
|
1367
|
+
ctx,
|
|
1368
|
+
args.siteId,
|
|
1369
|
+
args.from,
|
|
1370
|
+
args.to,
|
|
1371
|
+
"affiliateRollups",
|
|
1372
|
+
);
|
|
1373
|
+
const rollups = useRollups
|
|
1374
|
+
? await loadAffiliateDailyRollups(
|
|
1375
|
+
ctx,
|
|
1376
|
+
args.siteId,
|
|
1377
|
+
affiliates.map((affiliate) => affiliate._id),
|
|
1378
|
+
args.from,
|
|
1379
|
+
args.to,
|
|
1380
|
+
)
|
|
1381
|
+
: null;
|
|
1382
|
+
const visits = new Map<
|
|
1383
|
+
string,
|
|
1384
|
+
{
|
|
1385
|
+
visits: number;
|
|
1386
|
+
visitors: Set<string>;
|
|
1387
|
+
sketch: VisitorSketch;
|
|
1388
|
+
}
|
|
1389
|
+
>();
|
|
1390
|
+
const totals = new Map<
|
|
1391
|
+
string,
|
|
1392
|
+
{
|
|
1393
|
+
conversions: number;
|
|
1394
|
+
revenueCents: number;
|
|
1395
|
+
commissionCents: number;
|
|
1396
|
+
}
|
|
1397
|
+
>();
|
|
1398
|
+
if (rollups) {
|
|
1399
|
+
for (const row of rollups) {
|
|
1400
|
+
const key = String(row.affiliateId);
|
|
1401
|
+
const traffic = visits.get(key) ?? {
|
|
1402
|
+
visits: 0,
|
|
1403
|
+
visitors: new Set<string>(),
|
|
1404
|
+
sketch: createVisitorSketch(),
|
|
1405
|
+
};
|
|
1406
|
+
traffic.visits += row.visits;
|
|
1407
|
+
traffic.sketch = unionVisitorSketches(
|
|
1408
|
+
traffic.sketch,
|
|
1409
|
+
row.visitorSketch,
|
|
1410
|
+
);
|
|
1411
|
+
visits.set(key, traffic);
|
|
1412
|
+
const total = totals.get(key) ?? {
|
|
1413
|
+
conversions: 0,
|
|
1414
|
+
revenueCents: 0,
|
|
1415
|
+
commissionCents: 0,
|
|
1416
|
+
};
|
|
1417
|
+
total.conversions += row.conversions;
|
|
1418
|
+
total.revenueCents += row.revenueCents;
|
|
1419
|
+
total.commissionCents += row.commissionCents;
|
|
1420
|
+
totals.set(key, total);
|
|
1421
|
+
}
|
|
1422
|
+
} else {
|
|
1423
|
+
await assertSourceAvailable(
|
|
1424
|
+
ctx,
|
|
1425
|
+
[args.siteId],
|
|
1426
|
+
["affiliateStats", "sessions"],
|
|
1427
|
+
args.from,
|
|
1428
|
+
);
|
|
1429
|
+
const stats = await ctx.db
|
|
1430
|
+
.query("affiliateDailyStats")
|
|
1431
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
1432
|
+
range
|
|
1433
|
+
.eq("siteId", args.siteId)
|
|
1434
|
+
.gte("bucketStart", Math.floor(args.from / DAY_MS) * DAY_MS)
|
|
1435
|
+
.lte("bucketStart", Math.floor(args.to / DAY_MS) * DAY_MS),
|
|
1436
|
+
)
|
|
1437
|
+
.take(MAX_REPORT_BUCKETS + 1);
|
|
1438
|
+
const sessions = await ctx.db
|
|
1439
|
+
.query("sessions")
|
|
1440
|
+
.withIndex("by_siteId_and_startedAt", (range) =>
|
|
1441
|
+
range
|
|
1442
|
+
.eq("siteId", args.siteId)
|
|
1443
|
+
.gte("startedAt", args.from)
|
|
1444
|
+
.lte("startedAt", args.to),
|
|
1445
|
+
)
|
|
1446
|
+
.take(MAX_REPORT_BUCKETS + 1);
|
|
1447
|
+
failIfIncomplete(stats, MAX_REPORT_BUCKETS, "affiliate stat buckets");
|
|
1448
|
+
failIfIncomplete(sessions, MAX_REPORT_BUCKETS, "affiliate sessions");
|
|
1449
|
+
for (const session of sessions) {
|
|
1450
|
+
if (!session.affiliateId) continue;
|
|
1451
|
+
const key = String(session.affiliateId);
|
|
1452
|
+
const total = visits.get(key) ?? {
|
|
1453
|
+
visits: 0,
|
|
1454
|
+
visitors: new Set<string>(),
|
|
1455
|
+
sketch: createVisitorSketch(),
|
|
1456
|
+
};
|
|
1457
|
+
total.visits += 1;
|
|
1458
|
+
total.visitors.add(session.visitorId);
|
|
1459
|
+
visits.set(key, total);
|
|
1460
|
+
}
|
|
1461
|
+
for (const row of stats) {
|
|
1462
|
+
const key = String(row.affiliateId);
|
|
1463
|
+
const total = totals.get(key) ?? {
|
|
1464
|
+
conversions: 0,
|
|
1465
|
+
revenueCents: 0,
|
|
1466
|
+
commissionCents: 0,
|
|
1467
|
+
};
|
|
1468
|
+
total.conversions += row.conversions;
|
|
1469
|
+
total.revenueCents += row.revenueCents;
|
|
1470
|
+
total.commissionCents += row.commissionCents;
|
|
1471
|
+
totals.set(key, total);
|
|
1472
|
+
}
|
|
674
1473
|
}
|
|
675
1474
|
return {
|
|
676
1475
|
affiliates: affiliates.map((affiliate) => {
|
|
@@ -684,17 +1483,811 @@ export const affiliatesReport = query({
|
|
|
684
1483
|
...affiliate,
|
|
685
1484
|
...performance,
|
|
686
1485
|
visits: traffic?.visits ?? 0,
|
|
687
|
-
uniqueVisitors:
|
|
1486
|
+
uniqueVisitors: rollups
|
|
1487
|
+
? estimateVisitors(traffic?.sketch ?? createVisitorSketch())
|
|
1488
|
+
: (traffic?.visitors.size ?? 0),
|
|
688
1489
|
};
|
|
689
1490
|
}),
|
|
690
|
-
truncated:
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
1491
|
+
truncated: false,
|
|
1492
|
+
metadata: {
|
|
1493
|
+
basis: rollups ? ("dailyRollup" as const) : ("boundedLegacy" as const),
|
|
1494
|
+
coverage: await coverageForRange(
|
|
1495
|
+
ctx,
|
|
1496
|
+
[args.siteId],
|
|
1497
|
+
args.from,
|
|
1498
|
+
args.to,
|
|
1499
|
+
{
|
|
1500
|
+
datasets: rollups
|
|
1501
|
+
? ["affiliateRollups"]
|
|
1502
|
+
: ["affiliateStats", "sessions"],
|
|
1503
|
+
complete: true,
|
|
1504
|
+
generation: rollups ? 1 : null,
|
|
1505
|
+
},
|
|
1506
|
+
),
|
|
1507
|
+
range: rangeSemantics(args.from, args.to),
|
|
1508
|
+
uniqueVisitors: {
|
|
1509
|
+
basis: rollups
|
|
1510
|
+
? ("estimatedUnique" as const)
|
|
1511
|
+
: ("exactLegacy" as const),
|
|
1512
|
+
exact: !rollups,
|
|
1513
|
+
},
|
|
1514
|
+
},
|
|
1515
|
+
};
|
|
1516
|
+
},
|
|
1517
|
+
});
|
|
1518
|
+
|
|
1519
|
+
const vitalStatsValidator = v.object({
|
|
1520
|
+
count: v.number(),
|
|
1521
|
+
p75: v.number(),
|
|
1522
|
+
average: v.number(),
|
|
1523
|
+
good: v.number(),
|
|
1524
|
+
needsImprovement: v.number(),
|
|
1525
|
+
poor: v.number(),
|
|
1526
|
+
});
|
|
1527
|
+
|
|
1528
|
+
export const vitalsReport = query({
|
|
1529
|
+
args: { siteId: v.id("sites"), from: v.number(), to: v.number() },
|
|
1530
|
+
returns: v.object({
|
|
1531
|
+
metrics: v.array(
|
|
1532
|
+
vitalStatsValidator.extend({
|
|
1533
|
+
metric: vitalMetricValidator,
|
|
1534
|
+
devices: v.array(vitalStatsValidator.extend({ device: v.string() })),
|
|
1535
|
+
pages: v.array(vitalStatsValidator.extend({ page: v.string() })),
|
|
1536
|
+
timeSeries: v.array(
|
|
1537
|
+
v.object({
|
|
1538
|
+
timestamp: v.number(),
|
|
1539
|
+
count: v.number(),
|
|
1540
|
+
p75: v.number(),
|
|
1541
|
+
}),
|
|
1542
|
+
),
|
|
1543
|
+
}),
|
|
1544
|
+
),
|
|
1545
|
+
truncated: v.boolean(),
|
|
1546
|
+
metadata: reportMetadataValidator,
|
|
1547
|
+
}),
|
|
1548
|
+
handler: async (ctx, args) => {
|
|
1549
|
+
validateRange(args.from, args.to, MAX_FEATURE_ROLLUP_DAYS);
|
|
1550
|
+
requireAlignedRange(args.from, args.to, DAY_MS, "vitals report");
|
|
1551
|
+
await assertSourceAvailable(
|
|
1552
|
+
ctx,
|
|
1553
|
+
[args.siteId],
|
|
1554
|
+
["vitalRollups"],
|
|
1555
|
+
args.from,
|
|
1556
|
+
);
|
|
1557
|
+
const rows = await ctx.db
|
|
1558
|
+
.query("vitalRollups")
|
|
1559
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
1560
|
+
range
|
|
1561
|
+
.eq("siteId", args.siteId)
|
|
1562
|
+
.gte("bucketStart", args.from)
|
|
1563
|
+
.lte("bucketStart", args.to),
|
|
1564
|
+
)
|
|
1565
|
+
.take(MAX_VITALS_REPORT_ROWS + 1);
|
|
1566
|
+
failIfIncomplete(rows, MAX_VITALS_REPORT_ROWS, "vital rollups");
|
|
1567
|
+
|
|
1568
|
+
type Accumulated = { count: number; sum: number; histogram: number[] };
|
|
1569
|
+
const fold = <Key>(
|
|
1570
|
+
groups: Map<VitalMetric, Map<Key, Accumulated>>,
|
|
1571
|
+
metric: VitalMetric,
|
|
1572
|
+
key: Key,
|
|
1573
|
+
entry: { count: number; sum: number; histogram: number[] },
|
|
1574
|
+
) => {
|
|
1575
|
+
const group = groups.get(metric) ?? new Map<Key, Accumulated>();
|
|
1576
|
+
const current = group.get(key) ?? { count: 0, sum: 0, histogram: [] };
|
|
1577
|
+
current.count += entry.count;
|
|
1578
|
+
current.sum += entry.sum;
|
|
1579
|
+
current.histogram = mergeVitalHistograms(
|
|
1580
|
+
current.histogram,
|
|
1581
|
+
entry.histogram,
|
|
1582
|
+
);
|
|
1583
|
+
group.set(key, current);
|
|
1584
|
+
groups.set(metric, group);
|
|
1585
|
+
};
|
|
1586
|
+
const byDevice = new Map<VitalMetric, Map<string, Accumulated>>();
|
|
1587
|
+
const byPage = new Map<VitalMetric, Map<string, Accumulated>>();
|
|
1588
|
+
const byDay = new Map<VitalMetric, Map<number, Accumulated>>();
|
|
1589
|
+
for (const row of rows) {
|
|
1590
|
+
for (const entry of row.metrics) {
|
|
1591
|
+
if (row.page === VITAL_ALL) {
|
|
1592
|
+
// A device row carries the complete signal once, so overall totals
|
|
1593
|
+
// and the time series derive from this family alone.
|
|
1594
|
+
fold(byDevice, entry.metric, row.device, entry);
|
|
1595
|
+
fold(byDay, entry.metric, row.bucketStart, entry);
|
|
1596
|
+
} else {
|
|
1597
|
+
fold(byPage, entry.metric, row.page, entry);
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
}
|
|
1601
|
+
const stats = (metric: VitalMetric, accumulated: Accumulated) => ({
|
|
1602
|
+
count: accumulated.count,
|
|
1603
|
+
p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
|
|
1604
|
+
average:
|
|
1605
|
+
accumulated.count > 0
|
|
1606
|
+
? Math.round(accumulated.sum / accumulated.count)
|
|
1607
|
+
: 0,
|
|
1608
|
+
...vitalRatingCounts(accumulated.histogram, metric),
|
|
1609
|
+
});
|
|
1610
|
+
const metrics = VITAL_METRICS.map((metric) => {
|
|
1611
|
+
const overall: Accumulated = { count: 0, sum: 0, histogram: [] };
|
|
1612
|
+
for (const accumulated of byDevice.get(metric)?.values() ?? []) {
|
|
1613
|
+
overall.count += accumulated.count;
|
|
1614
|
+
overall.sum += accumulated.sum;
|
|
1615
|
+
overall.histogram = mergeVitalHistograms(
|
|
1616
|
+
overall.histogram,
|
|
1617
|
+
accumulated.histogram,
|
|
1618
|
+
);
|
|
1619
|
+
}
|
|
1620
|
+
return {
|
|
1621
|
+
metric,
|
|
1622
|
+
...stats(metric, overall),
|
|
1623
|
+
devices: [...(byDevice.get(metric) ?? [])]
|
|
1624
|
+
.map(([device, accumulated]) => ({
|
|
1625
|
+
device,
|
|
1626
|
+
...stats(metric, accumulated),
|
|
1627
|
+
}))
|
|
1628
|
+
.sort((left, right) => right.count - left.count),
|
|
1629
|
+
pages: [...(byPage.get(metric) ?? [])]
|
|
1630
|
+
.map(([page, accumulated]) => ({
|
|
1631
|
+
page,
|
|
1632
|
+
...stats(metric, accumulated),
|
|
1633
|
+
}))
|
|
1634
|
+
.sort(
|
|
1635
|
+
(left, right) =>
|
|
1636
|
+
right.count - left.count || left.page.localeCompare(right.page),
|
|
1637
|
+
)
|
|
1638
|
+
.slice(0, VITALS_REPORT_PAGE_LIMIT),
|
|
1639
|
+
timeSeries: [...(byDay.get(metric) ?? [])]
|
|
1640
|
+
.sort(([left], [right]) => left - right)
|
|
1641
|
+
.map(([timestamp, accumulated]) => ({
|
|
1642
|
+
timestamp,
|
|
1643
|
+
count: accumulated.count,
|
|
1644
|
+
p75: vitalHistogramPercentile(accumulated.histogram, 0.75),
|
|
1645
|
+
})),
|
|
1646
|
+
};
|
|
1647
|
+
});
|
|
1648
|
+
return {
|
|
1649
|
+
metrics,
|
|
1650
|
+
truncated: false,
|
|
1651
|
+
metadata: {
|
|
1652
|
+
basis: "dailyRollup" as const,
|
|
1653
|
+
coverage: await coverageForRange(
|
|
1654
|
+
ctx,
|
|
1655
|
+
[args.siteId],
|
|
1656
|
+
args.from,
|
|
1657
|
+
args.to,
|
|
1658
|
+
{ datasets: ["vitalRollups"], complete: true },
|
|
1659
|
+
),
|
|
1660
|
+
range: rangeSemantics(args.from, args.to),
|
|
1661
|
+
},
|
|
694
1662
|
};
|
|
695
1663
|
},
|
|
696
1664
|
});
|
|
697
1665
|
|
|
1666
|
+
const coverageDatasetValidator = v.union(
|
|
1667
|
+
v.literal("overviewHour"),
|
|
1668
|
+
v.literal("overviewDay"),
|
|
1669
|
+
v.literal("overviewLocalDay"),
|
|
1670
|
+
v.literal("events"),
|
|
1671
|
+
v.literal("sessions"),
|
|
1672
|
+
v.literal("conversions"),
|
|
1673
|
+
v.literal("goals"),
|
|
1674
|
+
v.literal("funnels"),
|
|
1675
|
+
v.literal("affiliates"),
|
|
1676
|
+
v.literal("vitals"),
|
|
1677
|
+
v.literal("siteMap"),
|
|
1678
|
+
);
|
|
1679
|
+
|
|
1680
|
+
export const dataCoverage = query({
|
|
1681
|
+
args: {
|
|
1682
|
+
siteId: v.id("sites"),
|
|
1683
|
+
from: v.number(),
|
|
1684
|
+
to: v.number(),
|
|
1685
|
+
},
|
|
1686
|
+
returns: v.object({
|
|
1687
|
+
siteId: v.id("sites"),
|
|
1688
|
+
from: v.number(),
|
|
1689
|
+
to: v.number(),
|
|
1690
|
+
datasets: v.array(
|
|
1691
|
+
v.object({
|
|
1692
|
+
dataset: coverageDatasetValidator,
|
|
1693
|
+
coverage: resultCoverageValidator,
|
|
1694
|
+
}),
|
|
1695
|
+
),
|
|
1696
|
+
}),
|
|
1697
|
+
handler: async (ctx, args) => {
|
|
1698
|
+
validateRange(args.from, args.to, MAX_REPORT_RANGE_DAYS);
|
|
1699
|
+
const datasets = [
|
|
1700
|
+
"overviewHour",
|
|
1701
|
+
"overviewDay",
|
|
1702
|
+
"overviewLocalDay",
|
|
1703
|
+
"events",
|
|
1704
|
+
"sessions",
|
|
1705
|
+
"conversions",
|
|
1706
|
+
"goals",
|
|
1707
|
+
"funnels",
|
|
1708
|
+
"affiliates",
|
|
1709
|
+
"vitals",
|
|
1710
|
+
"siteMap",
|
|
1711
|
+
] as const;
|
|
1712
|
+
return {
|
|
1713
|
+
siteId: args.siteId,
|
|
1714
|
+
from: args.from,
|
|
1715
|
+
to: args.to,
|
|
1716
|
+
datasets: await Promise.all(
|
|
1717
|
+
datasets.map(async (dataset) => ({
|
|
1718
|
+
dataset,
|
|
1719
|
+
coverage: await datasetCoverageForRange(
|
|
1720
|
+
ctx,
|
|
1721
|
+
args.siteId,
|
|
1722
|
+
dataset,
|
|
1723
|
+
args.from,
|
|
1724
|
+
args.to,
|
|
1725
|
+
),
|
|
1726
|
+
})),
|
|
1727
|
+
),
|
|
1728
|
+
};
|
|
1729
|
+
},
|
|
1730
|
+
});
|
|
1731
|
+
|
|
1732
|
+
async function dailyRollupsAvailable(
|
|
1733
|
+
ctx: QueryCtx,
|
|
1734
|
+
siteId: Id<"sites">,
|
|
1735
|
+
from: number,
|
|
1736
|
+
to: number,
|
|
1737
|
+
dataset: "goalRollups" | "funnelRollups" | "affiliateRollups",
|
|
1738
|
+
) {
|
|
1739
|
+
if (!isAlignedRange(from, to, DAY_MS)) return false;
|
|
1740
|
+
const coverage = await ctx.db
|
|
1741
|
+
.query("analyticsCoverage")
|
|
1742
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
1743
|
+
.unique();
|
|
1744
|
+
if (!coverage || coverage.rollupCompleteFrom > from) {
|
|
1745
|
+
return false;
|
|
1746
|
+
}
|
|
1747
|
+
const retainedBefore = coverage.retained.find(
|
|
1748
|
+
(item) => item.dataset === dataset,
|
|
1749
|
+
)?.before;
|
|
1750
|
+
return retainedBefore === undefined || from >= retainedBefore;
|
|
1751
|
+
}
|
|
1752
|
+
|
|
1753
|
+
async function loadGoalDailyRollups(
|
|
1754
|
+
ctx: QueryCtx,
|
|
1755
|
+
siteId: Id<"sites">,
|
|
1756
|
+
goalIds: Id<"goals">[],
|
|
1757
|
+
from: number,
|
|
1758
|
+
to: number,
|
|
1759
|
+
) {
|
|
1760
|
+
const days = Math.floor((to - from) / DAY_MS) + 1;
|
|
1761
|
+
requireFeatureRollupBudget(goalIds.length, days, "goal daily rollups");
|
|
1762
|
+
const rows: Doc<"goalDailyRollups">[] = [];
|
|
1763
|
+
for (const goalId of goalIds) {
|
|
1764
|
+
const perGoalLimit = days * REPORT_ROLLUP_SHARDS;
|
|
1765
|
+
const goalRows = await ctx.db
|
|
1766
|
+
.query("goalDailyRollups")
|
|
1767
|
+
.withIndex("by_siteId_and_goalId_and_bucketStart", (range) =>
|
|
1768
|
+
range
|
|
1769
|
+
.eq("siteId", siteId)
|
|
1770
|
+
.eq("goalId", goalId)
|
|
1771
|
+
.gte("bucketStart", from)
|
|
1772
|
+
.lte("bucketStart", to),
|
|
1773
|
+
)
|
|
1774
|
+
.take(perGoalLimit + 1);
|
|
1775
|
+
failIfIncomplete(goalRows, perGoalLimit, "goal daily rollups");
|
|
1776
|
+
rows.push(...goalRows);
|
|
1777
|
+
await requireFeatureReadHeadroom(ctx, "goal daily rollups");
|
|
1778
|
+
}
|
|
1779
|
+
return rows;
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
async function loadFunnelDailyRollups(
|
|
1783
|
+
ctx: QueryCtx,
|
|
1784
|
+
siteId: Id<"sites">,
|
|
1785
|
+
funnelIds: Id<"funnels">[],
|
|
1786
|
+
from: number,
|
|
1787
|
+
to: number,
|
|
1788
|
+
) {
|
|
1789
|
+
const days = Math.floor((to - from) / DAY_MS) + 1;
|
|
1790
|
+
requireFeatureRollupBudget(funnelIds.length, days, "funnel daily rollups");
|
|
1791
|
+
const rows: Doc<"funnelDailyRollups">[] = [];
|
|
1792
|
+
for (const funnelId of funnelIds) {
|
|
1793
|
+
const perFunnelLimit = days * REPORT_ROLLUP_SHARDS;
|
|
1794
|
+
const funnelRows = await ctx.db
|
|
1795
|
+
.query("funnelDailyRollups")
|
|
1796
|
+
.withIndex("by_siteId_and_funnelId_and_bucketStart", (range) =>
|
|
1797
|
+
range
|
|
1798
|
+
.eq("siteId", siteId)
|
|
1799
|
+
.eq("funnelId", funnelId)
|
|
1800
|
+
.gte("bucketStart", from)
|
|
1801
|
+
.lte("bucketStart", to),
|
|
1802
|
+
)
|
|
1803
|
+
.take(perFunnelLimit + 1);
|
|
1804
|
+
failIfIncomplete(funnelRows, perFunnelLimit, "funnel daily rollups");
|
|
1805
|
+
rows.push(...funnelRows);
|
|
1806
|
+
await requireFeatureReadHeadroom(ctx, "funnel daily rollups");
|
|
1807
|
+
}
|
|
1808
|
+
return rows;
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
async function loadAffiliateDailyRollups(
|
|
1812
|
+
ctx: QueryCtx,
|
|
1813
|
+
siteId: Id<"sites">,
|
|
1814
|
+
affiliateIds: Id<"affiliates">[],
|
|
1815
|
+
from: number,
|
|
1816
|
+
to: number,
|
|
1817
|
+
) {
|
|
1818
|
+
const days = Math.floor((to - from) / DAY_MS) + 1;
|
|
1819
|
+
requireFeatureRollupBudget(
|
|
1820
|
+
affiliateIds.length,
|
|
1821
|
+
days,
|
|
1822
|
+
"affiliate daily rollups",
|
|
1823
|
+
);
|
|
1824
|
+
const rows: Doc<"affiliateDailyRollups">[] = [];
|
|
1825
|
+
for (const affiliateId of affiliateIds) {
|
|
1826
|
+
const perAffiliateLimit = days * REPORT_ROLLUP_SHARDS;
|
|
1827
|
+
const affiliateRows = await ctx.db
|
|
1828
|
+
.query("affiliateDailyRollups")
|
|
1829
|
+
.withIndex("by_siteId_and_affiliateId_and_bucketStart", (range) =>
|
|
1830
|
+
range
|
|
1831
|
+
.eq("siteId", siteId)
|
|
1832
|
+
.eq("affiliateId", affiliateId)
|
|
1833
|
+
.gte("bucketStart", from)
|
|
1834
|
+
.lte("bucketStart", to),
|
|
1835
|
+
)
|
|
1836
|
+
.take(perAffiliateLimit + 1);
|
|
1837
|
+
failIfIncomplete(
|
|
1838
|
+
affiliateRows,
|
|
1839
|
+
perAffiliateLimit,
|
|
1840
|
+
"affiliate daily rollups",
|
|
1841
|
+
);
|
|
1842
|
+
rows.push(...affiliateRows);
|
|
1843
|
+
await requireFeatureReadHeadroom(ctx, "affiliate daily rollups");
|
|
1844
|
+
}
|
|
1845
|
+
return rows;
|
|
1846
|
+
}
|
|
1847
|
+
|
|
1848
|
+
function requireFeatureRollupBudget(
|
|
1849
|
+
entities: number,
|
|
1850
|
+
days: number,
|
|
1851
|
+
dataset: string,
|
|
1852
|
+
) {
|
|
1853
|
+
const maximumRows = entities * days * REPORT_ROLLUP_SHARDS;
|
|
1854
|
+
if (days > MAX_FEATURE_ROLLUP_DAYS || maximumRows > MAX_FEATURE_ROLLUP_ROWS) {
|
|
1855
|
+
fail("REPORT_INCOMPLETE", `${dataset} exceeds its complete read budget`, {
|
|
1856
|
+
dataset,
|
|
1857
|
+
maximumDays: MAX_FEATURE_ROLLUP_DAYS,
|
|
1858
|
+
maximumRows: MAX_FEATURE_ROLLUP_ROWS,
|
|
1859
|
+
});
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
async function hasFeatureReadHeadroom(ctx: QueryCtx) {
|
|
1864
|
+
const metrics = await ctx.meta.getTransactionMetrics();
|
|
1865
|
+
return (
|
|
1866
|
+
metrics.bytesRead.remaining >= FEATURE_REPORT_BYTE_RESERVE &&
|
|
1867
|
+
metrics.documentsRead.remaining >= FEATURE_REPORT_DOCUMENT_RESERVE
|
|
1868
|
+
);
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
async function requireFeatureReadHeadroom(ctx: QueryCtx, dataset: string) {
|
|
1872
|
+
const metrics = await ctx.meta.getTransactionMetrics();
|
|
1873
|
+
if (
|
|
1874
|
+
metrics.bytesRead.remaining < FEATURE_REPORT_BYTE_RESERVE ||
|
|
1875
|
+
metrics.documentsRead.remaining < FEATURE_REPORT_DOCUMENT_RESERVE
|
|
1876
|
+
) {
|
|
1877
|
+
fail(
|
|
1878
|
+
"REPORT_INCOMPLETE",
|
|
1879
|
+
`${dataset} exceeds its byte or document budget`,
|
|
1880
|
+
{
|
|
1881
|
+
dataset,
|
|
1882
|
+
bytesRead: metrics.bytesRead.used,
|
|
1883
|
+
documentsRead: metrics.documentsRead.used,
|
|
1884
|
+
},
|
|
1885
|
+
);
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
function failIfIncomplete<T>(rows: T[], maximum: number, dataset: string) {
|
|
1890
|
+
if (rows.length > maximum) {
|
|
1891
|
+
fail(
|
|
1892
|
+
"REPORT_INCOMPLETE",
|
|
1893
|
+
`${dataset} exceeds the complete report read budget`,
|
|
1894
|
+
{
|
|
1895
|
+
maximumRows: maximum,
|
|
1896
|
+
dataset,
|
|
1897
|
+
},
|
|
1898
|
+
);
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
|
|
1902
|
+
async function coverageForRange(
|
|
1903
|
+
ctx: QueryCtx,
|
|
1904
|
+
siteIds: Id<"sites">[],
|
|
1905
|
+
from: number,
|
|
1906
|
+
to: number,
|
|
1907
|
+
options: {
|
|
1908
|
+
datasets: CoverageSource[];
|
|
1909
|
+
complete?: boolean;
|
|
1910
|
+
generation?: 1 | null;
|
|
1911
|
+
},
|
|
1912
|
+
) {
|
|
1913
|
+
const controls = await Promise.all(
|
|
1914
|
+
siteIds.map((siteId) =>
|
|
1915
|
+
ctx.db
|
|
1916
|
+
.query("analyticsCoverage")
|
|
1917
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
1918
|
+
.unique(),
|
|
1919
|
+
),
|
|
1920
|
+
);
|
|
1921
|
+
const sourceRanges = await Promise.all(
|
|
1922
|
+
siteIds.map((siteId) => sourceSetRange(ctx, siteId, options.datasets)),
|
|
1923
|
+
);
|
|
1924
|
+
const retainedBefore = controls.reduce<number | null>((latest, control) => {
|
|
1925
|
+
const before = control?.retained
|
|
1926
|
+
.filter((item) =>
|
|
1927
|
+
options.datasets.some((dataset) => dataset === item.dataset),
|
|
1928
|
+
)
|
|
1929
|
+
.reduce<number | undefined>(
|
|
1930
|
+
(latest, item) => Math.max(latest ?? 0, item.before),
|
|
1931
|
+
undefined,
|
|
1932
|
+
);
|
|
1933
|
+
return before === undefined ? latest : Math.max(latest ?? 0, before);
|
|
1934
|
+
}, null);
|
|
1935
|
+
const availableFrom =
|
|
1936
|
+
options.generation === 1
|
|
1937
|
+
? controls.length > 0 && controls.every(Boolean)
|
|
1938
|
+
? Math.max(...controls.map((control) => control!.rollupCompleteFrom))
|
|
1939
|
+
: null
|
|
1940
|
+
: sourceRanges.length > 0 && sourceRanges.every(Boolean)
|
|
1941
|
+
? Math.max(...sourceRanges.map((range) => range!.from))
|
|
1942
|
+
: null;
|
|
1943
|
+
const availableThrough =
|
|
1944
|
+
sourceRanges.length > 0 && sourceRanges.every(Boolean)
|
|
1945
|
+
? Math.min(...sourceRanges.map((range) => range!.through))
|
|
1946
|
+
: null;
|
|
1947
|
+
let state: "complete" | "partial" | "retained" | "unavailable";
|
|
1948
|
+
if (retainedBefore !== null && to < retainedBefore) {
|
|
1949
|
+
state = "unavailable";
|
|
1950
|
+
} else if (retainedBefore !== null && from < retainedBefore) {
|
|
1951
|
+
state = "partial";
|
|
1952
|
+
} else if (
|
|
1953
|
+
options.complete ||
|
|
1954
|
+
(availableFrom !== null &&
|
|
1955
|
+
availableThrough !== null &&
|
|
1956
|
+
from >= availableFrom &&
|
|
1957
|
+
to <= availableThrough)
|
|
1958
|
+
) {
|
|
1959
|
+
state = "complete";
|
|
1960
|
+
} else if (sourceRanges.some(Boolean)) {
|
|
1961
|
+
state = "retained";
|
|
1962
|
+
} else {
|
|
1963
|
+
state = "unavailable";
|
|
1964
|
+
}
|
|
1965
|
+
return {
|
|
1966
|
+
state,
|
|
1967
|
+
generation: options.generation ?? null,
|
|
1968
|
+
availableFrom,
|
|
1969
|
+
availableThrough,
|
|
1970
|
+
retainedBefore,
|
|
1971
|
+
};
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
async function assertSourceAvailable(
|
|
1975
|
+
ctx: QueryCtx,
|
|
1976
|
+
siteIds: Id<"sites">[],
|
|
1977
|
+
sources: CoverageSource[],
|
|
1978
|
+
from: number,
|
|
1979
|
+
) {
|
|
1980
|
+
const rows = await Promise.all(
|
|
1981
|
+
siteIds.map((siteId) =>
|
|
1982
|
+
ctx.db
|
|
1983
|
+
.query("analyticsCoverage")
|
|
1984
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
1985
|
+
.unique(),
|
|
1986
|
+
),
|
|
1987
|
+
);
|
|
1988
|
+
const cutoff = rows.reduce<number | null>((latest, row) => {
|
|
1989
|
+
for (const retained of row?.retained ?? []) {
|
|
1990
|
+
if (sources.some((source) => source === retained.dataset)) {
|
|
1991
|
+
latest = Math.max(latest ?? 0, retained.before);
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
return latest;
|
|
1995
|
+
}, null);
|
|
1996
|
+
if (cutoff !== null && from < cutoff) {
|
|
1997
|
+
fail(
|
|
1998
|
+
"REPORT_INCOMPLETE",
|
|
1999
|
+
"requested report range intersects retained data",
|
|
2000
|
+
{
|
|
2001
|
+
retainedBefore: cutoff,
|
|
2002
|
+
},
|
|
2003
|
+
);
|
|
2004
|
+
}
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
async function datasetCoverageForRange(
|
|
2008
|
+
ctx: QueryCtx,
|
|
2009
|
+
siteId: Id<"sites">,
|
|
2010
|
+
dataset: PublicCoverageDataset,
|
|
2011
|
+
from: number,
|
|
2012
|
+
to: number,
|
|
2013
|
+
) {
|
|
2014
|
+
if (dataset === "siteMap") {
|
|
2015
|
+
return siteMapDatasetCoverage(ctx, siteId, from, to);
|
|
2016
|
+
}
|
|
2017
|
+
// Vitals have no legacy fallback source; the rollup either covers the range
|
|
2018
|
+
// or the coverage states say how much of it exists.
|
|
2019
|
+
if (dataset === "vitals") {
|
|
2020
|
+
return coverageForRange(ctx, [siteId], from, to, {
|
|
2021
|
+
datasets: ["vitalRollups"],
|
|
2022
|
+
complete: false,
|
|
2023
|
+
});
|
|
2024
|
+
}
|
|
2025
|
+
const sourceOptions: CoverageSource[][] =
|
|
2026
|
+
dataset === "goals"
|
|
2027
|
+
? [["goalRollups"], ["goalCompletions"]]
|
|
2028
|
+
: dataset === "funnels"
|
|
2029
|
+
? [["funnelRollups"], ["funnelSteps"]]
|
|
2030
|
+
: dataset === "affiliates"
|
|
2031
|
+
? [["affiliateRollups"], ["affiliateStats", "sessions"]]
|
|
2032
|
+
: [[dataset]];
|
|
2033
|
+
const options = (
|
|
2034
|
+
await Promise.all(
|
|
2035
|
+
sourceOptions.map(async (sources) => {
|
|
2036
|
+
const rollupDataset = sources[0] as
|
|
2037
|
+
"goalRollups" | "funnelRollups" | "affiliateRollups";
|
|
2038
|
+
const isRollup = rollupDataset.endsWith("Rollups");
|
|
2039
|
+
const complete = isRollup
|
|
2040
|
+
? await dailyRollupsAvailable(ctx, siteId, from, to, rollupDataset)
|
|
2041
|
+
: false;
|
|
2042
|
+
if (isRollup && !complete) return null;
|
|
2043
|
+
return coverageForRange(ctx, [siteId], from, to, {
|
|
2044
|
+
datasets: sources,
|
|
2045
|
+
complete,
|
|
2046
|
+
generation: complete ? 1 : null,
|
|
2047
|
+
});
|
|
2048
|
+
}),
|
|
2049
|
+
)
|
|
2050
|
+
).filter((option): option is NonNullable<typeof option> => option !== null);
|
|
2051
|
+
return options.sort(
|
|
2052
|
+
(left, right) => coverageRank(right.state) - coverageRank(left.state),
|
|
2053
|
+
)[0]!;
|
|
2054
|
+
}
|
|
2055
|
+
|
|
2056
|
+
/**
|
|
2057
|
+
* The site map's coverage without reading its rows: it starts at the site's
|
|
2058
|
+
* `siteMapSince` and ends where retention cut it. The report itself may still
|
|
2059
|
+
* narrow the window when the range exceeds its row budget.
|
|
2060
|
+
*/
|
|
2061
|
+
async function siteMapDatasetCoverage(
|
|
2062
|
+
ctx: QueryCtx,
|
|
2063
|
+
siteId: Id<"sites">,
|
|
2064
|
+
from: number,
|
|
2065
|
+
to: number,
|
|
2066
|
+
) {
|
|
2067
|
+
const control = await ctx.db
|
|
2068
|
+
.query("analyticsCoverage")
|
|
2069
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
2070
|
+
.unique();
|
|
2071
|
+
const since = control?.siteMapSince ?? null;
|
|
2072
|
+
const retainedBefore =
|
|
2073
|
+
control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
|
|
2074
|
+
?.before ?? null;
|
|
2075
|
+
const unavailable =
|
|
2076
|
+
since === null ||
|
|
2077
|
+
to < since ||
|
|
2078
|
+
(retainedBefore !== null && to < retainedBefore);
|
|
2079
|
+
const state = unavailable
|
|
2080
|
+
? ("unavailable" as const)
|
|
2081
|
+
: from < Math.max(since, retainedBefore ?? 0)
|
|
2082
|
+
? ("partial" as const)
|
|
2083
|
+
: ("complete" as const);
|
|
2084
|
+
return {
|
|
2085
|
+
state,
|
|
2086
|
+
generation: 1 as const,
|
|
2087
|
+
availableFrom: since,
|
|
2088
|
+
availableThrough: since === null ? null : to,
|
|
2089
|
+
retainedBefore,
|
|
2090
|
+
};
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
type PublicCoverageDataset =
|
|
2094
|
+
| "overviewHour"
|
|
2095
|
+
| "overviewDay"
|
|
2096
|
+
| "overviewLocalDay"
|
|
2097
|
+
| "events"
|
|
2098
|
+
| "sessions"
|
|
2099
|
+
| "conversions"
|
|
2100
|
+
| "goals"
|
|
2101
|
+
| "funnels"
|
|
2102
|
+
| "affiliates"
|
|
2103
|
+
| "vitals"
|
|
2104
|
+
| "siteMap";
|
|
2105
|
+
|
|
2106
|
+
type CoverageSource =
|
|
2107
|
+
| "overviewHour"
|
|
2108
|
+
| "overviewDay"
|
|
2109
|
+
| "overviewLocalDay"
|
|
2110
|
+
| "events"
|
|
2111
|
+
| "sessions"
|
|
2112
|
+
| "conversions"
|
|
2113
|
+
| "goalCompletions"
|
|
2114
|
+
| "funnelSteps"
|
|
2115
|
+
| "affiliateStats"
|
|
2116
|
+
| "goalRollups"
|
|
2117
|
+
| "funnelRollups"
|
|
2118
|
+
| "affiliateRollups"
|
|
2119
|
+
| "vitalRollups";
|
|
2120
|
+
|
|
2121
|
+
type SourceRange = { from: number; through: number };
|
|
2122
|
+
|
|
2123
|
+
async function sourceSetRange(
|
|
2124
|
+
ctx: QueryCtx,
|
|
2125
|
+
siteId: Id<"sites">,
|
|
2126
|
+
sources: CoverageSource[],
|
|
2127
|
+
): Promise<SourceRange | null> {
|
|
2128
|
+
const ranges = (
|
|
2129
|
+
await Promise.all(sources.map((source) => sourceRange(ctx, siteId, source)))
|
|
2130
|
+
).filter((range): range is SourceRange => range !== null);
|
|
2131
|
+
if (ranges.length === 0) return null;
|
|
2132
|
+
return {
|
|
2133
|
+
from: Math.min(...ranges.map((range) => range.from)),
|
|
2134
|
+
through: Math.max(...ranges.map((range) => range.through)),
|
|
2135
|
+
};
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
async function sourceRange(
|
|
2139
|
+
ctx: QueryCtx,
|
|
2140
|
+
siteId: Id<"sites">,
|
|
2141
|
+
source: CoverageSource,
|
|
2142
|
+
): Promise<SourceRange | null> {
|
|
2143
|
+
if (
|
|
2144
|
+
source === "overviewHour" ||
|
|
2145
|
+
source === "overviewDay" ||
|
|
2146
|
+
source === "overviewLocalDay"
|
|
2147
|
+
) {
|
|
2148
|
+
const granularity =
|
|
2149
|
+
source === "overviewHour"
|
|
2150
|
+
? ("hour" as const)
|
|
2151
|
+
: source === "overviewDay"
|
|
2152
|
+
? ("day" as const)
|
|
2153
|
+
: ("localDay" as const);
|
|
2154
|
+
const [first, last] = await Promise.all([
|
|
2155
|
+
ctx.db
|
|
2156
|
+
.query("aggregateBuckets")
|
|
2157
|
+
.withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
|
|
2158
|
+
range.eq("siteId", siteId).eq("granularity", granularity),
|
|
2159
|
+
)
|
|
2160
|
+
.order("asc")
|
|
2161
|
+
.take(1),
|
|
2162
|
+
ctx.db
|
|
2163
|
+
.query("aggregateBuckets")
|
|
2164
|
+
.withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
|
|
2165
|
+
range.eq("siteId", siteId).eq("granularity", granularity),
|
|
2166
|
+
)
|
|
2167
|
+
.order("desc")
|
|
2168
|
+
.take(1),
|
|
2169
|
+
]);
|
|
2170
|
+
return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
|
|
2171
|
+
}
|
|
2172
|
+
if (source === "events") {
|
|
2173
|
+
const [first, last] = await Promise.all([
|
|
2174
|
+
ctx.db
|
|
2175
|
+
.query("eventBatches")
|
|
2176
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
2177
|
+
range.eq("siteId", siteId),
|
|
2178
|
+
)
|
|
2179
|
+
.order("asc")
|
|
2180
|
+
.take(1),
|
|
2181
|
+
ctx.db
|
|
2182
|
+
.query("eventBatches")
|
|
2183
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
2184
|
+
range.eq("siteId", siteId),
|
|
2185
|
+
)
|
|
2186
|
+
.order("desc")
|
|
2187
|
+
.take(1),
|
|
2188
|
+
]);
|
|
2189
|
+
return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
|
|
2190
|
+
}
|
|
2191
|
+
if (source === "sessions") {
|
|
2192
|
+
const [first, last] = await Promise.all([
|
|
2193
|
+
ctx.db
|
|
2194
|
+
.query("sessions")
|
|
2195
|
+
.withIndex("by_siteId_and_startedAt", (range) =>
|
|
2196
|
+
range.eq("siteId", siteId),
|
|
2197
|
+
)
|
|
2198
|
+
.order("asc")
|
|
2199
|
+
.take(1),
|
|
2200
|
+
ctx.db
|
|
2201
|
+
.query("sessions")
|
|
2202
|
+
.withIndex("by_siteId_and_lastSeenAt", (range) =>
|
|
2203
|
+
range.eq("siteId", siteId),
|
|
2204
|
+
)
|
|
2205
|
+
.order("desc")
|
|
2206
|
+
.take(1),
|
|
2207
|
+
]);
|
|
2208
|
+
return bounds(first[0]?.startedAt, last[0]?.lastSeenAt);
|
|
2209
|
+
}
|
|
2210
|
+
if (source === "conversions") {
|
|
2211
|
+
const [first, last] = await Promise.all([
|
|
2212
|
+
ctx.db
|
|
2213
|
+
.query("conversions")
|
|
2214
|
+
.withIndex("by_siteId_and_timestamp", (range) =>
|
|
2215
|
+
range.eq("siteId", siteId),
|
|
2216
|
+
)
|
|
2217
|
+
.order("asc")
|
|
2218
|
+
.take(1),
|
|
2219
|
+
ctx.db
|
|
2220
|
+
.query("conversions")
|
|
2221
|
+
.withIndex("by_siteId_and_timestamp", (range) =>
|
|
2222
|
+
range.eq("siteId", siteId),
|
|
2223
|
+
)
|
|
2224
|
+
.order("desc")
|
|
2225
|
+
.take(1),
|
|
2226
|
+
]);
|
|
2227
|
+
return bounds(first[0]?.timestamp, last[0]?.timestamp);
|
|
2228
|
+
}
|
|
2229
|
+
if (source === "goalCompletions") {
|
|
2230
|
+
const [first, last] = await Promise.all([
|
|
2231
|
+
ctx.db
|
|
2232
|
+
.query("goalCompletions")
|
|
2233
|
+
.withIndex("by_siteId_and_timestamp", (range) =>
|
|
2234
|
+
range.eq("siteId", siteId),
|
|
2235
|
+
)
|
|
2236
|
+
.order("asc")
|
|
2237
|
+
.take(1),
|
|
2238
|
+
ctx.db
|
|
2239
|
+
.query("goalCompletions")
|
|
2240
|
+
.withIndex("by_siteId_and_timestamp", (range) =>
|
|
2241
|
+
range.eq("siteId", siteId),
|
|
2242
|
+
)
|
|
2243
|
+
.order("desc")
|
|
2244
|
+
.take(1),
|
|
2245
|
+
]);
|
|
2246
|
+
return bounds(first[0]?.timestamp, last[0]?.timestamp);
|
|
2247
|
+
}
|
|
2248
|
+
const table =
|
|
2249
|
+
source === "funnelSteps"
|
|
2250
|
+
? ("funnelStepBuckets" as const)
|
|
2251
|
+
: source === "affiliateStats"
|
|
2252
|
+
? ("affiliateDailyStats" as const)
|
|
2253
|
+
: source === "goalRollups"
|
|
2254
|
+
? ("goalDailyRollups" as const)
|
|
2255
|
+
: source === "funnelRollups"
|
|
2256
|
+
? ("funnelDailyRollups" as const)
|
|
2257
|
+
: source === "vitalRollups"
|
|
2258
|
+
? ("vitalRollups" as const)
|
|
2259
|
+
: ("affiliateDailyRollups" as const);
|
|
2260
|
+
const [first, last] = await Promise.all([
|
|
2261
|
+
ctx.db
|
|
2262
|
+
.query(table)
|
|
2263
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
2264
|
+
range.eq("siteId", siteId),
|
|
2265
|
+
)
|
|
2266
|
+
.order("asc")
|
|
2267
|
+
.take(1),
|
|
2268
|
+
ctx.db
|
|
2269
|
+
.query(table)
|
|
2270
|
+
.withIndex("by_siteId_and_bucketStart", (range) =>
|
|
2271
|
+
range.eq("siteId", siteId),
|
|
2272
|
+
)
|
|
2273
|
+
.order("desc")
|
|
2274
|
+
.take(1),
|
|
2275
|
+
]);
|
|
2276
|
+
return bounds(first[0]?.bucketStart, last[0]?.bucketStart);
|
|
2277
|
+
}
|
|
2278
|
+
|
|
2279
|
+
function bounds(from: number | undefined, through: number | undefined) {
|
|
2280
|
+
return from === undefined || through === undefined ? null : { from, through };
|
|
2281
|
+
}
|
|
2282
|
+
|
|
2283
|
+
function coverageRank(
|
|
2284
|
+
state: "complete" | "partial" | "retained" | "unavailable",
|
|
2285
|
+
) {
|
|
2286
|
+
if (state === "complete") return 3;
|
|
2287
|
+
if (state === "retained") return 2;
|
|
2288
|
+
if (state === "partial") return 1;
|
|
2289
|
+
return 0;
|
|
2290
|
+
}
|
|
698
2291
|
|
|
699
2292
|
type MetricTotals = {
|
|
700
2293
|
events: number;
|
|
@@ -706,6 +2299,9 @@ type MetricTotals = {
|
|
|
706
2299
|
visitors: number;
|
|
707
2300
|
conversions: number;
|
|
708
2301
|
revenueCents: number;
|
|
2302
|
+
durationMs: number;
|
|
2303
|
+
continuedSessions: number;
|
|
2304
|
+
newVisitors: number;
|
|
709
2305
|
};
|
|
710
2306
|
|
|
711
2307
|
function emptyTotals(): MetricTotals {
|
|
@@ -719,10 +2315,23 @@ function emptyTotals(): MetricTotals {
|
|
|
719
2315
|
visitors: 0,
|
|
720
2316
|
conversions: 0,
|
|
721
2317
|
revenueCents: 0,
|
|
2318
|
+
durationMs: 0,
|
|
2319
|
+
continuedSessions: 0,
|
|
2320
|
+
newVisitors: 0,
|
|
722
2321
|
};
|
|
723
2322
|
}
|
|
724
2323
|
|
|
725
|
-
function addMetrics(
|
|
2324
|
+
function addMetrics(
|
|
2325
|
+
target: MetricTotals,
|
|
2326
|
+
source: Omit<
|
|
2327
|
+
MetricTotals,
|
|
2328
|
+
"durationMs" | "continuedSessions" | "newVisitors"
|
|
2329
|
+
> & {
|
|
2330
|
+
durationMs?: number;
|
|
2331
|
+
continuedSessions?: number;
|
|
2332
|
+
newVisitors?: number;
|
|
2333
|
+
},
|
|
2334
|
+
) {
|
|
726
2335
|
target.events += source.events;
|
|
727
2336
|
target.pageviews += source.pageviews;
|
|
728
2337
|
target.clicks += source.clicks;
|
|
@@ -732,27 +2341,77 @@ function addMetrics(target: MetricTotals, source: MetricTotals) {
|
|
|
732
2341
|
target.visitors += source.visitors;
|
|
733
2342
|
target.conversions += source.conversions;
|
|
734
2343
|
target.revenueCents += source.revenueCents;
|
|
2344
|
+
target.durationMs += source.durationMs ?? 0;
|
|
2345
|
+
target.continuedSessions += source.continuedSessions ?? 0;
|
|
2346
|
+
target.newVisitors += source.newVisitors ?? 0;
|
|
735
2347
|
}
|
|
736
2348
|
|
|
737
2349
|
function validateSiteIds<T>(siteIds: T[]): T[] {
|
|
738
2350
|
const unique = [...new Set(siteIds)];
|
|
739
|
-
if (
|
|
740
|
-
|
|
2351
|
+
if (
|
|
2352
|
+
unique.length < 1 ||
|
|
2353
|
+
unique.length > MAX_REPORT_SITES ||
|
|
2354
|
+
unique.length !== siteIds.length
|
|
2355
|
+
) {
|
|
2356
|
+
fail(
|
|
2357
|
+
"INVALID_ARGUMENT",
|
|
2358
|
+
`siteIds must contain 1-${MAX_REPORT_SITES} unique IDs`,
|
|
2359
|
+
);
|
|
741
2360
|
}
|
|
742
2361
|
return unique;
|
|
743
2362
|
}
|
|
744
2363
|
|
|
745
2364
|
function validateRange(from: number, to: number, maxDays: number) {
|
|
746
2365
|
if (
|
|
747
|
-
!Number.
|
|
748
|
-
!Number.
|
|
2366
|
+
!Number.isSafeInteger(from) ||
|
|
2367
|
+
!Number.isSafeInteger(to) ||
|
|
749
2368
|
from < 0 ||
|
|
750
|
-
to
|
|
751
|
-
to - from
|
|
2369
|
+
to < from ||
|
|
2370
|
+
to - from >= maxDays * DAY_MS
|
|
752
2371
|
) {
|
|
753
|
-
fail(
|
|
2372
|
+
fail(
|
|
2373
|
+
"INVALID_ARGUMENT",
|
|
2374
|
+
`range must use inclusive integer milliseconds and span at most ${maxDays} UTC days`,
|
|
2375
|
+
);
|
|
754
2376
|
}
|
|
755
|
-
return { from
|
|
2377
|
+
return { from, to };
|
|
2378
|
+
}
|
|
2379
|
+
|
|
2380
|
+
function isAlignedRange(from: number, to: number, intervalMs: number) {
|
|
2381
|
+
return from % intervalMs === 0 && (to + 1) % intervalMs === 0;
|
|
2382
|
+
}
|
|
2383
|
+
|
|
2384
|
+
function requireAlignedRange(
|
|
2385
|
+
from: number,
|
|
2386
|
+
to: number,
|
|
2387
|
+
intervalMs: number,
|
|
2388
|
+
report: string,
|
|
2389
|
+
siteTimezone?: string,
|
|
2390
|
+
) {
|
|
2391
|
+
if (!isAlignedRange(from, to, intervalMs)) {
|
|
2392
|
+
fail(
|
|
2393
|
+
"REPORT_INCOMPLETE",
|
|
2394
|
+
siteTimezone
|
|
2395
|
+
? `${report} requires complete inclusive UTC bucket boundaries, or whole calendar days in ${siteTimezone}`
|
|
2396
|
+
: `${report} requires complete inclusive UTC bucket boundaries`,
|
|
2397
|
+
{
|
|
2398
|
+
from,
|
|
2399
|
+
to,
|
|
2400
|
+
intervalMs,
|
|
2401
|
+
timezone: "UTC",
|
|
2402
|
+
siteTimezone: siteTimezone ?? null,
|
|
2403
|
+
},
|
|
2404
|
+
);
|
|
2405
|
+
}
|
|
2406
|
+
}
|
|
2407
|
+
|
|
2408
|
+
function rangeSemantics(from: number, to: number, timezone = "UTC") {
|
|
2409
|
+
return {
|
|
2410
|
+
from,
|
|
2411
|
+
to,
|
|
2412
|
+
boundary: "inclusive" as const,
|
|
2413
|
+
timezone,
|
|
2414
|
+
};
|
|
756
2415
|
}
|
|
757
2416
|
|
|
758
2417
|
type KeysetPaginationInput = {
|
|
@@ -881,22 +2540,23 @@ function keysetPaginationResult<T>(
|
|
|
881
2540
|
) {
|
|
882
2541
|
const page = rows.slice(0, pagination.numItems);
|
|
883
2542
|
const isDone = rows.length <= pagination.numItems;
|
|
884
|
-
const next =
|
|
885
|
-
? cursorFor(page[page.length - 1])
|
|
886
|
-
: pagination.cursor;
|
|
2543
|
+
const next =
|
|
2544
|
+
page.length > 0 ? cursorFor(page[page.length - 1]) : pagination.cursor;
|
|
887
2545
|
return {
|
|
888
2546
|
page,
|
|
889
2547
|
isDone,
|
|
890
|
-
continueCursor: btoa(
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
2548
|
+
continueCursor: btoa(
|
|
2549
|
+
JSON.stringify({
|
|
2550
|
+
v: 2,
|
|
2551
|
+
k: pagination.scope.kind,
|
|
2552
|
+
s: pagination.scope.siteId,
|
|
2553
|
+
n: pagination.scope.sessionId ?? null,
|
|
2554
|
+
f: pagination.scope.from,
|
|
2555
|
+
t: pagination.scope.to,
|
|
2556
|
+
x: next?.timestamp ?? pagination.scope.to,
|
|
2557
|
+
c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
|
|
2558
|
+
}),
|
|
2559
|
+
),
|
|
900
2560
|
};
|
|
901
2561
|
}
|
|
902
2562
|
|
|
@@ -908,26 +2568,28 @@ function journeyPaginationResult(
|
|
|
908
2568
|
const isDone = rows.length <= pagination.numItems;
|
|
909
2569
|
const last = page.at(-1);
|
|
910
2570
|
const next = last
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
2571
|
+
? {
|
|
2572
|
+
timestamp: last.timestamp,
|
|
2573
|
+
creationTime: last._creationTime,
|
|
2574
|
+
id: last._id,
|
|
915
2575
|
}
|
|
916
2576
|
: pagination.cursor;
|
|
917
2577
|
return {
|
|
918
2578
|
page,
|
|
919
2579
|
isDone,
|
|
920
|
-
continueCursor: btoa(
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
2580
|
+
continueCursor: btoa(
|
|
2581
|
+
JSON.stringify({
|
|
2582
|
+
v: 4,
|
|
2583
|
+
k: pagination.scope.kind,
|
|
2584
|
+
s: pagination.scope.siteId,
|
|
2585
|
+
n: pagination.scope.sessionId,
|
|
2586
|
+
f: pagination.scope.from,
|
|
2587
|
+
t: pagination.scope.to,
|
|
2588
|
+
x: next?.timestamp ?? pagination.scope.to,
|
|
2589
|
+
c: next?.creationTime ?? Number.MAX_SAFE_INTEGER,
|
|
2590
|
+
i: next?.id ?? "~",
|
|
2591
|
+
}),
|
|
2592
|
+
),
|
|
931
2593
|
};
|
|
932
2594
|
}
|
|
933
2595
|
|
|
@@ -936,8 +2598,15 @@ function validatePagination(options: {
|
|
|
936
2598
|
maximumRowsRead?: number;
|
|
937
2599
|
maximumBytesRead?: number;
|
|
938
2600
|
}) {
|
|
939
|
-
if (
|
|
940
|
-
|
|
2601
|
+
if (
|
|
2602
|
+
!Number.isSafeInteger(options.numItems) ||
|
|
2603
|
+
options.numItems < 1 ||
|
|
2604
|
+
options.numItems > 100
|
|
2605
|
+
) {
|
|
2606
|
+
fail(
|
|
2607
|
+
"INVALID_ARGUMENT",
|
|
2608
|
+
"pagination numItems must be an integer between 1 and 100",
|
|
2609
|
+
);
|
|
941
2610
|
}
|
|
942
2611
|
if (
|
|
943
2612
|
options.maximumRowsRead !== undefined &&
|
|
@@ -945,7 +2614,10 @@ function validatePagination(options: {
|
|
|
945
2614
|
options.maximumRowsRead < 1 ||
|
|
946
2615
|
options.maximumRowsRead > 1_000)
|
|
947
2616
|
) {
|
|
948
|
-
fail(
|
|
2617
|
+
fail(
|
|
2618
|
+
"INVALID_ARGUMENT",
|
|
2619
|
+
"maximumRowsRead must be an integer between 1 and 1000",
|
|
2620
|
+
);
|
|
949
2621
|
}
|
|
950
2622
|
if (
|
|
951
2623
|
options.maximumBytesRead !== undefined &&
|
|
@@ -953,6 +2625,9 @@ function validatePagination(options: {
|
|
|
953
2625
|
options.maximumBytesRead < 1 ||
|
|
954
2626
|
options.maximumBytesRead > 8_000_000)
|
|
955
2627
|
) {
|
|
956
|
-
fail(
|
|
2628
|
+
fail(
|
|
2629
|
+
"INVALID_ARGUMENT",
|
|
2630
|
+
"maximumBytesRead must be an integer between 1 and 8000000",
|
|
2631
|
+
);
|
|
957
2632
|
}
|
|
958
2633
|
}
|