@iann29/rastro 0.1.0-alpha.9 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +246 -57
- package/agent/integration.md +223 -65
- package/agent/manifest.json +34 -13
- package/agent/manifest.schema.json +53 -11
- package/dist/client/federation.d.ts +196 -11
- package/dist/client/federation.d.ts.map +1 -1
- package/dist/client/federation.js +160 -6
- package/dist/client/federation.js.map +1 -1
- package/dist/client/heat.d.ts +37 -0
- package/dist/client/heat.d.ts.map +1 -0
- package/dist/client/heat.js +43 -0
- package/dist/client/heat.js.map +1 -0
- package/dist/client/index.d.ts +441 -98
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +207 -17
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +6 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +50 -13
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/constants.d.ts +3 -0
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +6 -0
- package/dist/component/constants.js.map +1 -1
- package/dist/component/coverage.d.ts +9 -0
- package/dist/component/coverage.d.ts.map +1 -1
- package/dist/component/coverage.js +36 -6
- package/dist/component/coverage.js.map +1 -1
- package/dist/component/geo.d.ts +2 -0
- package/dist/component/geo.d.ts.map +1 -1
- package/dist/component/geo.js +56 -19
- package/dist/component/geo.js.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +48 -27
- package/dist/component/http.js.map +1 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +223 -304
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/localTime.d.ts +25 -0
- package/dist/component/localTime.d.ts.map +1 -0
- package/dist/component/localTime.js +126 -0
- package/dist/component/localTime.js.map +1 -0
- package/dist/component/reports.d.ts +44 -13
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +193 -38
- package/dist/component/reports.js.map +1 -1
- package/dist/component/retention.js +26 -8
- package/dist/component/retention.js.map +1 -1
- package/dist/component/rollupStore.d.ts +320 -0
- package/dist/component/rollupStore.d.ts.map +1 -0
- package/dist/component/rollupStore.js +596 -0
- package/dist/component/rollupStore.js.map +1 -0
- package/dist/component/rollups.d.ts +20 -0
- package/dist/component/rollups.d.ts.map +1 -0
- package/dist/component/rollups.js +73 -0
- package/dist/component/rollups.js.map +1 -0
- package/dist/component/sanitize.d.ts +5 -0
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +15 -0
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +345 -11
- package/dist/component/schema.js +25 -0
- package/dist/component/schema.js.map +1 -1
- package/dist/component/sites.d.ts +12 -0
- package/dist/component/sites.d.ts.map +1 -1
- package/dist/component/sites.js +30 -0
- package/dist/component/sites.js.map +1 -1
- package/dist/component/validators.d.ts +34 -13
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +23 -2
- package/dist/component/validators.js.map +1 -1
- package/dist/tracker/generated.d.ts +3 -0
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +3 -0
- package/dist/tracker/generated.js.map +1 -1
- package/docs/federation-setup.md +104 -27
- package/docs/federation.md +168 -31
- package/docs/upgrading.md +158 -21
- package/llms.txt +8 -6
- package/package.json +2 -1
- package/src/component/_generated/api.ts +6 -0
- package/src/component/_generated/component.ts +47 -12
- package/src/component/constants.ts +6 -0
- package/src/component/coverage.ts +46 -6
- package/src/component/geo.ts +82 -28
- package/src/component/http.ts +61 -26
- package/src/component/ingest.ts +327 -439
- package/src/component/localTime.ts +167 -0
- package/src/component/reports.ts +261 -39
- package/src/component/retention.ts +25 -7
- package/src/component/rollupStore.ts +799 -0
- package/src/component/rollups.ts +82 -0
- package/src/component/sanitize.ts +14 -0
- package/src/component/schema.ts +35 -0
- package/src/component/sites.ts +32 -0
- package/src/component/validators.ts +27 -1
- package/src/tracker/generated.ts +12 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { DAY_MS, HOUR_MS } from "./constants.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Local calendar days for a site's timezone. Aggregates are bucketed by UTC
|
|
5
|
+
* hour and UTC day; a site with a timezone also gets a `localDay` bucket per
|
|
6
|
+
* calendar day in that zone (see foldAggregates), so "yesterday" in São Paulo
|
|
7
|
+
* ends at midnight in São Paulo, not at 21:00. Every function here uses the
|
|
8
|
+
* runtime's Intl data — the same data `validateTimezone` relies on.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
const formatters = new Map<string, Intl.DateTimeFormat>();
|
|
12
|
+
|
|
13
|
+
function formatter(timeZone: string): Intl.DateTimeFormat {
|
|
14
|
+
let cached = formatters.get(timeZone);
|
|
15
|
+
if (!cached) {
|
|
16
|
+
cached = new Intl.DateTimeFormat("en-US", {
|
|
17
|
+
timeZone,
|
|
18
|
+
hourCycle: "h23",
|
|
19
|
+
year: "numeric",
|
|
20
|
+
month: "2-digit",
|
|
21
|
+
day: "2-digit",
|
|
22
|
+
hour: "2-digit",
|
|
23
|
+
minute: "2-digit",
|
|
24
|
+
second: "2-digit",
|
|
25
|
+
});
|
|
26
|
+
formatters.set(timeZone, cached);
|
|
27
|
+
}
|
|
28
|
+
return cached;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
type LocalParts = {
|
|
32
|
+
year: number;
|
|
33
|
+
month: number;
|
|
34
|
+
day: number;
|
|
35
|
+
hour: number;
|
|
36
|
+
minute: number;
|
|
37
|
+
second: number;
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
function localParts(timestamp: number, timeZone: string): LocalParts {
|
|
41
|
+
const parts: Partial<LocalParts> = {};
|
|
42
|
+
for (const part of formatter(timeZone).formatToParts(timestamp)) {
|
|
43
|
+
if (part.type === "literal") continue;
|
|
44
|
+
parts[part.type as keyof LocalParts] = Number(part.value);
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
year: parts.year ?? 1970,
|
|
48
|
+
month: parts.month ?? 1,
|
|
49
|
+
day: parts.day ?? 1,
|
|
50
|
+
// Some engines print 24 for midnight even with hourCycle h23.
|
|
51
|
+
hour: (parts.hour ?? 0) % 24,
|
|
52
|
+
minute: parts.minute ?? 0,
|
|
53
|
+
second: parts.second ?? 0,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** The zone's offset from UTC at `timestamp`, in milliseconds (east is positive). */
|
|
58
|
+
export function localOffsetMs(timestamp: number, timeZone: string): number {
|
|
59
|
+
const parts = localParts(timestamp, timeZone);
|
|
60
|
+
const asUtc = Date.UTC(
|
|
61
|
+
parts.year,
|
|
62
|
+
parts.month - 1,
|
|
63
|
+
parts.day,
|
|
64
|
+
parts.hour,
|
|
65
|
+
parts.minute,
|
|
66
|
+
parts.second,
|
|
67
|
+
);
|
|
68
|
+
return asUtc - (timestamp - (((timestamp % 1000) + 1000) % 1000));
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function localDateKey(timestamp: number, timeZone: string): number {
|
|
72
|
+
const parts = localParts(timestamp, timeZone);
|
|
73
|
+
return parts.year * 10_000 + parts.month * 100 + parts.day;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The first instant of the local calendar day that contains `timestamp`, as
|
|
78
|
+
* a UTC millisecond timestamp. Where a transition removes midnight (Chile
|
|
79
|
+
* springs forward at 00:00), the day starts at the first instant that exists.
|
|
80
|
+
*/
|
|
81
|
+
export function localDayStart(timestamp: number, timeZone: string): number {
|
|
82
|
+
const parts = localParts(timestamp, timeZone);
|
|
83
|
+
const target = parts.year * 10_000 + parts.month * 100 + parts.day;
|
|
84
|
+
const guess = Date.UTC(parts.year, parts.month - 1, parts.day);
|
|
85
|
+
// The offset in force at UTC midnight is usually the one at local midnight
|
|
86
|
+
// too; around a transition it is not, so a second and third candidate use
|
|
87
|
+
// the offset observed at the previous guess, and the earliest instant that
|
|
88
|
+
// still falls on the target date wins.
|
|
89
|
+
const first = guess - localOffsetMs(guess, timeZone);
|
|
90
|
+
const second = guess - localOffsetMs(first, timeZone);
|
|
91
|
+
const third = guess - localOffsetMs(second, timeZone);
|
|
92
|
+
const candidates = [first, second, third]
|
|
93
|
+
.filter((candidate) => localDateKey(candidate, timeZone) === target)
|
|
94
|
+
.sort((left, right) => left - right);
|
|
95
|
+
return candidates[0] ?? first;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** The start of the local day after the one starting at `dayStart`. */
|
|
99
|
+
export function nextLocalDayStart(dayStart: number, timeZone: string): number {
|
|
100
|
+
return localDayStart(dayStart + DAY_MS + 12 * HOUR_MS, timeZone);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** The start of the local day before the one starting at `dayStart`. */
|
|
104
|
+
export function previousLocalDayStart(
|
|
105
|
+
dayStart: number,
|
|
106
|
+
timeZone: string,
|
|
107
|
+
): number {
|
|
108
|
+
return localDayStart(dayStart - 12 * HOUR_MS, timeZone);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Every local day start from `from` through `to`, when `from` is one. */
|
|
112
|
+
export function localDayStarts(
|
|
113
|
+
from: number,
|
|
114
|
+
to: number,
|
|
115
|
+
timeZone: string,
|
|
116
|
+
): number[] {
|
|
117
|
+
const starts: number[] = [];
|
|
118
|
+
let cursor = from;
|
|
119
|
+
while (cursor <= to) {
|
|
120
|
+
starts.push(cursor);
|
|
121
|
+
cursor = nextLocalDayStart(cursor, timeZone);
|
|
122
|
+
}
|
|
123
|
+
return starts;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Whether `[from, to]` covers whole local days in the zone, inclusively. */
|
|
127
|
+
export function isLocalDayAligned(
|
|
128
|
+
from: number,
|
|
129
|
+
to: number,
|
|
130
|
+
timeZone: string,
|
|
131
|
+
): boolean {
|
|
132
|
+
return (
|
|
133
|
+
localDayStart(from, timeZone) === from &&
|
|
134
|
+
localDayStart(to + 1, timeZone) === to + 1
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A site's timezone when it needs its own day buckets: undefined for no
|
|
140
|
+
* timezone and for zones that never leave UTC, whose local days are the UTC
|
|
141
|
+
* days already kept.
|
|
142
|
+
*/
|
|
143
|
+
export function localDayTimezone(
|
|
144
|
+
timezone: string | undefined | null,
|
|
145
|
+
): string | undefined {
|
|
146
|
+
if (!timezone) return undefined;
|
|
147
|
+
try {
|
|
148
|
+
const year = new Date().getUTCFullYear();
|
|
149
|
+
const january = Date.UTC(year, 0, 1);
|
|
150
|
+
const july = Date.UTC(year, 6, 1);
|
|
151
|
+
return localOffsetMs(january, timezone) === 0 &&
|
|
152
|
+
localOffsetMs(july, timezone) === 0
|
|
153
|
+
? undefined
|
|
154
|
+
: timezone;
|
|
155
|
+
} catch {
|
|
156
|
+
return undefined;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** One timezone shared by every site, or undefined when they differ. */
|
|
161
|
+
export function sharedLocalTimezone(
|
|
162
|
+
timezones: Array<string | undefined>,
|
|
163
|
+
): string | undefined {
|
|
164
|
+
const distinct = new Set(timezones);
|
|
165
|
+
if (distinct.size !== 1) return undefined;
|
|
166
|
+
return timezones[0];
|
|
167
|
+
}
|
package/src/component/reports.ts
CHANGED
|
@@ -39,6 +39,13 @@ import {
|
|
|
39
39
|
type VisitorSketch,
|
|
40
40
|
} from "./cardinality.js";
|
|
41
41
|
import { fail } from "./errors.js";
|
|
42
|
+
import {
|
|
43
|
+
isLocalDayAligned,
|
|
44
|
+
localDayStart,
|
|
45
|
+
localDayStarts,
|
|
46
|
+
localDayTimezone,
|
|
47
|
+
sharedLocalTimezone,
|
|
48
|
+
} from "./localTime.js";
|
|
42
49
|
import {
|
|
43
50
|
compareJourneyEvents,
|
|
44
51
|
hydrateEventContexts,
|
|
@@ -96,6 +103,18 @@ const liveSessionDocumentValidator = v.object({
|
|
|
96
103
|
eventCount: v.number(),
|
|
97
104
|
previousPath: v.optional(v.string()),
|
|
98
105
|
movedAt: v.optional(v.number()),
|
|
106
|
+
entryPath: v.optional(v.string()),
|
|
107
|
+
returning: v.optional(v.boolean()),
|
|
108
|
+
utmCampaign: v.optional(v.string()),
|
|
109
|
+
affiliateSlug: v.optional(v.string()),
|
|
110
|
+
conversionCount: v.optional(v.number()),
|
|
111
|
+
funnel: v.optional(
|
|
112
|
+
v.object({
|
|
113
|
+
funnelId: v.id("funnels"),
|
|
114
|
+
step: v.number(),
|
|
115
|
+
steps: v.number(),
|
|
116
|
+
}),
|
|
117
|
+
),
|
|
99
118
|
});
|
|
100
119
|
const conversionDocumentValidator = v.object({
|
|
101
120
|
_id: v.id("conversions"),
|
|
@@ -126,6 +145,12 @@ const metricTotalsValidator = v.object({
|
|
|
126
145
|
visitors: v.number(),
|
|
127
146
|
conversions: v.number(),
|
|
128
147
|
revenueCents: v.number(),
|
|
148
|
+
// Engagement: summed session duration, sessions that reached a second
|
|
149
|
+
// pageview, and first-ever sessions of a visitor. Zero for buckets older
|
|
150
|
+
// than the counters; metadata.engagement says whether the range is covered.
|
|
151
|
+
durationMs: v.number(),
|
|
152
|
+
continuedSessions: v.number(),
|
|
153
|
+
newVisitors: v.number(),
|
|
129
154
|
});
|
|
130
155
|
const timePointValidator = metricTotalsValidator.extend({
|
|
131
156
|
timestamp: v.number(),
|
|
@@ -135,6 +160,14 @@ const topItemValidator = v.object({
|
|
|
135
160
|
count: v.number(),
|
|
136
161
|
revenueCents: v.number(),
|
|
137
162
|
});
|
|
163
|
+
// What a breakdown's `count` is: pageviews for pages and sessions for the
|
|
164
|
+
// other dimensions once every bucket in the range carries those counters,
|
|
165
|
+
// raw event volume for ranges that reach back before they existed.
|
|
166
|
+
const breakdownBasisValidator = v.union(
|
|
167
|
+
v.literal("pageviews"),
|
|
168
|
+
v.literal("sessions"),
|
|
169
|
+
v.literal("eventVolume"),
|
|
170
|
+
);
|
|
138
171
|
const coverageStateValidator = v.union(
|
|
139
172
|
v.literal("complete"),
|
|
140
173
|
v.literal("partial"),
|
|
@@ -152,7 +185,8 @@ const rangeSemanticsValidator = v.object({
|
|
|
152
185
|
from: v.number(),
|
|
153
186
|
to: v.number(),
|
|
154
187
|
boundary: v.literal("inclusive"),
|
|
155
|
-
|
|
188
|
+
// "UTC", or the site timezone whose calendar days a daily overview used.
|
|
189
|
+
timezone: v.string(),
|
|
156
190
|
});
|
|
157
191
|
const reportMetadataValidator = v.object({
|
|
158
192
|
basis: v.union(v.literal("dailyRollup"), v.literal("boundedLegacy")),
|
|
@@ -182,6 +216,10 @@ export const overview = query({
|
|
|
182
216
|
topBrowsers: v.array(topItemValidator),
|
|
183
217
|
topEvents: v.array(topItemValidator),
|
|
184
218
|
topGoals: v.array(topItemValidator),
|
|
219
|
+
// The session's lowercase utm_medium, ranked like campaigns.
|
|
220
|
+
topMediums: v.array(topItemValidator),
|
|
221
|
+
// Destination hosts of outbound clicks, ranked by clicks.
|
|
222
|
+
topOutbound: v.array(topItemValidator),
|
|
185
223
|
metadata: v.object({
|
|
186
224
|
visitors: v.object({
|
|
187
225
|
basis: v.union(
|
|
@@ -194,15 +232,25 @@ export const overview = query({
|
|
|
194
232
|
precision: v.union(v.number(), v.null()),
|
|
195
233
|
}),
|
|
196
234
|
breakdowns: v.object({
|
|
197
|
-
topSources:
|
|
198
|
-
topCampaigns:
|
|
199
|
-
topPages:
|
|
200
|
-
topCountries:
|
|
201
|
-
topDevices:
|
|
202
|
-
topBrowsers:
|
|
235
|
+
topSources: breakdownBasisValidator,
|
|
236
|
+
topCampaigns: breakdownBasisValidator,
|
|
237
|
+
topPages: breakdownBasisValidator,
|
|
238
|
+
topCountries: breakdownBasisValidator,
|
|
239
|
+
topDevices: breakdownBasisValidator,
|
|
240
|
+
topBrowsers: breakdownBasisValidator,
|
|
203
241
|
topEvents: v.literal("eventVolume"),
|
|
204
242
|
topGoals: v.literal("goalCompletionVolume"),
|
|
243
|
+
topMediums: breakdownBasisValidator,
|
|
244
|
+
topOutbound: v.literal("eventVolume"),
|
|
205
245
|
}),
|
|
246
|
+
// Since when every site in the request records the medium and outbound
|
|
247
|
+
// slots; null while one of them has not stamped it. A range that starts
|
|
248
|
+
// earlier ranks those two breakdowns from partial data.
|
|
249
|
+
dimensionsSince: v.union(v.number(), v.null()),
|
|
250
|
+
// "counted" when every bucket in the range carries the engagement
|
|
251
|
+
// counters, so bounce rate, duration, pages per session and new
|
|
252
|
+
// visitors can be derived from the totals; "unavailable" otherwise.
|
|
253
|
+
engagement: v.union(v.literal("counted"), v.literal("unavailable")),
|
|
206
254
|
coverage: resultCoverageValidator,
|
|
207
255
|
range: rangeSemanticsValidator,
|
|
208
256
|
features: v.object({
|
|
@@ -216,11 +264,13 @@ export const overview = query({
|
|
|
216
264
|
handler: async (ctx, args) => {
|
|
217
265
|
const siteIds = validateSiteIds(args.siteIds);
|
|
218
266
|
const currencies = new Set<string>();
|
|
267
|
+
const siteTimezones: Array<string | undefined> = [];
|
|
219
268
|
for (const siteId of siteIds) {
|
|
220
269
|
const site = await ctx.db.get("sites", siteId);
|
|
221
270
|
if (!site)
|
|
222
271
|
fail("NOT_FOUND", "site not found", { siteId: String(siteId) });
|
|
223
272
|
currencies.add(site.currency ?? "USD");
|
|
273
|
+
siteTimezones.push(localDayTimezone(site.timezone));
|
|
224
274
|
}
|
|
225
275
|
if (currencies.size !== 1) {
|
|
226
276
|
fail("CURRENCY_MISMATCH", "overview sites must use the same currency", {
|
|
@@ -238,10 +288,64 @@ export const overview = query({
|
|
|
238
288
|
fail("INVALID_ARGUMENT", "hourly reports are limited to 24 hours");
|
|
239
289
|
}
|
|
240
290
|
const intervalMs = interval === "hour" ? HOUR_MS : DAY_MS;
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const
|
|
291
|
+
// Sites that share a timezone answer a daily range that covers whole
|
|
292
|
+
// calendar days in that zone from their local-day buckets; any other
|
|
293
|
+
// daily range must cover whole UTC days, as before.
|
|
294
|
+
const siteTimezone =
|
|
295
|
+
interval === "day" ? sharedLocalTimezone(siteTimezones) : undefined;
|
|
296
|
+
const localDays =
|
|
297
|
+
siteTimezone !== undefined && isLocalDayAligned(from, to, siteTimezone);
|
|
298
|
+
if (!localDays) {
|
|
299
|
+
requireAlignedRange(
|
|
300
|
+
from,
|
|
301
|
+
to,
|
|
302
|
+
intervalMs,
|
|
303
|
+
`${interval} overview`,
|
|
304
|
+
siteTimezone,
|
|
305
|
+
);
|
|
306
|
+
}
|
|
307
|
+
const granularity: "hour" | "day" | "localDay" = localDays
|
|
308
|
+
? "localDay"
|
|
309
|
+
: interval;
|
|
310
|
+
const controls = await Promise.all(
|
|
311
|
+
siteIds.map((siteId) =>
|
|
312
|
+
ctx.db
|
|
313
|
+
.query("analyticsCoverage")
|
|
314
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
315
|
+
.unique(),
|
|
316
|
+
),
|
|
317
|
+
);
|
|
318
|
+
// Local-day buckets are complete from each site's stamp (its first ingest
|
|
319
|
+
// after the upgrade or after a timezone change, plus the safety delay).
|
|
320
|
+
// Buckets before it belong to another zone's days or to no complete day
|
|
321
|
+
// and are left out; the coverage says since when the days are whole.
|
|
322
|
+
let localDaysFrom = Number.POSITIVE_INFINITY;
|
|
323
|
+
if (localDays) {
|
|
324
|
+
localDaysFrom = controls.reduce(
|
|
325
|
+
(latest, control) =>
|
|
326
|
+
control?.localDaySince === undefined
|
|
327
|
+
? Number.POSITIVE_INFINITY
|
|
328
|
+
: Math.max(latest, control.localDaySince),
|
|
329
|
+
0,
|
|
330
|
+
);
|
|
331
|
+
}
|
|
332
|
+
const bucketStarts = localDays
|
|
333
|
+
? localDayStarts(from, to, siteTimezone!)
|
|
334
|
+
: [];
|
|
335
|
+
const bucketStartSet = new Set(bucketStarts);
|
|
336
|
+
const localDaysFirstBucket =
|
|
337
|
+
localDays && Number.isFinite(localDaysFrom)
|
|
338
|
+
? localDayStart(localDaysFrom, siteTimezone!)
|
|
339
|
+
: Number.POSITIVE_INFINITY;
|
|
340
|
+
const firstBucket = localDays
|
|
341
|
+
? bucketStarts[0]!
|
|
342
|
+
: Math.floor(from / intervalMs) * intervalMs;
|
|
343
|
+
const lastBucket = localDays
|
|
344
|
+
? bucketStarts[bucketStarts.length - 1]!
|
|
345
|
+
: Math.floor(to / intervalMs) * intervalMs;
|
|
346
|
+
const bucketCount = localDays
|
|
347
|
+
? bucketStarts.length
|
|
348
|
+
: Math.floor((lastBucket - firstBucket) / intervalMs) + 1;
|
|
245
349
|
const perSiteLimit =
|
|
246
350
|
bucketCount *
|
|
247
351
|
(interval === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
|
|
@@ -260,12 +364,20 @@ export const overview = query({
|
|
|
260
364
|
let visitorsComplete = true;
|
|
261
365
|
const dimensions = new Map<
|
|
262
366
|
string,
|
|
263
|
-
{
|
|
367
|
+
{
|
|
368
|
+
value: string;
|
|
369
|
+
count: number;
|
|
370
|
+
revenueCents: number;
|
|
371
|
+
pageviews: number;
|
|
372
|
+
sessions: number;
|
|
373
|
+
}
|
|
264
374
|
>();
|
|
265
375
|
const overviewDataset =
|
|
266
|
-
|
|
376
|
+
granularity === "hour"
|
|
267
377
|
? ("overviewHour" as const)
|
|
268
|
-
:
|
|
378
|
+
: granularity === "day"
|
|
379
|
+
? ("overviewDay" as const)
|
|
380
|
+
: ("overviewLocalDay" as const);
|
|
269
381
|
|
|
270
382
|
await assertSourceAvailable(ctx, siteIds, [overviewDataset], from);
|
|
271
383
|
|
|
@@ -275,13 +387,20 @@ export const overview = query({
|
|
|
275
387
|
.withIndex("by_siteId_and_granularity_and_bucketStart", (range) =>
|
|
276
388
|
range
|
|
277
389
|
.eq("siteId", siteId)
|
|
278
|
-
.eq("granularity",
|
|
390
|
+
.eq("granularity", granularity)
|
|
279
391
|
.gte("bucketStart", firstBucket)
|
|
280
392
|
.lte("bucketStart", lastBucket),
|
|
281
393
|
)
|
|
282
394
|
.take(perSiteLimit + 1);
|
|
283
395
|
failIfIncomplete(buckets, perSiteLimit, "overview aggregate buckets");
|
|
284
396
|
for (const bucket of buckets) {
|
|
397
|
+
if (
|
|
398
|
+
localDays &&
|
|
399
|
+
(!bucketStartSet.has(bucket.bucketStart) ||
|
|
400
|
+
bucket.bucketStart < localDaysFirstBucket)
|
|
401
|
+
) {
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
285
404
|
addMetrics(totals, bucket);
|
|
286
405
|
const point = points.get(bucket.bucketStart) ?? emptyTotals();
|
|
287
406
|
addMetrics(point, bucket);
|
|
@@ -308,18 +427,48 @@ export const overview = query({
|
|
|
308
427
|
value: dimension.value,
|
|
309
428
|
count: 0,
|
|
310
429
|
revenueCents: 0,
|
|
430
|
+
pageviews: 0,
|
|
431
|
+
sessions: 0,
|
|
311
432
|
};
|
|
312
433
|
current.count += dimension.count;
|
|
313
434
|
current.revenueCents += dimension.revenueCents;
|
|
435
|
+
current.pageviews += dimension.pageviews ?? 0;
|
|
436
|
+
current.sessions += dimension.sessions ?? 0;
|
|
314
437
|
dimensions.set(key, current);
|
|
315
438
|
}
|
|
316
439
|
}
|
|
317
440
|
}
|
|
318
441
|
|
|
319
|
-
|
|
442
|
+
// Pages rank by pageviews and the other dimensions by sessions once every
|
|
443
|
+
// site's counters are proven for the whole range; before that stamp a
|
|
444
|
+
// bucket may hold slots without them, so the ranking stays event volume.
|
|
445
|
+
const countersProven = controls.every(
|
|
446
|
+
(control) =>
|
|
447
|
+
control?.countersSince !== undefined && from >= control.countersSince,
|
|
448
|
+
);
|
|
449
|
+
const pageBasis = countersProven
|
|
450
|
+
? ("pageviews" as const)
|
|
451
|
+
: ("eventVolume" as const);
|
|
452
|
+
const sessionBasis = countersProven
|
|
453
|
+
? ("sessions" as const)
|
|
454
|
+
: ("eventVolume" as const);
|
|
455
|
+
const top = (
|
|
456
|
+
type: string,
|
|
457
|
+
basis: "pageviews" | "sessions" | "eventVolume",
|
|
458
|
+
) =>
|
|
320
459
|
[...dimensions.entries()]
|
|
321
460
|
.filter(([key]) => key.startsWith(`${type}:`))
|
|
322
|
-
.map(([,
|
|
461
|
+
.map(([, item]) => ({
|
|
462
|
+
value: item.value,
|
|
463
|
+
count:
|
|
464
|
+
basis === "pageviews"
|
|
465
|
+
? item.pageviews
|
|
466
|
+
: basis === "sessions"
|
|
467
|
+
? item.sessions
|
|
468
|
+
: item.count,
|
|
469
|
+
revenueCents: item.revenueCents,
|
|
470
|
+
}))
|
|
471
|
+
.filter((item) => item.count > 0 || item.revenueCents > 0)
|
|
323
472
|
.sort(
|
|
324
473
|
(left, right) =>
|
|
325
474
|
right.count - left.count || left.value.localeCompare(right.value),
|
|
@@ -337,6 +486,19 @@ export const overview = query({
|
|
|
337
486
|
datasets: [overviewDataset],
|
|
338
487
|
complete: visitorsComplete,
|
|
339
488
|
});
|
|
489
|
+
if (localDays) {
|
|
490
|
+
// The stamp, not the oldest row, says where the site's days begin.
|
|
491
|
+
const availableFrom = Number.isFinite(localDaysFrom)
|
|
492
|
+
? Math.max(coverage.availableFrom ?? 0, localDaysFrom)
|
|
493
|
+
: null;
|
|
494
|
+
coverage.availableFrom = availableFrom;
|
|
495
|
+
if (
|
|
496
|
+
coverage.state === "complete" &&
|
|
497
|
+
(availableFrom === null || from < availableFrom)
|
|
498
|
+
) {
|
|
499
|
+
coverage.state = availableFrom === null ? "unavailable" : "partial";
|
|
500
|
+
}
|
|
501
|
+
}
|
|
340
502
|
const visitorBasis: "estimatedUnique" | "bucketOccurrences" =
|
|
341
503
|
visitorsComplete ? "estimatedUnique" : "bucketOccurrences";
|
|
342
504
|
return {
|
|
@@ -348,14 +510,16 @@ export const overview = query({
|
|
|
348
510
|
timeSeries: [...points.entries()]
|
|
349
511
|
.sort(([left], [right]) => left - right)
|
|
350
512
|
.map(([timestamp, metrics]) => ({ timestamp, ...metrics })),
|
|
351
|
-
topSources: top("source"),
|
|
352
|
-
topCampaigns: top("campaign"),
|
|
353
|
-
topPages: top("page"),
|
|
354
|
-
topCountries: top("country"),
|
|
355
|
-
topDevices: top("device"),
|
|
356
|
-
topBrowsers: top("browser"),
|
|
357
|
-
topEvents: top("event"),
|
|
358
|
-
topGoals: top("goal"),
|
|
513
|
+
topSources: top("source", sessionBasis),
|
|
514
|
+
topCampaigns: top("campaign", sessionBasis),
|
|
515
|
+
topPages: top("page", pageBasis),
|
|
516
|
+
topCountries: top("country", sessionBasis),
|
|
517
|
+
topDevices: top("device", sessionBasis),
|
|
518
|
+
topBrowsers: top("browser", sessionBasis),
|
|
519
|
+
topEvents: top("event", "eventVolume"),
|
|
520
|
+
topGoals: top("goal", "eventVolume"),
|
|
521
|
+
topMediums: top("medium", sessionBasis),
|
|
522
|
+
topOutbound: top("outbound", "eventVolume"),
|
|
359
523
|
metadata: {
|
|
360
524
|
visitors: {
|
|
361
525
|
basis: visitorBasis,
|
|
@@ -365,17 +529,27 @@ export const overview = query({
|
|
|
365
529
|
precision: visitorsComplete ? 6 : null,
|
|
366
530
|
},
|
|
367
531
|
breakdowns: {
|
|
368
|
-
topSources:
|
|
369
|
-
topCampaigns:
|
|
370
|
-
topPages:
|
|
371
|
-
topCountries:
|
|
372
|
-
topDevices:
|
|
373
|
-
topBrowsers:
|
|
532
|
+
topSources: sessionBasis,
|
|
533
|
+
topCampaigns: sessionBasis,
|
|
534
|
+
topPages: pageBasis,
|
|
535
|
+
topCountries: sessionBasis,
|
|
536
|
+
topDevices: sessionBasis,
|
|
537
|
+
topBrowsers: sessionBasis,
|
|
374
538
|
topEvents: "eventVolume" as const,
|
|
375
539
|
topGoals: "goalCompletionVolume" as const,
|
|
540
|
+
topMediums: sessionBasis,
|
|
541
|
+
topOutbound: "eventVolume" as const,
|
|
376
542
|
},
|
|
543
|
+
dimensionsSince: controls.every(
|
|
544
|
+
(control) => control?.dimensionsSince !== undefined,
|
|
545
|
+
)
|
|
546
|
+
? Math.max(...controls.map((control) => control!.dimensionsSince!))
|
|
547
|
+
: null,
|
|
548
|
+
engagement: countersProven
|
|
549
|
+
? ("counted" as const)
|
|
550
|
+
: ("unavailable" as const),
|
|
377
551
|
coverage,
|
|
378
|
-
range: rangeSemantics(from, to),
|
|
552
|
+
range: rangeSemantics(from, to, localDays ? siteTimezone : "UTC"),
|
|
379
553
|
features: {
|
|
380
554
|
sessionReplay: "unsupported" as const,
|
|
381
555
|
errorInsights: "unsupported" as const,
|
|
@@ -447,6 +621,14 @@ export const liveVisitors = query({
|
|
|
447
621
|
eventCount: row.eventCount,
|
|
448
622
|
...(row.previousPath ? { previousPath: row.previousPath } : {}),
|
|
449
623
|
...(row.movedAt !== undefined ? { movedAt: row.movedAt } : {}),
|
|
624
|
+
...(row.entryPath ? { entryPath: row.entryPath } : {}),
|
|
625
|
+
...(row.returning !== undefined ? { returning: row.returning } : {}),
|
|
626
|
+
...(row.utmCampaign ? { utmCampaign: row.utmCampaign } : {}),
|
|
627
|
+
...(row.affiliateSlug ? { affiliateSlug: row.affiliateSlug } : {}),
|
|
628
|
+
...(row.conversionCount !== undefined
|
|
629
|
+
? { conversionCount: row.conversionCount }
|
|
630
|
+
: {}),
|
|
631
|
+
...(row.funnel ? { funnel: row.funnel } : {}),
|
|
450
632
|
}));
|
|
451
633
|
},
|
|
452
634
|
});
|
|
@@ -1523,6 +1705,7 @@ export const vitalsReport = query({
|
|
|
1523
1705
|
const coverageDatasetValidator = v.union(
|
|
1524
1706
|
v.literal("overviewHour"),
|
|
1525
1707
|
v.literal("overviewDay"),
|
|
1708
|
+
v.literal("overviewLocalDay"),
|
|
1526
1709
|
v.literal("events"),
|
|
1527
1710
|
v.literal("sessions"),
|
|
1528
1711
|
v.literal("conversions"),
|
|
@@ -1555,6 +1738,7 @@ export const dataCoverage = query({
|
|
|
1555
1738
|
const datasets = [
|
|
1556
1739
|
"overviewHour",
|
|
1557
1740
|
"overviewDay",
|
|
1741
|
+
"overviewLocalDay",
|
|
1558
1742
|
"events",
|
|
1559
1743
|
"sessions",
|
|
1560
1744
|
"conversions",
|
|
@@ -1948,6 +2132,7 @@ async function siteMapDatasetCoverage(
|
|
|
1948
2132
|
type PublicCoverageDataset =
|
|
1949
2133
|
| "overviewHour"
|
|
1950
2134
|
| "overviewDay"
|
|
2135
|
+
| "overviewLocalDay"
|
|
1951
2136
|
| "events"
|
|
1952
2137
|
| "sessions"
|
|
1953
2138
|
| "conversions"
|
|
@@ -1960,6 +2145,7 @@ type PublicCoverageDataset =
|
|
|
1960
2145
|
type CoverageSource =
|
|
1961
2146
|
| "overviewHour"
|
|
1962
2147
|
| "overviewDay"
|
|
2148
|
+
| "overviewLocalDay"
|
|
1963
2149
|
| "events"
|
|
1964
2150
|
| "sessions"
|
|
1965
2151
|
| "conversions"
|
|
@@ -1993,9 +2179,17 @@ async function sourceRange(
|
|
|
1993
2179
|
siteId: Id<"sites">,
|
|
1994
2180
|
source: CoverageSource,
|
|
1995
2181
|
): Promise<SourceRange | null> {
|
|
1996
|
-
if (
|
|
2182
|
+
if (
|
|
2183
|
+
source === "overviewHour" ||
|
|
2184
|
+
source === "overviewDay" ||
|
|
2185
|
+
source === "overviewLocalDay"
|
|
2186
|
+
) {
|
|
1997
2187
|
const granularity =
|
|
1998
|
-
source === "overviewHour"
|
|
2188
|
+
source === "overviewHour"
|
|
2189
|
+
? ("hour" as const)
|
|
2190
|
+
: source === "overviewDay"
|
|
2191
|
+
? ("day" as const)
|
|
2192
|
+
: ("localDay" as const);
|
|
1999
2193
|
const [first, last] = await Promise.all([
|
|
2000
2194
|
ctx.db
|
|
2001
2195
|
.query("aggregateBuckets")
|
|
@@ -2144,6 +2338,9 @@ type MetricTotals = {
|
|
|
2144
2338
|
visitors: number;
|
|
2145
2339
|
conversions: number;
|
|
2146
2340
|
revenueCents: number;
|
|
2341
|
+
durationMs: number;
|
|
2342
|
+
continuedSessions: number;
|
|
2343
|
+
newVisitors: number;
|
|
2147
2344
|
};
|
|
2148
2345
|
|
|
2149
2346
|
function emptyTotals(): MetricTotals {
|
|
@@ -2157,10 +2354,23 @@ function emptyTotals(): MetricTotals {
|
|
|
2157
2354
|
visitors: 0,
|
|
2158
2355
|
conversions: 0,
|
|
2159
2356
|
revenueCents: 0,
|
|
2357
|
+
durationMs: 0,
|
|
2358
|
+
continuedSessions: 0,
|
|
2359
|
+
newVisitors: 0,
|
|
2160
2360
|
};
|
|
2161
2361
|
}
|
|
2162
2362
|
|
|
2163
|
-
function addMetrics(
|
|
2363
|
+
function addMetrics(
|
|
2364
|
+
target: MetricTotals,
|
|
2365
|
+
source: Omit<
|
|
2366
|
+
MetricTotals,
|
|
2367
|
+
"durationMs" | "continuedSessions" | "newVisitors"
|
|
2368
|
+
> & {
|
|
2369
|
+
durationMs?: number;
|
|
2370
|
+
continuedSessions?: number;
|
|
2371
|
+
newVisitors?: number;
|
|
2372
|
+
},
|
|
2373
|
+
) {
|
|
2164
2374
|
target.events += source.events;
|
|
2165
2375
|
target.pageviews += source.pageviews;
|
|
2166
2376
|
target.clicks += source.clicks;
|
|
@@ -2170,6 +2380,9 @@ function addMetrics(target: MetricTotals, source: MetricTotals) {
|
|
|
2170
2380
|
target.visitors += source.visitors;
|
|
2171
2381
|
target.conversions += source.conversions;
|
|
2172
2382
|
target.revenueCents += source.revenueCents;
|
|
2383
|
+
target.durationMs += source.durationMs ?? 0;
|
|
2384
|
+
target.continuedSessions += source.continuedSessions ?? 0;
|
|
2385
|
+
target.newVisitors += source.newVisitors ?? 0;
|
|
2173
2386
|
}
|
|
2174
2387
|
|
|
2175
2388
|
function validateSiteIds<T>(siteIds: T[]): T[] {
|
|
@@ -2212,22 +2425,31 @@ function requireAlignedRange(
|
|
|
2212
2425
|
to: number,
|
|
2213
2426
|
intervalMs: number,
|
|
2214
2427
|
report: string,
|
|
2428
|
+
siteTimezone?: string,
|
|
2215
2429
|
) {
|
|
2216
2430
|
if (!isAlignedRange(from, to, intervalMs)) {
|
|
2217
2431
|
fail(
|
|
2218
2432
|
"REPORT_INCOMPLETE",
|
|
2219
|
-
|
|
2220
|
-
|
|
2433
|
+
siteTimezone
|
|
2434
|
+
? `${report} requires complete inclusive UTC bucket boundaries, or whole calendar days in ${siteTimezone}`
|
|
2435
|
+
: `${report} requires complete inclusive UTC bucket boundaries`,
|
|
2436
|
+
{
|
|
2437
|
+
from,
|
|
2438
|
+
to,
|
|
2439
|
+
intervalMs,
|
|
2440
|
+
timezone: "UTC",
|
|
2441
|
+
siteTimezone: siteTimezone ?? null,
|
|
2442
|
+
},
|
|
2221
2443
|
);
|
|
2222
2444
|
}
|
|
2223
2445
|
}
|
|
2224
2446
|
|
|
2225
|
-
function rangeSemantics(from: number, to: number) {
|
|
2447
|
+
function rangeSemantics(from: number, to: number, timezone = "UTC") {
|
|
2226
2448
|
return {
|
|
2227
2449
|
from,
|
|
2228
2450
|
to,
|
|
2229
2451
|
boundary: "inclusive" as const,
|
|
2230
|
-
timezone
|
|
2452
|
+
timezone,
|
|
2231
2453
|
};
|
|
2232
2454
|
}
|
|
2233
2455
|
|