@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +521 -66
- package/agent/integration.md +661 -0
- package/agent/manifest.json +187 -0
- package/agent/manifest.schema.json +411 -0
- package/dist/client/federation.d.ts +236 -0
- package/dist/client/federation.d.ts.map +1 -0
- package/dist/client/federation.js +196 -0
- package/dist/client/federation.js.map +1 -0
- package/dist/client/index.d.ts +2467 -18
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +461 -32
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +16 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +302 -4
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/server.d.ts +4 -0
- package/dist/component/_generated/server.d.ts.map +1 -1
- package/dist/component/_generated/server.js.map +1 -1
- package/dist/component/affiliates.d.ts.map +1 -1
- package/dist/component/affiliates.js +6 -2
- package/dist/component/affiliates.js.map +1 -1
- package/dist/component/cardinality.d.ts +12 -0
- package/dist/component/cardinality.d.ts.map +1 -0
- package/dist/component/cardinality.js +94 -0
- package/dist/component/cardinality.js.map +1 -0
- package/dist/component/constants.d.ts +25 -1
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +38 -1
- package/dist/component/constants.js.map +1 -1
- package/dist/component/convex.config.d.ts +6 -1
- package/dist/component/convex.config.js +9 -1
- package/dist/component/convex.config.js.map +1 -1
- package/dist/component/coverage.d.ts +19 -0
- package/dist/component/coverage.d.ts.map +1 -0
- package/dist/component/coverage.js +30 -0
- package/dist/component/coverage.js.map +1 -0
- package/dist/component/diagnostics.d.ts +9 -0
- package/dist/component/diagnostics.d.ts.map +1 -0
- package/dist/component/diagnostics.js +47 -0
- package/dist/component/diagnostics.js.map +1 -0
- package/dist/component/errors.d.ts +1 -1
- package/dist/component/errors.d.ts.map +1 -1
- package/dist/component/errors.js.map +1 -1
- package/dist/component/eventStore.d.ts +21 -9
- package/dist/component/eventStore.d.ts.map +1 -1
- package/dist/component/eventStore.js +142 -152
- package/dist/component/eventStore.js.map +1 -1
- package/dist/component/funnels.d.ts.map +1 -1
- package/dist/component/funnels.js +5 -3
- package/dist/component/funnels.js.map +1 -1
- package/dist/component/geo.d.ts +71 -0
- package/dist/component/geo.d.ts.map +1 -0
- package/dist/component/geo.js +611 -0
- package/dist/component/geo.js.map +1 -0
- package/dist/component/goals.d.ts.map +1 -1
- package/dist/component/goals.js +8 -5
- package/dist/component/goals.js.map +1 -1
- package/dist/component/guards.d.ts.map +1 -1
- package/dist/component/guards.js.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +248 -49
- package/dist/component/http.js.map +1 -1
- package/dist/component/identity.d.ts +13 -0
- package/dist/component/identity.d.ts.map +1 -0
- package/dist/component/identity.js +58 -0
- package/dist/component/identity.js.map +1 -0
- package/dist/component/ingest.d.ts +3 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +604 -55
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/live.d.ts.map +1 -1
- package/dist/component/live.js +7 -6
- package/dist/component/live.js.map +1 -1
- package/dist/component/reports.d.ts +259 -10
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +1110 -124
- package/dist/component/reports.js.map +1 -1
- package/dist/component/retention.d.ts +75 -1
- package/dist/component/retention.d.ts.map +1 -1
- package/dist/component/retention.js +537 -32
- package/dist/component/retention.js.map +1 -1
- package/dist/component/sanitize.d.ts +24 -1
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +96 -16
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +378 -62
- package/dist/component/schema.js +178 -20
- package/dist/component/schema.js.map +1 -1
- package/dist/component/sites.d.ts.map +1 -1
- package/dist/component/sites.js +11 -7
- package/dist/component/sites.js.map +1 -1
- package/dist/component/useragent.d.ts +9 -0
- package/dist/component/useragent.d.ts.map +1 -0
- package/dist/component/useragent.js +152 -0
- package/dist/component/useragent.js.map +1 -0
- package/dist/component/validators.d.ts +101 -65
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +19 -13
- package/dist/component/validators.js.map +1 -1
- package/dist/component/visitors.d.ts +19 -0
- package/dist/component/visitors.d.ts.map +1 -0
- package/dist/component/visitors.js +86 -0
- package/dist/component/visitors.js.map +1 -0
- package/dist/component/vitals.d.ts +41 -0
- package/dist/component/vitals.d.ts.map +1 -0
- package/dist/component/vitals.js +115 -0
- package/dist/component/vitals.js.map +1 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js.map +1 -1
- package/dist/tracker/generated.d.ts +8 -4
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +8 -4
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.d.ts +1 -1
- package/dist/tracker/tracker.d.ts.map +1 -1
- package/dist/tracker/tracker.js +57 -20
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker/vitals.d.ts +10 -0
- package/dist/tracker/vitals.d.ts.map +1 -0
- package/dist/tracker/vitals.js +140 -0
- package/dist/tracker/vitals.js.map +1 -0
- package/dist/tracker.min.js +1 -1
- package/dist/vitals.min.js +1 -0
- package/docs/benchmarks/2026-08-20-realistic.md +76 -76
- package/docs/benchmarks/2026-08-21-formal-certification.md +353 -0
- package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
- package/docs/federation-setup.md +397 -0
- package/docs/federation.md +262 -0
- package/docs/upgrading.md +259 -0
- package/llms.txt +71 -0
- package/package.json +55 -11
- package/scripts/benchmark-ingest.mjs +175 -73
- package/scripts/generate-federation-keys.mjs +20 -0
- package/src/component/_generated/api.ts +16 -0
- package/src/component/_generated/component.ts +366 -4
- package/src/component/_generated/server.ts +4 -0
- package/src/component/affiliates.ts +20 -5
- package/src/component/cardinality.ts +117 -0
- package/src/component/constants.ts +38 -1
- package/src/component/convex.config.ts +11 -1
- package/src/component/coverage.ts +36 -0
- package/src/component/diagnostics.ts +65 -0
- package/src/component/errors.ts +2 -1
- package/src/component/eventStore.ts +217 -193
- package/src/component/funnels.ts +19 -16
- package/src/component/geo.ts +781 -0
- package/src/component/goals.ts +29 -21
- package/src/component/guards.ts +3 -1
- package/src/component/http.ts +357 -57
- package/src/component/identity.ts +74 -0
- package/src/component/ingest.ts +1040 -135
- package/src/component/live.ts +13 -7
- package/src/component/reports.ts +1675 -183
- package/src/component/retention.ts +770 -96
- package/src/component/sanitize.ts +144 -29
- package/src/component/schema.ts +200 -21
- package/src/component/sites.ts +27 -12
- package/src/component/useragent.ts +171 -0
- package/src/component/validators.ts +29 -13
- package/src/component/visitors.ts +116 -0
- package/src/component/vitals.ts +146 -0
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
CLOCK_SKEW_TOLERANCE_MS,
|
|
2
3
|
MAX_BATCH_BYTES,
|
|
3
4
|
MAX_PROPERTIES,
|
|
4
5
|
MAX_SITE_DOMAINS,
|
|
@@ -8,6 +9,7 @@ import type {
|
|
|
8
9
|
IngestContext,
|
|
9
10
|
TrackerEvent,
|
|
10
11
|
} from "./validators.js";
|
|
12
|
+
import { isValidVitalValue, isVitalMetric } from "./vitals.js";
|
|
11
13
|
|
|
12
14
|
const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
|
13
15
|
const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
|
|
@@ -62,9 +64,10 @@ export function sanitizePath(value: string): string {
|
|
|
62
64
|
const input = requireString(value, "path", 2048);
|
|
63
65
|
let path: string;
|
|
64
66
|
try {
|
|
65
|
-
path =
|
|
66
|
-
|
|
67
|
-
|
|
67
|
+
path =
|
|
68
|
+
input.startsWith("http://") || input.startsWith("https://")
|
|
69
|
+
? new URL(input).pathname
|
|
70
|
+
: input.split(/[?#]/, 1)[0];
|
|
68
71
|
} catch {
|
|
69
72
|
throw new Error("path is invalid");
|
|
70
73
|
}
|
|
@@ -73,6 +76,37 @@ export function sanitizePath(value: string): string {
|
|
|
73
76
|
return cleaned || "/";
|
|
74
77
|
}
|
|
75
78
|
|
|
79
|
+
const ROUTE_ID = ":id";
|
|
80
|
+
const NUMERIC_SEGMENT = /^\d+$/;
|
|
81
|
+
const UUID_SEGMENT =
|
|
82
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
83
|
+
const HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Collapses the identifiers that unmistakably vary per record — numbers,
|
|
87
|
+
* UUIDs, and long hexadecimal tokens — into `:id`, so `/blog/42` and
|
|
88
|
+
* `/blog/43` roll up as one route in the site map. Slugs are pages of their
|
|
89
|
+
* own and stay as they are. A trailing slash is dropped because both
|
|
90
|
+
* spellings serve the same page. Only the map uses this; every other report
|
|
91
|
+
* keeps the sanitized path exactly as the tracker sent it.
|
|
92
|
+
*/
|
|
93
|
+
export function normalizeRoute(path: string): string {
|
|
94
|
+
const trimmed =
|
|
95
|
+
path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
|
|
96
|
+
const segments = trimmed.split("/");
|
|
97
|
+
for (let index = 1; index < segments.length; index += 1) {
|
|
98
|
+
const segment = segments[index] ?? "";
|
|
99
|
+
if (
|
|
100
|
+
NUMERIC_SEGMENT.test(segment) ||
|
|
101
|
+
UUID_SEGMENT.test(segment) ||
|
|
102
|
+
HEX_SEGMENT.test(segment)
|
|
103
|
+
) {
|
|
104
|
+
segments[index] = ROUTE_ID;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return segments.join("/") || "/";
|
|
108
|
+
}
|
|
109
|
+
|
|
76
110
|
export function sanitizePublicUrl(value: string): string | undefined {
|
|
77
111
|
const input = cleanString(value, 2048);
|
|
78
112
|
if (!input) return undefined;
|
|
@@ -111,15 +145,42 @@ export function sanitizeProperties(
|
|
|
111
145
|
return Object.keys(result).length > 0 ? result : undefined;
|
|
112
146
|
}
|
|
113
147
|
|
|
148
|
+
/**
|
|
149
|
+
* Re-anchors a batch whose sender's clock disagrees with ours. `sentAt` is the
|
|
150
|
+
* sender's clock at flush time, so `now - sentAt` is the clock offset alone: an
|
|
151
|
+
* event queued long before the flush keeps its age, and nothing is inferred
|
|
152
|
+
* from the events themselves. Offsets inside the tolerance are latency or a
|
|
153
|
+
* slightly drifted clock and leave the batch untouched, so a well-set client's
|
|
154
|
+
* timestamps stay exactly as sent.
|
|
155
|
+
*/
|
|
156
|
+
export function alignClock<T extends { timestamp: number }>(
|
|
157
|
+
events: T[],
|
|
158
|
+
sentAt: number | undefined,
|
|
159
|
+
now: number,
|
|
160
|
+
): T[] {
|
|
161
|
+
if (sentAt === undefined || !Number.isFinite(sentAt)) return events;
|
|
162
|
+
const offset = now - sentAt;
|
|
163
|
+
if (Math.abs(offset) <= CLOCK_SKEW_TOLERANCE_MS) return events;
|
|
164
|
+
return events.map((event) => ({
|
|
165
|
+
...event,
|
|
166
|
+
timestamp: event.timestamp + offset,
|
|
167
|
+
}));
|
|
168
|
+
}
|
|
169
|
+
|
|
114
170
|
export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
|
|
115
|
-
if (!Number.isFinite(event.timestamp))
|
|
171
|
+
if (!Number.isFinite(event.timestamp))
|
|
172
|
+
throw new Error("timestamp is invalid");
|
|
116
173
|
if (
|
|
117
174
|
event.timestamp < now - MAX_EVENT_AGE_MS ||
|
|
118
175
|
event.timestamp > now + MAX_FUTURE_SKEW_MS
|
|
119
176
|
) {
|
|
120
177
|
throw new Error("timestamp is outside the accepted window");
|
|
121
178
|
}
|
|
122
|
-
if (
|
|
179
|
+
if (
|
|
180
|
+
!Number.isSafeInteger(event.sequence) ||
|
|
181
|
+
event.sequence < 0 ||
|
|
182
|
+
event.sequence > 1_000_000_000
|
|
183
|
+
) {
|
|
123
184
|
throw new Error("sequence is out of range");
|
|
124
185
|
}
|
|
125
186
|
if (event.revenueCents !== undefined && !isValidMoney(event.revenueCents)) {
|
|
@@ -130,9 +191,20 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
|
|
|
130
191
|
throw new Error("name is required for custom events");
|
|
131
192
|
}
|
|
132
193
|
if (event.type === "conversion" && !name) name = "conversion";
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
194
|
+
if (event.type === "vital") {
|
|
195
|
+
if (!name || !isVitalMetric(name)) {
|
|
196
|
+
throw new Error("vital name must be a Web Vitals metric");
|
|
197
|
+
}
|
|
198
|
+
if (event.value === undefined || !isValidVitalValue(event.value)) {
|
|
199
|
+
throw new Error("vital value is out of range");
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
const currency =
|
|
203
|
+
event.currency === undefined
|
|
204
|
+
? event.type === "conversion"
|
|
205
|
+
? "USD"
|
|
206
|
+
: undefined
|
|
207
|
+
: sanitizeCurrency(event.currency);
|
|
136
208
|
return {
|
|
137
209
|
eventId: sanitizeOpaqueId(event.eventId, "eventId"),
|
|
138
210
|
sessionId: sanitizeOpaqueId(event.sessionId, "sessionId"),
|
|
@@ -140,19 +212,26 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
|
|
|
140
212
|
type: event.type,
|
|
141
213
|
name,
|
|
142
214
|
path: sanitizePath(event.path),
|
|
143
|
-
referrer:
|
|
144
|
-
|
|
145
|
-
|
|
215
|
+
referrer:
|
|
216
|
+
event.referrer === undefined
|
|
217
|
+
? undefined
|
|
218
|
+
: sanitizePublicUrl(event.referrer),
|
|
146
219
|
timestamp: Math.trunc(event.timestamp),
|
|
147
220
|
sequence: event.sequence,
|
|
148
|
-
target:
|
|
149
|
-
|
|
150
|
-
|
|
221
|
+
target:
|
|
222
|
+
event.target === undefined
|
|
223
|
+
? undefined
|
|
224
|
+
: cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
|
|
151
225
|
href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
|
|
152
226
|
properties: sanitizeProperties(event.properties),
|
|
153
227
|
revenueCents: event.revenueCents,
|
|
154
228
|
currency,
|
|
155
229
|
affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
|
|
230
|
+
// Only measurements keep a value; a stray number on any other type is noise.
|
|
231
|
+
value:
|
|
232
|
+
event.type === "vital" && event.value !== undefined
|
|
233
|
+
? Math.round(event.value)
|
|
234
|
+
: undefined,
|
|
156
235
|
};
|
|
157
236
|
}
|
|
158
237
|
|
|
@@ -167,7 +246,9 @@ function optionalAffiliateSlug(value: string | undefined): string | undefined {
|
|
|
167
246
|
|
|
168
247
|
export function normalizeDomains(domains: string[]): string[] {
|
|
169
248
|
if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
|
|
170
|
-
throw new Error(
|
|
249
|
+
throw new Error(
|
|
250
|
+
`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`,
|
|
251
|
+
);
|
|
171
252
|
}
|
|
172
253
|
const normalized = domains.map(normalizeDomainPattern);
|
|
173
254
|
return [...new Set(normalized)].sort();
|
|
@@ -185,7 +266,11 @@ export function normalizeDomainPattern(value: string): string {
|
|
|
185
266
|
throw new Error("domains cannot include paths or query strings");
|
|
186
267
|
}
|
|
187
268
|
hostname = url.hostname;
|
|
188
|
-
} else if (
|
|
269
|
+
} else if (
|
|
270
|
+
hostInput.includes("/") ||
|
|
271
|
+
hostInput.includes("?") ||
|
|
272
|
+
hostInput.includes("#")
|
|
273
|
+
) {
|
|
189
274
|
throw new Error("domain is invalid");
|
|
190
275
|
}
|
|
191
276
|
hostname = hostname.replace(/\.$/, "");
|
|
@@ -214,7 +299,9 @@ export function originAllowed(origin: string, domains: string[]): boolean {
|
|
|
214
299
|
});
|
|
215
300
|
}
|
|
216
301
|
|
|
217
|
-
export function validateTimezone(
|
|
302
|
+
export function validateTimezone(
|
|
303
|
+
value: string | undefined,
|
|
304
|
+
): string | undefined {
|
|
218
305
|
if (value === undefined || value.trim() === "") return undefined;
|
|
219
306
|
const timezone = cleanString(value, 80);
|
|
220
307
|
try {
|
|
@@ -225,17 +312,33 @@ export function validateTimezone(value: string | undefined): string | undefined
|
|
|
225
312
|
}
|
|
226
313
|
}
|
|
227
314
|
|
|
228
|
-
export function sanitizeContext(
|
|
315
|
+
export function sanitizeContext(
|
|
316
|
+
context: IngestContext | undefined,
|
|
317
|
+
): IngestContext {
|
|
229
318
|
const latitude = finiteCoordinate(context?.latitude, -90, 90);
|
|
230
319
|
const longitude = finiteCoordinate(context?.longitude, -180, 180);
|
|
231
320
|
return {
|
|
232
|
-
country: context?.country
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
321
|
+
country: context?.country
|
|
322
|
+
? cleanString(context.country.toUpperCase(), 2) || undefined
|
|
323
|
+
: undefined,
|
|
324
|
+
city: context?.city
|
|
325
|
+
? cleanString(context.city, 80) || undefined
|
|
326
|
+
: undefined,
|
|
327
|
+
latitude:
|
|
328
|
+
latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
|
|
329
|
+
longitude:
|
|
330
|
+
longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
|
|
331
|
+
browser: context?.browser
|
|
332
|
+
? cleanString(context.browser, 32) || undefined
|
|
333
|
+
: undefined,
|
|
237
334
|
os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
|
|
238
|
-
device: context?.device
|
|
335
|
+
device: context?.device
|
|
336
|
+
? cleanString(context.device, 24) || undefined
|
|
337
|
+
: undefined,
|
|
338
|
+
visitorKey:
|
|
339
|
+
context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
|
|
340
|
+
? cleanString(context.visitorKey, 128) || undefined
|
|
341
|
+
: undefined,
|
|
239
342
|
};
|
|
240
343
|
}
|
|
241
344
|
|
|
@@ -249,10 +352,21 @@ export function isValidMoney(value: number): boolean {
|
|
|
249
352
|
return Number.isSafeInteger(value) && value >= 0 && value <= 1_000_000_000;
|
|
250
353
|
}
|
|
251
354
|
|
|
252
|
-
export function
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
355
|
+
export function measureBatchBytes(
|
|
356
|
+
siteId: string,
|
|
357
|
+
events: TrackerEvent[],
|
|
358
|
+
): { batchBytes: number; eventByteCounts: number[] } {
|
|
359
|
+
const encoder = new TextEncoder();
|
|
360
|
+
const eventByteCounts = events.map(
|
|
361
|
+
(event) => encoder.encode(JSON.stringify(event)).byteLength,
|
|
362
|
+
);
|
|
363
|
+
// The empty envelope already includes both array brackets; only commas remain.
|
|
364
|
+
const batchBytes =
|
|
365
|
+
encoder.encode(JSON.stringify({ siteId, events: [] })).byteLength +
|
|
366
|
+
eventByteCounts.reduce((total, byteCount) => total + byteCount, 0) +
|
|
367
|
+
Math.max(0, events.length - 1);
|
|
368
|
+
if (batchBytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
|
|
369
|
+
return { batchBytes, eventByteCounts };
|
|
256
370
|
}
|
|
257
371
|
|
|
258
372
|
export function sourceFromReferrer(referrer: string | undefined): string {
|
|
@@ -280,6 +394,7 @@ function finiteCoordinate(
|
|
|
280
394
|
maximum: number,
|
|
281
395
|
): number | undefined {
|
|
282
396
|
if (value === undefined) return undefined;
|
|
283
|
-
if (!Number.isFinite(value) || value < minimum || value > maximum)
|
|
397
|
+
if (!Number.isFinite(value) || value < minimum || value > maximum)
|
|
398
|
+
return undefined;
|
|
284
399
|
return value;
|
|
285
400
|
}
|
package/src/component/schema.ts
CHANGED
|
@@ -3,12 +3,13 @@ import { v } from "convex/values";
|
|
|
3
3
|
import {
|
|
4
4
|
affiliateFieldsValidator,
|
|
5
5
|
aggregateFieldsValidator,
|
|
6
|
-
|
|
6
|
+
batchedEventValidator,
|
|
7
7
|
funnelFieldsValidator,
|
|
8
8
|
goalFieldsValidator,
|
|
9
9
|
sessionFieldsValidator,
|
|
10
10
|
siteFieldsValidator,
|
|
11
|
-
|
|
11
|
+
visitorSketchValidator,
|
|
12
|
+
vitalMetricValidator,
|
|
12
13
|
} from "./validators.js";
|
|
13
14
|
|
|
14
15
|
export default defineSchema({
|
|
@@ -17,7 +18,12 @@ export default defineSchema({
|
|
|
17
18
|
.index("by_ownerId_and_name", ["ownerId", "name"])
|
|
18
19
|
.index("by_networkId", ["networkId"]),
|
|
19
20
|
|
|
20
|
-
sessions: defineTable(
|
|
21
|
+
sessions: defineTable(
|
|
22
|
+
sessionFieldsValidator.extend({
|
|
23
|
+
geoLookupAttemptedAt: v.optional(v.number()),
|
|
24
|
+
lastPageviewAt: v.optional(v.number()),
|
|
25
|
+
}),
|
|
26
|
+
)
|
|
21
27
|
.index("by_siteId_and_sessionId", ["siteId", "sessionId"])
|
|
22
28
|
.index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
|
|
23
29
|
.index("by_siteId_and_startedAt", ["siteId", "startedAt"])
|
|
@@ -48,6 +54,8 @@ export default defineSchema({
|
|
|
48
54
|
lastHeartbeatEventId: v.optional(v.string()),
|
|
49
55
|
recentHeartbeatEventIds: v.optional(v.array(v.string())),
|
|
50
56
|
eventCount: v.number(),
|
|
57
|
+
previousPath: v.optional(v.string()),
|
|
58
|
+
movedAt: v.optional(v.number()),
|
|
51
59
|
})
|
|
52
60
|
.index("by_siteId_and_sessionId", ["siteId", "sessionId"])
|
|
53
61
|
.index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
|
|
@@ -59,25 +67,19 @@ export default defineSchema({
|
|
|
59
67
|
scheduledAt: v.number(),
|
|
60
68
|
}).index("by_siteId", ["siteId"]),
|
|
61
69
|
|
|
62
|
-
|
|
63
|
-
.
|
|
64
|
-
.
|
|
65
|
-
.
|
|
66
|
-
|
|
67
|
-
"sessionId",
|
|
68
|
-
"timestamp",
|
|
69
|
-
])
|
|
70
|
-
.index("by_siteId_and_visitorId_and_timestamp", {
|
|
71
|
-
fields: ["siteId", "visitorId", "timestamp"],
|
|
72
|
-
staged: true,
|
|
73
|
-
}),
|
|
70
|
+
geoLookupBudgets: defineTable({
|
|
71
|
+
siteId: v.id("sites"),
|
|
72
|
+
dayStart: v.number(),
|
|
73
|
+
count: v.number(),
|
|
74
|
+
}).index("by_siteId", ["siteId"]),
|
|
74
75
|
|
|
75
76
|
eventBatches: defineTable({
|
|
76
77
|
siteId: v.id("sites"),
|
|
77
78
|
sessionId: v.string(),
|
|
78
79
|
visitorId: v.string(),
|
|
79
80
|
bucketStart: v.number(),
|
|
80
|
-
events: v.array(
|
|
81
|
+
events: v.array(batchedEventValidator),
|
|
82
|
+
migrationVersion: v.optional(v.literal(1)),
|
|
81
83
|
})
|
|
82
84
|
.index("by_siteId_and_sessionId_and_bucketStart", [
|
|
83
85
|
"siteId",
|
|
@@ -85,10 +87,11 @@ export default defineSchema({
|
|
|
85
87
|
"bucketStart",
|
|
86
88
|
])
|
|
87
89
|
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
|
|
88
|
-
.index("by_siteId_and_visitorId_and_bucketStart",
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
.index("by_siteId_and_visitorId_and_bucketStart", [
|
|
91
|
+
"siteId",
|
|
92
|
+
"visitorId",
|
|
93
|
+
"bucketStart",
|
|
94
|
+
]),
|
|
92
95
|
|
|
93
96
|
aggregateBuckets: defineTable(aggregateFieldsValidator)
|
|
94
97
|
.index("by_siteId_and_granularity_and_bucketStart_and_shard", [
|
|
@@ -103,6 +106,163 @@ export default defineSchema({
|
|
|
103
106
|
"bucketStart",
|
|
104
107
|
]),
|
|
105
108
|
|
|
109
|
+
routeDailyRollups: defineTable({
|
|
110
|
+
siteId: v.id("sites"),
|
|
111
|
+
bucketStart: v.number(),
|
|
112
|
+
shard: v.number(),
|
|
113
|
+
generation: v.literal(1),
|
|
114
|
+
route: v.string(),
|
|
115
|
+
pageviews: v.number(),
|
|
116
|
+
entries: v.number(),
|
|
117
|
+
continued: v.number(),
|
|
118
|
+
dwellMs: v.number(),
|
|
119
|
+
dwellSamples: v.number(),
|
|
120
|
+
visitorSketch: visitorSketchValidator,
|
|
121
|
+
outgoing: v.array(v.object({ route: v.string(), count: v.number() })),
|
|
122
|
+
})
|
|
123
|
+
.index("by_siteId_and_route_and_bucketStart_and_shard", [
|
|
124
|
+
"siteId",
|
|
125
|
+
"route",
|
|
126
|
+
"bucketStart",
|
|
127
|
+
"shard",
|
|
128
|
+
])
|
|
129
|
+
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
|
|
130
|
+
|
|
131
|
+
goalDailyRollups: defineTable({
|
|
132
|
+
siteId: v.id("sites"),
|
|
133
|
+
goalId: v.id("goals"),
|
|
134
|
+
bucketStart: v.number(),
|
|
135
|
+
shard: v.number(),
|
|
136
|
+
generation: v.literal(1),
|
|
137
|
+
goalKey: v.string(),
|
|
138
|
+
completions: v.number(),
|
|
139
|
+
valueCents: v.number(),
|
|
140
|
+
})
|
|
141
|
+
.index("by_siteId_and_goalId_and_bucketStart_and_shard", [
|
|
142
|
+
"siteId",
|
|
143
|
+
"goalId",
|
|
144
|
+
"bucketStart",
|
|
145
|
+
"shard",
|
|
146
|
+
])
|
|
147
|
+
.index("by_siteId_and_goalId_and_bucketStart", [
|
|
148
|
+
"siteId",
|
|
149
|
+
"goalId",
|
|
150
|
+
"bucketStart",
|
|
151
|
+
])
|
|
152
|
+
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
|
|
153
|
+
|
|
154
|
+
funnelDailyRollups: defineTable({
|
|
155
|
+
siteId: v.id("sites"),
|
|
156
|
+
funnelId: v.id("funnels"),
|
|
157
|
+
bucketStart: v.number(),
|
|
158
|
+
shard: v.number(),
|
|
159
|
+
generation: v.literal(1),
|
|
160
|
+
steps: v.array(
|
|
161
|
+
v.object({
|
|
162
|
+
step: v.number(),
|
|
163
|
+
entrants: v.number(),
|
|
164
|
+
completions: v.number(),
|
|
165
|
+
}),
|
|
166
|
+
),
|
|
167
|
+
})
|
|
168
|
+
.index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
|
|
169
|
+
"siteId",
|
|
170
|
+
"funnelId",
|
|
171
|
+
"bucketStart",
|
|
172
|
+
"shard",
|
|
173
|
+
])
|
|
174
|
+
.index("by_siteId_and_funnelId_and_bucketStart", [
|
|
175
|
+
"siteId",
|
|
176
|
+
"funnelId",
|
|
177
|
+
"bucketStart",
|
|
178
|
+
])
|
|
179
|
+
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
|
|
180
|
+
|
|
181
|
+
// Field Web Vitals. One row per site/day/shard and either a page (device
|
|
182
|
+
// "(all)") or a device class (page "(all)"); the row folds every metric's
|
|
183
|
+
// histogram together so a report reads five metrics for one document. Pages
|
|
184
|
+
// are bounded like dimension slots: at most VITAL_PAGE_SLOTS distinct pages
|
|
185
|
+
// per day, then "(other)".
|
|
186
|
+
vitalRollups: defineTable({
|
|
187
|
+
siteId: v.id("sites"),
|
|
188
|
+
bucketStart: v.number(),
|
|
189
|
+
device: v.string(),
|
|
190
|
+
page: v.string(),
|
|
191
|
+
shard: v.number(),
|
|
192
|
+
metrics: v.array(
|
|
193
|
+
v.object({
|
|
194
|
+
metric: vitalMetricValidator,
|
|
195
|
+
count: v.number(),
|
|
196
|
+
sum: v.number(),
|
|
197
|
+
histogram: v.array(v.number()),
|
|
198
|
+
}),
|
|
199
|
+
),
|
|
200
|
+
})
|
|
201
|
+
.index("by_key", ["siteId", "bucketStart", "device", "page", "shard"])
|
|
202
|
+
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
|
|
203
|
+
|
|
204
|
+
affiliateDailyRollups: defineTable({
|
|
205
|
+
siteId: v.id("sites"),
|
|
206
|
+
affiliateId: v.id("affiliates"),
|
|
207
|
+
bucketStart: v.number(),
|
|
208
|
+
shard: v.number(),
|
|
209
|
+
generation: v.literal(1),
|
|
210
|
+
affiliateSlug: v.string(),
|
|
211
|
+
visits: v.number(),
|
|
212
|
+
visitorSketch: visitorSketchValidator,
|
|
213
|
+
conversions: v.number(),
|
|
214
|
+
revenueCents: v.number(),
|
|
215
|
+
commissionCents: v.number(),
|
|
216
|
+
})
|
|
217
|
+
.index("by_siteId_and_affiliateId_and_bucketStart_and_shard", [
|
|
218
|
+
"siteId",
|
|
219
|
+
"affiliateId",
|
|
220
|
+
"bucketStart",
|
|
221
|
+
"shard",
|
|
222
|
+
])
|
|
223
|
+
.index("by_siteId_and_affiliateId_and_bucketStart", [
|
|
224
|
+
"siteId",
|
|
225
|
+
"affiliateId",
|
|
226
|
+
"bucketStart",
|
|
227
|
+
])
|
|
228
|
+
.index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
|
|
229
|
+
|
|
230
|
+
analyticsCoverage: defineTable({
|
|
231
|
+
siteId: v.id("sites"),
|
|
232
|
+
generation: v.literal(1),
|
|
233
|
+
rollupCompleteFrom: v.number(),
|
|
234
|
+
siteMapSince: v.optional(v.number()),
|
|
235
|
+
retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
|
|
236
|
+
updatedAt: v.number(),
|
|
237
|
+
}).index("by_siteId", ["siteId"]),
|
|
238
|
+
|
|
239
|
+
retentionPolicies: defineTable({
|
|
240
|
+
siteId: v.id("sites"),
|
|
241
|
+
eventsDays: v.number(),
|
|
242
|
+
sessionsDays: v.number(),
|
|
243
|
+
conversionsDays: v.number(),
|
|
244
|
+
aggregatesDays: v.number(),
|
|
245
|
+
reportsDays: v.number(),
|
|
246
|
+
updatedAt: v.number(),
|
|
247
|
+
}).index("by_siteId", ["siteId"]),
|
|
248
|
+
|
|
249
|
+
retentionStates: defineTable({
|
|
250
|
+
siteId: v.id("sites"),
|
|
251
|
+
kind: v.string(),
|
|
252
|
+
before: v.number(),
|
|
253
|
+
status: v.union(
|
|
254
|
+
v.literal("pending"),
|
|
255
|
+
v.literal("running"),
|
|
256
|
+
v.literal("complete"),
|
|
257
|
+
),
|
|
258
|
+
generation: v.number(),
|
|
259
|
+
attempt: v.number(),
|
|
260
|
+
leaseExpiresAt: v.optional(v.number()),
|
|
261
|
+
scheduledAt: v.optional(v.number()),
|
|
262
|
+
deleted: v.number(),
|
|
263
|
+
updatedAt: v.number(),
|
|
264
|
+
}).index("by_siteId_and_kind", ["siteId", "kind"]),
|
|
265
|
+
|
|
106
266
|
goals: defineTable(goalFieldsValidator)
|
|
107
267
|
.index("by_siteId", ["siteId"])
|
|
108
268
|
.index("by_siteId_and_key", ["siteId", "key"]),
|
|
@@ -119,7 +279,11 @@ export default defineSchema({
|
|
|
119
279
|
valueCents: v.number(),
|
|
120
280
|
})
|
|
121
281
|
.index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
|
|
122
|
-
.index("by_siteId_and_goalId_and_timestamp", [
|
|
282
|
+
.index("by_siteId_and_goalId_and_timestamp", [
|
|
283
|
+
"siteId",
|
|
284
|
+
"goalId",
|
|
285
|
+
"timestamp",
|
|
286
|
+
])
|
|
123
287
|
.index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
|
|
124
288
|
|
|
125
289
|
funnels: defineTable(funnelFieldsValidator)
|
|
@@ -171,6 +335,21 @@ export default defineSchema({
|
|
|
171
335
|
.index("by_siteId", ["siteId"])
|
|
172
336
|
.index("by_siteId_and_slug", ["siteId", "slug"]),
|
|
173
337
|
|
|
338
|
+
visitorSecrets: defineTable({
|
|
339
|
+
key: v.literal("current"),
|
|
340
|
+
value: v.string(),
|
|
341
|
+
createdAt: v.number(),
|
|
342
|
+
}).index("by_key", ["key"]),
|
|
343
|
+
|
|
344
|
+
visitorAliases: defineTable({
|
|
345
|
+
siteId: v.id("sites"),
|
|
346
|
+
visitorId: v.string(),
|
|
347
|
+
previousVisitorId: v.string(),
|
|
348
|
+
linkedAt: v.number(),
|
|
349
|
+
})
|
|
350
|
+
.index("by_siteId_and_visitorId", ["siteId", "visitorId"])
|
|
351
|
+
.index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
|
|
352
|
+
|
|
174
353
|
visitorAttributions: defineTable({
|
|
175
354
|
siteId: v.id("sites"),
|
|
176
355
|
visitorId: v.string(),
|
package/src/component/sites.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { v } from "convex/values";
|
|
2
2
|
import { internalQuery, mutation, query } from "./_generated/server.js";
|
|
3
3
|
import { MAX_SITES_PER_OWNER } from "./constants.js";
|
|
4
|
+
import { ensureAnalyticsControl } from "./coverage.js";
|
|
4
5
|
import { fail } from "./errors.js";
|
|
5
6
|
import {
|
|
6
7
|
cleanString,
|
|
@@ -31,7 +32,8 @@ export const create = mutation({
|
|
|
31
32
|
handler: async (ctx, args) => {
|
|
32
33
|
const ownerId = cleanString(args.ownerId, 128);
|
|
33
34
|
const name = cleanString(args.name, 120);
|
|
34
|
-
if (!ownerId || !name)
|
|
35
|
+
if (!ownerId || !name)
|
|
36
|
+
fail("INVALID_SITE", "ownerId and name are required");
|
|
35
37
|
|
|
36
38
|
let domains: string[];
|
|
37
39
|
let timezone: string | undefined;
|
|
@@ -45,7 +47,10 @@ export const create = mutation({
|
|
|
45
47
|
: undefined;
|
|
46
48
|
currency = sanitizeCurrency(args.currency ?? "USD");
|
|
47
49
|
} catch (error) {
|
|
48
|
-
fail(
|
|
50
|
+
fail(
|
|
51
|
+
"INVALID_SITE",
|
|
52
|
+
error instanceof Error ? error.message : "invalid site",
|
|
53
|
+
);
|
|
49
54
|
}
|
|
50
55
|
const cookieless = args.cookieless ?? true;
|
|
51
56
|
|
|
@@ -64,8 +69,12 @@ export const create = mutation({
|
|
|
64
69
|
(existing.currency ?? "USD") !== currency ||
|
|
65
70
|
existing.cookieless !== cookieless
|
|
66
71
|
) {
|
|
67
|
-
fail(
|
|
72
|
+
fail(
|
|
73
|
+
"CONFLICT",
|
|
74
|
+
"site name is already associated with different settings",
|
|
75
|
+
);
|
|
68
76
|
}
|
|
77
|
+
await ensureAnalyticsControl(ctx, existing._id, Date.now());
|
|
69
78
|
return existing._id;
|
|
70
79
|
}
|
|
71
80
|
|
|
@@ -80,7 +89,7 @@ export const create = mutation({
|
|
|
80
89
|
}
|
|
81
90
|
|
|
82
91
|
const now = Date.now();
|
|
83
|
-
|
|
92
|
+
const siteId = await ctx.db.insert("sites", {
|
|
84
93
|
ownerId,
|
|
85
94
|
name,
|
|
86
95
|
domains,
|
|
@@ -91,6 +100,8 @@ export const create = mutation({
|
|
|
91
100
|
createdAt: now,
|
|
92
101
|
updatedAt: now,
|
|
93
102
|
});
|
|
103
|
+
await ensureAnalyticsControl(ctx, siteId, now);
|
|
104
|
+
return siteId;
|
|
94
105
|
},
|
|
95
106
|
});
|
|
96
107
|
|
|
@@ -133,22 +144,26 @@ export const update = mutation({
|
|
|
133
144
|
patch.name = name;
|
|
134
145
|
}
|
|
135
146
|
try {
|
|
136
|
-
if (args.domains !== undefined)
|
|
147
|
+
if (args.domains !== undefined)
|
|
148
|
+
patch.domains = normalizeDomains(args.domains);
|
|
137
149
|
if (args.timezone !== undefined) {
|
|
138
|
-
patch.timezone =
|
|
139
|
-
? undefined
|
|
140
|
-
: validateTimezone(args.timezone);
|
|
150
|
+
patch.timezone =
|
|
151
|
+
args.timezone === null ? undefined : validateTimezone(args.timezone);
|
|
141
152
|
}
|
|
142
153
|
if (args.networkId !== undefined) {
|
|
143
|
-
patch.networkId =
|
|
144
|
-
|
|
145
|
-
|
|
154
|
+
patch.networkId =
|
|
155
|
+
args.networkId === null
|
|
156
|
+
? undefined
|
|
157
|
+
: sanitizeOpaqueId(args.networkId, "networkId");
|
|
146
158
|
}
|
|
147
159
|
if (args.currency !== undefined) {
|
|
148
160
|
patch.currency = sanitizeCurrency(args.currency);
|
|
149
161
|
}
|
|
150
162
|
} catch (error) {
|
|
151
|
-
fail(
|
|
163
|
+
fail(
|
|
164
|
+
"INVALID_SITE",
|
|
165
|
+
error instanceof Error ? error.message : "invalid site",
|
|
166
|
+
);
|
|
152
167
|
}
|
|
153
168
|
if (
|
|
154
169
|
patch.currency !== undefined &&
|