@iann29/rastro 0.1.0-alpha.1 → 0.1.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/README.md +521 -66
  2. package/agent/integration.md +661 -0
  3. package/agent/manifest.json +187 -0
  4. package/agent/manifest.schema.json +411 -0
  5. package/dist/client/federation.d.ts +236 -0
  6. package/dist/client/federation.d.ts.map +1 -0
  7. package/dist/client/federation.js +196 -0
  8. package/dist/client/federation.js.map +1 -0
  9. package/dist/client/index.d.ts +2467 -18
  10. package/dist/client/index.d.ts.map +1 -1
  11. package/dist/client/index.js +461 -32
  12. package/dist/client/index.js.map +1 -1
  13. package/dist/component/_generated/api.d.ts +16 -0
  14. package/dist/component/_generated/api.d.ts.map +1 -1
  15. package/dist/component/_generated/api.js.map +1 -1
  16. package/dist/component/_generated/component.d.ts +302 -4
  17. package/dist/component/_generated/component.d.ts.map +1 -1
  18. package/dist/component/_generated/server.d.ts +4 -0
  19. package/dist/component/_generated/server.d.ts.map +1 -1
  20. package/dist/component/_generated/server.js.map +1 -1
  21. package/dist/component/affiliates.d.ts.map +1 -1
  22. package/dist/component/affiliates.js +6 -2
  23. package/dist/component/affiliates.js.map +1 -1
  24. package/dist/component/cardinality.d.ts +12 -0
  25. package/dist/component/cardinality.d.ts.map +1 -0
  26. package/dist/component/cardinality.js +94 -0
  27. package/dist/component/cardinality.js.map +1 -0
  28. package/dist/component/constants.d.ts +25 -1
  29. package/dist/component/constants.d.ts.map +1 -1
  30. package/dist/component/constants.js +38 -1
  31. package/dist/component/constants.js.map +1 -1
  32. package/dist/component/convex.config.d.ts +6 -1
  33. package/dist/component/convex.config.js +9 -1
  34. package/dist/component/convex.config.js.map +1 -1
  35. package/dist/component/coverage.d.ts +19 -0
  36. package/dist/component/coverage.d.ts.map +1 -0
  37. package/dist/component/coverage.js +30 -0
  38. package/dist/component/coverage.js.map +1 -0
  39. package/dist/component/diagnostics.d.ts +9 -0
  40. package/dist/component/diagnostics.d.ts.map +1 -0
  41. package/dist/component/diagnostics.js +47 -0
  42. package/dist/component/diagnostics.js.map +1 -0
  43. package/dist/component/errors.d.ts +1 -1
  44. package/dist/component/errors.d.ts.map +1 -1
  45. package/dist/component/errors.js.map +1 -1
  46. package/dist/component/eventStore.d.ts +21 -9
  47. package/dist/component/eventStore.d.ts.map +1 -1
  48. package/dist/component/eventStore.js +142 -152
  49. package/dist/component/eventStore.js.map +1 -1
  50. package/dist/component/funnels.d.ts.map +1 -1
  51. package/dist/component/funnels.js +5 -3
  52. package/dist/component/funnels.js.map +1 -1
  53. package/dist/component/geo.d.ts +71 -0
  54. package/dist/component/geo.d.ts.map +1 -0
  55. package/dist/component/geo.js +611 -0
  56. package/dist/component/geo.js.map +1 -0
  57. package/dist/component/goals.d.ts.map +1 -1
  58. package/dist/component/goals.js +8 -5
  59. package/dist/component/goals.js.map +1 -1
  60. package/dist/component/guards.d.ts.map +1 -1
  61. package/dist/component/guards.js.map +1 -1
  62. package/dist/component/http.d.ts.map +1 -1
  63. package/dist/component/http.js +248 -49
  64. package/dist/component/http.js.map +1 -1
  65. package/dist/component/identity.d.ts +13 -0
  66. package/dist/component/identity.d.ts.map +1 -0
  67. package/dist/component/identity.js +58 -0
  68. package/dist/component/identity.js.map +1 -0
  69. package/dist/component/ingest.d.ts +3 -1
  70. package/dist/component/ingest.d.ts.map +1 -1
  71. package/dist/component/ingest.js +604 -55
  72. package/dist/component/ingest.js.map +1 -1
  73. package/dist/component/live.d.ts.map +1 -1
  74. package/dist/component/live.js +7 -6
  75. package/dist/component/live.js.map +1 -1
  76. package/dist/component/reports.d.ts +259 -10
  77. package/dist/component/reports.d.ts.map +1 -1
  78. package/dist/component/reports.js +1110 -124
  79. package/dist/component/reports.js.map +1 -1
  80. package/dist/component/retention.d.ts +75 -1
  81. package/dist/component/retention.d.ts.map +1 -1
  82. package/dist/component/retention.js +537 -32
  83. package/dist/component/retention.js.map +1 -1
  84. package/dist/component/sanitize.d.ts +24 -1
  85. package/dist/component/sanitize.d.ts.map +1 -1
  86. package/dist/component/sanitize.js +96 -16
  87. package/dist/component/sanitize.js.map +1 -1
  88. package/dist/component/schema.d.ts +378 -62
  89. package/dist/component/schema.js +178 -20
  90. package/dist/component/schema.js.map +1 -1
  91. package/dist/component/sites.d.ts.map +1 -1
  92. package/dist/component/sites.js +11 -7
  93. package/dist/component/sites.js.map +1 -1
  94. package/dist/component/useragent.d.ts +9 -0
  95. package/dist/component/useragent.d.ts.map +1 -0
  96. package/dist/component/useragent.js +152 -0
  97. package/dist/component/useragent.js.map +1 -0
  98. package/dist/component/validators.d.ts +101 -65
  99. package/dist/component/validators.d.ts.map +1 -1
  100. package/dist/component/validators.js +19 -13
  101. package/dist/component/validators.js.map +1 -1
  102. package/dist/component/visitors.d.ts +19 -0
  103. package/dist/component/visitors.d.ts.map +1 -0
  104. package/dist/component/visitors.js +86 -0
  105. package/dist/component/visitors.js.map +1 -0
  106. package/dist/component/vitals.d.ts +41 -0
  107. package/dist/component/vitals.d.ts.map +1 -0
  108. package/dist/component/vitals.js +115 -0
  109. package/dist/component/vitals.js.map +1 -0
  110. package/dist/react/index.d.ts.map +1 -1
  111. package/dist/react/index.js.map +1 -1
  112. package/dist/tracker/generated.d.ts +8 -4
  113. package/dist/tracker/generated.d.ts.map +1 -1
  114. package/dist/tracker/generated.js +8 -4
  115. package/dist/tracker/generated.js.map +1 -1
  116. package/dist/tracker/tracker.d.ts +1 -1
  117. package/dist/tracker/tracker.d.ts.map +1 -1
  118. package/dist/tracker/tracker.js +57 -20
  119. package/dist/tracker/tracker.js.map +1 -1
  120. package/dist/tracker/vitals.d.ts +10 -0
  121. package/dist/tracker/vitals.d.ts.map +1 -0
  122. package/dist/tracker/vitals.js +140 -0
  123. package/dist/tracker/vitals.js.map +1 -0
  124. package/dist/tracker.min.js +1 -1
  125. package/dist/vitals.min.js +1 -0
  126. package/docs/benchmarks/2026-08-20-realistic.md +76 -76
  127. package/docs/benchmarks/2026-08-21-formal-certification.md +353 -0
  128. package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
  129. package/docs/federation-setup.md +397 -0
  130. package/docs/federation.md +262 -0
  131. package/docs/upgrading.md +259 -0
  132. package/llms.txt +71 -0
  133. package/package.json +55 -11
  134. package/scripts/benchmark-ingest.mjs +175 -73
  135. package/scripts/generate-federation-keys.mjs +20 -0
  136. package/src/component/_generated/api.ts +16 -0
  137. package/src/component/_generated/component.ts +366 -4
  138. package/src/component/_generated/server.ts +4 -0
  139. package/src/component/affiliates.ts +20 -5
  140. package/src/component/cardinality.ts +117 -0
  141. package/src/component/constants.ts +38 -1
  142. package/src/component/convex.config.ts +11 -1
  143. package/src/component/coverage.ts +36 -0
  144. package/src/component/diagnostics.ts +65 -0
  145. package/src/component/errors.ts +2 -1
  146. package/src/component/eventStore.ts +217 -193
  147. package/src/component/funnels.ts +19 -16
  148. package/src/component/geo.ts +781 -0
  149. package/src/component/goals.ts +29 -21
  150. package/src/component/guards.ts +3 -1
  151. package/src/component/http.ts +357 -57
  152. package/src/component/identity.ts +74 -0
  153. package/src/component/ingest.ts +1040 -135
  154. package/src/component/live.ts +13 -7
  155. package/src/component/reports.ts +1675 -183
  156. package/src/component/retention.ts +770 -96
  157. package/src/component/sanitize.ts +144 -29
  158. package/src/component/schema.ts +200 -21
  159. package/src/component/sites.ts +27 -12
  160. package/src/component/useragent.ts +171 -0
  161. package/src/component/validators.ts +29 -13
  162. package/src/component/visitors.ts +116 -0
  163. package/src/component/vitals.ts +146 -0
@@ -1,4 +1,5 @@
1
1
  import {
2
+ CLOCK_SKEW_TOLERANCE_MS,
2
3
  MAX_BATCH_BYTES,
3
4
  MAX_PROPERTIES,
4
5
  MAX_SITE_DOMAINS,
@@ -8,6 +9,7 @@ import type {
8
9
  IngestContext,
9
10
  TrackerEvent,
10
11
  } from "./validators.js";
12
+ import { isValidVitalValue, isVitalMetric } from "./vitals.js";
11
13
 
12
14
  const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
13
15
  const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -62,9 +64,10 @@ export function sanitizePath(value: string): string {
62
64
  const input = requireString(value, "path", 2048);
63
65
  let path: string;
64
66
  try {
65
- path = input.startsWith("http://") || input.startsWith("https://")
66
- ? new URL(input).pathname
67
- : input.split(/[?#]/, 1)[0];
67
+ path =
68
+ input.startsWith("http://") || input.startsWith("https://")
69
+ ? new URL(input).pathname
70
+ : input.split(/[?#]/, 1)[0];
68
71
  } catch {
69
72
  throw new Error("path is invalid");
70
73
  }
@@ -73,6 +76,37 @@ export function sanitizePath(value: string): string {
73
76
  return cleaned || "/";
74
77
  }
75
78
 
79
+ const ROUTE_ID = ":id";
80
+ const NUMERIC_SEGMENT = /^\d+$/;
81
+ const UUID_SEGMENT =
82
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
83
+ const HEX_SEGMENT = /^[0-9a-f]{16,}$/i;
84
+
85
+ /**
86
+ * Collapses the identifiers that unmistakably vary per record — numbers,
87
+ * UUIDs, and long hexadecimal tokens — into `:id`, so `/blog/42` and
88
+ * `/blog/43` roll up as one route in the site map. Slugs are pages of their
89
+ * own and stay as they are. A trailing slash is dropped because both
90
+ * spellings serve the same page. Only the map uses this; every other report
91
+ * keeps the sanitized path exactly as the tracker sent it.
92
+ */
93
+ export function normalizeRoute(path: string): string {
94
+ const trimmed =
95
+ path.length > 1 && path.endsWith("/") ? path.slice(0, -1) : path;
96
+ const segments = trimmed.split("/");
97
+ for (let index = 1; index < segments.length; index += 1) {
98
+ const segment = segments[index] ?? "";
99
+ if (
100
+ NUMERIC_SEGMENT.test(segment) ||
101
+ UUID_SEGMENT.test(segment) ||
102
+ HEX_SEGMENT.test(segment)
103
+ ) {
104
+ segments[index] = ROUTE_ID;
105
+ }
106
+ }
107
+ return segments.join("/") || "/";
108
+ }
109
+
76
110
  export function sanitizePublicUrl(value: string): string | undefined {
77
111
  const input = cleanString(value, 2048);
78
112
  if (!input) return undefined;
@@ -111,15 +145,42 @@ export function sanitizeProperties(
111
145
  return Object.keys(result).length > 0 ? result : undefined;
112
146
  }
113
147
 
148
+ /**
149
+ * Re-anchors a batch whose sender's clock disagrees with ours. `sentAt` is the
150
+ * sender's clock at flush time, so `now - sentAt` is the clock offset alone: an
151
+ * event queued long before the flush keeps its age, and nothing is inferred
152
+ * from the events themselves. Offsets inside the tolerance are latency or a
153
+ * slightly drifted clock and leave the batch untouched, so a well-set client's
154
+ * timestamps stay exactly as sent.
155
+ */
156
+ export function alignClock<T extends { timestamp: number }>(
157
+ events: T[],
158
+ sentAt: number | undefined,
159
+ now: number,
160
+ ): T[] {
161
+ if (sentAt === undefined || !Number.isFinite(sentAt)) return events;
162
+ const offset = now - sentAt;
163
+ if (Math.abs(offset) <= CLOCK_SKEW_TOLERANCE_MS) return events;
164
+ return events.map((event) => ({
165
+ ...event,
166
+ timestamp: event.timestamp + offset,
167
+ }));
168
+ }
169
+
114
170
  export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
115
- if (!Number.isFinite(event.timestamp)) throw new Error("timestamp is invalid");
171
+ if (!Number.isFinite(event.timestamp))
172
+ throw new Error("timestamp is invalid");
116
173
  if (
117
174
  event.timestamp < now - MAX_EVENT_AGE_MS ||
118
175
  event.timestamp > now + MAX_FUTURE_SKEW_MS
119
176
  ) {
120
177
  throw new Error("timestamp is outside the accepted window");
121
178
  }
122
- if (!Number.isSafeInteger(event.sequence) || event.sequence < 0 || event.sequence > 1_000_000_000) {
179
+ if (
180
+ !Number.isSafeInteger(event.sequence) ||
181
+ event.sequence < 0 ||
182
+ event.sequence > 1_000_000_000
183
+ ) {
123
184
  throw new Error("sequence is out of range");
124
185
  }
125
186
  if (event.revenueCents !== undefined && !isValidMoney(event.revenueCents)) {
@@ -130,9 +191,20 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
130
191
  throw new Error("name is required for custom events");
131
192
  }
132
193
  if (event.type === "conversion" && !name) name = "conversion";
133
- const currency = event.currency === undefined
134
- ? event.type === "conversion" ? "USD" : undefined
135
- : sanitizeCurrency(event.currency);
194
+ if (event.type === "vital") {
195
+ if (!name || !isVitalMetric(name)) {
196
+ throw new Error("vital name must be a Web Vitals metric");
197
+ }
198
+ if (event.value === undefined || !isValidVitalValue(event.value)) {
199
+ throw new Error("vital value is out of range");
200
+ }
201
+ }
202
+ const currency =
203
+ event.currency === undefined
204
+ ? event.type === "conversion"
205
+ ? "USD"
206
+ : undefined
207
+ : sanitizeCurrency(event.currency);
136
208
  return {
137
209
  eventId: sanitizeOpaqueId(event.eventId, "eventId"),
138
210
  sessionId: sanitizeOpaqueId(event.sessionId, "sessionId"),
@@ -140,19 +212,26 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
140
212
  type: event.type,
141
213
  name,
142
214
  path: sanitizePath(event.path),
143
- referrer: event.referrer === undefined
144
- ? undefined
145
- : sanitizePublicUrl(event.referrer),
215
+ referrer:
216
+ event.referrer === undefined
217
+ ? undefined
218
+ : sanitizePublicUrl(event.referrer),
146
219
  timestamp: Math.trunc(event.timestamp),
147
220
  sequence: event.sequence,
148
- target: event.target === undefined
149
- ? undefined
150
- : cleanString(event.target, 160) || undefined,
221
+ target:
222
+ event.target === undefined
223
+ ? undefined
224
+ : cleanString(event.target.replace(/\s+/g, " "), 160) || undefined,
151
225
  href: event.href === undefined ? undefined : sanitizePublicUrl(event.href),
152
226
  properties: sanitizeProperties(event.properties),
153
227
  revenueCents: event.revenueCents,
154
228
  currency,
155
229
  affiliateSlug: optionalAffiliateSlug(event.affiliateSlug),
230
+ // Only measurements keep a value; a stray number on any other type is noise.
231
+ value:
232
+ event.type === "vital" && event.value !== undefined
233
+ ? Math.round(event.value)
234
+ : undefined,
156
235
  };
157
236
  }
158
237
 
@@ -167,7 +246,9 @@ function optionalAffiliateSlug(value: string | undefined): string | undefined {
167
246
 
168
247
  export function normalizeDomains(domains: string[]): string[] {
169
248
  if (domains.length === 0 || domains.length > MAX_SITE_DOMAINS) {
170
- throw new Error(`domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`);
249
+ throw new Error(
250
+ `domains must contain between 1 and ${MAX_SITE_DOMAINS} entries`,
251
+ );
171
252
  }
172
253
  const normalized = domains.map(normalizeDomainPattern);
173
254
  return [...new Set(normalized)].sort();
@@ -185,7 +266,11 @@ export function normalizeDomainPattern(value: string): string {
185
266
  throw new Error("domains cannot include paths or query strings");
186
267
  }
187
268
  hostname = url.hostname;
188
- } else if (hostInput.includes("/") || hostInput.includes("?") || hostInput.includes("#")) {
269
+ } else if (
270
+ hostInput.includes("/") ||
271
+ hostInput.includes("?") ||
272
+ hostInput.includes("#")
273
+ ) {
189
274
  throw new Error("domain is invalid");
190
275
  }
191
276
  hostname = hostname.replace(/\.$/, "");
@@ -214,7 +299,9 @@ export function originAllowed(origin: string, domains: string[]): boolean {
214
299
  });
215
300
  }
216
301
 
217
- export function validateTimezone(value: string | undefined): string | undefined {
302
+ export function validateTimezone(
303
+ value: string | undefined,
304
+ ): string | undefined {
218
305
  if (value === undefined || value.trim() === "") return undefined;
219
306
  const timezone = cleanString(value, 80);
220
307
  try {
@@ -225,17 +312,33 @@ export function validateTimezone(value: string | undefined): string | undefined
225
312
  }
226
313
  }
227
314
 
228
- export function sanitizeContext(context: IngestContext | undefined): IngestContext {
315
+ export function sanitizeContext(
316
+ context: IngestContext | undefined,
317
+ ): IngestContext {
229
318
  const latitude = finiteCoordinate(context?.latitude, -90, 90);
230
319
  const longitude = finiteCoordinate(context?.longitude, -180, 180);
231
320
  return {
232
- country: context?.country ? cleanString(context.country.toUpperCase(), 2) || undefined : undefined,
233
- city: context?.city ? cleanString(context.city, 80) || undefined : undefined,
234
- latitude: latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
235
- longitude: longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
236
- browser: context?.browser ? cleanString(context.browser, 32) || undefined : undefined,
321
+ country: context?.country
322
+ ? cleanString(context.country.toUpperCase(), 2) || undefined
323
+ : undefined,
324
+ city: context?.city
325
+ ? cleanString(context.city, 80) || undefined
326
+ : undefined,
327
+ latitude:
328
+ latitude === undefined ? undefined : Math.round(latitude * 10) / 10,
329
+ longitude:
330
+ longitude === undefined ? undefined : Math.round(longitude * 10) / 10,
331
+ browser: context?.browser
332
+ ? cleanString(context.browser, 32) || undefined
333
+ : undefined,
237
334
  os: context?.os ? cleanString(context.os, 32) || undefined : undefined,
238
- device: context?.device ? cleanString(context.device, 24) || undefined : undefined,
335
+ device: context?.device
336
+ ? cleanString(context.device, 24) || undefined
337
+ : undefined,
338
+ visitorKey:
339
+ context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
340
+ ? cleanString(context.visitorKey, 128) || undefined
341
+ : undefined,
239
342
  };
240
343
  }
241
344
 
@@ -249,10 +352,21 @@ export function isValidMoney(value: number): boolean {
249
352
  return Number.isSafeInteger(value) && value >= 0 && value <= 1_000_000_000;
250
353
  }
251
354
 
252
- export function ensureBatchBytes(value: unknown): number {
253
- const bytes = new TextEncoder().encode(JSON.stringify(value)).byteLength;
254
- if (bytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
255
- return bytes;
355
+ export function measureBatchBytes(
356
+ siteId: string,
357
+ events: TrackerEvent[],
358
+ ): { batchBytes: number; eventByteCounts: number[] } {
359
+ const encoder = new TextEncoder();
360
+ const eventByteCounts = events.map(
361
+ (event) => encoder.encode(JSON.stringify(event)).byteLength,
362
+ );
363
+ // The empty envelope already includes both array brackets; only commas remain.
364
+ const batchBytes =
365
+ encoder.encode(JSON.stringify({ siteId, events: [] })).byteLength +
366
+ eventByteCounts.reduce((total, byteCount) => total + byteCount, 0) +
367
+ Math.max(0, events.length - 1);
368
+ if (batchBytes > MAX_BATCH_BYTES) throw new Error("batch exceeds 64 KiB");
369
+ return { batchBytes, eventByteCounts };
256
370
  }
257
371
 
258
372
  export function sourceFromReferrer(referrer: string | undefined): string {
@@ -280,6 +394,7 @@ function finiteCoordinate(
280
394
  maximum: number,
281
395
  ): number | undefined {
282
396
  if (value === undefined) return undefined;
283
- if (!Number.isFinite(value) || value < minimum || value > maximum) return undefined;
397
+ if (!Number.isFinite(value) || value < minimum || value > maximum)
398
+ return undefined;
284
399
  return value;
285
400
  }
@@ -3,12 +3,13 @@ import { v } from "convex/values";
3
3
  import {
4
4
  affiliateFieldsValidator,
5
5
  aggregateFieldsValidator,
6
- storedEventFieldsValidator,
6
+ batchedEventValidator,
7
7
  funnelFieldsValidator,
8
8
  goalFieldsValidator,
9
9
  sessionFieldsValidator,
10
10
  siteFieldsValidator,
11
- trackerEventValidator,
11
+ visitorSketchValidator,
12
+ vitalMetricValidator,
12
13
  } from "./validators.js";
13
14
 
14
15
  export default defineSchema({
@@ -17,7 +18,12 @@ export default defineSchema({
17
18
  .index("by_ownerId_and_name", ["ownerId", "name"])
18
19
  .index("by_networkId", ["networkId"]),
19
20
 
20
- sessions: defineTable(sessionFieldsValidator)
21
+ sessions: defineTable(
22
+ sessionFieldsValidator.extend({
23
+ geoLookupAttemptedAt: v.optional(v.number()),
24
+ lastPageviewAt: v.optional(v.number()),
25
+ }),
26
+ )
21
27
  .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
22
28
  .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
23
29
  .index("by_siteId_and_startedAt", ["siteId", "startedAt"])
@@ -48,6 +54,8 @@ export default defineSchema({
48
54
  lastHeartbeatEventId: v.optional(v.string()),
49
55
  recentHeartbeatEventIds: v.optional(v.array(v.string())),
50
56
  eventCount: v.number(),
57
+ previousPath: v.optional(v.string()),
58
+ movedAt: v.optional(v.number()),
51
59
  })
52
60
  .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
53
61
  .index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
@@ -59,25 +67,19 @@ export default defineSchema({
59
67
  scheduledAt: v.number(),
60
68
  }).index("by_siteId", ["siteId"]),
61
69
 
62
- events: defineTable(storedEventFieldsValidator)
63
- .index("by_siteId_and_eventId", ["siteId", "eventId"])
64
- .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
65
- .index("by_siteId_and_sessionId_and_timestamp", [
66
- "siteId",
67
- "sessionId",
68
- "timestamp",
69
- ])
70
- .index("by_siteId_and_visitorId_and_timestamp", {
71
- fields: ["siteId", "visitorId", "timestamp"],
72
- staged: true,
73
- }),
70
+ geoLookupBudgets: defineTable({
71
+ siteId: v.id("sites"),
72
+ dayStart: v.number(),
73
+ count: v.number(),
74
+ }).index("by_siteId", ["siteId"]),
74
75
 
75
76
  eventBatches: defineTable({
76
77
  siteId: v.id("sites"),
77
78
  sessionId: v.string(),
78
79
  visitorId: v.string(),
79
80
  bucketStart: v.number(),
80
- events: v.array(trackerEventValidator),
81
+ events: v.array(batchedEventValidator),
82
+ migrationVersion: v.optional(v.literal(1)),
81
83
  })
82
84
  .index("by_siteId_and_sessionId_and_bucketStart", [
83
85
  "siteId",
@@ -85,10 +87,11 @@ export default defineSchema({
85
87
  "bucketStart",
86
88
  ])
87
89
  .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"])
88
- .index("by_siteId_and_visitorId_and_bucketStart", {
89
- fields: ["siteId", "visitorId", "bucketStart"],
90
- staged: true,
91
- }),
90
+ .index("by_siteId_and_visitorId_and_bucketStart", [
91
+ "siteId",
92
+ "visitorId",
93
+ "bucketStart",
94
+ ]),
92
95
 
93
96
  aggregateBuckets: defineTable(aggregateFieldsValidator)
94
97
  .index("by_siteId_and_granularity_and_bucketStart_and_shard", [
@@ -103,6 +106,163 @@ export default defineSchema({
103
106
  "bucketStart",
104
107
  ]),
105
108
 
109
+ routeDailyRollups: defineTable({
110
+ siteId: v.id("sites"),
111
+ bucketStart: v.number(),
112
+ shard: v.number(),
113
+ generation: v.literal(1),
114
+ route: v.string(),
115
+ pageviews: v.number(),
116
+ entries: v.number(),
117
+ continued: v.number(),
118
+ dwellMs: v.number(),
119
+ dwellSamples: v.number(),
120
+ visitorSketch: visitorSketchValidator,
121
+ outgoing: v.array(v.object({ route: v.string(), count: v.number() })),
122
+ })
123
+ .index("by_siteId_and_route_and_bucketStart_and_shard", [
124
+ "siteId",
125
+ "route",
126
+ "bucketStart",
127
+ "shard",
128
+ ])
129
+ .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
130
+
131
+ goalDailyRollups: defineTable({
132
+ siteId: v.id("sites"),
133
+ goalId: v.id("goals"),
134
+ bucketStart: v.number(),
135
+ shard: v.number(),
136
+ generation: v.literal(1),
137
+ goalKey: v.string(),
138
+ completions: v.number(),
139
+ valueCents: v.number(),
140
+ })
141
+ .index("by_siteId_and_goalId_and_bucketStart_and_shard", [
142
+ "siteId",
143
+ "goalId",
144
+ "bucketStart",
145
+ "shard",
146
+ ])
147
+ .index("by_siteId_and_goalId_and_bucketStart", [
148
+ "siteId",
149
+ "goalId",
150
+ "bucketStart",
151
+ ])
152
+ .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
153
+
154
+ funnelDailyRollups: defineTable({
155
+ siteId: v.id("sites"),
156
+ funnelId: v.id("funnels"),
157
+ bucketStart: v.number(),
158
+ shard: v.number(),
159
+ generation: v.literal(1),
160
+ steps: v.array(
161
+ v.object({
162
+ step: v.number(),
163
+ entrants: v.number(),
164
+ completions: v.number(),
165
+ }),
166
+ ),
167
+ })
168
+ .index("by_siteId_and_funnelId_and_bucketStart_and_shard", [
169
+ "siteId",
170
+ "funnelId",
171
+ "bucketStart",
172
+ "shard",
173
+ ])
174
+ .index("by_siteId_and_funnelId_and_bucketStart", [
175
+ "siteId",
176
+ "funnelId",
177
+ "bucketStart",
178
+ ])
179
+ .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
180
+
181
+ // Field Web Vitals. One row per site/day/shard and either a page (device
182
+ // "(all)") or a device class (page "(all)"); the row folds every metric's
183
+ // histogram together so a report reads five metrics for one document. Pages
184
+ // are bounded like dimension slots: at most VITAL_PAGE_SLOTS distinct pages
185
+ // per day, then "(other)".
186
+ vitalRollups: defineTable({
187
+ siteId: v.id("sites"),
188
+ bucketStart: v.number(),
189
+ device: v.string(),
190
+ page: v.string(),
191
+ shard: v.number(),
192
+ metrics: v.array(
193
+ v.object({
194
+ metric: vitalMetricValidator,
195
+ count: v.number(),
196
+ sum: v.number(),
197
+ histogram: v.array(v.number()),
198
+ }),
199
+ ),
200
+ })
201
+ .index("by_key", ["siteId", "bucketStart", "device", "page", "shard"])
202
+ .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
203
+
204
+ affiliateDailyRollups: defineTable({
205
+ siteId: v.id("sites"),
206
+ affiliateId: v.id("affiliates"),
207
+ bucketStart: v.number(),
208
+ shard: v.number(),
209
+ generation: v.literal(1),
210
+ affiliateSlug: v.string(),
211
+ visits: v.number(),
212
+ visitorSketch: visitorSketchValidator,
213
+ conversions: v.number(),
214
+ revenueCents: v.number(),
215
+ commissionCents: v.number(),
216
+ })
217
+ .index("by_siteId_and_affiliateId_and_bucketStart_and_shard", [
218
+ "siteId",
219
+ "affiliateId",
220
+ "bucketStart",
221
+ "shard",
222
+ ])
223
+ .index("by_siteId_and_affiliateId_and_bucketStart", [
224
+ "siteId",
225
+ "affiliateId",
226
+ "bucketStart",
227
+ ])
228
+ .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
229
+
230
+ analyticsCoverage: defineTable({
231
+ siteId: v.id("sites"),
232
+ generation: v.literal(1),
233
+ rollupCompleteFrom: v.number(),
234
+ siteMapSince: v.optional(v.number()),
235
+ retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
236
+ updatedAt: v.number(),
237
+ }).index("by_siteId", ["siteId"]),
238
+
239
+ retentionPolicies: defineTable({
240
+ siteId: v.id("sites"),
241
+ eventsDays: v.number(),
242
+ sessionsDays: v.number(),
243
+ conversionsDays: v.number(),
244
+ aggregatesDays: v.number(),
245
+ reportsDays: v.number(),
246
+ updatedAt: v.number(),
247
+ }).index("by_siteId", ["siteId"]),
248
+
249
+ retentionStates: defineTable({
250
+ siteId: v.id("sites"),
251
+ kind: v.string(),
252
+ before: v.number(),
253
+ status: v.union(
254
+ v.literal("pending"),
255
+ v.literal("running"),
256
+ v.literal("complete"),
257
+ ),
258
+ generation: v.number(),
259
+ attempt: v.number(),
260
+ leaseExpiresAt: v.optional(v.number()),
261
+ scheduledAt: v.optional(v.number()),
262
+ deleted: v.number(),
263
+ updatedAt: v.number(),
264
+ }).index("by_siteId_and_kind", ["siteId", "kind"]),
265
+
106
266
  goals: defineTable(goalFieldsValidator)
107
267
  .index("by_siteId", ["siteId"])
108
268
  .index("by_siteId_and_key", ["siteId", "key"]),
@@ -119,7 +279,11 @@ export default defineSchema({
119
279
  valueCents: v.number(),
120
280
  })
121
281
  .index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
122
- .index("by_siteId_and_goalId_and_timestamp", ["siteId", "goalId", "timestamp"])
282
+ .index("by_siteId_and_goalId_and_timestamp", [
283
+ "siteId",
284
+ "goalId",
285
+ "timestamp",
286
+ ])
123
287
  .index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
124
288
 
125
289
  funnels: defineTable(funnelFieldsValidator)
@@ -171,6 +335,21 @@ export default defineSchema({
171
335
  .index("by_siteId", ["siteId"])
172
336
  .index("by_siteId_and_slug", ["siteId", "slug"]),
173
337
 
338
+ visitorSecrets: defineTable({
339
+ key: v.literal("current"),
340
+ value: v.string(),
341
+ createdAt: v.number(),
342
+ }).index("by_key", ["key"]),
343
+
344
+ visitorAliases: defineTable({
345
+ siteId: v.id("sites"),
346
+ visitorId: v.string(),
347
+ previousVisitorId: v.string(),
348
+ linkedAt: v.number(),
349
+ })
350
+ .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
351
+ .index("by_siteId_and_previousVisitorId", ["siteId", "previousVisitorId"]),
352
+
174
353
  visitorAttributions: defineTable({
175
354
  siteId: v.id("sites"),
176
355
  visitorId: v.string(),
@@ -1,6 +1,7 @@
1
1
  import { v } from "convex/values";
2
2
  import { internalQuery, mutation, query } from "./_generated/server.js";
3
3
  import { MAX_SITES_PER_OWNER } from "./constants.js";
4
+ import { ensureAnalyticsControl } from "./coverage.js";
4
5
  import { fail } from "./errors.js";
5
6
  import {
6
7
  cleanString,
@@ -31,7 +32,8 @@ export const create = mutation({
31
32
  handler: async (ctx, args) => {
32
33
  const ownerId = cleanString(args.ownerId, 128);
33
34
  const name = cleanString(args.name, 120);
34
- if (!ownerId || !name) fail("INVALID_SITE", "ownerId and name are required");
35
+ if (!ownerId || !name)
36
+ fail("INVALID_SITE", "ownerId and name are required");
35
37
 
36
38
  let domains: string[];
37
39
  let timezone: string | undefined;
@@ -45,7 +47,10 @@ export const create = mutation({
45
47
  : undefined;
46
48
  currency = sanitizeCurrency(args.currency ?? "USD");
47
49
  } catch (error) {
48
- fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
50
+ fail(
51
+ "INVALID_SITE",
52
+ error instanceof Error ? error.message : "invalid site",
53
+ );
49
54
  }
50
55
  const cookieless = args.cookieless ?? true;
51
56
 
@@ -64,8 +69,12 @@ export const create = mutation({
64
69
  (existing.currency ?? "USD") !== currency ||
65
70
  existing.cookieless !== cookieless
66
71
  ) {
67
- fail("CONFLICT", "site name is already associated with different settings");
72
+ fail(
73
+ "CONFLICT",
74
+ "site name is already associated with different settings",
75
+ );
68
76
  }
77
+ await ensureAnalyticsControl(ctx, existing._id, Date.now());
69
78
  return existing._id;
70
79
  }
71
80
 
@@ -80,7 +89,7 @@ export const create = mutation({
80
89
  }
81
90
 
82
91
  const now = Date.now();
83
- return await ctx.db.insert("sites", {
92
+ const siteId = await ctx.db.insert("sites", {
84
93
  ownerId,
85
94
  name,
86
95
  domains,
@@ -91,6 +100,8 @@ export const create = mutation({
91
100
  createdAt: now,
92
101
  updatedAt: now,
93
102
  });
103
+ await ensureAnalyticsControl(ctx, siteId, now);
104
+ return siteId;
94
105
  },
95
106
  });
96
107
 
@@ -133,22 +144,26 @@ export const update = mutation({
133
144
  patch.name = name;
134
145
  }
135
146
  try {
136
- if (args.domains !== undefined) patch.domains = normalizeDomains(args.domains);
147
+ if (args.domains !== undefined)
148
+ patch.domains = normalizeDomains(args.domains);
137
149
  if (args.timezone !== undefined) {
138
- patch.timezone = args.timezone === null
139
- ? undefined
140
- : validateTimezone(args.timezone);
150
+ patch.timezone =
151
+ args.timezone === null ? undefined : validateTimezone(args.timezone);
141
152
  }
142
153
  if (args.networkId !== undefined) {
143
- patch.networkId = args.networkId === null
144
- ? undefined
145
- : sanitizeOpaqueId(args.networkId, "networkId");
154
+ patch.networkId =
155
+ args.networkId === null
156
+ ? undefined
157
+ : sanitizeOpaqueId(args.networkId, "networkId");
146
158
  }
147
159
  if (args.currency !== undefined) {
148
160
  patch.currency = sanitizeCurrency(args.currency);
149
161
  }
150
162
  } catch (error) {
151
- fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
163
+ fail(
164
+ "INVALID_SITE",
165
+ error instanceof Error ? error.message : "invalid site",
166
+ );
152
167
  }
153
168
  if (
154
169
  patch.currency !== undefined &&