@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +79 -30
  2. package/dist/component/_generated/api.d.ts +5 -5
  3. package/dist/component/_generated/api.d.ts.map +1 -1
  4. package/dist/component/_generated/api.js.map +1 -1
  5. package/dist/component/_generated/component.d.ts +10 -0
  6. package/dist/component/_generated/component.d.ts.map +1 -1
  7. package/dist/component/_generated/server.d.ts +4 -0
  8. package/dist/component/_generated/server.d.ts.map +1 -1
  9. package/dist/component/_generated/server.js.map +1 -1
  10. package/dist/component/convex.config.d.ts +7 -2
  11. package/dist/component/convex.config.d.ts.map +1 -1
  12. package/dist/component/convex.config.js +9 -4
  13. package/dist/component/convex.config.js.map +1 -1
  14. package/dist/component/diagnostics.d.ts +9 -0
  15. package/dist/component/diagnostics.d.ts.map +1 -0
  16. package/dist/component/diagnostics.js +47 -0
  17. package/dist/component/diagnostics.js.map +1 -0
  18. package/dist/component/eventStore.d.ts +11 -9
  19. package/dist/component/eventStore.d.ts.map +1 -1
  20. package/dist/component/eventStore.js +55 -227
  21. package/dist/component/eventStore.js.map +1 -1
  22. package/dist/component/geo.d.ts +71 -0
  23. package/dist/component/geo.d.ts.map +1 -0
  24. package/dist/component/geo.js +610 -0
  25. package/dist/component/geo.js.map +1 -0
  26. package/dist/component/http.d.ts.map +1 -1
  27. package/dist/component/http.js +124 -11
  28. package/dist/component/http.js.map +1 -1
  29. package/dist/component/ingest.d.ts.map +1 -1
  30. package/dist/component/ingest.js +34 -14
  31. package/dist/component/ingest.js.map +1 -1
  32. package/dist/component/reports.d.ts +8 -8
  33. package/dist/component/reports.d.ts.map +1 -1
  34. package/dist/component/reports.js +2 -1
  35. package/dist/component/reports.js.map +1 -1
  36. package/dist/component/schema.d.ts +15 -52
  37. package/dist/component/schema.js +14 -16
  38. package/dist/component/schema.js.map +1 -1
  39. package/dist/component/validators.d.ts +3 -50
  40. package/dist/component/validators.d.ts.map +1 -1
  41. package/dist/component/validators.js +4 -12
  42. package/dist/component/validators.js.map +1 -1
  43. package/dist/tracker/generated.d.ts +4 -4
  44. package/dist/tracker/generated.d.ts.map +1 -1
  45. package/dist/tracker/generated.js +4 -4
  46. package/dist/tracker/generated.js.map +1 -1
  47. package/dist/tracker/tracker.js +7 -6
  48. package/dist/tracker/tracker.js.map +1 -1
  49. package/dist/tracker.min.js +1 -1
  50. package/docs/benchmarks/2026-08-20-realistic.md +1 -1
  51. package/package.json +2 -6
  52. package/scripts/benchmark-ingest.mjs +81 -32
  53. package/src/component/_generated/api.ts +5 -5
  54. package/src/component/_generated/component.ts +13 -0
  55. package/src/component/_generated/server.ts +4 -0
  56. package/src/component/convex.config.ts +11 -5
  57. package/src/component/diagnostics.ts +64 -0
  58. package/src/component/eventStore.ts +78 -280
  59. package/src/component/geo.ts +779 -0
  60. package/src/component/http.ts +184 -17
  61. package/src/component/ingest.ts +49 -15
  62. package/src/component/reports.ts +4 -1
  63. package/src/component/schema.ts +15 -16
  64. package/src/component/validators.ts +4 -13
  65. package/src/test.ts +0 -2
  66. package/dist/component/migrations.d.ts +0 -17
  67. package/dist/component/migrations.d.ts.map +0 -1
  68. package/dist/component/migrations.js +0 -44
  69. package/dist/component/migrations.js.map +0 -1
  70. package/src/component/migrations.ts +0 -52
@@ -1,19 +1,32 @@
1
1
  import { httpRouter } from "convex/server";
2
2
  import { ConvexError } from "convex/values";
3
3
  import type { Id } from "./_generated/dataModel.js";
4
- import { api } from "./_generated/api.js";
5
- import { httpAction } from "./_generated/server.js";
6
- import { MAX_BATCH_BYTES } from "./constants.js";
4
+ import { api, internal } from "./_generated/api.js";
5
+ import { env, httpAction } from "./_generated/server.js";
6
+ import {
7
+ MAX_BATCH_BYTES,
8
+ MAX_BATCH_EVENT_GROUPS,
9
+ MAX_BATCH_EVENTS,
10
+ } from "./constants.js";
11
+ import {
12
+ classifyIp,
13
+ geoIpDailyLimit,
14
+ geoIpStatus,
15
+ isTrustedProxyRequest,
16
+ readRequestMetadata,
17
+ resolveClientIp,
18
+ resolveGeoIp,
19
+ } from "./geo.js";
7
20
  import { isPlainRecord } from "./guards.js";
21
+ import { sanitizeEvent } from "./sanitize.js";
8
22
  import type { TrackerEvent } from "./validators.js";
9
23
  import {
10
24
  TRACKER_GZIP_BASE64,
11
25
  TRACKER_GZIP_BYTES,
12
26
  } from "../tracker/generated.js";
13
27
 
14
- const TRACKER_GZIP = Uint8Array.from(
15
- atob(TRACKER_GZIP_BASE64),
16
- (character) => character.charCodeAt(0),
28
+ const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
29
+ character.charCodeAt(0),
17
30
  );
18
31
 
19
32
  const http = httpRouter();
@@ -51,7 +64,8 @@ http.route({
51
64
  {
52
65
  error: {
53
66
  code: "INVALID_BODY",
54
- message: error instanceof Error ? error.message : "invalid JSON body",
67
+ message:
68
+ error instanceof Error ? error.message : "invalid JSON body",
55
69
  },
56
70
  },
57
71
  400,
@@ -59,12 +73,15 @@ http.route({
59
73
  );
60
74
  }
61
75
 
62
- const userAgent = request.headers.get("user-agent") ?? "";
76
+ const trustProxy = env.RASTRO_TRUST_PROXY === "true";
77
+ const metadata = await readRequestMetadata(ctx);
78
+ const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
79
+ const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
80
+ const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
81
+ const userAgent =
82
+ metadata.userAgent ?? request.headers.get("user-agent") ?? "";
63
83
  const context = {
64
- country: firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]),
65
- city: decodeHeader(firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"])),
66
- latitude: numericHeader(request, ["x-vercel-ip-latitude", "cf-iplatitude"]),
67
- longitude: numericHeader(request, ["x-vercel-ip-longitude", "cf-iplongitude"]),
84
+ ...edgeGeo,
68
85
  ...classifyUserAgent(userAgent),
69
86
  };
70
87
 
@@ -75,6 +92,43 @@ http.route({
75
92
  events: payload.events,
76
93
  context,
77
94
  });
95
+ if (
96
+ clientIp.ip &&
97
+ !hasCoordinates(edgeGeo) &&
98
+ geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
99
+ "ready"
100
+ ) {
101
+ const events = enrichmentCandidateEvents(payload.events);
102
+ if (events) {
103
+ try {
104
+ const claim = await ctx.runMutation(
105
+ internal.geo.claimEnrichment,
106
+ {
107
+ siteId: payload.siteId as Id<"sites">,
108
+ events,
109
+ dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
110
+ },
111
+ );
112
+ if (claim.sessionIds.length > 0) {
113
+ const providerGeo = await resolveGeoIp(
114
+ clientIp.ip,
115
+ env.RASTRO_GEOIP_PROVIDER!,
116
+ env.RASTRO_GEOIP_TOKEN,
117
+ );
118
+ if (providerGeo) {
119
+ await ctx.runMutation(internal.geo.applyEnrichment, {
120
+ siteId: payload.siteId as Id<"sites">,
121
+ sessionIds: claim.sessionIds,
122
+ events: claim.events,
123
+ context: providerGeo,
124
+ });
125
+ }
126
+ }
127
+ } catch {
128
+ // Analytics ingestion remains available if optional enrichment fails.
129
+ }
130
+ }
131
+ }
78
132
  return json(result, 200, headers);
79
133
  } catch (error) {
80
134
  if (error instanceof ConvexError) {
@@ -127,13 +181,35 @@ http.route({
127
181
  }),
128
182
  });
129
183
 
130
-
131
184
  http.route({
132
185
  path: "/health",
133
186
  method: "GET",
134
- handler: httpAction(async () => {
187
+ handler: httpAction(async (ctx, request) => {
188
+ const trustProxy = env.RASTRO_TRUST_PROXY === "true";
189
+ const metadata = await readRequestMetadata(ctx);
190
+ const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
191
+ const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
192
+ const provider = env.RASTRO_GEOIP_PROVIDER;
135
193
  return json(
136
- { ok: true, component: "@iann29/rastro" },
194
+ {
195
+ ok: true,
196
+ component: "@iann29/rastro",
197
+ requestMetadata: {
198
+ runtimeIpKind: classifyIp(metadata.ip),
199
+ clientIpAvailable: clientIp.ip !== null,
200
+ clientIpSource: clientIp.source,
201
+ userAgentAvailable: metadata.userAgent !== null,
202
+ },
203
+ geoIp: {
204
+ provider: provider ?? null,
205
+ status: geoIpStatus(provider, env.RASTRO_GEOIP_TOKEN),
206
+ dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
207
+ trustedProxy: trustProxy,
208
+ proxyRequestTrusted,
209
+ edgeHeadersAvailable:
210
+ proxyRequestTrusted && Object.keys(geoFromHeaders(request)).length > 0,
211
+ },
212
+ },
137
213
  200,
138
214
  {
139
215
  "Access-Control-Allow-Origin": "*",
@@ -143,8 +219,15 @@ http.route({
143
219
  }),
144
220
  });
145
221
 
146
- function parsePayload(value: unknown): { siteId: string; events: TrackerEvent[] } {
147
- if (!isPlainRecord(value) || typeof value.siteId !== "string" || !Array.isArray(value.events)) {
222
+ function parsePayload(value: unknown): {
223
+ siteId: string;
224
+ events: TrackerEvent[];
225
+ } {
226
+ if (
227
+ !isPlainRecord(value) ||
228
+ typeof value.siteId !== "string" ||
229
+ !Array.isArray(value.events)
230
+ ) {
148
231
  throw new Error("body must contain siteId and events");
149
232
  }
150
233
  return { siteId: value.siteId, events: value.events.map(parseEvent) };
@@ -266,6 +349,90 @@ function optionalNumber(value: unknown, field: string): number | undefined {
266
349
  return value;
267
350
  }
268
351
 
352
+ function enrichmentCandidateEvents(events: TrackerEvent[]):
353
+ | Array<{
354
+ eventId: string;
355
+ sessionId: string;
356
+ visitorId: string;
357
+ timestamp: number;
358
+ sequence: number;
359
+ }>
360
+ | undefined {
361
+ if (events.length === 0 || events.length > MAX_BATCH_EVENTS) return undefined;
362
+ const now = Date.now();
363
+ const sanitized = events.flatMap((event) => {
364
+ try {
365
+ return [sanitizeEvent(event, now)];
366
+ } catch {
367
+ return [];
368
+ }
369
+ });
370
+ const sessionIds = [...new Set(sanitized.map((event) => event.sessionId))];
371
+ if (
372
+ !sanitized.some((event) => event.sequence === 0) ||
373
+ sessionIds.length > MAX_BATCH_EVENT_GROUPS ||
374
+ sessionIds.some(
375
+ (sessionId) => sessionId.length === 0 || sessionId.length > 128,
376
+ )
377
+ ) {
378
+ return undefined;
379
+ }
380
+ return sanitized.map(
381
+ ({ eventId, sessionId, visitorId, timestamp, sequence }) => ({
382
+ eventId,
383
+ sessionId,
384
+ visitorId,
385
+ timestamp,
386
+ sequence,
387
+ }),
388
+ );
389
+ }
390
+
391
+ function geoFromHeaders(request: Request): {
392
+ country?: string;
393
+ city?: string;
394
+ latitude?: number;
395
+ longitude?: number;
396
+ } {
397
+ const country = firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]);
398
+ const city = decodeHeader(
399
+ firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"]),
400
+ );
401
+ const latitude = numericHeader(request, [
402
+ "x-vercel-ip-latitude",
403
+ "cf-iplatitude",
404
+ ]);
405
+ const longitude = numericHeader(request, [
406
+ "x-vercel-ip-longitude",
407
+ "cf-iplongitude",
408
+ ]);
409
+ return {
410
+ ...(country ? { country } : {}),
411
+ ...(city ? { city } : {}),
412
+ ...(validCoordinates(latitude, longitude)
413
+ ? { latitude, longitude }
414
+ : {}),
415
+ };
416
+ }
417
+
418
+ function hasCoordinates(value: {
419
+ latitude?: number;
420
+ longitude?: number;
421
+ }): boolean {
422
+ return value.latitude !== undefined && value.longitude !== undefined;
423
+ }
424
+
425
+ function validCoordinates(
426
+ latitude: number | undefined,
427
+ longitude: number | undefined,
428
+ ): latitude is number {
429
+ return latitude !== undefined &&
430
+ longitude !== undefined &&
431
+ latitude >= -90 &&
432
+ latitude <= 90 &&
433
+ longitude >= -180 &&
434
+ longitude <= 180;
435
+ }
269
436
 
270
437
  function firstHeader(request: Request, names: string[]): string | undefined {
271
438
  for (const name of names) {
@@ -99,6 +99,10 @@ type SessionState = {
99
99
  isNewVisitorDay: boolean;
100
100
  startedAt: number;
101
101
  source: string;
102
+ country?: string;
103
+ city?: string;
104
+ latitude?: number;
105
+ longitude?: number;
102
106
  pageviewCount: number;
103
107
  eventCount: number;
104
108
  affiliateId?: Id<"affiliates">;
@@ -236,7 +240,6 @@ export const ingestBatch = mutation({
236
240
  siteId: args.siteId,
237
241
  sessionId: group.sessionId,
238
242
  bucketStart: group.bucketStart,
239
- candidateEventIds: group.eventIds,
240
243
  });
241
244
  const newCount = [...group.eventIds].filter((eventId) =>
242
245
  !eventIds.has(eventId)
@@ -356,7 +359,7 @@ export const ingestBatch = mutation({
356
359
  .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
357
360
  .take(MAX_FUNNELS_PER_SITE);
358
361
  }
359
- await accountEvent(ctx, {
362
+ const session = await accountEvent(ctx, {
360
363
  siteId: args.siteId,
361
364
  event,
362
365
  context,
@@ -370,7 +373,12 @@ export const ingestBatch = mutation({
370
373
  now,
371
374
  batchState,
372
375
  });
373
- bufferEventBatch(args.siteId, event, batchState.eventBatches);
376
+ bufferEventBatch(
377
+ args.siteId,
378
+ event,
379
+ batchState.eventBatches,
380
+ session.country ?? "unknown",
381
+ );
374
382
  accepted += 1;
375
383
  }
376
384
  await flushBatchState(ctx, batchState);
@@ -467,7 +475,7 @@ export const trackConversion = mutation({
467
475
  };
468
476
 
469
477
  const aggregateDeltas: AggregateDeltas = new Map();
470
- await accountEvent(ctx, {
478
+ const session = await accountEvent(ctx, {
471
479
  siteId: args.siteId,
472
480
  event,
473
481
  context: conversionContext,
@@ -481,7 +489,12 @@ export const trackConversion = mutation({
481
489
  now: Date.now(),
482
490
  });
483
491
  if (!existingTelemetry && !telemetry.saturated) {
484
- await insertEventBatch(ctx, args.siteId, event);
492
+ await insertEventBatch(
493
+ ctx,
494
+ args.siteId,
495
+ event,
496
+ session.country ?? "unknown",
497
+ );
485
498
  }
486
499
  await flushAggregates(ctx, aggregateDeltas);
487
500
  return { accepted: 1, duplicates: 0, rejected: 0 };
@@ -612,7 +625,7 @@ async function accountEvent(
612
625
  now: number;
613
626
  batchState?: BatchState;
614
627
  },
615
- ) {
628
+ ): Promise<SessionState> {
616
629
  const { event } = input;
617
630
  const session = input.recordTelemetry
618
631
  ? await upsertSession(
@@ -700,7 +713,7 @@ async function accountEvent(
700
713
  const dimensions: Dimension[] = [
701
714
  { type: "source", value: session.source },
702
715
  { type: "page", value: event.path },
703
- { type: "country", value: input.context.country ?? "unknown" },
716
+ { type: "country", value: session.country ?? "unknown" },
704
717
  { type: "device", value: input.context.device ?? "unknown" },
705
718
  { type: "browser", value: input.context.browser ?? "Other" },
706
719
  {
@@ -733,6 +746,7 @@ async function accountEvent(
733
746
  input.batchState,
734
747
  );
735
748
  }
749
+ return session;
736
750
  }
737
751
 
738
752
  async function accountTrustedSession(
@@ -755,6 +769,10 @@ async function accountTrustedSession(
755
769
  isNewVisitorDay: false,
756
770
  startedAt: existing.startedAt,
757
771
  source: existing.source,
772
+ country: existing.country,
773
+ city: existing.city,
774
+ latitude: existing.latitude,
775
+ longitude: existing.longitude,
758
776
  pageviewCount: existing.pageviewCount,
759
777
  eventCount: existing.eventCount,
760
778
  affiliateId: existing.affiliateId,
@@ -832,6 +850,10 @@ async function upsertSession(
832
850
  Math.floor(event.timestamp / DAY_MS),
833
851
  startedAt,
834
852
  source: existing.source,
853
+ country: context.country ?? existing.country,
854
+ city: context.city ?? existing.city,
855
+ latitude: context.latitude ?? existing.latitude,
856
+ longitude: context.longitude ?? existing.longitude,
835
857
  pageviewCount:
836
858
  existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
837
859
  eventCount: existing.eventCount + 1,
@@ -910,6 +932,10 @@ async function upsertSession(
910
932
  Math.floor(event.timestamp / DAY_MS),
911
933
  startedAt: event.timestamp,
912
934
  source,
935
+ country: context.country,
936
+ city: context.city,
937
+ latitude: context.latitude,
938
+ longitude: context.longitude,
913
939
  pageviewCount: event.type === "pageview" ? 1 : 0,
914
940
  eventCount: 1,
915
941
  ...(attribution ?? {}),
@@ -1102,6 +1128,10 @@ async function accountHeartbeat(
1102
1128
  isNewVisitorDay: false,
1103
1129
  startedAt,
1104
1130
  source: session.source,
1131
+ country: context.country ?? session.country,
1132
+ city: context.city ?? session.city,
1133
+ latitude: context.latitude ?? session.latitude,
1134
+ longitude: context.longitude ?? session.longitude,
1105
1135
  pageviewCount: session.pageviewCount,
1106
1136
  eventCount: session.eventCount,
1107
1137
  affiliateSlug: session.affiliateSlug,
@@ -1289,6 +1319,10 @@ async function updateLiveSession(
1289
1319
  )
1290
1320
  .unique();
1291
1321
  const deadline = now + LIVE_SESSION_TTL_MS;
1322
+ const country = context.country ?? session.country;
1323
+ const city = context.city ?? session.city;
1324
+ const latitude = context.latitude ?? session.latitude;
1325
+ const longitude = context.longitude ?? session.longitude;
1292
1326
  if (existing) {
1293
1327
  const recentHeartbeatEventIds = event.type === "heartbeat"
1294
1328
  ? [
@@ -1304,10 +1338,10 @@ async function updateLiveSession(
1304
1338
  path: event.path,
1305
1339
  lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
1306
1340
  expiresAt: deadline,
1307
- ...(context.country ? { country: context.country } : {}),
1308
- ...(context.city ? { city: context.city } : {}),
1309
- ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
1310
- ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
1341
+ ...(country ? { country } : {}),
1342
+ ...(city ? { city } : {}),
1343
+ ...(latitude !== undefined ? { latitude } : {}),
1344
+ ...(longitude !== undefined ? { longitude } : {}),
1311
1345
  browser: context.browser ?? existing.browser,
1312
1346
  os: context.os ?? existing.os,
1313
1347
  source: session.source,
@@ -1338,10 +1372,10 @@ async function updateLiveSession(
1338
1372
  startedAt: event.timestamp,
1339
1373
  lastSeenAt: event.timestamp,
1340
1374
  expiresAt: deadline,
1341
- ...(context.country ? { country: context.country } : {}),
1342
- ...(context.city ? { city: context.city } : {}),
1343
- ...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
1344
- ...(context.longitude !== undefined ? { longitude: context.longitude } : {}),
1375
+ ...(country ? { country } : {}),
1376
+ ...(city ? { city } : {}),
1377
+ ...(latitude !== undefined ? { latitude } : {}),
1378
+ ...(longitude !== undefined ? { longitude } : {}),
1345
1379
  browser: context.browser ?? "Other",
1346
1380
  os: context.os ?? "Other",
1347
1381
  source: session.source,
@@ -331,7 +331,10 @@ export const listSessions = query({
331
331
  .take(take)),
332
332
  );
333
333
  }
334
- return keysetPaginationResult(rows, pagination, (row) => ({
334
+ const publicRows = rows.map(
335
+ ({ geoLookupAttemptedAt: _attemptedAt, ...row }) => row,
336
+ );
337
+ return keysetPaginationResult(publicRows, pagination, (row) => ({
335
338
  timestamp: row.lastSeenAt,
336
339
  creationTime: row._creationTime,
337
340
  }));
@@ -4,7 +4,6 @@ import {
4
4
  affiliateFieldsValidator,
5
5
  aggregateFieldsValidator,
6
6
  batchedEventValidator,
7
- storedEventFieldsValidator,
8
7
  funnelFieldsValidator,
9
8
  goalFieldsValidator,
10
9
  sessionFieldsValidator,
@@ -17,7 +16,11 @@ export default defineSchema({
17
16
  .index("by_ownerId_and_name", ["ownerId", "name"])
18
17
  .index("by_networkId", ["networkId"]),
19
18
 
20
- sessions: defineTable(sessionFieldsValidator)
19
+ sessions: defineTable(
20
+ sessionFieldsValidator.extend({
21
+ geoLookupAttemptedAt: v.optional(v.number()),
22
+ }),
23
+ )
21
24
  .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
22
25
  .index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
23
26
  .index("by_siteId_and_startedAt", ["siteId", "startedAt"])
@@ -59,19 +62,11 @@ export default defineSchema({
59
62
  scheduledAt: v.number(),
60
63
  }).index("by_siteId", ["siteId"]),
61
64
 
62
- events: defineTable(storedEventFieldsValidator)
63
- .index("by_siteId_and_eventId", ["siteId", "eventId"])
64
- .index("by_siteId_and_timestamp", ["siteId", "timestamp"])
65
- .index("by_siteId_and_sessionId_and_timestamp", [
66
- "siteId",
67
- "sessionId",
68
- "timestamp",
69
- ])
70
- .index("by_siteId_and_visitorId_and_timestamp", [
71
- "siteId",
72
- "visitorId",
73
- "timestamp",
74
- ]),
65
+ geoLookupBudgets: defineTable({
66
+ siteId: v.id("sites"),
67
+ dayStart: v.number(),
68
+ count: v.number(),
69
+ }).index("by_siteId", ["siteId"]),
75
70
 
76
71
  eventBatches: defineTable({
77
72
  siteId: v.id("sites"),
@@ -122,7 +117,11 @@ export default defineSchema({
122
117
  valueCents: v.number(),
123
118
  })
124
119
  .index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
125
- .index("by_siteId_and_goalId_and_timestamp", ["siteId", "goalId", "timestamp"])
120
+ .index("by_siteId_and_goalId_and_timestamp", [
121
+ "siteId",
122
+ "goalId",
123
+ "timestamp",
124
+ ])
126
125
  .index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
127
126
 
128
127
  funnels: defineTable(funnelFieldsValidator)
@@ -50,7 +50,10 @@ export const ingestContextValidator = v.object({
50
50
  });
51
51
 
52
52
  export const batchedEventValidator = trackerEventValidator.extend(
53
- ingestContextValidator.fields,
53
+ {
54
+ ...ingestContextValidator.fields,
55
+ aggregateCountry: v.optional(v.string()),
56
+ },
54
57
  );
55
58
 
56
59
  export const siteFieldsValidator = v.object({
@@ -123,18 +126,6 @@ export const eventFieldsValidator = v.object({
123
126
  device: v.string(),
124
127
  });
125
128
 
126
- export const storedEventFieldsValidator = eventFieldsValidator
127
- .omit("country", "city", "latitude", "longitude", "browser", "os", "device")
128
- .extend({
129
- country: v.optional(v.string()),
130
- city: v.optional(v.string()),
131
- latitude: v.optional(v.number()),
132
- longitude: v.optional(v.number()),
133
- browser: v.optional(v.string()),
134
- os: v.optional(v.string()),
135
- device: v.optional(v.string()),
136
- });
137
-
138
129
  export const dimensionTypeValidator = v.union(
139
130
  v.literal("source"),
140
131
  v.literal("page"),
package/src/test.ts CHANGED
@@ -1,5 +1,4 @@
1
1
  /// <reference types="vite/client" />
2
- import migrationsComponent from "@convex-dev/migrations/test";
3
2
  import type { TestConvex } from "convex-test";
4
3
  import type { GenericSchema, SchemaDefinition } from "convex/server";
5
4
  import schema from "./component/schema.js";
@@ -16,6 +15,5 @@ export function register(
16
15
  name: string = "rastroAnalytics",
17
16
  ) {
18
17
  t.registerComponent(name, schema, modules);
19
- migrationsComponent.register(t, `${name}/migrations`);
20
18
  }
21
19
  export default { register, schema, modules };
@@ -1,17 +0,0 @@
1
- export declare const migrateLegacyEvents: import("convex/server").RegisteredMutation<"internal", {
2
- fn?: string | undefined;
3
- cursor?: string | null | undefined;
4
- batchSize?: number | undefined;
5
- dryRun?: boolean | undefined;
6
- next?: string[] | undefined;
7
- reset?: boolean | undefined;
8
- oneBatchOnly?: boolean | undefined;
9
- }, Promise<any>>;
10
- export declare const legacyEventsStatus: import("convex/server").RegisteredQuery<"internal", {}, Promise<{
11
- complete: boolean;
12
- sourceEmpty: boolean;
13
- migrationState: "canceled" | "failed" | "inProgress" | "success" | "unknown";
14
- processed: number;
15
- sampleRemainingId: import("convex/values").GenericId<"events"> | null;
16
- }>>;
17
- //# sourceMappingURL=migrations.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,mBAAmB;;;;;;;;gBAI9B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;GAiC7B,CAAC"}
@@ -1,44 +0,0 @@
1
- import { Migrations } from "@convex-dev/migrations";
2
- import { v } from "convex/values";
3
- import { components } from "./_generated/api.js";
4
- import { internalMutation, internalQuery } from "./_generated/server.js";
5
- import { migrateLegacyEventToBatch } from "./eventStore.js";
6
- import schema from "./schema.js";
7
- const migrations = new Migrations(components.migrations, {
8
- internalMutation,
9
- schema,
10
- });
11
- export const migrateLegacyEvents = migrations.define({
12
- table: "events",
13
- batchSize: 16,
14
- migrateOne: migrateLegacyEventToBatch,
15
- });
16
- export const legacyEventsStatus = internalQuery({
17
- args: {},
18
- returns: v.object({
19
- complete: v.boolean(),
20
- sourceEmpty: v.boolean(),
21
- migrationState: v.union(v.literal("inProgress"), v.literal("success"), v.literal("failed"), v.literal("canceled"), v.literal("unknown")),
22
- processed: v.number(),
23
- sampleRemainingId: v.union(v.id("events"), v.null()),
24
- }),
25
- handler: async (ctx) => {
26
- const [remaining, statuses] = await Promise.all([
27
- ctx.db.query("events").first(),
28
- migrations.getStatus(ctx, {
29
- migrations: ["migrations:migrateLegacyEvents"],
30
- }),
31
- ]);
32
- const migrationStatus = statuses[0];
33
- const migrationState = migrationStatus?.state ?? "unknown";
34
- const sourceEmpty = remaining === null;
35
- return {
36
- complete: sourceEmpty && migrationState === "success",
37
- sourceEmpty,
38
- migrationState,
39
- processed: migrationStatus?.processed ?? 0,
40
- sampleRemainingId: remaining?._id ?? null,
41
- };
42
- },
43
- });
44
- //# sourceMappingURL=migrations.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,EAAE;IACvD,gBAAgB;IAChB,MAAM;CACP,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC,MAAM,CAAC;IACnD,KAAK,EAAE,QAAQ;IACf,SAAS,EAAE,EAAE;IACb,UAAU,EAAE,yBAAyB;CACtC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAC9C,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE;QACrB,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE;QACxB,cAAc,EAAE,CAAC,CAAC,KAAK,CACrB,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EACvB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EACnB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EACrB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CACrB;QACD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;KACrD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE;YAC9B,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE;gBACxB,UAAU,EAAE,CAAC,gCAAgC,CAAC;aAC/C,CAAC;SACH,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,eAAe,EAAE,KAAK,IAAI,SAAS,CAAC;QAC3D,MAAM,WAAW,GAAG,SAAS,KAAK,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,WAAW,IAAI,cAAc,KAAK,SAAS;YACrD,WAAW;YACX,cAAc;YACd,SAAS,EAAE,eAAe,EAAE,SAAS,IAAI,CAAC;YAC1C,iBAAiB,EAAE,SAAS,EAAE,GAAG,IAAI,IAAI;SAC1C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
@@ -1,52 +0,0 @@
1
- import { Migrations } from "@convex-dev/migrations";
2
- import { v } from "convex/values";
3
- import { components } from "./_generated/api.js";
4
- import { internalMutation, internalQuery } from "./_generated/server.js";
5
- import { migrateLegacyEventToBatch } from "./eventStore.js";
6
- import schema from "./schema.js";
7
-
8
- const migrations = new Migrations(components.migrations, {
9
- internalMutation,
10
- schema,
11
- });
12
-
13
- export const migrateLegacyEvents = migrations.define({
14
- table: "events",
15
- batchSize: 16,
16
- migrateOne: migrateLegacyEventToBatch,
17
- });
18
-
19
- export const legacyEventsStatus = internalQuery({
20
- args: {},
21
- returns: v.object({
22
- complete: v.boolean(),
23
- sourceEmpty: v.boolean(),
24
- migrationState: v.union(
25
- v.literal("inProgress"),
26
- v.literal("success"),
27
- v.literal("failed"),
28
- v.literal("canceled"),
29
- v.literal("unknown"),
30
- ),
31
- processed: v.number(),
32
- sampleRemainingId: v.union(v.id("events"), v.null()),
33
- }),
34
- handler: async (ctx) => {
35
- const [remaining, statuses] = await Promise.all([
36
- ctx.db.query("events").first(),
37
- migrations.getStatus(ctx, {
38
- migrations: ["migrations:migrateLegacyEvents"],
39
- }),
40
- ]);
41
- const migrationStatus = statuses[0];
42
- const migrationState = migrationStatus?.state ?? "unknown";
43
- const sourceEmpty = remaining === null;
44
- return {
45
- complete: sourceEmpty && migrationState === "success",
46
- sourceEmpty,
47
- migrationState,
48
- processed: migrationStatus?.processed ?? 0,
49
- sampleRemainingId: remaining?._id ?? null,
50
- };
51
- },
52
- });