@iann29/rastro 0.1.0-alpha.2 → 0.1.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -30
- package/dist/component/_generated/api.d.ts +5 -5
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +10 -0
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/server.d.ts +4 -0
- package/dist/component/_generated/server.d.ts.map +1 -1
- package/dist/component/_generated/server.js.map +1 -1
- package/dist/component/convex.config.d.ts +7 -2
- package/dist/component/convex.config.d.ts.map +1 -1
- package/dist/component/convex.config.js +9 -4
- package/dist/component/convex.config.js.map +1 -1
- package/dist/component/diagnostics.d.ts +9 -0
- package/dist/component/diagnostics.d.ts.map +1 -0
- package/dist/component/diagnostics.js +47 -0
- package/dist/component/diagnostics.js.map +1 -0
- package/dist/component/eventStore.d.ts +11 -9
- package/dist/component/eventStore.d.ts.map +1 -1
- package/dist/component/eventStore.js +55 -227
- package/dist/component/eventStore.js.map +1 -1
- package/dist/component/geo.d.ts +71 -0
- package/dist/component/geo.d.ts.map +1 -0
- package/dist/component/geo.js +610 -0
- package/dist/component/geo.js.map +1 -0
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +124 -11
- package/dist/component/http.js.map +1 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +34 -14
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/reports.d.ts +8 -8
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +2 -1
- package/dist/component/reports.js.map +1 -1
- package/dist/component/schema.d.ts +15 -52
- package/dist/component/schema.js +14 -16
- package/dist/component/schema.js.map +1 -1
- package/dist/component/validators.d.ts +3 -50
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +4 -12
- package/dist/component/validators.js.map +1 -1
- package/dist/tracker/generated.d.ts +4 -4
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +4 -4
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.js +7 -6
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker.min.js +1 -1
- package/docs/benchmarks/2026-08-20-realistic.md +1 -1
- package/package.json +2 -6
- package/scripts/benchmark-ingest.mjs +81 -32
- package/src/component/_generated/api.ts +5 -5
- package/src/component/_generated/component.ts +13 -0
- package/src/component/_generated/server.ts +4 -0
- package/src/component/convex.config.ts +11 -5
- package/src/component/diagnostics.ts +64 -0
- package/src/component/eventStore.ts +78 -280
- package/src/component/geo.ts +779 -0
- package/src/component/http.ts +184 -17
- package/src/component/ingest.ts +49 -15
- package/src/component/reports.ts +4 -1
- package/src/component/schema.ts +15 -16
- package/src/component/validators.ts +4 -13
- package/src/test.ts +0 -2
- package/dist/component/migrations.d.ts +0 -17
- package/dist/component/migrations.d.ts.map +0 -1
- package/dist/component/migrations.js +0 -44
- package/dist/component/migrations.js.map +0 -1
- package/src/component/migrations.ts +0 -52
package/src/component/http.ts
CHANGED
|
@@ -1,19 +1,32 @@
|
|
|
1
1
|
import { httpRouter } from "convex/server";
|
|
2
2
|
import { ConvexError } from "convex/values";
|
|
3
3
|
import type { Id } from "./_generated/dataModel.js";
|
|
4
|
-
import { api } from "./_generated/api.js";
|
|
5
|
-
import { httpAction } from "./_generated/server.js";
|
|
6
|
-
import {
|
|
4
|
+
import { api, internal } from "./_generated/api.js";
|
|
5
|
+
import { env, httpAction } from "./_generated/server.js";
|
|
6
|
+
import {
|
|
7
|
+
MAX_BATCH_BYTES,
|
|
8
|
+
MAX_BATCH_EVENT_GROUPS,
|
|
9
|
+
MAX_BATCH_EVENTS,
|
|
10
|
+
} from "./constants.js";
|
|
11
|
+
import {
|
|
12
|
+
classifyIp,
|
|
13
|
+
geoIpDailyLimit,
|
|
14
|
+
geoIpStatus,
|
|
15
|
+
isTrustedProxyRequest,
|
|
16
|
+
readRequestMetadata,
|
|
17
|
+
resolveClientIp,
|
|
18
|
+
resolveGeoIp,
|
|
19
|
+
} from "./geo.js";
|
|
7
20
|
import { isPlainRecord } from "./guards.js";
|
|
21
|
+
import { sanitizeEvent } from "./sanitize.js";
|
|
8
22
|
import type { TrackerEvent } from "./validators.js";
|
|
9
23
|
import {
|
|
10
24
|
TRACKER_GZIP_BASE64,
|
|
11
25
|
TRACKER_GZIP_BYTES,
|
|
12
26
|
} from "../tracker/generated.js";
|
|
13
27
|
|
|
14
|
-
const TRACKER_GZIP = Uint8Array.from(
|
|
15
|
-
|
|
16
|
-
(character) => character.charCodeAt(0),
|
|
28
|
+
const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
|
|
29
|
+
character.charCodeAt(0),
|
|
17
30
|
);
|
|
18
31
|
|
|
19
32
|
const http = httpRouter();
|
|
@@ -51,7 +64,8 @@ http.route({
|
|
|
51
64
|
{
|
|
52
65
|
error: {
|
|
53
66
|
code: "INVALID_BODY",
|
|
54
|
-
message:
|
|
67
|
+
message:
|
|
68
|
+
error instanceof Error ? error.message : "invalid JSON body",
|
|
55
69
|
},
|
|
56
70
|
},
|
|
57
71
|
400,
|
|
@@ -59,12 +73,15 @@ http.route({
|
|
|
59
73
|
);
|
|
60
74
|
}
|
|
61
75
|
|
|
62
|
-
const
|
|
76
|
+
const trustProxy = env.RASTRO_TRUST_PROXY === "true";
|
|
77
|
+
const metadata = await readRequestMetadata(ctx);
|
|
78
|
+
const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
|
|
79
|
+
const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
|
|
80
|
+
const edgeGeo = proxyRequestTrusted ? geoFromHeaders(request) : {};
|
|
81
|
+
const userAgent =
|
|
82
|
+
metadata.userAgent ?? request.headers.get("user-agent") ?? "";
|
|
63
83
|
const context = {
|
|
64
|
-
|
|
65
|
-
city: decodeHeader(firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"])),
|
|
66
|
-
latitude: numericHeader(request, ["x-vercel-ip-latitude", "cf-iplatitude"]),
|
|
67
|
-
longitude: numericHeader(request, ["x-vercel-ip-longitude", "cf-iplongitude"]),
|
|
84
|
+
...edgeGeo,
|
|
68
85
|
...classifyUserAgent(userAgent),
|
|
69
86
|
};
|
|
70
87
|
|
|
@@ -75,6 +92,43 @@ http.route({
|
|
|
75
92
|
events: payload.events,
|
|
76
93
|
context,
|
|
77
94
|
});
|
|
95
|
+
if (
|
|
96
|
+
clientIp.ip &&
|
|
97
|
+
!hasCoordinates(edgeGeo) &&
|
|
98
|
+
geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
|
|
99
|
+
"ready"
|
|
100
|
+
) {
|
|
101
|
+
const events = enrichmentCandidateEvents(payload.events);
|
|
102
|
+
if (events) {
|
|
103
|
+
try {
|
|
104
|
+
const claim = await ctx.runMutation(
|
|
105
|
+
internal.geo.claimEnrichment,
|
|
106
|
+
{
|
|
107
|
+
siteId: payload.siteId as Id<"sites">,
|
|
108
|
+
events,
|
|
109
|
+
dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
|
|
110
|
+
},
|
|
111
|
+
);
|
|
112
|
+
if (claim.sessionIds.length > 0) {
|
|
113
|
+
const providerGeo = await resolveGeoIp(
|
|
114
|
+
clientIp.ip,
|
|
115
|
+
env.RASTRO_GEOIP_PROVIDER!,
|
|
116
|
+
env.RASTRO_GEOIP_TOKEN,
|
|
117
|
+
);
|
|
118
|
+
if (providerGeo) {
|
|
119
|
+
await ctx.runMutation(internal.geo.applyEnrichment, {
|
|
120
|
+
siteId: payload.siteId as Id<"sites">,
|
|
121
|
+
sessionIds: claim.sessionIds,
|
|
122
|
+
events: claim.events,
|
|
123
|
+
context: providerGeo,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
} catch {
|
|
128
|
+
// Analytics ingestion remains available if optional enrichment fails.
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
78
132
|
return json(result, 200, headers);
|
|
79
133
|
} catch (error) {
|
|
80
134
|
if (error instanceof ConvexError) {
|
|
@@ -127,13 +181,35 @@ http.route({
|
|
|
127
181
|
}),
|
|
128
182
|
});
|
|
129
183
|
|
|
130
|
-
|
|
131
184
|
http.route({
|
|
132
185
|
path: "/health",
|
|
133
186
|
method: "GET",
|
|
134
|
-
handler: httpAction(async () => {
|
|
187
|
+
handler: httpAction(async (ctx, request) => {
|
|
188
|
+
const trustProxy = env.RASTRO_TRUST_PROXY === "true";
|
|
189
|
+
const metadata = await readRequestMetadata(ctx);
|
|
190
|
+
const proxyRequestTrusted = isTrustedProxyRequest(metadata.ip, trustProxy);
|
|
191
|
+
const clientIp = resolveClientIp(metadata.ip, request.headers, trustProxy);
|
|
192
|
+
const provider = env.RASTRO_GEOIP_PROVIDER;
|
|
135
193
|
return json(
|
|
136
|
-
{
|
|
194
|
+
{
|
|
195
|
+
ok: true,
|
|
196
|
+
component: "@iann29/rastro",
|
|
197
|
+
requestMetadata: {
|
|
198
|
+
runtimeIpKind: classifyIp(metadata.ip),
|
|
199
|
+
clientIpAvailable: clientIp.ip !== null,
|
|
200
|
+
clientIpSource: clientIp.source,
|
|
201
|
+
userAgentAvailable: metadata.userAgent !== null,
|
|
202
|
+
},
|
|
203
|
+
geoIp: {
|
|
204
|
+
provider: provider ?? null,
|
|
205
|
+
status: geoIpStatus(provider, env.RASTRO_GEOIP_TOKEN),
|
|
206
|
+
dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
|
|
207
|
+
trustedProxy: trustProxy,
|
|
208
|
+
proxyRequestTrusted,
|
|
209
|
+
edgeHeadersAvailable:
|
|
210
|
+
proxyRequestTrusted && Object.keys(geoFromHeaders(request)).length > 0,
|
|
211
|
+
},
|
|
212
|
+
},
|
|
137
213
|
200,
|
|
138
214
|
{
|
|
139
215
|
"Access-Control-Allow-Origin": "*",
|
|
@@ -143,8 +219,15 @@ http.route({
|
|
|
143
219
|
}),
|
|
144
220
|
});
|
|
145
221
|
|
|
146
|
-
function parsePayload(value: unknown): {
|
|
147
|
-
|
|
222
|
+
function parsePayload(value: unknown): {
|
|
223
|
+
siteId: string;
|
|
224
|
+
events: TrackerEvent[];
|
|
225
|
+
} {
|
|
226
|
+
if (
|
|
227
|
+
!isPlainRecord(value) ||
|
|
228
|
+
typeof value.siteId !== "string" ||
|
|
229
|
+
!Array.isArray(value.events)
|
|
230
|
+
) {
|
|
148
231
|
throw new Error("body must contain siteId and events");
|
|
149
232
|
}
|
|
150
233
|
return { siteId: value.siteId, events: value.events.map(parseEvent) };
|
|
@@ -266,6 +349,90 @@ function optionalNumber(value: unknown, field: string): number | undefined {
|
|
|
266
349
|
return value;
|
|
267
350
|
}
|
|
268
351
|
|
|
352
|
+
function enrichmentCandidateEvents(events: TrackerEvent[]):
|
|
353
|
+
| Array<{
|
|
354
|
+
eventId: string;
|
|
355
|
+
sessionId: string;
|
|
356
|
+
visitorId: string;
|
|
357
|
+
timestamp: number;
|
|
358
|
+
sequence: number;
|
|
359
|
+
}>
|
|
360
|
+
| undefined {
|
|
361
|
+
if (events.length === 0 || events.length > MAX_BATCH_EVENTS) return undefined;
|
|
362
|
+
const now = Date.now();
|
|
363
|
+
const sanitized = events.flatMap((event) => {
|
|
364
|
+
try {
|
|
365
|
+
return [sanitizeEvent(event, now)];
|
|
366
|
+
} catch {
|
|
367
|
+
return [];
|
|
368
|
+
}
|
|
369
|
+
});
|
|
370
|
+
const sessionIds = [...new Set(sanitized.map((event) => event.sessionId))];
|
|
371
|
+
if (
|
|
372
|
+
!sanitized.some((event) => event.sequence === 0) ||
|
|
373
|
+
sessionIds.length > MAX_BATCH_EVENT_GROUPS ||
|
|
374
|
+
sessionIds.some(
|
|
375
|
+
(sessionId) => sessionId.length === 0 || sessionId.length > 128,
|
|
376
|
+
)
|
|
377
|
+
) {
|
|
378
|
+
return undefined;
|
|
379
|
+
}
|
|
380
|
+
return sanitized.map(
|
|
381
|
+
({ eventId, sessionId, visitorId, timestamp, sequence }) => ({
|
|
382
|
+
eventId,
|
|
383
|
+
sessionId,
|
|
384
|
+
visitorId,
|
|
385
|
+
timestamp,
|
|
386
|
+
sequence,
|
|
387
|
+
}),
|
|
388
|
+
);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function geoFromHeaders(request: Request): {
|
|
392
|
+
country?: string;
|
|
393
|
+
city?: string;
|
|
394
|
+
latitude?: number;
|
|
395
|
+
longitude?: number;
|
|
396
|
+
} {
|
|
397
|
+
const country = firstHeader(request, ["x-vercel-ip-country", "cf-ipcountry"]);
|
|
398
|
+
const city = decodeHeader(
|
|
399
|
+
firstHeader(request, ["x-vercel-ip-city", "cf-ipcity"]),
|
|
400
|
+
);
|
|
401
|
+
const latitude = numericHeader(request, [
|
|
402
|
+
"x-vercel-ip-latitude",
|
|
403
|
+
"cf-iplatitude",
|
|
404
|
+
]);
|
|
405
|
+
const longitude = numericHeader(request, [
|
|
406
|
+
"x-vercel-ip-longitude",
|
|
407
|
+
"cf-iplongitude",
|
|
408
|
+
]);
|
|
409
|
+
return {
|
|
410
|
+
...(country ? { country } : {}),
|
|
411
|
+
...(city ? { city } : {}),
|
|
412
|
+
...(validCoordinates(latitude, longitude)
|
|
413
|
+
? { latitude, longitude }
|
|
414
|
+
: {}),
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function hasCoordinates(value: {
|
|
419
|
+
latitude?: number;
|
|
420
|
+
longitude?: number;
|
|
421
|
+
}): boolean {
|
|
422
|
+
return value.latitude !== undefined && value.longitude !== undefined;
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function validCoordinates(
|
|
426
|
+
latitude: number | undefined,
|
|
427
|
+
longitude: number | undefined,
|
|
428
|
+
): latitude is number {
|
|
429
|
+
return latitude !== undefined &&
|
|
430
|
+
longitude !== undefined &&
|
|
431
|
+
latitude >= -90 &&
|
|
432
|
+
latitude <= 90 &&
|
|
433
|
+
longitude >= -180 &&
|
|
434
|
+
longitude <= 180;
|
|
435
|
+
}
|
|
269
436
|
|
|
270
437
|
function firstHeader(request: Request, names: string[]): string | undefined {
|
|
271
438
|
for (const name of names) {
|
package/src/component/ingest.ts
CHANGED
|
@@ -99,6 +99,10 @@ type SessionState = {
|
|
|
99
99
|
isNewVisitorDay: boolean;
|
|
100
100
|
startedAt: number;
|
|
101
101
|
source: string;
|
|
102
|
+
country?: string;
|
|
103
|
+
city?: string;
|
|
104
|
+
latitude?: number;
|
|
105
|
+
longitude?: number;
|
|
102
106
|
pageviewCount: number;
|
|
103
107
|
eventCount: number;
|
|
104
108
|
affiliateId?: Id<"affiliates">;
|
|
@@ -236,7 +240,6 @@ export const ingestBatch = mutation({
|
|
|
236
240
|
siteId: args.siteId,
|
|
237
241
|
sessionId: group.sessionId,
|
|
238
242
|
bucketStart: group.bucketStart,
|
|
239
|
-
candidateEventIds: group.eventIds,
|
|
240
243
|
});
|
|
241
244
|
const newCount = [...group.eventIds].filter((eventId) =>
|
|
242
245
|
!eventIds.has(eventId)
|
|
@@ -356,7 +359,7 @@ export const ingestBatch = mutation({
|
|
|
356
359
|
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
357
360
|
.take(MAX_FUNNELS_PER_SITE);
|
|
358
361
|
}
|
|
359
|
-
await accountEvent(ctx, {
|
|
362
|
+
const session = await accountEvent(ctx, {
|
|
360
363
|
siteId: args.siteId,
|
|
361
364
|
event,
|
|
362
365
|
context,
|
|
@@ -370,7 +373,12 @@ export const ingestBatch = mutation({
|
|
|
370
373
|
now,
|
|
371
374
|
batchState,
|
|
372
375
|
});
|
|
373
|
-
bufferEventBatch(
|
|
376
|
+
bufferEventBatch(
|
|
377
|
+
args.siteId,
|
|
378
|
+
event,
|
|
379
|
+
batchState.eventBatches,
|
|
380
|
+
session.country ?? "unknown",
|
|
381
|
+
);
|
|
374
382
|
accepted += 1;
|
|
375
383
|
}
|
|
376
384
|
await flushBatchState(ctx, batchState);
|
|
@@ -467,7 +475,7 @@ export const trackConversion = mutation({
|
|
|
467
475
|
};
|
|
468
476
|
|
|
469
477
|
const aggregateDeltas: AggregateDeltas = new Map();
|
|
470
|
-
await accountEvent(ctx, {
|
|
478
|
+
const session = await accountEvent(ctx, {
|
|
471
479
|
siteId: args.siteId,
|
|
472
480
|
event,
|
|
473
481
|
context: conversionContext,
|
|
@@ -481,7 +489,12 @@ export const trackConversion = mutation({
|
|
|
481
489
|
now: Date.now(),
|
|
482
490
|
});
|
|
483
491
|
if (!existingTelemetry && !telemetry.saturated) {
|
|
484
|
-
await insertEventBatch(
|
|
492
|
+
await insertEventBatch(
|
|
493
|
+
ctx,
|
|
494
|
+
args.siteId,
|
|
495
|
+
event,
|
|
496
|
+
session.country ?? "unknown",
|
|
497
|
+
);
|
|
485
498
|
}
|
|
486
499
|
await flushAggregates(ctx, aggregateDeltas);
|
|
487
500
|
return { accepted: 1, duplicates: 0, rejected: 0 };
|
|
@@ -612,7 +625,7 @@ async function accountEvent(
|
|
|
612
625
|
now: number;
|
|
613
626
|
batchState?: BatchState;
|
|
614
627
|
},
|
|
615
|
-
) {
|
|
628
|
+
): Promise<SessionState> {
|
|
616
629
|
const { event } = input;
|
|
617
630
|
const session = input.recordTelemetry
|
|
618
631
|
? await upsertSession(
|
|
@@ -700,7 +713,7 @@ async function accountEvent(
|
|
|
700
713
|
const dimensions: Dimension[] = [
|
|
701
714
|
{ type: "source", value: session.source },
|
|
702
715
|
{ type: "page", value: event.path },
|
|
703
|
-
{ type: "country", value:
|
|
716
|
+
{ type: "country", value: session.country ?? "unknown" },
|
|
704
717
|
{ type: "device", value: input.context.device ?? "unknown" },
|
|
705
718
|
{ type: "browser", value: input.context.browser ?? "Other" },
|
|
706
719
|
{
|
|
@@ -733,6 +746,7 @@ async function accountEvent(
|
|
|
733
746
|
input.batchState,
|
|
734
747
|
);
|
|
735
748
|
}
|
|
749
|
+
return session;
|
|
736
750
|
}
|
|
737
751
|
|
|
738
752
|
async function accountTrustedSession(
|
|
@@ -755,6 +769,10 @@ async function accountTrustedSession(
|
|
|
755
769
|
isNewVisitorDay: false,
|
|
756
770
|
startedAt: existing.startedAt,
|
|
757
771
|
source: existing.source,
|
|
772
|
+
country: existing.country,
|
|
773
|
+
city: existing.city,
|
|
774
|
+
latitude: existing.latitude,
|
|
775
|
+
longitude: existing.longitude,
|
|
758
776
|
pageviewCount: existing.pageviewCount,
|
|
759
777
|
eventCount: existing.eventCount,
|
|
760
778
|
affiliateId: existing.affiliateId,
|
|
@@ -832,6 +850,10 @@ async function upsertSession(
|
|
|
832
850
|
Math.floor(event.timestamp / DAY_MS),
|
|
833
851
|
startedAt,
|
|
834
852
|
source: existing.source,
|
|
853
|
+
country: context.country ?? existing.country,
|
|
854
|
+
city: context.city ?? existing.city,
|
|
855
|
+
latitude: context.latitude ?? existing.latitude,
|
|
856
|
+
longitude: context.longitude ?? existing.longitude,
|
|
835
857
|
pageviewCount:
|
|
836
858
|
existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
|
|
837
859
|
eventCount: existing.eventCount + 1,
|
|
@@ -910,6 +932,10 @@ async function upsertSession(
|
|
|
910
932
|
Math.floor(event.timestamp / DAY_MS),
|
|
911
933
|
startedAt: event.timestamp,
|
|
912
934
|
source,
|
|
935
|
+
country: context.country,
|
|
936
|
+
city: context.city,
|
|
937
|
+
latitude: context.latitude,
|
|
938
|
+
longitude: context.longitude,
|
|
913
939
|
pageviewCount: event.type === "pageview" ? 1 : 0,
|
|
914
940
|
eventCount: 1,
|
|
915
941
|
...(attribution ?? {}),
|
|
@@ -1102,6 +1128,10 @@ async function accountHeartbeat(
|
|
|
1102
1128
|
isNewVisitorDay: false,
|
|
1103
1129
|
startedAt,
|
|
1104
1130
|
source: session.source,
|
|
1131
|
+
country: context.country ?? session.country,
|
|
1132
|
+
city: context.city ?? session.city,
|
|
1133
|
+
latitude: context.latitude ?? session.latitude,
|
|
1134
|
+
longitude: context.longitude ?? session.longitude,
|
|
1105
1135
|
pageviewCount: session.pageviewCount,
|
|
1106
1136
|
eventCount: session.eventCount,
|
|
1107
1137
|
affiliateSlug: session.affiliateSlug,
|
|
@@ -1289,6 +1319,10 @@ async function updateLiveSession(
|
|
|
1289
1319
|
)
|
|
1290
1320
|
.unique();
|
|
1291
1321
|
const deadline = now + LIVE_SESSION_TTL_MS;
|
|
1322
|
+
const country = context.country ?? session.country;
|
|
1323
|
+
const city = context.city ?? session.city;
|
|
1324
|
+
const latitude = context.latitude ?? session.latitude;
|
|
1325
|
+
const longitude = context.longitude ?? session.longitude;
|
|
1292
1326
|
if (existing) {
|
|
1293
1327
|
const recentHeartbeatEventIds = event.type === "heartbeat"
|
|
1294
1328
|
? [
|
|
@@ -1304,10 +1338,10 @@ async function updateLiveSession(
|
|
|
1304
1338
|
path: event.path,
|
|
1305
1339
|
lastSeenAt: Math.max(existing.lastSeenAt, event.timestamp),
|
|
1306
1340
|
expiresAt: deadline,
|
|
1307
|
-
...(
|
|
1308
|
-
...(
|
|
1309
|
-
...(
|
|
1310
|
-
...(
|
|
1341
|
+
...(country ? { country } : {}),
|
|
1342
|
+
...(city ? { city } : {}),
|
|
1343
|
+
...(latitude !== undefined ? { latitude } : {}),
|
|
1344
|
+
...(longitude !== undefined ? { longitude } : {}),
|
|
1311
1345
|
browser: context.browser ?? existing.browser,
|
|
1312
1346
|
os: context.os ?? existing.os,
|
|
1313
1347
|
source: session.source,
|
|
@@ -1338,10 +1372,10 @@ async function updateLiveSession(
|
|
|
1338
1372
|
startedAt: event.timestamp,
|
|
1339
1373
|
lastSeenAt: event.timestamp,
|
|
1340
1374
|
expiresAt: deadline,
|
|
1341
|
-
...(
|
|
1342
|
-
...(
|
|
1343
|
-
...(
|
|
1344
|
-
...(
|
|
1375
|
+
...(country ? { country } : {}),
|
|
1376
|
+
...(city ? { city } : {}),
|
|
1377
|
+
...(latitude !== undefined ? { latitude } : {}),
|
|
1378
|
+
...(longitude !== undefined ? { longitude } : {}),
|
|
1345
1379
|
browser: context.browser ?? "Other",
|
|
1346
1380
|
os: context.os ?? "Other",
|
|
1347
1381
|
source: session.source,
|
package/src/component/reports.ts
CHANGED
|
@@ -331,7 +331,10 @@ export const listSessions = query({
|
|
|
331
331
|
.take(take)),
|
|
332
332
|
);
|
|
333
333
|
}
|
|
334
|
-
|
|
334
|
+
const publicRows = rows.map(
|
|
335
|
+
({ geoLookupAttemptedAt: _attemptedAt, ...row }) => row,
|
|
336
|
+
);
|
|
337
|
+
return keysetPaginationResult(publicRows, pagination, (row) => ({
|
|
335
338
|
timestamp: row.lastSeenAt,
|
|
336
339
|
creationTime: row._creationTime,
|
|
337
340
|
}));
|
package/src/component/schema.ts
CHANGED
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
affiliateFieldsValidator,
|
|
5
5
|
aggregateFieldsValidator,
|
|
6
6
|
batchedEventValidator,
|
|
7
|
-
storedEventFieldsValidator,
|
|
8
7
|
funnelFieldsValidator,
|
|
9
8
|
goalFieldsValidator,
|
|
10
9
|
sessionFieldsValidator,
|
|
@@ -17,7 +16,11 @@ export default defineSchema({
|
|
|
17
16
|
.index("by_ownerId_and_name", ["ownerId", "name"])
|
|
18
17
|
.index("by_networkId", ["networkId"]),
|
|
19
18
|
|
|
20
|
-
sessions: defineTable(
|
|
19
|
+
sessions: defineTable(
|
|
20
|
+
sessionFieldsValidator.extend({
|
|
21
|
+
geoLookupAttemptedAt: v.optional(v.number()),
|
|
22
|
+
}),
|
|
23
|
+
)
|
|
21
24
|
.index("by_siteId_and_sessionId", ["siteId", "sessionId"])
|
|
22
25
|
.index("by_siteId_and_lastSeenAt", ["siteId", "lastSeenAt"])
|
|
23
26
|
.index("by_siteId_and_startedAt", ["siteId", "startedAt"])
|
|
@@ -59,19 +62,11 @@ export default defineSchema({
|
|
|
59
62
|
scheduledAt: v.number(),
|
|
60
63
|
}).index("by_siteId", ["siteId"]),
|
|
61
64
|
|
|
62
|
-
|
|
63
|
-
.
|
|
64
|
-
.
|
|
65
|
-
.
|
|
66
|
-
|
|
67
|
-
"sessionId",
|
|
68
|
-
"timestamp",
|
|
69
|
-
])
|
|
70
|
-
.index("by_siteId_and_visitorId_and_timestamp", [
|
|
71
|
-
"siteId",
|
|
72
|
-
"visitorId",
|
|
73
|
-
"timestamp",
|
|
74
|
-
]),
|
|
65
|
+
geoLookupBudgets: defineTable({
|
|
66
|
+
siteId: v.id("sites"),
|
|
67
|
+
dayStart: v.number(),
|
|
68
|
+
count: v.number(),
|
|
69
|
+
}).index("by_siteId", ["siteId"]),
|
|
75
70
|
|
|
76
71
|
eventBatches: defineTable({
|
|
77
72
|
siteId: v.id("sites"),
|
|
@@ -122,7 +117,11 @@ export default defineSchema({
|
|
|
122
117
|
valueCents: v.number(),
|
|
123
118
|
})
|
|
124
119
|
.index("by_siteId_and_goalId_and_eventId", ["siteId", "goalId", "eventId"])
|
|
125
|
-
.index("by_siteId_and_goalId_and_timestamp", [
|
|
120
|
+
.index("by_siteId_and_goalId_and_timestamp", [
|
|
121
|
+
"siteId",
|
|
122
|
+
"goalId",
|
|
123
|
+
"timestamp",
|
|
124
|
+
])
|
|
126
125
|
.index("by_siteId_and_timestamp", ["siteId", "timestamp"]),
|
|
127
126
|
|
|
128
127
|
funnels: defineTable(funnelFieldsValidator)
|
|
@@ -50,7 +50,10 @@ export const ingestContextValidator = v.object({
|
|
|
50
50
|
});
|
|
51
51
|
|
|
52
52
|
export const batchedEventValidator = trackerEventValidator.extend(
|
|
53
|
-
|
|
53
|
+
{
|
|
54
|
+
...ingestContextValidator.fields,
|
|
55
|
+
aggregateCountry: v.optional(v.string()),
|
|
56
|
+
},
|
|
54
57
|
);
|
|
55
58
|
|
|
56
59
|
export const siteFieldsValidator = v.object({
|
|
@@ -123,18 +126,6 @@ export const eventFieldsValidator = v.object({
|
|
|
123
126
|
device: v.string(),
|
|
124
127
|
});
|
|
125
128
|
|
|
126
|
-
export const storedEventFieldsValidator = eventFieldsValidator
|
|
127
|
-
.omit("country", "city", "latitude", "longitude", "browser", "os", "device")
|
|
128
|
-
.extend({
|
|
129
|
-
country: v.optional(v.string()),
|
|
130
|
-
city: v.optional(v.string()),
|
|
131
|
-
latitude: v.optional(v.number()),
|
|
132
|
-
longitude: v.optional(v.number()),
|
|
133
|
-
browser: v.optional(v.string()),
|
|
134
|
-
os: v.optional(v.string()),
|
|
135
|
-
device: v.optional(v.string()),
|
|
136
|
-
});
|
|
137
|
-
|
|
138
129
|
export const dimensionTypeValidator = v.union(
|
|
139
130
|
v.literal("source"),
|
|
140
131
|
v.literal("page"),
|
package/src/test.ts
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
/// <reference types="vite/client" />
|
|
2
|
-
import migrationsComponent from "@convex-dev/migrations/test";
|
|
3
2
|
import type { TestConvex } from "convex-test";
|
|
4
3
|
import type { GenericSchema, SchemaDefinition } from "convex/server";
|
|
5
4
|
import schema from "./component/schema.js";
|
|
@@ -16,6 +15,5 @@ export function register(
|
|
|
16
15
|
name: string = "rastroAnalytics",
|
|
17
16
|
) {
|
|
18
17
|
t.registerComponent(name, schema, modules);
|
|
19
|
-
migrationsComponent.register(t, `${name}/migrations`);
|
|
20
18
|
}
|
|
21
19
|
export default { register, schema, modules };
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
export declare const migrateLegacyEvents: import("convex/server").RegisteredMutation<"internal", {
|
|
2
|
-
fn?: string | undefined;
|
|
3
|
-
cursor?: string | null | undefined;
|
|
4
|
-
batchSize?: number | undefined;
|
|
5
|
-
dryRun?: boolean | undefined;
|
|
6
|
-
next?: string[] | undefined;
|
|
7
|
-
reset?: boolean | undefined;
|
|
8
|
-
oneBatchOnly?: boolean | undefined;
|
|
9
|
-
}, Promise<any>>;
|
|
10
|
-
export declare const legacyEventsStatus: import("convex/server").RegisteredQuery<"internal", {}, Promise<{
|
|
11
|
-
complete: boolean;
|
|
12
|
-
sourceEmpty: boolean;
|
|
13
|
-
migrationState: "canceled" | "failed" | "inProgress" | "success" | "unknown";
|
|
14
|
-
processed: number;
|
|
15
|
-
sampleRemainingId: import("convex/values").GenericId<"events"> | null;
|
|
16
|
-
}>>;
|
|
17
|
-
//# sourceMappingURL=migrations.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"migrations.d.ts","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAYA,eAAO,MAAM,mBAAmB;;;;;;;;gBAI9B,CAAC;AAEH,eAAO,MAAM,kBAAkB;;;;;;GAiC7B,CAAC"}
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
import { Migrations } from "@convex-dev/migrations";
|
|
2
|
-
import { v } from "convex/values";
|
|
3
|
-
import { components } from "./_generated/api.js";
|
|
4
|
-
import { internalMutation, internalQuery } from "./_generated/server.js";
|
|
5
|
-
import { migrateLegacyEventToBatch } from "./eventStore.js";
|
|
6
|
-
import schema from "./schema.js";
|
|
7
|
-
const migrations = new Migrations(components.migrations, {
|
|
8
|
-
internalMutation,
|
|
9
|
-
schema,
|
|
10
|
-
});
|
|
11
|
-
export const migrateLegacyEvents = migrations.define({
|
|
12
|
-
table: "events",
|
|
13
|
-
batchSize: 16,
|
|
14
|
-
migrateOne: migrateLegacyEventToBatch,
|
|
15
|
-
});
|
|
16
|
-
export const legacyEventsStatus = internalQuery({
|
|
17
|
-
args: {},
|
|
18
|
-
returns: v.object({
|
|
19
|
-
complete: v.boolean(),
|
|
20
|
-
sourceEmpty: v.boolean(),
|
|
21
|
-
migrationState: v.union(v.literal("inProgress"), v.literal("success"), v.literal("failed"), v.literal("canceled"), v.literal("unknown")),
|
|
22
|
-
processed: v.number(),
|
|
23
|
-
sampleRemainingId: v.union(v.id("events"), v.null()),
|
|
24
|
-
}),
|
|
25
|
-
handler: async (ctx) => {
|
|
26
|
-
const [remaining, statuses] = await Promise.all([
|
|
27
|
-
ctx.db.query("events").first(),
|
|
28
|
-
migrations.getStatus(ctx, {
|
|
29
|
-
migrations: ["migrations:migrateLegacyEvents"],
|
|
30
|
-
}),
|
|
31
|
-
]);
|
|
32
|
-
const migrationStatus = statuses[0];
|
|
33
|
-
const migrationState = migrationStatus?.state ?? "unknown";
|
|
34
|
-
const sourceEmpty = remaining === null;
|
|
35
|
-
return {
|
|
36
|
-
complete: sourceEmpty && migrationState === "success",
|
|
37
|
-
sourceEmpty,
|
|
38
|
-
migrationState,
|
|
39
|
-
processed: migrationStatus?.processed ?? 0,
|
|
40
|
-
sampleRemainingId: remaining?._id ?? null,
|
|
41
|
-
};
|
|
42
|
-
},
|
|
43
|
-
});
|
|
44
|
-
//# sourceMappingURL=migrations.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/component/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,CAAC,EAAE,MAAM,eAAe,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,EAAE;IACvD,gBAAgB;IAChB,MAAM;CACP,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAAG,UAAU,CAAC,MAAM,CAAC;IACnD,KAAK,EAAE,QAAQ;IACf,SAAS,EAAE,EAAE;IACb,UAAU,EAAE,yBAAyB;CACtC,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAC9C,IAAI,EAAE,EAAE;IACR,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC;QAChB,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE;QACrB,WAAW,EAAE,CAAC,CAAC,OAAO,EAAE;QACxB,cAAc,EAAE,CAAC,CAAC,KAAK,CACrB,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,EACvB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,EACpB,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,EACnB,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,EACrB,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CACrB;QACD,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;QACrB,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;KACrD,CAAC;IACF,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;QACrB,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE;YAC9B,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE;gBACxB,UAAU,EAAE,CAAC,gCAAgC,CAAC;aAC/C,CAAC;SACH,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,cAAc,GAAG,eAAe,EAAE,KAAK,IAAI,SAAS,CAAC;QAC3D,MAAM,WAAW,GAAG,SAAS,KAAK,IAAI,CAAC;QACvC,OAAO;YACL,QAAQ,EAAE,WAAW,IAAI,cAAc,KAAK,SAAS;YACrD,WAAW;YACX,cAAc;YACd,SAAS,EAAE,eAAe,EAAE,SAAS,IAAI,CAAC;YAC1C,iBAAiB,EAAE,SAAS,EAAE,GAAG,IAAI,IAAI;SAC1C,CAAC;IACJ,CAAC;CACF,CAAC,CAAC"}
|
|
@@ -1,52 +0,0 @@
|
|
|
1
|
-
import { Migrations } from "@convex-dev/migrations";
|
|
2
|
-
import { v } from "convex/values";
|
|
3
|
-
import { components } from "./_generated/api.js";
|
|
4
|
-
import { internalMutation, internalQuery } from "./_generated/server.js";
|
|
5
|
-
import { migrateLegacyEventToBatch } from "./eventStore.js";
|
|
6
|
-
import schema from "./schema.js";
|
|
7
|
-
|
|
8
|
-
const migrations = new Migrations(components.migrations, {
|
|
9
|
-
internalMutation,
|
|
10
|
-
schema,
|
|
11
|
-
});
|
|
12
|
-
|
|
13
|
-
export const migrateLegacyEvents = migrations.define({
|
|
14
|
-
table: "events",
|
|
15
|
-
batchSize: 16,
|
|
16
|
-
migrateOne: migrateLegacyEventToBatch,
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
export const legacyEventsStatus = internalQuery({
|
|
20
|
-
args: {},
|
|
21
|
-
returns: v.object({
|
|
22
|
-
complete: v.boolean(),
|
|
23
|
-
sourceEmpty: v.boolean(),
|
|
24
|
-
migrationState: v.union(
|
|
25
|
-
v.literal("inProgress"),
|
|
26
|
-
v.literal("success"),
|
|
27
|
-
v.literal("failed"),
|
|
28
|
-
v.literal("canceled"),
|
|
29
|
-
v.literal("unknown"),
|
|
30
|
-
),
|
|
31
|
-
processed: v.number(),
|
|
32
|
-
sampleRemainingId: v.union(v.id("events"), v.null()),
|
|
33
|
-
}),
|
|
34
|
-
handler: async (ctx) => {
|
|
35
|
-
const [remaining, statuses] = await Promise.all([
|
|
36
|
-
ctx.db.query("events").first(),
|
|
37
|
-
migrations.getStatus(ctx, {
|
|
38
|
-
migrations: ["migrations:migrateLegacyEvents"],
|
|
39
|
-
}),
|
|
40
|
-
]);
|
|
41
|
-
const migrationStatus = statuses[0];
|
|
42
|
-
const migrationState = migrationStatus?.state ?? "unknown";
|
|
43
|
-
const sourceEmpty = remaining === null;
|
|
44
|
-
return {
|
|
45
|
-
complete: sourceEmpty && migrationState === "success",
|
|
46
|
-
sourceEmpty,
|
|
47
|
-
migrationState,
|
|
48
|
-
processed: migrationStatus?.processed ?? 0,
|
|
49
|
-
sampleRemainingId: remaining?._id ?? null,
|
|
50
|
-
};
|
|
51
|
-
},
|
|
52
|
-
});
|