@iann29/rastro 0.1.0-alpha.5 → 0.1.0-alpha.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +82 -29
- package/agent/integration.md +17 -15
- package/agent/manifest.json +7 -5
- package/agent/manifest.schema.json +8 -5
- package/dist/client/federation.d.ts +34 -8
- package/dist/client/federation.d.ts.map +1 -1
- package/dist/client/federation.js +17 -3
- package/dist/client/federation.js.map +1 -1
- package/dist/client/index.d.ts +418 -21
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +191 -41
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +2 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +63 -7
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/affiliates.d.ts.map +1 -1
- package/dist/component/affiliates.js +6 -2
- package/dist/component/affiliates.js.map +1 -1
- package/dist/component/cardinality.d.ts.map +1 -1
- package/dist/component/cardinality.js.map +1 -1
- package/dist/component/constants.d.ts +7 -1
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +13 -1
- package/dist/component/constants.js.map +1 -1
- package/dist/component/diagnostics.d.ts.map +1 -1
- package/dist/component/diagnostics.js.map +1 -1
- package/dist/component/eventStore.d.ts +2 -1
- package/dist/component/eventStore.d.ts.map +1 -1
- package/dist/component/eventStore.js +13 -10
- package/dist/component/eventStore.js.map +1 -1
- package/dist/component/funnels.d.ts.map +1 -1
- package/dist/component/funnels.js +5 -3
- package/dist/component/funnels.js.map +1 -1
- package/dist/component/geo.d.ts.map +1 -1
- package/dist/component/geo.js +3 -2
- package/dist/component/geo.js.map +1 -1
- package/dist/component/goals.d.ts.map +1 -1
- package/dist/component/goals.js +8 -5
- package/dist/component/goals.js.map +1 -1
- package/dist/component/guards.d.ts.map +1 -1
- package/dist/component/guards.js.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +51 -17
- package/dist/component/http.js.map +1 -1
- package/dist/component/ingest.d.ts +2 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +143 -23
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/live.d.ts.map +1 -1
- package/dist/component/live.js +1 -3
- package/dist/component/live.js.map +1 -1
- package/dist/component/reports.d.ts +60 -3
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +215 -49
- package/dist/component/reports.js.map +1 -1
- package/dist/component/retention.d.ts +4 -4
- package/dist/component/retention.d.ts.map +1 -1
- package/dist/component/retention.js +54 -22
- package/dist/component/retention.js.map +1 -1
- package/dist/component/sanitize.d.ts +11 -0
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +58 -11
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +47 -5
- package/dist/component/schema.js +21 -1
- package/dist/component/schema.js.map +1 -1
- package/dist/component/sites.d.ts.map +1 -1
- package/dist/component/sites.js +6 -6
- package/dist/component/sites.js.map +1 -1
- package/dist/component/validators.d.ts +14 -9
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +5 -1
- package/dist/component/validators.js.map +1 -1
- package/dist/component/visitors.d.ts.map +1 -1
- package/dist/component/visitors.js.map +1 -1
- package/dist/component/vitals.d.ts +41 -0
- package/dist/component/vitals.d.ts.map +1 -0
- package/dist/component/vitals.js +115 -0
- package/dist/component/vitals.js.map +1 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js.map +1 -1
- package/dist/tracker/generated.d.ts +8 -4
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +8 -4
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.d.ts.map +1 -1
- package/dist/tracker/tracker.js +3 -1
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker/vitals.d.ts +10 -0
- package/dist/tracker/vitals.d.ts.map +1 -0
- package/dist/tracker/vitals.js +140 -0
- package/dist/tracker/vitals.js.map +1 -0
- package/dist/tracker.min.js +1 -1
- package/dist/vitals.min.js +1 -0
- package/docs/benchmarks/2026-08-20-realistic.md +71 -71
- package/docs/benchmarks/2026-08-21-formal-certification.md +50 -30
- package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
- package/docs/upgrading.md +46 -8
- package/llms.txt +5 -1
- package/package.json +17 -3
- package/scripts/benchmark-ingest.mjs +101 -48
- package/src/component/_generated/api.ts +2 -0
- package/src/component/_generated/component.ts +69 -4
- package/src/component/affiliates.ts +20 -5
- package/src/component/cardinality.ts +8 -7
- package/src/component/constants.ts +13 -1
- package/src/component/diagnostics.ts +3 -2
- package/src/component/eventStore.ts +53 -46
- package/src/component/funnels.ts +19 -16
- package/src/component/geo.ts +16 -14
- package/src/component/goals.ts +29 -21
- package/src/component/guards.ts +3 -1
- package/src/component/http.ts +93 -54
- package/src/component/ingest.ts +320 -127
- package/src/component/live.ts +5 -4
- package/src/component/reports.ts +563 -219
- package/src/component/retention.ts +221 -98
- package/src/component/sanitize.ts +99 -29
- package/src/component/schema.ts +31 -5
- package/src/component/sites.ts +22 -11
- package/src/component/validators.ts +16 -7
- package/src/component/visitors.ts +16 -5
- package/src/component/vitals.ts +146 -0
package/src/component/ingest.ts
CHANGED
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
REPORT_ROLLUP_SHARDS,
|
|
25
25
|
SECONDARY_AGGREGATE_SHARDS,
|
|
26
26
|
SITE_INGEST_SHARDS,
|
|
27
|
+
VITAL_PAGE_SLOTS,
|
|
27
28
|
} from "./constants.js";
|
|
28
29
|
import {
|
|
29
30
|
addVisitor,
|
|
@@ -31,6 +32,15 @@ import {
|
|
|
31
32
|
unionVisitorSketches,
|
|
32
33
|
type VisitorSketch,
|
|
33
34
|
} from "./cardinality.js";
|
|
35
|
+
import {
|
|
36
|
+
addToVitalHistogram,
|
|
37
|
+
emptyVitalHistogram,
|
|
38
|
+
mergeVitalHistograms,
|
|
39
|
+
vitalDevice,
|
|
40
|
+
VITAL_ALL,
|
|
41
|
+
VITAL_OTHER_PAGES,
|
|
42
|
+
type VitalMetric,
|
|
43
|
+
} from "./vitals.js";
|
|
34
44
|
import { fail } from "./errors.js";
|
|
35
45
|
import { ensureAnalyticsControl } from "./coverage.js";
|
|
36
46
|
import {
|
|
@@ -111,11 +121,14 @@ type FunnelRollupDelta = {
|
|
|
111
121
|
funnelId: Id<"funnels">;
|
|
112
122
|
bucketStart: number;
|
|
113
123
|
shard: number;
|
|
114
|
-
steps: Map<
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
124
|
+
steps: Map<
|
|
125
|
+
number,
|
|
126
|
+
{
|
|
127
|
+
step: number;
|
|
128
|
+
entrants: number;
|
|
129
|
+
completions: number;
|
|
130
|
+
}
|
|
131
|
+
>;
|
|
119
132
|
};
|
|
120
133
|
type AffiliateRollupDelta = {
|
|
121
134
|
siteId: Id<"sites">;
|
|
@@ -129,10 +142,22 @@ type AffiliateRollupDelta = {
|
|
|
129
142
|
revenueCents: number;
|
|
130
143
|
commissionCents: number;
|
|
131
144
|
};
|
|
145
|
+
type VitalRollupDelta = {
|
|
146
|
+
siteId: Id<"sites">;
|
|
147
|
+
bucketStart: number;
|
|
148
|
+
device: string;
|
|
149
|
+
page: string;
|
|
150
|
+
shard: number;
|
|
151
|
+
metrics: Map<
|
|
152
|
+
VitalMetric,
|
|
153
|
+
{ count: number; sum: number; histogram: number[] }
|
|
154
|
+
>;
|
|
155
|
+
};
|
|
132
156
|
type ReportRollupDeltas = {
|
|
133
157
|
goals: Map<string, GoalRollupDelta>;
|
|
134
158
|
funnels: Map<string, FunnelRollupDelta>;
|
|
135
159
|
affiliates: Map<string, AffiliateRollupDelta>;
|
|
160
|
+
vitals: Map<string, VitalRollupDelta>;
|
|
136
161
|
};
|
|
137
162
|
type AttributionSnapshot = {
|
|
138
163
|
affiliateId: Id<"affiliates">;
|
|
@@ -175,8 +200,9 @@ type CachedRow<T> = {
|
|
|
175
200
|
dirty: boolean;
|
|
176
201
|
};
|
|
177
202
|
|
|
178
|
-
type CachedDocument<
|
|
179
|
-
|
|
203
|
+
type CachedDocument<
|
|
204
|
+
TableName extends keyof import("./_generated/dataModel.js").DataModel,
|
|
205
|
+
> = Omit<Doc<TableName>, "_creationTime"> & { _creationTime?: number };
|
|
180
206
|
|
|
181
207
|
type BatchState = {
|
|
182
208
|
sessions: Map<string, CachedRow<CachedDocument<"sessions">>>;
|
|
@@ -207,13 +233,20 @@ export const ingestBatch = mutation({
|
|
|
207
233
|
});
|
|
208
234
|
}
|
|
209
235
|
if (args.events.length < 1 || args.events.length > MAX_BATCH_EVENTS) {
|
|
210
|
-
fail(
|
|
211
|
-
|
|
212
|
-
|
|
236
|
+
fail(
|
|
237
|
+
"BATCH_TOO_LARGE",
|
|
238
|
+
`events must contain 1-${MAX_BATCH_EVENTS} items`,
|
|
239
|
+
{
|
|
240
|
+
limit: MAX_BATCH_EVENTS,
|
|
241
|
+
},
|
|
242
|
+
);
|
|
213
243
|
}
|
|
214
244
|
let eventByteCounts: number[];
|
|
215
245
|
try {
|
|
216
|
-
({ eventByteCounts } = measureBatchBytes(
|
|
246
|
+
({ eventByteCounts } = measureBatchBytes(
|
|
247
|
+
String(args.siteId),
|
|
248
|
+
args.events,
|
|
249
|
+
));
|
|
217
250
|
} catch {
|
|
218
251
|
fail("BATCH_TOO_LARGE", "batch exceeds 64 KiB", { limitBytes: 65_536 });
|
|
219
252
|
}
|
|
@@ -264,7 +297,7 @@ export const ingestBatch = mutation({
|
|
|
264
297
|
const bucketStart = eventBucketStart(event.timestamp);
|
|
265
298
|
const groupKey = eventBatchKey(event.sessionId, bucketStart);
|
|
266
299
|
requestGroups.add(groupKey);
|
|
267
|
-
if (event.type === "heartbeat") continue;
|
|
300
|
+
if (event.type === "heartbeat" || event.type === "vital") continue;
|
|
268
301
|
const eventIds = eventIdsByGroup.get(groupKey) ?? new Set<string>();
|
|
269
302
|
if (eventIds.has(event.eventId)) {
|
|
270
303
|
duplicateInBatch.add(eventIndex);
|
|
@@ -295,8 +328,8 @@ export const ingestBatch = mutation({
|
|
|
295
328
|
sessionId: group.sessionId,
|
|
296
329
|
bucketStart: group.bucketStart,
|
|
297
330
|
});
|
|
298
|
-
const newCount = [...group.eventIds].filter(
|
|
299
|
-
!eventIds.has(eventId)
|
|
331
|
+
const newCount = [...group.eventIds].filter(
|
|
332
|
+
(eventId) => !eventIds.has(eventId),
|
|
300
333
|
).length;
|
|
301
334
|
if (
|
|
302
335
|
newCount > 0 &&
|
|
@@ -335,12 +368,7 @@ export const ingestBatch = mutation({
|
|
|
335
368
|
continue;
|
|
336
369
|
}
|
|
337
370
|
const liveSession = (
|
|
338
|
-
await loadLiveSession(
|
|
339
|
-
ctx,
|
|
340
|
-
args.siteId,
|
|
341
|
-
event.sessionId,
|
|
342
|
-
batchState,
|
|
343
|
-
)
|
|
371
|
+
await loadLiveSession(ctx, args.siteId, event.sessionId, batchState)
|
|
344
372
|
).current;
|
|
345
373
|
if (
|
|
346
374
|
liveSession &&
|
|
@@ -376,10 +404,42 @@ export const ingestBatch = mutation({
|
|
|
376
404
|
continue;
|
|
377
405
|
}
|
|
378
406
|
|
|
407
|
+
if (event.type === "vital") {
|
|
408
|
+
// A vital annotates telemetry its session already produced; it never
|
|
409
|
+
// creates sessions, live presence, event batches, or aggregate counts.
|
|
410
|
+
if (!existingSession) {
|
|
411
|
+
rejected += 1;
|
|
412
|
+
continue;
|
|
413
|
+
}
|
|
414
|
+
if (
|
|
415
|
+
!(await consumeRateLimit(
|
|
416
|
+
ctx,
|
|
417
|
+
args.siteId,
|
|
418
|
+
event.sessionId,
|
|
419
|
+
eventByteCounts[eventIndex],
|
|
420
|
+
now,
|
|
421
|
+
sessionRateWindows,
|
|
422
|
+
))
|
|
423
|
+
) {
|
|
424
|
+
rejected += 1;
|
|
425
|
+
continue;
|
|
426
|
+
}
|
|
427
|
+
foldVitalRollups(
|
|
428
|
+
reportRollupDeltas,
|
|
429
|
+
args.siteId,
|
|
430
|
+
event,
|
|
431
|
+
vitalDevice(context.device),
|
|
432
|
+
);
|
|
433
|
+
accepted += 1;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
|
|
379
437
|
if (
|
|
380
438
|
duplicateInBatch.has(eventIndex) ||
|
|
381
439
|
persistedEventIds
|
|
382
|
-
.get(
|
|
440
|
+
.get(
|
|
441
|
+
eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)),
|
|
442
|
+
)
|
|
383
443
|
?.has(event.eventId)
|
|
384
444
|
) {
|
|
385
445
|
duplicates += 1;
|
|
@@ -418,8 +478,8 @@ export const ingestBatch = mutation({
|
|
|
418
478
|
event,
|
|
419
479
|
context,
|
|
420
480
|
existingSession,
|
|
421
|
-
goals: canMatchDefinition ? goals ?? [] : [],
|
|
422
|
-
funnels: canMatchDefinition ? funnels ?? [] : [],
|
|
481
|
+
goals: canMatchDefinition ? (goals ?? []) : [],
|
|
482
|
+
funnels: canMatchDefinition ? (funnels ?? []) : [],
|
|
423
483
|
aggregateDeltas,
|
|
424
484
|
reportRollupDeltas,
|
|
425
485
|
recordTelemetry: true,
|
|
@@ -451,7 +511,10 @@ export const trackConversion = mutation({
|
|
|
451
511
|
const site = await ctx.db.get("sites", args.siteId);
|
|
452
512
|
if (!site) fail("NOT_FOUND", "site not found");
|
|
453
513
|
if (!isValidMoney(args.revenueCents)) {
|
|
454
|
-
fail(
|
|
514
|
+
fail(
|
|
515
|
+
"INVALID_ARGUMENT",
|
|
516
|
+
"revenueCents must be an integer between 0 and 1000000000",
|
|
517
|
+
);
|
|
455
518
|
}
|
|
456
519
|
let event: TrackerEvent;
|
|
457
520
|
let trustedGoalKey: string | undefined;
|
|
@@ -476,7 +539,10 @@ export const trackConversion = mutation({
|
|
|
476
539
|
Date.now(),
|
|
477
540
|
);
|
|
478
541
|
} catch (error) {
|
|
479
|
-
fail(
|
|
542
|
+
fail(
|
|
543
|
+
"INVALID_ARGUMENT",
|
|
544
|
+
error instanceof Error ? error.message : "invalid conversion",
|
|
545
|
+
);
|
|
480
546
|
}
|
|
481
547
|
|
|
482
548
|
const siteCurrency = site.currency ?? "USD";
|
|
@@ -496,10 +562,14 @@ export const trackConversion = mutation({
|
|
|
496
562
|
return { accepted: 0, duplicates: 1, rejected: 0 };
|
|
497
563
|
}
|
|
498
564
|
if (event.currency !== siteCurrency) {
|
|
499
|
-
fail(
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
565
|
+
fail(
|
|
566
|
+
"CURRENCY_MISMATCH",
|
|
567
|
+
"conversion currency does not match site currency",
|
|
568
|
+
{
|
|
569
|
+
siteCurrency,
|
|
570
|
+
conversionCurrency: event.currency ?? "USD",
|
|
571
|
+
},
|
|
572
|
+
);
|
|
503
573
|
}
|
|
504
574
|
|
|
505
575
|
const telemetry = await findTelemetry(ctx, args.siteId, event);
|
|
@@ -659,11 +729,7 @@ async function flushBatchState(ctx: MutationCtx, batchState: BatchState) {
|
|
|
659
729
|
function withoutSystemFields<
|
|
660
730
|
T extends { _id: unknown; _creationTime?: number },
|
|
661
731
|
>(document: T): Omit<T, "_id" | "_creationTime"> {
|
|
662
|
-
const {
|
|
663
|
-
_id,
|
|
664
|
-
_creationTime,
|
|
665
|
-
...fields
|
|
666
|
-
} = document;
|
|
732
|
+
const { _id, _creationTime, ...fields } = document;
|
|
667
733
|
void _id;
|
|
668
734
|
void _creationTime;
|
|
669
735
|
return fields;
|
|
@@ -702,8 +768,8 @@ async function accountEvent(
|
|
|
702
768
|
: await accountTrustedSession(ctx, input.existingSession, event);
|
|
703
769
|
|
|
704
770
|
const matchedGoalKeys: string[] = [];
|
|
705
|
-
const matchingGoals = input.goals.filter(
|
|
706
|
-
goal.active && goalMatches(goal, event)
|
|
771
|
+
const matchingGoals = input.goals.filter(
|
|
772
|
+
(goal) => goal.active && goalMatches(goal, event),
|
|
707
773
|
);
|
|
708
774
|
if (matchingGoals.length > 1) {
|
|
709
775
|
fail("LIMIT_EXCEEDED", "active goals must use unique matchers");
|
|
@@ -714,33 +780,36 @@ async function accountEvent(
|
|
|
714
780
|
eventBucketStart(event.timestamp),
|
|
715
781
|
event.eventId,
|
|
716
782
|
]);
|
|
717
|
-
const scopedPrior = (
|
|
718
|
-
.
|
|
719
|
-
|
|
720
|
-
range
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
783
|
+
const scopedPrior = (
|
|
784
|
+
await ctx.db
|
|
785
|
+
.query("goalCompletions")
|
|
786
|
+
.withIndex("by_siteId_and_goalId_and_eventId", (range) =>
|
|
787
|
+
range
|
|
788
|
+
.eq("siteId", input.siteId)
|
|
789
|
+
.eq("goalId", goal._id)
|
|
790
|
+
.eq("eventId", completionEventId),
|
|
791
|
+
)
|
|
792
|
+
.take(2)
|
|
793
|
+
).find((completion) => completion.idempotencyVersion === 2);
|
|
727
794
|
const legacyPrior = scopedPrior
|
|
728
795
|
? undefined
|
|
729
|
-
: (
|
|
730
|
-
.
|
|
731
|
-
|
|
732
|
-
range
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
796
|
+
: (
|
|
797
|
+
await ctx.db
|
|
798
|
+
.query("goalCompletions")
|
|
799
|
+
.withIndex("by_siteId_and_goalId_and_eventId", (range) =>
|
|
800
|
+
range
|
|
801
|
+
.eq("siteId", input.siteId)
|
|
802
|
+
.eq("goalId", goal._id)
|
|
803
|
+
.eq("eventId", event.eventId),
|
|
804
|
+
)
|
|
805
|
+
.take(2)
|
|
806
|
+
).find(
|
|
807
|
+
(completion) =>
|
|
739
808
|
completion.idempotencyVersion === undefined &&
|
|
740
809
|
completion.sessionId === event.sessionId &&
|
|
741
810
|
eventBucketStart(completion.timestamp) ===
|
|
742
|
-
eventBucketStart(event.timestamp)
|
|
743
|
-
|
|
811
|
+
eventBucketStart(event.timestamp),
|
|
812
|
+
);
|
|
744
813
|
if (scopedPrior || legacyPrior) continue;
|
|
745
814
|
await ctx.db.insert("goalCompletions", {
|
|
746
815
|
siteId: input.siteId,
|
|
@@ -753,12 +822,7 @@ async function accountEvent(
|
|
|
753
822
|
timestamp: event.timestamp,
|
|
754
823
|
valueCents: goal.valueCents ?? 0,
|
|
755
824
|
});
|
|
756
|
-
foldGoalRollup(
|
|
757
|
-
input.reportRollupDeltas,
|
|
758
|
-
input.siteId,
|
|
759
|
-
event,
|
|
760
|
-
goal,
|
|
761
|
-
);
|
|
825
|
+
foldGoalRollup(input.reportRollupDeltas, input.siteId, event, goal);
|
|
762
826
|
matchedGoalKeys.push(goal.key);
|
|
763
827
|
}
|
|
764
828
|
|
|
@@ -798,7 +862,7 @@ async function accountEvent(
|
|
|
798
862
|
{ type: "browser", value: input.context.browser ?? "Other" },
|
|
799
863
|
{
|
|
800
864
|
type: "event",
|
|
801
|
-
value: event.type === "custom" ? event.name ?? "custom" : event.type,
|
|
865
|
+
value: event.type === "custom" ? (event.name ?? "custom") : event.type,
|
|
802
866
|
},
|
|
803
867
|
...matchedGoalKeys.map((value): Dimension => ({ type: "goal", value })),
|
|
804
868
|
];
|
|
@@ -908,15 +972,18 @@ async function upsertSession(
|
|
|
908
972
|
const updates = {
|
|
909
973
|
startedAt,
|
|
910
974
|
lastSeenAt,
|
|
911
|
-
entryPath:
|
|
912
|
-
|
|
975
|
+
entryPath:
|
|
976
|
+
event.timestamp < existing.startedAt ? event.path : existing.entryPath,
|
|
977
|
+
exitPath:
|
|
978
|
+
event.timestamp >= existing.lastSeenAt ? event.path : existing.exitPath,
|
|
913
979
|
eventCount: existing.eventCount + 1,
|
|
914
|
-
pageviewCount:
|
|
980
|
+
pageviewCount:
|
|
981
|
+
existing.pageviewCount + (event.type === "pageview" ? 1 : 0),
|
|
915
982
|
durationMs: Math.max(0, lastSeenAt - startedAt),
|
|
916
983
|
conversionCount: existing.conversionCount + Number(financialConversion),
|
|
917
984
|
revenueCents:
|
|
918
985
|
existing.revenueCents +
|
|
919
|
-
(financialConversion ? event.revenueCents ?? 0 : 0),
|
|
986
|
+
(financialConversion ? (event.revenueCents ?? 0) : 0),
|
|
920
987
|
...(financialConversion && event.currency
|
|
921
988
|
? { currency: event.currency }
|
|
922
989
|
: {}),
|
|
@@ -927,7 +994,9 @@ async function upsertSession(
|
|
|
927
994
|
...(context.country ? { country: context.country } : {}),
|
|
928
995
|
...(context.city ? { city: context.city } : {}),
|
|
929
996
|
...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
|
|
930
|
-
...(context.longitude !== undefined
|
|
997
|
+
...(context.longitude !== undefined
|
|
998
|
+
? { longitude: context.longitude }
|
|
999
|
+
: {}),
|
|
931
1000
|
browser: context.browser ?? existing.browser,
|
|
932
1001
|
os: context.os ?? existing.os,
|
|
933
1002
|
device: context.device ?? existing.device,
|
|
@@ -1000,7 +1069,9 @@ async function upsertSession(
|
|
|
1000
1069
|
...(context.country ? { country: context.country } : {}),
|
|
1001
1070
|
...(context.city ? { city: context.city } : {}),
|
|
1002
1071
|
...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
|
|
1003
|
-
...(context.longitude !== undefined
|
|
1072
|
+
...(context.longitude !== undefined
|
|
1073
|
+
? { longitude: context.longitude }
|
|
1074
|
+
: {}),
|
|
1004
1075
|
browser: context.browser ?? "Other",
|
|
1005
1076
|
os: context.os ?? "Other",
|
|
1006
1077
|
device: context.device ?? "unknown",
|
|
@@ -1008,7 +1079,7 @@ async function upsertSession(
|
|
|
1008
1079
|
pageviewCount: event.type === "pageview" ? 1 : 0,
|
|
1009
1080
|
durationMs: 0,
|
|
1010
1081
|
conversionCount: Number(financialConversion),
|
|
1011
|
-
revenueCents: financialConversion ? event.revenueCents ?? 0 : 0,
|
|
1082
|
+
revenueCents: financialConversion ? (event.revenueCents ?? 0) : 0,
|
|
1012
1083
|
...(financialConversion && event.currency
|
|
1013
1084
|
? { currency: event.currency }
|
|
1014
1085
|
: {}),
|
|
@@ -1056,14 +1127,12 @@ async function upsertSession(
|
|
|
1056
1127
|
};
|
|
1057
1128
|
}
|
|
1058
1129
|
|
|
1059
|
-
function attributionSnapshot(
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
},
|
|
1066
|
-
): AttributionSnapshot | undefined {
|
|
1130
|
+
function attributionSnapshot(value: {
|
|
1131
|
+
affiliateId?: Id<"affiliates">;
|
|
1132
|
+
affiliateSlug?: string;
|
|
1133
|
+
affiliateAttributedAt?: number;
|
|
1134
|
+
affiliateExpiresAt?: number;
|
|
1135
|
+
}): AttributionSnapshot | undefined {
|
|
1067
1136
|
if (
|
|
1068
1137
|
!value.affiliateId ||
|
|
1069
1138
|
!value.affiliateSlug ||
|
|
@@ -1080,7 +1149,6 @@ function attributionSnapshot(
|
|
|
1080
1149
|
};
|
|
1081
1150
|
}
|
|
1082
1151
|
|
|
1083
|
-
|
|
1084
1152
|
function attributionApplies(
|
|
1085
1153
|
attribution: AttributionSnapshot,
|
|
1086
1154
|
timestamp: number,
|
|
@@ -1211,14 +1279,15 @@ async function accountHeartbeat(
|
|
|
1211
1279
|
const updates = {
|
|
1212
1280
|
startedAt,
|
|
1213
1281
|
lastSeenAt,
|
|
1214
|
-
exitPath:
|
|
1215
|
-
? event.path
|
|
1216
|
-
: session.exitPath,
|
|
1282
|
+
exitPath:
|
|
1283
|
+
event.timestamp >= session.lastSeenAt ? event.path : session.exitPath,
|
|
1217
1284
|
durationMs: Math.max(0, lastSeenAt - startedAt),
|
|
1218
1285
|
...(context.country ? { country: context.country } : {}),
|
|
1219
1286
|
...(context.city ? { city: context.city } : {}),
|
|
1220
1287
|
...(context.latitude !== undefined ? { latitude: context.latitude } : {}),
|
|
1221
|
-
...(context.longitude !== undefined
|
|
1288
|
+
...(context.longitude !== undefined
|
|
1289
|
+
? { longitude: context.longitude }
|
|
1290
|
+
: {}),
|
|
1222
1291
|
browser: context.browser ?? session.browser,
|
|
1223
1292
|
os: context.os ?? session.os,
|
|
1224
1293
|
device: context.device ?? session.device,
|
|
@@ -1265,7 +1334,10 @@ async function consumeSiteRateLimit(
|
|
|
1265
1334
|
): Promise<void> {
|
|
1266
1335
|
const windowStart =
|
|
1267
1336
|
Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
|
|
1268
|
-
const increments = new Map<
|
|
1337
|
+
const increments = new Map<
|
|
1338
|
+
number,
|
|
1339
|
+
{ eventCount: number; byteCount: number }
|
|
1340
|
+
>();
|
|
1269
1341
|
for (let index = 0; index < events.length; index += 1) {
|
|
1270
1342
|
const event = events[index];
|
|
1271
1343
|
const byteCount = eventByteCounts[index];
|
|
@@ -1290,9 +1362,8 @@ async function consumeSiteRateLimit(
|
|
|
1290
1362
|
.order("desc")
|
|
1291
1363
|
.take(1);
|
|
1292
1364
|
const storedWindow = rows[0] ?? null;
|
|
1293
|
-
const window =
|
|
1294
|
-
? storedWindow
|
|
1295
|
-
: null;
|
|
1365
|
+
const window =
|
|
1366
|
+
storedWindow?.windowStart === windowStart ? storedWindow : null;
|
|
1296
1367
|
if (
|
|
1297
1368
|
(window?.eventCount ?? 0) + increment.eventCount >
|
|
1298
1369
|
MAX_SITE_EVENTS_PER_SHARD_WINDOW ||
|
|
@@ -1304,10 +1375,8 @@ async function consumeSiteRateLimit(
|
|
|
1304
1375
|
windowMs: RATE_LIMIT_WINDOW_MS,
|
|
1305
1376
|
eventLimitPerShard: MAX_SITE_EVENTS_PER_SHARD_WINDOW,
|
|
1306
1377
|
byteLimitPerShard: MAX_SITE_BYTES_PER_SHARD_WINDOW,
|
|
1307
|
-
siteEventLimit:
|
|
1308
|
-
|
|
1309
|
-
siteByteLimit:
|
|
1310
|
-
SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
|
|
1378
|
+
siteEventLimit: SITE_INGEST_SHARDS * MAX_SITE_EVENTS_PER_SHARD_WINDOW,
|
|
1379
|
+
siteByteLimit: SITE_INGEST_SHARDS * MAX_SITE_BYTES_PER_SHARD_WINDOW,
|
|
1311
1380
|
});
|
|
1312
1381
|
}
|
|
1313
1382
|
windows.push({ shard, increment, window: storedWindow });
|
|
@@ -1348,7 +1417,8 @@ async function consumeRateLimit(
|
|
|
1348
1417
|
now: number,
|
|
1349
1418
|
states: Map<string, SessionRateWindowState>,
|
|
1350
1419
|
): Promise<boolean> {
|
|
1351
|
-
const windowStart =
|
|
1420
|
+
const windowStart =
|
|
1421
|
+
Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
|
|
1352
1422
|
let state = states.get(sessionId);
|
|
1353
1423
|
if (!state) {
|
|
1354
1424
|
const rows = await ctx.db
|
|
@@ -1386,7 +1456,8 @@ async function flushSessionRateLimits(
|
|
|
1386
1456
|
states: Map<string, SessionRateWindowState>,
|
|
1387
1457
|
now: number,
|
|
1388
1458
|
) {
|
|
1389
|
-
const windowStart =
|
|
1459
|
+
const windowStart =
|
|
1460
|
+
Math.floor(now / RATE_LIMIT_WINDOW_MS) * RATE_LIMIT_WINDOW_MS;
|
|
1390
1461
|
for (const state of states.values()) {
|
|
1391
1462
|
if (!state.dirty) continue;
|
|
1392
1463
|
if (state.window) {
|
|
@@ -1436,15 +1507,16 @@ async function updateLiveSession(
|
|
|
1436
1507
|
const latitude = context.latitude ?? session.latitude;
|
|
1437
1508
|
const longitude = context.longitude ?? session.longitude;
|
|
1438
1509
|
if (existing) {
|
|
1439
|
-
const recentHeartbeatEventIds =
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
(existing.
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1510
|
+
const recentHeartbeatEventIds =
|
|
1511
|
+
event.type === "heartbeat"
|
|
1512
|
+
? [
|
|
1513
|
+
...(existing.recentHeartbeatEventIds ??
|
|
1514
|
+
(existing.lastHeartbeatEventId
|
|
1515
|
+
? [existing.lastHeartbeatEventId]
|
|
1516
|
+
: [])),
|
|
1517
|
+
event.eventId,
|
|
1518
|
+
].slice(-8)
|
|
1519
|
+
: undefined;
|
|
1448
1520
|
const updates = {
|
|
1449
1521
|
visitorId: event.visitorId,
|
|
1450
1522
|
path: event.path,
|
|
@@ -1601,7 +1673,11 @@ async function advanceFunnel(
|
|
|
1601
1673
|
foldFunnelRollup(reportRollupDeltas, siteId, funnel._id, event, 0, false);
|
|
1602
1674
|
return;
|
|
1603
1675
|
}
|
|
1604
|
-
if (
|
|
1676
|
+
if (
|
|
1677
|
+
progress.completedAt !== undefined ||
|
|
1678
|
+
event.timestamp < progress.lastStepAt
|
|
1679
|
+
)
|
|
1680
|
+
return;
|
|
1605
1681
|
if (event.timestamp - progress.startedAt > funnel.conversionWindowMs) {
|
|
1606
1682
|
if (!matchesFirst) return;
|
|
1607
1683
|
const updated = {
|
|
@@ -1799,6 +1875,7 @@ function createReportRollupDeltas(): ReportRollupDeltas {
|
|
|
1799
1875
|
goals: new Map(),
|
|
1800
1876
|
funnels: new Map(),
|
|
1801
1877
|
affiliates: new Map(),
|
|
1878
|
+
vitals: new Map(),
|
|
1802
1879
|
};
|
|
1803
1880
|
}
|
|
1804
1881
|
|
|
@@ -1938,6 +2015,120 @@ function foldAffiliateConversionRollup(
|
|
|
1938
2015
|
deltas.affiliates.set(coordinates.key, current);
|
|
1939
2016
|
}
|
|
1940
2017
|
|
|
2018
|
+
function foldVitalRollups(
|
|
2019
|
+
deltas: ReportRollupDeltas,
|
|
2020
|
+
siteId: Id<"sites">,
|
|
2021
|
+
event: TrackerEvent,
|
|
2022
|
+
device: string,
|
|
2023
|
+
) {
|
|
2024
|
+
const metric = event.name as VitalMetric;
|
|
2025
|
+
const value = event.value ?? 0;
|
|
2026
|
+
const bucketStart = Math.floor(event.timestamp / DAY_MS) * DAY_MS;
|
|
2027
|
+
const shard = stableHash(event.sessionId) % REPORT_ROLLUP_SHARDS;
|
|
2028
|
+
// Each measurement lands twice: on its page (across devices) and on its
|
|
2029
|
+
// device class (across pages), so both breakdowns stay exact and bounded.
|
|
2030
|
+
for (const target of [
|
|
2031
|
+
{ device: VITAL_ALL, page: event.path },
|
|
2032
|
+
{ device, page: VITAL_ALL },
|
|
2033
|
+
]) {
|
|
2034
|
+
const key = JSON.stringify([
|
|
2035
|
+
siteId,
|
|
2036
|
+
bucketStart,
|
|
2037
|
+
target.device,
|
|
2038
|
+
target.page,
|
|
2039
|
+
shard,
|
|
2040
|
+
]);
|
|
2041
|
+
const row = deltas.vitals.get(key) ?? {
|
|
2042
|
+
siteId,
|
|
2043
|
+
bucketStart,
|
|
2044
|
+
device: target.device,
|
|
2045
|
+
page: target.page,
|
|
2046
|
+
shard,
|
|
2047
|
+
metrics: new Map<
|
|
2048
|
+
VitalMetric,
|
|
2049
|
+
{ count: number; sum: number; histogram: number[] }
|
|
2050
|
+
>(),
|
|
2051
|
+
};
|
|
2052
|
+
const entry = row.metrics.get(metric) ?? {
|
|
2053
|
+
count: 0,
|
|
2054
|
+
sum: 0,
|
|
2055
|
+
histogram: emptyVitalHistogram(),
|
|
2056
|
+
};
|
|
2057
|
+
entry.count += 1;
|
|
2058
|
+
entry.sum += value;
|
|
2059
|
+
entry.histogram = addToVitalHistogram(entry.histogram, value);
|
|
2060
|
+
row.metrics.set(metric, entry);
|
|
2061
|
+
deltas.vitals.set(key, row);
|
|
2062
|
+
}
|
|
2063
|
+
}
|
|
2064
|
+
|
|
2065
|
+
async function upsertVitalRollup(ctx: MutationCtx, delta: VitalRollupDelta) {
|
|
2066
|
+
const existing = await ctx.db
|
|
2067
|
+
.query("vitalRollups")
|
|
2068
|
+
.withIndex("by_key", (range) =>
|
|
2069
|
+
range
|
|
2070
|
+
.eq("siteId", delta.siteId)
|
|
2071
|
+
.eq("bucketStart", delta.bucketStart)
|
|
2072
|
+
.eq("device", delta.device)
|
|
2073
|
+
.eq("page", delta.page)
|
|
2074
|
+
.eq("shard", delta.shard),
|
|
2075
|
+
)
|
|
2076
|
+
.unique();
|
|
2077
|
+
if (
|
|
2078
|
+
!existing &&
|
|
2079
|
+
delta.device === VITAL_ALL &&
|
|
2080
|
+
delta.page !== VITAL_OTHER_PAGES
|
|
2081
|
+
) {
|
|
2082
|
+
// A new page row spends one bounded read to respect the daily page slots;
|
|
2083
|
+
// past the cap, the day's remaining pages fold into "(other)".
|
|
2084
|
+
const pageRows = await ctx.db
|
|
2085
|
+
.query("vitalRollups")
|
|
2086
|
+
.withIndex("by_key", (range) =>
|
|
2087
|
+
range
|
|
2088
|
+
.eq("siteId", delta.siteId)
|
|
2089
|
+
.eq("bucketStart", delta.bucketStart)
|
|
2090
|
+
.eq("device", VITAL_ALL),
|
|
2091
|
+
)
|
|
2092
|
+
.take(VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS + 1);
|
|
2093
|
+
if (pageRows.length > VITAL_PAGE_SLOTS * REPORT_ROLLUP_SHARDS) {
|
|
2094
|
+
await upsertVitalRollup(ctx, { ...delta, page: VITAL_OTHER_PAGES });
|
|
2095
|
+
return;
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
const metrics = existing?.metrics.map((entry) => ({ ...entry })) ?? [];
|
|
2099
|
+
for (const [metric, addition] of delta.metrics) {
|
|
2100
|
+
const entry = metrics.find((candidate) => candidate.metric === metric);
|
|
2101
|
+
if (entry) {
|
|
2102
|
+
entry.count += addition.count;
|
|
2103
|
+
entry.sum += addition.sum;
|
|
2104
|
+
entry.histogram = mergeVitalHistograms(
|
|
2105
|
+
entry.histogram,
|
|
2106
|
+
addition.histogram,
|
|
2107
|
+
);
|
|
2108
|
+
} else {
|
|
2109
|
+
metrics.push({
|
|
2110
|
+
metric,
|
|
2111
|
+
count: addition.count,
|
|
2112
|
+
sum: addition.sum,
|
|
2113
|
+
histogram: [...addition.histogram],
|
|
2114
|
+
});
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
const fields = {
|
|
2118
|
+
siteId: delta.siteId,
|
|
2119
|
+
bucketStart: delta.bucketStart,
|
|
2120
|
+
device: delta.device,
|
|
2121
|
+
page: delta.page,
|
|
2122
|
+
shard: delta.shard,
|
|
2123
|
+
metrics,
|
|
2124
|
+
};
|
|
2125
|
+
if (existing) {
|
|
2126
|
+
await ctx.db.replace("vitalRollups", existing._id, fields);
|
|
2127
|
+
} else {
|
|
2128
|
+
await ctx.db.insert("vitalRollups", fields);
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
|
|
1941
2132
|
async function flushReportRollups(
|
|
1942
2133
|
ctx: MutationCtx,
|
|
1943
2134
|
deltas: ReportRollupDeltas,
|
|
@@ -2010,12 +2201,14 @@ async function flushReportRollups(
|
|
|
2010
2201
|
for (const delta of deltas.affiliates.values()) {
|
|
2011
2202
|
const existing = await ctx.db
|
|
2012
2203
|
.query("affiliateDailyRollups")
|
|
2013
|
-
.withIndex(
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2204
|
+
.withIndex(
|
|
2205
|
+
"by_siteId_and_affiliateId_and_bucketStart_and_shard",
|
|
2206
|
+
(range) =>
|
|
2207
|
+
range
|
|
2208
|
+
.eq("siteId", delta.siteId)
|
|
2209
|
+
.eq("affiliateId", delta.affiliateId)
|
|
2210
|
+
.eq("bucketStart", delta.bucketStart)
|
|
2211
|
+
.eq("shard", delta.shard),
|
|
2019
2212
|
)
|
|
2020
2213
|
.unique();
|
|
2021
2214
|
const fields = {
|
|
@@ -2032,8 +2225,7 @@ async function flushReportRollups(
|
|
|
2032
2225
|
),
|
|
2033
2226
|
conversions: (existing?.conversions ?? 0) + delta.conversions,
|
|
2034
2227
|
revenueCents: (existing?.revenueCents ?? 0) + delta.revenueCents,
|
|
2035
|
-
commissionCents:
|
|
2036
|
-
(existing?.commissionCents ?? 0) + delta.commissionCents,
|
|
2228
|
+
commissionCents: (existing?.commissionCents ?? 0) + delta.commissionCents,
|
|
2037
2229
|
};
|
|
2038
2230
|
if (existing) {
|
|
2039
2231
|
await ctx.db.replace("affiliateDailyRollups", existing._id, fields);
|
|
@@ -2041,6 +2233,9 @@ async function flushReportRollups(
|
|
|
2041
2233
|
await ctx.db.insert("affiliateDailyRollups", fields);
|
|
2042
2234
|
}
|
|
2043
2235
|
}
|
|
2236
|
+
for (const delta of deltas.vitals.values()) {
|
|
2237
|
+
await upsertVitalRollup(ctx, delta);
|
|
2238
|
+
}
|
|
2044
2239
|
}
|
|
2045
2240
|
|
|
2046
2241
|
function foldAggregates(
|
|
@@ -2052,16 +2247,15 @@ function foldAggregates(
|
|
|
2052
2247
|
recordTelemetry: boolean,
|
|
2053
2248
|
financialConversion: boolean,
|
|
2054
2249
|
) {
|
|
2055
|
-
const revenueCents = financialConversion ? event.revenueCents ?? 0 : 0;
|
|
2056
|
-
const dimensionDeltas = dimensions.map(
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
}),
|
|
2062
|
-
);
|
|
2250
|
+
const revenueCents = financialConversion ? (event.revenueCents ?? 0) : 0;
|
|
2251
|
+
const dimensionDeltas = dimensions.map((dimension): DimensionDelta => ({
|
|
2252
|
+
...dimension,
|
|
2253
|
+
count: Number(recordTelemetry),
|
|
2254
|
+
revenueCents,
|
|
2255
|
+
}));
|
|
2063
2256
|
for (const granularity of ["hour", "day"] as const) {
|
|
2064
|
-
const shard =
|
|
2257
|
+
const shard =
|
|
2258
|
+
stableHash(event.sessionId) %
|
|
2065
2259
|
(granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
|
|
2066
2260
|
const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
|
|
2067
2261
|
const bucketStart = Math.floor(event.timestamp / interval) * interval;
|
|
@@ -2196,10 +2390,7 @@ function mergeDimensions(
|
|
|
2196
2390
|
(candidate) => candidate.type === addition.type,
|
|
2197
2391
|
);
|
|
2198
2392
|
if (typeSlots.length < DIMENSION_SLOTS - 1) {
|
|
2199
|
-
const occupied = Array.from(
|
|
2200
|
-
{ length: DIMENSION_SLOTS },
|
|
2201
|
-
() => false,
|
|
2202
|
-
);
|
|
2393
|
+
const occupied = Array.from({ length: DIMENSION_SLOTS }, () => false);
|
|
2203
2394
|
for (const candidate of typeSlots) occupied[candidate.slot] = true;
|
|
2204
2395
|
let slot = stableHash(addition.value) % DIMENSION_SLOTS;
|
|
2205
2396
|
while (occupied[slot]) slot = (slot + 1) % DIMENSION_SLOTS;
|
|
@@ -2251,5 +2442,7 @@ function stringProperty(
|
|
|
2251
2442
|
key: string,
|
|
2252
2443
|
): string | undefined {
|
|
2253
2444
|
const value = properties?.[key];
|
|
2254
|
-
return typeof value === "string"
|
|
2445
|
+
return typeof value === "string"
|
|
2446
|
+
? cleanString(value, 120) || undefined
|
|
2447
|
+
: undefined;
|
|
2255
2448
|
}
|