@iann29/rastro 0.1.0-alpha.0 → 0.1.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +579 -57
- package/agent/integration.md +661 -0
- package/agent/manifest.json +187 -0
- package/agent/manifest.schema.json +411 -0
- package/dist/client/federation.d.ts +236 -0
- package/dist/client/federation.d.ts.map +1 -0
- package/dist/client/federation.js +196 -0
- package/dist/client/federation.js.map +1 -0
- package/dist/client/index.d.ts +2467 -18
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +461 -32
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +18 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +302 -4
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/_generated/server.d.ts +4 -0
- package/dist/component/_generated/server.d.ts.map +1 -1
- package/dist/component/_generated/server.js.map +1 -1
- package/dist/component/affiliates.d.ts.map +1 -1
- package/dist/component/affiliates.js +6 -2
- package/dist/component/affiliates.js.map +1 -1
- package/dist/component/cardinality.d.ts +12 -0
- package/dist/component/cardinality.d.ts.map +1 -0
- package/dist/component/cardinality.js +94 -0
- package/dist/component/cardinality.js.map +1 -0
- package/dist/component/constants.d.ts +34 -7
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +47 -7
- package/dist/component/constants.js.map +1 -1
- package/dist/component/convex.config.d.ts +6 -1
- package/dist/component/convex.config.js +9 -1
- package/dist/component/convex.config.js.map +1 -1
- package/dist/component/coverage.d.ts +19 -0
- package/dist/component/coverage.d.ts.map +1 -0
- package/dist/component/coverage.js +30 -0
- package/dist/component/coverage.js.map +1 -0
- package/dist/component/diagnostics.d.ts +9 -0
- package/dist/component/diagnostics.d.ts.map +1 -0
- package/dist/component/diagnostics.js +47 -0
- package/dist/component/diagnostics.js.map +1 -0
- package/dist/component/errors.d.ts +1 -1
- package/dist/component/errors.d.ts.map +1 -1
- package/dist/component/errors.js.map +1 -1
- package/dist/component/eventStore.d.ts +106 -0
- package/dist/component/eventStore.d.ts.map +1 -0
- package/dist/component/eventStore.js +294 -0
- package/dist/component/eventStore.js.map +1 -0
- package/dist/component/funnels.d.ts.map +1 -1
- package/dist/component/funnels.js +5 -3
- package/dist/component/funnels.js.map +1 -1
- package/dist/component/geo.d.ts +71 -0
- package/dist/component/geo.d.ts.map +1 -0
- package/dist/component/geo.js +611 -0
- package/dist/component/geo.js.map +1 -0
- package/dist/component/goals.d.ts.map +1 -1
- package/dist/component/goals.js +25 -9
- package/dist/component/goals.js.map +1 -1
- package/dist/component/guards.d.ts.map +1 -1
- package/dist/component/guards.js.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +284 -53
- package/dist/component/http.js.map +1 -1
- package/dist/component/identity.d.ts +13 -0
- package/dist/component/identity.d.ts.map +1 -0
- package/dist/component/identity.js +58 -0
- package/dist/component/identity.js.map +1 -0
- package/dist/component/ingest.d.ts +3 -1
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +1105 -243
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/live.d.ts +8 -0
- package/dist/component/live.d.ts.map +1 -1
- package/dist/component/live.js +74 -9
- package/dist/component/live.js.map +1 -1
- package/dist/component/reports.d.ts +273 -24
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +1320 -150
- package/dist/component/reports.js.map +1 -1
- package/dist/component/retention.d.ts +75 -1
- package/dist/component/retention.d.ts.map +1 -1
- package/dist/component/retention.js +541 -31
- package/dist/component/retention.js.map +1 -1
- package/dist/component/sanitize.d.ts +24 -1
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +107 -19
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +462 -55
- package/dist/component/schema.js +195 -15
- package/dist/component/schema.js.map +1 -1
- package/dist/component/sites.d.ts.map +1 -1
- package/dist/component/sites.js +38 -25
- package/dist/component/sites.js.map +1 -1
- package/dist/component/useragent.d.ts +9 -0
- package/dist/component/useragent.d.ts.map +1 -0
- package/dist/component/useragent.js +152 -0
- package/dist/component/useragent.js.map +1 -0
- package/dist/component/validators.d.ts +101 -16
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +19 -2
- package/dist/component/validators.js.map +1 -1
- package/dist/component/visitors.d.ts +19 -0
- package/dist/component/visitors.d.ts.map +1 -0
- package/dist/component/visitors.js +86 -0
- package/dist/component/visitors.js.map +1 -0
- package/dist/component/vitals.d.ts +41 -0
- package/dist/component/vitals.d.ts.map +1 -0
- package/dist/component/vitals.js +115 -0
- package/dist/component/vitals.js.map +1 -0
- package/dist/react/index.d.ts.map +1 -1
- package/dist/react/index.js.map +1 -1
- package/dist/tracker/generated.d.ts +8 -4
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +8 -4
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.d.ts +1 -1
- package/dist/tracker/tracker.d.ts.map +1 -1
- package/dist/tracker/tracker.js +81 -33
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker/vitals.d.ts +10 -0
- package/dist/tracker/vitals.d.ts.map +1 -0
- package/dist/tracker/vitals.js +140 -0
- package/dist/tracker/vitals.js.map +1 -0
- package/dist/tracker.min.js +1 -1
- package/dist/vitals.min.js +1 -0
- package/docs/benchmarks/2026-08-20-realistic.md +171 -0
- package/docs/benchmarks/2026-08-21-formal-certification.md +353 -0
- package/docs/benchmarks/2026-08-30-alpha6-recertification.md +206 -0
- package/docs/federation-setup.md +397 -0
- package/docs/federation.md +262 -0
- package/docs/upgrading.md +259 -0
- package/llms.txt +71 -0
- package/package.json +61 -11
- package/scripts/benchmark-ingest.mjs +601 -0
- package/scripts/generate-federation-keys.mjs +20 -0
- package/src/component/_generated/api.ts +18 -0
- package/src/component/_generated/component.ts +366 -4
- package/src/component/_generated/server.ts +4 -0
- package/src/component/affiliates.ts +20 -5
- package/src/component/cardinality.ts +117 -0
- package/src/component/constants.ts +47 -7
- package/src/component/convex.config.ts +11 -1
- package/src/component/coverage.ts +36 -0
- package/src/component/diagnostics.ts +65 -0
- package/src/component/errors.ts +2 -1
- package/src/component/eventStore.ts +489 -0
- package/src/component/funnels.ts +19 -16
- package/src/component/geo.ts +781 -0
- package/src/component/goals.ts +45 -16
- package/src/component/guards.ts +3 -1
- package/src/component/http.ts +403 -60
- package/src/component/identity.ts +74 -0
- package/src/component/ingest.ts +1652 -291
- package/src/component/live.ts +90 -9
- package/src/component/reports.ts +1929 -199
- package/src/component/retention.ts +774 -97
- package/src/component/sanitize.ts +154 -32
- package/src/component/schema.ts +219 -15
- package/src/component/sites.ts +56 -28
- package/src/component/useragent.ts +171 -0
- package/src/component/validators.ts +29 -1
- package/src/component/visitors.ts +116 -0
- package/src/component/vitals.ts +146 -0
package/src/component/sites.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { v } from "convex/values";
|
|
2
2
|
import { internalQuery, mutation, query } from "./_generated/server.js";
|
|
3
3
|
import { MAX_SITES_PER_OWNER } from "./constants.js";
|
|
4
|
+
import { ensureAnalyticsControl } from "./coverage.js";
|
|
4
5
|
import { fail } from "./errors.js";
|
|
5
6
|
import {
|
|
6
7
|
cleanString,
|
|
@@ -31,7 +32,27 @@ export const create = mutation({
|
|
|
31
32
|
handler: async (ctx, args) => {
|
|
32
33
|
const ownerId = cleanString(args.ownerId, 128);
|
|
33
34
|
const name = cleanString(args.name, 120);
|
|
34
|
-
if (!ownerId || !name)
|
|
35
|
+
if (!ownerId || !name)
|
|
36
|
+
fail("INVALID_SITE", "ownerId and name are required");
|
|
37
|
+
|
|
38
|
+
let domains: string[];
|
|
39
|
+
let timezone: string | undefined;
|
|
40
|
+
let networkId: string | undefined;
|
|
41
|
+
let currency: string;
|
|
42
|
+
try {
|
|
43
|
+
domains = normalizeDomains(args.domains);
|
|
44
|
+
timezone = validateTimezone(args.timezone);
|
|
45
|
+
networkId = args.networkId
|
|
46
|
+
? sanitizeOpaqueId(args.networkId, "networkId")
|
|
47
|
+
: undefined;
|
|
48
|
+
currency = sanitizeCurrency(args.currency ?? "USD");
|
|
49
|
+
} catch (error) {
|
|
50
|
+
fail(
|
|
51
|
+
"INVALID_SITE",
|
|
52
|
+
error instanceof Error ? error.message : "invalid site",
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
const cookieless = args.cookieless ?? true;
|
|
35
56
|
|
|
36
57
|
const existing = await ctx.db
|
|
37
58
|
.query("sites")
|
|
@@ -39,7 +60,23 @@ export const create = mutation({
|
|
|
39
60
|
range.eq("ownerId", ownerId).eq("name", name),
|
|
40
61
|
)
|
|
41
62
|
.unique();
|
|
42
|
-
if (existing)
|
|
63
|
+
if (existing) {
|
|
64
|
+
if (
|
|
65
|
+
existing.domains.length !== domains.length ||
|
|
66
|
+
existing.domains.some((domain, index) => domain !== domains[index]) ||
|
|
67
|
+
existing.networkId !== networkId ||
|
|
68
|
+
existing.timezone !== timezone ||
|
|
69
|
+
(existing.currency ?? "USD") !== currency ||
|
|
70
|
+
existing.cookieless !== cookieless
|
|
71
|
+
) {
|
|
72
|
+
fail(
|
|
73
|
+
"CONFLICT",
|
|
74
|
+
"site name is already associated with different settings",
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
await ensureAnalyticsControl(ctx, existing._id, Date.now());
|
|
78
|
+
return existing._id;
|
|
79
|
+
}
|
|
43
80
|
|
|
44
81
|
const sites = await ctx.db
|
|
45
82
|
.query("sites")
|
|
@@ -51,32 +88,20 @@ export const create = mutation({
|
|
|
51
88
|
});
|
|
52
89
|
}
|
|
53
90
|
|
|
54
|
-
let domains: string[];
|
|
55
|
-
let timezone: string | undefined;
|
|
56
|
-
let networkId: string | undefined;
|
|
57
|
-
let currency: string;
|
|
58
|
-
try {
|
|
59
|
-
domains = normalizeDomains(args.domains);
|
|
60
|
-
timezone = validateTimezone(args.timezone);
|
|
61
|
-
networkId = args.networkId
|
|
62
|
-
? sanitizeOpaqueId(args.networkId, "networkId")
|
|
63
|
-
: undefined;
|
|
64
|
-
currency = sanitizeCurrency(args.currency ?? "USD");
|
|
65
|
-
} catch (error) {
|
|
66
|
-
fail("INVALID_SITE", error instanceof Error ? error.message : "invalid site");
|
|
67
|
-
}
|
|
68
91
|
const now = Date.now();
|
|
69
|
-
|
|
92
|
+
const siteId = await ctx.db.insert("sites", {
|
|
70
93
|
ownerId,
|
|
71
94
|
name,
|
|
72
95
|
domains,
|
|
73
96
|
networkId,
|
|
74
97
|
timezone,
|
|
75
98
|
currency,
|
|
76
|
-
cookieless
|
|
99
|
+
cookieless,
|
|
77
100
|
createdAt: now,
|
|
78
101
|
updatedAt: now,
|
|
79
102
|
});
|
|
103
|
+
await ensureAnalyticsControl(ctx, siteId, now);
|
|
104
|
+
return siteId;
|
|
80
105
|
},
|
|
81
106
|
});
|
|
82
107
|
|
|
@@ -119,27 +144,30 @@ export const update = mutation({
|
|
|
119
144
|
patch.name = name;
|
|
120
145
|
}
|
|
121
146
|
try {
|
|
122
|
-
if (args.domains !== undefined)
|
|
147
|
+
if (args.domains !== undefined)
|
|
148
|
+
patch.domains = normalizeDomains(args.domains);
|
|
123
149
|
if (args.timezone !== undefined) {
|
|
124
|
-
patch.timezone =
|
|
125
|
-
? undefined
|
|
126
|
-
: validateTimezone(args.timezone);
|
|
150
|
+
patch.timezone =
|
|
151
|
+
args.timezone === null ? undefined : validateTimezone(args.timezone);
|
|
127
152
|
}
|
|
128
153
|
if (args.networkId !== undefined) {
|
|
129
|
-
patch.networkId =
|
|
130
|
-
|
|
131
|
-
|
|
154
|
+
patch.networkId =
|
|
155
|
+
args.networkId === null
|
|
156
|
+
? undefined
|
|
157
|
+
: sanitizeOpaqueId(args.networkId, "networkId");
|
|
132
158
|
}
|
|
133
159
|
if (args.currency !== undefined) {
|
|
134
160
|
patch.currency = sanitizeCurrency(args.currency);
|
|
135
161
|
}
|
|
136
162
|
} catch (error) {
|
|
137
|
-
fail(
|
|
163
|
+
fail(
|
|
164
|
+
"INVALID_SITE",
|
|
165
|
+
error instanceof Error ? error.message : "invalid site",
|
|
166
|
+
);
|
|
138
167
|
}
|
|
139
168
|
if (
|
|
140
169
|
patch.currency !== undefined &&
|
|
141
|
-
site.currency !==
|
|
142
|
-
site.currency !== patch.currency
|
|
170
|
+
(site.currency ?? "USD") !== patch.currency
|
|
143
171
|
) {
|
|
144
172
|
fail("CONFLICT", "site currency cannot be changed");
|
|
145
173
|
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
export type ClientClassification = {
|
|
2
|
+
browser: string;
|
|
3
|
+
os: string;
|
|
4
|
+
device: string;
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
// The tracker never reports the client: everything here is derived from headers
|
|
8
|
+
// the browser sets itself. Chromium sends the low-entropy `Sec-CH-UA` hints on
|
|
9
|
+
// cross-origin requests without an `Accept-CH` opt-in, which is the only way to
|
|
10
|
+
// tell Brave apart from Chrome — Brave deliberately ships Chrome's User-Agent.
|
|
11
|
+
// Firefox and Safari implement no client hints, so the User-Agent stays the
|
|
12
|
+
// fallback for them and for insecure origins, where no hint is sent at all.
|
|
13
|
+
const MAX_HINT_LENGTH = 512;
|
|
14
|
+
const MAX_HINT_BRANDS = 16;
|
|
15
|
+
|
|
16
|
+
const BRAND_PATTERN =
|
|
17
|
+
/"((?:[^"\\]|\\.)*)"(?:\s*;\s*v\s*=\s*"(?:[^"\\]|\\.)*")?/g;
|
|
18
|
+
const PLATFORM_PATTERN = /^\s*"((?:[^"\\]|\\.)*)"\s*$/;
|
|
19
|
+
|
|
20
|
+
// `Sec-CH-UA` is attacker-controlled on a public ingestion route, so brands are
|
|
21
|
+
// allowlisted: anything unrecognized (including the GREASE entries Chromium
|
|
22
|
+
// injects, such as `Not.A/Brand`) falls back to the User-Agent instead of
|
|
23
|
+
// entering the browser dimension verbatim.
|
|
24
|
+
const HINT_BRANDS = new Map<string, string>([
|
|
25
|
+
["brave", "Brave"],
|
|
26
|
+
["chromium", "Chromium"],
|
|
27
|
+
["google chrome", "Chrome"],
|
|
28
|
+
["microsoft edge", "Edge"],
|
|
29
|
+
["opera", "Opera"],
|
|
30
|
+
["opera gx", "Opera"],
|
|
31
|
+
["samsung internet", "Samsung Internet"],
|
|
32
|
+
["vivaldi", "Vivaldi"],
|
|
33
|
+
["yandex", "Yandex"],
|
|
34
|
+
]);
|
|
35
|
+
|
|
36
|
+
// Every Chromium brand list contains "Chromium", and Chrome adds "Google
|
|
37
|
+
// Chrome" beside it; both only win when no fork identifies itself.
|
|
38
|
+
const GENERIC_BRANDS = new Set(["Chrome", "Chromium"]);
|
|
39
|
+
|
|
40
|
+
const HINT_PLATFORMS = new Map<string, string>([
|
|
41
|
+
["android", "Android"],
|
|
42
|
+
["chrome os", "ChromeOS"],
|
|
43
|
+
["chromium os", "ChromeOS"],
|
|
44
|
+
["ios", "iOS"],
|
|
45
|
+
["linux", "Linux"],
|
|
46
|
+
["macos", "macOS"],
|
|
47
|
+
["windows", "Windows"],
|
|
48
|
+
]);
|
|
49
|
+
|
|
50
|
+
// Ordered: every Chromium fork carries `Chrome/`, and every Chromium and Gecko
|
|
51
|
+
// browser on iOS carries `Safari/`, so the specific tokens must be tested
|
|
52
|
+
// first. Brave has no token at all and is only detectable through the hints.
|
|
53
|
+
const AGENT_BROWSERS: readonly (readonly [RegExp, string])[] = [
|
|
54
|
+
[/Edg(?:A|iOS)?\//, "Edge"],
|
|
55
|
+
[/(?:Firefox|FxiOS)\//, "Firefox"],
|
|
56
|
+
[/OPR\//, "Opera"],
|
|
57
|
+
[/SamsungBrowser\//, "Samsung Internet"],
|
|
58
|
+
[/Vivaldi\//, "Vivaldi"],
|
|
59
|
+
[/YaBrowser\//, "Yandex"],
|
|
60
|
+
[/(?:Chrome|CriOS)\//, "Chrome"],
|
|
61
|
+
[/Safari\//, "Safari"],
|
|
62
|
+
];
|
|
63
|
+
|
|
64
|
+
// Ordered: Android and ChromeOS user agents also carry the `Linux` token.
|
|
65
|
+
const AGENT_PLATFORMS: readonly (readonly [RegExp, string])[] = [
|
|
66
|
+
[/Android/, "Android"],
|
|
67
|
+
[/iPhone|iPad|iPod/, "iOS"],
|
|
68
|
+
[/CrOS/, "ChromeOS"],
|
|
69
|
+
[/Windows/, "Windows"],
|
|
70
|
+
[/Mac OS X/, "macOS"],
|
|
71
|
+
[/Linux/, "Linux"],
|
|
72
|
+
];
|
|
73
|
+
|
|
74
|
+
// Ordered: tablet user agents that carry `Android` must not be read as mobile.
|
|
75
|
+
const AGENT_DEVICES: readonly (readonly [RegExp, string])[] = [
|
|
76
|
+
[/iPad|Tablet/, "tablet"],
|
|
77
|
+
[/Mobile|Android|iPhone/, "mobile"],
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
export function classifyClient(
|
|
81
|
+
userAgent: string,
|
|
82
|
+
headers: Headers,
|
|
83
|
+
): ClientClassification {
|
|
84
|
+
const agent = classifyUserAgent(userAgent);
|
|
85
|
+
const brand = brandFromClientHints(headers.get("sec-ch-ua"));
|
|
86
|
+
const platform = platformFromClientHints(headers.get("sec-ch-ua-platform"));
|
|
87
|
+
const mobile = mobileFromClientHints(headers.get("sec-ch-ua-mobile"));
|
|
88
|
+
return {
|
|
89
|
+
browser: brand ?? agent.browser,
|
|
90
|
+
os: platform ?? agent.os,
|
|
91
|
+
device: deviceFromClientHints(mobile, platform) ?? agent.device,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export function classifyUserAgent(userAgent: string): ClientClassification {
|
|
96
|
+
return {
|
|
97
|
+
browser: matchAgent(AGENT_BROWSERS, userAgent) ?? "Other",
|
|
98
|
+
os: matchAgent(AGENT_PLATFORMS, userAgent) ?? "Other",
|
|
99
|
+
device: matchAgent(AGENT_DEVICES, userAgent) ?? "desktop",
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function matchAgent(
|
|
104
|
+
table: readonly (readonly [RegExp, string])[],
|
|
105
|
+
userAgent: string,
|
|
106
|
+
): string | undefined {
|
|
107
|
+
for (const [pattern, value] of table) {
|
|
108
|
+
if (pattern.test(userAgent)) return value;
|
|
109
|
+
}
|
|
110
|
+
return undefined;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function brandFromClientHints(header: string | null): string | undefined {
|
|
114
|
+
if (!header || header.length > MAX_HINT_LENGTH) return undefined;
|
|
115
|
+
let generic: string | undefined;
|
|
116
|
+
let seen = 0;
|
|
117
|
+
for (const match of header.matchAll(BRAND_PATTERN)) {
|
|
118
|
+
if (seen >= MAX_HINT_BRANDS) break;
|
|
119
|
+
seen += 1;
|
|
120
|
+
const brand = HINT_BRANDS.get(unquote(match[1]).trim().toLowerCase());
|
|
121
|
+
if (!brand) continue;
|
|
122
|
+
if (!GENERIC_BRANDS.has(brand)) return brand;
|
|
123
|
+
if (brand === "Chrome" || generic === undefined) generic = brand;
|
|
124
|
+
}
|
|
125
|
+
return generic;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function platformFromClientHints(header: string | null): string | undefined {
|
|
129
|
+
if (!header || header.length > MAX_HINT_LENGTH) return undefined;
|
|
130
|
+
const match = PLATFORM_PATTERN.exec(header);
|
|
131
|
+
if (!match) return undefined;
|
|
132
|
+
return HINT_PLATFORMS.get(unquote(match[1]).trim().toLowerCase());
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function mobileFromClientHints(header: string | null): boolean | undefined {
|
|
136
|
+
if (header === null) return undefined;
|
|
137
|
+
const value = header.trim();
|
|
138
|
+
if (value === "?1") return true;
|
|
139
|
+
if (value === "?0") return false;
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function deviceFromClientHints(
|
|
144
|
+
mobile: boolean | undefined,
|
|
145
|
+
platform: string | undefined,
|
|
146
|
+
): string | undefined {
|
|
147
|
+
if (mobile === undefined) return undefined;
|
|
148
|
+
if (mobile) return "mobile";
|
|
149
|
+
// Chrome on Android tablets reports `Sec-CH-UA-Mobile: ?0`. The explicit
|
|
150
|
+
// `Sec-CH-UA-Form-Factors` hint is high entropy, so it would need an
|
|
151
|
+
// `Accept-CH` opt-in and a permissions-policy delegation from every tracked
|
|
152
|
+
// site; "Android but not mobile" is the only tablet signal available here.
|
|
153
|
+
if (platform === "Android") return "tablet";
|
|
154
|
+
return "desktop";
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function unquote(value: string): string {
|
|
158
|
+
return value.replace(/\\(.)/g, "$1");
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
// Self-declared crawlers, link unfurlers, uptime monitors, HTTP libraries, and
|
|
162
|
+
// headless browsers. A URL inside the User-Agent is the crawler convention
|
|
163
|
+
// ("+https://…/bot.html"); no browser ships one. An empty User-Agent is left
|
|
164
|
+
// alone because some runtimes do not expose it at all, and a scripted browser
|
|
165
|
+
// that spoofs a stock User-Agent is indistinguishable from a person here.
|
|
166
|
+
const BOT_PATTERN =
|
|
167
|
+
/bot|crawl|spider|slurp|headless|phantomjs|lighthouse|pingdom|gtmetrix|uptime|monitor|scrap|fetch|curl\/|wget\/|python|java\/|go-http-client|okhttp|axios|libwww|httpclient|facebookexternalhit|whatsapp|embedly|quora link preview|preview|mediapartners|feedfetcher|validator|https?:\/\//i;
|
|
168
|
+
|
|
169
|
+
export function isKnownBot(userAgent: string): boolean {
|
|
170
|
+
return BOT_PATTERN.test(userAgent);
|
|
171
|
+
}
|
|
@@ -6,7 +6,17 @@ export const eventTypeValidator = v.union(
|
|
|
6
6
|
v.literal("custom"),
|
|
7
7
|
v.literal("conversion"),
|
|
8
8
|
v.literal("heartbeat"),
|
|
9
|
+
v.literal("leave"),
|
|
9
10
|
v.literal("outbound"),
|
|
11
|
+
v.literal("vital"),
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
export const vitalMetricValidator = v.union(
|
|
15
|
+
v.literal("LCP"),
|
|
16
|
+
v.literal("CLS"),
|
|
17
|
+
v.literal("INP"),
|
|
18
|
+
v.literal("FCP"),
|
|
19
|
+
v.literal("TTFB"),
|
|
10
20
|
);
|
|
11
21
|
|
|
12
22
|
export const propertyValueValidator = v.union(
|
|
@@ -37,6 +47,9 @@ export const trackerEventValidator = v.object({
|
|
|
37
47
|
revenueCents: v.optional(v.number()),
|
|
38
48
|
currency: v.optional(v.string()),
|
|
39
49
|
affiliateSlug: v.optional(v.string()),
|
|
50
|
+
// Web Vitals measurement: milliseconds, or CLS scaled by 1000. Only "vital"
|
|
51
|
+
// events carry it; sanitization strips it from every other type.
|
|
52
|
+
value: v.optional(v.number()),
|
|
40
53
|
});
|
|
41
54
|
|
|
42
55
|
export const ingestContextValidator = v.object({
|
|
@@ -47,6 +60,12 @@ export const ingestContextValidator = v.object({
|
|
|
47
60
|
browser: v.optional(v.string()),
|
|
48
61
|
os: v.optional(v.string()),
|
|
49
62
|
device: v.optional(v.string()),
|
|
63
|
+
visitorKey: v.optional(v.string()),
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
export const batchedEventValidator = trackerEventValidator.extend({
|
|
67
|
+
...ingestContextValidator.fields,
|
|
68
|
+
aggregateCountry: v.optional(v.string()),
|
|
50
69
|
});
|
|
51
70
|
|
|
52
71
|
export const siteFieldsValidator = v.object({
|
|
@@ -65,6 +84,7 @@ export const sessionFieldsValidator = v.object({
|
|
|
65
84
|
siteId: v.id("sites"),
|
|
66
85
|
sessionId: v.string(),
|
|
67
86
|
visitorId: v.string(),
|
|
87
|
+
visitorKey: v.optional(v.string()),
|
|
68
88
|
startedAt: v.number(),
|
|
69
89
|
lastSeenAt: v.number(),
|
|
70
90
|
entryPath: v.string(),
|
|
@@ -121,6 +141,7 @@ export const eventFieldsValidator = v.object({
|
|
|
121
141
|
|
|
122
142
|
export const dimensionTypeValidator = v.union(
|
|
123
143
|
v.literal("source"),
|
|
144
|
+
v.literal("campaign"),
|
|
124
145
|
v.literal("page"),
|
|
125
146
|
v.literal("country"),
|
|
126
147
|
v.literal("device"),
|
|
@@ -137,6 +158,12 @@ export const dimensionSlotValidator = v.object({
|
|
|
137
158
|
revenueCents: v.number(),
|
|
138
159
|
});
|
|
139
160
|
|
|
161
|
+
export const visitorSketchValidator = v.object({
|
|
162
|
+
version: v.literal(1),
|
|
163
|
+
precision: v.number(),
|
|
164
|
+
registers: v.bytes(),
|
|
165
|
+
});
|
|
166
|
+
|
|
140
167
|
export const aggregateFieldsValidator = v.object({
|
|
141
168
|
siteId: v.id("sites"),
|
|
142
169
|
granularity: v.union(v.literal("hour"), v.literal("day")),
|
|
@@ -149,6 +176,8 @@ export const aggregateFieldsValidator = v.object({
|
|
|
149
176
|
outboundClicks: v.number(),
|
|
150
177
|
sessions: v.number(),
|
|
151
178
|
visitors: v.number(),
|
|
179
|
+
visitorSketch: v.optional(visitorSketchValidator),
|
|
180
|
+
visitorSketchComplete: v.optional(v.boolean()),
|
|
152
181
|
conversions: v.number(),
|
|
153
182
|
revenueCents: v.number(),
|
|
154
183
|
dimensions: v.array(dimensionSlotValidator),
|
|
@@ -218,7 +247,6 @@ export const trustedConversionValidator = v.object({
|
|
|
218
247
|
properties: v.optional(eventPropertiesValidator),
|
|
219
248
|
});
|
|
220
249
|
|
|
221
|
-
|
|
222
250
|
export type TrackerEvent = Infer<typeof trackerEventValidator>;
|
|
223
251
|
export type IngestContext = Infer<typeof ingestContextValidator>;
|
|
224
252
|
export type EventProperties = Infer<typeof eventPropertiesValidator>;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { v } from "convex/values";
|
|
2
|
+
import { MAX_VISITOR_ALIASES } from "./constants.js";
|
|
3
|
+
import { fail } from "./errors.js";
|
|
4
|
+
import { sanitizeOpaqueId } from "./sanitize.js";
|
|
5
|
+
import type { Id } from "./_generated/dataModel.js";
|
|
6
|
+
import { mutation, type QueryCtx } from "./_generated/server.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Folds an anonymous visitor id into the pseudonymous identity the host
|
|
10
|
+
* supplies after signup or sign-in. Links are one level deep: an alias can
|
|
11
|
+
* belong to one identity, an identity can never become an alias, and an
|
|
12
|
+
* alias can never own aliases, so journeys stay bounded and unambiguous.
|
|
13
|
+
*/
|
|
14
|
+
export const link = mutation({
|
|
15
|
+
args: {
|
|
16
|
+
siteId: v.id("sites"),
|
|
17
|
+
visitorId: v.string(),
|
|
18
|
+
previousVisitorId: v.string(),
|
|
19
|
+
},
|
|
20
|
+
returns: v.object({ linked: v.boolean(), aliasCount: v.number() }),
|
|
21
|
+
handler: async (ctx, args) => {
|
|
22
|
+
const site = await ctx.db.get("sites", args.siteId);
|
|
23
|
+
if (!site) fail("NOT_FOUND", "site not found");
|
|
24
|
+
let visitorId: string;
|
|
25
|
+
let previousVisitorId: string;
|
|
26
|
+
try {
|
|
27
|
+
visitorId = sanitizeOpaqueId(args.visitorId, "visitorId");
|
|
28
|
+
previousVisitorId = sanitizeOpaqueId(
|
|
29
|
+
args.previousVisitorId,
|
|
30
|
+
"previousVisitorId",
|
|
31
|
+
);
|
|
32
|
+
} catch (error) {
|
|
33
|
+
fail(
|
|
34
|
+
"INVALID_ARGUMENT",
|
|
35
|
+
error instanceof Error ? error.message : "invalid visitor id",
|
|
36
|
+
);
|
|
37
|
+
}
|
|
38
|
+
if (visitorId === previousVisitorId) {
|
|
39
|
+
fail("INVALID_ARGUMENT", "previousVisitorId must differ from visitorId");
|
|
40
|
+
}
|
|
41
|
+
if (await findAlias(ctx, args.siteId, visitorId)) {
|
|
42
|
+
fail("CONFLICT", "visitorId is already linked to another visitor");
|
|
43
|
+
}
|
|
44
|
+
const previousOwnsAliases = await ctx.db
|
|
45
|
+
.query("visitorAliases")
|
|
46
|
+
.withIndex("by_siteId_and_visitorId", (range) =>
|
|
47
|
+
range.eq("siteId", args.siteId).eq("visitorId", previousVisitorId),
|
|
48
|
+
)
|
|
49
|
+
.first();
|
|
50
|
+
if (previousOwnsAliases) {
|
|
51
|
+
fail("CONFLICT", "previousVisitorId already owns linked visitors");
|
|
52
|
+
}
|
|
53
|
+
const aliases = await ctx.db
|
|
54
|
+
.query("visitorAliases")
|
|
55
|
+
.withIndex("by_siteId_and_visitorId", (range) =>
|
|
56
|
+
range.eq("siteId", args.siteId).eq("visitorId", visitorId),
|
|
57
|
+
)
|
|
58
|
+
.take(MAX_VISITOR_ALIASES);
|
|
59
|
+
const existing = await findAlias(ctx, args.siteId, previousVisitorId);
|
|
60
|
+
if (existing) {
|
|
61
|
+
if (existing.visitorId !== visitorId) {
|
|
62
|
+
fail(
|
|
63
|
+
"CONFLICT",
|
|
64
|
+
"previousVisitorId is already linked to a different visitor",
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
return { linked: false, aliasCount: aliases.length };
|
|
68
|
+
}
|
|
69
|
+
if (aliases.length >= MAX_VISITOR_ALIASES) {
|
|
70
|
+
fail(
|
|
71
|
+
"LIMIT_EXCEEDED",
|
|
72
|
+
`a visitor can hold at most ${MAX_VISITOR_ALIASES} aliases`,
|
|
73
|
+
{
|
|
74
|
+
limit: MAX_VISITOR_ALIASES,
|
|
75
|
+
},
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
await ctx.db.insert("visitorAliases", {
|
|
79
|
+
siteId: args.siteId,
|
|
80
|
+
visitorId,
|
|
81
|
+
previousVisitorId,
|
|
82
|
+
linkedAt: Date.now(),
|
|
83
|
+
});
|
|
84
|
+
return { linked: true, aliasCount: aliases.length + 1 };
|
|
85
|
+
},
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
/** The identity a visitor id resolves to, followed by every alias it owns. */
|
|
89
|
+
export async function resolveVisitorIdentities(
|
|
90
|
+
ctx: QueryCtx,
|
|
91
|
+
siteId: Id<"sites">,
|
|
92
|
+
visitorId: string,
|
|
93
|
+
): Promise<string[]> {
|
|
94
|
+
const alias = await findAlias(ctx, siteId, visitorId);
|
|
95
|
+
const identity = alias?.visitorId ?? visitorId;
|
|
96
|
+
const aliases = await ctx.db
|
|
97
|
+
.query("visitorAliases")
|
|
98
|
+
.withIndex("by_siteId_and_visitorId", (range) =>
|
|
99
|
+
range.eq("siteId", siteId).eq("visitorId", identity),
|
|
100
|
+
)
|
|
101
|
+
.take(MAX_VISITOR_ALIASES);
|
|
102
|
+
return [identity, ...aliases.map((row) => row.previousVisitorId)];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function findAlias(
|
|
106
|
+
ctx: QueryCtx,
|
|
107
|
+
siteId: Id<"sites">,
|
|
108
|
+
previousVisitorId: string,
|
|
109
|
+
) {
|
|
110
|
+
return ctx.db
|
|
111
|
+
.query("visitorAliases")
|
|
112
|
+
.withIndex("by_siteId_and_previousVisitorId", (range) =>
|
|
113
|
+
range.eq("siteId", siteId).eq("previousVisitorId", previousVisitorId),
|
|
114
|
+
)
|
|
115
|
+
.unique();
|
|
116
|
+
}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import { VITAL_HISTOGRAM_EDGES, MAX_VITAL_VALUE } from "./constants.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Field-measured Web Vitals. Values are integers: milliseconds for the time
|
|
5
|
+
* metrics, and CLS scaled by 1000 so one shared histogram covers every metric.
|
|
6
|
+
*/
|
|
7
|
+
export const VITAL_METRICS = ["LCP", "CLS", "INP", "FCP", "TTFB"] as const;
|
|
8
|
+
|
|
9
|
+
export type VitalMetric = (typeof VITAL_METRICS)[number];
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The Google-published rating thresholds. Every threshold is also a histogram
|
|
13
|
+
* edge, so good/needs-improvement/poor counts are exact, never interpolated.
|
|
14
|
+
*/
|
|
15
|
+
export const VITAL_THRESHOLDS: Record<
|
|
16
|
+
VitalMetric,
|
|
17
|
+
{ good: number; poor: number }
|
|
18
|
+
> = {
|
|
19
|
+
LCP: { good: 2_500, poor: 4_000 },
|
|
20
|
+
CLS: { good: 100, poor: 250 },
|
|
21
|
+
INP: { good: 200, poor: 500 },
|
|
22
|
+
FCP: { good: 1_800, poor: 3_000 },
|
|
23
|
+
TTFB: { good: 800, poor: 1_800 },
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/** Sentinel row keys; sanitized paths always start with "/" so neither collides. */
|
|
27
|
+
export const VITAL_ALL = "(all)";
|
|
28
|
+
export const VITAL_OTHER_PAGES = "(other)";
|
|
29
|
+
|
|
30
|
+
/** Bounded device classes; anything else folds into "unknown". */
|
|
31
|
+
export const VITAL_DEVICES = ["desktop", "mobile", "tablet"] as const;
|
|
32
|
+
|
|
33
|
+
export function vitalDevice(device: string | undefined): string {
|
|
34
|
+
return device && (VITAL_DEVICES as readonly string[]).includes(device)
|
|
35
|
+
? device
|
|
36
|
+
: "unknown";
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export const VITAL_HISTOGRAM_BUCKETS = VITAL_HISTOGRAM_EDGES.length + 1;
|
|
40
|
+
|
|
41
|
+
export function isVitalMetric(value: string): value is VitalMetric {
|
|
42
|
+
return (VITAL_METRICS as readonly string[]).includes(value);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function isValidVitalValue(value: number): boolean {
|
|
46
|
+
return Number.isFinite(value) && value >= 0 && value <= MAX_VITAL_VALUE;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function emptyVitalHistogram(): number[] {
|
|
50
|
+
return Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Bucket i holds values in (edge[i-1], edge[i]]; the last bucket is open. */
|
|
54
|
+
export function vitalHistogramBucket(value: number): number {
|
|
55
|
+
for (let index = 0; index < VITAL_HISTOGRAM_EDGES.length; index += 1) {
|
|
56
|
+
if (value <= VITAL_HISTOGRAM_EDGES[index]) return index;
|
|
57
|
+
}
|
|
58
|
+
return VITAL_HISTOGRAM_EDGES.length;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function addToVitalHistogram(
|
|
62
|
+
histogram: number[],
|
|
63
|
+
value: number,
|
|
64
|
+
): number[] {
|
|
65
|
+
const next = normalizedVitalHistogram(histogram);
|
|
66
|
+
next[vitalHistogramBucket(value)] += 1;
|
|
67
|
+
return next;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function mergeVitalHistograms(
|
|
71
|
+
left: number[],
|
|
72
|
+
right: number[],
|
|
73
|
+
): number[] {
|
|
74
|
+
const result = normalizedVitalHistogram(left);
|
|
75
|
+
const addition = normalizedVitalHistogram(right);
|
|
76
|
+
for (let index = 0; index < result.length; index += 1) {
|
|
77
|
+
result[index] += addition[index];
|
|
78
|
+
}
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Percentile estimated by linear interpolation inside the winning bucket. The
|
|
84
|
+
* top bucket is open-ended, so estimates saturate at the highest edge; a
|
|
85
|
+
* reported p75 equal to that edge means "at least this much".
|
|
86
|
+
*/
|
|
87
|
+
export function vitalHistogramPercentile(
|
|
88
|
+
histogram: number[],
|
|
89
|
+
fraction: number,
|
|
90
|
+
): number {
|
|
91
|
+
const buckets = normalizedVitalHistogram(histogram);
|
|
92
|
+
const total = buckets.reduce((sum, count) => sum + count, 0);
|
|
93
|
+
if (total === 0) return 0;
|
|
94
|
+
const target = Math.ceil(total * fraction);
|
|
95
|
+
let cumulative = 0;
|
|
96
|
+
for (let index = 0; index < buckets.length; index += 1) {
|
|
97
|
+
const count = buckets[index];
|
|
98
|
+
if (count === 0) continue;
|
|
99
|
+
if (cumulative + count >= target) {
|
|
100
|
+
const lower = index === 0 ? 0 : VITAL_HISTOGRAM_EDGES[index - 1];
|
|
101
|
+
const upper =
|
|
102
|
+
index < VITAL_HISTOGRAM_EDGES.length
|
|
103
|
+
? VITAL_HISTOGRAM_EDGES[index]
|
|
104
|
+
: VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
|
|
105
|
+
const position = (target - cumulative) / count;
|
|
106
|
+
return Math.round(Math.min(upper, lower + position * (upper - lower)));
|
|
107
|
+
}
|
|
108
|
+
cumulative += count;
|
|
109
|
+
}
|
|
110
|
+
return VITAL_HISTOGRAM_EDGES[VITAL_HISTOGRAM_EDGES.length - 1];
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Exact rating counts; every threshold is a bucket edge by construction. */
|
|
114
|
+
export function vitalRatingCounts(
|
|
115
|
+
histogram: number[],
|
|
116
|
+
metric: VitalMetric,
|
|
117
|
+
): { good: number; needsImprovement: number; poor: number } {
|
|
118
|
+
const buckets = normalizedVitalHistogram(histogram);
|
|
119
|
+
const thresholds = VITAL_THRESHOLDS[metric];
|
|
120
|
+
let good = 0;
|
|
121
|
+
let needsImprovement = 0;
|
|
122
|
+
let poor = 0;
|
|
123
|
+
for (let index = 0; index < buckets.length; index += 1) {
|
|
124
|
+
const upper =
|
|
125
|
+
index < VITAL_HISTOGRAM_EDGES.length
|
|
126
|
+
? VITAL_HISTOGRAM_EDGES[index]
|
|
127
|
+
: Number.POSITIVE_INFINITY;
|
|
128
|
+
if (upper <= thresholds.good) good += buckets[index];
|
|
129
|
+
else if (upper <= thresholds.poor) needsImprovement += buckets[index];
|
|
130
|
+
else poor += buckets[index];
|
|
131
|
+
}
|
|
132
|
+
return { good, needsImprovement, poor };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function normalizedVitalHistogram(histogram: number[]): number[] {
|
|
136
|
+
const result = Array.from({ length: VITAL_HISTOGRAM_BUCKETS }, () => 0);
|
|
137
|
+
for (
|
|
138
|
+
let index = 0;
|
|
139
|
+
index < Math.min(histogram.length, result.length);
|
|
140
|
+
index += 1
|
|
141
|
+
) {
|
|
142
|
+
const count = histogram[index];
|
|
143
|
+
result[index] = Number.isSafeInteger(count) && count > 0 ? count : 0;
|
|
144
|
+
}
|
|
145
|
+
return result;
|
|
146
|
+
}
|