@iann29/rastro 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +122 -17
- package/agent/integration.md +26 -10
- package/agent/manifest.json +10 -5
- package/agent/manifest.schema.json +14 -5
- package/dist/client/federation.d.ts +28 -8
- package/dist/client/federation.d.ts.map +1 -1
- package/dist/client/federation.js +12 -1
- package/dist/client/federation.js.map +1 -1
- package/dist/client/index.d.ts +262 -9
- package/dist/client/index.d.ts.map +1 -1
- package/dist/client/index.js +119 -2
- package/dist/client/index.js.map +1 -1
- package/dist/component/_generated/api.d.ts +6 -0
- package/dist/component/_generated/api.d.ts.map +1 -1
- package/dist/component/_generated/api.js.map +1 -1
- package/dist/component/_generated/component.d.ts +71 -1
- package/dist/component/_generated/component.d.ts.map +1 -1
- package/dist/component/constants.d.ts +2 -0
- package/dist/component/constants.d.ts.map +1 -1
- package/dist/component/constants.js +7 -0
- package/dist/component/constants.js.map +1 -1
- package/dist/component/coverage.d.ts +1 -0
- package/dist/component/coverage.d.ts.map +1 -1
- package/dist/component/coverage.js +6 -1
- package/dist/component/coverage.js.map +1 -1
- package/dist/component/eventStore.d.ts +2 -0
- package/dist/component/eventStore.d.ts.map +1 -1
- package/dist/component/http.d.ts.map +1 -1
- package/dist/component/http.js +51 -1
- package/dist/component/http.js.map +1 -1
- package/dist/component/ingest.d.ts +2 -0
- package/dist/component/ingest.d.ts.map +1 -1
- package/dist/component/ingest.js +42 -3
- package/dist/component/ingest.js.map +1 -1
- package/dist/component/origin.d.ts +70 -0
- package/dist/component/origin.d.ts.map +1 -0
- package/dist/component/origin.js +230 -0
- package/dist/component/origin.js.map +1 -0
- package/dist/component/platforms.d.ts +33 -0
- package/dist/component/platforms.d.ts.map +1 -0
- package/dist/component/platforms.js +328 -0
- package/dist/component/platforms.js.map +1 -0
- package/dist/component/reports.d.ts +27 -72
- package/dist/component/reports.d.ts.map +1 -1
- package/dist/component/reports.js +51 -10
- package/dist/component/reports.js.map +1 -1
- package/dist/component/sanitize.d.ts +5 -0
- package/dist/component/sanitize.d.ts.map +1 -1
- package/dist/component/sanitize.js +33 -1
- package/dist/component/sanitize.js.map +1 -1
- package/dist/component/schema.d.ts +62 -7
- package/dist/component/schema.js +16 -1
- package/dist/component/schema.js.map +1 -1
- package/dist/component/trackedLinks.d.ts +91 -0
- package/dist/component/trackedLinks.d.ts.map +1 -0
- package/dist/component/trackedLinks.js +314 -0
- package/dist/component/trackedLinks.js.map +1 -0
- package/dist/component/useragent.d.ts +6 -0
- package/dist/component/useragent.d.ts.map +1 -1
- package/dist/component/useragent.js +9 -0
- package/dist/component/useragent.js.map +1 -1
- package/dist/component/validators.d.ts +51 -11
- package/dist/component/validators.d.ts.map +1 -1
- package/dist/component/validators.js +35 -1
- package/dist/component/validators.js.map +1 -1
- package/dist/tracker/generated.d.ts +6 -6
- package/dist/tracker/generated.d.ts.map +1 -1
- package/dist/tracker/generated.js +6 -6
- package/dist/tracker/generated.js.map +1 -1
- package/dist/tracker/tracker.d.ts +1 -0
- package/dist/tracker/tracker.d.ts.map +1 -1
- package/dist/tracker/tracker.js +16 -4
- package/dist/tracker/tracker.js.map +1 -1
- package/dist/tracker.min.js +1 -1
- package/docs/upgrading.md +152 -19
- package/llms.txt +6 -1
- package/package.json +2 -2
- package/src/component/_generated/api.ts +6 -0
- package/src/component/_generated/component.ts +152 -1
- package/src/component/constants.ts +7 -0
- package/src/component/coverage.ts +7 -1
- package/src/component/http.ts +56 -1
- package/src/component/ingest.ts +54 -1
- package/src/component/origin.ts +273 -0
- package/src/component/platforms.ts +359 -0
- package/src/component/reports.ts +74 -10
- package/src/component/sanitize.ts +39 -1
- package/src/component/schema.ts +18 -0
- package/src/component/trackedLinks.ts +384 -0
- package/src/component/useragent.ts +11 -0
- package/src/component/validators.ts +71 -0
- package/src/tracker/generated.ts +6 -6
package/src/component/reports.ts
CHANGED
|
@@ -55,6 +55,8 @@ import {
|
|
|
55
55
|
type RawJourneyEvent,
|
|
56
56
|
} from "./eventStore.js";
|
|
57
57
|
import { isPlainRecord } from "./guards.js";
|
|
58
|
+
import { classifyOrigin } from "./origin.js";
|
|
59
|
+
import { externalReferrer } from "./sanitize.js";
|
|
58
60
|
import {
|
|
59
61
|
affiliateFieldsValidator,
|
|
60
62
|
eventFieldsValidator,
|
|
@@ -220,6 +222,11 @@ export const overview = query({
|
|
|
220
222
|
topMediums: v.array(topItemValidator),
|
|
221
223
|
// Destination hosts of outbound clicks, ranked by clicks.
|
|
222
224
|
topOutbound: v.array(topItemValidator),
|
|
225
|
+
// The session's origin (origin.ts), ranked like sources: platform ids,
|
|
226
|
+
// channel ids, and the evidence that decided them.
|
|
227
|
+
topPlatforms: v.array(topItemValidator),
|
|
228
|
+
topChannels: v.array(topItemValidator),
|
|
229
|
+
topEvidence: v.array(topItemValidator),
|
|
223
230
|
metadata: v.object({
|
|
224
231
|
visitors: v.object({
|
|
225
232
|
basis: v.union(
|
|
@@ -242,6 +249,9 @@ export const overview = query({
|
|
|
242
249
|
topGoals: v.literal("goalCompletionVolume"),
|
|
243
250
|
topMediums: breakdownBasisValidator,
|
|
244
251
|
topOutbound: v.literal("eventVolume"),
|
|
252
|
+
topPlatforms: breakdownBasisValidator,
|
|
253
|
+
topChannels: breakdownBasisValidator,
|
|
254
|
+
topEvidence: breakdownBasisValidator,
|
|
245
255
|
}),
|
|
246
256
|
// Since when every site in the request records the medium and outbound
|
|
247
257
|
// slots; null while one of them has not stamped it. A range that starts
|
|
@@ -536,6 +546,9 @@ export const overview = query({
|
|
|
536
546
|
topGoals: top("goal", "eventVolume"),
|
|
537
547
|
topMediums: top("medium", sessionBasis),
|
|
538
548
|
topOutbound: top("outbound", "eventVolume"),
|
|
549
|
+
topPlatforms: top("platform", sessionBasis),
|
|
550
|
+
topChannels: top("channel", sessionBasis),
|
|
551
|
+
topEvidence: top("evidence", sessionBasis),
|
|
539
552
|
metadata: {
|
|
540
553
|
visitors: {
|
|
541
554
|
basis: visitorBasis,
|
|
@@ -555,6 +568,9 @@ export const overview = query({
|
|
|
555
568
|
topGoals: "goalCompletionVolume" as const,
|
|
556
569
|
topMediums: sessionBasis,
|
|
557
570
|
topOutbound: "eventVolume" as const,
|
|
571
|
+
topPlatforms: sessionBasis,
|
|
572
|
+
topChannels: sessionBasis,
|
|
573
|
+
topEvidence: sessionBasis,
|
|
558
574
|
},
|
|
559
575
|
dimensionsSince: controls.every(
|
|
560
576
|
(control) => control?.dimensionsSince !== undefined,
|
|
@@ -948,6 +964,27 @@ async function readRouteRollups(
|
|
|
948
964
|
return { rows, completeFrom: cutAt + DAY_MS };
|
|
949
965
|
}
|
|
950
966
|
|
|
967
|
+
type SessionRow = Omit<
|
|
968
|
+
Doc<"sessions">,
|
|
969
|
+
"geoLookupAttemptedAt" | "lastPageviewAt"
|
|
970
|
+
>;
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* A session recorded before origins existed, classified on read from what it
|
|
974
|
+
* kept (utm fields, referrer, affiliate) and marked `legacy`: it carries no
|
|
975
|
+
* click IDs or in-app signal, and it never enters the aggregates.
|
|
976
|
+
*/
|
|
977
|
+
function withOrigin(session: SessionRow, domains: string[]): SessionRow {
|
|
978
|
+
if (session.platform !== undefined) return session;
|
|
979
|
+
const origin = classifyOrigin({
|
|
980
|
+
utmSource: session.utmSource,
|
|
981
|
+
utmMedium: session.utmMedium,
|
|
982
|
+
referrer: externalReferrer(session.referrer, domains),
|
|
983
|
+
affiliate: session.affiliateSlug !== undefined,
|
|
984
|
+
});
|
|
985
|
+
return { ...session, ...origin, evidence: "legacy" };
|
|
986
|
+
}
|
|
987
|
+
|
|
951
988
|
/** Metadata for a selected session, independent of list pagination. */
|
|
952
989
|
export const getSession = query({
|
|
953
990
|
args: { siteId: v.id("sites"), sessionId: v.string() },
|
|
@@ -965,7 +1002,8 @@ export const getSession = query({
|
|
|
965
1002
|
lastPageviewAt: _pageview,
|
|
966
1003
|
...result
|
|
967
1004
|
} = session;
|
|
968
|
-
|
|
1005
|
+
const site = await ctx.db.get("sites", args.siteId);
|
|
1006
|
+
return withOrigin(result, site?.domains ?? []);
|
|
969
1007
|
},
|
|
970
1008
|
});
|
|
971
1009
|
|
|
@@ -1104,12 +1142,13 @@ export const listSessions = query({
|
|
|
1104
1142
|
.take(take)),
|
|
1105
1143
|
);
|
|
1106
1144
|
}
|
|
1145
|
+
const site = await ctx.db.get("sites", args.siteId);
|
|
1107
1146
|
const publicRows = rows.map(
|
|
1108
1147
|
({
|
|
1109
1148
|
geoLookupAttemptedAt: _attemptedAt,
|
|
1110
1149
|
lastPageviewAt: _lastPageviewAt,
|
|
1111
1150
|
...row
|
|
1112
|
-
}) => row,
|
|
1151
|
+
}) => withOrigin(row, site?.domains ?? []),
|
|
1113
1152
|
);
|
|
1114
1153
|
return keysetPaginationResult(publicRows, pagination, (row) => ({
|
|
1115
1154
|
timestamp: row.lastSeenAt,
|
|
@@ -1826,6 +1865,7 @@ const coverageDatasetValidator = v.union(
|
|
|
1826
1865
|
v.literal("affiliates"),
|
|
1827
1866
|
v.literal("vitals"),
|
|
1828
1867
|
v.literal("siteMap"),
|
|
1868
|
+
v.literal("origin"),
|
|
1829
1869
|
);
|
|
1830
1870
|
|
|
1831
1871
|
export const dataCoverage = query({
|
|
@@ -1859,6 +1899,7 @@ export const dataCoverage = query({
|
|
|
1859
1899
|
"affiliates",
|
|
1860
1900
|
"vitals",
|
|
1861
1901
|
"siteMap",
|
|
1902
|
+
"origin",
|
|
1862
1903
|
] as const;
|
|
1863
1904
|
return {
|
|
1864
1905
|
siteId: args.siteId,
|
|
@@ -2163,7 +2204,26 @@ async function datasetCoverageForRange(
|
|
|
2163
2204
|
to: number,
|
|
2164
2205
|
) {
|
|
2165
2206
|
if (dataset === "siteMap") {
|
|
2166
|
-
return
|
|
2207
|
+
return stampedDatasetCoverage(
|
|
2208
|
+
ctx,
|
|
2209
|
+
siteId,
|
|
2210
|
+
from,
|
|
2211
|
+
to,
|
|
2212
|
+
"siteMapSince",
|
|
2213
|
+
siteMapCoverageDataset,
|
|
2214
|
+
);
|
|
2215
|
+
}
|
|
2216
|
+
// Origins exist in sessions and aggregate slots from the site's stamp, and
|
|
2217
|
+
// the overview reads them from the daily aggregates retention trims.
|
|
2218
|
+
if (dataset === "origin") {
|
|
2219
|
+
return stampedDatasetCoverage(
|
|
2220
|
+
ctx,
|
|
2221
|
+
siteId,
|
|
2222
|
+
from,
|
|
2223
|
+
to,
|
|
2224
|
+
"originSince",
|
|
2225
|
+
"overviewDay",
|
|
2226
|
+
);
|
|
2167
2227
|
}
|
|
2168
2228
|
// Vitals have no legacy fallback source; the rollup either covers the range
|
|
2169
2229
|
// or the coverage states say how much of it exists.
|
|
@@ -2205,23 +2265,26 @@ async function datasetCoverageForRange(
|
|
|
2205
2265
|
}
|
|
2206
2266
|
|
|
2207
2267
|
/**
|
|
2208
|
-
*
|
|
2209
|
-
* `siteMapSince` and ends where retention cut
|
|
2210
|
-
* narrow the window when the range exceeds
|
|
2268
|
+
* A stamped dataset's coverage without reading its rows: it starts at the
|
|
2269
|
+
* site's stamp (`siteMapSince`, `originSince`) and ends where retention cut
|
|
2270
|
+
* it. The site map report may still narrow the window when the range exceeds
|
|
2271
|
+
* its row budget.
|
|
2211
2272
|
*/
|
|
2212
|
-
async function
|
|
2273
|
+
async function stampedDatasetCoverage(
|
|
2213
2274
|
ctx: QueryCtx,
|
|
2214
2275
|
siteId: Id<"sites">,
|
|
2215
2276
|
from: number,
|
|
2216
2277
|
to: number,
|
|
2278
|
+
stamp: "siteMapSince" | "originSince",
|
|
2279
|
+
retainedDataset: string,
|
|
2217
2280
|
) {
|
|
2218
2281
|
const control = await ctx.db
|
|
2219
2282
|
.query("analyticsCoverage")
|
|
2220
2283
|
.withIndex("by_siteId", (range) => range.eq("siteId", siteId))
|
|
2221
2284
|
.unique();
|
|
2222
|
-
const since = control?.
|
|
2285
|
+
const since = control?.[stamp] ?? null;
|
|
2223
2286
|
const retainedBefore =
|
|
2224
|
-
control?.retained.find((item) => item.dataset ===
|
|
2287
|
+
control?.retained.find((item) => item.dataset === retainedDataset)
|
|
2225
2288
|
?.before ?? null;
|
|
2226
2289
|
const unavailable =
|
|
2227
2290
|
since === null ||
|
|
@@ -2252,7 +2315,8 @@ type PublicCoverageDataset =
|
|
|
2252
2315
|
| "funnels"
|
|
2253
2316
|
| "affiliates"
|
|
2254
2317
|
| "vitals"
|
|
2255
|
-
| "siteMap"
|
|
2318
|
+
| "siteMap"
|
|
2319
|
+
| "origin";
|
|
2256
2320
|
|
|
2257
2321
|
type CoverageSource =
|
|
2258
2322
|
| "overviewHour"
|
|
@@ -10,6 +10,7 @@ import type {
|
|
|
10
10
|
TrackerEvent,
|
|
11
11
|
} from "./validators.js";
|
|
12
12
|
import { isValidVitalValue, isVitalMetric } from "./vitals.js";
|
|
13
|
+
import { isPlatform, knownClickIds } from "./origin.js";
|
|
13
14
|
|
|
14
15
|
const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
|
|
15
16
|
const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
|
|
@@ -121,6 +122,27 @@ export function sanitizePublicUrl(value: string): string | undefined {
|
|
|
121
122
|
}
|
|
122
123
|
}
|
|
123
124
|
|
|
125
|
+
const ANDROID_PACKAGE = /^[a-z][a-z0-9_]*(?:\.[a-z0-9_]+)+$/;
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* A referrer as sanitizePublicUrl keeps it, or the Android app that opened
|
|
129
|
+
* the link: `android-app://<package>/…` keeps the package alone.
|
|
130
|
+
*/
|
|
131
|
+
function sanitizeReferrer(value: string): string | undefined {
|
|
132
|
+
const input = cleanString(value, 2048);
|
|
133
|
+
if (!input.toLowerCase().startsWith("android-app://")) {
|
|
134
|
+
return sanitizePublicUrl(input);
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
const pkg = new URL(input).hostname.toLowerCase();
|
|
138
|
+
return pkg.length <= 120 && ANDROID_PACKAGE.test(pkg)
|
|
139
|
+
? `android-app://${pkg}`
|
|
140
|
+
: undefined;
|
|
141
|
+
} catch {
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
124
146
|
/**
|
|
125
147
|
* The host of a public URL the way the outbound dimension names it: lowercase,
|
|
126
148
|
* without port, path, or query. A relative path has no host.
|
|
@@ -229,7 +251,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
|
|
|
229
251
|
referrer:
|
|
230
252
|
event.referrer === undefined
|
|
231
253
|
? undefined
|
|
232
|
-
:
|
|
254
|
+
: sanitizeReferrer(event.referrer),
|
|
233
255
|
timestamp: Math.trunc(event.timestamp),
|
|
234
256
|
sequence: event.sequence,
|
|
235
257
|
target:
|
|
@@ -246,6 +268,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
|
|
|
246
268
|
event.type === "vital" && event.value !== undefined
|
|
247
269
|
? Math.round(event.value)
|
|
248
270
|
: undefined,
|
|
271
|
+
clid: event.clid === undefined ? undefined : knownClickIds(event.clid),
|
|
249
272
|
};
|
|
250
273
|
}
|
|
251
274
|
|
|
@@ -353,6 +376,8 @@ export function sanitizeContext(
|
|
|
353
376
|
context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
|
|
354
377
|
? cleanString(context.visitorKey, 128) || undefined
|
|
355
378
|
: undefined,
|
|
379
|
+
inApp:
|
|
380
|
+
context?.inApp && isPlatform(context.inApp) ? context.inApp : undefined,
|
|
356
381
|
};
|
|
357
382
|
}
|
|
358
383
|
|
|
@@ -393,6 +418,19 @@ export function sourceFromReferrer(referrer: string | undefined): string {
|
|
|
393
418
|
}
|
|
394
419
|
}
|
|
395
420
|
|
|
421
|
+
/**
|
|
422
|
+
* The referrer unless it is internal: a relative path or one of the site's
|
|
423
|
+
* own domains never names where a session came from.
|
|
424
|
+
*/
|
|
425
|
+
export function externalReferrer(
|
|
426
|
+
referrer: string | undefined,
|
|
427
|
+
domains: string[],
|
|
428
|
+
): string | undefined {
|
|
429
|
+
if (!referrer || referrer.startsWith("/") || originAllowed(referrer, domains))
|
|
430
|
+
return undefined;
|
|
431
|
+
return referrer;
|
|
432
|
+
}
|
|
433
|
+
|
|
396
434
|
export function stableHash(value: string): number {
|
|
397
435
|
let hash = 2_166_136_261;
|
|
398
436
|
for (let index = 0; index < value.length; index += 1) {
|
package/src/component/schema.ts
CHANGED
|
@@ -8,6 +8,7 @@ import {
|
|
|
8
8
|
goalFieldsValidator,
|
|
9
9
|
sessionFieldsValidator,
|
|
10
10
|
siteFieldsValidator,
|
|
11
|
+
trackedLinkFieldsValidator,
|
|
11
12
|
visitorSketchValidator,
|
|
12
13
|
vitalMetricValidator,
|
|
13
14
|
} from "./validators.js";
|
|
@@ -268,6 +269,8 @@ export default defineSchema({
|
|
|
268
269
|
localDaySince: v.optional(v.number()),
|
|
269
270
|
// Since when the medium and outbound dimension slots are recorded.
|
|
270
271
|
dimensionsSince: v.optional(v.number()),
|
|
272
|
+
// Since when sessions carry their origin and the origin slots exist.
|
|
273
|
+
originSince: v.optional(v.number()),
|
|
271
274
|
retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
|
|
272
275
|
updatedAt: v.number(),
|
|
273
276
|
}).index("by_siteId", ["siteId"]),
|
|
@@ -371,6 +374,21 @@ export default defineSchema({
|
|
|
371
374
|
.index("by_siteId", ["siteId"])
|
|
372
375
|
.index("by_siteId_and_slug", ["siteId", "slug"]),
|
|
373
376
|
|
|
377
|
+
// Unique by slug across the deployment: the redirect route names no site.
|
|
378
|
+
trackedLinks: defineTable(trackedLinkFieldsValidator)
|
|
379
|
+
.index("by_siteId", ["siteId"])
|
|
380
|
+
.index("by_slug", ["slug"]),
|
|
381
|
+
|
|
382
|
+
// One row per link, UTC day and shard, plus each shard's all-time row
|
|
383
|
+
// under dayStart 0 (see trackedLinks.ts).
|
|
384
|
+
trackedLinkClicks: defineTable({
|
|
385
|
+
linkId: v.id("trackedLinks"),
|
|
386
|
+
dayStart: v.number(),
|
|
387
|
+
shard: v.number(),
|
|
388
|
+
clicks: v.number(),
|
|
389
|
+
bots: v.number(),
|
|
390
|
+
}).index("by_linkId_and_dayStart_and_shard", ["linkId", "dayStart", "shard"]),
|
|
391
|
+
|
|
374
392
|
visitorSecrets: defineTable({
|
|
375
393
|
key: v.literal("current"),
|
|
376
394
|
value: v.string(),
|
|
@@ -0,0 +1,384 @@
|
|
|
1
|
+
import { v, type Infer } from "convex/values";
|
|
2
|
+
import { internal } from "./_generated/api.js";
|
|
3
|
+
import { internalMutation, mutation, query } from "./_generated/server.js";
|
|
4
|
+
import {
|
|
5
|
+
DAY_MS,
|
|
6
|
+
MAX_TRACKED_LINKS_PER_SITE,
|
|
7
|
+
TRACKED_LINK_CLICK_SHARDS,
|
|
8
|
+
} from "./constants.js";
|
|
9
|
+
import { fail } from "./errors.js";
|
|
10
|
+
import { cleanString, originAllowed, sanitizeOpaqueId } from "./sanitize.js";
|
|
11
|
+
import {
|
|
12
|
+
trackedLinkChannelValidator,
|
|
13
|
+
trackedLinkFieldsValidator,
|
|
14
|
+
} from "./validators.js";
|
|
15
|
+
|
|
16
|
+
const SLUG = /^[a-z0-9][a-z0-9-]{1,30}[a-z0-9]$/;
|
|
17
|
+
const PLATFORM = /^[a-z0-9_]{2,24}$/;
|
|
18
|
+
const MAX_DESTINATION_LENGTH = 2048;
|
|
19
|
+
const MAX_CAMPAIGN_LENGTH = 64;
|
|
20
|
+
const RECENT_DAYS = 30;
|
|
21
|
+
// Each shard's all-time counter sits beside the daily ones under this day, so
|
|
22
|
+
// a link's total is TRACKED_LINK_CLICK_SHARDS rows however long it has lived.
|
|
23
|
+
const ALL_TIME = 0;
|
|
24
|
+
const PURGE_BATCH = 500;
|
|
25
|
+
|
|
26
|
+
type Channel = Infer<typeof trackedLinkChannelValidator>;
|
|
27
|
+
|
|
28
|
+
const trackedLinkDocumentValidator = trackedLinkFieldsValidator.extend({
|
|
29
|
+
_id: v.id("trackedLinks"),
|
|
30
|
+
_creationTime: v.number(),
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const redirectValidator = trackedLinkFieldsValidator.pick(
|
|
34
|
+
"slug",
|
|
35
|
+
"destination",
|
|
36
|
+
"platform",
|
|
37
|
+
"channel",
|
|
38
|
+
"campaign",
|
|
39
|
+
);
|
|
40
|
+
|
|
41
|
+
export function isTrackedLinkSlug(value: string): boolean {
|
|
42
|
+
return SLUG.test(value);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Where a click lands: the destination with the link's utm_* appended. A key
|
|
47
|
+
* the destination already carries wins, and its own query keeps its exact
|
|
48
|
+
* encoding because the tags are appended as text.
|
|
49
|
+
*/
|
|
50
|
+
export function trackedLinkLocation(
|
|
51
|
+
link: Infer<typeof redirectValidator>,
|
|
52
|
+
): string {
|
|
53
|
+
const url = new URL(link.destination);
|
|
54
|
+
const utm: Array<[string, string]> = [
|
|
55
|
+
["utm_source", link.platform],
|
|
56
|
+
["utm_medium", link.channel],
|
|
57
|
+
["utm_campaign", link.campaign ?? link.slug],
|
|
58
|
+
["utm_content", link.slug],
|
|
59
|
+
];
|
|
60
|
+
const tags = new URLSearchParams(
|
|
61
|
+
utm.filter(([key]) => !url.searchParams.has(key)),
|
|
62
|
+
).toString();
|
|
63
|
+
if (tags) url.search = url.search ? `${url.search}&${tags}` : `?${tags}`;
|
|
64
|
+
return url.href;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function sanitizeLinkSlug(value: string): string {
|
|
68
|
+
const slug = value.trim().toLowerCase();
|
|
69
|
+
if (!isTrackedLinkSlug(slug)) {
|
|
70
|
+
fail(
|
|
71
|
+
"INVALID_ARGUMENT",
|
|
72
|
+
"slug must be 3 to 32 lowercase letters, digits or hyphens, starting and ending with a letter or digit",
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
return slug;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function sanitizePlatform(value: string): string {
|
|
79
|
+
const platform = value.trim().toLowerCase();
|
|
80
|
+
if (!PLATFORM.test(platform)) {
|
|
81
|
+
fail(
|
|
82
|
+
"INVALID_ARGUMENT",
|
|
83
|
+
"platform must be 2 to 24 lowercase letters, digits or underscores",
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
return platform;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function sanitizeCampaign(value: string | undefined): string | undefined {
|
|
90
|
+
return value === undefined
|
|
91
|
+
? undefined
|
|
92
|
+
: cleanString(value, MAX_CAMPAIGN_LENGTH) || undefined;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// No open redirect: the destination is an absolute http(s) URL on one of the
|
|
96
|
+
// site's own domains, the same rule the ingestion origin check applies.
|
|
97
|
+
function sanitizeDestination(value: string, domains: string[]): string {
|
|
98
|
+
let url: URL | undefined;
|
|
99
|
+
try {
|
|
100
|
+
if (value.length <= MAX_DESTINATION_LENGTH) {
|
|
101
|
+
url = new URL(cleanString(value, MAX_DESTINATION_LENGTH));
|
|
102
|
+
}
|
|
103
|
+
} catch {
|
|
104
|
+
url = undefined;
|
|
105
|
+
}
|
|
106
|
+
if (
|
|
107
|
+
!url ||
|
|
108
|
+
(url.protocol !== "http:" && url.protocol !== "https:") ||
|
|
109
|
+
url.username ||
|
|
110
|
+
url.password
|
|
111
|
+
) {
|
|
112
|
+
fail(
|
|
113
|
+
"INVALID_ARGUMENT",
|
|
114
|
+
"destination must be an absolute http(s) URL without credentials",
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
if (!originAllowed(url.href, domains)) {
|
|
118
|
+
fail(
|
|
119
|
+
"INVALID_ARGUMENT",
|
|
120
|
+
"destination must be on one of the site's domains",
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
return url.href;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export const create = mutation({
|
|
127
|
+
args: {
|
|
128
|
+
siteId: v.id("sites"),
|
|
129
|
+
slug: v.string(),
|
|
130
|
+
destination: v.string(),
|
|
131
|
+
platform: v.string(),
|
|
132
|
+
channel: trackedLinkChannelValidator,
|
|
133
|
+
campaign: v.optional(v.string()),
|
|
134
|
+
createdBy: v.string(),
|
|
135
|
+
},
|
|
136
|
+
returns: v.id("trackedLinks"),
|
|
137
|
+
handler: async (ctx, args) => {
|
|
138
|
+
const site = await ctx.db.get("sites", args.siteId);
|
|
139
|
+
if (!site) fail("NOT_FOUND", "site not found");
|
|
140
|
+
const slug = sanitizeLinkSlug(args.slug);
|
|
141
|
+
const destination = sanitizeDestination(args.destination, site.domains);
|
|
142
|
+
const platform = sanitizePlatform(args.platform);
|
|
143
|
+
let createdBy: string;
|
|
144
|
+
try {
|
|
145
|
+
createdBy = sanitizeOpaqueId(args.createdBy, "createdBy");
|
|
146
|
+
} catch (error) {
|
|
147
|
+
fail(
|
|
148
|
+
"INVALID_ARGUMENT",
|
|
149
|
+
error instanceof Error ? error.message : "invalid createdBy",
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
const taken = await ctx.db
|
|
153
|
+
.query("trackedLinks")
|
|
154
|
+
.withIndex("by_slug", (range) => range.eq("slug", slug))
|
|
155
|
+
.unique();
|
|
156
|
+
if (taken) fail("CONFLICT", "another tracked link already uses this slug");
|
|
157
|
+
const links = await ctx.db
|
|
158
|
+
.query("trackedLinks")
|
|
159
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
160
|
+
.take(MAX_TRACKED_LINKS_PER_SITE);
|
|
161
|
+
if (links.length >= MAX_TRACKED_LINKS_PER_SITE) {
|
|
162
|
+
fail("LIMIT_EXCEEDED", "tracked link limit reached", {
|
|
163
|
+
limit: MAX_TRACKED_LINKS_PER_SITE,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
return await ctx.db.insert("trackedLinks", {
|
|
167
|
+
siteId: args.siteId,
|
|
168
|
+
slug,
|
|
169
|
+
destination,
|
|
170
|
+
platform,
|
|
171
|
+
channel: args.channel,
|
|
172
|
+
campaign: sanitizeCampaign(args.campaign),
|
|
173
|
+
createdAt: Date.now(),
|
|
174
|
+
createdBy,
|
|
175
|
+
});
|
|
176
|
+
},
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
/** Everything but the slug, which every link already printed carries. */
|
|
180
|
+
export const update = mutation({
|
|
181
|
+
args: {
|
|
182
|
+
siteId: v.id("sites"),
|
|
183
|
+
linkId: v.id("trackedLinks"),
|
|
184
|
+
destination: v.optional(v.string()),
|
|
185
|
+
platform: v.optional(v.string()),
|
|
186
|
+
channel: v.optional(trackedLinkChannelValidator),
|
|
187
|
+
campaign: v.optional(v.union(v.string(), v.null())),
|
|
188
|
+
disabled: v.optional(v.boolean()),
|
|
189
|
+
},
|
|
190
|
+
returns: v.null(),
|
|
191
|
+
handler: async (ctx, args) => {
|
|
192
|
+
const link = await ctx.db.get("trackedLinks", args.linkId);
|
|
193
|
+
if (!link || link.siteId !== args.siteId) {
|
|
194
|
+
fail("NOT_FOUND", "tracked link not found");
|
|
195
|
+
}
|
|
196
|
+
const patch: {
|
|
197
|
+
destination?: string;
|
|
198
|
+
platform?: string;
|
|
199
|
+
channel?: Channel;
|
|
200
|
+
campaign?: string;
|
|
201
|
+
disabledAt?: number;
|
|
202
|
+
} = {};
|
|
203
|
+
if (args.destination !== undefined) {
|
|
204
|
+
const site = await ctx.db.get("sites", args.siteId);
|
|
205
|
+
if (!site) fail("NOT_FOUND", "site not found");
|
|
206
|
+
patch.destination = sanitizeDestination(args.destination, site.domains);
|
|
207
|
+
}
|
|
208
|
+
if (args.platform !== undefined) {
|
|
209
|
+
patch.platform = sanitizePlatform(args.platform);
|
|
210
|
+
}
|
|
211
|
+
if (args.channel !== undefined) patch.channel = args.channel;
|
|
212
|
+
if (args.campaign !== undefined) {
|
|
213
|
+
patch.campaign = sanitizeCampaign(args.campaign ?? undefined);
|
|
214
|
+
}
|
|
215
|
+
if (args.disabled !== undefined) {
|
|
216
|
+
patch.disabledAt = args.disabled
|
|
217
|
+
? (link.disabledAt ?? Date.now())
|
|
218
|
+
: undefined;
|
|
219
|
+
}
|
|
220
|
+
await ctx.db.patch("trackedLinks", args.linkId, patch);
|
|
221
|
+
return null;
|
|
222
|
+
},
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
export const remove = mutation({
|
|
226
|
+
args: { siteId: v.id("sites"), linkId: v.id("trackedLinks") },
|
|
227
|
+
returns: v.null(),
|
|
228
|
+
handler: async (ctx, args): Promise<null> => {
|
|
229
|
+
const link = await ctx.db.get("trackedLinks", args.linkId);
|
|
230
|
+
if (!link || link.siteId !== args.siteId) {
|
|
231
|
+
fail("NOT_FOUND", "tracked link not found");
|
|
232
|
+
}
|
|
233
|
+
await ctx.db.delete("trackedLinks", args.linkId);
|
|
234
|
+
await ctx.scheduler.runAfter(0, internal.trackedLinks.purgeClicks, {
|
|
235
|
+
linkId: args.linkId,
|
|
236
|
+
});
|
|
237
|
+
return null;
|
|
238
|
+
},
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* The site's links, newest first, with their clicks and bot hits: all time,
|
|
243
|
+
* and over the 30 UTC days ending on the day of `now`. The caller passes the
|
|
244
|
+
* clock so the subscription does not go stale.
|
|
245
|
+
*/
|
|
246
|
+
export const list = query({
|
|
247
|
+
args: { siteId: v.id("sites"), now: v.number() },
|
|
248
|
+
returns: v.array(
|
|
249
|
+
trackedLinkDocumentValidator.extend({
|
|
250
|
+
clicks: v.number(),
|
|
251
|
+
bots: v.number(),
|
|
252
|
+
clicksLast30Days: v.number(),
|
|
253
|
+
botsLast30Days: v.number(),
|
|
254
|
+
}),
|
|
255
|
+
),
|
|
256
|
+
handler: async (ctx, args) => {
|
|
257
|
+
if (!Number.isSafeInteger(args.now) || args.now < RECENT_DAYS * DAY_MS) {
|
|
258
|
+
fail("INVALID_ARGUMENT", "now must be a Unix time in milliseconds");
|
|
259
|
+
}
|
|
260
|
+
const since =
|
|
261
|
+
Math.floor(args.now / DAY_MS) * DAY_MS - (RECENT_DAYS - 1) * DAY_MS;
|
|
262
|
+
const links = await ctx.db
|
|
263
|
+
.query("trackedLinks")
|
|
264
|
+
.withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
|
|
265
|
+
.order("desc")
|
|
266
|
+
.take(MAX_TRACKED_LINKS_PER_SITE);
|
|
267
|
+
const sum = (rows: Array<{ clicks: number; bots: number }>) =>
|
|
268
|
+
rows.reduce(
|
|
269
|
+
(total, row) => ({
|
|
270
|
+
clicks: total.clicks + row.clicks,
|
|
271
|
+
bots: total.bots + row.bots,
|
|
272
|
+
}),
|
|
273
|
+
{ clicks: 0, bots: 0 },
|
|
274
|
+
);
|
|
275
|
+
// Up to 4 + 120 counter rows per link, so a full site of 200 links whose
|
|
276
|
+
// every link fills every shard every day reads about 24,800 rows, inside
|
|
277
|
+
// the 32,000 documents a query may read. Raising either cap needs a
|
|
278
|
+
// finished day's shards folded into one row first.
|
|
279
|
+
return await Promise.all(
|
|
280
|
+
links.map(async (link) => {
|
|
281
|
+
const total = sum(
|
|
282
|
+
await ctx.db
|
|
283
|
+
.query("trackedLinkClicks")
|
|
284
|
+
.withIndex("by_linkId_and_dayStart_and_shard", (range) =>
|
|
285
|
+
range.eq("linkId", link._id).eq("dayStart", ALL_TIME),
|
|
286
|
+
)
|
|
287
|
+
.take(TRACKED_LINK_CLICK_SHARDS),
|
|
288
|
+
);
|
|
289
|
+
const recent = sum(
|
|
290
|
+
await ctx.db
|
|
291
|
+
.query("trackedLinkClicks")
|
|
292
|
+
.withIndex("by_linkId_and_dayStart_and_shard", (range) =>
|
|
293
|
+
range
|
|
294
|
+
.eq("linkId", link._id)
|
|
295
|
+
.gte("dayStart", since)
|
|
296
|
+
.lt("dayStart", since + RECENT_DAYS * DAY_MS),
|
|
297
|
+
)
|
|
298
|
+
.take(RECENT_DAYS * TRACKED_LINK_CLICK_SHARDS),
|
|
299
|
+
);
|
|
300
|
+
return {
|
|
301
|
+
...link,
|
|
302
|
+
clicks: total.clicks,
|
|
303
|
+
bots: total.bots,
|
|
304
|
+
clicksLast30Days: recent.clicks,
|
|
305
|
+
botsLast30Days: recent.bots,
|
|
306
|
+
};
|
|
307
|
+
}),
|
|
308
|
+
);
|
|
309
|
+
},
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
/**
|
|
313
|
+
* Counts one click on a live link and returns where it goes, or null for an
|
|
314
|
+
* unknown or disabled slug. Bots are counted apart and never as clicks. The
|
|
315
|
+
* destination is checked against the site's domains again, so a domain the
|
|
316
|
+
* site has since dropped stops redirecting.
|
|
317
|
+
*/
|
|
318
|
+
export const recordClick = internalMutation({
|
|
319
|
+
args: { slug: v.string(), bot: v.boolean() },
|
|
320
|
+
returns: v.union(redirectValidator, v.null()),
|
|
321
|
+
handler: async (ctx, args) => {
|
|
322
|
+
const link = await ctx.db
|
|
323
|
+
.query("trackedLinks")
|
|
324
|
+
.withIndex("by_slug", (range) => range.eq("slug", args.slug))
|
|
325
|
+
.unique();
|
|
326
|
+
if (!link || link.disabledAt !== undefined) return null;
|
|
327
|
+
const site = await ctx.db.get("sites", link.siteId);
|
|
328
|
+
if (!site || !originAllowed(link.destination, site.domains)) return null;
|
|
329
|
+
const now = Date.now();
|
|
330
|
+
const shard = Math.floor(Math.random() * TRACKED_LINK_CLICK_SHARDS);
|
|
331
|
+
for (const dayStart of [Math.floor(now / DAY_MS) * DAY_MS, ALL_TIME]) {
|
|
332
|
+
const row = await ctx.db
|
|
333
|
+
.query("trackedLinkClicks")
|
|
334
|
+
.withIndex("by_linkId_and_dayStart_and_shard", (range) =>
|
|
335
|
+
range
|
|
336
|
+
.eq("linkId", link._id)
|
|
337
|
+
.eq("dayStart", dayStart)
|
|
338
|
+
.eq("shard", shard),
|
|
339
|
+
)
|
|
340
|
+
.unique();
|
|
341
|
+
if (row) {
|
|
342
|
+
await ctx.db.patch(
|
|
343
|
+
"trackedLinkClicks",
|
|
344
|
+
row._id,
|
|
345
|
+
args.bot ? { bots: row.bots + 1 } : { clicks: row.clicks + 1 },
|
|
346
|
+
);
|
|
347
|
+
} else {
|
|
348
|
+
await ctx.db.insert("trackedLinkClicks", {
|
|
349
|
+
linkId: link._id,
|
|
350
|
+
dayStart,
|
|
351
|
+
shard,
|
|
352
|
+
clicks: args.bot ? 0 : 1,
|
|
353
|
+
bots: args.bot ? 1 : 0,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
slug: link.slug,
|
|
359
|
+
destination: link.destination,
|
|
360
|
+
platform: link.platform,
|
|
361
|
+
channel: link.channel,
|
|
362
|
+
...(link.campaign === undefined ? {} : { campaign: link.campaign }),
|
|
363
|
+
};
|
|
364
|
+
},
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
/** Deletes a removed link's counters a batch at a time. */
|
|
368
|
+
export const purgeClicks = internalMutation({
|
|
369
|
+
args: { linkId: v.id("trackedLinks") },
|
|
370
|
+
returns: v.null(),
|
|
371
|
+
handler: async (ctx, args): Promise<null> => {
|
|
372
|
+
const rows = await ctx.db
|
|
373
|
+
.query("trackedLinkClicks")
|
|
374
|
+
.withIndex("by_linkId_and_dayStart_and_shard", (range) =>
|
|
375
|
+
range.eq("linkId", args.linkId),
|
|
376
|
+
)
|
|
377
|
+
.take(PURGE_BATCH);
|
|
378
|
+
for (const row of rows) await ctx.db.delete("trackedLinkClicks", row._id);
|
|
379
|
+
if (rows.length === PURGE_BATCH) {
|
|
380
|
+
await ctx.scheduler.runAfter(0, internal.trackedLinks.purgeClicks, args);
|
|
381
|
+
}
|
|
382
|
+
return null;
|
|
383
|
+
},
|
|
384
|
+
});
|