@iann29/rastro 0.4.0 → 0.6.0

package/src/component/reports.ts

@@ -55,6 +55,8 @@ import {
   type RawJourneyEvent,
 } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
+import { classifyOrigin } from "./origin.js";
+import { externalReferrer } from "./sanitize.js";
 import {
   affiliateFieldsValidator,
   eventFieldsValidator,
@@ -62,8 +64,9 @@ import {
   goalFieldsValidator,
   sessionFieldsValidator,
   vitalMetricValidator,
+  visitorProfileValidator,
 } from "./validators.js";
-import { resolveVisitorIdentities } from "./visitors.js";
+import { resolveVisitorIdentities, withVisitorProfiles } from "./visitors.js";
 import {
   mergeVitalHistograms,
   VITAL_ALL,
@@ -74,6 +77,7 @@ import {
 } from "./vitals.js";
 
 const sessionDocumentValidator = sessionFieldsValidator.extend({
+  profile: v.optional(visitorProfileValidator),
   _id: v.id("sessions"),
   _creationTime: v.number(),
 });
@@ -82,6 +86,7 @@ const eventDocumentValidator = eventFieldsValidator.extend({
   _creationTime: v.number(),
 });
 const liveSessionDocumentValidator = v.object({
+  profile: v.optional(visitorProfileValidator),
   _id: v.id("liveSessions"),
   _creationTime: v.number(),
   siteId: v.id("sites"),
@@ -220,6 +225,11 @@ export const overview = query({
     topMediums: v.array(topItemValidator),
     // Destination hosts of outbound clicks, ranked by clicks.
     topOutbound: v.array(topItemValidator),
+    // The session's origin (origin.ts), ranked like sources: platform ids,
+    // channel ids, and the evidence that decided them.
+    topPlatforms: v.array(topItemValidator),
+    topChannels: v.array(topItemValidator),
+    topEvidence: v.array(topItemValidator),
     metadata: v.object({
       visitors: v.object({
         basis: v.union(
@@ -242,6 +252,9 @@ export const overview = query({
         topGoals: v.literal("goalCompletionVolume"),
         topMediums: breakdownBasisValidator,
         topOutbound: v.literal("eventVolume"),
+        topPlatforms: breakdownBasisValidator,
+        topChannels: breakdownBasisValidator,
+        topEvidence: breakdownBasisValidator,
       }),
       // Since when every site in the request records the medium and outbound
       // slots; null while one of them has not stamped it. A range that starts
@@ -536,6 +549,9 @@ export const overview = query({
       topGoals: top("goal", "eventVolume"),
       topMediums: top("medium", sessionBasis),
       topOutbound: top("outbound", "eventVolume"),
+      topPlatforms: top("platform", sessionBasis),
+      topChannels: top("channel", sessionBasis),
+      topEvidence: top("evidence", sessionBasis),
       metadata: {
         visitors: {
           basis: visitorBasis,
@@ -555,6 +571,9 @@ export const overview = query({
           topGoals: "goalCompletionVolume" as const,
           topMediums: sessionBasis,
           topOutbound: "eventVolume" as const,
+          topPlatforms: sessionBasis,
+          topChannels: sessionBasis,
+          topEvidence: sessionBasis,
         },
         dimensionsSince: controls.every(
           (control) => control?.dimensionsSince !== undefined,
@@ -612,7 +631,7 @@ export const liveVisitors = query({
           .take(limit)),
       );
     }
-    return rows
+    const visible = rows
       .sort((left, right) => right.lastSeenAt - left.lastSeenAt)
       .slice(0, limit)
       .map((row) => ({
@@ -646,6 +665,7 @@ export const liveVisitors = query({
           : {}),
         ...(row.funnel ? { funnel: row.funnel } : {}),
       }));
+    return withVisitorProfiles(ctx, visible);
   },
 });
 
@@ -948,6 +968,27 @@ async function readRouteRollups(
   return { rows, completeFrom: cutAt + DAY_MS };
 }
 
+type SessionRow = Omit<
+  Doc<"sessions">,
+  "geoLookupAttemptedAt" | "lastPageviewAt"
+>;
+
+/**
+ * A session recorded before origins existed, classified on read from what it
+ * kept (utm fields, referrer, affiliate) and marked `legacy`: it carries no
+ * click IDs or in-app signal, and it never enters the aggregates.
+ */
+function withOrigin(session: SessionRow, domains: string[]): SessionRow {
+  if (session.platform !== undefined) return session;
+  const origin = classifyOrigin({
+    utmSource: session.utmSource,
+    utmMedium: session.utmMedium,
+    referrer: externalReferrer(session.referrer, domains),
+    affiliate: session.affiliateSlug !== undefined,
+  });
+  return { ...session, ...origin, evidence: "legacy" };
+}
+
 /** Metadata for a selected session, independent of list pagination. */
 export const getSession = query({
   args: { siteId: v.id("sites"), sessionId: v.string() },
@@ -965,7 +1006,10 @@ export const getSession = query({
       lastPageviewAt: _pageview,
       ...result
     } = session;
-    return result;
+    const site = await ctx.db.get("sites", args.siteId);
+    return (
+      await withVisitorProfiles(ctx, [withOrigin(result, site?.domains ?? [])])
+    )[0];
   },
 });
 
@@ -1104,17 +1148,19 @@ export const listSessions = query({
           .take(take)),
       );
     }
+    const site = await ctx.db.get("sites", args.siteId);
     const publicRows = rows.map(
       ({
         geoLookupAttemptedAt: _attemptedAt,
         lastPageviewAt: _lastPageviewAt,
         ...row
-      }) => row,
+      }) => withOrigin(row, site?.domains ?? []),
     );
-    return keysetPaginationResult(publicRows, pagination, (row) => ({
+    const result = keysetPaginationResult(publicRows, pagination, (row) => ({
       timestamp: row.lastSeenAt,
       creationTime: row._creationTime,
     }));
+    return { ...result, page: await withVisitorProfiles(ctx, result.page) };
   },
 });
 
@@ -1826,6 +1872,7 @@ const coverageDatasetValidator = v.union(
   v.literal("affiliates"),
   v.literal("vitals"),
   v.literal("siteMap"),
+  v.literal("origin"),
 );
 
 export const dataCoverage = query({
@@ -1859,6 +1906,7 @@ export const dataCoverage = query({
       "affiliates",
       "vitals",
       "siteMap",
+      "origin",
     ] as const;
     return {
       siteId: args.siteId,
@@ -2163,7 +2211,26 @@ async function datasetCoverageForRange(
   to: number,
 ) {
   if (dataset === "siteMap") {
-    return siteMapDatasetCoverage(ctx, siteId, from, to);
+    return stampedDatasetCoverage(
+      ctx,
+      siteId,
+      from,
+      to,
+      "siteMapSince",
+      siteMapCoverageDataset,
+    );
+  }
+  // Origins exist in sessions and aggregate slots from the site's stamp, and
+  // the overview reads them from the daily aggregates retention trims.
+  if (dataset === "origin") {
+    return stampedDatasetCoverage(
+      ctx,
+      siteId,
+      from,
+      to,
+      "originSince",
+      "overviewDay",
+    );
   }
   // Vitals have no legacy fallback source; the rollup either covers the range
   // or the coverage states say how much of it exists.
@@ -2205,23 +2272,26 @@ async function datasetCoverageForRange(
 }
 
 /**
- * The site map's coverage without reading its rows: it starts at the site's
- * `siteMapSince` and ends where retention cut it. The report itself may still
- * narrow the window when the range exceeds its row budget.
+ * A stamped dataset's coverage without reading its rows: it starts at the
+ * site's stamp (`siteMapSince`, `originSince`) and ends where retention cut
+ * it. The site map report may still narrow the window when the range exceeds
+ * its row budget.
  */
-async function siteMapDatasetCoverage(
+async function stampedDatasetCoverage(
   ctx: QueryCtx,
   siteId: Id<"sites">,
   from: number,
   to: number,
+  stamp: "siteMapSince" | "originSince",
+  retainedDataset: string,
 ) {
   const control = await ctx.db
     .query("analyticsCoverage")
     .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
     .unique();
-  const since = control?.siteMapSince ?? null;
+  const since = control?.[stamp] ?? null;
   const retainedBefore =
-    control?.retained.find((item) => item.dataset === siteMapCoverageDataset)
+    control?.retained.find((item) => item.dataset === retainedDataset)
       ?.before ?? null;
   const unavailable =
     since === null ||
@@ -2252,7 +2322,8 @@ type PublicCoverageDataset =
   | "funnels"
   | "affiliates"
   | "vitals"
-  | "siteMap";
+  | "siteMap"
+  | "origin";
 
 type CoverageSource =
   | "overviewHour"

package/src/component/sanitize.ts

@@ -10,6 +10,7 @@ import type {
   TrackerEvent,
 } from "./validators.js";
 import { isValidVitalValue, isVitalMetric } from "./vitals.js";
+import { isPlatform, knownClickIds } from "./origin.js";
 
 const HOST_LABEL = /^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/;
 const SAFE_KEY = /^[A-Za-z][A-Za-z0-9_.-]{0,39}$/;
@@ -121,6 +122,27 @@ export function sanitizePublicUrl(value: string): string | undefined {
   }
 }
 
+const ANDROID_PACKAGE = /^[a-z][a-z0-9_]*(?:\.[a-z0-9_]+)+$/;
+
+/**
+ * A referrer as sanitizePublicUrl keeps it, or the Android app that opened
+ * the link: `android-app://<package>/…` keeps the package alone.
+ */
+function sanitizeReferrer(value: string): string | undefined {
+  const input = cleanString(value, 2048);
+  if (!input.toLowerCase().startsWith("android-app://")) {
+    return sanitizePublicUrl(input);
+  }
+  try {
+    const pkg = new URL(input).hostname.toLowerCase();
+    return pkg.length <= 120 && ANDROID_PACKAGE.test(pkg)
+      ? `android-app://${pkg}`
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 /**
  * The host of a public URL the way the outbound dimension names it: lowercase,
  * without port, path, or query. A relative path has no host.
@@ -229,7 +251,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
     referrer:
       event.referrer === undefined
         ? undefined
-        : sanitizePublicUrl(event.referrer),
+        : sanitizeReferrer(event.referrer),
     timestamp: Math.trunc(event.timestamp),
     sequence: event.sequence,
     target:
@@ -246,6 +268,7 @@ export function sanitizeEvent(event: TrackerEvent, now: number): TrackerEvent {
       event.type === "vital" && event.value !== undefined
         ? Math.round(event.value)
         : undefined,
+    clid: event.clid === undefined ? undefined : knownClickIds(event.clid),
   };
 }
 
@@ -353,6 +376,8 @@ export function sanitizeContext(
       context?.visitorKey && SAFE_OPAQUE_ID.test(context.visitorKey)
         ? cleanString(context.visitorKey, 128) || undefined
         : undefined,
+    inApp:
+      context?.inApp && isPlatform(context.inApp) ? context.inApp : undefined,
   };
 }
 
@@ -393,6 +418,19 @@ export function sourceFromReferrer(referrer: string | undefined): string {
   }
 }
 
+/**
+ * The referrer unless it is internal: a relative path or one of the site's
+ * own domains never names where a session came from.
+ */
+export function externalReferrer(
+  referrer: string | undefined,
+  domains: string[],
+): string | undefined {
+  if (!referrer || referrer.startsWith("/") || originAllowed(referrer, domains))
+    return undefined;
+  return referrer;
+}
+
 export function stableHash(value: string): number {
   let hash = 2_166_136_261;
   for (let index = 0; index < value.length; index += 1) {

package/src/component/schema.ts

@@ -8,7 +8,10 @@ import {
   goalFieldsValidator,
   sessionFieldsValidator,
   siteFieldsValidator,
+  trackedLinkFieldsValidator,
   visitorSketchValidator,
+  visitorProfileValidator,
+  personAttributeValueValidator,
   vitalMetricValidator,
 } from "./validators.js";
 import {
@@ -268,6 +271,8 @@ export default defineSchema({
     localDaySince: v.optional(v.number()),
     // Since when the medium and outbound dimension slots are recorded.
     dimensionsSince: v.optional(v.number()),
+    // Since when sessions carry their origin and the origin slots exist.
+    originSince: v.optional(v.number()),
     retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
     updatedAt: v.number(),
   }).index("by_siteId", ["siteId"]),
@@ -371,12 +376,60 @@ export default defineSchema({
     .index("by_siteId", ["siteId"])
     .index("by_siteId_and_slug", ["siteId", "slug"]),
 
+  // Unique by slug across the deployment: the redirect route names no site.
+  trackedLinks: defineTable(trackedLinkFieldsValidator)
+    .index("by_siteId", ["siteId"])
+    .index("by_slug", ["slug"]),
+
+  // One row per link, UTC day and shard, plus each shard's all-time row
+  // under dayStart 0 (see trackedLinks.ts).
+  trackedLinkClicks: defineTable({
+    linkId: v.id("trackedLinks"),
+    dayStart: v.number(),
+    shard: v.number(),
+    clicks: v.number(),
+    bots: v.number(),
+  }).index("by_linkId_and_dayStart_and_shard", ["linkId", "dayStart", "shard"]),
+
   visitorSecrets: defineTable({
     key: v.literal("current"),
     value: v.string(),
     createdAt: v.number(),
   }).index("by_key", ["key"]),
 
+  visitorProfiles: defineTable(
+    visitorProfileValidator.extend({
+      siteId: v.id("sites"),
+      searchName: v.optional(v.string()),
+      searchEmail: v.optional(v.string()),
+    }),
+  )
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"])
+    .index("by_siteId_and_searchName_and_visitorId", [
+      "siteId",
+      "searchName",
+      "visitorId",
+    ])
+    .index("by_siteId_and_searchEmail_and_visitorId", [
+      "siteId",
+      "searchEmail",
+      "visitorId",
+    ]),
+
+  visitorAttributes: defineTable({
+    siteId: v.id("sites"),
+    visitorId: v.string(),
+    key: v.string(),
+    value: personAttributeValueValidator,
+  })
+    .index("by_siteId_and_key_and_value_and_visitorId", [
+      "siteId",
+      "key",
+      "value",
+      "visitorId",
+    ])
+    .index("by_siteId_and_visitorId", ["siteId", "visitorId"]),
+
   visitorAliases: defineTable({
     siteId: v.id("sites"),
     visitorId: v.string(),