@iann29/rastro 0.3.0 → 0.5.0

package/src/component/_generated/component.ts

@@ -177,11 +177,13 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         "mutation",
         "internal",
         {
+          clientKey?: string;
           context?: {
             browser?: string;
             city?: string;
             country?: string;
             device?: string;
+            inApp?: string;
             latitude?: number;
             longitude?: number;
             os?: string;
@@ -189,6 +191,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           };
           events: Array<{
             affiliateSlug?: string;
+            clid?: Array<string>;
             currency?: string;
             eventId: string;
             href?: string;
@@ -309,7 +312,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "funnels"
               | "affiliates"
               | "vitals"
-              | "siteMap";
+              | "siteMap"
+              | "origin";
           }>;
           from: number;
           siteId: string;
@@ -374,6 +378,19 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           affiliateId?: string;
           affiliateSlug?: string;
           browser: string;
+          channel?:
+            | "direct"
+            | "organic_search"
+            | "paid_search"
+            | "organic_social"
+            | "paid_social"
+            | "messaging"
+            | "email"
+            | "affiliate"
+            | "ai"
+            | "display"
+            | "referral"
+            | "offline";
           city?: string;
           conversionCount: number;
           country?: string;
@@ -382,6 +399,14 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           durationMs: number;
           entryPath: string;
           eventCount: number;
+          evidence?:
+            | "utm"
+            | "clickId"
+            | "referrer"
+            | "inApp"
+            | "affiliate"
+            | "none"
+            | "legacy";
           exitPath: string;
           lastSeenAt: number;
           latitude?: number;
@@ -389,6 +414,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           newVisitor?: boolean;
           os: string;
           pageviewCount: number;
+          platform?: string;
           referrer?: string;
           revenueCents: number;
           sessionId: string;
@@ -512,6 +538,19 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             affiliateId?: string;
             affiliateSlug?: string;
             browser: string;
+            channel?:
+              | "direct"
+              | "organic_search"
+              | "paid_search"
+              | "organic_social"
+              | "paid_social"
+              | "messaging"
+              | "email"
+              | "affiliate"
+              | "ai"
+              | "display"
+              | "referral"
+              | "offline";
             city?: string;
             conversionCount: number;
             country?: string;
@@ -520,6 +559,14 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             durationMs: number;
             entryPath: string;
             eventCount: number;
+            evidence?:
+              | "utm"
+              | "clickId"
+              | "referrer"
+              | "inApp"
+              | "affiliate"
+              | "none"
+              | "legacy";
             exitPath: string;
             lastSeenAt: number;
             latitude?: number;
@@ -527,6 +574,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             newVisitor?: boolean;
             os: string;
             pageviewCount: number;
+            platform?: string;
             referrer?: string;
             revenueCents: number;
             sessionId: string;
@@ -596,13 +644,16 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             breakdowns: {
               topBrowsers: "pageviews" | "sessions" | "eventVolume";
               topCampaigns: "pageviews" | "sessions" | "eventVolume";
+              topChannels: "pageviews" | "sessions" | "eventVolume";
               topCountries: "pageviews" | "sessions" | "eventVolume";
               topDevices: "pageviews" | "sessions" | "eventVolume";
               topEvents: "eventVolume";
+              topEvidence: "pageviews" | "sessions" | "eventVolume";
               topGoals: "goalCompletionVolume";
               topMediums: "pageviews" | "sessions" | "eventVolume";
               topOutbound: "eventVolume";
               topPages: "pageviews" | "sessions" | "eventVolume";
+              topPlatforms: "pageviews" | "sessions" | "eventVolume";
               topSources: "pageviews" | "sessions" | "eventVolume";
             };
             coverage: {
@@ -660,6 +711,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topChannels: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topCountries: Array<{
             count: number;
             revenueCents: number;
@@ -675,6 +731,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topEvidence: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topGoals: Array<{
             count: number;
             revenueCents: number;
@@ -695,6 +756,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             revenueCents: number;
             value: string;
           }>;
+          topPlatforms: Array<{
+            count: number;
+            revenueCents: number;
+            value: string;
+          }>;
           topSources: Array<{
             count: number;
             revenueCents: number;
@@ -1139,6 +1205,92 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         Name
       >;
     };
+    trackedLinks: {
+      create: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          campaign?: string;
+          channel:
+            | "organic_social"
+            | "paid_social"
+            | "messaging"
+            | "email"
+            | "affiliate"
+            | "display"
+            | "referral"
+            | "offline";
+          createdBy: string;
+          destination: string;
+          platform: string;
+          siteId: string;
+          slug: string;
+        },
+        string,
+        Name
+      >;
+      list: FunctionReference<
+        "query",
+        "internal",
+        { now: number; siteId: string },
+        Array<{
+          _creationTime: number;
+          _id: string;
+          bots: number;
+          botsLast30Days: number;
+          campaign?: string;
+          channel:
+            | "organic_social"
+            | "paid_social"
+            | "messaging"
+            | "email"
+            | "affiliate"
+            | "display"
+            | "referral"
+            | "offline";
+          clicks: number;
+          clicksLast30Days: number;
+          createdAt: number;
+          createdBy: string;
+          destination: string;
+          disabledAt?: number;
+          platform: string;
+          siteId: string;
+          slug: string;
+        }>,
+        Name
+      >;
+      remove: FunctionReference<
+        "mutation",
+        "internal",
+        { linkId: string; siteId: string },
+        null,
+        Name
+      >;
+      update: FunctionReference<
+        "mutation",
+        "internal",
+        {
+          campaign?: string | null;
+          channel?:
+            | "organic_social"
+            | "paid_social"
+            | "messaging"
+            | "email"
+            | "affiliate"
+            | "display"
+            | "referral"
+            | "offline";
+          destination?: string;
+          disabled?: boolean;
+          linkId: string;
+          platform?: string;
+          siteId: string;
+        },
+        null,
+        Name
+      >;
+    };
     visitors: {
       link: FunctionReference<
         "mutation",

package/src/component/_generated/server.ts

@@ -39,6 +39,7 @@ import type { DataModel } from "./dataModel.js";
 type Env = {
   readonly CONVEX_CLOUD_URL: string;
   readonly CONVEX_SITE_URL: string;
+  readonly RASTRO_CLIENT_RATE_LIMIT: "off" | undefined;
   readonly RASTRO_GEOIP_DAILY_LIMIT: string | undefined;
   readonly RASTRO_GEOIP_PROVIDER: "ipinfo" | "ipwhois" | undefined;
   readonly RASTRO_GEOIP_TOKEN: string | undefined;

package/src/component/constants.ts

@@ -4,9 +4,20 @@ export const MAX_BATCH_BYTES = 64 * 1024;
 export const MAX_EVENTS_PER_SESSION_WINDOW = 120;
 export const RATE_LIMIT_WINDOW_MS = 60_000;
 export const MAX_BYTES_PER_SESSION_WINDOW = 256 * 1024;
-export const SITE_INGEST_SHARDS = 4_096;
-export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 300;
-export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 512 * 1024;
+// A site admits at most SITE_INGEST_SHARDS × the per-shard caps a minute:
+// 384,000 events and 512 MiB. Shards keep every POST off one shared row per
+// site; random session IDs can only fill them evenly, while a real peak lands
+// unevenly — the realistic benchmark (~100k events/min) peaks near 1,050
+// events in its hottest shard-minute, so fewer shards would need a looser cap
+// per shard and more would need a looser cap per site.
+export const SITE_INGEST_SHARDS = 256;
+export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 1_500;
+export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 2 * 1024 * 1024;
+// Per client network (an IPv4 address or an IPv6 /64, keyed by a salted daily
+// hash) per site and minute. Generous on purpose: an office or carrier-grade
+// NAT puts many visitors behind one address, and they all share this budget.
+export const MAX_EVENTS_PER_CLIENT_WINDOW = 1_200;
+export const MAX_BYTES_PER_CLIENT_WINDOW = 2 * 1024 * 1024;
 export const LIVE_SESSION_TTL_MS = 90_000;
 export const LIVE_LEAVE_GRACE_MS = 10_000;
 export const LIVE_SWEEP_INTERVAL_MS = 5_000;
@@ -20,6 +31,13 @@ export const MAX_SITES_PER_OWNER = 100;
 export const MAX_GOALS_PER_SITE = 50;
 export const MAX_FUNNELS_PER_SITE = 20;
 export const MAX_AFFILIATES_PER_SITE = 100;
+export const MAX_TRACKED_LINKS_PER_SITE = 200;
+// A tracked link's clicks spread over this many counter rows, so a link that
+// goes viral writes at four times one document's rate. Four and not more
+// because every shard is read back: the list reads a link's all-time shards
+// plus its last 30 days, and a full site of 200 links must stay under the
+// 32,000 documents a query may read (see trackedLinks.ts).
+export const TRACKED_LINK_CLICK_SHARDS = 4;
 export const MAX_FUNNEL_STEPS = 10;
 export const MAX_PROPERTIES = 16;
 export const MAX_REPORT_SITES = 10;
@@ -44,6 +62,16 @@ export const RETENTION_LEASE_MS = 60_000;
 export const ROLLUP_FOLD_BATCH_DOCS = 200;
 export const ROLLUP_FOLD_MAX_ROWS = 1_500;
 export const ROLLUP_FOLD_LEASE_MS = 60_000;
+// A batch that finds no fold pending schedules one this far out, and a run
+// that folded anything schedules the next one as far again, so one run per
+// second serves every batch of that second; only a run that finds the queue
+// empty releases the lease.
+export const ROLLUP_FOLD_DEBOUNCE_MS = 1_000;
+// A run that throws retries its oldest batch alone; a batch that fails this
+// many runs in a row is set aside as a dead letter, kept this long (30 days)
+// for inspection, so one bad batch cannot stall a site's reports.
+export const ROLLUP_FOLD_MAX_FAILURES = 3;
+export const ROLLUP_DEAD_LETTER_RETENTION_MS = 30 * 86_400_000;
 // Shared Web Vitals histogram edges. Every Google rating threshold (LCP
 // 2500/4000, CLS×1000 100/250, INP 200/500, FCP 1800/3000, TTFB 800/1800) is
 // an edge, so rating counts stay exact while one layout serves all metrics.

package/src/component/convex.config.ts

@@ -9,5 +9,8 @@ export default defineComponent("rastroAnalytics", {
     RASTRO_GEOIP_TOKEN: v.optional(v.string()),
     RASTRO_GEOIP_DAILY_LIMIT: v.optional(v.string()),
     RASTRO_TRUST_PROXY: v.optional(v.literal("true")),
+    // "off" drops the per-client ingest budget, for a load-test canary whose
+    // driver sends every request from one address.
+    RASTRO_CLIENT_RATE_LIMIT: v.optional(v.literal("off")),
   },
 });

package/src/component/coverage.ts

@@ -17,12 +17,14 @@ export async function ensureAnalyticsControl(
     // first ingest after the upgrade, with the same safety delay as rollups:
     // the site map rows, and the alpha.11 counters (pageviews and sessions
     // per dimension slot; duration, continued sessions and new visitors per
-    // bucket), and the 0.1.0 dimensions (utm_medium and outbound hosts).
+    // bucket), the 0.1.0 dimensions (utm_medium and outbound hosts), and
+    // the session origins (platform, channel and evidence).
     const stamps: {
       siteMapSince?: number;
       countersSince?: number;
       localDaySince?: number;
       dimensionsSince?: number;
+      originSince?: number;
     } = {};
     if (existing.siteMapSince === undefined) {
       stamps.siteMapSince = now + ROLLUP_SAFETY_DELAY_MS;
@@ -36,6 +38,9 @@ export async function ensureAnalyticsControl(
     if (existing.dimensionsSince === undefined) {
       stamps.dimensionsSince = now + ROLLUP_SAFETY_DELAY_MS;
     }
+    if (existing.originSince === undefined) {
+      stamps.originSince = now + ROLLUP_SAFETY_DELAY_MS;
+    }
     if (Object.keys(stamps).length === 0) return existing;
     await ctx.db.patch("analyticsCoverage", existing._id, {
       ...stamps,
@@ -51,6 +56,7 @@ export async function ensureAnalyticsControl(
     countersSince: now + ROLLUP_SAFETY_DELAY_MS,
     localDaySince: now + ROLLUP_SAFETY_DELAY_MS,
     dimensionsSince: now + ROLLUP_SAFETY_DELAY_MS,
+    originSince: now + ROLLUP_SAFETY_DELAY_MS,
     retained,
     updatedAt: now,
   };

package/src/component/geo.ts

@@ -510,7 +510,7 @@ function classifyIpv6(ip: string): Exclude<IpKind, "unavailable"> {
   return (groups[0]! & 0xe000) === 0x2000 ? "public" : "private";
 }
 
-function parseIpv6(ip: string): number[] | undefined {
+export function parseIpv6(ip: string): number[] | undefined {
   let normalized = ip;
   const dottedTail = normalized.match(/(\d+\.\d+\.\d+\.\d+)$/)?.[1];
   if (dottedTail) {

package/src/component/http.ts

@@ -20,8 +20,10 @@ import {
 } from "./geo.js";
 import { isPlainRecord } from "./guards.js";
 import { alignClock, sanitizeEvent } from "./sanitize.js";
-import { deriveVisitorKey, randomSecret } from "./identity.js";
-import { classifyClient, isKnownBot } from "./useragent.js";
+import { deriveClientKey, deriveVisitorKey, randomSecret } from "./identity.js";
+import { isTrackedLinkSlug, trackedLinkLocation } from "./trackedLinks.js";
+import { classifyClient, inAppPlatform, isKnownBot } from "./useragent.js";
+import { knownClickIds } from "./origin.js";
 import type { TrackerEvent } from "./validators.js";
 import {
   RASTRO_VERSION,
@@ -174,11 +176,26 @@ http.route({
             userAgent,
           })
         : undefined;
+    const inApp = inAppPlatform(userAgent);
     const context = {
       ...edgeGeo,
       ...classifyClient(userAgent, request.headers),
+      ...(inApp ? { inApp } : {}),
       ...(visitorKey ? { visitorKey } : {}),
     };
+    // Origin is a browser control: any other sender writes the header it
+    // likes. The per-client budget is keyed like the visitor key above (salted,
+    // daily, never the address); without a usable client IP only the per-site
+    // shards bound the batch.
+    const clientKey =
+      clientIp.ip && env.RASTRO_CLIENT_RATE_LIMIT !== "off"
+        ? await deriveClientKey({
+            secret: await visitorSecret(ctx),
+            now: Date.now(),
+            siteId: payload.siteId,
+            ip: clientIp.ip,
+          })
+        : undefined;
 
     try {
       const result = await ctx.runMutation(api.ingest.ingestBatch, {
@@ -186,6 +203,7 @@ http.route({
         origin,
         events,
         context,
+        ...(clientKey ? { clientKey } : {}),
       });
       if (
         clientIp.ip &&
@@ -290,6 +308,46 @@ http.route({
   ),
 });
 
+// A tracked link. The click is counted — as a bot when the agent says it is
+// one, so link unfurlers never inflate the clicks but still get the page — and
+// the visitor goes on with the link's utm_* appended. Unknown and disabled
+// slugs answer 404.
+http.route({
+  pathPrefix: "/l/",
+  method: "GET",
+  handler: httpAction(async (ctx, request) => {
+    const slug = (
+      new URL(request.url).pathname.split("/").filter(Boolean).pop() ?? ""
+    ).toLowerCase();
+    const link = isTrackedLinkSlug(slug)
+      ? await ctx.runMutation(internal.trackedLinks.recordClick, {
+          slug,
+          bot: isKnownBot(
+            (await readRequestMetadata(ctx)).userAgent ??
+              request.headers.get("user-agent") ??
+              "",
+          ),
+        })
+      : null;
+    if (!link) {
+      return new Response("Not found", {
+        status: 404,
+        headers: {
+          "Content-Type": "text/plain; charset=utf-8",
+          "Cache-Control": "no-store",
+        },
+      });
+    }
+    return new Response(null, {
+      status: 302,
+      headers: {
+        Location: trackedLinkLocation(link),
+        "Cache-Control": "no-store",
+      },
+    });
+  }),
+});
+
 http.route({
   path: "/health",
   method: "GET",
@@ -420,6 +478,7 @@ function parseEvent(value: unknown): TrackerEvent {
     throw new Error("event is missing required fields");
   }
   const properties = parseProperties(value.properties);
+  const clid = parseClickIds(value.clid);
   return {
     eventId: value.eventId,
     sessionId: value.sessionId,
@@ -453,9 +512,19 @@ function parseEvent(value: unknown): TrackerEvent {
     ...(optionalNumber(value.value, "value") !== undefined
       ? { value: optionalNumber(value.value, "value") }
       : {}),
+    ...(clid ? { clid } : {}),
   };
 }
 
+/** Click-ID key names: unknown keys are dropped, never a reason to reject. */
+function parseClickIds(value: unknown): string[] | undefined {
+  if (value === undefined) return undefined;
+  if (!Array.isArray(value) || value.some((key) => typeof key !== "string")) {
+    throw new Error("clid must be an array of strings");
+  }
+  return knownClickIds(value as string[]);
+}
+
 function parseProperties(value: unknown): TrackerEvent["properties"] {
   if (value === undefined) return undefined;
   if (!isPlainRecord(value)) throw new Error("properties must be an object");

package/src/component/identity.ts

@@ -1,5 +1,6 @@
 import { v } from "convex/values";
 import { DAY_MS } from "./constants.js";
+import { parseIpv6 } from "./geo.js";
 import { internalMutation, internalQuery } from "./_generated/server.js";
 
 /**
@@ -57,9 +58,47 @@ export async function deriveVisitorKey(input: {
   ip: string;
   userAgent: string;
 }): Promise<string> {
-  const dayStart = Math.floor(input.now / DAY_MS) * DAY_MS;
+  return saltedDailyKey(input.secret, input.now, [
+    input.siteId,
+    input.ip,
+    input.userAgent,
+  ]);
+}
+
+/**
+ * The per-client ingest budget's key: the same salted daily hash, over the
+ * client's network instead of its device — the IPv4 address, or the /64 of an
+ * IPv6 address, the block one subscriber gets and could otherwise walk through
+ * one address per request.
+ */
+export async function deriveClientKey(input: {
+  secret: string;
+  now: number;
+  siteId: string;
+  ip: string;
+}): Promise<string> {
+  const groups = parseIpv6(input.ip.toLowerCase());
+  const network = groups
+    ? `${groups
+        .slice(0, 4)
+        .map((group) => group.toString(16))
+        .join(":")}::/64`
+    : input.ip;
+  return saltedDailyKey(input.secret, input.now, [
+    "client",
+    input.siteId,
+    network,
+  ]);
+}
+
+async function saltedDailyKey(
+  secret: string,
+  now: number,
+  parts: string[],
+): Promise<string> {
+  const dayStart = Math.floor(now / DAY_MS) * DAY_MS;
   const material = new TextEncoder().encode(
-    [input.secret, dayStart, input.siteId, input.ip, input.userAgent].join(" "),
+    [secret, dayStart, ...parts].join(" "),
   );
   const digest = new Uint8Array(
     await crypto.subtle.digest("SHA-256", material),