@iann29/rastro 0.1.0-alpha.9 → 0.1.0

package/src/component/geo.ts

@@ -15,6 +15,7 @@ import {
 import { eventBatchKey, eventBucketStart } from "./eventStore.js";
 import { isPlainRecord } from "./guards.js";
 import { sanitizeContext, stableHash } from "./sanitize.js";
+import { localDayStart, localDayTimezone } from "./localTime.js";
 import { trackerEventValidator, type IngestContext } from "./validators.js";
 
 export type GeoIpProvider = "ipinfo" | "ipwhois";
@@ -55,6 +56,7 @@ const geoAggregateEventValidator = v.object({
   sessionId: v.string(),
   timestamp: v.number(),
   priorCountry: v.string(),
+  pageview: v.optional(v.boolean()),
 });
 
 export const probe = internalAction({
@@ -132,7 +134,10 @@ export const claimEnrichment = internalMutation({
     if (groups.size > MAX_BATCH_EVENT_GROUPS) {
       return { sessionIds: [], events: [] };
     }
-    const persistedEvents = new Map<string, Map<string, string | undefined>>();
+    const persistedEvents = new Map<
+      string,
+      Map<string, { country: string | undefined; pageview: boolean }>
+    >();
     await Promise.all(
       [...groups].map(async ([key, group]) => {
         const batches = await ctx.db
@@ -154,7 +159,10 @@ export const claimEnrichment = internalMutation({
                   (event) =>
                     [
                       event.eventId,
-                      event.aggregateCountry ?? event.country,
+                      {
+                        country: event.aggregateCountry ?? event.country,
+                        pageview: event.type === "pageview",
+                      },
                     ] as const,
                 ),
             ),
@@ -230,6 +238,7 @@ export const claimEnrichment = internalMutation({
       sessionId: string;
       timestamp: number;
       priorCountry: string;
+      pageview: boolean;
     }> = [];
     for (const event of args.events) {
       const key = `${eventBatchKey(event.sessionId, eventBucketStart(event.timestamp))}:${event.eventId}`;
@@ -245,14 +254,15 @@ export const claimEnrichment = internalMutation({
         continue;
       }
       seenEvents.add(key);
-      const priorCountry = persistedEvents
+      const persisted = persistedEvents
         .get(eventBatchKey(event.sessionId, eventBucketStart(event.timestamp)))
         ?.get(event.eventId);
-      if (priorCountry === undefined) continue;
+      if (persisted?.country === undefined) continue;
       correctionEvents.push({
         sessionId: event.sessionId,
         timestamp: event.timestamp,
-        priorCountry,
+        priorCountry: persisted.country,
+        pageview: persisted.pageview,
       });
     }
     return { sessionIds, events: correctionEvents };
@@ -276,7 +286,10 @@ export const applyEnrichment = internalMutation({
       return 0;
     }
     const context = sanitizeContext(args.context);
+    const site = await ctx.db.get("sites", args.siteId);
+    const timezone = localDayTimezone(site?.timezone);
     const sessionCountries = new Map<string, string>();
+    const sessionStarts = new Map<string, number>();
     let updated = 0;
     for (const sessionId of new Set(args.sessionIds)) {
       const session = await ctx.db
@@ -288,6 +301,7 @@ export const applyEnrichment = internalMutation({
       if (!session || session.geoLookupAttemptedAt === undefined) continue;
       const sessionCountry = session.country ?? context.country;
       if (sessionCountry) sessionCountries.set(sessionId, sessionCountry);
+      sessionStarts.set(sessionId, session.startedAt);
       const sessionGeo = missingGeo(session, context);
       if (Object.keys(sessionGeo).length > 0) {
         await ctx.db.patch("sessions", session._id, sessionGeo);
@@ -312,6 +326,8 @@ export const applyEnrichment = internalMutation({
         args.siteId,
         args.events,
         sessionCountries,
+        sessionStarts,
+        timezone,
       );
     }
     return updated;
@@ -546,24 +562,34 @@ async function correctCountryAggregates(
     sessionId: string;
     timestamp: number;
     priorCountry: string;
+    pageview?: boolean;
   }>,
   sessionCountries: Map<string, string>,
+  sessionStarts: Map<string, number>,
+  timezone: string | undefined,
 ): Promise<void> {
   const corrections = new Map<
     string,
     {
-      granularity: "hour" | "day";
+      granularity: "hour" | "day" | "localDay";
       bucketStart: number;
       shard: number;
-      moves: Map<string, { from: string; to: string; count: number }>;
+      moves: Map<string, CountryMove>;
     }
   >();
+  const granularities: ReadonlyArray<"hour" | "day" | "localDay"> = timezone
+    ? ["hour", "day", "localDay"]
+    : ["hour", "day"];
   for (const event of events) {
     const country = sessionCountries.get(event.sessionId);
     if (!country || country === event.priorCountry) continue;
-    for (const granularity of ["hour", "day"] as const) {
-      const interval = granularity === "hour" ? HOUR_MS : DAY_MS;
-      const bucketStart = Math.floor(event.timestamp / interval) * interval;
+    for (const granularity of granularities) {
+      const bucketStart =
+        granularity === "localDay" && timezone
+          ? localDayStart(event.timestamp, timezone)
+          : Math.floor(
+              event.timestamp / (granularity === "hour" ? HOUR_MS : DAY_MS),
+            ) * (granularity === "hour" ? HOUR_MS : DAY_MS);
       const shard =
         stableHash(event.sessionId) %
         (granularity === "hour" ? HOURLY_AGGREGATE_SHARDS : AGGREGATE_SHARDS);
@@ -572,15 +598,23 @@ async function correctCountryAggregates(
         granularity,
         bucketStart,
         shard,
-        moves: new Map<string, { from: string; to: string; count: number }>(),
+        moves: new Map<string, CountryMove>(),
       };
       const moveKey = `${event.priorCountry}:${country}`;
       const move = correction.moves.get(moveKey) ?? {
         from: event.priorCountry,
         to: country,
         count: 0,
+        pageviews: 0,
+        sessions: 0,
       };
       move.count += 1;
+      move.pageviews += Number(event.pageview === true);
+      // The session counter was folded with the session's first event, so it
+      // moves with the bucket that holds that event and no other.
+      move.sessions += Number(
+        sessionStarts.get(event.sessionId) === event.timestamp,
+      );
       correction.moves.set(moveKey, move);
       corrections.set(key, correction);
     }
@@ -602,12 +636,7 @@ async function correctCountryAggregates(
     let dimensions = bucket.dimensions;
     let changed = false;
     for (const move of correction.moves.values()) {
-      const corrected = moveCountryCount(
-        dimensions,
-        move.from,
-        move.to,
-        move.count,
-      );
+      const corrected = moveCountryCount(dimensions, move);
       if (corrected) {
         dimensions = corrected;
         changed = true;
@@ -619,11 +648,17 @@ async function correctCountryAggregates(
   }
 }
 
+type CountryMove = {
+  from: string;
+  to: string;
+  count: number;
+  pageviews: number;
+  sessions: number;
+};
+
 function moveCountryCount(
   current: Doc<"aggregateBuckets">["dimensions"],
-  from: string,
-  to: string,
-  requested: number,
+  { from, to, ...requested }: CountryMove,
 ): Doc<"aggregateBuckets">["dimensions"] | undefined {
   const dimensions = current.map((slot) => ({ ...slot }));
   let sourceIndex = dimensions.findIndex(
@@ -636,9 +671,15 @@ function moveCountryCount(
   }
   if (sourceIndex < 0) return undefined;
   const source = dimensions[sourceIndex]!;
-  const moved = Math.min(source.count, requested);
-  if (moved <= 0) return undefined;
-  source.count -= moved;
+  const moved = {
+    count: Math.min(source.count, requested.count),
+    pageviews: Math.min(source.pageviews ?? 0, requested.pageviews),
+    sessions: Math.min(source.sessions ?? 0, requested.sessions),
+  };
+  if (moved.count <= 0) return undefined;
+  source.count -= moved.count;
+  if (source.pageviews !== undefined) source.pageviews -= moved.pageviews;
+  if (source.sessions !== undefined) source.sessions -= moved.sessions;
   if (source.count === 0 && source.revenueCents === 0) {
     dimensions.splice(sourceIndex, 1);
   }
@@ -646,7 +687,7 @@ function moveCountryCount(
     (slot) => slot.type === "country" && slot.value === to,
   );
   if (target) {
-    target.count += moved;
+    addCountryMove(target, moved);
     return dimensions;
   }
   const countrySlots = dimensions.filter((slot) => slot.type === "country");
@@ -658,14 +699,16 @@ function moveCountryCount(
       type: "country",
       slot,
       value: to,
-      count: moved,
+      count: moved.count,
       revenueCents: 0,
+      pageviews: moved.pageviews,
+      sessions: moved.sessions,
     });
     return dimensions;
   }
   const overflow = countrySlots.find((slot) => slot.value === "(other)");
   if (overflow) {
-    overflow.count += moved;
+    addCountryMove(overflow, moved);
     return dimensions;
   }
   const occupied = new Set(countrySlots.map((slot) => slot.slot));
@@ -676,8 +719,10 @@ function moveCountryCount(
       type: "country",
       slot,
       value: "(other)",
-      count: moved,
+      count: moved.count,
       revenueCents: 0,
+      pageviews: moved.pageviews,
+      sessions: moved.sessions,
     });
     return dimensions;
   }
@@ -686,10 +731,19 @@ function moveCountryCount(
     if (candidate.count < replacement.count) replacement = candidate;
   }
   replacement.value = "(other)";
-  replacement.count += moved;
+  addCountryMove(replacement, moved);
   return dimensions;
 }
 
+function addCountryMove(
+  slot: Doc<"aggregateBuckets">["dimensions"][number],
+  moved: { count: number; pageviews: number; sessions: number },
+) {
+  slot.count += moved.count;
+  slot.pageviews = (slot.pageviews ?? 0) + moved.pageviews;
+  slot.sessions = (slot.sessions ?? 0) + moved.sessions;
+}
+
 function missingGeo(existing: GeoContext, context: GeoContext): GeoContext {
   const coordinates =
     (existing.latitude === undefined || existing.longitude === undefined) &&

package/src/component/http.ts

@@ -24,10 +24,13 @@ import { deriveVisitorKey, randomSecret } from "./identity.js";
 import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
 import {
+  RASTRO_VERSION,
   TRACKER_GZIP_BASE64,
   TRACKER_GZIP_BYTES,
+  TRACKER_HASH,
   VITALS_GZIP_BASE64,
   VITALS_GZIP_BYTES,
+  VITALS_HASH,
 } from "../tracker/generated.js";
 
 const TRACKER_GZIP = Uint8Array.from(atob(TRACKER_GZIP_BASE64), (character) =>
@@ -37,6 +40,52 @@ const VITALS_GZIP = Uint8Array.from(atob(VITALS_GZIP_BASE64), (character) =>
   character.charCodeAt(0),
 );
 
+// Five minutes fresh, then up to an hour of stale-while-revalidate: a visitor
+// who returns after a deploy gets at most one stale load before the browser's
+// background revalidation, and that revalidation is a 304 against the ETag.
+const SCRIPT_CACHE_CONTROL = "public, max-age=300, stale-while-revalidate=3600";
+
+/**
+ * Serves one pre-gzipped module with a strong ETag from its build-time content
+ * hash. A matching If-None-Match answers 304 with the same caching and CORS
+ * headers, so a revalidation costs a header exchange instead of the script.
+ */
+function scriptResponse(
+  request: Request,
+  body: Uint8Array<ArrayBuffer>,
+  byteLength: number,
+  hash: string,
+): Response {
+  const etag = `"${hash}"`;
+  const headers = {
+    ETag: etag,
+    "Cache-Control": SCRIPT_CACHE_CONTROL,
+    "Access-Control-Allow-Origin": "*",
+    Vary: "Accept-Encoding",
+  };
+  if (etagMatches(request.headers.get("if-none-match"), etag)) {
+    return new Response(null, { status: 304, headers });
+  }
+  return new Response(body, {
+    status: 200,
+    headers: {
+      ...headers,
+      "Content-Type": "text/javascript; charset=utf-8",
+      "Content-Encoding": "gzip",
+      "Content-Length": String(byteLength),
+    },
+  });
+}
+
+/** RFC 9110 If-None-Match: a list of entity tags or "*", weak markers ignored. */
+function etagMatches(header: string | null, etag: string): boolean {
+  if (!header) return false;
+  return header.split(",").some((candidate) => {
+    const value = candidate.trim().replace(/^W\//, "");
+    return value === "*" || value === etag;
+  });
+}
+
 const http = httpRouter();
 
 http.route({
@@ -228,37 +277,17 @@ http.route({
 http.route({
   path: "/tracker.js",
   method: "GET",
-  handler: httpAction(async () => {
-    return new Response(TRACKER_GZIP, {
-      status: 200,
-      headers: {
-        "Content-Type": "text/javascript; charset=utf-8",
-        "Content-Encoding": "gzip",
-        "Content-Length": String(TRACKER_GZIP_BYTES),
-        "Cache-Control": "public, max-age=300, stale-while-revalidate=86400",
-        "Access-Control-Allow-Origin": "*",
-        Vary: "Accept-Encoding",
-      },
-    });
-  }),
+  handler: httpAction(async (_ctx, request) =>
+    scriptResponse(request, TRACKER_GZIP, TRACKER_GZIP_BYTES, TRACKER_HASH),
+  ),
 });
 
 http.route({
   path: "/vitals.js",
   method: "GET",
-  handler: httpAction(async () => {
-    return new Response(VITALS_GZIP, {
-      status: 200,
-      headers: {
-        "Content-Type": "text/javascript; charset=utf-8",
-        "Content-Encoding": "gzip",
-        "Content-Length": String(VITALS_GZIP_BYTES),
-        "Cache-Control": "public, max-age=300, stale-while-revalidate=86400",
-        "Access-Control-Allow-Origin": "*",
-        Vary: "Accept-Encoding",
-      },
-    });
-  }),
+  handler: httpAction(async (_ctx, request) =>
+    scriptResponse(request, VITALS_GZIP, VITALS_GZIP_BYTES, VITALS_HASH),
+  ),
 });
 
 http.route({
@@ -301,6 +330,11 @@ http.route({
           siteMap: "dailyRollup",
           liveDepartures: "leaveBeacon",
         },
+        tracker: {
+          version: RASTRO_VERSION,
+          hash: TRACKER_HASH,
+          vitalsHash: VITALS_HASH,
+        },
       },
       200,
       {
@@ -369,6 +403,7 @@ function parseEvent(value: unknown): TrackerEvent {
     type !== "custom" &&
     type !== "conversion" &&
     type !== "heartbeat" &&
+    type !== "leave" &&
     type !== "outbound" &&
     type !== "vital"
   ) {