@iann29/rastro 0.1.0-alpha.8 → 0.1.0

package/src/component/rollups.ts

@@ -0,0 +1,82 @@
+import { v } from "convex/values";
+import { internal } from "./_generated/api.js";
+import { internalMutation } from "./_generated/server.js";
+import {
+  ROLLUP_FOLD_BATCH_DOCS,
+  ROLLUP_FOLD_LEASE_MS,
+  ROLLUP_FOLD_MAX_ROWS,
+} from "./constants.js";
+import {
+  countReportRollupRows,
+  createReportRollupDeltas,
+  flushReportRollups,
+  mergeQueuedRollups,
+} from "./rollupStore.js";
+
+/**
+ * Folds a site's queued report rollup deltas into the rollup rows. Exactly
+ * one chain runs per site: ingestion starts one when no lease is held, every
+ * run carries the token of the lease it belongs to, and a run whose token is
+ * no longer current exits without touching anything. A run that finds more
+ * queued documents than it could take continues at once; a run that drains
+ * the queue releases the lease. A batch that lands while a run is draining
+ * is never stranded: the run read the queue range the batch inserted into
+ * and the batch read the lease the run released, so one of the two retries
+ * and sees the other's work.
+ */
+export const fold = internalMutation({
+  args: { siteId: v.id("sites"), token: v.string() },
+  returns: v.object({
+    folded: v.number(),
+    remaining: v.boolean(),
+    stale: v.boolean(),
+  }),
+  handler: async (ctx, args) => {
+    const state = await ctx.db
+      .query("rollupFoldStates")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    if (!state || state.token !== args.token) {
+      return { folded: 0, remaining: false, stale: true };
+    }
+    const now = Date.now();
+    const queued = await ctx.db
+      .query("reportRollupQueue")
+      .withIndex("by_siteId_and_createdAt", (range) =>
+        range.eq("siteId", args.siteId),
+      )
+      .order("asc")
+      .take(ROLLUP_FOLD_BATCH_DOCS + 1);
+    if (queued.length === 0) {
+      if (state.leaseUntil > now) {
+        await ctx.db.patch("rollupFoldStates", state._id, { leaseUntil: now });
+      }
+      return { folded: 0, remaining: false, stale: false };
+    }
+    const deltas = createReportRollupDeltas();
+    let taken = 0;
+    for (const document of queued.slice(0, ROLLUP_FOLD_BATCH_DOCS)) {
+      mergeQueuedRollups(deltas, document);
+      taken += 1;
+      if (countReportRollupRows(deltas) >= ROLLUP_FOLD_MAX_ROWS) break;
+    }
+    await flushReportRollups(ctx, deltas);
+    for (const document of queued.slice(0, taken)) {
+      await ctx.db.delete("reportRollupQueue", document._id);
+    }
+    const remaining = queued.length > taken;
+    if (remaining) {
+      // Renew only when the lease is halfway gone, so the state document is
+      // rewritten about twice a minute under load rather than on every run.
+      if (state.leaseUntil - now < ROLLUP_FOLD_LEASE_MS / 2) {
+        await ctx.db.patch("rollupFoldStates", state._id, {
+          leaseUntil: now + ROLLUP_FOLD_LEASE_MS,
+        });
+      }
+      await ctx.scheduler.runAfter(0, internal.rollups.fold, args);
+    } else {
+      await ctx.db.patch("rollupFoldStates", state._id, { leaseUntil: now });
+    }
+    return { folded: taken, remaining, stale: false };
+  },
+});

package/src/component/sanitize.ts

@@ -121,6 +121,20 @@ export function sanitizePublicUrl(value: string): string | undefined {
   }
 }
 
+/**
+ * The host of a public URL the way the outbound dimension names it: lowercase,
+ * without port, path, or query. A relative path has no host.
+ */
+export function publicUrlHost(value: string | undefined): string | undefined {
+  if (!value || value.startsWith("/")) return undefined;
+  try {
+    const host = new URL(value).hostname.toLowerCase();
+    return host ? host.slice(0, 120) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
 export function sanitizeProperties(
   properties: EventProperties | undefined,
 ): EventProperties | undefined {

package/src/component/schema.ts

@@ -11,6 +11,10 @@ import {
   visitorSketchValidator,
   vitalMetricValidator,
 } from "./validators.js";
+import {
+  reportRollupQueueFields,
+  rollupFoldStateFields,
+} from "./rollupStore.js";
 
 export default defineSchema({
   sites: defineTable(siteFieldsValidator)
@@ -56,6 +60,22 @@ export default defineSchema({
     eventCount: v.number(),
     previousPath: v.optional(v.string()),
     movedAt: v.optional(v.number()),
+    // What the live view needs to tell intent apart (since 0.1.0): where the
+    // session came in, whether the visitor had a history, the campaign and
+    // affiliate it carries, conversions so far, and the furthest funnel step
+    // the visitor has reached.
+    entryPath: v.optional(v.string()),
+    returning: v.optional(v.boolean()),
+    utmCampaign: v.optional(v.string()),
+    affiliateSlug: v.optional(v.string()),
+    conversionCount: v.optional(v.number()),
+    funnel: v.optional(
+      v.object({
+        funnelId: v.id("funnels"),
+        step: v.number(),
+        steps: v.number(),
+      }),
+    ),
   })
     .index("by_siteId_and_sessionId", ["siteId", "sessionId"])
     .index("by_siteId_and_expiresAt", ["siteId", "expiresAt"])
@@ -227,11 +247,26 @@ export default defineSchema({
     ])
     .index("by_siteId_and_bucketStart", ["siteId", "bucketStart"]),
 
+  // Rollup deltas wait here between ingestion and the fold job; see
+  // rollupStore.ts. Rows are deleted as they are folded.
+  reportRollupQueue: defineTable(reportRollupQueueFields).index(
+    "by_siteId_and_createdAt",
+    ["siteId", "createdAt"],
+  ),
+
+  rollupFoldStates: defineTable(rollupFoldStateFields).index("by_siteId", [
+    "siteId",
+  ]),
+
   analyticsCoverage: defineTable({
     siteId: v.id("sites"),
     generation: v.literal(1),
     rollupCompleteFrom: v.number(),
     siteMapSince: v.optional(v.number()),
+    countersSince: v.optional(v.number()),
+    localDaySince: v.optional(v.number()),
+    // Since when the medium and outbound dimension slots are recorded.
+    dimensionsSince: v.optional(v.number()),
     retained: v.array(v.object({ dataset: v.string(), before: v.number() })),
     updatedAt: v.number(),
   }).index("by_siteId", ["siteId"]),

package/src/component/sites.ts

@@ -3,6 +3,8 @@ import { internalQuery, mutation, query } from "./_generated/server.js";
 import { MAX_SITES_PER_OWNER } from "./constants.js";
 import { ensureAnalyticsControl } from "./coverage.js";
 import { fail } from "./errors.js";
+import { resetLocalDayCoverage } from "./coverage.js";
+import { localDayTimezone } from "./localTime.js";
 import {
   cleanString,
   normalizeDomains,
@@ -173,6 +175,9 @@ export const update = mutation({
     }
     if (args.cookieless !== undefined) patch.cookieless = args.cookieless;
     await ctx.db.patch("sites", args.siteId, patch);
+    if (args.timezone !== undefined && site.timezone !== patch.timezone) {
+      await resetLocalDayCoverage(ctx, args.siteId, patch.updatedAt);
+    }
     return null;
   },
 });
@@ -185,6 +190,33 @@ export const get = query({
   },
 });
 
+/**
+ * The calendar days a site's daily overview can answer in: the timezone its
+ * local-day buckets follow and the instant from which they are complete, or
+ * null when the site keeps none (no timezone, or a zone that never leaves
+ * UTC, whose days are the UTC days already kept).
+ */
+export const localDays = query({
+  args: { siteId: v.id("sites") },
+  returns: v.union(
+    v.object({ timezone: v.string(), since: v.number() }),
+    v.null(),
+  ),
+  handler: async (ctx, args) => {
+    const site = await ctx.db.get("sites", args.siteId);
+    const timezone = localDayTimezone(site?.timezone);
+    if (!timezone) return null;
+    const control = await ctx.db
+      .query("analyticsCoverage")
+      .withIndex("by_siteId", (range) => range.eq("siteId", args.siteId))
+      .unique();
+    // A site that has not ingested since the upgrade has no stamp yet; its
+    // days begin with its first batch, which is later than now.
+    if (control?.localDaySince === undefined) return null;
+    return { timezone, since: control.localDaySince };
+  },
+});
+
 export const listByOwner = query({
   args: {
     ownerId: v.string(),

package/src/component/validators.ts

@@ -6,6 +6,7 @@ export const eventTypeValidator = v.union(
   v.literal("custom"),
   v.literal("conversion"),
   v.literal("heartbeat"),
+  v.literal("leave"),
   v.literal("outbound"),
   v.literal("vital"),
 );
@@ -84,6 +85,9 @@ export const sessionFieldsValidator = v.object({
   sessionId: v.string(),
   visitorId: v.string(),
   visitorKey: v.optional(v.string()),
+  // Since 0.1.0: whether nothing was recorded under the visitor before this
+  // session (a host id linked to anonymous history at sign-up is a return).
+  newVisitor: v.optional(v.boolean()),
   startedAt: v.number(),
   lastSeenAt: v.number(),
   entryPath: v.string(),
@@ -147,6 +151,10 @@ export const dimensionTypeValidator = v.union(
   v.literal("browser"),
   v.literal("event"),
   v.literal("goal"),
+  // Since 0.1.0: the lowercase utm_medium of the session, and the destination
+  // host of an outbound click (never its path).
+  v.literal("medium"),
+  v.literal("outbound"),
 );
 
 export const dimensionSlotValidator = v.object({
@@ -155,6 +163,11 @@ export const dimensionSlotValidator = v.object({
   value: v.string(),
   count: v.number(),
   revenueCents: v.number(),
+  // Slots written since alpha.11 also count the pageviews and the sessions
+  // (a session's first event) they saw, so pages rank by views and every
+  // other dimension by sessions instead of by raw event volume.
+  pageviews: v.optional(v.number()),
+  sessions: v.optional(v.number()),
 });
 
 export const visitorSketchValidator = v.object({
@@ -163,9 +176,17 @@ export const visitorSketchValidator = v.object({
   registers: v.bytes(),
 });
 
+export const aggregateGranularityValidator = v.union(
+  v.literal("hour"),
+  v.literal("day"),
+  // One bucket per calendar day in the site's timezone, written alongside
+  // the UTC day for sites with a timezone (see localTime.ts).
+  v.literal("localDay"),
+);
+
 export const aggregateFieldsValidator = v.object({
   siteId: v.id("sites"),
-  granularity: v.union(v.literal("hour"), v.literal("day")),
+  granularity: aggregateGranularityValidator,
   bucketStart: v.number(),
   shard: v.number(),
   events: v.number(),
@@ -180,6 +201,12 @@ export const aggregateFieldsValidator = v.object({
   conversions: v.number(),
   revenueCents: v.number(),
   dimensions: v.array(dimensionSlotValidator),
+  // Engagement counters recorded since alpha.11: how much session duration
+  // grew from the activity that landed in this bucket, sessions that reached
+  // a second pageview here, and sessions of visitors never seen before.
+  durationMs: v.optional(v.number()),
+  continuedSessions: v.optional(v.number()),
+  newVisitors: v.optional(v.number()),
 });
 
 export const goalMatchTypeValidator = v.union(

package/src/tracker/generated.ts

@@ -0,0 +1,12 @@
+// Generated by scripts/build-tracker.mjs. Do not edit.
+export const RASTRO_VERSION = "0.1.0";
+export const TRACKER_SOURCE = "(()=>{const e=document.currentScript,t=e?.dataset.site,s=e?.dataset.endpoint||e?.src&&new URL(\"events\",e.src).href;if(!t||!s)return;const o=()=>crypto.randomUUID?.()||Math.random()+\"\",n=new URLSearchParams(location.search),r=n.get(\"ref\")?.slice(0,64);let a,i,c;for(const e of[\"source\",\"medium\",\"campaign\",\"term\",\"content\"]){const t=n.get(\"utm_\"+e)?.slice(0,64);t&&((a??={})[\"utm_\"+e]=t)}try{i=sessionStorage._r||o(),sessionStorage._r=i,c=r||sessionStorage._a,c&&(sessionStorage._a=c),a||=sessionStorage._u&&JSON.parse(sessionStorage._u),a&&(sessionStorage._u=JSON.stringify(a))}catch{i=o(),c=r}let d=e.dataset.visitor;try{d||=localStorage._rv||=o()}catch{}d||=i;let l,p=0;const g=[],h=document.referrer?new URL(document.referrer).origin:void 0,u=e=>{const o=g.splice(0,50);if(!o.length)return;const n=()=>new Blob([JSON.stringify({siteId:t,events:o,sentAt:Date.now()})]);let r=n();for(;r.size>64e3&&o.length>1;)g.unshift(o.pop()),r=n();e&&navigator.sendBeacon(s,r)||fetch(s,{method:\"POST\",body:r,keepalive:1}).catch(()=>{}),g.length&&u(e)},m=(e,t,s)=>{const o=Date.now();g.push({eventId:`${i}.${o.toString(36)}.${p}`,sessionId:i,visitorId:d,type:e,name:t,path:location.pathname.slice(0,256),referrer:h,timestamp:o,sequence:p++,affiliateSlug:c,...s}),clearTimeout(l),l=setTimeout(u,1200)};let v;const f=()=>{const e=location.pathname;e!==v&&(v=e,m(\"pageview\",void 0,a&&{properties:a}))};window.rastro=(e,t,s)=>{if(\"context\"===e)return{sessionId:i,visitorId:d};\"event\"===e&&m(\"custom\",`${t||\"event\"}`.slice(0,80),s&&JSON.stringify(s).length>2e4?void 0:{properties:s})},document.addEventListener(\"click\",e=>{const t=e.target?.closest(\"a,button,[data-rastro-event]\");if(!t)return;const s=t.dataset.rastroEvent,o=t.href,n=o&&new URL(o),r=/^https?:$/.test(n?.protocol),a=r&&n.origin!==location.origin;m(s?\"custom\":a?\"outbound\":\"click\",s,{target:(t.dataset.rastroLabel||t.innerText||t.tagName).slice(0,64),href:r?((a?n.origin:\"\")+n.pathname).slice(0,256):void 0})},1);const S=e=>{const t=history[e];history[e]=(...e)=>{t.apply(history,e),f()}};S(\"pushState\"),S(\"replaceState\"),addEventListener(\"popstate\",f),addEventListener(\"pagehide\",()=>{m(\"leave\"),u(1)}),addEventListener(\"pageshow\",e=>{e.persisted&&(m(\"heartbeat\"),u(1))}),setInterval(()=>{m(\"heartbeat\"),u()},2e4),f()})();";
+export const TRACKER_GZIP_BASE64 = "H4sIAAAAAAACA3VV227cNhD9FZswBBKeMLbjGIUEZtEgfXDhJkE3eTLcmKZGEhGJVMnROtuV/r2QVuv1JXmTZsi5nDlnyLlQ7zbGu0gHqHJvugYdSdOFgI6WJtiWgBQuZK5JRyQZLSHExxZ0eeuto77HhYzBJInD+4Ovf19xhit0FBngaBeyClhktuCH1PeHUQSkLrhsm92rsRQT1i15GbTLffP16+WHheSi7//SVM1GLo4ZA6fmHEvUwVSfddBN5LU3mqx3Mk5WAUE5WSJxFrBgYiFjbQ3yE7g4F1mNdKDBgskKH/gMwYEvrln0XTDIgDWY265hwIxuWm1Lx4ARhsniHaEjdiNm9GiXqqPmGzvGZ9koSTjXi4XaDOJ6d+ZGkRgorDdWRYzRerckH3SJ8lvoe88FvDArC0aFvn/u0GCShL+wKiNA9/2L8F2S/Ln89FG2OkR8ca0ToH8SrVPTnUjButIWa66FGIwmU22sGqs1KgwjrLnCB3asbLTkQza2mfe9GmdU7/tZ9f14dQ4zjCfsNJoaWnUyc6NU1zdQ7ekZsMAQMCx2RHvhEdIHW1qXrrzND06gU/jAc69KGdt5Nm9PxMRIL2t0JVVPSekmUo5Z3tf+jl8/638ziuEyTwm2RE89RHT0O6UfNKF0/p6LQdxsuRaU42LiWhZktP/hu4tzfJMku8zvTjNRys7FyhbEvWx9y8VEYS4yTBKnV7bU5IOM6PL3qI13PEIQfV8gmYpH2DRIlc9T9vnT8guDO5+v0wDfEVtd2xWmp4OQE9B8kv0goJyTJ0nHUQzQKI5AEMUjtPa9ZKVsu1jxzdTuZZ7eHm3sII82XpJfTrDwNxditLTD7Y68l3lqYabBZZ7mQOsWUwSnG0wJWk1V+qDc8W90PIjn7O2FgN1Y0wrINhhJN+0E9r8dOoNpe3wMuihsbTXhsu7K1ICUMg4CTI06fLEN+o54LaBWEWn338Hp2cmJGKYBreahF+rJUnxRWoaHSq2ShK8UQsNZq0tcWbxnMJNNJ8mmDb7FQBZjqgchhuzeutzfy6AjBf8IZVvw7TL5QUwphTMBN78Ab8i2S3U6myQNZ6aL5BsGt0cb6vvZO9w+APjbiYA4y31P3Sh2tDvD88W28PRx1XEQAzzoSuf5H2PgKxsJHQbOTG3NdwZ7WZFCSTqUSAtpah8xEmca7joi7+B63Aevtt2/mmq8YVvl0VPJRUUPu2N7fEoMXtH0foBTfv/C+FEgr/+piNq4SI9eSxrTuoVsgydvfC1Aq5Akbt4Hh2o/z60la3hc7EBM9YL5ju5853KW7nqMsNk2lvLntV3pO6z7nqR1DsMX/EHjD+nyo25QPH4BYCw+DYvxFdjlThkTx3tiiSekn1fXOIZTMYOzVI/xrmwkH9bXeJPtPxWXUuLILJK6bes1n32AAgouhiFbcjbKeEmakAlYjs9jW2uDO8vLabe+jZMTip+6dYmVzZHBJJ2Gsxr1agzV8VMx/OpKrPz9lkIoWwxxdOZJwhvOKtSB7lDTHGMMEpEuHWFY6Zrv8jw9JwY4w/Ntn4KL7H/zTKaM3wgAAA==";
+export const TRACKER_RAW_BYTES = 2271;
+export const TRACKER_GZIP_BYTES = 1177;
+export const TRACKER_HASH = "14dc90ee3a9b5ce4";
+export const VITALS_SOURCE = "(()=>{const t=document.currentScript,e=t?.dataset.site,n=t?.dataset.endpoint||t?.src&&new URL(\"events\",t.src).href;if(!e||!n)return;const r=()=>crypto.randomUUID?.()||Math.random()+\"\";let a;try{a=sessionStorage._r||r(),sessionStorage._r=a}catch{a=r()}const s=t.dataset.visitor||a,o=location.pathname.slice(0,256),i={},c=[],d=(t,e,n)=>{try{const r=new PerformanceObserver(t=>e(t.getEntries()));r.observe({type:t,buffered:1,...n}),c.push({observer:r,handler:e})}catch{}},h=performance.getEntriesByType?.(\"navigation\")[0];h&&h.responseStart>0&&(i.TTFB=h.responseStart),d(\"paint\",t=>{for(const e of t)\"first-contentful-paint\"===e.name&&(i.FCP=e.startTime)}),d(\"largest-contentful-paint\",t=>{const e=t[t.length-1];e&&(i.LCP=e.startTime)});let l=0,f=0,m=0;d(\"layout-shift\",t=>{for(const e of t)e.hadRecentInput||(l&&e.startTime-m<1e3&&e.startTime-f<5e3?l+=e.value:(l=e.value,f=e.startTime),m=e.startTime,i.CLS=Math.max(i.CLS||0,1e3*l))});const u=[],p=new Set,v=t=>{for(const e of t)e.interactionId&&(p.add(e.interactionId),u.push(e.duration));if(!u.length)return;u.sort((t,e)=>e-t),u.length=Math.min(u.length,10);const e=performance.interactionCount||p.size;i.INP=u[Math.min(u.length-1,Math.floor(e/50))]};d(\"event\",v,{durationThreshold:40}),d(\"first-input\",v);let g=0;const y=()=>{if(g)return;for(const{observer:t,handler:e}of c)e(t.takeRecords()),t.disconnect();const t=Date.now(),r=Object.keys(i).map((e,n)=>({eventId:`${a}.${t.toString(36)}.v${n}`,sessionId:a,visitorId:s,type:\"vital\",name:e,value:Math.round(i[e]),path:o,timestamp:t,sequence:n}));if(!r.length)return;g=1;const d=new Blob([JSON.stringify({siteId:e,events:r,sentAt:t})]),h=navigator.sendBeacon?.bind(navigator);h&&h(n,d)||fetch(n,{method:\"POST\",body:d,keepalive:1}).catch(()=>{})};document.addEventListener(\"visibilitychange\",()=>{\"hidden\"===document.visibilityState&&y()}),addEventListener(\"pagehide\",y)})();";
+export const VITALS_GZIP_BASE64 = "H4sIAAAAAAACA3VV72/bNhD9V1IhMMiV5uxl6Qe5TLCkLeAha4I4+RQEK02eLC4SqZEnd5qk/32gZOfn+k06kXfv3b17IoSKk1Y5G/AAhXaqLsEiV7X3YHGlvKmQgcBTriXKAMiDQWD2eQSsrpyx2HV4yoNXk4mF7we31xckgS1YDAnDGKc895AtTEbeQde9s9QD1t4uxupeRCjKNxU67qXVrry9XX465YR23R8S812Q0PdJsigAD+QCfdNKESAE4+wKnZcb4H/6rvOEsjdhIXslUeWtFJ7QfiwbBD4S2Zpg0Pmuk8yJwimJxlleScytLIGHwiggM/bL8QfKjGh7psTdPdOCIANmYyMjoD2d2IQr8JnzpbQKLtcB/BY8QXECBPkG8LNFbyAQSunCczceIC02FaTI1nWWgQedzhnn3PaUKV7VISft7qRPPcul1QX4FHq6I9f3LBfVU91nhc6am6aCU04SK7dmM9BL6N3sfpFPJjn3ECpnA6xQejyZTSbE8JubL2fi1SfKNEkqaSwmDMVJmzlPRtJw4LIDpElmfMCpchbBYlYX0/G0EAJ4bOWQ+sv5lQAeYsYbUwLth7yF9Bv4v8tDqV0ZgXfIC7AbzKfz+8WY7+JNvkElhZixTMxYKWaLIX/japyG3GQ/gg88l/oaFFhc2qrGriPFZPIs9bT8OIejl6Hs4zEcnRbvBfCtLGpISbF/ZNkLXKx8/soMP79YiUHgpfyHDK9dN2NzOPqpoJHFCK6OWqsGVa0A2Vb8ALuxCF6qONulnkxIxaXW5FWcsnrUEnBd+0EIlA6bWe/6ul/OmgfnkUSJU3ECU4xXxyM70MaSfYTNZ3u48EKDz4qfuzpaRcWD+RcWhi+/Xon67k2q6ZwNsaxwzhP4+XhG6X0fJzh4SsK2rN1Dv8k9hNwVOv11NqpoFKCJ00vYdhTCRsx22JrBalqTkc2e5mMjn5YLny2Xyw4UjVuL8gGuQTmv49oy5NoE5awFhWRPHcUnicCt+04o8+Jy/Rco5A/QBGIoL2VFyGgXpB24LHX67bCVPT9skaNboTd2Q44+0J5vD1vbf9tb2VKnku1MaqnTwAanSLYGZZGwuFgpsFF9o2G62mpi7uCesuhiqWNoSggoyypFFuDvGqyC1Pa74ftXw9+I+Y6THoR3Vrg1uft9dfmVhwGlyRrSxj/CUqfARrdPPQtg8TdMsaf3lOVi5zbO8wBWn4FUzp7ytbGaPH6igwcRyzTtugxQxee2BMydTpOry9VNwtZON6lmDwCVLMwW0nlP+eB6ZBhoT/vF4y9Mav054rkwAcGCJ0ns3NoUBhuVS7uBhA23ktxoDTa60+Plp6MrlAiTSUOiP73NWckN5EZDwhraU0IX/wGQzzziUQcAAA==";
+export const VITALS_RAW_BYTES = 1873;
+export const VITALS_GZIP_BYTES = 1042;
+export const VITALS_HASH = "ec87741c5dfdd0cb";