@iann29/rastro 0.1.0-alpha.6 → 0.1.0-alpha.8

package/docs/federation-setup.md

@@ -304,6 +304,8 @@ export const {
   goalsReport,
   funnelsReport,
   affiliatesReport,
+  vitalsReport,
+  siteMap,
   dataCoverage,
 } = federated;
 ```

package/docs/federation.md

@@ -48,6 +48,8 @@ The exact generated surface is:
 - `funnelsReport`
 - `affiliatesReport`
 - `dataCoverage`
+- `vitalsReport` (optional capability `vitals`, hosts from `alpha.6`)
+- `siteMap` (optional capability `siteMap`, hosts from `alpha.8`)
 
 There are no ingestion, configuration-write, retention-write, raw-site, or site
 mutation functions in this surface. The read-only goal, funnel, and affiliate
@@ -185,6 +187,8 @@ export const {
   goalsReport,
   funnelsReport,
   affiliatesReport,
+  vitalsReport,
+  siteMap,
   dataCoverage,
 } = federated;
 ```

package/docs/upgrading.md

@@ -6,14 +6,17 @@ deployment.
 
 ## Release matrix
 
-| Current state                                         | Required path                                                             | Success condition                                               |
-| ----------------------------------------------------- | ------------------------------------------------------------------------- | --------------------------------------------------------------- |
-| Fresh deployment                                      | Install the intended release directly                                     | Component deploys and `/rastro/health` succeeds                 |
-| `0.1.0-alpha.0` with maintained data                  | Install `0.1.0-alpha.1` first and wait for staged visitor-index backfills | Every staged index reports complete before installing `alpha.2` |
-| `0.1.0-alpha.1`                                       | Wait for staged visitor-index backfills, then install `alpha.2`           | Backfills complete before active readers deploy                 |
-| `0.1.0-alpha.2` with legacy `events` rows             | Stay on `alpha.2` and run/resume migration                                | Durable success plus empty source                               |
-| `0.1.0-alpha.3` fresh/canonical deployment            | Rehearse the next alpha normally                                          | Schema, reports, tracker, and retention checks pass             |
-| Any release whose packed declarations lack federation | Stop and request an eligible exact registry release                       | Packed declarations contain the federation exports              |
+| Current state                                         | Required path                                                                       | Success condition                                                |
+| ----------------------------------------------------- | ----------------------------------------------------------------------------------- | ---------------------------------------------------------------- |
+| Fresh deployment                                      | Install the intended release directly                                               | Component deploys and `/rastro/health` succeeds                  |
+| `0.1.0-alpha.0` with maintained data                  | Install `0.1.0-alpha.1` first and wait for staged visitor-index backfills           | Every staged index reports complete before installing `alpha.2`  |
+| `0.1.0-alpha.1`                                       | Wait for staged visitor-index backfills, then install `alpha.2`                     | Backfills complete before active readers deploy                  |
+| `0.1.0-alpha.2` with legacy `events` rows             | Stay on `alpha.2` and run/resume migration                                          | Durable success plus empty source                                |
+| `0.1.0-alpha.3` fresh/canonical deployment            | Rehearse the next alpha normally                                                    | Schema, reports, tracker, and retention checks pass              |
+| `0.1.0-alpha.3` through `alpha.5` with canonical data | Install `0.1.0-alpha.6` directly; Web Vitals tables are additive                    | Schema deploys and `/rastro/health` reports `webVitals: "optIn"` |
+| `0.1.0-alpha.6`                                       | Install `0.1.0-alpha.7` directly; no schema change                                  | `GET /rastro/tracker.js` contains `sentAt`                       |
+| `0.1.0-alpha.7`                                       | Install `0.1.0-alpha.8` directly; site map tables are additive; re-export `siteMap` | `/rastro/health` reports `siteMap: "dailyRollup"`                |
+| Any release whose packed declarations lack federation | Stop and request an eligible exact registry release                                 | Packed declarations contain the federation exports               |
 
 Do not upgrade a populated legacy deployment directly to a release that removes
 the `events` table. Schema acceptance alone does not prove that telemetry was
@@ -93,6 +96,80 @@ The visitor-time indexes on legacy and canonical telemetry follow this rule.
 Check the release notes for the exact intermediate version before upgrading a
 large deployment.
 
+## Web Vitals (alpha.6)
+
+`0.1.0-alpha.6` adds field Web Vitals as a purely additive upgrade: a new
+`vitalRollups` component table, the `vital` event type, the `vitalsReport`
+query, the retention kind `vitalRollups` (governed by `reportsDays`), and the
+pre-gzipped `GET /rastro/vitals.js` module. No data migration and no staged
+indexes are involved — install and deploy through the host's normal workflow.
+
+Measurement is opt-in per page. Nothing reports until the host adds the second
+script tag documented in the README ("Field Web Vitals (optional)"); the base
+tracker is byte-for-byte unchanged. After deploying, the federation manifest
+advertises the optional `vitals` capability; in the central dashboard, open
+**Conexões** and click **Verificar novamente** on the active connection so the
+stored capability list picks it up and the **Vitais** view replaces its upgrade
+notice. A dashboard talking to an older host shows that notice by design — the
+protocol-v1 baseline does not require the capability.
+
+Acceptance for this upgrade: `/rastro/health` reports
+`features.webVitals: "optIn"`, `GET /rastro/vitals.js` serves the module, and
+after one real page load is hidden, `vitalsReport` shows the measurement.
+
+## Clock alignment (alpha.7)
+
+`0.1.0-alpha.7` changes no schema. The served `tracker.js` and `vitals.js` stamp
+each batch with `sentAt`, and `POST /rastro/events` re-anchors a batch whose
+sender clock is more than 30 seconds from the server's before ingestion. Live
+presence lasts five minutes after the last event or heartbeat instead of one.
+Browsers keep the previous tracker for at most the five-minute cache plus one
+background revalidation; a batch without `sentAt` is ingested exactly as before.
+
+Acceptance for this upgrade: `/rastro/health` succeeds, `GET /rastro/tracker.js`
+contains `sentAt`, and a visitor whose device clock is minutes off appears in
+**Ao vivo** within seconds of loading a page.
+
+## Site map (alpha.8)
+
+`0.1.0-alpha.8` adds the route map as a purely additive upgrade: a new
+`routeDailyRollups` component table fed at ingestion, optional fields on
+existing tables (`liveSessions.previousPath`/`movedAt`,
+`sessions.lastPageviewAt`, `analyticsCoverage.siteMapSince`), the `siteMap`
+query, and the retention kind `siteMapRollups` (governed by `reportsDays`). No
+data migration and no staged indexes are involved — install and deploy through
+the host's normal workflow. Ingestion writes one extra rollup document per
+distinct route in a batch.
+
+One federation edit is required. The connector manifest is static package
+metadata: after this upgrade it advertises the optional `siteMap` capability
+whether or not the host module exports the function, and connection verification
+checks the manifest's shape, not each function's existence. A host that upgrades
+the package without re-exporting the query leaves the dashboard's **Mapa** view
+failing with a missing-function error. Add `siteMap` to the destructured export
+in `convex/rastroFederation.ts` before deploying:
+
+```ts
+export const {
+  manifest,
+  // ...the existing exports...
+  vitalsReport,
+  siteMap,
+  dataCoverage,
+} = federated;
+```
+
+The map has no history to backfill: coverage starts at `siteMapSince`, stamped
+by the first ingest after the upgrade, and the dashboard names the date its map
+data begins. In the central dashboard, open **Conexões** and click **Verificar
+novamente** on the active connection so the stored capability list picks up
+`siteMap` and the **Mapa** view replaces its upgrade notice.
+
+Acceptance for this upgrade: `/rastro/health` reports
+`features.siteMap: "dailyRollup"`, and after one real visit that navigates
+between two pages, `siteMap` returns both routes and the transition between
+them.
+
 ## Federation releases
 
 Federation is additive to component telemetry but requires host tables,
@@ -120,6 +197,10 @@ control-plane issuer.
 - Staged indexes completed before their readers were activated.
 - `/rastro/health` succeeds on the HTTP-actions URL.
 - A real tracker request is accepted from an allowed origin.
+- When the host opts into Web Vitals: `/rastro/vitals.js` serves and
+  `vitalsReport` reflects a real page load.
+- On `alpha.8` or later: the federation module re-exports `siteMap` and, after a
+  real two-page visit, `siteMap` returns both routes and their transition.
 - Protected overview, journey, goal, funnel, and affiliate reports used by the
   host still run.
 - Trusted conversion idempotency and currency checks still pass.

package/llms.txt

@@ -16,6 +16,7 @@ Do not infer federation availability from this website, a dist-tag, or a source
 ## Architecture
 
 - Tracker and telemetry: browser -> customer HTTP-actions URL -> customer Rastro component.
+- Optional field Web Vitals (alpha.6+): a second browser module at `/rastro/vitals.js` reports LCP, CLS, INP, FCP, and TTFB to the same events endpoint, into bounded daily histograms.
 - Reactive dashboard reads: amagerastro.com browser -> customer functions URL -> `rastroFederation:*`.
 - Control plane: account, organization, connection metadata, short-lived token issuance, and revocation.
 - The control plane does not proxy or persist customer analytics events.
@@ -29,7 +30,7 @@ Do not infer federation availability from this website, a dist-tag, or a source
 - Resolve an indexed, authoritative local grant on every federated query.
 - Grant one through ten explicit site IDs. Never grant by owner-wide enumeration.
 - Use the canonical module name `convex/rastroFederation.ts`.
-- Expose only the manifest, connection status, site summaries, reports, journeys, conversions, affiliates, funnels, goals, and coverage listed in the manifest.
+- Expose only the manifest, connection status, site summaries, reports, journeys, conversions, affiliates, funnels, goals, web vitals, and coverage listed in the manifest.
 - Use the functions URL as JWT audience and the HTTP-actions URL for tracker ingestion.
 - Rehearse on an explicit development target. Production changes require fresh operator consent.
 - Validate manifest, authenticated connection status, redacted site list, real ingestion, reactive live data, and local-first revocation.
@@ -51,7 +52,10 @@ Do not infer federation availability from this website, a dist-tag, or a source
 - Hourly overview maximum: 24 hours.
 - Funnel report maximum: 90 complete UTC days.
 - Affiliate reports require complete UTC-day boundaries.
+- Vitals report maximum: 90 complete UTC days; `vitals` is an optional capability that hosts before `alpha.6` do not advertise.
+- Site map maximum: 90 complete UTC days, read newest day first within 8,000 route rows; older days it could not read are declared as partial coverage. `siteMap` is an optional capability that hosts before `alpha.8` do not advertise, and its data starts at the first ingest after that upgrade.
 - Maximum live visitors: 500.
+- Live window: five minutes since the visitor's last event or heartbeat. `POST /rastro/events` re-anchors a batch whose `sentAt` is more than 30 seconds from server time, so a wrong device clock decides neither liveness nor hour buckets.
 - Maximum journey events: 500.
 - Token lifetime: 10 minutes, RS256, permission `analytics:read`.
 
@@ -63,3 +67,5 @@ Do not infer federation availability from this website, a dist-tag, or a source
 - Audit retained routes, identity strategy, sessions, live location, campaigns, affiliates, conversions, custom properties, retention, and intended organization membership before granting federation; stop if their sensitivity or audience cannot be established.
 - Overview ranges require complete UTC hour or day buckets. Manual pagination honors `numItems` and `cursor` only.
 - `dataCoverage` echoes the authorized site ID but returns no event, visitor, or session IDs.
+- `vitalsReport` returns p75, averages, and exact good/needs-improvement/poor counts per metric, page, and device — never individual measurements.
+- `siteMap` returns per-route totals (pageviews, estimated visitors, entries, derived exits and bounces, time on page) and route-to-route transition counts, with numeric, UUID, and long hexadecimal segments collapsed to `:id` — never sessions, visitor IDs, or query strings. `liveVisitors` adds `previousPath` and `movedAt` when a visitor moves between routes.

package/package.json

@@ -6,7 +6,7 @@
   "bugs": {
     "url": "https://github.com/amageweb/amage-rastro/issues"
   },
-  "version": "0.1.0-alpha.6",
+  "version": "0.1.0-alpha.8",
   "license": "Apache-2.0",
   "publishConfig": {
     "access": "public"

package/src/component/_generated/component.ts

@@ -306,7 +306,8 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "goals"
               | "funnels"
               | "affiliates"
-              | "vitals";
+              | "vitals"
+              | "siteMap";
           }>;
           from: number;
           siteId: string;
@@ -515,9 +516,11 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
           lastSeenAt: number;
           latitude?: number;
           longitude?: number;
+          movedAt?: number;
           os: string;
           pageviewCount: number;
           path: string;
+          previousPath?: string;
           sessionId: string;
           siteId: string;
           source: string;
@@ -703,6 +706,54 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
         },
         Name
       >;
+      siteMap: FunctionReference<
+        "query",
+        "internal",
+        { from: number; siteIds: Array<string>; to: number },
+        {
+          from: number;
+          metadata: {
+            basis: "dailyRollup" | "boundedLegacy";
+            coverage: {
+              availableFrom: number | null;
+              availableThrough: number | null;
+              generation: number | null;
+              retainedBefore: number | null;
+              state: "complete" | "partial" | "retained" | "unavailable";
+            };
+            limits: {
+              routesPerDay: number;
+              rows: number;
+              transitionSlots: number;
+            };
+            range: {
+              boundary: "inclusive";
+              from: number;
+              timezone: "UTC";
+              to: number;
+            };
+            visitors: { basis: "estimatedUnique"; exact: boolean };
+          };
+          routes: Array<{
+            bounces: number;
+            daily: Array<{ bucketStart: number; pageviews: number }>;
+            dwellMs: number;
+            dwellSamples: number;
+            entries: number;
+            exits: number;
+            pageviews: number;
+            route: string;
+            transitionsIn: number;
+            transitionsOut: number;
+            visitors: number;
+          }>;
+          to: number;
+          transitions: Array<{ count: number; from: string; to: string }>;
+          truncated: { routes: boolean; transitions: boolean };
+          window: { from: number; to: number } | null;
+        },
+        Name
+      >;
       visitorJourney: FunctionReference<
         "query",
         "internal",
@@ -828,6 +879,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "funnelRollups"
             | "affiliateRollups"
             | "vitalRollups"
+            | "siteMapRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
@@ -863,6 +915,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
             | "funnelRollups"
             | "affiliateRollups"
             | "vitalRollups"
+            | "siteMapRollups"
             | "attributions"
             | "liveSessions"
             | "ingestWindows"
@@ -909,6 +962,7 @@ export type ComponentApi<Name extends string | undefined = string | undefined> =
               | "funnelRollups"
               | "affiliateRollups"
               | "vitalRollups"
+              | "siteMapRollups"
               | "attributions"
               | "liveSessions"
               | "ingestWindows"

package/src/component/constants.ts

@@ -7,8 +7,9 @@ export const MAX_BYTES_PER_SESSION_WINDOW = 256 * 1024;
 export const SITE_INGEST_SHARDS = 4_096;
 export const MAX_SITE_EVENTS_PER_SHARD_WINDOW = 300;
 export const MAX_SITE_BYTES_PER_SHARD_WINDOW = 512 * 1024;
-export const LIVE_SESSION_TTL_MS = 60_000;
+export const LIVE_SESSION_TTL_MS = 5 * 60_000;
 export const LIVE_SWEEP_INTERVAL_MS = 5_000;
+export const CLOCK_SKEW_TOLERANCE_MS = 30_000;
 export const AGGREGATE_SHARDS = 128;
 export const HOURLY_AGGREGATE_SHARDS = 256;
 export const SECONDARY_AGGREGATE_SHARDS = 16;
@@ -53,3 +54,14 @@ export const HOUR_MS = 3_600_000;
 export const MINUTE_MS = 60_000;
 export const RETENTION_POLICY_INTERVAL_MS = DAY_MS;
 export const MAX_VISITOR_ALIASES = 10;
+// Site map rollups: one row per (site, route, UTC day, shard). A day keeps at
+// most this many route rows per shard before new routes fold into "(other)";
+// each row keeps this many outgoing targets before they fold the same way. A
+// report reads at most MAX_SITE_MAP_ROWS rows, newest day first — about
+// 5 MiB of rows, under the 8 MiB a function may read once the feature-report
+// byte reserve is kept — and declares the older days it could not read as
+// partial coverage: 20 days at the daily cap, 40 for a 100-route site.
+export const MAX_SITE_MAP_ROUTES_PER_DAY = 200;
+export const ROUTE_TRANSITION_SLOTS = 24;
+export const MAX_SITE_MAP_ROWS = 8_000;
+export const SITE_MAP_OTHER_ROUTE = "(other)";

package/src/component/coverage.ts

@@ -12,11 +12,22 @@ export async function ensureAnalyticsControl(
     .query("analyticsCoverage")
     .withIndex("by_siteId", (range) => range.eq("siteId", siteId))
     .unique();
-  if (existing) return existing;
+  if (existing) {
+    if (existing.siteMapSince !== undefined) return existing;
+    // A site that predates the site map starts its map coverage at the first
+    // ingest after the upgrade; the same safety delay applies as for rollups.
+    const siteMapSince = now + ROLLUP_SAFETY_DELAY_MS;
+    await ctx.db.patch("analyticsCoverage", existing._id, {
+      siteMapSince,
+      updatedAt: now,
+    });
+    return { ...existing, siteMapSince, updatedAt: now };
+  }
   const fields = {
     siteId,
     generation: 1 as const,
     rollupCompleteFrom: now + ROLLUP_SAFETY_DELAY_MS,
+    siteMapSince: now + ROLLUP_SAFETY_DELAY_MS,
     retained,
     updatedAt: now,
   };

package/src/component/http.ts

@@ -19,7 +19,7 @@ import {
   resolveGeoIp,
 } from "./geo.js";
 import { isPlainRecord } from "./guards.js";
-import { sanitizeEvent } from "./sanitize.js";
+import { alignClock, sanitizeEvent } from "./sanitize.js";
 import { deriveVisitorKey, randomSecret } from "./identity.js";
 import { classifyClient, isKnownBot } from "./useragent.js";
 import type { TrackerEvent } from "./validators.js";
@@ -62,7 +62,7 @@ http.route({
       );
     }
 
-    let payload: { siteId: string; events: TrackerEvent[] };
+    let payload: { siteId: string; events: TrackerEvent[]; sentAt?: number };
     try {
       const bytes = await readLimitedBody(request, MAX_BATCH_BYTES);
       const parsed: unknown = JSON.parse(new TextDecoder().decode(bytes));
@@ -87,6 +87,12 @@ http.route({
         headers,
       );
     }
+    // The sender's clock says when its events happened, but a clock minutes
+    // off would keep the visitor out of the live view and file the traffic
+    // under another hour. The batch also says when it was sent, and that offset
+    // re-anchors every event here, before the transaction and the geo
+    // enrichment read them, so both see the same timestamps.
+    const events = alignClock(payload.events, payload.sentAt, Date.now());
 
     const trustProxy = env.RASTRO_TRUST_PROXY === "true";
     const metadata = await readRequestMetadata(ctx);
@@ -99,14 +105,14 @@ http.route({
     // counter shape the tracker expects and names the reason in a header.
     if (isKnownBot(userAgent)) {
       return json(
-        { accepted: 0, duplicates: 0, rejected: payload.events.length },
+        { accepted: 0, duplicates: 0, rejected: events.length },
         200,
         { ...headers, "X-Rastro-Rejected": "bot" },
       );
     }
     // Anonymous batches (visitorId === sessionId) get a salted daily key so
     // repeat visits count once; the IP itself stops here.
-    const anonymous = payload.events.some(
+    const anonymous = events.some(
       (event) => event.visitorId === event.sessionId,
     );
     const visitorKey =
@@ -129,7 +135,7 @@ http.route({
       const result = await ctx.runMutation(api.ingest.ingestBatch, {
         siteId: payload.siteId as Id<"sites">,
         origin,
-        events: payload.events,
+        events,
         context,
       });
       if (
@@ -138,13 +144,13 @@ http.route({
         geoIpStatus(env.RASTRO_GEOIP_PROVIDER, env.RASTRO_GEOIP_TOKEN) ===
           "ready"
       ) {
-        const events = enrichmentCandidateEvents(payload.events);
-        if (events) {
+        const candidates = enrichmentCandidateEvents(events);
+        if (candidates) {
           let stage: GeoIpFailureStage = "claim";
           try {
             const claim = await ctx.runMutation(internal.geo.claimEnrichment, {
               siteId: payload.siteId as Id<"sites">,
-              events,
+              events: candidates,
               dailyLimit: geoIpDailyLimit(env.RASTRO_GEOIP_DAILY_LIMIT),
             });
             if (claim.sessionIds.length > 0) {
@@ -292,6 +298,7 @@ http.route({
               ? "dailyHash"
               : "session",
           webVitals: "optIn",
+          siteMap: "dailyRollup",
         },
       },
       200,
@@ -306,6 +313,7 @@ http.route({
 function parsePayload(value: unknown): {
   siteId: string;
   events: TrackerEvent[];
+  sentAt?: number;
 } {
   if (
     !isPlainRecord(value) ||
@@ -314,7 +322,12 @@ function parsePayload(value: unknown): {
   ) {
     throw new Error("body must contain siteId and events");
   }
-  return { siteId: value.siteId, events: value.events.map(parseEvent) };
+  const sentAt = optionalNumber(value.sentAt, "sentAt");
+  return {
+    siteId: value.siteId,
+    events: value.events.map(parseEvent),
+    ...(sentAt === undefined ? {} : { sentAt }),
+  };
 }
 
 class PayloadTooLargeError extends Error {}