@tokenoftrust/storefront-runner 2.0.3 → 2.0.5

package/apps/storefront/src/lib/tracking/interactionCollector.ts

@@ -0,0 +1,202 @@
+/**
+ * dt-collector-v1 — the browser-side delivery destination for
+ * `TotInteraction` (`@tot/public-runtime`'s declarative interaction-tracking
+ * contract). This module OWNS batching + transport; it does not install the
+ * click listener and does not gate on consent/environment — those are
+ * `interaction-tracking.ts` and `dt-consent-environment-gate` respectively.
+ *
+ * Wiring (owned by whichever module composes the gate in front of delivery,
+ * e.g. `dt-consent-environment-gate`):
+ *
+ *   import { onInteraction } from "@tot/public-runtime";
+ *   import { collectInteraction } from "@/lib/tracking/interactionCollector";
+ *   onInteraction((interaction) => {
+ *     if (consentAndEnvironmentAllow()) collectInteraction(interaction);
+ *   });
+ *
+ * or, where the caller controls install directly:
+ *
+ *   installInteractionTracking({ deliver: collectInteraction });
+ *
+ * PAYLOAD GUARDRAIL: this module reads only the five `TotInteraction` fields
+ * handed to it, plus what it synthesizes itself (`ts`, `correlationId`,
+ * `seq`). No DOM access, no `textContent`/`href`/form reads — same prohibition
+ * as `interaction-tracking.ts`'s listener; see that module's guardrail test
+ * (`tests/interaction-tracking.test.ts`) for the pattern this module must not
+ * violate either.
+ */
+import type { TotInteraction } from "@tot/public-runtime";
+
+/** Where the browser beacon POSTs interaction batches. */
+export const INTERACTION_BEACON_PATH = "/api/tracking/interaction";
+
+/** Max events held in memory before a forced flush — bounds one beacon's size
+ *  and the worst case if flush triggers never fire (e.g. a test/SSR context). */
+export const MAX_QUEUE_SIZE = 20;
+
+/** The wire shape POSTed to `INTERACTION_BEACON_PATH` — `TotInteraction`'s five
+ *  fields plus this module's own synthesized correlation fields. Must stay in
+ *  lockstep with `INTERACTION_WIRE_ALLOWLIST`
+ *  (`apps/storefront/src/lib/tracking/collectorAllowlist.ts`). */
+export interface InteractionWireEvent {
+  type: TotInteraction["type"];
+  el: string;
+  action: string | null;
+  placement: string | null;
+  scope: string | null;
+  index: number | null;
+  /** Client emit clock, ISO-8601. */
+  ts: string;
+  /** Groups every event from one collector lifetime (one page load). */
+  correlationId: string;
+  /** Monotonic within `correlationId`, starting at 0. */
+  seq: number;
+}
+
+/** Build the wire event for one interaction. Pure — no DOM, no clock side
+ *  effects beyond the `now` parameter, so it is unit-testable without faking
+ *  globals. */
+export function toWireEvent(
+  interaction: TotInteraction,
+  correlationId: string,
+  seq: number,
+  now: () => string = () => new Date().toISOString(),
+): InteractionWireEvent {
+  return {
+    type: interaction.type,
+    el: interaction.el,
+    action: interaction.action,
+    placement: interaction.placement,
+    scope: interaction.scope,
+    index: interaction.index,
+    ts: now(),
+    correlationId,
+    seq,
+  };
+}
+
+/** A sink for a finished batch. The default posts one JSON beacon. */
+export type InteractionBatchSender = (events: InteractionWireEvent[]) => void;
+
+/** Default sender: one `sendBeacon` per flushed batch, falling back to a
+ *  keepalive `fetch` where `sendBeacon` is absent (mirrors
+ *  `analytics/webVitals.ts`'s `beaconReporter`). Never throws into the page. */
+export function beaconBatchSender(endpoint: string = INTERACTION_BEACON_PATH): InteractionBatchSender {
+  return (events) => {
+    if (events.length === 0 || typeof navigator === "undefined") return;
+    const body = JSON.stringify({ events });
+    try {
+      if (typeof navigator.sendBeacon === "function") {
+        navigator.sendBeacon(endpoint, body);
+      } else if (typeof fetch === "function") {
+        void fetch(endpoint, { method: "POST", body, keepalive: true });
+      }
+    } catch {
+      // Never let telemetry throw into the page.
+    }
+  };
+}
+
+function randomCorrelationId(): string {
+  const c: Crypto | undefined = (globalThis as { crypto?: Crypto }).crypto;
+  if (c?.randomUUID) return c.randomUUID();
+  return `ic-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+}
+
+/**
+ * A collector instance: batches interactions in memory and flushes them to
+ * `send` on demand, on queue overflow, or when the caller wires
+ * `flush`-on-`visibilitychange`/`pagehide` (see {@link installFlushTriggers}).
+ * Exposed as a factory (rather than one module-level singleton) so tests can
+ * construct an isolated instance with a fake sender and clock.
+ */
+export interface InteractionCollector {
+  /** Enqueue one interaction; synthesizes `ts`/`correlationId`/`seq`. Flushes
+   *  immediately once the queue reaches `MAX_QUEUE_SIZE`. */
+  collect(interaction: TotInteraction): void;
+  /** Send whatever is queued right now (no-op if empty). */
+  flush(): void;
+}
+
+export function createInteractionCollector(
+  send: InteractionBatchSender = beaconBatchSender(),
+  correlationId: string = randomCorrelationId(),
+  now?: () => string,
+): InteractionCollector {
+  let queue: InteractionWireEvent[] = [];
+  let seq = 0;
+
+  const flush = (): void => {
+    if (queue.length === 0) return;
+    const batch = queue;
+    queue = [];
+    send(batch);
+  };
+
+  const collect = (interaction: TotInteraction): void => {
+    queue.push(toWireEvent(interaction, correlationId, seq, now));
+    seq += 1;
+    if (queue.length >= MAX_QUEUE_SIZE) flush();
+  };
+
+  return { collect, flush };
+}
+
+// ---------------------------------------------------------------------------
+// Browser wiring (thin — everything above is pure + unit-tested).
+// ---------------------------------------------------------------------------
+
+/**
+ * Wire `collector.flush` to the two reliable end-of-visibility moments (same
+ * pair `analytics/webVitals.ts#startWebVitals` uses): the page first going
+ * hidden, and `pagehide` — the last point mobile Safari guarantees a beacon
+ * can still be sent. No-op teardown when there is no `document`/`window`
+ * (SSR, tests).
+ */
+export function installFlushTriggers(collector: InteractionCollector): () => void {
+  if (typeof document === "undefined" || typeof window === "undefined") return () => {};
+  const onVisibility = (): void => {
+    if (document.visibilityState === "hidden") collector.flush();
+  };
+  document.addEventListener("visibilitychange", onVisibility, { capture: true });
+  window.addEventListener("pagehide", collector.flush, { capture: true });
+  return () => {
+    document.removeEventListener("visibilitychange", onVisibility, { capture: true });
+    window.removeEventListener("pagehide", collector.flush, { capture: true });
+  };
+}
+
+let defaultCollector: InteractionCollector | null = null;
+let defaultTeardown: (() => void) | null = null;
+
+function getDefaultCollector(): InteractionCollector {
+  if (!defaultCollector) {
+    defaultCollector = createInteractionCollector();
+    defaultTeardown = installFlushTriggers(defaultCollector);
+  }
+  return defaultCollector;
+}
+
+/**
+ * The default delivery destination: `installInteractionTracking({ deliver:
+ * collectInteraction })` or forward into it from an `onInteraction` subscriber
+ * (that is what the consent/environment gate does). Lazily creates the
+ * module-level collector and its flush wiring on first call, so importing
+ * this module has no side effects until an interaction actually arrives.
+ */
+export function collectInteraction(interaction: TotInteraction): void {
+  getDefaultCollector().collect(interaction);
+}
+
+/** Force-flush the default collector (e.g. from a manual "send now" hook). */
+export function flushInteractions(): void {
+  defaultCollector?.flush();
+}
+
+/** Test-only: drop the module-level singleton so a fresh collector/wiring is
+ *  created on the next {@link collectInteraction} call. */
+export function resetDefaultCollectorForTests(): void {
+  defaultTeardown?.();
+  defaultCollector = null;
+  defaultTeardown = null;
+}

package/apps/storefront/src/lib/tracking/interactionHealth.ts

@@ -0,0 +1,252 @@
+/**
+ * The interaction-tracking chain's HEALTH ASSERTION — the thing that makes a
+ * broken collection chain loud instead of silent.
+ *
+ * Why it exists. Every other part of this system is correctly fail-quiet at
+ * runtime: the delegated listener returns without an event when there is no
+ * `data-tot-el` up the ancestor chain (that is the `tracking: "none"` opt-out,
+ * where ABSENCE is the declaration), and the consent/environment gate drops a
+ * disallowed interaction with no error. Both are right. Together they mean a
+ * surface that was never stamped at all produces exactly the same observable
+ * result as a surface nobody clicked — which is the failure mode that made the
+ * legacy mixin useless for years: no way to tell "collection is broken" from
+ * "legitimately idle".
+ *
+ * This module is the one place that CAN tell them apart, because it looks at
+ * the rendered markup rather than at the event stream:
+ *
+ *   - `mount_missing` — the document carries actionable elements but not the
+ *     serialized listener, so no click on it can ever be seen. FATAL.
+ *   - `chrome_unstamped` — the surface carries actionable elements and NOT ONE
+ *     of them carries `data-tot-el`. FATAL. Zero-of-N is not an opt-out
+ *     declaration; a real opt-out is per element, and an author opting out of
+ *     every single element still leaves the OTHER elements stamped. This is
+ *     the finding that covers the **sharedChrome bypass path**: a tenant that
+ *     ships a hand-authored `content/<id>/chrome.html` bypasses governed
+ *     chrome entirely, so `validateChromeConfig` never sees it,
+ *     `trackingAttributes()` never stamps it, and `rewriteTrackingIds`
+ *     (dt-id-writeback) explicitly refuses to touch chrome. Nothing else in
+ *     the chain would ever say a word about it.
+ *   - `stamp_incomplete` — elements carry `data-tot-el` but no
+ *     `data-tot-action`. NOT fatal: the publish-time safety net
+ *     (`publishSafetyNet.ts`) stamps identity only, by design, and there is no
+ *     derivation path from identity to intent. Reported because those rows
+ *     land in the store un-analyzable, which is worth seeing.
+ *
+ * NO DOM-TEXT READS: only `tagName` and the parsed attribute list are ever
+ * read, matching the payload guardrail the listener and every consumer are
+ * held to. Actionability is decided by `@tot/public-runtime`'s shared
+ * `isTrackableNode` — the same oracle the derivation engine and the writeback
+ * rewriter use, so "actionable" cannot mean three different things.
+ */
+import { parse } from "parse5";
+import {
+  INTERACTION_TRACKING_BOOT_JS,
+  TRACKING_ATTR_ACTION,
+  TRACKING_ATTR_EL,
+  isTrackableNode,
+  type StructuralNode,
+} from "@tot/public-runtime";
+
+/** The finding codes, in the order this module reports them. */
+export type TrackingHealthCode = "mount_missing" | "chrome_unstamped" | "stamp_incomplete";
+
+/** The codes that mean the chain is BROKEN rather than merely imperfect. */
+const FATAL_CODES: ReadonlySet<TrackingHealthCode> = new Set<TrackingHealthCode>([
+  "mount_missing",
+  "chrome_unstamped",
+]);
+
+export interface TrackingHealthFinding {
+  code: TrackingHealthCode;
+  /** Human-actionable one-liner; carries counts and tags, never content. */
+  detail: string;
+  fatal: boolean;
+}
+
+export interface TrackingHealthReport {
+  /** True iff there is no FATAL finding — i.e. a click here can be collected. */
+  ok: boolean;
+  /** Elements an interaction can originate from (`isTrackableNode`). */
+  actionable: number;
+  /** Of those, how many carry `data-tot-el`. */
+  stamped: number;
+  /** Of the stamped ones, how many also carry `data-tot-action`. */
+  attributed: number;
+  findings: TrackingHealthFinding[];
+}
+
+export interface TrackingHealthOptions {
+  /**
+   * Require the serialized listener to be present. True for a whole document,
+   * false for a chrome WRAPPER inspected before `rawHtmlResponse()` has
+   * injected the listener into it.
+   */
+  requireMount?: boolean;
+  /** A label for the log/error line — e.g. `chrome.html (tenant acme)`. */
+  surface?: string;
+}
+
+/** Thrown by {@link assertTrackingChainHealthy} on a FATAL finding. */
+export class TrackingChainBrokenError extends Error {
+  readonly report: TrackingHealthReport;
+
+  constructor(report: TrackingHealthReport, surface: string) {
+    const codes = report.findings.filter((f) => f.fatal).map((f) => f.code);
+    super(
+      `interaction tracking is BROKEN on ${surface} [${codes.join(", ")}] — ` +
+        report.findings
+          .filter((f) => f.fatal)
+          .map((f) => f.detail)
+          .join("; ") +
+        ` (actionable=${report.actionable}, stamped=${report.stamped}, ` +
+        `attributed=${report.attributed}). A click on this surface produces NO ` +
+        `retrievable event, and nothing else in the chain reports it: the listener ` +
+        `returns silently when no ${TRACKING_ATTR_EL} is present, which is also how a ` +
+        `legitimate \`tracking: "none"\` opt-out looks.`,
+    );
+    this.name = "TrackingChainBrokenError";
+    this.report = report;
+  }
+}
+
+interface P5Attr {
+  name: string;
+  value: string;
+}
+interface P5Node {
+  nodeName: string;
+  tagName?: string;
+  attrs?: P5Attr[];
+  childNodes?: P5Node[];
+}
+
+function nonEmpty(v: string | undefined): boolean {
+  return typeof v === "string" && v.trim().length > 0;
+}
+
+/**
+ * Inspect one rendered surface (a full document, a chrome wrapper, or a body
+ * fragment). PURE — no I/O, no logging, no throwing. `html` that parses to
+ * nothing actionable is healthy by definition: a surface with no actionable
+ * element has nothing to collect, and saying otherwise would make every
+ * text-only page a false alarm.
+ */
+export function inspectTrackingChain(
+  html: string,
+  options: TrackingHealthOptions = {},
+): TrackingHealthReport {
+  let actionable = 0;
+  let stamped = 0;
+  let attributed = 0;
+  const unstampedTags = new Set<string>();
+
+  const walk = (node: P5Node): void => {
+    if (node.tagName) {
+      const attrs: Record<string, string> = {};
+      for (const a of node.attrs ?? []) attrs[a.name] = a.value;
+      const structural: StructuralNode = { tag: node.tagName.toLowerCase(), attrs };
+      const hasEl = nonEmpty(attrs[TRACKING_ATTR_EL]);
+      if (isTrackableNode(structural)) {
+        actionable += 1;
+        if (hasEl) {
+          stamped += 1;
+          if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
+        } else {
+          unstampedTags.add(structural.tag);
+        }
+      } else if (hasEl) {
+        // Stamped but not actionable by the shared oracle (e.g. a stamped
+        // wrapper). Counted so `stamped` never understates the surface.
+        stamped += 1;
+        if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
+      }
+    }
+    for (const child of node.childNodes ?? []) walk(child);
+  };
+  walk(parse(html) as P5Node);
+
+  const findings: TrackingHealthFinding[] = [];
+
+  if (options.requireMount && actionable > 0 && !html.includes(INTERACTION_TRACKING_BOOT_JS)) {
+    findings.push({
+      code: "mount_missing",
+      detail: `${actionable} actionable element(s) but the serialized interaction-tracking listener is not in this document`,
+      fatal: true,
+    });
+  }
+
+  if (actionable > 0 && stamped === 0) {
+    findings.push({
+      code: "chrome_unstamped",
+      detail:
+        `${actionable} actionable element(s) (${[...unstampedTags].sort().join(", ")}) and NOT ONE ` +
+        `carries ${TRACKING_ATTR_EL} — an unstamped surface, not a per-element opt-out`,
+      fatal: true,
+    });
+  } else if (stamped > attributed) {
+    findings.push({
+      code: "stamp_incomplete",
+      detail: `${stamped - attributed} of ${stamped} stamped element(s) carry no ${TRACKING_ATTR_ACTION}, so their rows land un-analyzable`,
+      fatal: false,
+    });
+  }
+
+  return {
+    ok: !findings.some((f) => f.fatal),
+    actionable,
+    stamped,
+    attributed,
+    findings,
+  };
+}
+
+/**
+ * {@link inspectTrackingChain}, but a FATAL finding THROWS. For build/publish
+ * gates and tests — anywhere a broken chain should stop the line rather than
+ * be noted.
+ */
+export function assertTrackingChainHealthy(
+  html: string,
+  options: TrackingHealthOptions = {},
+): TrackingHealthReport {
+  const report = inspectTrackingChain(html, options);
+  if (!report.ok) throw new TrackingChainBrokenError(report, options.surface ?? "this surface");
+  return report;
+}
+
+/**
+ * The render-path form: NEVER throws, emits ONE distinct structured line per
+ * distinct outcome so a broken surface is visible in Workers Logs. Returns the
+ * report so a caller can act on it.
+ *
+ * Non-throwing on purpose — a tenant whose hand-authored chrome was never
+ * stamped should get a loud log line, not a blank page.
+ */
+const reported = new Set<string>();
+/** Bound the dedupe set so a long-lived isolate cannot grow it without limit. */
+const REPORTED_MAX = 64;
+
+export function reportTrackingChainHealth(
+  html: string,
+  options: TrackingHealthOptions = {},
+): TrackingHealthReport {
+  const report = inspectTrackingChain(html, options);
+  if (report.findings.length === 0) return report;
+  const surface = options.surface ?? "unknown-surface";
+  const key = `${surface}|${report.findings.map((f) => f.code).join(",")}|${report.actionable}/${report.stamped}`;
+  if (reported.has(key)) return report;
+  if (reported.size >= REPORTED_MAX) reported.clear();
+  reported.add(key);
+  for (const finding of report.findings) {
+    const line = `[tracking:health] ${finding.fatal ? "BROKEN" : "degraded"} ${finding.code} on ${surface}: ${finding.detail}`;
+    if (finding.fatal) console.error(line);
+    else console.warn(line);
+  }
+  return report;
+}
+
+/** Test-only: forget which lines have already been emitted. */
+export function resetTrackingHealthReportingForTests(): void {
+  reported.clear();
+}

package/apps/storefront/src/lib/tracking/interactionSchema.ts

@@ -0,0 +1,107 @@
+/**
+ * The RAW interaction-event table — the schema of a TENANT'S OWN D1 database.
+ *
+ * PER-TENANT, NOT PER-COLUMN. Every other D1 table in this repo lives in a
+ * shared database and carries a `tenant_id` column as its isolation boundary
+ * (see `d1/schema-apps.ts`). This one does NOT work that way: each tenant gets
+ * its OWN D1 database holding only its own rows, so cross-tenant leakage is
+ * impossible BY CONSTRUCTION rather than by a correct `WHERE tenant_id = ?` in
+ * every query. A missing predicate here cannot reach another tenant's data —
+ * there is no other tenant's data in the file.
+ *
+ * `tenant_id` is still stored on the row (redundantly, one constant value per
+ * database) deliberately: it makes an exported/backed-up/rolled-up row
+ * self-describing, and it lets a mis-routed insert be DETECTED (a row whose
+ * `tenant_id` disagrees with the database's owner is a routing bug, and
+ * `dt-tenant-isolation` can assert on exactly that).
+ *
+ * WHY D1 AND NOT A SHARED STORE: D1 is not billed per database — thousands of
+ * databases cost the same as one — so per-tenant databases are the endorsed
+ * shape, not a workaround. The hard limit is 10GB PER DATABASE (~30M rows at a
+ * 90-day window for a busy tenant), which is a per-tenant ceiling rather than a
+ * global one; `interaction_store_meta` below carries the counters
+ * `dt-prune-and-rollup` needs to surface a tenant approaching it.
+ *
+ * RETENTION is a 90-day window, enforced by a DELETE-based prune in a later
+ * unit (`dt-prune-and-rollup`) — D1 has no TTL. This schema only has to make
+ * that prune cheap, which is what `idx_interaction_events_ingested_at` is for:
+ * `DELETE FROM interaction_events WHERE ingested_at < ?` is an index range
+ * scan, not a table scan. `ingested_at` (the SERVER receive clock) is the prune
+ * key, never `ts` (a CLIENT clock a beacon can skew arbitrarily).
+ *
+ * SCHEMA IS APPLIED, NOT MIGRATED. A per-tenant database is created on demand
+ * at runtime, so it cannot ride the numbered `migrations-apps/` lineage (which
+ * `wrangler d1 migrations apply` runs against ONE named binding). Instead this
+ * DDL is idempotent (`IF NOT EXISTS` throughout) and applied at provision time;
+ * `INTERACTION_SCHEMA_VERSION` is recorded in `interaction_store_meta` so a
+ * future version bump can be detected and re-applied per tenant.
+ */
+
+/** Bumped whenever `INTERACTION_D1_SCHEMA` changes; recorded per database. */
+export const INTERACTION_SCHEMA_VERSION = 1;
+
+/**
+ * Idempotent DDL for one tenant's raw-interaction database. Multi-statement:
+ * applied via `Queryable.exec`, which both the D1 binding and the D1 HTTP API
+ * accept as a semicolon-separated script.
+ */
+export const INTERACTION_D1_SCHEMA = `
+CREATE TABLE IF NOT EXISTS interaction_events (
+  -- Owning tenant. One constant value per database (see the module note): the
+  -- isolation boundary is the DATABASE, this column is provenance + a
+  -- mis-routing tripwire.
+  tenant_id TEXT NOT NULL,
+  -- Stream discriminator, always "interaction" today. Present from row one so a
+  -- second stream can land in the same table without a migration.
+  stream TEXT NOT NULL,
+  -- The six TotInteraction fields, flattened for SQL aggregation.
+  type TEXT NOT NULL,
+  el TEXT NOT NULL,
+  action TEXT,
+  placement TEXT,
+  scope TEXT,
+  idx INTEGER,
+  -- Groups the events of one collector lifetime (~one page load).
+  correlation_id TEXT NOT NULL,
+  -- Monotonic within correlation_id, assigned by the client collector.
+  seq INTEGER NOT NULL,
+  -- Client emit clock (ISO-8601). Informational: never the prune or ordering key.
+  ts TEXT NOT NULL,
+  -- Server receive clock (ISO-8601). THE retention/prune key.
+  ingested_at TEXT NOT NULL,
+  -- (correlation_id, seq) is the collector's own natural identity, so a
+  -- retried/duplicated beacon re-inserting the same event is a no-op under
+  -- INSERT OR IGNORE rather than a double-count.
+  PRIMARY KEY (correlation_id, seq)
+);
+CREATE INDEX IF NOT EXISTS idx_interaction_events_ingested_at
+  ON interaction_events (ingested_at);
+CREATE INDEX IF NOT EXISTS idx_interaction_events_stream_ingested_at
+  ON interaction_events (stream, ingested_at);
+CREATE INDEX IF NOT EXISTS idx_interaction_events_el
+  ON interaction_events (el, ingested_at);
+CREATE TABLE IF NOT EXISTS interaction_store_meta (
+  -- Single-row table. The constant primary key makes that a schema invariant.
+  -- (Keep every inline comment free of semicolons: this DDL is executed as one
+  -- semicolon-separated script.)
+  id INTEGER PRIMARY KEY CHECK (id = 1),
+  tenant_id TEXT NOT NULL,
+  schema_version INTEGER NOT NULL,
+  created_at TEXT NOT NULL,
+  updated_at TEXT NOT NULL
+);
+`;
+
+/** Statements that stamp/refresh the single `interaction_store_meta` row. */
+export function interactionMetaUpsert(tenantId: string, at: string): {
+  sql: string;
+  params: readonly unknown[];
+} {
+  return {
+    sql: `INSERT INTO interaction_store_meta (id, tenant_id, schema_version, created_at, updated_at)
+          VALUES (1, ?, ?, ?, ?)
+          ON CONFLICT(id) DO UPDATE SET schema_version = excluded.schema_version,
+                                        updated_at = excluded.updated_at`,
+    params: [tenantId, INTERACTION_SCHEMA_VERSION, at, at],
+  };
+}