@tokenoftrust/storefront-runner 2.0.4 → 2.0.5

package/apps/storefront/src/lib/tracking/collectorIngest.ts

@@ -0,0 +1,188 @@
+/**
+ * Ingest validation for `POST /api/tracking/interaction` (dt-collector-v1).
+ *
+ * PURE (no I/O) — a thin auth-free parse/validate shell, same split as
+ * `activity/ingest.ts`: this module decides accept/reject, the route
+ * (`pages/api/tracking/interaction.ts`) does auth/transport/persistence.
+ *
+ * DEFAULT-DENY ALLOWLIST (reuses the posture of `activity/redaction.ts` /
+ * `activity/ingest.ts`'s `createActivityEvent()` re-validation, not their
+ * code — there is no per-action catalog for a fixed-shape event):
+ *   1. Only keys in `INTERACTION_WIRE_ALLOWLIST` are ever read from the raw
+ *      body. Every other key is DROPPED and COUNTED (`droppedKeys`) — never
+ *      stored, never logged with its value.
+ *   2. A declared key with the WRONG TYPE is never coerced — the whole event
+ *      is rejected with a reason naming the field, exactly like a malformed
+ *      `activity` event is rejected rather than papered over.
+ *
+ * TENANT + STREAM ISOLATION. `tenant_id` and `stream` are NEVER read from the
+ * posted body (they are not even in the allowlist) — the route stamps both
+ * server-side on every accepted event: `tenant_id` from the authenticated
+ * `context.locals.tenant.tenant_id`, `stream` as the fixed literal this module
+ * exports (`INTERACTION_STREAM`). A client cannot claim another tenant's scope
+ * or another stream.
+ */
+import { isIsoTimestamp } from "@tot/public-runtime";
+import { isAllowedInteractionKey } from "./collectorAllowlist.js";
+
+/** The single stream this collector accepts in V1. Never client-supplied —
+ *  stamped server-side so `dt-storage-per-tenant-d1` has a stable discriminator
+ *  column ready before a second stream (e.g. a future custom `track()` call)
+ *  exists. */
+export const INTERACTION_STREAM = "interaction" as const;
+
+/** Interaction kinds this collector accepts. Mirrors `TotInteraction["type"]`
+ *  (currently "click" only — V1 has no other interaction kind). */
+const INTERACTION_TYPES = new Set(["click"]);
+
+/** A validated wire event, before tenant/stream are stamped. */
+export interface InteractionWireEvent {
+  type: "click";
+  el: string;
+  action: string | null;
+  placement: string | null;
+  scope: string | null;
+  index: number | null;
+  /** Client emit clock (ISO-8601). Informational only — never trusted for
+   *  ordering guarantees; the store also has its own receive clock. */
+  ts: string;
+  /** Groups events emitted by one collector lifetime (roughly: one page load). */
+  correlationId: string;
+  /** Monotonic within `correlationId`, assigned by the client collector. */
+  seq: number;
+}
+
+/** The stored/forwarded shape: a validated wire event plus the server stamp. */
+export interface StampedInteractionEvent extends InteractionWireEvent {
+  tenant_id: string;
+  stream: typeof INTERACTION_STREAM;
+  /** Server receive clock (ISO-8601), stamped once per ingest. */
+  ingested_at: string;
+}
+
+export interface InteractionRejection {
+  index: number;
+  reason:
+    | "not_object"
+    | "unknown_type"
+    | "el_invalid"
+    | "action_invalid"
+    | "placement_invalid"
+    | "scope_invalid"
+    | "index_invalid"
+    | "ts_invalid"
+    | "correlation_id_invalid"
+    | "seq_invalid";
+}
+
+export interface AcceptedInteraction {
+  event: StampedInteractionEvent;
+  /** Non-allowlisted keys present on the raw body, dropped before validation
+   *  (fact-of, never the value) — for audit / alerting, same convention as
+   *  `RedactionFieldHit`. */
+  droppedKeys: string[];
+}
+
+export interface InteractionIngestOutcome {
+  accepted: AcceptedInteraction[];
+  rejected: InteractionRejection[];
+}
+
+const MAX_CORRELATION_ID_LEN = 128;
+
+function nullableString(v: unknown): v is string | null {
+  return v === null || v === undefined || typeof v === "string";
+}
+
+function asNullableString(v: unknown): string | null {
+  return typeof v === "string" ? v : null;
+}
+
+/**
+ * Validate + stamp one posted interaction under the authenticated tenant.
+ * Strips any non-allowlisted key first (dropped + counted, never stored),
+ * then validates each allowlisted field's TYPE — a wrong-typed field rejects
+ * the whole event rather than being coerced.
+ */
+export function ingestInteraction(
+  index: number,
+  raw: unknown,
+  tenantId: string,
+  ingestedAt: string,
+): AcceptedInteraction | InteractionRejection {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return { index, reason: "not_object" };
+  const e = raw as Record<string, unknown>;
+
+  const droppedKeys = Object.keys(e).filter((k) => !isAllowedInteractionKey(k));
+
+  const type = e.type;
+  if (typeof type !== "string" || !INTERACTION_TYPES.has(type)) return { index, reason: "unknown_type" };
+
+  const el = e.el;
+  if (typeof el !== "string" || el.trim().length === 0) return { index, reason: "el_invalid" };
+
+  if (!nullableString(e.action)) return { index, reason: "action_invalid" };
+  if (!nullableString(e.placement)) return { index, reason: "placement_invalid" };
+  if (!nullableString(e.scope)) return { index, reason: "scope_invalid" };
+
+  const rawIndex = e.index;
+  let idx: number | null;
+  if (rawIndex === null || rawIndex === undefined) {
+    idx = null;
+  } else if (typeof rawIndex === "number" && Number.isInteger(rawIndex) && rawIndex >= 0) {
+    idx = rawIndex;
+  } else {
+    return { index, reason: "index_invalid" };
+  }
+
+  const ts = e.ts;
+  if (!isIsoTimestamp(ts)) return { index, reason: "ts_invalid" };
+
+  const correlationId = e.correlationId;
+  if (
+    typeof correlationId !== "string" ||
+    correlationId.trim().length === 0 ||
+    correlationId.length > MAX_CORRELATION_ID_LEN
+  ) {
+    return { index, reason: "correlation_id_invalid" };
+  }
+
+  const seq = e.seq;
+  if (typeof seq !== "number" || !Number.isInteger(seq) || seq < 0) {
+    return { index, reason: "seq_invalid" };
+  }
+
+  const event: StampedInteractionEvent = {
+    type: "click",
+    el,
+    action: asNullableString(e.action),
+    placement: asNullableString(e.placement),
+    scope: asNullableString(e.scope),
+    index: idx,
+    ts,
+    correlationId,
+    seq,
+    // Server-stamped, never from the client.
+    tenant_id: tenantId,
+    stream: INTERACTION_STREAM,
+    ingested_at: ingestedAt,
+  };
+
+  return { event, droppedKeys };
+}
+
+/** Validate + stamp a whole posted batch under the authenticated tenant. */
+export function ingestInteractions(
+  events: unknown[],
+  tenantId: string,
+  ingestedAt: string,
+): InteractionIngestOutcome {
+  const accepted: AcceptedInteraction[] = [];
+  const rejected: InteractionRejection[] = [];
+  events.forEach((raw, i) => {
+    const r = ingestInteraction(i, raw, tenantId, ingestedAt);
+    if ("event" in r) accepted.push(r);
+    else rejected.push(r);
+  });
+  return { accepted, rejected };
+}

package/apps/storefront/src/lib/tracking/d1Control.ts

@@ -0,0 +1,180 @@
+/**
+ * Cloudflare D1 control + data plane over the REST API — how a PER-TENANT D1
+ * database is created and queried from inside the Worker.
+ *
+ * WHY REST AND NOT A BINDING. Worker bindings are STATIC: `wrangler.toml`
+ * declares a fixed set of `[[d1_databases]]` at build time, so there is no
+ * `env.lookupD1(name)` and no way for a request to bind a database that did not
+ * exist when the Worker was built. One database per tenant is therefore only
+ * reachable through the D1 HTTP API (`/accounts/:account/d1/database/...`),
+ * which addresses a database by UUID at call time. The shared bindings
+ * (`CATALOG_DB`, `STOREFRONT_APPS_DB`) stay exactly as they are; this module is
+ * additive and used ONLY for the per-tenant raw-interaction stores.
+ *
+ * CREDENTIALS. `CF_ACCOUNT_ID` + `TRACKING_D1_API_TOKEN` (a `wrangler secret`,
+ * scoped to D1:edit). Either one absent ⇒ `readD1Control()` returns undefined
+ * and every caller treats that as "sink unavailable, skip it" — never an error,
+ * the same posture `readAnalytics` + `activity/store.ts` already establish for
+ * an unbound sink. Nothing here ever throws to the caller.
+ *
+ * The query adapter satisfies the same `Queryable` seam the binding-backed
+ * stores use (`d1/catalog.ts`), so the insert code in `interactionStore.ts` is
+ * transport-agnostic and unit-testable against an in-memory sqlite.
+ */
+import { readEnv } from "../env.js";
+import type { Queryable, QueryStatement } from "../d1/catalog.js";
+
+const CF_API_BASE = "https://api.cloudflare.com/client/v4";
+
+/** Resolved REST credentials for the D1 API. */
+export interface D1ControlConfig {
+  accountId: string;
+  apiToken: string;
+  /** Injected for tests; defaults to the global `fetch`. */
+  fetchImpl?: typeof fetch;
+  /** Injected for tests; defaults to the public Cloudflare API. */
+  apiBase?: string;
+}
+
+/**
+ * Read the D1 REST credentials from the runtime env. Undefined when either is
+ * unset — i.e. the per-tenant store is not configured in this environment
+ * (local `astro dev`, an un-provisioned tier). Callers SKIP, never fail.
+ */
+export async function readD1Control(): Promise<D1ControlConfig | undefined> {
+  const [accountId, apiToken] = await Promise.all([
+    readEnv("CF_ACCOUNT_ID"),
+    readEnv("TRACKING_D1_API_TOKEN"),
+  ]);
+  if (!accountId || !apiToken) return undefined;
+  return { accountId, apiToken };
+}
+
+interface CfEnvelope<T> {
+  success?: boolean;
+  result?: T;
+  errors?: Array<{ code?: number; message?: string }>;
+}
+
+async function cfRequest<T>(
+  cfg: D1ControlConfig,
+  path: string,
+  init?: { method?: string; body?: unknown },
+): Promise<CfEnvelope<T>> {
+  const doFetch = cfg.fetchImpl ?? fetch;
+  const res = await doFetch(`${cfg.apiBase ?? CF_API_BASE}/accounts/${cfg.accountId}${path}`, {
+    method: init?.method ?? "GET",
+    headers: {
+      authorization: `Bearer ${cfg.apiToken}`,
+      "content-type": "application/json",
+    },
+    body: init?.body === undefined ? undefined : JSON.stringify(init.body),
+  });
+  const parsed = (await res.json()) as CfEnvelope<T>;
+  if (!res.ok || parsed.success === false) {
+    const detail = parsed.errors?.map((e) => e.message).filter(Boolean).join("; ") || `HTTP ${res.status}`;
+    throw new Error(`d1 api ${path} failed: ${detail}`);
+  }
+  return parsed;
+}
+
+interface D1DatabaseSummary {
+  uuid?: string;
+  name?: string;
+}
+
+/** The UUID of the D1 database with this exact name, or null if none exists. */
+export async function findDatabaseByName(
+  cfg: D1ControlConfig,
+  name: string,
+): Promise<string | null> {
+  const body = await cfRequest<D1DatabaseSummary[]>(
+    cfg,
+    `/d1/database?name=${encodeURIComponent(name)}&per_page=100`,
+  );
+  // `?name=` is a SUBSTRING filter on Cloudflare's side, so match exactly here —
+  // otherwise `…-dt-acme` could resolve to `…-dt-acme-staging` and route one
+  // tenant's rows into another tenant's database.
+  const exact = (body.result ?? []).find((d) => d.name === name);
+  return exact?.uuid ?? null;
+}
+
+/** Create a D1 database and return its UUID. */
+export async function createDatabase(cfg: D1ControlConfig, name: string): Promise<string> {
+  const body = await cfRequest<D1DatabaseSummary>(cfg, "/d1/database", {
+    method: "POST",
+    body: { name },
+  });
+  const uuid = body.result?.uuid;
+  if (!uuid) throw new Error(`d1 api create ${name} returned no uuid`);
+  return uuid;
+}
+
+/**
+ * Find-or-create, tolerant of a concurrent creator: two beacons for a brand-new
+ * tenant can race here, and the loser's `create` fails with "already exists" —
+ * which is a SUCCESS for our purposes, so we re-resolve by name instead of
+ * propagating it.
+ */
+export async function ensureDatabase(cfg: D1ControlConfig, name: string): Promise<string> {
+  const existing = await findDatabaseByName(cfg, name);
+  if (existing) return existing;
+  try {
+    return await createDatabase(cfg, name);
+  } catch (err) {
+    const raced = await findDatabaseByName(cfg, name);
+    if (raced) return raced;
+    throw err;
+  }
+}
+
+interface D1QueryResult {
+  results?: unknown[];
+}
+
+/** POST one (or a semicolon-separated script of) statement(s) to a database. */
+async function query(
+  cfg: D1ControlConfig,
+  databaseUuid: string,
+  sql: string,
+  params: readonly unknown[],
+): Promise<unknown[]> {
+  const body = await cfRequest<D1QueryResult[]>(cfg, `/d1/database/${databaseUuid}/query`, {
+    method: "POST",
+    body: { sql, params },
+  });
+  // One entry per statement; flatten to the rows of the LAST statement, which is
+  // what a single-statement `all`/`first` caller means.
+  const blocks = body.result ?? [];
+  return (blocks[blocks.length - 1]?.results ?? []) as unknown[];
+}
+
+/**
+ * Adapt a per-tenant database (addressed by UUID over REST) to the same
+ * `Queryable` seam the D1 bindings use. `batch` is emulated as a
+ * semicolon-joined script in ONE HTTP call — D1's `/query` runs a multi-
+ * statement script inside one implicit transaction, which is what the binding's
+ * `batch()` guarantees too.
+ */
+export function fromD1Rest(cfg: D1ControlConfig, databaseUuid: string): Queryable {
+  return {
+    all: async <T>(sql: string, ...params: unknown[]) =>
+      (await query(cfg, databaseUuid, sql, params)) as T[],
+    first: async <T>(sql: string, ...params: unknown[]) =>
+      (((await query(cfg, databaseUuid, sql, params))[0] ?? null) as T | null),
+    run: async (sql: string, ...params: unknown[]) => {
+      await query(cfg, databaseUuid, sql, params);
+    },
+    exec: async (sql: string) => {
+      await query(cfg, databaseUuid, sql, []);
+    },
+    batch: async (statements: readonly QueryStatement[]) => {
+      if (statements.length === 0) return;
+      // The REST `params` array is positional across the whole script, so the
+      // placeholders of every statement bind in order.
+      const sql = statements.map((s) => s.sql.trim().replace(/;$/, "")).join(";\n");
+      const params = statements.flatMap((s) => [...(s.params ?? [])]);
+      await query(cfg, databaseUuid, sql, params);
+    },
+  };
+}

package/apps/storefront/src/lib/tracking/interactionCollector.ts

@@ -0,0 +1,202 @@
+/**
+ * dt-collector-v1 — the browser-side delivery destination for
+ * `TotInteraction` (`@tot/public-runtime`'s declarative interaction-tracking
+ * contract). This module OWNS batching + transport; it does not install the
+ * click listener and does not gate on consent/environment — those are
+ * `interaction-tracking.ts` and `dt-consent-environment-gate` respectively.
+ *
+ * Wiring (owned by whichever module composes the gate in front of delivery,
+ * e.g. `dt-consent-environment-gate`):
+ *
+ *   import { onInteraction } from "@tot/public-runtime";
+ *   import { collectInteraction } from "@/lib/tracking/interactionCollector";
+ *   onInteraction((interaction) => {
+ *     if (consentAndEnvironmentAllow()) collectInteraction(interaction);
+ *   });
+ *
+ * or, where the caller controls install directly:
+ *
+ *   installInteractionTracking({ deliver: collectInteraction });
+ *
+ * PAYLOAD GUARDRAIL: this module reads only the five `TotInteraction` fields
+ * handed to it, plus what it synthesizes itself (`ts`, `correlationId`,
+ * `seq`). No DOM access, no `textContent`/`href`/form reads — same prohibition
+ * as `interaction-tracking.ts`'s listener; see that module's guardrail test
+ * (`tests/interaction-tracking.test.ts`) for the pattern this module must not
+ * violate either.
+ */
+import type { TotInteraction } from "@tot/public-runtime";
+
+/** Where the browser beacon POSTs interaction batches. */
+export const INTERACTION_BEACON_PATH = "/api/tracking/interaction";
+
+/** Max events held in memory before a forced flush — bounds one beacon's size
+ *  and the worst case if flush triggers never fire (e.g. a test/SSR context). */
+export const MAX_QUEUE_SIZE = 20;
+
+/** The wire shape POSTed to `INTERACTION_BEACON_PATH` — `TotInteraction`'s five
+ *  fields plus this module's own synthesized correlation fields. Must stay in
+ *  lockstep with `INTERACTION_WIRE_ALLOWLIST`
+ *  (`apps/storefront/src/lib/tracking/collectorAllowlist.ts`). */
+export interface InteractionWireEvent {
+  type: TotInteraction["type"];
+  el: string;
+  action: string | null;
+  placement: string | null;
+  scope: string | null;
+  index: number | null;
+  /** Client emit clock, ISO-8601. */
+  ts: string;
+  /** Groups every event from one collector lifetime (one page load). */
+  correlationId: string;
+  /** Monotonic within `correlationId`, starting at 0. */
+  seq: number;
+}
+
+/** Build the wire event for one interaction. Pure — no DOM, no clock side
+ *  effects beyond the `now` parameter, so it is unit-testable without faking
+ *  globals. */
+export function toWireEvent(
+  interaction: TotInteraction,
+  correlationId: string,
+  seq: number,
+  now: () => string = () => new Date().toISOString(),
+): InteractionWireEvent {
+  return {
+    type: interaction.type,
+    el: interaction.el,
+    action: interaction.action,
+    placement: interaction.placement,
+    scope: interaction.scope,
+    index: interaction.index,
+    ts: now(),
+    correlationId,
+    seq,
+  };
+}
+
+/** A sink for a finished batch. The default posts one JSON beacon. */
+export type InteractionBatchSender = (events: InteractionWireEvent[]) => void;
+
+/** Default sender: one `sendBeacon` per flushed batch, falling back to a
+ *  keepalive `fetch` where `sendBeacon` is absent (mirrors
+ *  `analytics/webVitals.ts`'s `beaconReporter`). Never throws into the page. */
+export function beaconBatchSender(endpoint: string = INTERACTION_BEACON_PATH): InteractionBatchSender {
+  return (events) => {
+    if (events.length === 0 || typeof navigator === "undefined") return;
+    const body = JSON.stringify({ events });
+    try {
+      if (typeof navigator.sendBeacon === "function") {
+        navigator.sendBeacon(endpoint, body);
+      } else if (typeof fetch === "function") {
+        void fetch(endpoint, { method: "POST", body, keepalive: true });
+      }
+    } catch {
+      // Never let telemetry throw into the page.
+    }
+  };
+}
+
+function randomCorrelationId(): string {
+  const c: Crypto | undefined = (globalThis as { crypto?: Crypto }).crypto;
+  if (c?.randomUUID) return c.randomUUID();
+  return `ic-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+}
+
+/**
+ * A collector instance: batches interactions in memory and flushes them to
+ * `send` on demand, on queue overflow, or when the caller wires
+ * `flush`-on-`visibilitychange`/`pagehide` (see {@link installFlushTriggers}).
+ * Exposed as a factory (rather than one module-level singleton) so tests can
+ * construct an isolated instance with a fake sender and clock.
+ */
+export interface InteractionCollector {
+  /** Enqueue one interaction; synthesizes `ts`/`correlationId`/`seq`. Flushes
+   *  immediately once the queue reaches `MAX_QUEUE_SIZE`. */
+  collect(interaction: TotInteraction): void;
+  /** Send whatever is queued right now (no-op if empty). */
+  flush(): void;
+}
+
+export function createInteractionCollector(
+  send: InteractionBatchSender = beaconBatchSender(),
+  correlationId: string = randomCorrelationId(),
+  now?: () => string,
+): InteractionCollector {
+  let queue: InteractionWireEvent[] = [];
+  let seq = 0;
+
+  const flush = (): void => {
+    if (queue.length === 0) return;
+    const batch = queue;
+    queue = [];
+    send(batch);
+  };
+
+  const collect = (interaction: TotInteraction): void => {
+    queue.push(toWireEvent(interaction, correlationId, seq, now));
+    seq += 1;
+    if (queue.length >= MAX_QUEUE_SIZE) flush();
+  };
+
+  return { collect, flush };
+}
+
+// ---------------------------------------------------------------------------
+// Browser wiring (thin — everything above is pure + unit-tested).
+// ---------------------------------------------------------------------------
+
+/**
+ * Wire `collector.flush` to the two reliable end-of-visibility moments (same
+ * pair `analytics/webVitals.ts#startWebVitals` uses): the page first going
+ * hidden, and `pagehide` — the last point mobile Safari guarantees a beacon
+ * can still be sent. No-op teardown when there is no `document`/`window`
+ * (SSR, tests).
+ */
+export function installFlushTriggers(collector: InteractionCollector): () => void {
+  if (typeof document === "undefined" || typeof window === "undefined") return () => {};
+  const onVisibility = (): void => {
+    if (document.visibilityState === "hidden") collector.flush();
+  };
+  document.addEventListener("visibilitychange", onVisibility, { capture: true });
+  window.addEventListener("pagehide", collector.flush, { capture: true });
+  return () => {
+    document.removeEventListener("visibilitychange", onVisibility, { capture: true });
+    window.removeEventListener("pagehide", collector.flush, { capture: true });
+  };
+}
+
+let defaultCollector: InteractionCollector | null = null;
+let defaultTeardown: (() => void) | null = null;
+
+function getDefaultCollector(): InteractionCollector {
+  if (!defaultCollector) {
+    defaultCollector = createInteractionCollector();
+    defaultTeardown = installFlushTriggers(defaultCollector);
+  }
+  return defaultCollector;
+}
+
+/**
+ * The default delivery destination: `installInteractionTracking({ deliver:
+ * collectInteraction })` or forward into it from an `onInteraction` subscriber
+ * (that is what the consent/environment gate does). Lazily creates the
+ * module-level collector and its flush wiring on first call, so importing
+ * this module has no side effects until an interaction actually arrives.
+ */
+export function collectInteraction(interaction: TotInteraction): void {
+  getDefaultCollector().collect(interaction);
+}
+
+/** Force-flush the default collector (e.g. from a manual "send now" hook). */
+export function flushInteractions(): void {
+  defaultCollector?.flush();
+}
+
+/** Test-only: drop the module-level singleton so a fresh collector/wiring is
+ *  created on the next {@link collectInteraction} call. */
+export function resetDefaultCollectorForTests(): void {
+  defaultTeardown?.();
+  defaultCollector = null;
+  defaultTeardown = null;
+}