@tokenoftrust/storefront-runner 2.0.4 → 2.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/apps/storefront/migrations-apps/0011_dt_tracking_tenant_databases.sql +23 -0
  2. package/apps/storefront/node_modules/.bin/vitest +2 -2
  3. package/apps/storefront/node_modules/.vite/deps/_metadata.json +19 -19
  4. package/apps/storefront/node_modules/.vite/deps_ssr/_metadata.json +3 -3
  5. package/apps/storefront/src/components/chrome/MegaMenu.astro +11 -1
  6. package/apps/storefront/src/components/chrome/NavDropdown.astro +49 -7
  7. package/apps/storefront/src/components/chrome/SiteFooter.astro +29 -4
  8. package/apps/storefront/src/components/chrome/SiteHeader.astro +39 -6
  9. package/apps/storefront/src/components/chrome/UtilityNav.astro +13 -1
  10. package/apps/storefront/src/components/islands/MobileNav.tsx +42 -2
  11. package/apps/storefront/src/layouts/Layout.astro +64 -6
  12. package/apps/storefront/src/lib/chrome/ctaIdentity.ts +21 -0
  13. package/apps/storefront/src/lib/chrome/examples/tokenoftrust.chrome.example.json +22 -22
  14. package/apps/storefront/src/lib/chrome/trackingPlacements.ts +36 -0
  15. package/apps/storefront/src/lib/d1/schema-apps.ts +57 -0
  16. package/apps/storefront/src/lib/rawChrome.ts +73 -14
  17. package/apps/storefront/src/lib/rawMarketingHtml.ts +30 -0
  18. package/apps/storefront/src/lib/shared/developerInvites.ts +6 -0
  19. package/apps/storefront/src/lib/storyblok/provider.ts +21 -5
  20. package/apps/storefront/src/lib/tracking/collectorAllowlist.ts +40 -0
  21. package/apps/storefront/src/lib/tracking/collectorIngest.ts +188 -0
  22. package/apps/storefront/src/lib/tracking/d1Control.ts +180 -0
  23. package/apps/storefront/src/lib/tracking/interactionCollector.ts +202 -0
  24. package/apps/storefront/src/lib/tracking/interactionHealth.ts +252 -0
  25. package/apps/storefront/src/lib/tracking/interactionSchema.ts +107 -0
  26. package/apps/storefront/src/lib/tracking/interactionStore.ts +0 -0
  27. package/apps/storefront/src/lib/tracking/pruneRollup.ts +260 -0
  28. package/apps/storefront/src/lib/tracking/publishSafetyNet.ts +162 -0
  29. package/apps/storefront/src/lib/tracking/tenantD1Registry.ts +214 -0
  30. package/apps/storefront/src/lib/tracking/tenantReadAccess.ts +138 -0
  31. package/apps/storefront/src/lib/tracking-writeback/rewriteTrackingIds.ts +175 -0
  32. package/apps/storefront/src/lib/tracking-writeback/writebackCandidate.ts +110 -0
  33. package/apps/storefront/src/pages/api/admin/interaction-insights.ts +101 -0
  34. package/apps/storefront/src/pages/api/tracking/interaction.ts +123 -0
  35. package/apps/storefront/src/pages/style-guide/[tenant]/[theme].astro +33 -14
  36. package/apps/storefront/src/workers/pruneRollupWorker.ts +120 -0
  37. package/package.json +1 -1
  38. package/packages/public-runtime/node_modules/.bin/vitest +2 -2
  39. package/packages/public-runtime/src/chrome.ts +18 -7
  40. package/packages/public-runtime/src/index.ts +10 -0
  41. package/packages/public-runtime/src/integration/index.ts +1 -0
  42. package/packages/public-runtime/src/integration/interaction-tracking-gate.ts +142 -0
  43. package/packages/public-runtime/src/interaction-tracking.ts +364 -0
  44. package/packages/public-runtime/src/tracking-id-derivation.ts +1008 -0
  45. package/scripts/tenant/validate.mjs +306 -1
  46. package/scripts/tot-dev.mjs +59 -3
  47. package/tenants/home/public/js/storefront.js +6 -87
@@ -0,0 +1,252 @@
1
+ /**
2
+ * The interaction-tracking chain's HEALTH ASSERTION — the thing that makes a
3
+ * broken collection chain loud instead of silent.
4
+ *
5
+ * Why it exists. Every other part of this system is correctly fail-quiet at
6
+ * runtime: the delegated listener returns without an event when there is no
7
+ * `data-tot-el` up the ancestor chain (that is the `tracking: "none"` opt-out,
8
+ * where ABSENCE is the declaration), and the consent/environment gate drops a
9
+ * disallowed interaction with no error. Both are right. Together they mean a
10
+ * surface that was never stamped at all produces exactly the same observable
11
+ * result as a surface nobody clicked — which is the failure mode that made the
12
+ * legacy mixin useless for years: no way to tell "collection is broken" from
13
+ * "legitimately idle".
14
+ *
15
+ * This module is the one place that CAN tell them apart, because it looks at
16
+ * the rendered markup rather than at the event stream:
17
+ *
18
+ * - `mount_missing` — the document carries actionable elements but not the
19
+ * serialized listener, so no click on it can ever be seen. FATAL.
20
+ * - `chrome_unstamped` — the surface carries actionable elements and NOT ONE
21
+ * of them carries `data-tot-el`. FATAL. Zero-of-N is not an opt-out
22
+ * declaration; a real opt-out is per element, and an author opting out of
23
+ * every single element still leaves the OTHER elements stamped. This is
24
+ * the finding that covers the **sharedChrome bypass path**: a tenant that
25
+ * ships a hand-authored `content/<id>/chrome.html` bypasses governed
26
+ * chrome entirely, so `validateChromeConfig` never sees it,
27
+ * `trackingAttributes()` never stamps it, and `rewriteTrackingIds`
28
+ * (dt-id-writeback) explicitly refuses to touch chrome. Nothing else in
29
+ * the chain would ever say a word about it.
30
+ * - `stamp_incomplete` — elements carry `data-tot-el` but no
31
+ * `data-tot-action`. NOT fatal: the publish-time safety net
32
+ * (`publishSafetyNet.ts`) stamps identity only, by design, and there is no
33
+ * derivation path from identity to intent. Reported because those rows
34
+ * land in the store un-analyzable, which is worth seeing.
35
+ *
36
+ * NO DOM-TEXT READS: only `tagName` and the parsed attribute list are ever
37
+ * read, matching the payload guardrail the listener and every consumer are
38
+ * held to. Actionability is decided by `@tot/public-runtime`'s shared
39
+ * `isTrackableNode` — the same oracle the derivation engine and the writeback
40
+ * rewriter use, so "actionable" cannot mean three different things.
41
+ */
42
+ import { parse } from "parse5";
43
+ import {
44
+ INTERACTION_TRACKING_BOOT_JS,
45
+ TRACKING_ATTR_ACTION,
46
+ TRACKING_ATTR_EL,
47
+ isTrackableNode,
48
+ type StructuralNode,
49
+ } from "@tot/public-runtime";
50
+
51
+ /** The finding codes, in the order this module reports them. */
52
+ export type TrackingHealthCode = "mount_missing" | "chrome_unstamped" | "stamp_incomplete";
53
+
54
+ /** The codes that mean the chain is BROKEN rather than merely imperfect. */
55
+ const FATAL_CODES: ReadonlySet<TrackingHealthCode> = new Set<TrackingHealthCode>([
56
+ "mount_missing",
57
+ "chrome_unstamped",
58
+ ]);
59
+
60
+ export interface TrackingHealthFinding {
61
+ code: TrackingHealthCode;
62
+ /** Human-actionable one-liner; carries counts and tags, never content. */
63
+ detail: string;
64
+ fatal: boolean;
65
+ }
66
+
67
+ export interface TrackingHealthReport {
68
+ /** True iff there is no FATAL finding — i.e. a click here can be collected. */
69
+ ok: boolean;
70
+ /** Elements an interaction can originate from (`isTrackableNode`). */
71
+ actionable: number;
72
+ /** Of those, how many carry `data-tot-el`. */
73
+ stamped: number;
74
+ /** Of the stamped ones, how many also carry `data-tot-action`. */
75
+ attributed: number;
76
+ findings: TrackingHealthFinding[];
77
+ }
78
+
79
+ export interface TrackingHealthOptions {
80
+ /**
81
+ * Require the serialized listener to be present. True for a whole document,
82
+ * false for a chrome WRAPPER inspected before `rawHtmlResponse()` has
83
+ * injected the listener into it.
84
+ */
85
+ requireMount?: boolean;
86
+ /** A label for the log/error line — e.g. `chrome.html (tenant acme)`. */
87
+ surface?: string;
88
+ }
89
+
90
+ /** Thrown by {@link assertTrackingChainHealthy} on a FATAL finding. */
91
+ export class TrackingChainBrokenError extends Error {
92
+ readonly report: TrackingHealthReport;
93
+
94
+ constructor(report: TrackingHealthReport, surface: string) {
95
+ const codes = report.findings.filter((f) => f.fatal).map((f) => f.code);
96
+ super(
97
+ `interaction tracking is BROKEN on ${surface} [${codes.join(", ")}] — ` +
98
+ report.findings
99
+ .filter((f) => f.fatal)
100
+ .map((f) => f.detail)
101
+ .join("; ") +
102
+ ` (actionable=${report.actionable}, stamped=${report.stamped}, ` +
103
+ `attributed=${report.attributed}). A click on this surface produces NO ` +
104
+ `retrievable event, and nothing else in the chain reports it: the listener ` +
105
+ `returns silently when no ${TRACKING_ATTR_EL} is present, which is also how a ` +
106
+ `legitimate \`tracking: "none"\` opt-out looks.`,
107
+ );
108
+ this.name = "TrackingChainBrokenError";
109
+ this.report = report;
110
+ }
111
+ }
112
+
113
+ interface P5Attr {
114
+ name: string;
115
+ value: string;
116
+ }
117
+ interface P5Node {
118
+ nodeName: string;
119
+ tagName?: string;
120
+ attrs?: P5Attr[];
121
+ childNodes?: P5Node[];
122
+ }
123
+
124
+ function nonEmpty(v: string | undefined): boolean {
125
+ return typeof v === "string" && v.trim().length > 0;
126
+ }
127
+
128
+ /**
129
+ * Inspect one rendered surface (a full document, a chrome wrapper, or a body
130
+ * fragment). PURE — no I/O, no logging, no throwing. `html` that parses to
131
+ * nothing actionable is healthy by definition: a surface with no actionable
132
+ * element has nothing to collect, and saying otherwise would make every
133
+ * text-only page a false alarm.
134
+ */
135
+ export function inspectTrackingChain(
136
+ html: string,
137
+ options: TrackingHealthOptions = {},
138
+ ): TrackingHealthReport {
139
+ let actionable = 0;
140
+ let stamped = 0;
141
+ let attributed = 0;
142
+ const unstampedTags = new Set<string>();
143
+
144
+ const walk = (node: P5Node): void => {
145
+ if (node.tagName) {
146
+ const attrs: Record<string, string> = {};
147
+ for (const a of node.attrs ?? []) attrs[a.name] = a.value;
148
+ const structural: StructuralNode = { tag: node.tagName.toLowerCase(), attrs };
149
+ const hasEl = nonEmpty(attrs[TRACKING_ATTR_EL]);
150
+ if (isTrackableNode(structural)) {
151
+ actionable += 1;
152
+ if (hasEl) {
153
+ stamped += 1;
154
+ if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
155
+ } else {
156
+ unstampedTags.add(structural.tag);
157
+ }
158
+ } else if (hasEl) {
159
+ // Stamped but not actionable by the shared oracle (e.g. a stamped
160
+ // wrapper). Counted so `stamped` never understates the surface.
161
+ stamped += 1;
162
+ if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
163
+ }
164
+ }
165
+ for (const child of node.childNodes ?? []) walk(child);
166
+ };
167
+ walk(parse(html) as P5Node);
168
+
169
+ const findings: TrackingHealthFinding[] = [];
170
+
171
+ if (options.requireMount && actionable > 0 && !html.includes(INTERACTION_TRACKING_BOOT_JS)) {
172
+ findings.push({
173
+ code: "mount_missing",
174
+ detail: `${actionable} actionable element(s) but the serialized interaction-tracking listener is not in this document`,
175
+ fatal: true,
176
+ });
177
+ }
178
+
179
+ if (actionable > 0 && stamped === 0) {
180
+ findings.push({
181
+ code: "chrome_unstamped",
182
+ detail:
183
+ `${actionable} actionable element(s) (${[...unstampedTags].sort().join(", ")}) and NOT ONE ` +
184
+ `carries ${TRACKING_ATTR_EL} — an unstamped surface, not a per-element opt-out`,
185
+ fatal: true,
186
+ });
187
+ } else if (stamped > attributed) {
188
+ findings.push({
189
+ code: "stamp_incomplete",
190
+ detail: `${stamped - attributed} of ${stamped} stamped element(s) carry no ${TRACKING_ATTR_ACTION}, so their rows land un-analyzable`,
191
+ fatal: false,
192
+ });
193
+ }
194
+
195
+ return {
196
+ ok: !findings.some((f) => f.fatal),
197
+ actionable,
198
+ stamped,
199
+ attributed,
200
+ findings,
201
+ };
202
+ }
203
+
204
+ /**
205
+ * {@link inspectTrackingChain}, but a FATAL finding THROWS. For build/publish
206
+ * gates and tests — anywhere a broken chain should stop the line rather than
207
+ * be noted.
208
+ */
209
+ export function assertTrackingChainHealthy(
210
+ html: string,
211
+ options: TrackingHealthOptions = {},
212
+ ): TrackingHealthReport {
213
+ const report = inspectTrackingChain(html, options);
214
+ if (!report.ok) throw new TrackingChainBrokenError(report, options.surface ?? "this surface");
215
+ return report;
216
+ }
217
+
218
+ /**
219
+ * The render-path form: NEVER throws, emits ONE distinct structured line per
220
+ * distinct outcome so a broken surface is visible in Workers Logs. Returns the
221
+ * report so a caller can act on it.
222
+ *
223
+ * Non-throwing on purpose — a tenant whose hand-authored chrome was never
224
+ * stamped should get a loud log line, not a blank page.
225
+ */
226
+ const reported = new Set<string>();
227
+ /** Bound the dedupe set so a long-lived isolate cannot grow it without limit. */
228
+ const REPORTED_MAX = 64;
229
+
230
+ export function reportTrackingChainHealth(
231
+ html: string,
232
+ options: TrackingHealthOptions = {},
233
+ ): TrackingHealthReport {
234
+ const report = inspectTrackingChain(html, options);
235
+ if (report.findings.length === 0) return report;
236
+ const surface = options.surface ?? "unknown-surface";
237
+ const key = `${surface}|${report.findings.map((f) => f.code).join(",")}|${report.actionable}/${report.stamped}`;
238
+ if (reported.has(key)) return report;
239
+ if (reported.size >= REPORTED_MAX) reported.clear();
240
+ reported.add(key);
241
+ for (const finding of report.findings) {
242
+ const line = `[tracking:health] ${finding.fatal ? "BROKEN" : "degraded"} ${finding.code} on ${surface}: ${finding.detail}`;
243
+ if (finding.fatal) console.error(line);
244
+ else console.warn(line);
245
+ }
246
+ return report;
247
+ }
248
+
249
+ /** Test-only: forget which lines have already been emitted. */
250
+ export function resetTrackingHealthReportingForTests(): void {
251
+ reported.clear();
252
+ }
@@ -0,0 +1,107 @@
1
+ /**
2
+ * The RAW interaction-event table — the schema of a TENANT'S OWN D1 database.
3
+ *
4
+ * PER-TENANT, NOT PER-COLUMN. Every other D1 table in this repo lives in a
5
+ * shared database and carries a `tenant_id` column as its isolation boundary
6
+ * (see `d1/schema-apps.ts`). This one does NOT work that way: each tenant gets
7
+ * its OWN D1 database holding only its own rows, so cross-tenant leakage is
8
+ * impossible BY CONSTRUCTION rather than by a correct `WHERE tenant_id = ?` in
9
+ * every query. A missing predicate here cannot reach another tenant's data —
10
+ * there is no other tenant's data in the file.
11
+ *
12
+ * `tenant_id` is still stored on the row (redundantly, one constant value per
13
+ * database) deliberately: it makes an exported/backed-up/rolled-up row
14
+ * self-describing, and it lets a mis-routed insert be DETECTED (a row whose
15
+ * `tenant_id` disagrees with the database's owner is a routing bug, and
16
+ * `dt-tenant-isolation` can assert on exactly that).
17
+ *
18
+ * WHY D1 AND NOT A SHARED STORE: D1 is not billed per database — thousands of
19
+ * databases cost the same as one — so per-tenant databases are the endorsed
20
+ * shape, not a workaround. The hard limit is 10GB PER DATABASE (~30M rows at a
21
+ * 90-day window for a busy tenant), which is a per-tenant ceiling rather than a
22
+ * global one; `interaction_store_meta` below carries the counters
23
+ * `dt-prune-and-rollup` needs to surface a tenant approaching it.
24
+ *
25
+ * RETENTION is a 90-day window, enforced by a DELETE-based prune in a later
26
+ * unit (`dt-prune-and-rollup`) — D1 has no TTL. This schema only has to make
27
+ * that prune cheap, which is what `idx_interaction_events_ingested_at` is for:
28
+ * `DELETE FROM interaction_events WHERE ingested_at < ?` is an index range
29
+ * scan, not a table scan. `ingested_at` (the SERVER receive clock) is the prune
30
+ * key, never `ts` (a CLIENT clock a beacon can skew arbitrarily).
31
+ *
32
+ * SCHEMA IS APPLIED, NOT MIGRATED. A per-tenant database is created on demand
33
+ * at runtime, so it cannot ride the numbered `migrations-apps/` lineage (which
34
+ * `wrangler d1 migrations apply` runs against ONE named binding). Instead this
35
+ * DDL is idempotent (`IF NOT EXISTS` throughout) and applied at provision time;
36
+ * `INTERACTION_SCHEMA_VERSION` is recorded in `interaction_store_meta` so a
37
+ * future version bump can be detected and re-applied per tenant.
38
+ */
39
+
40
+ /** Bumped whenever `INTERACTION_D1_SCHEMA` changes; recorded per database. */
41
+ export const INTERACTION_SCHEMA_VERSION = 1;
42
+
43
+ /**
44
+ * Idempotent DDL for one tenant's raw-interaction database. Multi-statement:
45
+ * applied via `Queryable.exec`, which both the D1 binding and the D1 HTTP API
46
+ * accept as a semicolon-separated script.
47
+ */
48
+ export const INTERACTION_D1_SCHEMA = `
49
+ CREATE TABLE IF NOT EXISTS interaction_events (
50
+ -- Owning tenant. One constant value per database (see the module note): the
51
+ -- isolation boundary is the DATABASE, this column is provenance + a
52
+ -- mis-routing tripwire.
53
+ tenant_id TEXT NOT NULL,
54
+ -- Stream discriminator, always "interaction" today. Present from row one so a
55
+ -- second stream can land in the same table without a migration.
56
+ stream TEXT NOT NULL,
57
+ -- The six TotInteraction fields, flattened for SQL aggregation.
58
+ type TEXT NOT NULL,
59
+ el TEXT NOT NULL,
60
+ action TEXT,
61
+ placement TEXT,
62
+ scope TEXT,
63
+ idx INTEGER,
64
+ -- Groups the events of one collector lifetime (~one page load).
65
+ correlation_id TEXT NOT NULL,
66
+ -- Monotonic within correlation_id, assigned by the client collector.
67
+ seq INTEGER NOT NULL,
68
+ -- Client emit clock (ISO-8601). Informational: never the prune or ordering key.
69
+ ts TEXT NOT NULL,
70
+ -- Server receive clock (ISO-8601). THE retention/prune key.
71
+ ingested_at TEXT NOT NULL,
72
+ -- (correlation_id, seq) is the collector's own natural identity, so a
73
+ -- retried/duplicated beacon re-inserting the same event is a no-op under
74
+ -- INSERT OR IGNORE rather than a double-count.
75
+ PRIMARY KEY (correlation_id, seq)
76
+ );
77
+ CREATE INDEX IF NOT EXISTS idx_interaction_events_ingested_at
78
+ ON interaction_events (ingested_at);
79
+ CREATE INDEX IF NOT EXISTS idx_interaction_events_stream_ingested_at
80
+ ON interaction_events (stream, ingested_at);
81
+ CREATE INDEX IF NOT EXISTS idx_interaction_events_el
82
+ ON interaction_events (el, ingested_at);
83
+ CREATE TABLE IF NOT EXISTS interaction_store_meta (
84
+ -- Single-row table. The constant primary key makes that a schema invariant.
85
+ -- (Keep every inline comment free of semicolons: this DDL is executed as one
86
+ -- semicolon-separated script.)
87
+ id INTEGER PRIMARY KEY CHECK (id = 1),
88
+ tenant_id TEXT NOT NULL,
89
+ schema_version INTEGER NOT NULL,
90
+ created_at TEXT NOT NULL,
91
+ updated_at TEXT NOT NULL
92
+ );
93
+ `;
94
+
95
+ /** Statements that stamp/refresh the single `interaction_store_meta` row. */
96
+ export function interactionMetaUpsert(tenantId: string, at: string): {
97
+ sql: string;
98
+ params: readonly unknown[];
99
+ } {
100
+ return {
101
+ sql: `INSERT INTO interaction_store_meta (id, tenant_id, schema_version, created_at, updated_at)
102
+ VALUES (1, ?, ?, ?, ?)
103
+ ON CONFLICT(id) DO UPDATE SET schema_version = excluded.schema_version,
104
+ updated_at = excluded.updated_at`,
105
+ params: [tenantId, INTERACTION_SCHEMA_VERSION, at, at],
106
+ };
107
+ }
@@ -0,0 +1,260 @@
1
+ /**
2
+ * Scheduled PRUNE + ROLLUP for the per-tenant raw-interaction D1 stores
3
+ * (`dt-storage-per-tenant-d1`).
4
+ *
5
+ * WHAT THIS DOES, per tenant, per sweep:
6
+ * 1. READ the rows aging out of the 90-day window (`ingested_at < cutoff`,
7
+ * the SERVER receive clock — never the client `ts`, which a beacon can
8
+ * skew arbitrarily).
9
+ * 2. ROLL UP those rows via the already-built, pure `aggregateInteractions()`
10
+ * and write them to the dedicated `INTERACTION_ANALYTICS` dataset via
11
+ * the already-built `writeInteractionAnalytics()` — durable long-term
12
+ * aggregates survive the delete below. Both are reused verbatim from
13
+ * `interactionStore.ts`, never reimplemented here.
14
+ * 3. DELETE those same rows. `DELETE FROM interaction_events WHERE
15
+ * ingested_at < ?` is an index range scan
16
+ * (`idx_interaction_events_ingested_at`), not a table scan.
17
+ * 4. MEASURE the database's current size (`PRAGMA page_count * page_size`)
18
+ * and warn when a tenant is approaching D1's 10GB-per-database ceiling —
19
+ * the point is to surface this BEFORE writes start failing, not after.
20
+ * 5. WRITE BACK the `tracking_tenant_databases` bookkeeping columns
21
+ * (`last_pruned_at`, `pruned_rows`, `size_bytes`, `size_checked_at`) that
22
+ * `dt-storage-per-tenant-d1` added for exactly this purpose.
23
+ *
24
+ * WORK QUEUE. One sweep processes a bounded BATCH of tenants
25
+ * (`ORDER BY last_pruned_at ASC` — SQLite sorts NULL first, so a tenant that
26
+ * has never been pruned is swept immediately), not the whole fleet in one
27
+ * invocation. A frequent-enough cron (see `workers/pruneRollupWorker.ts`)
28
+ * cycles fairly through every tenant over time instead of risking one run
29
+ * exceeding the Worker's CPU/wall-clock budget as the tenant count grows.
30
+ *
31
+ * NOT DONE HERE (by design, per the binding decisions this unit follows):
32
+ * - No DROP-based time-partitioned schema. The DROP-vs-DELETE cost
33
+ * asymmetry noted in `d1-cost-basis-verified` is UNVERIFIED inference;
34
+ * this sticks to the straightforward `DELETE ... WHERE ingested_at < ?`
35
+ * against the single `interaction_events` table dt-storage-per-tenant-d1
36
+ * already built. A future optimization pass can revisit it once verified.
37
+ * - No outbound alert transport. Same posture as `activity/alerts.ts`: this
38
+ * module DETECTS a tenant approaching the ceiling and logs it
39
+ * (`console.warn`, grep-able in Workers Logs) plus returns it in the
40
+ * structured sweep result — where it goes from there (a dashboard, a
41
+ * polled endpoint) is a sink's job, not this module's.
42
+ *
43
+ * BEST-EFFORT PER TENANT. One tenant's prune failing (a REST error, a
44
+ * transient D1 fault) is caught, logged, and recorded in that tenant's
45
+ * outcome — it never aborts the rest of the batch.
46
+ */
47
+ import type { Queryable } from "../d1/catalog.js";
48
+ import type { AnalyticsEngineLike } from "../activity/store.js";
49
+ import { fromD1Rest, type D1ControlConfig } from "./d1Control.js";
50
+ import { aggregateInteractions, writeInteractionAnalytics } from "./interactionStore.js";
51
+ import type { StampedInteractionEvent } from "./collectorIngest.js";
52
+
53
+ /** The settled retention window (`settled-operational-choices`). */
54
+ export const DEFAULT_RETENTION_DAYS = 90;
55
+
56
+ /** How many tenants one sweep invocation processes. */
57
+ export const DEFAULT_BATCH_SIZE = 25;
58
+
59
+ /** D1's documented per-database ceiling. */
60
+ export const D1_SIZE_CEILING_BYTES = 10 * 1024 * 1024 * 1024;
61
+
62
+ /** Warn at 80% of the ceiling — well before writes start failing. */
63
+ const DEFAULT_SIZE_WARNING_RATIO = 0.8;
64
+
65
+ /** One row of the prune work queue. */
66
+ export interface TenantPruneRow {
67
+ tenant_id: string;
68
+ database_uuid: string;
69
+ last_pruned_at: string | null;
70
+ }
71
+
72
+ export interface PruneSweepDeps {
73
+ /** The shared STOREFRONT_APPS_DB handle holding the directory table. */
74
+ apps: Queryable;
75
+ /** D1 REST credentials + transport, for reaching each tenant's own database. */
76
+ control: D1ControlConfig;
77
+ /** The dedicated interaction AE dataset. Absent ⇒ rollup is skipped, prune
78
+ * still runs (never block retention on an unavailable aggregates sink). */
79
+ analytics?: AnalyticsEngineLike;
80
+ /** ISO-8601 "now" — injected so the sweep is deterministic under test. */
81
+ now: string;
82
+ retentionDays?: number;
83
+ batchSize?: number;
84
+ /** Absolute byte threshold for the ceiling warning; defaults to 80% of
85
+ * `D1_SIZE_CEILING_BYTES`. */
86
+ sizeWarningBytes?: number;
87
+ }
88
+
89
+ export interface TenantPruneOutcome {
90
+ tenantId: string;
91
+ prunedRows: number;
92
+ aggregatesWritten: number;
93
+ sizeBytes: number | null;
94
+ /** True when `sizeBytes` is at/above the warning threshold. */
95
+ sizeWarning: boolean;
96
+ /** Set only when this tenant's sweep failed; the batch still continues. */
97
+ error?: string;
98
+ }
99
+
100
+ export interface PruneSweepResult {
101
+ tenantsSwept: number;
102
+ totalPrunedRows: number;
103
+ totalAggregatesWritten: number;
104
+ tenants: TenantPruneOutcome[];
105
+ }
106
+
107
+ /** Map one raw `interaction_events` row (snake_case D1 columns) back to the
108
+ * `StampedInteractionEvent` shape `aggregateInteractions()` expects. PURE. */
109
+ export function rawRowToEvent(row: Record<string, unknown>): StampedInteractionEvent {
110
+ return {
111
+ type: "click",
112
+ el: String(row.el ?? ""),
113
+ action: (row.action as string | null) ?? null,
114
+ placement: (row.placement as string | null) ?? null,
115
+ scope: (row.scope as string | null) ?? null,
116
+ index: (row.idx as number | null) ?? null,
117
+ ts: String(row.ts ?? ""),
118
+ correlationId: String(row.correlation_id ?? ""),
119
+ seq: Number(row.seq ?? 0),
120
+ tenant_id: String(row.tenant_id ?? ""),
121
+ stream: "interaction",
122
+ ingested_at: String(row.ingested_at ?? ""),
123
+ };
124
+ }
125
+
126
+ /** The prune work queue: oldest-`last_pruned_at`-first (NULL — never pruned —
127
+ * sorts first in SQLite ASC). Never throws. */
128
+ export async function listPruneCandidates(
129
+ apps: Queryable,
130
+ batchSize: number,
131
+ ): Promise<TenantPruneRow[]> {
132
+ try {
133
+ return await apps.all<TenantPruneRow>(
134
+ `SELECT tenant_id, database_uuid, last_pruned_at
135
+ FROM tracking_tenant_databases
136
+ ORDER BY last_pruned_at ASC
137
+ LIMIT ?`,
138
+ batchSize,
139
+ );
140
+ } catch (err) {
141
+ console.error(
142
+ `[tracking:prune] directory read failed: ${err instanceof Error ? err.message : String(err)}`,
143
+ );
144
+ return [];
145
+ }
146
+ }
147
+
148
+ /** `PRAGMA page_count * page_size` — the current on-disk size of one tenant's
149
+ * database. Returns null (never throws) when the pragmas are unavailable. */
150
+ async function readDatabaseSizeBytes(db: Queryable): Promise<number | null> {
151
+ try {
152
+ const pageCount = await db.first<{ page_count: number }>("PRAGMA page_count");
153
+ const pageSize = await db.first<{ page_size: number }>("PRAGMA page_size");
154
+ if (!pageCount || !pageSize) return null;
155
+ const count = Number(pageCount.page_count);
156
+ const size = Number(pageSize.page_size);
157
+ if (!Number.isFinite(count) || !Number.isFinite(size)) return null;
158
+ return count * size;
159
+ } catch {
160
+ return null;
161
+ }
162
+ }
163
+
164
+ /** Prune + roll up ONE tenant's database. Never throws. */
165
+ export async function pruneTenant(
166
+ row: TenantPruneRow,
167
+ deps: PruneSweepDeps,
168
+ ): Promise<TenantPruneOutcome> {
169
+ const retentionDays = deps.retentionDays ?? DEFAULT_RETENTION_DAYS;
170
+ const sizeWarningBytes =
171
+ deps.sizeWarningBytes ?? Math.floor(D1_SIZE_CEILING_BYTES * DEFAULT_SIZE_WARNING_RATIO);
172
+ const cutoff = new Date(
173
+ Date.parse(deps.now) - retentionDays * 24 * 60 * 60 * 1000,
174
+ ).toISOString();
175
+
176
+ try {
177
+ const db = fromD1Rest(deps.control, row.database_uuid);
178
+
179
+ // Read the aging-out rows BEFORE deleting them — the rollup needs them.
180
+ const aging = await db.all<Record<string, unknown>>(
181
+ `SELECT tenant_id, stream, type, el, action, placement, scope, idx, correlation_id, seq, ts, ingested_at
182
+ FROM interaction_events WHERE ingested_at < ?`,
183
+ cutoff,
184
+ );
185
+
186
+ let aggregatesWritten = 0;
187
+ if (aging.length > 0) {
188
+ if (deps.analytics) {
189
+ const aggregates = aggregateInteractions(aging.map(rawRowToEvent));
190
+ writeInteractionAnalytics(deps.analytics, aggregates);
191
+ aggregatesWritten = aggregates.length;
192
+ }
193
+ // The DELETE itself never depends on the rollup succeeding — retention
194
+ // must not stall because the aggregates sink is unavailable.
195
+ await db.run(`DELETE FROM interaction_events WHERE ingested_at < ?`, cutoff);
196
+ }
197
+
198
+ const sizeBytes = await readDatabaseSizeBytes(db);
199
+ const sizeWarning = sizeBytes != null && sizeBytes >= sizeWarningBytes;
200
+ if (sizeWarning) {
201
+ console.warn(
202
+ `[tracking:prune] tenant ${row.tenant_id} database at ${sizeBytes} bytes — ` +
203
+ `approaching the ${D1_SIZE_CEILING_BYTES}-byte D1 per-database ceiling`,
204
+ );
205
+ }
206
+
207
+ await deps.apps.run(
208
+ `UPDATE tracking_tenant_databases
209
+ SET last_pruned_at = ?, pruned_rows = pruned_rows + ?, size_bytes = ?, size_checked_at = ?
210
+ WHERE tenant_id = ?`,
211
+ deps.now,
212
+ aging.length,
213
+ sizeBytes,
214
+ deps.now,
215
+ row.tenant_id,
216
+ );
217
+
218
+ return {
219
+ tenantId: row.tenant_id,
220
+ prunedRows: aging.length,
221
+ aggregatesWritten,
222
+ sizeBytes,
223
+ sizeWarning,
224
+ };
225
+ } catch (err) {
226
+ const message = err instanceof Error ? err.message : String(err);
227
+ console.error(`[tracking:prune] sweep failed for ${row.tenant_id}: ${message}`);
228
+ return {
229
+ tenantId: row.tenant_id,
230
+ prunedRows: 0,
231
+ aggregatesWritten: 0,
232
+ sizeBytes: null,
233
+ sizeWarning: false,
234
+ error: message,
235
+ };
236
+ }
237
+ }
238
+
239
+ /**
240
+ * Run one bounded batch of the prune sweep: pull the oldest-pruned tenants
241
+ * off the work queue and prune+roll-up each in turn. Sequential, not
242
+ * parallel — keeps one invocation's D1 REST call volume predictable rather
243
+ * than fanning every tenant in the batch out concurrently.
244
+ */
245
+ export async function runPruneSweep(deps: PruneSweepDeps): Promise<PruneSweepResult> {
246
+ const batchSize = deps.batchSize ?? DEFAULT_BATCH_SIZE;
247
+ const candidates = await listPruneCandidates(deps.apps, batchSize);
248
+
249
+ const tenants: TenantPruneOutcome[] = [];
250
+ for (const row of candidates) {
251
+ tenants.push(await pruneTenant(row, deps));
252
+ }
253
+
254
+ return {
255
+ tenantsSwept: tenants.length,
256
+ totalPrunedRows: tenants.reduce((n, t) => n + t.prunedRows, 0),
257
+ totalAggregatesWritten: tenants.reduce((n, t) => n + t.aggregatesWritten, 0),
258
+ tenants,
259
+ };
260
+ }