@tokenoftrust/storefront-runner 2.0.4 → 2.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/storefront/migrations-apps/0011_dt_tracking_tenant_databases.sql +23 -0
- package/apps/storefront/node_modules/.bin/vitest +2 -2
- package/apps/storefront/node_modules/.vite/deps/_metadata.json +19 -19
- package/apps/storefront/node_modules/.vite/deps_ssr/_metadata.json +3 -3
- package/apps/storefront/src/components/chrome/MegaMenu.astro +11 -1
- package/apps/storefront/src/components/chrome/NavDropdown.astro +49 -7
- package/apps/storefront/src/components/chrome/SiteFooter.astro +29 -4
- package/apps/storefront/src/components/chrome/SiteHeader.astro +39 -6
- package/apps/storefront/src/components/chrome/UtilityNav.astro +13 -1
- package/apps/storefront/src/components/islands/MobileNav.tsx +42 -2
- package/apps/storefront/src/layouts/Layout.astro +64 -6
- package/apps/storefront/src/lib/chrome/ctaIdentity.ts +21 -0
- package/apps/storefront/src/lib/chrome/examples/tokenoftrust.chrome.example.json +22 -22
- package/apps/storefront/src/lib/chrome/trackingPlacements.ts +36 -0
- package/apps/storefront/src/lib/d1/schema-apps.ts +57 -0
- package/apps/storefront/src/lib/rawChrome.ts +73 -14
- package/apps/storefront/src/lib/rawMarketingHtml.ts +30 -0
- package/apps/storefront/src/lib/shared/developerInvites.ts +6 -0
- package/apps/storefront/src/lib/storyblok/provider.ts +21 -5
- package/apps/storefront/src/lib/tracking/collectorAllowlist.ts +40 -0
- package/apps/storefront/src/lib/tracking/collectorIngest.ts +188 -0
- package/apps/storefront/src/lib/tracking/d1Control.ts +180 -0
- package/apps/storefront/src/lib/tracking/interactionCollector.ts +202 -0
- package/apps/storefront/src/lib/tracking/interactionHealth.ts +252 -0
- package/apps/storefront/src/lib/tracking/interactionSchema.ts +107 -0
- package/apps/storefront/src/lib/tracking/interactionStore.ts +0 -0
- package/apps/storefront/src/lib/tracking/pruneRollup.ts +260 -0
- package/apps/storefront/src/lib/tracking/publishSafetyNet.ts +162 -0
- package/apps/storefront/src/lib/tracking/tenantD1Registry.ts +214 -0
- package/apps/storefront/src/lib/tracking/tenantReadAccess.ts +138 -0
- package/apps/storefront/src/lib/tracking-writeback/rewriteTrackingIds.ts +175 -0
- package/apps/storefront/src/lib/tracking-writeback/writebackCandidate.ts +110 -0
- package/apps/storefront/src/pages/api/admin/interaction-insights.ts +101 -0
- package/apps/storefront/src/pages/api/tracking/interaction.ts +123 -0
- package/apps/storefront/src/pages/style-guide/[tenant]/[theme].astro +33 -14
- package/apps/storefront/src/workers/pruneRollupWorker.ts +120 -0
- package/package.json +1 -1
- package/packages/public-runtime/node_modules/.bin/vitest +2 -2
- package/packages/public-runtime/src/chrome.ts +18 -7
- package/packages/public-runtime/src/index.ts +10 -0
- package/packages/public-runtime/src/integration/index.ts +1 -0
- package/packages/public-runtime/src/integration/interaction-tracking-gate.ts +142 -0
- package/packages/public-runtime/src/interaction-tracking.ts +364 -0
- package/packages/public-runtime/src/tracking-id-derivation.ts +1008 -0
- package/scripts/tenant/validate.mjs +306 -1
- package/scripts/tot-dev.mjs +59 -3
- package/tenants/home/public/js/storefront.js +6 -87
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The interaction-tracking chain's HEALTH ASSERTION — the thing that makes a
|
|
3
|
+
* broken collection chain loud instead of silent.
|
|
4
|
+
*
|
|
5
|
+
* Why it exists. Every other part of this system is correctly fail-quiet at
|
|
6
|
+
* runtime: the delegated listener returns without an event when there is no
|
|
7
|
+
* `data-tot-el` up the ancestor chain (that is the `tracking: "none"` opt-out,
|
|
8
|
+
* where ABSENCE is the declaration), and the consent/environment gate drops a
|
|
9
|
+
* disallowed interaction with no error. Both are right. Together they mean a
|
|
10
|
+
* surface that was never stamped at all produces exactly the same observable
|
|
11
|
+
* result as a surface nobody clicked — which is the failure mode that made the
|
|
12
|
+
* legacy mixin useless for years: no way to tell "collection is broken" from
|
|
13
|
+
* "legitimately idle".
|
|
14
|
+
*
|
|
15
|
+
* This module is the one place that CAN tell them apart, because it looks at
|
|
16
|
+
* the rendered markup rather than at the event stream:
|
|
17
|
+
*
|
|
18
|
+
* - `mount_missing` — the document carries actionable elements but not the
|
|
19
|
+
* serialized listener, so no click on it can ever be seen. FATAL.
|
|
20
|
+
* - `chrome_unstamped` — the surface carries actionable elements and NOT ONE
|
|
21
|
+
* of them carries `data-tot-el`. FATAL. Zero-of-N is not an opt-out
|
|
22
|
+
* declaration; a real opt-out is per element, and an author opting out of
|
|
23
|
+
* every single element still leaves the OTHER elements stamped. This is
|
|
24
|
+
* the finding that covers the **sharedChrome bypass path**: a tenant that
|
|
25
|
+
* ships a hand-authored `content/<id>/chrome.html` bypasses governed
|
|
26
|
+
* chrome entirely, so `validateChromeConfig` never sees it,
|
|
27
|
+
* `trackingAttributes()` never stamps it, and `rewriteTrackingIds`
|
|
28
|
+
* (dt-id-writeback) explicitly refuses to touch chrome. Nothing else in
|
|
29
|
+
* the chain would ever say a word about it.
|
|
30
|
+
* - `stamp_incomplete` — elements carry `data-tot-el` but no
|
|
31
|
+
* `data-tot-action`. NOT fatal: the publish-time safety net
|
|
32
|
+
* (`publishSafetyNet.ts`) stamps identity only, by design, and there is no
|
|
33
|
+
* derivation path from identity to intent. Reported because those rows
|
|
34
|
+
* land in the store un-analyzable, which is worth seeing.
|
|
35
|
+
*
|
|
36
|
+
* NO DOM-TEXT READS: only `tagName` and the parsed attribute list are ever
|
|
37
|
+
* read, matching the payload guardrail the listener and every consumer are
|
|
38
|
+
* held to. Actionability is decided by `@tot/public-runtime`'s shared
|
|
39
|
+
* `isTrackableNode` — the same oracle the derivation engine and the writeback
|
|
40
|
+
* rewriter use, so "actionable" cannot mean three different things.
|
|
41
|
+
*/
|
|
42
|
+
import { parse } from "parse5";
|
|
43
|
+
import {
|
|
44
|
+
INTERACTION_TRACKING_BOOT_JS,
|
|
45
|
+
TRACKING_ATTR_ACTION,
|
|
46
|
+
TRACKING_ATTR_EL,
|
|
47
|
+
isTrackableNode,
|
|
48
|
+
type StructuralNode,
|
|
49
|
+
} from "@tot/public-runtime";
|
|
50
|
+
|
|
51
|
+
/** The finding codes, in the order this module reports them. */
|
|
52
|
+
export type TrackingHealthCode = "mount_missing" | "chrome_unstamped" | "stamp_incomplete";
|
|
53
|
+
|
|
54
|
+
/** The codes that mean the chain is BROKEN rather than merely imperfect. */
|
|
55
|
+
const FATAL_CODES: ReadonlySet<TrackingHealthCode> = new Set<TrackingHealthCode>([
|
|
56
|
+
"mount_missing",
|
|
57
|
+
"chrome_unstamped",
|
|
58
|
+
]);
|
|
59
|
+
|
|
60
|
+
export interface TrackingHealthFinding {
|
|
61
|
+
code: TrackingHealthCode;
|
|
62
|
+
/** Human-actionable one-liner; carries counts and tags, never content. */
|
|
63
|
+
detail: string;
|
|
64
|
+
fatal: boolean;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface TrackingHealthReport {
|
|
68
|
+
/** True iff there is no FATAL finding — i.e. a click here can be collected. */
|
|
69
|
+
ok: boolean;
|
|
70
|
+
/** Elements an interaction can originate from (`isTrackableNode`). */
|
|
71
|
+
actionable: number;
|
|
72
|
+
/** Of those, how many carry `data-tot-el`. */
|
|
73
|
+
stamped: number;
|
|
74
|
+
/** Of the stamped ones, how many also carry `data-tot-action`. */
|
|
75
|
+
attributed: number;
|
|
76
|
+
findings: TrackingHealthFinding[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export interface TrackingHealthOptions {
|
|
80
|
+
/**
|
|
81
|
+
* Require the serialized listener to be present. True for a whole document,
|
|
82
|
+
* false for a chrome WRAPPER inspected before `rawHtmlResponse()` has
|
|
83
|
+
* injected the listener into it.
|
|
84
|
+
*/
|
|
85
|
+
requireMount?: boolean;
|
|
86
|
+
/** A label for the log/error line — e.g. `chrome.html (tenant acme)`. */
|
|
87
|
+
surface?: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Thrown by {@link assertTrackingChainHealthy} on a FATAL finding. */
|
|
91
|
+
export class TrackingChainBrokenError extends Error {
|
|
92
|
+
readonly report: TrackingHealthReport;
|
|
93
|
+
|
|
94
|
+
constructor(report: TrackingHealthReport, surface: string) {
|
|
95
|
+
const codes = report.findings.filter((f) => f.fatal).map((f) => f.code);
|
|
96
|
+
super(
|
|
97
|
+
`interaction tracking is BROKEN on ${surface} [${codes.join(", ")}] — ` +
|
|
98
|
+
report.findings
|
|
99
|
+
.filter((f) => f.fatal)
|
|
100
|
+
.map((f) => f.detail)
|
|
101
|
+
.join("; ") +
|
|
102
|
+
` (actionable=${report.actionable}, stamped=${report.stamped}, ` +
|
|
103
|
+
`attributed=${report.attributed}). A click on this surface produces NO ` +
|
|
104
|
+
`retrievable event, and nothing else in the chain reports it: the listener ` +
|
|
105
|
+
`returns silently when no ${TRACKING_ATTR_EL} is present, which is also how a ` +
|
|
106
|
+
`legitimate \`tracking: "none"\` opt-out looks.`,
|
|
107
|
+
);
|
|
108
|
+
this.name = "TrackingChainBrokenError";
|
|
109
|
+
this.report = report;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
interface P5Attr {
|
|
114
|
+
name: string;
|
|
115
|
+
value: string;
|
|
116
|
+
}
|
|
117
|
+
interface P5Node {
|
|
118
|
+
nodeName: string;
|
|
119
|
+
tagName?: string;
|
|
120
|
+
attrs?: P5Attr[];
|
|
121
|
+
childNodes?: P5Node[];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function nonEmpty(v: string | undefined): boolean {
|
|
125
|
+
return typeof v === "string" && v.trim().length > 0;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Inspect one rendered surface (a full document, a chrome wrapper, or a body
|
|
130
|
+
* fragment). PURE — no I/O, no logging, no throwing. `html` that parses to
|
|
131
|
+
* nothing actionable is healthy by definition: a surface with no actionable
|
|
132
|
+
* element has nothing to collect, and saying otherwise would make every
|
|
133
|
+
* text-only page a false alarm.
|
|
134
|
+
*/
|
|
135
|
+
export function inspectTrackingChain(
|
|
136
|
+
html: string,
|
|
137
|
+
options: TrackingHealthOptions = {},
|
|
138
|
+
): TrackingHealthReport {
|
|
139
|
+
let actionable = 0;
|
|
140
|
+
let stamped = 0;
|
|
141
|
+
let attributed = 0;
|
|
142
|
+
const unstampedTags = new Set<string>();
|
|
143
|
+
|
|
144
|
+
const walk = (node: P5Node): void => {
|
|
145
|
+
if (node.tagName) {
|
|
146
|
+
const attrs: Record<string, string> = {};
|
|
147
|
+
for (const a of node.attrs ?? []) attrs[a.name] = a.value;
|
|
148
|
+
const structural: StructuralNode = { tag: node.tagName.toLowerCase(), attrs };
|
|
149
|
+
const hasEl = nonEmpty(attrs[TRACKING_ATTR_EL]);
|
|
150
|
+
if (isTrackableNode(structural)) {
|
|
151
|
+
actionable += 1;
|
|
152
|
+
if (hasEl) {
|
|
153
|
+
stamped += 1;
|
|
154
|
+
if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
|
|
155
|
+
} else {
|
|
156
|
+
unstampedTags.add(structural.tag);
|
|
157
|
+
}
|
|
158
|
+
} else if (hasEl) {
|
|
159
|
+
// Stamped but not actionable by the shared oracle (e.g. a stamped
|
|
160
|
+
// wrapper). Counted so `stamped` never understates the surface.
|
|
161
|
+
stamped += 1;
|
|
162
|
+
if (nonEmpty(attrs[TRACKING_ATTR_ACTION])) attributed += 1;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
for (const child of node.childNodes ?? []) walk(child);
|
|
166
|
+
};
|
|
167
|
+
walk(parse(html) as P5Node);
|
|
168
|
+
|
|
169
|
+
const findings: TrackingHealthFinding[] = [];
|
|
170
|
+
|
|
171
|
+
if (options.requireMount && actionable > 0 && !html.includes(INTERACTION_TRACKING_BOOT_JS)) {
|
|
172
|
+
findings.push({
|
|
173
|
+
code: "mount_missing",
|
|
174
|
+
detail: `${actionable} actionable element(s) but the serialized interaction-tracking listener is not in this document`,
|
|
175
|
+
fatal: true,
|
|
176
|
+
});
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
if (actionable > 0 && stamped === 0) {
|
|
180
|
+
findings.push({
|
|
181
|
+
code: "chrome_unstamped",
|
|
182
|
+
detail:
|
|
183
|
+
`${actionable} actionable element(s) (${[...unstampedTags].sort().join(", ")}) and NOT ONE ` +
|
|
184
|
+
`carries ${TRACKING_ATTR_EL} — an unstamped surface, not a per-element opt-out`,
|
|
185
|
+
fatal: true,
|
|
186
|
+
});
|
|
187
|
+
} else if (stamped > attributed) {
|
|
188
|
+
findings.push({
|
|
189
|
+
code: "stamp_incomplete",
|
|
190
|
+
detail: `${stamped - attributed} of ${stamped} stamped element(s) carry no ${TRACKING_ATTR_ACTION}, so their rows land un-analyzable`,
|
|
191
|
+
fatal: false,
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
return {
|
|
196
|
+
ok: !findings.some((f) => f.fatal),
|
|
197
|
+
actionable,
|
|
198
|
+
stamped,
|
|
199
|
+
attributed,
|
|
200
|
+
findings,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* {@link inspectTrackingChain}, but a FATAL finding THROWS. For build/publish
|
|
206
|
+
* gates and tests — anywhere a broken chain should stop the line rather than
|
|
207
|
+
* be noted.
|
|
208
|
+
*/
|
|
209
|
+
export function assertTrackingChainHealthy(
|
|
210
|
+
html: string,
|
|
211
|
+
options: TrackingHealthOptions = {},
|
|
212
|
+
): TrackingHealthReport {
|
|
213
|
+
const report = inspectTrackingChain(html, options);
|
|
214
|
+
if (!report.ok) throw new TrackingChainBrokenError(report, options.surface ?? "this surface");
|
|
215
|
+
return report;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* The render-path form: NEVER throws, emits ONE distinct structured line per
|
|
220
|
+
* distinct outcome so a broken surface is visible in Workers Logs. Returns the
|
|
221
|
+
* report so a caller can act on it.
|
|
222
|
+
*
|
|
223
|
+
* Non-throwing on purpose — a tenant whose hand-authored chrome was never
|
|
224
|
+
* stamped should get a loud log line, not a blank page.
|
|
225
|
+
*/
|
|
226
|
+
const reported = new Set<string>();
|
|
227
|
+
/** Bound the dedupe set so a long-lived isolate cannot grow it without limit. */
|
|
228
|
+
const REPORTED_MAX = 64;
|
|
229
|
+
|
|
230
|
+
export function reportTrackingChainHealth(
|
|
231
|
+
html: string,
|
|
232
|
+
options: TrackingHealthOptions = {},
|
|
233
|
+
): TrackingHealthReport {
|
|
234
|
+
const report = inspectTrackingChain(html, options);
|
|
235
|
+
if (report.findings.length === 0) return report;
|
|
236
|
+
const surface = options.surface ?? "unknown-surface";
|
|
237
|
+
const key = `${surface}|${report.findings.map((f) => f.code).join(",")}|${report.actionable}/${report.stamped}`;
|
|
238
|
+
if (reported.has(key)) return report;
|
|
239
|
+
if (reported.size >= REPORTED_MAX) reported.clear();
|
|
240
|
+
reported.add(key);
|
|
241
|
+
for (const finding of report.findings) {
|
|
242
|
+
const line = `[tracking:health] ${finding.fatal ? "BROKEN" : "degraded"} ${finding.code} on ${surface}: ${finding.detail}`;
|
|
243
|
+
if (finding.fatal) console.error(line);
|
|
244
|
+
else console.warn(line);
|
|
245
|
+
}
|
|
246
|
+
return report;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Test-only: forget which lines have already been emitted. */
|
|
250
|
+
export function resetTrackingHealthReportingForTests(): void {
|
|
251
|
+
reported.clear();
|
|
252
|
+
}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The RAW interaction-event table — the schema of a TENANT'S OWN D1 database.
|
|
3
|
+
*
|
|
4
|
+
* PER-TENANT, NOT PER-COLUMN. Every other D1 table in this repo lives in a
|
|
5
|
+
* shared database and carries a `tenant_id` column as its isolation boundary
|
|
6
|
+
* (see `d1/schema-apps.ts`). This one does NOT work that way: each tenant gets
|
|
7
|
+
* its OWN D1 database holding only its own rows, so cross-tenant leakage is
|
|
8
|
+
* impossible BY CONSTRUCTION rather than by a correct `WHERE tenant_id = ?` in
|
|
9
|
+
* every query. A missing predicate here cannot reach another tenant's data —
|
|
10
|
+
* there is no other tenant's data in the file.
|
|
11
|
+
*
|
|
12
|
+
* `tenant_id` is still stored on the row (redundantly, one constant value per
|
|
13
|
+
* database) deliberately: it makes an exported/backed-up/rolled-up row
|
|
14
|
+
* self-describing, and it lets a mis-routed insert be DETECTED (a row whose
|
|
15
|
+
* `tenant_id` disagrees with the database's owner is a routing bug, and
|
|
16
|
+
* `dt-tenant-isolation` can assert on exactly that).
|
|
17
|
+
*
|
|
18
|
+
* WHY D1 AND NOT A SHARED STORE: D1 is not billed per database — thousands of
|
|
19
|
+
* databases cost the same as one — so per-tenant databases are the endorsed
|
|
20
|
+
* shape, not a workaround. The hard limit is 10GB PER DATABASE (~30M rows at a
|
|
21
|
+
* 90-day window for a busy tenant), which is a per-tenant ceiling rather than a
|
|
22
|
+
* global one; `interaction_store_meta` below carries the counters
|
|
23
|
+
* `dt-prune-and-rollup` needs to surface a tenant approaching it.
|
|
24
|
+
*
|
|
25
|
+
* RETENTION is a 90-day window, enforced by a DELETE-based prune in a later
|
|
26
|
+
* unit (`dt-prune-and-rollup`) — D1 has no TTL. This schema only has to make
|
|
27
|
+
* that prune cheap, which is what `idx_interaction_events_ingested_at` is for:
|
|
28
|
+
* `DELETE FROM interaction_events WHERE ingested_at < ?` is an index range
|
|
29
|
+
* scan, not a table scan. `ingested_at` (the SERVER receive clock) is the prune
|
|
30
|
+
* key, never `ts` (a CLIENT clock a beacon can skew arbitrarily).
|
|
31
|
+
*
|
|
32
|
+
* SCHEMA IS APPLIED, NOT MIGRATED. A per-tenant database is created on demand
|
|
33
|
+
* at runtime, so it cannot ride the numbered `migrations-apps/` lineage (which
|
|
34
|
+
* `wrangler d1 migrations apply` runs against ONE named binding). Instead this
|
|
35
|
+
* DDL is idempotent (`IF NOT EXISTS` throughout) and applied at provision time;
|
|
36
|
+
* `INTERACTION_SCHEMA_VERSION` is recorded in `interaction_store_meta` so a
|
|
37
|
+
* future version bump can be detected and re-applied per tenant.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
/** Bumped whenever `INTERACTION_D1_SCHEMA` changes; recorded per database. */
|
|
41
|
+
export const INTERACTION_SCHEMA_VERSION = 1;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Idempotent DDL for one tenant's raw-interaction database. Multi-statement:
|
|
45
|
+
* applied via `Queryable.exec`, which both the D1 binding and the D1 HTTP API
|
|
46
|
+
* accept as a semicolon-separated script.
|
|
47
|
+
*/
|
|
48
|
+
export const INTERACTION_D1_SCHEMA = `
|
|
49
|
+
CREATE TABLE IF NOT EXISTS interaction_events (
|
|
50
|
+
-- Owning tenant. One constant value per database (see the module note): the
|
|
51
|
+
-- isolation boundary is the DATABASE, this column is provenance + a
|
|
52
|
+
-- mis-routing tripwire.
|
|
53
|
+
tenant_id TEXT NOT NULL,
|
|
54
|
+
-- Stream discriminator, always "interaction" today. Present from row one so a
|
|
55
|
+
-- second stream can land in the same table without a migration.
|
|
56
|
+
stream TEXT NOT NULL,
|
|
57
|
+
-- The six TotInteraction fields, flattened for SQL aggregation.
|
|
58
|
+
type TEXT NOT NULL,
|
|
59
|
+
el TEXT NOT NULL,
|
|
60
|
+
action TEXT,
|
|
61
|
+
placement TEXT,
|
|
62
|
+
scope TEXT,
|
|
63
|
+
idx INTEGER,
|
|
64
|
+
-- Groups the events of one collector lifetime (~one page load).
|
|
65
|
+
correlation_id TEXT NOT NULL,
|
|
66
|
+
-- Monotonic within correlation_id, assigned by the client collector.
|
|
67
|
+
seq INTEGER NOT NULL,
|
|
68
|
+
-- Client emit clock (ISO-8601). Informational: never the prune or ordering key.
|
|
69
|
+
ts TEXT NOT NULL,
|
|
70
|
+
-- Server receive clock (ISO-8601). THE retention/prune key.
|
|
71
|
+
ingested_at TEXT NOT NULL,
|
|
72
|
+
-- (correlation_id, seq) is the collector's own natural identity, so a
|
|
73
|
+
-- retried/duplicated beacon re-inserting the same event is a no-op under
|
|
74
|
+
-- INSERT OR IGNORE rather than a double-count.
|
|
75
|
+
PRIMARY KEY (correlation_id, seq)
|
|
76
|
+
);
|
|
77
|
+
CREATE INDEX IF NOT EXISTS idx_interaction_events_ingested_at
|
|
78
|
+
ON interaction_events (ingested_at);
|
|
79
|
+
CREATE INDEX IF NOT EXISTS idx_interaction_events_stream_ingested_at
|
|
80
|
+
ON interaction_events (stream, ingested_at);
|
|
81
|
+
CREATE INDEX IF NOT EXISTS idx_interaction_events_el
|
|
82
|
+
ON interaction_events (el, ingested_at);
|
|
83
|
+
CREATE TABLE IF NOT EXISTS interaction_store_meta (
|
|
84
|
+
-- Single-row table. The constant primary key makes that a schema invariant.
|
|
85
|
+
-- (Keep every inline comment free of semicolons: this DDL is executed as one
|
|
86
|
+
-- semicolon-separated script.)
|
|
87
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
88
|
+
tenant_id TEXT NOT NULL,
|
|
89
|
+
schema_version INTEGER NOT NULL,
|
|
90
|
+
created_at TEXT NOT NULL,
|
|
91
|
+
updated_at TEXT NOT NULL
|
|
92
|
+
);
|
|
93
|
+
`;
|
|
94
|
+
|
|
95
|
+
/** Statements that stamp/refresh the single `interaction_store_meta` row. */
|
|
96
|
+
export function interactionMetaUpsert(tenantId: string, at: string): {
|
|
97
|
+
sql: string;
|
|
98
|
+
params: readonly unknown[];
|
|
99
|
+
} {
|
|
100
|
+
return {
|
|
101
|
+
sql: `INSERT INTO interaction_store_meta (id, tenant_id, schema_version, created_at, updated_at)
|
|
102
|
+
VALUES (1, ?, ?, ?, ?)
|
|
103
|
+
ON CONFLICT(id) DO UPDATE SET schema_version = excluded.schema_version,
|
|
104
|
+
updated_at = excluded.updated_at`,
|
|
105
|
+
params: [tenantId, INTERACTION_SCHEMA_VERSION, at, at],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
Binary file
|
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scheduled PRUNE + ROLLUP for the per-tenant raw-interaction D1 stores
|
|
3
|
+
* (`dt-storage-per-tenant-d1`).
|
|
4
|
+
*
|
|
5
|
+
* WHAT THIS DOES, per tenant, per sweep:
|
|
6
|
+
* 1. READ the rows aging out of the 90-day window (`ingested_at < cutoff`,
|
|
7
|
+
* the SERVER receive clock — never the client `ts`, which a beacon can
|
|
8
|
+
* skew arbitrarily).
|
|
9
|
+
* 2. ROLL UP those rows via the already-built, pure `aggregateInteractions()`
|
|
10
|
+
* and write them to the dedicated `INTERACTION_ANALYTICS` dataset via
|
|
11
|
+
* the already-built `writeInteractionAnalytics()` — durable long-term
|
|
12
|
+
* aggregates survive the delete below. Both are reused verbatim from
|
|
13
|
+
* `interactionStore.ts`, never reimplemented here.
|
|
14
|
+
* 3. DELETE those same rows. `DELETE FROM interaction_events WHERE
|
|
15
|
+
* ingested_at < ?` is an index range scan
|
|
16
|
+
* (`idx_interaction_events_ingested_at`), not a table scan.
|
|
17
|
+
* 4. MEASURE the database's current size (`PRAGMA page_count * page_size`)
|
|
18
|
+
* and warn when a tenant is approaching D1's 10GB-per-database ceiling —
|
|
19
|
+
* the point is to surface this BEFORE writes start failing, not after.
|
|
20
|
+
* 5. WRITE BACK the `tracking_tenant_databases` bookkeeping columns
|
|
21
|
+
* (`last_pruned_at`, `pruned_rows`, `size_bytes`, `size_checked_at`) that
|
|
22
|
+
* `dt-storage-per-tenant-d1` added for exactly this purpose.
|
|
23
|
+
*
|
|
24
|
+
* WORK QUEUE. One sweep processes a bounded BATCH of tenants
|
|
25
|
+
* (`ORDER BY last_pruned_at ASC` — SQLite sorts NULL first, so a tenant that
|
|
26
|
+
* has never been pruned is swept immediately), not the whole fleet in one
|
|
27
|
+
* invocation. A frequent-enough cron (see `workers/pruneRollupWorker.ts`)
|
|
28
|
+
* cycles fairly through every tenant over time instead of risking one run
|
|
29
|
+
* exceeding the Worker's CPU/wall-clock budget as the tenant count grows.
|
|
30
|
+
*
|
|
31
|
+
* NOT DONE HERE (by design, per the binding decisions this unit follows):
|
|
32
|
+
* - No DROP-based time-partitioned schema. The DROP-vs-DELETE cost
|
|
33
|
+
* asymmetry noted in `d1-cost-basis-verified` is UNVERIFIED inference;
|
|
34
|
+
* this sticks to the straightforward `DELETE ... WHERE ingested_at < ?`
|
|
35
|
+
* against the single `interaction_events` table dt-storage-per-tenant-d1
|
|
36
|
+
* already built. A future optimization pass can revisit it once verified.
|
|
37
|
+
* - No outbound alert transport. Same posture as `activity/alerts.ts`: this
|
|
38
|
+
* module DETECTS a tenant approaching the ceiling and logs it
|
|
39
|
+
* (`console.warn`, grep-able in Workers Logs) plus returns it in the
|
|
40
|
+
* structured sweep result — where it goes from there (a dashboard, a
|
|
41
|
+
* polled endpoint) is a sink's job, not this module's.
|
|
42
|
+
*
|
|
43
|
+
* BEST-EFFORT PER TENANT. One tenant's prune failing (a REST error, a
|
|
44
|
+
* transient D1 fault) is caught, logged, and recorded in that tenant's
|
|
45
|
+
* outcome — it never aborts the rest of the batch.
|
|
46
|
+
*/
|
|
47
|
+
import type { Queryable } from "../d1/catalog.js";
|
|
48
|
+
import type { AnalyticsEngineLike } from "../activity/store.js";
|
|
49
|
+
import { fromD1Rest, type D1ControlConfig } from "./d1Control.js";
|
|
50
|
+
import { aggregateInteractions, writeInteractionAnalytics } from "./interactionStore.js";
|
|
51
|
+
import type { StampedInteractionEvent } from "./collectorIngest.js";
|
|
52
|
+
|
|
53
|
+
/** The settled retention window (`settled-operational-choices`). */
|
|
54
|
+
export const DEFAULT_RETENTION_DAYS = 90;
|
|
55
|
+
|
|
56
|
+
/** How many tenants one sweep invocation processes. */
|
|
57
|
+
export const DEFAULT_BATCH_SIZE = 25;
|
|
58
|
+
|
|
59
|
+
/** D1's documented per-database ceiling. */
|
|
60
|
+
export const D1_SIZE_CEILING_BYTES = 10 * 1024 * 1024 * 1024;
|
|
61
|
+
|
|
62
|
+
/** Warn at 80% of the ceiling — well before writes start failing. */
|
|
63
|
+
const DEFAULT_SIZE_WARNING_RATIO = 0.8;
|
|
64
|
+
|
|
65
|
+
/** One row of the prune work queue. */
|
|
66
|
+
export interface TenantPruneRow {
|
|
67
|
+
tenant_id: string;
|
|
68
|
+
database_uuid: string;
|
|
69
|
+
last_pruned_at: string | null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export interface PruneSweepDeps {
|
|
73
|
+
/** The shared STOREFRONT_APPS_DB handle holding the directory table. */
|
|
74
|
+
apps: Queryable;
|
|
75
|
+
/** D1 REST credentials + transport, for reaching each tenant's own database. */
|
|
76
|
+
control: D1ControlConfig;
|
|
77
|
+
/** The dedicated interaction AE dataset. Absent ⇒ rollup is skipped, prune
|
|
78
|
+
* still runs (never block retention on an unavailable aggregates sink). */
|
|
79
|
+
analytics?: AnalyticsEngineLike;
|
|
80
|
+
/** ISO-8601 "now" — injected so the sweep is deterministic under test. */
|
|
81
|
+
now: string;
|
|
82
|
+
retentionDays?: number;
|
|
83
|
+
batchSize?: number;
|
|
84
|
+
/** Absolute byte threshold for the ceiling warning; defaults to 80% of
|
|
85
|
+
* `D1_SIZE_CEILING_BYTES`. */
|
|
86
|
+
sizeWarningBytes?: number;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export interface TenantPruneOutcome {
|
|
90
|
+
tenantId: string;
|
|
91
|
+
prunedRows: number;
|
|
92
|
+
aggregatesWritten: number;
|
|
93
|
+
sizeBytes: number | null;
|
|
94
|
+
/** True when `sizeBytes` is at/above the warning threshold. */
|
|
95
|
+
sizeWarning: boolean;
|
|
96
|
+
/** Set only when this tenant's sweep failed; the batch still continues. */
|
|
97
|
+
error?: string;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface PruneSweepResult {
|
|
101
|
+
tenantsSwept: number;
|
|
102
|
+
totalPrunedRows: number;
|
|
103
|
+
totalAggregatesWritten: number;
|
|
104
|
+
tenants: TenantPruneOutcome[];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Map one raw `interaction_events` row (snake_case D1 columns) back to the
|
|
108
|
+
* `StampedInteractionEvent` shape `aggregateInteractions()` expects. PURE. */
|
|
109
|
+
export function rawRowToEvent(row: Record<string, unknown>): StampedInteractionEvent {
|
|
110
|
+
return {
|
|
111
|
+
type: "click",
|
|
112
|
+
el: String(row.el ?? ""),
|
|
113
|
+
action: (row.action as string | null) ?? null,
|
|
114
|
+
placement: (row.placement as string | null) ?? null,
|
|
115
|
+
scope: (row.scope as string | null) ?? null,
|
|
116
|
+
index: (row.idx as number | null) ?? null,
|
|
117
|
+
ts: String(row.ts ?? ""),
|
|
118
|
+
correlationId: String(row.correlation_id ?? ""),
|
|
119
|
+
seq: Number(row.seq ?? 0),
|
|
120
|
+
tenant_id: String(row.tenant_id ?? ""),
|
|
121
|
+
stream: "interaction",
|
|
122
|
+
ingested_at: String(row.ingested_at ?? ""),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** The prune work queue: oldest-`last_pruned_at`-first (NULL — never pruned —
|
|
127
|
+
* sorts first in SQLite ASC). Never throws. */
|
|
128
|
+
export async function listPruneCandidates(
|
|
129
|
+
apps: Queryable,
|
|
130
|
+
batchSize: number,
|
|
131
|
+
): Promise<TenantPruneRow[]> {
|
|
132
|
+
try {
|
|
133
|
+
return await apps.all<TenantPruneRow>(
|
|
134
|
+
`SELECT tenant_id, database_uuid, last_pruned_at
|
|
135
|
+
FROM tracking_tenant_databases
|
|
136
|
+
ORDER BY last_pruned_at ASC
|
|
137
|
+
LIMIT ?`,
|
|
138
|
+
batchSize,
|
|
139
|
+
);
|
|
140
|
+
} catch (err) {
|
|
141
|
+
console.error(
|
|
142
|
+
`[tracking:prune] directory read failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
143
|
+
);
|
|
144
|
+
return [];
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** `PRAGMA page_count * page_size` — the current on-disk size of one tenant's
|
|
149
|
+
* database. Returns null (never throws) when the pragmas are unavailable. */
|
|
150
|
+
async function readDatabaseSizeBytes(db: Queryable): Promise<number | null> {
|
|
151
|
+
try {
|
|
152
|
+
const pageCount = await db.first<{ page_count: number }>("PRAGMA page_count");
|
|
153
|
+
const pageSize = await db.first<{ page_size: number }>("PRAGMA page_size");
|
|
154
|
+
if (!pageCount || !pageSize) return null;
|
|
155
|
+
const count = Number(pageCount.page_count);
|
|
156
|
+
const size = Number(pageSize.page_size);
|
|
157
|
+
if (!Number.isFinite(count) || !Number.isFinite(size)) return null;
|
|
158
|
+
return count * size;
|
|
159
|
+
} catch {
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Prune + roll up ONE tenant's database. Never throws. */
|
|
165
|
+
export async function pruneTenant(
|
|
166
|
+
row: TenantPruneRow,
|
|
167
|
+
deps: PruneSweepDeps,
|
|
168
|
+
): Promise<TenantPruneOutcome> {
|
|
169
|
+
const retentionDays = deps.retentionDays ?? DEFAULT_RETENTION_DAYS;
|
|
170
|
+
const sizeWarningBytes =
|
|
171
|
+
deps.sizeWarningBytes ?? Math.floor(D1_SIZE_CEILING_BYTES * DEFAULT_SIZE_WARNING_RATIO);
|
|
172
|
+
const cutoff = new Date(
|
|
173
|
+
Date.parse(deps.now) - retentionDays * 24 * 60 * 60 * 1000,
|
|
174
|
+
).toISOString();
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
const db = fromD1Rest(deps.control, row.database_uuid);
|
|
178
|
+
|
|
179
|
+
// Read the aging-out rows BEFORE deleting them — the rollup needs them.
|
|
180
|
+
const aging = await db.all<Record<string, unknown>>(
|
|
181
|
+
`SELECT tenant_id, stream, type, el, action, placement, scope, idx, correlation_id, seq, ts, ingested_at
|
|
182
|
+
FROM interaction_events WHERE ingested_at < ?`,
|
|
183
|
+
cutoff,
|
|
184
|
+
);
|
|
185
|
+
|
|
186
|
+
let aggregatesWritten = 0;
|
|
187
|
+
if (aging.length > 0) {
|
|
188
|
+
if (deps.analytics) {
|
|
189
|
+
const aggregates = aggregateInteractions(aging.map(rawRowToEvent));
|
|
190
|
+
writeInteractionAnalytics(deps.analytics, aggregates);
|
|
191
|
+
aggregatesWritten = aggregates.length;
|
|
192
|
+
}
|
|
193
|
+
// The DELETE itself never depends on the rollup succeeding — retention
|
|
194
|
+
// must not stall because the aggregates sink is unavailable.
|
|
195
|
+
await db.run(`DELETE FROM interaction_events WHERE ingested_at < ?`, cutoff);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
const sizeBytes = await readDatabaseSizeBytes(db);
|
|
199
|
+
const sizeWarning = sizeBytes != null && sizeBytes >= sizeWarningBytes;
|
|
200
|
+
if (sizeWarning) {
|
|
201
|
+
console.warn(
|
|
202
|
+
`[tracking:prune] tenant ${row.tenant_id} database at ${sizeBytes} bytes — ` +
|
|
203
|
+
`approaching the ${D1_SIZE_CEILING_BYTES}-byte D1 per-database ceiling`,
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
await deps.apps.run(
|
|
208
|
+
`UPDATE tracking_tenant_databases
|
|
209
|
+
SET last_pruned_at = ?, pruned_rows = pruned_rows + ?, size_bytes = ?, size_checked_at = ?
|
|
210
|
+
WHERE tenant_id = ?`,
|
|
211
|
+
deps.now,
|
|
212
|
+
aging.length,
|
|
213
|
+
sizeBytes,
|
|
214
|
+
deps.now,
|
|
215
|
+
row.tenant_id,
|
|
216
|
+
);
|
|
217
|
+
|
|
218
|
+
return {
|
|
219
|
+
tenantId: row.tenant_id,
|
|
220
|
+
prunedRows: aging.length,
|
|
221
|
+
aggregatesWritten,
|
|
222
|
+
sizeBytes,
|
|
223
|
+
sizeWarning,
|
|
224
|
+
};
|
|
225
|
+
} catch (err) {
|
|
226
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
227
|
+
console.error(`[tracking:prune] sweep failed for ${row.tenant_id}: ${message}`);
|
|
228
|
+
return {
|
|
229
|
+
tenantId: row.tenant_id,
|
|
230
|
+
prunedRows: 0,
|
|
231
|
+
aggregatesWritten: 0,
|
|
232
|
+
sizeBytes: null,
|
|
233
|
+
sizeWarning: false,
|
|
234
|
+
error: message,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Run one bounded batch of the prune sweep: pull the oldest-pruned tenants
|
|
241
|
+
* off the work queue and prune+roll-up each in turn. Sequential, not
|
|
242
|
+
* parallel — keeps one invocation's D1 REST call volume predictable rather
|
|
243
|
+
* than fanning every tenant in the batch out concurrently.
|
|
244
|
+
*/
|
|
245
|
+
export async function runPruneSweep(deps: PruneSweepDeps): Promise<PruneSweepResult> {
|
|
246
|
+
const batchSize = deps.batchSize ?? DEFAULT_BATCH_SIZE;
|
|
247
|
+
const candidates = await listPruneCandidates(deps.apps, batchSize);
|
|
248
|
+
|
|
249
|
+
const tenants: TenantPruneOutcome[] = [];
|
|
250
|
+
for (const row of candidates) {
|
|
251
|
+
tenants.push(await pruneTenant(row, deps));
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
return {
|
|
255
|
+
tenantsSwept: tenants.length,
|
|
256
|
+
totalPrunedRows: tenants.reduce((n, t) => n + t.prunedRows, 0),
|
|
257
|
+
totalAggregatesWritten: tenants.reduce((n, t) => n + t.aggregatesWritten, 0),
|
|
258
|
+
tenants,
|
|
259
|
+
};
|
|
260
|
+
}
|