blogwright-analytics 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (131) hide show
  1. package/README.md +162 -0
  2. package/dist/adapters/duckdb-ingest.d.ts +76 -0
  3. package/dist/adapters/duckdb-ingest.js +173 -0
  4. package/dist/adapters/duckdb-query.d.ts +56 -0
  5. package/dist/adapters/duckdb-query.js +80 -0
  6. package/dist/adapters/duckdb-session.d.ts +168 -0
  7. package/dist/adapters/duckdb-session.js +330 -0
  8. package/dist/app/_app/immutable/assets/0.BTQrrh5B.css +1 -0
  9. package/dist/app/_app/immutable/assets/2.CZSK3rT8.css +1 -0
  10. package/dist/app/_app/immutable/assets/BrushContext.D7c8UPey.css +1 -0
  11. package/dist/app/_app/immutable/assets/ChartAnnotations.CPxIG7Mw.css +1 -0
  12. package/dist/app/_app/immutable/assets/Circle.C5MKzgk2.css +1 -0
  13. package/dist/app/_app/immutable/assets/DefaultTooltip.C5-uctZ7.css +1 -0
  14. package/dist/app/_app/immutable/assets/Group.DV48xipa.css +1 -0
  15. package/dist/app/_app/immutable/assets/Labels.BxZ4NUVz.css +1 -0
  16. package/dist/app/_app/immutable/assets/Legend.CxnrE4Ye.css +1 -0
  17. package/dist/app/_app/immutable/assets/Line.fkmsECm9.css +1 -0
  18. package/dist/app/_app/immutable/assets/Path.CvpwNZ6g.css +1 -0
  19. package/dist/app/_app/immutable/assets/Rect.CtRaGMmQ.css +1 -0
  20. package/dist/app/_app/immutable/assets/Text.j9l35qB0.css +1 -0
  21. package/dist/app/_app/immutable/assets/TransformContext.Bs_HkpAk.css +1 -0
  22. package/dist/app/_app/immutable/assets/Voronoi.ce7atosu.css +1 -0
  23. package/dist/app/_app/immutable/chunks/-aNGNaBT.js +1 -0
  24. package/dist/app/_app/immutable/chunks/6djn-yLs.js +1 -0
  25. package/dist/app/_app/immutable/chunks/B1amyutE.js +1 -0
  26. package/dist/app/_app/immutable/chunks/B3vZDoek.js +1 -0
  27. package/dist/app/_app/immutable/chunks/B5KRA4hC.js +1 -0
  28. package/dist/app/_app/immutable/chunks/BClnVG6H.js +1 -0
  29. package/dist/app/_app/immutable/chunks/BID1NNRh.js +1 -0
  30. package/dist/app/_app/immutable/chunks/BR2LaRms.js +1 -0
  31. package/dist/app/_app/immutable/chunks/Bd1gDe3Y.js +1 -0
  32. package/dist/app/_app/immutable/chunks/Bjy-W4x2.js +81 -0
  33. package/dist/app/_app/immutable/chunks/Bl052uUt.js +1 -0
  34. package/dist/app/_app/immutable/chunks/Bye3lL0c.js +1 -0
  35. package/dist/app/_app/immutable/chunks/C58PZtCD.js +4 -0
  36. package/dist/app/_app/immutable/chunks/CAzydqEO.js +1 -0
  37. package/dist/app/_app/immutable/chunks/CCch3uox.js +1 -0
  38. package/dist/app/_app/immutable/chunks/CIlSMUH9.js +1 -0
  39. package/dist/app/_app/immutable/chunks/CO1vUXfR.js +1 -0
  40. package/dist/app/_app/immutable/chunks/CPbD8C65.js +5 -0
  41. package/dist/app/_app/immutable/chunks/CRTcXoMo.js +1 -0
  42. package/dist/app/_app/immutable/chunks/CjjyIQAO.js +1 -0
  43. package/dist/app/_app/immutable/chunks/CuXAxjvF.js +1 -0
  44. package/dist/app/_app/immutable/chunks/CvyVA_jC.js +1 -0
  45. package/dist/app/_app/immutable/chunks/CxGCFVdy.js +1 -0
  46. package/dist/app/_app/immutable/chunks/D0Ty6LN0.js +1 -0
  47. package/dist/app/_app/immutable/chunks/D2AaQUUW.js +1 -0
  48. package/dist/app/_app/immutable/chunks/D2BnX0Uk.js +3 -0
  49. package/dist/app/_app/immutable/chunks/DJc8C0NK.js +1 -0
  50. package/dist/app/_app/immutable/chunks/DKMlMI4a.js +1 -0
  51. package/dist/app/_app/immutable/chunks/DVXZkpbf.js +1 -0
  52. package/dist/app/_app/immutable/chunks/DVt8ukQ_.js +1 -0
  53. package/dist/app/_app/immutable/chunks/DZPlYdq_.js +1 -0
  54. package/dist/app/_app/immutable/chunks/Db0q5_zr.js +1 -0
  55. package/dist/app/_app/immutable/chunks/Dfvzj6n2.js +1 -0
  56. package/dist/app/_app/immutable/chunks/Dh958be7.js +1 -0
  57. package/dist/app/_app/immutable/chunks/DjKLLdnY.js +15 -0
  58. package/dist/app/_app/immutable/chunks/Doz7YX1W.js +1 -0
  59. package/dist/app/_app/immutable/chunks/DthYhn6Y.js +2 -0
  60. package/dist/app/_app/immutable/chunks/DtuTIrAM.js +1 -0
  61. package/dist/app/_app/immutable/chunks/HclGiUj8.js +1 -0
  62. package/dist/app/_app/immutable/chunks/Hx0TNsV3.js +1 -0
  63. package/dist/app/_app/immutable/chunks/RobXhXPM.js +1 -0
  64. package/dist/app/_app/immutable/chunks/V9ZjaxiY.js +1 -0
  65. package/dist/app/_app/immutable/chunks/Y5urAfNy.js +1 -0
  66. package/dist/app/_app/immutable/chunks/caXkbKD3.js +1 -0
  67. package/dist/app/_app/immutable/chunks/devYm2ud.js +1 -0
  68. package/dist/app/_app/immutable/chunks/mtZWP0zR.js +1 -0
  69. package/dist/app/_app/immutable/chunks/vDgBJUjM.js +1 -0
  70. package/dist/app/_app/immutable/chunks/xIq_fFFM.js +1 -0
  71. package/dist/app/_app/immutable/chunks/xihTtKlq.js +1 -0
  72. package/dist/app/_app/immutable/chunks/z05MoCFz.js +1 -0
  73. package/dist/app/_app/immutable/entry/app.CLAerUAN.js +2 -0
  74. package/dist/app/_app/immutable/entry/start.D3MqnNci.js +1 -0
  75. package/dist/app/_app/immutable/nodes/0.UTMEigHJ.js +1 -0
  76. package/dist/app/_app/immutable/nodes/1.Cn4f11bT.js +1 -0
  77. package/dist/app/_app/immutable/nodes/2.B39cIcr2.js +6 -0
  78. package/dist/app/_app/version.json +1 -0
  79. package/dist/app/index.html +82 -0
  80. package/dist/aws/clients.d.ts +70 -0
  81. package/dist/aws/clients.js +52 -0
  82. package/dist/aws/errors.d.ts +41 -0
  83. package/dist/aws/errors.js +70 -0
  84. package/dist/aws/firehose.d.ts +228 -0
  85. package/dist/aws/firehose.js +347 -0
  86. package/dist/aws/glue.d.ts +103 -0
  87. package/dist/aws/glue.js +225 -0
  88. package/dist/aws/lambda.d.ts +132 -0
  89. package/dist/aws/lambda.js +339 -0
  90. package/dist/aws/s3tables.d.ts +120 -0
  91. package/dist/aws/s3tables.js +281 -0
  92. package/dist/backfill.d.ts +100 -0
  93. package/dist/backfill.js +294 -0
  94. package/dist/commands.d.ts +124 -0
  95. package/dist/commands.js +336 -0
  96. package/dist/config.d.ts +162 -0
  97. package/dist/config.js +317 -0
  98. package/dist/fixture-ingest.d.ts +49 -0
  99. package/dist/fixture-ingest.js +43 -0
  100. package/dist/fixture-query.d.ts +39 -0
  101. package/dist/fixture-query.js +70 -0
  102. package/dist/index.d.ts +35 -0
  103. package/dist/index.js +35 -0
  104. package/dist/nodes.d.ts +404 -0
  105. package/dist/nodes.js +2708 -0
  106. package/dist/paths.d.ts +45 -0
  107. package/dist/paths.js +47 -0
  108. package/dist/plugin.d.ts +102 -0
  109. package/dist/plugin.js +248 -0
  110. package/dist/ports.d.ts +113 -0
  111. package/dist/ports.js +35 -0
  112. package/dist/queries.d.ts +301 -0
  113. package/dist/queries.js +414 -0
  114. package/dist/schema.d.ts +240 -0
  115. package/dist/schema.js +154 -0
  116. package/dist/server.d.ts +150 -0
  117. package/dist/server.js +499 -0
  118. package/dist/transform/bots.d.ts +47 -0
  119. package/dist/transform/bots.js +73 -0
  120. package/dist/transform/handler.d.ts +135 -0
  121. package/dist/transform/handler.js +177 -0
  122. package/dist/transform/map-record.d.ts +110 -0
  123. package/dist/transform/map-record.js +275 -0
  124. package/dist/transform/visitor-key.d.ts +83 -0
  125. package/dist/transform/visitor-key.js +120 -0
  126. package/dist/transform-bundle/index.mjs +21456 -0
  127. package/dist/transform-bundle/transform-manifest.json +4 -0
  128. package/dist/transform-hash.d.ts +135 -0
  129. package/dist/transform-hash.js +186 -0
  130. package/dist/write-transform-manifest.mjs +365 -0
  131. package/package.json +59 -0
@@ -0,0 +1,275 @@
1
+ /**
2
+ * `mapRecord`: one CloudFront standard-logging (v2) record in, one `page_views`
3
+ * row out - or a droppable result naming the column that could not be filled
4
+ * and the CloudFront field behind it. This is all five steps of
5
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md):
6
+ * rename each selected field to its column, derive `event_time` and the `day`
7
+ * partition from `timestamp(ms)`, replace the viewer IP with `visitor_key`,
8
+ * set `is_bot` from the user agent, and drop what the schema cannot accept.
9
+ *
10
+ * The viewer IP is an *input* here and never a value. It reaches `visitorKey`
11
+ * and nothing else: it has no `FIELD_TO_COLUMN` entry to be renamed through,
12
+ * and `map-record.test.ts` searches every value of every produced row for it.
13
+ * That search is the standing check that no later field addition puts a raw
14
+ * address in the warehouse.
15
+ *
16
+ * The salt secret is a parameter for the same reason the day is read off the
17
+ * record: this function has no clock and reads no secret. The caller supplies
18
+ * the long-lived stored secret - the transform reads it once at cold start, a
19
+ * backfill reads the same one - and the day's salt is derived here rather than
20
+ * by the caller, because the day comes from the record's own timestamp. A
21
+ * batch can straddle midnight, so one salt chosen for a whole batch would be
22
+ * the wrong day's for the records on the other side of it.
23
+ *
24
+ * Why the drop path exists at all: Firehose matches incoming JSON keys to
25
+ * Iceberg column names **exactly**, and a record it cannot match goes to the
26
+ * S3 error bucket with no error anywhere an operator will see it - the symptom
27
+ * is an empty dashboard. So this module never emits a key that is not a column
28
+ * and never emits a value whose JavaScript type is not the one the column
29
+ * stores. Where it cannot honour that, it returns the droppable result and the
30
+ * envelope (task 42) reports `ProcessingFailed` for that one record, which
31
+ * routes it to the error prefix without failing the batch.
32
+ *
33
+ * Every column and field name comes from `schema.ts`: the row is built by
34
+ * iterating `FIELD_TO_COLUMN`, the required set and the numeric set are
35
+ * derived from `PAGE_VIEWS_COLUMNS`, and the only names spelled here are the
36
+ * four derived columns and the one column the two of them read back
37
+ * (`user_agent`), each checked against `PageViewColumnName` so a typo is a
38
+ * compile error rather than a column Firehose silently drops.
39
+ *
40
+ * Three input decisions the spec left open, settled here:
41
+ *
42
+ * - **`-` and the empty string mean absent.** CloudFront writes `-` for a
43
+ * field the request had nothing to say for (no referrer, no query string).
44
+ * Writing that through would fill `referrer` with a wall of `-`; treating it
45
+ * as absent leaves the column null, which is what it means. For a *required*
46
+ * column it is a drop, not a null.
47
+ * - **A number where a string column is expected is rendered, not rejected.**
48
+ * `asn` is a string column that a JSON encoder may well emit unquoted;
49
+ * `String(64512)` is the same value in the type the column stores. Anything
50
+ * that is not a string or a number - an object, an array, a boolean - is a
51
+ * drop, because there is no rendering of it that is not a guess.
52
+ * - **A numeric column that does not parse is a drop, never a coerced value.**
53
+ * `Number('abc')` is `NaN` and `Number('')` is `0`; writing either would put
54
+ * a wrong number in the table, which is worse than losing the record.
55
+ *
56
+ * Nullability governs the *absent* case only. A value that is present but
57
+ * unusable drops the record whether or not the column is nullable: leaving a
58
+ * nullable column empty asserts the request had nothing to say for it, which
59
+ * is a different - and false - fact about that request.
60
+ *
61
+ * Pure: no clock (`event_time` comes from the record's own `timestamp(ms)`, in
62
+ * UTC), no secret read, no vendor SDK, no `fetch`. The one `node:` builtin
63
+ * this directory uses is `node:crypto`, reached through `visitor-key.ts` and
64
+ * only for the digests - the same import `packages/core/src/aws/s3.ts` makes.
65
+ * The record arrives already parsed and is trusted to be an object - JSON
66
+ * parsing and its failures are the handler's boundary, not this function's.
67
+ */
68
+ import { FIELD_TO_COLUMN, PAGE_VIEWS_COLUMNS, TIMESTAMP_MS_FIELD, VIEWER_IP_FIELD, } from '../schema.js';
69
+ import { isBotUserAgent } from './bots.js';
70
+ import { dailySalt, visitorKey } from './visitor-key.js';
71
+ /** CloudFront's marker for a field the request had nothing to say for. */
72
+ const CLOUDFRONT_EMPTY_VALUE = '-';
73
+ /** The `YYYY-MM-DD` prefix an ISO-8601 instant opens with, in characters. */
74
+ const ISO_DATE_LENGTH = 10;
75
+ /**
76
+ * An ISO-8601 instant with a four-digit year, as `toISOString` renders every
77
+ * year from 0000 to 9999. Outside that range it switches to an expanded year
78
+ * (`+058632-08-17T19:50:00.000Z`), whose first ten characters are not a date
79
+ * but would still slice cleanly into `day`. This is the check that stops that.
80
+ *
81
+ * Which wrong units it actually catches, taking 1788099825 as the instant:
82
+ *
83
+ * - **Microseconds** (`1788099825000000`) land in the year 58632. A `Date`
84
+ * holds it, `toISOString` renders it with an expanded year, and this regex
85
+ * rejects it. Dropped.
86
+ * - **Nanoseconds** (`1788099825000000000`) exceed the 8.64e15 ms either side
87
+ * of the epoch a `Date` can represent at all, so `eventTimeFrom`'s
88
+ * representability guard rejects it before this regex sees it. Dropped.
89
+ * - **Seconds** (`1788099825`) are *not* detected, and seconds are the
90
+ * likeliest wrong unit anyone would actually supply. Read as milliseconds
91
+ * the value is an ordinary instant - `1970-01-21T16:41:39.825Z` - so it
92
+ * passes both checks and writes a well-formed row under `day=1970-01-21`,
93
+ * a partition no query will look at and indistinguishable from a genuine
94
+ * 1970 record.
95
+ *
96
+ * The seconds case is a known hole, not an oversight: catching it needs a
97
+ * plausibility floor on the value rather than a shape check on the rendering,
98
+ * which is a different decision (what floor, and what to do at it) than this
99
+ * transform was scoped to make.
100
+ */
101
+ const ISO_INSTANT = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
102
+ /** Iceberg types the `PageView` row stores as a JavaScript `number`. */
103
+ const NUMERIC_ICEBERG_TYPES = new Set(['int', 'long', 'double']);
104
+ /** Columns Firehose may not write a null to; an absent value drops the record. */
105
+ const REQUIRED_COLUMNS = new Set(PAGE_VIEWS_COLUMNS.filter((column) => column.required).map((column) => column.name));
106
+ /** Columns whose value must be coerced to a number before it is written. */
107
+ const NUMERIC_COLUMNS = new Set(PAGE_VIEWS_COLUMNS.filter((column) => NUMERIC_ICEBERG_TYPES.has(column.icebergType)).map((column) => column.name));
108
+ /** The four columns this transform derives; checked against the table's names. */
109
+ const EVENT_TIME_COLUMN = 'event_time';
110
+ const DAY_COLUMN = 'day';
111
+ const VISITOR_KEY_COLUMN = 'visitor_key';
112
+ const IS_BOT_COLUMN = 'is_bot';
113
+ /** The mapped column both derived columns above read their user agent back from. */
114
+ const USER_AGENT_COLUMN = 'user_agent';
115
+ const ABSENT = { kind: 'absent' };
116
+ function invalid(detail) {
117
+ return { kind: 'invalid', detail };
118
+ }
119
+ /** The clause a drop reason ends with, for a field that yielded no value. */
120
+ function unfilledDetail(outcome) {
121
+ return outcome.kind === 'absent' ? 'is absent' : outcome.detail;
122
+ }
123
+ /** A string column's value, with CloudFront's absence markers read as absent. */
124
+ function stringFrom(raw) {
125
+ if (raw === undefined || raw === null)
126
+ return ABSENT;
127
+ if (typeof raw === 'number') {
128
+ return Number.isFinite(raw)
129
+ ? { kind: 'value', value: String(raw) }
130
+ : invalid(`holds ${String(raw)}, which is not a finite number`);
131
+ }
132
+ if (typeof raw !== 'string')
133
+ return invalid(`holds an unsupported ${typeof raw} value, which is not text`);
134
+ const value = raw.trim();
135
+ return value === '' || value === CLOUDFRONT_EMPTY_VALUE ? ABSENT : { kind: 'value', value };
136
+ }
137
+ /**
138
+ * A numeric column's value; anything that does not parse is a drop, not a
139
+ * `NaN`.
140
+ *
141
+ * Known gap: this coerces by JavaScript number, not by the column's Iceberg
142
+ * type, so an `int` or `long` column accepts a non-integral value as readily
143
+ * as a `double` does - `sc-status: '200.5'` is written through as `200.5`. No
144
+ * CloudFront field can produce such a value, so nothing here rounds or rejects
145
+ * one; a source that could would need an integrality check keyed on
146
+ * `icebergType`.
147
+ */
148
+ function numberFrom(raw) {
149
+ if (raw === undefined || raw === null)
150
+ return ABSENT;
151
+ if (typeof raw === 'number') {
152
+ return Number.isFinite(raw)
153
+ ? { kind: 'value', value: raw }
154
+ : invalid(`holds ${String(raw)}, which is not a finite number`);
155
+ }
156
+ if (typeof raw !== 'string')
157
+ return invalid(`holds an unsupported ${typeof raw} value, which is not a number`);
158
+ const text = raw.trim();
159
+ if (text === '' || text === CLOUDFRONT_EMPTY_VALUE)
160
+ return ABSENT;
161
+ const value = Number(text);
162
+ return Number.isFinite(value)
163
+ ? { kind: 'value', value }
164
+ : invalid(`holds "${raw}", which does not parse as a number`);
165
+ }
166
+ /** Reads one field the way its target column stores it. */
167
+ function columnValueFrom(raw, column) {
168
+ return NUMERIC_COLUMNS.has(column) ? numberFrom(raw) : stringFrom(raw);
169
+ }
170
+ /** `event_time` as a UTC ISO-8601 instant, from `timestamp(ms)`. */
171
+ function eventTimeFrom(raw) {
172
+ const milliseconds = numberFrom(raw);
173
+ if (milliseconds.kind !== 'value')
174
+ return milliseconds;
175
+ const instant = new Date(milliseconds.value);
176
+ if (Number.isNaN(instant.getTime())) {
177
+ return invalid(`holds ${String(milliseconds.value)}, which is not a representable instant`);
178
+ }
179
+ const eventTime = instant.toISOString();
180
+ return ISO_INSTANT.test(eventTime)
181
+ ? { kind: 'value', value: eventTime }
182
+ : invalid(`holds ${String(milliseconds.value)}, which falls outside the years 0000-9999`);
183
+ }
184
+ /** The `day` partition value: the UTC date `event_time` falls on. */
185
+ function dayFrom(eventTime) {
186
+ return eventTime.slice(0, ISO_DATE_LENGTH);
187
+ }
188
+ /**
189
+ * The user agent as the row will carry it - trimmed, with CloudFront's absence
190
+ * markers already read as absent - or `undefined` when the request named none.
191
+ * Both derived columns read it back from the row rather than from the record,
192
+ * so the text hashed into `visitor_key` and the text `is_bot` was decided from
193
+ * are exactly the text stored in `user_agent`. A divergence between the three
194
+ * would be invisible in the table and unfalsifiable from a query.
195
+ */
196
+ function userAgentOf(row) {
197
+ const userAgent = row[USER_AGENT_COLUMN];
198
+ return typeof userAgent === 'string' ? userAgent : undefined;
199
+ }
200
+ /** A drop naming the column, the field behind it, and what was wrong. */
201
+ function dropped(column, field, detail) {
202
+ return {
203
+ mapped: false,
204
+ column,
205
+ field,
206
+ reason: `page_views column "${column}" cannot be filled: CloudFront field "${field}" ${detail}`,
207
+ };
208
+ }
209
+ /** Both derived columns are lost together, because both read the same field. */
210
+ function droppedTimestamp(detail) {
211
+ return {
212
+ mapped: false,
213
+ column: EVENT_TIME_COLUMN,
214
+ field: TIMESTAMP_MS_FIELD,
215
+ reason: `page_views columns "${EVENT_TIME_COLUMN}" and "${DAY_COLUMN}" cannot be filled: CloudFront field "${TIMESTAMP_MS_FIELD}" ${detail}`,
216
+ };
217
+ }
218
+ /**
219
+ * Turns one CloudFront access-log record into a `page_views` row, or reports
220
+ * why it cannot. Never returns a partially populated row: the first column it
221
+ * cannot fill ends the mapping.
222
+ *
223
+ * `saltSecret` is the long-lived stored secret behind `visitor_key`, not the
224
+ * day's salt - that is derived here from the record's own day. It is required
225
+ * rather than optional on purpose: a row mapped without it would carry no
226
+ * visitor at all, or worse an unsalted digest, and either would look like a
227
+ * normal row in the table. `dailySalt` throws on an empty secret for the same
228
+ * reason, so a caller whose secret read failed fails its batch instead of
229
+ * quietly writing unprotected data.
230
+ */
231
+ export function mapRecord(record, saltSecret) {
232
+ const eventTime = eventTimeFrom(record[TIMESTAMP_MS_FIELD]);
233
+ if (eventTime.kind !== 'value')
234
+ return droppedTimestamp(unfilledDetail(eventTime));
235
+ const day = dayFrom(eventTime.value);
236
+ const columns = {
237
+ [EVENT_TIME_COLUMN]: eventTime.value,
238
+ [DAY_COLUMN]: day,
239
+ };
240
+ for (const [field, column] of Object.entries(FIELD_TO_COLUMN)) {
241
+ const outcome = columnValueFrom(record[field], column);
242
+ if (outcome.kind === 'invalid')
243
+ return dropped(column, field, outcome.detail);
244
+ if (outcome.kind === 'absent') {
245
+ if (REQUIRED_COLUMNS.has(column))
246
+ return dropped(column, field, unfilledDetail(outcome));
247
+ continue;
248
+ }
249
+ columns[column] = outcome.value;
250
+ }
251
+ // `is_bot` is set for every record, including the ones that named no agent:
252
+ // the column answers "did this agent name itself as a bot", and `false` is
253
+ // that question's answer for a request that named nothing.
254
+ const userAgent = userAgentOf(columns);
255
+ columns[IS_BOT_COLUMN] = isBotUserAgent(userAgent);
256
+ // The viewer IP goes in here and comes out as a digest. An unusable value
257
+ // drops the record like any other - the same rule the rest of this module
258
+ // follows - but an *absent* one leaves `visitor_key` empty rather than
259
+ // hashing the user agent alone, which would collapse every anonymous request
260
+ // in a day onto one fabricated returning visitor. A null visitor is exactly
261
+ // what an unknown one is, and `COUNT(DISTINCT visitor_key)` skips it.
262
+ const viewerIp = stringFrom(record[VIEWER_IP_FIELD]);
263
+ if (viewerIp.kind === 'invalid') {
264
+ return dropped(VISITOR_KEY_COLUMN, VIEWER_IP_FIELD, viewerIp.detail);
265
+ }
266
+ if (viewerIp.kind === 'value') {
267
+ columns[VISITOR_KEY_COLUMN] = visitorKey(viewerIp.value, userAgent ?? '', dailySalt(saltSecret, day));
268
+ }
269
+ // Every required column is filled above or the record has already dropped:
270
+ // `event_time` and `day` unconditionally, and the required mapped columns by
271
+ // the guard in the loop. `map-record.test.ts` pins that correspondence
272
+ // against `PAGE_VIEWS_COLUMNS` itself, so a column the table makes required
273
+ // later fails a test here rather than a Firehose write in production.
274
+ return { mapped: true, row: columns };
275
+ }
@@ -0,0 +1,83 @@
1
+ /**
2
+ * `visitor_key`: how a page view is attributed to a visitor without the table
3
+ * ever holding a value that identifies one. This is step 3 of
4
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md):
5
+ * the viewer IP is replaced by a SHA-256 digest over the IP, the user agent
6
+ * and a secret daily salt, and the raw address is written to no column.
7
+ *
8
+ * ## The salt decision (settled 2026-07-26)
9
+ *
10
+ * ONE long-lived random secret lives in Secrets Manager, and the per-day salt
11
+ * is **derived** from it: `dailySalt(secret, day) = HMAC-SHA256(secret, day)`.
12
+ * The secret is created once and never rewritten; nothing rotates in Secrets
13
+ * Manager. Both alternatives were considered and rejected:
14
+ *
15
+ * - **A salt derived from the date alone** is computable by anyone holding the
16
+ * table, and IPv4 is a 2^32 space - so every row brute-forces back to its
17
+ * source address in seconds of GPU time. The digest would look like
18
+ * pseudonymisation while providing none of it.
19
+ * - **Managed rotation of the stored secret** would add a rotation Lambda, a
20
+ * schedule and a second execution role - more moving parts than the thing
21
+ * they protect - to buy exactly the daily turnover that deriving already
22
+ * gives for the price of one HMAC per record.
23
+ *
24
+ * The consequence to know before touching the stored secret: replacing it
25
+ * after rows exist makes `visitor_key` incomparable across that boundary, and
26
+ * no reprocessing can repair it because the old salt is gone. That is why the
27
+ * node that provisions the secret creates it when absent and never rewrites
28
+ * it.
29
+ *
30
+ * Daily turnover is deliberate and has a query consequence: a unique-visitor
31
+ * figure is a per-day distinct count, and a range figure is the sum of those
32
+ * daily counts - never a `COUNT(DISTINCT visitor_key)` spanning days, which
33
+ * two salts make meaningless. One day is also the bound on what anyone holding
34
+ * the table and one brute-forced day of salt could ever correlate.
35
+ *
36
+ * ## Purity
37
+ *
38
+ * Both functions take every input as an argument: no Secrets Manager read, no
39
+ * environment variable, no clock, no wall-clock date anywhere. The secret
40
+ * arrives from the transform's cold-start read and the day from the record's
41
+ * own `timestamp(ms)` - which is what lets a backfill re-derive a historical
42
+ * day's salt and produce a byte-identical row for a record either path could
43
+ * have carried.
44
+ */
45
+ /**
46
+ * The salt for one UTC day: `HMAC-SHA256(secret, day)` over the long-lived
47
+ * stored secret and the `day` the record already carries (`YYYY-MM-DD`).
48
+ *
49
+ * The secret is the key and the day the message, not the other way round: the
50
+ * day is public, and only a secret key gives the output a value an attacker
51
+ * cannot compute.
52
+ *
53
+ * Throws rather than deriving from an empty secret or an empty day. An empty
54
+ * secret yields a salt anyone can recompute, and an empty day yields one salt
55
+ * for all time - both produce a `visitor_key` that looks protected and is not,
56
+ * which is worse than a failed batch that says so.
57
+ */
58
+ export declare function dailySalt(secret: string, day: string): string;
59
+ /**
60
+ * The pseudonymous visitor identifier: a SHA-256 digest over the viewer IP,
61
+ * the user agent and that day's salt, as lowercase hex.
62
+ *
63
+ * Why this construction is enough for what it is asked to do. The key must be
64
+ * stable within a day (so a visitor counts once), unguessable from the table
65
+ * (which stores `user_agent` in the clear beside it), and irreversible. A hash
66
+ * alone would satisfy none of the last two: an IPv4 address is a 32-bit space,
67
+ * so an unsalted digest is a lookup table. The secret salt is what removes
68
+ * that, and the daily salt is what bounds the correlation window to a day. The
69
+ * salt is hashed last, so no length-extension property of Merkle-Damgård
70
+ * applies to a secret prefix.
71
+ *
72
+ * An empty `userAgent` is a legitimate input, not an error: a request that
73
+ * sent no `User-Agent` header still has a viewer IP, and its visitor is still
74
+ * countable. An absent user agent and an empty one are deliberately the same
75
+ * input, because they are the same fact about the request.
76
+ *
77
+ * Throws when the IP or the salt is empty. Without a salt the digest is
78
+ * brute-forceable; without an IP the key would be a digest of the user agent
79
+ * alone, which would collapse every anonymous request in a day onto one
80
+ * fabricated "returning visitor". The caller leaves the column empty instead -
81
+ * an unknown visitor is what a null says.
82
+ */
83
+ export declare function visitorKey(ip: string, userAgent: string, salt: string): string;
@@ -0,0 +1,120 @@
1
+ /**
2
+ * `visitor_key`: how a page view is attributed to a visitor without the table
3
+ * ever holding a value that identifies one. This is step 3 of
4
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md):
5
+ * the viewer IP is replaced by a SHA-256 digest over the IP, the user agent
6
+ * and a secret daily salt, and the raw address is written to no column.
7
+ *
8
+ * ## The salt decision (settled 2026-07-26)
9
+ *
10
+ * ONE long-lived random secret lives in Secrets Manager, and the per-day salt
11
+ * is **derived** from it: `dailySalt(secret, day) = HMAC-SHA256(secret, day)`.
12
+ * The secret is created once and never rewritten; nothing rotates in Secrets
13
+ * Manager. Both alternatives were considered and rejected:
14
+ *
15
+ * - **A salt derived from the date alone** is computable by anyone holding the
16
+ * table, and IPv4 is a 2^32 space - so every row brute-forces back to its
17
+ * source address in seconds of GPU time. The digest would look like
18
+ * pseudonymisation while providing none of it.
19
+ * - **Managed rotation of the stored secret** would add a rotation Lambda, a
20
+ * schedule and a second execution role - more moving parts than the thing
21
+ * they protect - to buy exactly the daily turnover that deriving already
22
+ * gives for the price of one HMAC per record.
23
+ *
24
+ * The consequence to know before touching the stored secret: replacing it
25
+ * after rows exist makes `visitor_key` incomparable across that boundary, and
26
+ * no reprocessing can repair it because the old salt is gone. That is why the
27
+ * node that provisions the secret creates it when absent and never rewrites
28
+ * it.
29
+ *
30
+ * Daily turnover is deliberate and has a query consequence: a unique-visitor
31
+ * figure is a per-day distinct count, and a range figure is the sum of those
32
+ * daily counts - never a `COUNT(DISTINCT visitor_key)` spanning days, which
33
+ * two salts make meaningless. One day is also the bound on what anyone holding
34
+ * the table and one brute-forced day of salt could ever correlate.
35
+ *
36
+ * ## Purity
37
+ *
38
+ * Both functions take every input as an argument: no Secrets Manager read, no
39
+ * environment variable, no clock, no wall-clock date anywhere. The secret
40
+ * arrives from the transform's cold-start read and the day from the record's
41
+ * own `timestamp(ms)` - which is what lets a backfill re-derive a historical
42
+ * day's salt and produce a byte-identical row for a record either path could
43
+ * have carried.
44
+ */
45
+ import { createHash, createHmac } from 'node:crypto';
46
+ /** The one digest algorithm both derivations use. */
47
+ const DIGEST_ALGORITHM = 'sha256';
48
+ /** Digests are lowercase hex: 64 characters, safe in a `string` column. */
49
+ const DIGEST_ENCODING = 'hex';
50
+ /**
51
+ * One input, length-prefixed, so a concatenation of several is unambiguous:
52
+ * `("1.2.3", "45")` and `("1.2.34", "5")` both concatenate to `1.2.345` and
53
+ * would collide, while they frame to `5:1.2.32:45` and `6:1.2.341:5`, which do
54
+ * not. A plain separator character could not promise as much: a user agent is
55
+ * attacker-supplied text and may contain whatever separator was chosen, which
56
+ * would let two different visitors collide onto one key by construction.
57
+ *
58
+ * The count is in UTF-8 bytes because that is the encoding `update` applies to
59
+ * a string, so the frame describes the bytes actually hashed.
60
+ */
61
+ function framed(value) {
62
+ return `${Buffer.byteLength(value)}:${value}`;
63
+ }
64
+ /**
65
+ * The salt for one UTC day: `HMAC-SHA256(secret, day)` over the long-lived
66
+ * stored secret and the `day` the record already carries (`YYYY-MM-DD`).
67
+ *
68
+ * The secret is the key and the day the message, not the other way round: the
69
+ * day is public, and only a secret key gives the output a value an attacker
70
+ * cannot compute.
71
+ *
72
+ * Throws rather than deriving from an empty secret or an empty day. An empty
73
+ * secret yields a salt anyone can recompute, and an empty day yields one salt
74
+ * for all time - both produce a `visitor_key` that looks protected and is not,
75
+ * which is worse than a failed batch that says so.
76
+ */
77
+ export function dailySalt(secret, day) {
78
+ if (secret.trim() === '') {
79
+ throw new Error('dailySalt was given no salt secret: an empty secret makes every visitor_key recomputable by anyone holding the table');
80
+ }
81
+ if (day.trim() === '') {
82
+ throw new Error('dailySalt was given no day: an empty day gives every record the same salt, so the daily turnover visitor_key depends on never happens');
83
+ }
84
+ return createHmac(DIGEST_ALGORITHM, secret).update(day).digest(DIGEST_ENCODING);
85
+ }
86
+ /**
87
+ * The pseudonymous visitor identifier: a SHA-256 digest over the viewer IP,
88
+ * the user agent and that day's salt, as lowercase hex.
89
+ *
90
+ * Why this construction is enough for what it is asked to do. The key must be
91
+ * stable within a day (so a visitor counts once), unguessable from the table
92
+ * (which stores `user_agent` in the clear beside it), and irreversible. A hash
93
+ * alone would satisfy none of the last two: an IPv4 address is a 32-bit space,
94
+ * so an unsalted digest is a lookup table. The secret salt is what removes
95
+ * that, and the daily salt is what bounds the correlation window to a day. The
96
+ * salt is hashed last, so no length-extension property of Merkle-Damgård
97
+ * applies to a secret prefix.
98
+ *
99
+ * An empty `userAgent` is a legitimate input, not an error: a request that
100
+ * sent no `User-Agent` header still has a viewer IP, and its visitor is still
101
+ * countable. An absent user agent and an empty one are deliberately the same
102
+ * input, because they are the same fact about the request.
103
+ *
104
+ * Throws when the IP or the salt is empty. Without a salt the digest is
105
+ * brute-forceable; without an IP the key would be a digest of the user agent
106
+ * alone, which would collapse every anonymous request in a day onto one
107
+ * fabricated "returning visitor". The caller leaves the column empty instead -
108
+ * an unknown visitor is what a null says.
109
+ */
110
+ export function visitorKey(ip, userAgent, salt) {
111
+ if (ip.trim() === '') {
112
+ throw new Error('visitorKey was given no viewer IP: a key over the user agent alone would count every anonymous request in a day as one returning visitor');
113
+ }
114
+ if (salt.trim() === '') {
115
+ throw new Error('visitorKey was given no salt: an unsalted digest of an IPv4 address brute-forces in seconds, so the key would identify the visitor it exists to hide');
116
+ }
117
+ return createHash(DIGEST_ALGORITHM)
118
+ .update(framed(ip) + framed(userAgent) + framed(salt))
119
+ .digest(DIGEST_ENCODING);
120
+ }