blogwright-analytics 0.3.3

package/dist/transform/handler.d.ts

@@ -0,0 +1,135 @@
+/**
+ * The Firehose transform Lambda's envelope: the batch handler that wraps
+ * `mapRecord`. This is step 5 of
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md) -
+ * "drops records the schema cannot accept, emitting them to the Firehose error
+ * prefix rather than failing the batch" - plus the cold-start secret read step
+ * 3 calls for.
+ *
+ * It owns four things and nothing else: the envelope types, the base64 and
+ * JSON boundary, the cold-start read of the long-lived salt secret, and the
+ * translation of one `MapRecordResult` into one Firehose response entry. Every
+ * decision about *what a row is* - which field fills which column, which
+ * values are unusable, how `visitor_key` is derived and under which day's salt
+ * - belongs to `map-record.ts` and is forwarded here, never re-decided.
+ *
+ * ## The two outcomes, and why they are different
+ *
+ * Firehose's per-record vocabulary distinguishes them and so does this module:
+ *
+ * - **`Ok`** carries the mapped row, re-encoded. Firehose writes it.
+ * - **`ProcessingFailed`** carries no data. Firehose writes the *original*
+ *   record to the error prefix and delivers the rest of the batch. That is the
+ *   visible data-quality signal a record the schema cannot accept should
+ *   produce.
+ *
+ * Firehose's third result, `Dropped`, is deliberately unused: it discards the
+ * record silently, which is the failure mode this whole pipeline is written
+ * against. A record that cannot be mapped must end up somewhere an operator
+ * can read it.
+ *
+ * `recordId` is echoed verbatim onto every entry, `Ok` and `ProcessingFailed`
+ * alike. Firehose discards a response whose ids do not match the request's -
+ * not the mismatched entry, the response - so an id dropped from a failed
+ * entry loses the whole batch.
+ *
+ * ## Why there is no try/catch around the mapping
+ *
+ * `dailySalt` and `visitorKey` throw on an empty secret, day, IP or salt
+ * (`visitor-key.ts`), because an unsalted digest that looks protected is worse
+ * than a failed batch: the table stores `user_agent` in the clear beside
+ * `visitor_key`, and an unsalted SHA-256 of an IPv4 address is a lookup table
+ * over a 2^32 space. A blanket catch around `mapRecord` would convert those
+ * throws into `ProcessingFailed` for every record - the whole batch to the
+ * error prefix, no error anywhere, an empty dashboard, and nothing to say why.
+ * So the only `try` in this module is the one around `JSON.parse`, which is
+ * this module's own boundary. A throw from the mapping propagates, the
+ * invocation fails, and Firehose retries the batch and raises its own error
+ * metric. Loudly is the point.
+ *
+ * The same reasoning governs the secret: a failed or empty read fails the
+ * batch. There is no unsalted or date-only fallback, because a fallback would
+ * write unprotected data that looks protected, and no reprocessing could
+ * repair rows already written under it.
+ *
+ * ## No AWS SDK, no network, no clock
+ *
+ * The envelope types below are repo-owned declarations, not `@types/aws-lambda`
+ * - the package takes no dependency for a shape this small and this stable.
+ * The one thing this module cannot do without is the secret, and it reaches it
+ * through {@link SaltSecretStore}, a structural slice of core's own client
+ * (the `packages/pds/src/secret.ts` precedent). That import is type-only, so it
+ * erases at compile: this module carries no client, no signer and no transport,
+ * and its tests stub the store with a plain object rather than mocking a module
+ * or reaching a cloud.
+ *
+ * Which also means this module is not a composition root. Binding a real
+ * `SecretsManagerClient` - built over the us-east-1 signer, since the function
+ * and its secret both live there - is the bundle entry's job, and it is the
+ * caller that hands the constructed handler to Lambda.
+ */
+import type { SecretsManagerClient } from 'blogwright-core';
+/**
+ * One record as Firehose sends it. The event carries more - `invocationId`,
+ * `deliveryStreamArn`, `region`, and per-record source metadata - and this
+ * handler reads none of it, so none of it is declared: a field spelled here
+ * and never read is a field a later reader would believe is load-bearing.
+ */
+interface FirehoseTransformRecord {
+    /** Firehose's id for this record. Echoed onto the response entry unchanged. */
+    readonly recordId: string;
+    /** The record's payload, base64-encoded. */
+    readonly data: string;
+}
+/** The transform invocation's payload: one buffer of records to translate. */
+export interface FirehoseTransformRequest {
+    readonly records: readonly FirehoseTransformRecord[];
+}
+/**
+ * What Firehose does with one record. `Dropped` exists in the API and is not
+ * offered here - see the module comment.
+ */
+type FirehoseRecordResult = 'Ok' | 'ProcessingFailed';
+/** One record's outcome, in Firehose's vocabulary. */
+interface FirehoseTransformedRecord {
+    /** The request entry's id, unchanged. */
+    readonly recordId: string;
+    readonly result: FirehoseRecordResult;
+    /** The transformed payload, base64-encoded. Present only for `Ok`. */
+    readonly data?: string | undefined;
+}
+/** The transform's response: one entry per request record, in request order. */
+export interface FirehoseTransformResponse {
+    readonly records: readonly FirehoseTransformedRecord[];
+}
+/**
+ * The read this handler needs, as a structural slice of core's client rather
+ * than a hand-rolled function type, so the seam is checked against the real
+ * `SecretsManagerClient` and a test satisfies it with a plain object.
+ */
+export type SaltSecretStore = Pick<SecretsManagerClient, 'getSecretValue'>;
+/**
+ * The environment variable naming the Secrets Manager secret behind
+ * `visitor_key`, set on the function by the `analytics-transform-function`
+ * node (task 50). The *name* travels in the environment; the value never does,
+ * so it cannot be read off the function's configuration.
+ */
+export declare const SALT_SECRET_NAME_ENV = "ANALYTICS_SALT_SECRET_NAME";
+/** The environment as the runtime hands it over. */
+type Environment = Readonly<Record<string, string | undefined>>;
+/**
+ * Build the Lambda entry point over a Secrets Manager read.
+ *
+ * The secret's name is resolved now, so a function deployed without
+ * {@link SALT_SECRET_NAME_ENV} fails at initialisation rather than once per
+ * batch. Its *value* is read on the first invocation and cached for the life
+ * of the execution environment: a 60-second Firehose buffer is roughly 43,000
+ * invocations a month, so reading per invocation would spend more than half
+ * the price of the secret itself again on `GetSecretValue` calls, for nothing.
+ *
+ * A failed read is not cached. Caching the failure would let one throttled
+ * call disable an execution environment for its whole life, failing every
+ * batch it ever sees; leaving it uncached means the next invocation retries.
+ */
+export declare function createTransformHandler(secrets: SaltSecretStore, env?: Environment): (request: FirehoseTransformRequest) => Promise<FirehoseTransformResponse>;
+export {};

package/dist/transform/handler.js

@@ -0,0 +1,177 @@
+/**
+ * The Firehose transform Lambda's envelope: the batch handler that wraps
+ * `mapRecord`. This is step 5 of
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md) -
+ * "drops records the schema cannot accept, emitting them to the Firehose error
+ * prefix rather than failing the batch" - plus the cold-start secret read step
+ * 3 calls for.
+ *
+ * It owns four things and nothing else: the envelope types, the base64 and
+ * JSON boundary, the cold-start read of the long-lived salt secret, and the
+ * translation of one `MapRecordResult` into one Firehose response entry. Every
+ * decision about *what a row is* - which field fills which column, which
+ * values are unusable, how `visitor_key` is derived and under which day's salt
+ * - belongs to `map-record.ts` and is forwarded here, never re-decided.
+ *
+ * ## The two outcomes, and why they are different
+ *
+ * Firehose's per-record vocabulary distinguishes them and so does this module:
+ *
+ * - **`Ok`** carries the mapped row, re-encoded. Firehose writes it.
+ * - **`ProcessingFailed`** carries no data. Firehose writes the *original*
+ *   record to the error prefix and delivers the rest of the batch. That is the
+ *   visible data-quality signal a record the schema cannot accept should
+ *   produce.
+ *
+ * Firehose's third result, `Dropped`, is deliberately unused: it discards the
+ * record silently, which is the failure mode this whole pipeline is written
+ * against. A record that cannot be mapped must end up somewhere an operator
+ * can read it.
+ *
+ * `recordId` is echoed verbatim onto every entry, `Ok` and `ProcessingFailed`
+ * alike. Firehose discards a response whose ids do not match the request's -
+ * not the mismatched entry, the response - so an id dropped from a failed
+ * entry loses the whole batch.
+ *
+ * ## Why there is no try/catch around the mapping
+ *
+ * `dailySalt` and `visitorKey` throw on an empty secret, day, IP or salt
+ * (`visitor-key.ts`), because an unsalted digest that looks protected is worse
+ * than a failed batch: the table stores `user_agent` in the clear beside
+ * `visitor_key`, and an unsalted SHA-256 of an IPv4 address is a lookup table
+ * over a 2^32 space. A blanket catch around `mapRecord` would convert those
+ * throws into `ProcessingFailed` for every record - the whole batch to the
+ * error prefix, no error anywhere, an empty dashboard, and nothing to say why.
+ * So the only `try` in this module is the one around `JSON.parse`, which is
+ * this module's own boundary. A throw from the mapping propagates, the
+ * invocation fails, and Firehose retries the batch and raises its own error
+ * metric. Loudly is the point.
+ *
+ * The same reasoning governs the secret: a failed or empty read fails the
+ * batch. There is no unsalted or date-only fallback, because a fallback would
+ * write unprotected data that looks protected, and no reprocessing could
+ * repair rows already written under it.
+ *
+ * ## No AWS SDK, no network, no clock
+ *
+ * The envelope types below are repo-owned declarations, not `@types/aws-lambda`
+ * - the package takes no dependency for a shape this small and this stable.
+ * The one thing this module cannot do without is the secret, and it reaches it
+ * through {@link SaltSecretStore}, a structural slice of core's own client
+ * (the `packages/pds/src/secret.ts` precedent). That import is type-only, so it
+ * erases at compile: this module carries no client, no signer and no transport,
+ * and its tests stub the store with a plain object rather than mocking a module
+ * or reaching a cloud.
+ *
+ * Which also means this module is not a composition root. Binding a real
+ * `SecretsManagerClient` - built over the us-east-1 signer, since the function
+ * and its secret both live there - is the bundle entry's job, and it is the
+ * caller that hands the constructed handler to Lambda.
+ */
+import { mapRecord } from './map-record.js';
+/** Firehose writes the record. */
+const OK = 'Ok';
+/** Firehose writes the *original* record to the error prefix and carries on. */
+const PROCESSING_FAILED = 'ProcessingFailed';
+/**
+ * The environment variable naming the Secrets Manager secret behind
+ * `visitor_key`, set on the function by the `analytics-transform-function`
+ * node (task 50). The *name* travels in the environment; the value never does,
+ * so it cannot be read off the function's configuration.
+ */
+export const SALT_SECRET_NAME_ENV = 'ANALYTICS_SALT_SECRET_NAME';
+/** The secret's name, or a failure naming the variable that should carry it. */
+function requireSaltSecretName(env) {
+    const name = env[SALT_SECRET_NAME_ENV]?.trim();
+    if (name === undefined || name === '') {
+        throw new Error(`the analytics transform needs the salt secret's name in ${SALT_SECRET_NAME_ENV}: without it every visitor_key would be unsalted, so the function refuses to start`);
+    }
+    return name;
+}
+/**
+ * The stored secret, or a failure. Never the empty string and never a
+ * substitute: `dailySalt` would throw on a blank secret anyway, and this says
+ * which secret is blank and how to fix it.
+ *
+ * The value is returned untrimmed. It is opaque key material, and trimming it
+ * would derive a different salt from the same secret - orphaning every
+ * `visitor_key` already written.
+ */
+async function loadSaltSecret(secrets, secretName) {
+    const secret = await secrets.getSecretValue(secretName);
+    if (secret === undefined || secret.trim() === '') {
+        throw new Error(`the analytics salt secret "${secretName}" holds no value: an unsalted visitor_key would identify the visitor it exists to hide, so this batch fails instead - run \`blogwright analytics bootstrap\` to create the secret`);
+    }
+    return secret;
+}
+/**
+ * The record's payload as an object, or `undefined` when it is not one.
+ *
+ * `Buffer.from(data, 'base64')` never throws - it ignores what it cannot
+ * decode - so a corrupt payload surfaces here as JSON that will not parse.
+ * `mapRecord` documents that it trusts its record to be an object and that
+ * parsing is this module's boundary, which is why a non-object payload (a
+ * bare number, `null`, an array) is rejected here rather than there.
+ *
+ * This `try` covers `JSON.parse` and nothing else. Widening it to cover the
+ * mapping would turn `visitor-key.ts`'s deliberate throws into silent drops.
+ */
+function decodePayload(data) {
+    let parsed;
+    try {
+        parsed = JSON.parse(Buffer.from(data, 'base64').toString('utf8'));
+    }
+    catch {
+        return undefined;
+    }
+    if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed))
+        return undefined;
+    return parsed;
+}
+/**
+ * One record in, one response entry out. The record is mapped with the whole
+ * batch's secret but under its *own* day's salt, which `mapRecord` derives
+ * from the record's own `timestamp(ms)` - a buffer that straddles midnight
+ * carries records of two days, and one salt chosen per invocation would be the
+ * wrong day's for every record on the far side of it.
+ */
+function transformRecord(record, saltSecret) {
+    const payload = decodePayload(record.data);
+    if (payload === undefined)
+        return { recordId: record.recordId, result: PROCESSING_FAILED };
+    const mapped = mapRecord(payload, saltSecret);
+    if (!mapped.mapped)
+        return { recordId: record.recordId, result: PROCESSING_FAILED };
+    return {
+        recordId: record.recordId,
+        result: OK,
+        data: Buffer.from(JSON.stringify(mapped.row), 'utf8').toString('base64'),
+    };
+}
+/**
+ * Build the Lambda entry point over a Secrets Manager read.
+ *
+ * The secret's name is resolved now, so a function deployed without
+ * {@link SALT_SECRET_NAME_ENV} fails at initialisation rather than once per
+ * batch. Its *value* is read on the first invocation and cached for the life
+ * of the execution environment: a 60-second Firehose buffer is roughly 43,000
+ * invocations a month, so reading per invocation would spend more than half
+ * the price of the secret itself again on `GetSecretValue` calls, for nothing.
+ *
+ * A failed read is not cached. Caching the failure would let one throttled
+ * call disable an execution environment for its whole life, failing every
+ * batch it ever sees; leaving it uncached means the next invocation retries.
+ */
+export function createTransformHandler(secrets, env = process.env) {
+    const secretName = requireSaltSecretName(env);
+    let cached;
+    async function readSaltSecret() {
+        if (cached === undefined)
+            cached = await loadSaltSecret(secrets, secretName);
+        return cached;
+    }
+    return async function transformFirehoseRecords(request) {
+        const saltSecret = await readSaltSecret();
+        return { records: request.records.map((record) => transformRecord(record, saltSecret)) };
+    };
+}

package/dist/transform/map-record.d.ts

@@ -0,0 +1,110 @@
+/**
+ * `mapRecord`: one CloudFront standard-logging (v2) record in, one `page_views`
+ * row out - or a droppable result naming the column that could not be filled
+ * and the CloudFront field behind it. This is all five steps of
+ * [§Record transformation](../../../../.specs/changes/merged/2026-07-26-analytics_plugin.md):
+ * rename each selected field to its column, derive `event_time` and the `day`
+ * partition from `timestamp(ms)`, replace the viewer IP with `visitor_key`,
+ * set `is_bot` from the user agent, and drop what the schema cannot accept.
+ *
+ * The viewer IP is an *input* here and never a value. It reaches `visitorKey`
+ * and nothing else: it has no `FIELD_TO_COLUMN` entry to be renamed through,
+ * and `map-record.test.ts` searches every value of every produced row for it.
+ * That search is the standing check that no later field addition puts a raw
+ * address in the warehouse.
+ *
+ * The salt secret is a parameter for the same reason the day is read off the
+ * record: this function has no clock and reads no secret. The caller supplies
+ * the long-lived stored secret - the transform reads it once at cold start, a
+ * backfill reads the same one - and the day's salt is derived here rather than
+ * by the caller, because the day comes from the record's own timestamp. A
+ * batch can straddle midnight, so one salt chosen for a whole batch would be
+ * the wrong day's for the records on the other side of it.
+ *
+ * Why the drop path exists at all: Firehose matches incoming JSON keys to
+ * Iceberg column names **exactly**, and a record it cannot match goes to the
+ * S3 error bucket with no error anywhere an operator will see it - the symptom
+ * is an empty dashboard. So this module never emits a key that is not a column
+ * and never emits a value whose JavaScript type is not the one the column
+ * stores. Where it cannot honour that, it returns the droppable result and the
+ * envelope (task 42) reports `ProcessingFailed` for that one record, which
+ * routes it to the error prefix without failing the batch.
+ *
+ * Every column and field name comes from `schema.ts`: the row is built by
+ * iterating `FIELD_TO_COLUMN`, the required set and the numeric set are
+ * derived from `PAGE_VIEWS_COLUMNS`, and the only names spelled here are the
+ * four derived columns and the one column the two of them read back
+ * (`user_agent`), each checked against `PageViewColumnName` so a typo is a
+ * compile error rather than a column Firehose silently drops.
+ *
+ * Three input decisions the spec left open, settled here:
+ *
+ * - **`-` and the empty string mean absent.** CloudFront writes `-` for a
+ *   field the request had nothing to say for (no referrer, no query string).
+ *   Writing that through would fill `referrer` with a wall of `-`; treating it
+ *   as absent leaves the column null, which is what it means. For a *required*
+ *   column it is a drop, not a null.
+ * - **A number where a string column is expected is rendered, not rejected.**
+ *   `asn` is a string column that a JSON encoder may well emit unquoted;
+ *   `String(64512)` is the same value in the type the column stores. Anything
+ *   that is not a string or a number - an object, an array, a boolean - is a
+ *   drop, because there is no rendering of it that is not a guess.
+ * - **A numeric column that does not parse is a drop, never a coerced value.**
+ *   `Number('abc')` is `NaN` and `Number('')` is `0`; writing either would put
+ *   a wrong number in the table, which is worse than losing the record.
+ *
+ * Nullability governs the *absent* case only. A value that is present but
+ * unusable drops the record whether or not the column is nullable: leaving a
+ * nullable column empty asserts the request had nothing to say for it, which
+ * is a different - and false - fact about that request.
+ *
+ * Pure: no clock (`event_time` comes from the record's own `timestamp(ms)`, in
+ * UTC), no secret read, no vendor SDK, no `fetch`. The one `node:` builtin
+ * this directory uses is `node:crypto`, reached through `visitor-key.ts` and
+ * only for the digests - the same import `packages/core/src/aws/s3.ts` makes.
+ * The record arrives already parsed and is trusted to be an object - JSON
+ * parsing and its failures are the handler's boundary, not this function's.
+ */
+import { type PageView, type PageViewColumnName } from '../schema.js';
+/**
+ * One CloudFront access-log record as the delivery hands it over: the selected
+ * field names of `CLOUDFRONT_RECORD_FIELDS` against values of whatever type
+ * the JSON payload carried.
+ */
+export type CloudFrontRecord = Readonly<Record<string, unknown>>;
+/** A record that maps cleanly: the row is complete and safe to hand Firehose. */
+interface MappedRecord {
+    readonly mapped: true;
+    readonly row: PageView;
+}
+/**
+ * A record the schema cannot accept. It carries no row at all - a partially
+ * populated one is exactly the silent corruption the drop path exists to
+ * prevent.
+ */
+interface DroppedRecord {
+    readonly mapped: false;
+    /** The `page_views` column that could not be filled. */
+    readonly column: PageViewColumnName;
+    /** The CloudFront field that column reads. */
+    readonly field: string;
+    /** Names both, plus what was wrong with the value. */
+    readonly reason: string;
+}
+/** The outcome of mapping one record: a complete row, or a named drop. */
+export type MapRecordResult = MappedRecord | DroppedRecord;
+/**
+ * Turns one CloudFront access-log record into a `page_views` row, or reports
+ * why it cannot. Never returns a partially populated row: the first column it
+ * cannot fill ends the mapping.
+ *
+ * `saltSecret` is the long-lived stored secret behind `visitor_key`, not the
+ * day's salt - that is derived here from the record's own day. It is required
+ * rather than optional on purpose: a row mapped without it would carry no
+ * visitor at all, or worse an unsalted digest, and either would look like a
+ * normal row in the table. `dailySalt` throws on an empty secret for the same
+ * reason, so a caller whose secret read failed fails its batch instead of
+ * quietly writing unprotected data.
+ */
+export declare function mapRecord(record: CloudFrontRecord, saltSecret: string): MapRecordResult;
+export {};