blogwright-analytics 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -0
- package/dist/adapters/duckdb-ingest.d.ts +76 -0
- package/dist/adapters/duckdb-ingest.js +173 -0
- package/dist/adapters/duckdb-query.d.ts +56 -0
- package/dist/adapters/duckdb-query.js +80 -0
- package/dist/adapters/duckdb-session.d.ts +168 -0
- package/dist/adapters/duckdb-session.js +330 -0
- package/dist/app/_app/immutable/assets/0.BTQrrh5B.css +1 -0
- package/dist/app/_app/immutable/assets/2.CZSK3rT8.css +1 -0
- package/dist/app/_app/immutable/assets/BrushContext.D7c8UPey.css +1 -0
- package/dist/app/_app/immutable/assets/ChartAnnotations.CPxIG7Mw.css +1 -0
- package/dist/app/_app/immutable/assets/Circle.C5MKzgk2.css +1 -0
- package/dist/app/_app/immutable/assets/DefaultTooltip.C5-uctZ7.css +1 -0
- package/dist/app/_app/immutable/assets/Group.DV48xipa.css +1 -0
- package/dist/app/_app/immutable/assets/Labels.BxZ4NUVz.css +1 -0
- package/dist/app/_app/immutable/assets/Legend.CxnrE4Ye.css +1 -0
- package/dist/app/_app/immutable/assets/Line.fkmsECm9.css +1 -0
- package/dist/app/_app/immutable/assets/Path.CvpwNZ6g.css +1 -0
- package/dist/app/_app/immutable/assets/Rect.CtRaGMmQ.css +1 -0
- package/dist/app/_app/immutable/assets/Text.j9l35qB0.css +1 -0
- package/dist/app/_app/immutable/assets/TransformContext.Bs_HkpAk.css +1 -0
- package/dist/app/_app/immutable/assets/Voronoi.ce7atosu.css +1 -0
- package/dist/app/_app/immutable/chunks/-aNGNaBT.js +1 -0
- package/dist/app/_app/immutable/chunks/6djn-yLs.js +1 -0
- package/dist/app/_app/immutable/chunks/B1amyutE.js +1 -0
- package/dist/app/_app/immutable/chunks/B3vZDoek.js +1 -0
- package/dist/app/_app/immutable/chunks/B5KRA4hC.js +1 -0
- package/dist/app/_app/immutable/chunks/BClnVG6H.js +1 -0
- package/dist/app/_app/immutable/chunks/BID1NNRh.js +1 -0
- package/dist/app/_app/immutable/chunks/BR2LaRms.js +1 -0
- package/dist/app/_app/immutable/chunks/Bd1gDe3Y.js +1 -0
- package/dist/app/_app/immutable/chunks/Bjy-W4x2.js +81 -0
- package/dist/app/_app/immutable/chunks/Bl052uUt.js +1 -0
- package/dist/app/_app/immutable/chunks/Bye3lL0c.js +1 -0
- package/dist/app/_app/immutable/chunks/C58PZtCD.js +4 -0
- package/dist/app/_app/immutable/chunks/CAzydqEO.js +1 -0
- package/dist/app/_app/immutable/chunks/CCch3uox.js +1 -0
- package/dist/app/_app/immutable/chunks/CIlSMUH9.js +1 -0
- package/dist/app/_app/immutable/chunks/CO1vUXfR.js +1 -0
- package/dist/app/_app/immutable/chunks/CPbD8C65.js +5 -0
- package/dist/app/_app/immutable/chunks/CRTcXoMo.js +1 -0
- package/dist/app/_app/immutable/chunks/CjjyIQAO.js +1 -0
- package/dist/app/_app/immutable/chunks/CuXAxjvF.js +1 -0
- package/dist/app/_app/immutable/chunks/CvyVA_jC.js +1 -0
- package/dist/app/_app/immutable/chunks/CxGCFVdy.js +1 -0
- package/dist/app/_app/immutable/chunks/D0Ty6LN0.js +1 -0
- package/dist/app/_app/immutable/chunks/D2AaQUUW.js +1 -0
- package/dist/app/_app/immutable/chunks/D2BnX0Uk.js +3 -0
- package/dist/app/_app/immutable/chunks/DJc8C0NK.js +1 -0
- package/dist/app/_app/immutable/chunks/DKMlMI4a.js +1 -0
- package/dist/app/_app/immutable/chunks/DVXZkpbf.js +1 -0
- package/dist/app/_app/immutable/chunks/DVt8ukQ_.js +1 -0
- package/dist/app/_app/immutable/chunks/DZPlYdq_.js +1 -0
- package/dist/app/_app/immutable/chunks/Db0q5_zr.js +1 -0
- package/dist/app/_app/immutable/chunks/Dfvzj6n2.js +1 -0
- package/dist/app/_app/immutable/chunks/Dh958be7.js +1 -0
- package/dist/app/_app/immutable/chunks/DjKLLdnY.js +15 -0
- package/dist/app/_app/immutable/chunks/Doz7YX1W.js +1 -0
- package/dist/app/_app/immutable/chunks/DthYhn6Y.js +2 -0
- package/dist/app/_app/immutable/chunks/DtuTIrAM.js +1 -0
- package/dist/app/_app/immutable/chunks/HclGiUj8.js +1 -0
- package/dist/app/_app/immutable/chunks/Hx0TNsV3.js +1 -0
- package/dist/app/_app/immutable/chunks/RobXhXPM.js +1 -0
- package/dist/app/_app/immutable/chunks/V9ZjaxiY.js +1 -0
- package/dist/app/_app/immutable/chunks/Y5urAfNy.js +1 -0
- package/dist/app/_app/immutable/chunks/caXkbKD3.js +1 -0
- package/dist/app/_app/immutable/chunks/devYm2ud.js +1 -0
- package/dist/app/_app/immutable/chunks/mtZWP0zR.js +1 -0
- package/dist/app/_app/immutable/chunks/vDgBJUjM.js +1 -0
- package/dist/app/_app/immutable/chunks/xIq_fFFM.js +1 -0
- package/dist/app/_app/immutable/chunks/xihTtKlq.js +1 -0
- package/dist/app/_app/immutable/chunks/z05MoCFz.js +1 -0
- package/dist/app/_app/immutable/entry/app.CLAerUAN.js +2 -0
- package/dist/app/_app/immutable/entry/start.D3MqnNci.js +1 -0
- package/dist/app/_app/immutable/nodes/0.UTMEigHJ.js +1 -0
- package/dist/app/_app/immutable/nodes/1.Cn4f11bT.js +1 -0
- package/dist/app/_app/immutable/nodes/2.B39cIcr2.js +6 -0
- package/dist/app/_app/version.json +1 -0
- package/dist/app/index.html +82 -0
- package/dist/aws/clients.d.ts +70 -0
- package/dist/aws/clients.js +52 -0
- package/dist/aws/errors.d.ts +41 -0
- package/dist/aws/errors.js +70 -0
- package/dist/aws/firehose.d.ts +228 -0
- package/dist/aws/firehose.js +347 -0
- package/dist/aws/glue.d.ts +103 -0
- package/dist/aws/glue.js +225 -0
- package/dist/aws/lambda.d.ts +132 -0
- package/dist/aws/lambda.js +339 -0
- package/dist/aws/s3tables.d.ts +120 -0
- package/dist/aws/s3tables.js +281 -0
- package/dist/backfill.d.ts +100 -0
- package/dist/backfill.js +294 -0
- package/dist/commands.d.ts +124 -0
- package/dist/commands.js +336 -0
- package/dist/config.d.ts +162 -0
- package/dist/config.js +317 -0
- package/dist/fixture-ingest.d.ts +49 -0
- package/dist/fixture-ingest.js +43 -0
- package/dist/fixture-query.d.ts +39 -0
- package/dist/fixture-query.js +70 -0
- package/dist/index.d.ts +35 -0
- package/dist/index.js +35 -0
- package/dist/nodes.d.ts +404 -0
- package/dist/nodes.js +2708 -0
- package/dist/paths.d.ts +45 -0
- package/dist/paths.js +47 -0
- package/dist/plugin.d.ts +102 -0
- package/dist/plugin.js +248 -0
- package/dist/ports.d.ts +113 -0
- package/dist/ports.js +35 -0
- package/dist/queries.d.ts +301 -0
- package/dist/queries.js +414 -0
- package/dist/schema.d.ts +240 -0
- package/dist/schema.js +154 -0
- package/dist/server.d.ts +150 -0
- package/dist/server.js +499 -0
- package/dist/transform/bots.d.ts +47 -0
- package/dist/transform/bots.js +73 -0
- package/dist/transform/handler.d.ts +135 -0
- package/dist/transform/handler.js +177 -0
- package/dist/transform/map-record.d.ts +110 -0
- package/dist/transform/map-record.js +275 -0
- package/dist/transform/visitor-key.d.ts +83 -0
- package/dist/transform/visitor-key.js +120 -0
- package/dist/transform-bundle/index.mjs +21456 -0
- package/dist/transform-bundle/transform-manifest.json +4 -0
- package/dist/transform-hash.d.ts +135 -0
- package/dist/transform-hash.js +186 -0
- package/dist/write-transform-manifest.mjs +365 -0
- package/package.json +59 -0
package/dist/nodes.js
ADDED
|
@@ -0,0 +1,2708 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The analytics plugin's resource graph. It owns the AWS resources the
|
|
3
|
+
* CloudFront-logs-to-Iceberg pipeline is built from and nothing else: the
|
|
4
|
+
* site's own bucket, distribution and log group stay in the CLI's graph
|
|
5
|
+
* (`packages/cli/src/nodes.ts`) and are never touched from here. This module
|
|
6
|
+
* carries all twelve of them, in four chains. The table chain - the S3 Tables bucket,
|
|
7
|
+
* the namespace inside it, the `page_views` table, and the Glue federation
|
|
8
|
+
* Firehose reads that table through - runs `analytics-table-bucket` ->
|
|
9
|
+
* `analytics-namespace` -> `analytics-table` ->
|
|
10
|
+
* `analytics-catalog-integration`. The transform chain - the long-lived
|
|
11
|
+
* `visitor_key` salt, the Lambda execution role whose policy names that
|
|
12
|
+
* secret's ARN, and the record-transform function itself - runs
|
|
13
|
+
* `analytics-salt-secret` -> `analytics-transform-role` ->
|
|
14
|
+
* `analytics-transform-function`. The delivery chain - the bucket every record
|
|
15
|
+
* Firehose cannot deliver lands in, the role it assumes, and the stream itself
|
|
16
|
+
* - runs `analytics-error-bucket` -> `analytics-firehose-role` ->
|
|
17
|
+
* `analytics-firehose-stream`, and joins the other two chains through the
|
|
18
|
+
* role's four grants and the stream's destination. The vended-delivery chain -
|
|
19
|
+
* the CloudWatch delivery destination pointing at that stream and the delivery
|
|
20
|
+
* joining it to the site's log source - runs `analytics-log-destination` ->
|
|
21
|
+
* `analytics-log-delivery` and hangs off the stream. All four are wired through
|
|
22
|
+
* `dependsOn`, and a node depends on every node whose recorded ARN it
|
|
23
|
+
* interpolates. {@link buildAnalyticsNodes} at the foot of this module returns
|
|
24
|
+
* the assembled set, and `plugin.ts` hands it to the SPI's `Plugin.nodes`;
|
|
25
|
+
* assembling an array is all it does - nothing here reconciles anything.
|
|
26
|
+
*
|
|
27
|
+
* **The delivery source the last chain hangs off is the site's, and this module
|
|
28
|
+
* only ever reads it.** AWS permits exactly one delivery source per
|
|
29
|
+
* distribution, so the site's CloudWatch delivery and this plugin's Firehose
|
|
30
|
+
* delivery necessarily share one, and `packages/cli/src/nodes.ts`'s
|
|
31
|
+
* `logDeliveryNode` owns it. Nothing here calls `putDeliverySource` or
|
|
32
|
+
* `deleteDeliverySource`; the source's name is read off `ctx.names` and the
|
|
33
|
+
* evidence that the site has been bootstrapped off `ctx.siteState`, the SPI's
|
|
34
|
+
* read-only view of the site's state.
|
|
35
|
+
*
|
|
36
|
+
* **Everything in this graph is created in `us-east-1`, whatever
|
|
37
|
+
* `config.region` says.** CloudFront standard logging accepts a Firehose
|
|
38
|
+
* delivery stream only in that region, so the whole pipeline - and therefore
|
|
39
|
+
* the table the stream writes into - has to live there too. The pin is
|
|
40
|
+
* enforced in exactly one place, `aws/clients.ts`, which builds every client
|
|
41
|
+
* over the host's `signingUsEast1` signer; no node here picks a region for a
|
|
42
|
+
* request. {@link ANALYTICS_REGION} below is the same region as *text*, needed
|
|
43
|
+
* only because an ARN spells its region out and because every node `title`
|
|
44
|
+
* states the pin, so the bootstrap output an operator reads carries it. Ten of
|
|
45
|
+
* the twelve titles state it as the region they are created in; the two IAM
|
|
46
|
+
* role nodes state it as the pipeline they serve, because IAM is global and
|
|
47
|
+
* "created in us-east-1" is not a property a role has (§Region pinning says so
|
|
48
|
+
* in as many words) - a title claiming otherwise would be the pin stated
|
|
49
|
+
* falsely rather than stated.
|
|
50
|
+
*
|
|
51
|
+
* The nodes are core's {@link ResourceNode} over {@link PluginContext}, so the
|
|
52
|
+
* CLI's own engine (`topoSort`/`applyGraph`/`destroyGraph`,
|
|
53
|
+
* `packages/cli/src/graph.ts`) reconciles them unchanged - this package
|
|
54
|
+
* contributes nodes, never a second engine.
|
|
55
|
+
*
|
|
56
|
+
* **Creating the namespace and the table through the S3 Tables control-plane
|
|
57
|
+
* API is a supported Firehose source.** Verified 2026-07-26 against AWS's S3
|
|
58
|
+
* Tables + Firehose walkthrough, which creates both with `aws s3tables
|
|
59
|
+
* create-namespace` and `aws s3tables create-table` and then points the
|
|
60
|
+
* delivery stream at them. This is worth recording because Firehose's
|
|
61
|
+
* considerations page carries a limitation that reads as if it forbids exactly
|
|
62
|
+
* that - "only tables created through Iceberg's GlueCatalog API". That
|
|
63
|
+
* limitation applies to plain Iceberg-on-S3 tables registered in Glue, not to
|
|
64
|
+
* S3 Tables reached through the `s3tablescatalog` federation
|
|
65
|
+
* (`analytics-catalog-integration`), which is how this pipeline reaches them.
|
|
66
|
+
* Without this note the next reader re-litigates it.
|
|
67
|
+
*/
|
|
68
|
+
import { join } from 'node:path';
|
|
69
|
+
import { AwsError, pollUntil, REPRODUCIBLE_ZIP_MTIME, } from 'blogwright-core';
|
|
70
|
+
import { zipSync } from 'fflate';
|
|
71
|
+
import { createAnalyticsClients } from './aws/clients.js';
|
|
72
|
+
import { STREAM_APPEND_ONLY, } from './aws/firehose.js';
|
|
73
|
+
import { resolveAnalyticsConfig } from './config.js';
|
|
74
|
+
import { ANALYTICS_PACKAGE_DIR } from './paths.js';
|
|
75
|
+
import { CLOUDFRONT_RECORD_FIELDS, PAGE_VIEWS_COLUMNS, PAGE_VIEWS_PARTITION_COLUMN, } from './schema.js';
|
|
76
|
+
import { SALT_SECRET_NAME_ENV } from './transform/handler.js';
|
|
77
|
+
import { TRANSFORM_BUNDLE_DIR, TRANSFORM_BUNDLE_FILE, TRANSFORM_LAMBDA_HANDLER, TRANSFORM_MANIFEST_FILE, transformZipKey, } from './transform-hash.js';
|
|
78
|
+
/** The `analytics-table-bucket` node id, shared by its `id`, its state key and the edge into it. */
|
|
79
|
+
const TABLE_BUCKET_NODE = 'analytics-table-bucket';
|
|
80
|
+
/** The `analytics-namespace` node id. */
|
|
81
|
+
const NAMESPACE_NODE = 'analytics-namespace';
|
|
82
|
+
/** The `analytics-table` node id. */
|
|
83
|
+
const TABLE_NODE = 'analytics-table';
|
|
84
|
+
/** The `analytics-catalog-integration` node id. */
|
|
85
|
+
const CATALOG_NODE = 'analytics-catalog-integration';
|
|
86
|
+
/** The `analytics-salt-secret` node id, shared by its `id`, its state key and the edge into it. */
|
|
87
|
+
const SALT_SECRET_NODE = 'analytics-salt-secret';
|
|
88
|
+
/** The `analytics-transform-role` node id. */
|
|
89
|
+
const TRANSFORM_ROLE_NODE = 'analytics-transform-role';
|
|
90
|
+
/** The `analytics-transform-function` node id. */
|
|
91
|
+
const TRANSFORM_FUNCTION_NODE = 'analytics-transform-function';
|
|
92
|
+
/** The `analytics-error-bucket` node id, shared by its `id`, its state key and the edge into it. */
|
|
93
|
+
const ERROR_BUCKET_NODE = 'analytics-error-bucket';
|
|
94
|
+
/** The `analytics-firehose-role` node id. */
|
|
95
|
+
const FIREHOSE_ROLE_NODE = 'analytics-firehose-role';
|
|
96
|
+
/**
|
|
97
|
+
* The `analytics-firehose-stream` node id. Exported, alone among the twelve,
|
|
98
|
+
* because `analytics status` reads this node's recorded outputs back out of
|
|
99
|
+
* the scoped state its `read` hydrated - the stream's delivery health - and a
|
|
100
|
+
* second copy of the string in `commands.ts` would be a state key with two
|
|
101
|
+
* homes.
|
|
102
|
+
*/
|
|
103
|
+
export const FIREHOSE_STREAM_NODE = 'analytics-firehose-stream';
|
|
104
|
+
/** The `analytics-log-destination` node id. */
|
|
105
|
+
const LOG_DESTINATION_NODE = 'analytics-log-destination';
|
|
106
|
+
/**
|
|
107
|
+
* The `analytics-log-delivery` node id. Exported for `backfill.ts`, which
|
|
108
|
+
* reads {@link CREATED_DAY_KEY} out of this node's recorded outputs: the
|
|
109
|
+
* backfill's idempotency bound and the node that writes it must name the same
|
|
110
|
+
* state entry, and a second spelling of the id is the one way that could stop
|
|
111
|
+
* being true without anything noticing.
|
|
112
|
+
*/
|
|
113
|
+
export const LOG_DELIVERY_NODE = 'analytics-log-delivery';
|
|
114
|
+
/**
|
|
115
|
+
* The Glue catalog the S3 Tables integration registers itself under. The one
|
|
116
|
+
* name in this module that carries neither the environment nor the site, and
|
|
117
|
+
* deliberately: AWS's integration procedure creates exactly one catalog called
|
|
118
|
+
* `s3tablescatalog` per account and Region, and every S3 Tables table in the
|
|
119
|
+
* account is reached through it. A per-environment name derived here would not
|
|
120
|
+
* buy a second, private integration - it would create a catalog the S3 Tables
|
|
121
|
+
* integration itself never populates.
|
|
122
|
+
*/
|
|
123
|
+
const CATALOG_NAME = 's3tablescatalog';
|
|
124
|
+
/**
|
|
125
|
+
* The bucket segment of {@link federationSource}: the wildcard naming every
|
|
126
|
+
* table bucket in the account and Region rather than this environment's one.
|
|
127
|
+
* Named rather than inlined so it does not read as a stray character in an ARN.
|
|
128
|
+
*/
|
|
129
|
+
const ALL_TABLE_BUCKETS = '*';
|
|
130
|
+
/**
|
|
131
|
+
* The region every resource in this graph is created in - see the module
|
|
132
|
+
* comment. This constant is *not* what enforces the pin: `aws/clients.ts`
|
|
133
|
+
* does that, by building every client over `ctx.clients.signingUsEast1`. It
|
|
134
|
+
* exists because an ARN carries its region as text and `SigningClient` does
|
|
135
|
+
* not expose the region it signs in, so a node that has to name an ARN has to
|
|
136
|
+
* name the region too - and, since task 54, because every node's `title` states
|
|
137
|
+
* the pin out loud, which is how `applyGraph`'s `create <title>` lines carry
|
|
138
|
+
* the divergence from `config.region` into the bootstrap output an operator
|
|
139
|
+
* reads. Two different tests in `nodes.test.ts` pin the two
|
|
140
|
+
* halves and they are not interchangeable. The credential-scope assertion
|
|
141
|
+
* ("signs every call against us-east-1 while config.region says otherwise")
|
|
142
|
+
* reads the region back out of the SigV4 `Authorization` header, so it catches
|
|
143
|
+
* the *clients* drifting off the pin - but it is blind to this constant, and
|
|
144
|
+
* stays green if only this string changes, because a signed region is not an
|
|
145
|
+
* ARN. What catches that is every assertion that spells an S3 Tables bucket ARN
|
|
146
|
+
* out - the recorded request URLs, the recorded outputs, and the account-wide
|
|
147
|
+
* wildcard the catalog federation is registered over: setting this to
|
|
148
|
+
* `eu-west-1` reddens nineteen tests while the credential-scope test passes.
|
|
149
|
+
*/
|
|
150
|
+
const ANALYTICS_REGION = 'us-east-1';
|
|
151
|
+
/**
|
|
152
|
+
* Iceberg numbers schema fields from 1, so the nth column of
|
|
153
|
+
* `PAGE_VIEWS_COLUMNS` takes id n. See {@link pageViewsFields} for why the
|
|
154
|
+
* caller assigns ids at all.
|
|
155
|
+
*/
|
|
156
|
+
const FIRST_FIELD_ID = 1;
|
|
157
|
+
/**
|
|
158
|
+
* The transform the `page_views` partition applies to
|
|
159
|
+
* `PAGE_VIEWS_PARTITION_COLUMN`. `identity`, not Iceberg's `day` transform:
|
|
160
|
+
* `day` is already a `date` column that the transform Lambda computes from
|
|
161
|
+
* CloudFront's `timestamp(ms)` (`schema.ts`'s `DERIVED_COLUMNS`), so the
|
|
162
|
+
* partition value *is* the column value. Iceberg's `day` transform truncates a
|
|
163
|
+
* timestamp to a date, and would be right only if the table partitioned on
|
|
164
|
+
* `event_time` instead. This is a fact about the S3 Tables API's partition
|
|
165
|
+
* vocabulary rather than about the table's columns, which is why it lives here
|
|
166
|
+
* and not in `schema.ts`.
|
|
167
|
+
*/
|
|
168
|
+
const PARTITION_TRANSFORM = 'identity';
|
|
169
|
+
/**
|
|
170
|
+
* Record outputs under `nodeId` and hand back the live object, so a node
|
|
171
|
+
* records each identifier *as* its resource is created rather than after the
|
|
172
|
+
* chain completes - the discipline `packages/cli/src/nodes.ts:713-719` states
|
|
173
|
+
* and `packages/cli/src/nodes.ts:20-22` provides for the site's own nodes. A
|
|
174
|
+
* crash between two calls must still leave what was already created recorded
|
|
175
|
+
* in state for `destroy` to clean up.
|
|
176
|
+
*
|
|
177
|
+
* Unlike the CLI's helper this writes through `ctx.record`, which the SPI names
|
|
178
|
+
* as the only way a plugin's nodes may record outputs (core's `plugin.ts`),
|
|
179
|
+
* rather than assigning into `ctx.state.resources` behind its back. An existing
|
|
180
|
+
* entry is re-recorded rather than replaced, so a second call in the same run
|
|
181
|
+
* adds to what the first one wrote instead of dropping it.
|
|
182
|
+
*
|
|
183
|
+
* The handle is read back out of `state` rather than returned directly. The
|
|
184
|
+
* host that fills `record` today stores the object it is handed
|
|
185
|
+
* (`packages/cli/src/plugin-commands.ts`'s `toPluginContext`), which makes the
|
|
186
|
+
* two the same object - but one that stored a copy would leave every later
|
|
187
|
+
* write landing on an orphan and silently record nothing. The read-back costs
|
|
188
|
+
* a lookup and removes that dependency; `noUncheckedIndexedAccess` is why the
|
|
189
|
+
* `??` is spelled twice rather than once.
|
|
190
|
+
*
|
|
191
|
+
* Called only where a value is about to be written. A `read` that finds
|
|
192
|
+
* nothing must not call it: doing so would leave an empty entry in
|
|
193
|
+
* `state/<env>.analytics.json` for a resource that does not exist.
|
|
194
|
+
*/
|
|
195
|
+
function output(ctx, nodeId) {
|
|
196
|
+
const outputs = ctx.state.resources[nodeId] ?? {};
|
|
197
|
+
ctx.record(nodeId, outputs);
|
|
198
|
+
return ctx.state.resources[nodeId] ?? outputs;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* The plugin's own S3 Tables client. Built through
|
|
202
|
+
* {@link createAnalyticsClients}, never lifted off `ctx.clients`, which
|
|
203
|
+
* enumerates only core's own services and signs them in `config.region`.
|
|
204
|
+
*/
|
|
205
|
+
function s3tables(ctx) {
|
|
206
|
+
return createAnalyticsClients(ctx).s3tables;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* The plugin's own Glue client, built the same way {@link s3tables} is and for
|
|
210
|
+
* the same reason: core's bundle enumerates no `glue` service at all, and the
|
|
211
|
+
* one it does expose signs in `config.region`.
|
|
212
|
+
*/
|
|
213
|
+
function glue(ctx) {
|
|
214
|
+
return createAnalyticsClients(ctx).glue;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* The table bucket's ARN, in the fixed
|
|
218
|
+
* `arn:aws:s3tables:<region>:<accountId>:bucket/<name>` form. Derived rather
|
|
219
|
+
* than read back from the API because `getTableBucket` is ARN-keyed with no
|
|
220
|
+
* name-based lookup and `createTableBucket` deliberately returns no ARN
|
|
221
|
+
* (`aws/s3tables.ts`): a caller has to compute this before it can make either
|
|
222
|
+
* call, so there is nothing to hydrate it from.
|
|
223
|
+
*
|
|
224
|
+
* The name comes from {@link resolveAnalyticsConfig}, the only route to it -
|
|
225
|
+
* `ctx.pluginConfig.tableBucket` does not compile, because the default carries
|
|
226
|
+
* the environment and a bucket name derived without one makes staging and
|
|
227
|
+
* production resolve to the same Iceberg table. `config.ts` owns that rule and
|
|
228
|
+
* the test for it.
|
|
229
|
+
*/
|
|
230
|
+
function tableBucketArn(ctx) {
|
|
231
|
+
return s3TablesBucketArn(ctx, resolveAnalyticsConfig(ctx).tableBucket);
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* An S3 Tables bucket ARN, in the fixed
|
|
235
|
+
* `arn:aws:s3tables:<region>:<accountId>:bucket/<bucket>` form - the one place
|
|
236
|
+
* that form is spelled. {@link tableBucketArn} passes this environment's bucket
|
|
237
|
+
* name and {@link federationSource} passes {@link ALL_TABLE_BUCKETS}; the two
|
|
238
|
+
* have to agree on everything left of the last segment, because the catalog
|
|
239
|
+
* federation is checked against the wildcard form of the very ARN the table
|
|
240
|
+
* bucket is created under.
|
|
241
|
+
*/
|
|
242
|
+
function s3TablesBucketArn(ctx, bucket) {
|
|
243
|
+
return `arn:aws:s3tables:${ANALYTICS_REGION}:${ctx.accountId}:bucket/${bucket}`;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* `PAGE_VIEWS_COLUMNS` in the shape `createTable` accepts, so no column name,
|
|
247
|
+
* type or ordering is spelled a second time in this module.
|
|
248
|
+
*
|
|
249
|
+
* The ids are this function's own contribution, and it has to make one:
|
|
250
|
+
* `IcebergSchemaField.id` is optional in the S3 Tables API (auto-assigned when
|
|
251
|
+
* omitted) but required by the client, because the partition spec references a
|
|
252
|
+
* schema field by id and both travel in the same `CreateTable` request - an
|
|
253
|
+
* auto-assigned id would not exist yet for `sourceId` to name.
|
|
254
|
+
*
|
|
255
|
+
* The assignment is positional: the nth column of `PAGE_VIEWS_COLUMNS` gets id
|
|
256
|
+
* n, counting from {@link FIRST_FIELD_ID}. That is stable because
|
|
257
|
+
* `PAGE_VIEWS_COLUMNS` is an ordered `as const` tuple in the spec's own column
|
|
258
|
+
* order, so the same source always produces the same ids. Appending a column
|
|
259
|
+
* leaves every existing id untouched; reordering or removing one renumbers the
|
|
260
|
+
* columns after it, which is harmless here because the ids only ever have to
|
|
261
|
+
* agree *within one* `CreateTable` request - S3 Tables has no update-schema
|
|
262
|
+
* operation this payload is ever compared against, and an already-existing
|
|
263
|
+
* table's schema is not reconciled (`aws/s3tables.ts`'s `createTable`).
|
|
264
|
+
*/
|
|
265
|
+
function pageViewsFields() {
|
|
266
|
+
return PAGE_VIEWS_COLUMNS.map((column, index) => ({
|
|
267
|
+
name: column.name,
|
|
268
|
+
type: column.icebergType,
|
|
269
|
+
id: index + FIRST_FIELD_ID,
|
|
270
|
+
required: column.required,
|
|
271
|
+
}));
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* The `page_views` schema and partition spec, both derived from `schema.ts`.
|
|
275
|
+
* `fieldId` is left off the partition field: it is optional in the API and
|
|
276
|
+
* auto-assigned, and nothing in this request or any later one references it,
|
|
277
|
+
* so synthesising a second id would add a number with no reader.
|
|
278
|
+
*/
|
|
279
|
+
function pageViewsSchema() {
|
|
280
|
+
const fields = pageViewsFields();
|
|
281
|
+
const partitionSource = fields.find((field) => field.name === PAGE_VIEWS_PARTITION_COLUMN);
|
|
282
|
+
if (partitionSource === undefined) {
|
|
283
|
+
// Unreachable while `PAGE_VIEWS_PARTITION_COLUMN` is typed as a
|
|
284
|
+
// `PageViewColumnName`, which is derived from `PAGE_VIEWS_COLUMNS`. Raised
|
|
285
|
+
// rather than allowed to fall through to `sourceId: 0`, which would create
|
|
286
|
+
// a table partitioned on whatever field id 0 turned out to mean.
|
|
287
|
+
throw new Error(`analytics table schema has no "${PAGE_VIEWS_PARTITION_COLUMN}" column to partition by - PAGE_VIEWS_COLUMNS and PAGE_VIEWS_PARTITION_COLUMN in schema.ts have diverged`);
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
fields,
|
|
291
|
+
partitionSpec: [
|
|
292
|
+
{
|
|
293
|
+
name: partitionSource.name,
|
|
294
|
+
sourceId: partitionSource.id,
|
|
295
|
+
transform: PARTITION_TRANSFORM,
|
|
296
|
+
},
|
|
297
|
+
],
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Record the table bucket's identity. Shared by `read` and `create` because
|
|
302
|
+
* both record the same two values from the same two sources: the resolved
|
|
303
|
+
* config and {@link tableBucketArn}. `getTableBucket`'s response cannot
|
|
304
|
+
* disagree with either - the lookup is keyed by that very ARN - so the read
|
|
305
|
+
* path gains nothing by echoing the response back instead.
|
|
306
|
+
*/
|
|
307
|
+
function recordTableBucket(ctx) {
|
|
308
|
+
const out = output(ctx, TABLE_BUCKET_NODE);
|
|
309
|
+
out.name = resolveAnalyticsConfig(ctx).tableBucket;
|
|
310
|
+
out.arn = tableBucketArn(ctx);
|
|
311
|
+
}
|
|
312
|
+
/** Record the namespace's identity - its name and the bucket it lives in, which is all a namespace is. */
|
|
313
|
+
function recordNamespace(ctx) {
|
|
314
|
+
const out = output(ctx, NAMESPACE_NODE);
|
|
315
|
+
out.name = resolveAnalyticsConfig(ctx).namespace;
|
|
316
|
+
out.tableBucketArn = tableBucketArn(ctx);
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* The S3 Tables resource the catalog federates: **every** table bucket in this
|
|
320
|
+
* account and Region, which is what AWS's own integration procedure registers
|
|
321
|
+
* and what makes the catalog shared rather than this environment's own.
|
|
322
|
+
*
|
|
323
|
+
* Passing {@link tableBucketArn} here instead would look tidier and would break
|
|
324
|
+
* the one thing this node exists to get right. A federation registered against
|
|
325
|
+
* one environment's bucket is not one the next environment can adopt: staging
|
|
326
|
+
* would find a catalog federating production's bucket, and either adopt a
|
|
327
|
+
* federation that does not cover its own table or try to register a second one
|
|
328
|
+
* under the same account-scoped name. Every environment in the account derives
|
|
329
|
+
* this identical string, which is why two of them converge on one catalog
|
|
330
|
+
* instead of fighting over it.
|
|
331
|
+
*/
|
|
332
|
+
function federationSource(ctx) {
|
|
333
|
+
return s3TablesBucketArn(ctx, ALL_TABLE_BUCKETS);
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* The federation's source, checked against the one this account's pipeline has
|
|
337
|
+
* to be federated on - and a throw when it is not.
|
|
338
|
+
*
|
|
339
|
+
* This is the check that stops a successful *lookup* from standing in for a
|
|
340
|
+
* working *federation*. `EntityNotFoundException` is documented on both of
|
|
341
|
+
* `GlueClient`'s operations and means two different things. On `GetCatalog` it
|
|
342
|
+
* is "no such catalog" - but a **source-level** miss answers with it too (the
|
|
343
|
+
* S3 Tables resource the catalog federates does not exist, typically a wrong
|
|
344
|
+
* ARN), and `getCatalogFederation` maps both to `undefined`. So a federation
|
|
345
|
+
* that is broken at its source reads as one that is absent; `create` then runs,
|
|
346
|
+
* and `createCatalogFederation` swallows `FederatedResourceAlreadyExistsException`
|
|
347
|
+
* because a second environment genuinely must adopt what is there. Left
|
|
348
|
+
* unchecked the reconcile converges silently: `analytics bootstrap` reports the
|
|
349
|
+
* integration green over a federation that was never wired to a bucket, and the
|
|
350
|
+
* first symptom is Firehose routing every record to the error bucket.
|
|
351
|
+
*
|
|
352
|
+
* `CatalogFederation.sourceIdentifier` is what closes it, and is a required key
|
|
353
|
+
* of type `string | undefined` for this reason: it is `undefined` exactly when
|
|
354
|
+
* the catalog carries no `FederatedCatalog` at all, so a same-named catalog
|
|
355
|
+
* that is not federated, or is federated somewhere else, is distinguishable
|
|
356
|
+
* from this plugin's own. Both of the node's paths run through here - an
|
|
357
|
+
* adopted catalog is verified before it is recorded, and a created one is read
|
|
358
|
+
* back and verified rather than assumed - so no path records this node as
|
|
359
|
+
* satisfied on a catalog whose source was never checked.
|
|
360
|
+
*
|
|
361
|
+
* Only the source is checked, not `connectionName`. The source ARN is what
|
|
362
|
+
* decides whether the federation covers this account's table buckets; a catalog
|
|
363
|
+
* federating exactly those buckets through some connection other than
|
|
364
|
+
* `aws:s3tables` is not a state AWS's integration can produce, so a second
|
|
365
|
+
* condition would be a second way to fail with nothing new caught.
|
|
366
|
+
*/
|
|
367
|
+
function verifiedSource(ctx, federation) {
|
|
368
|
+
const source = federationSource(ctx);
|
|
369
|
+
if (federation.sourceIdentifier === source)
|
|
370
|
+
return source;
|
|
371
|
+
const found = federation.sourceIdentifier === undefined
|
|
372
|
+
? 'is not a federated catalog'
|
|
373
|
+
: `federates "${federation.sourceIdentifier}"`;
|
|
374
|
+
throw new Error(`Glue catalog "${federation.name}" ${found}, so it is not the S3 Tables integration this pipeline reads through - it has to federate "${source}". Adopting it would point Firehose at a catalog with no table behind it. Remove or rename that catalog, or enable the S3 Tables integration for this account and Region.`);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Record the adopted federation - after {@link verifiedSource} has passed, and
|
|
378
|
+
* never before it. {@link output} writes an entry into
|
|
379
|
+
* `state/<env>.analytics.json`, and an entry under this node's id is the claim
|
|
380
|
+
* that the pipeline has a catalog to read the table through, so it must not
|
|
381
|
+
* outlive the check that the catalog is the right one.
|
|
382
|
+
*/
|
|
383
|
+
function recordCatalogIntegration(ctx, federation) {
|
|
384
|
+
const source = verifiedSource(ctx, federation);
|
|
385
|
+
const out = output(ctx, CATALOG_NODE);
|
|
386
|
+
out.name = federation.name;
|
|
387
|
+
out.sourceIdentifier = source;
|
|
388
|
+
// The same guard `analytics-table` puts on its ARN, for the same reason:
|
|
389
|
+
// `normalizeCatalog` falls back to `''` for a body carrying no `ResourceArn`,
|
|
390
|
+
// and an empty string recorded under `arn` reads downstream as a real one.
|
|
391
|
+
if (federation.resourceArn)
|
|
392
|
+
out.arn = federation.resourceArn;
|
|
393
|
+
}
|
|
394
|
+
/** The S3 Tables bucket every analytics table lives in. */
|
|
395
|
+
export function analyticsTableBucketNode() {
|
|
396
|
+
return {
|
|
397
|
+
id: TABLE_BUCKET_NODE,
|
|
398
|
+
dependsOn: [],
|
|
399
|
+
title: 'S3 Tables bucket (us-east-1)',
|
|
400
|
+
async read(ctx) {
|
|
401
|
+
const bucket = await s3tables(ctx).getTableBucket(tableBucketArn(ctx));
|
|
402
|
+
if (bucket === undefined)
|
|
403
|
+
return false;
|
|
404
|
+
recordTableBucket(ctx);
|
|
405
|
+
return true;
|
|
406
|
+
},
|
|
407
|
+
async create(ctx) {
|
|
408
|
+
await s3tables(ctx).createTableBucket(resolveAnalyticsConfig(ctx).tableBucket);
|
|
409
|
+
recordTableBucket(ctx);
|
|
410
|
+
},
|
|
411
|
+
async delete(ctx) {
|
|
412
|
+
// Already-gone is not an error: the client swallows the 404 so a
|
|
413
|
+
// half-finished teardown is re-runnable.
|
|
414
|
+
await s3tables(ctx).deleteTableBucket(tableBucketArn(ctx));
|
|
415
|
+
},
|
|
416
|
+
};
|
|
417
|
+
}
|
|
418
|
+
/** The Iceberg namespace inside the table bucket. */
|
|
419
|
+
export function analyticsNamespaceNode() {
|
|
420
|
+
return {
|
|
421
|
+
id: NAMESPACE_NODE,
|
|
422
|
+
dependsOn: [TABLE_BUCKET_NODE],
|
|
423
|
+
title: 'S3 Tables namespace (us-east-1)',
|
|
424
|
+
async read(ctx) {
|
|
425
|
+
const namespace = await s3tables(ctx).getNamespace(tableBucketArn(ctx), resolveAnalyticsConfig(ctx).namespace);
|
|
426
|
+
if (namespace === undefined)
|
|
427
|
+
return false;
|
|
428
|
+
recordNamespace(ctx);
|
|
429
|
+
return true;
|
|
430
|
+
},
|
|
431
|
+
async create(ctx) {
|
|
432
|
+
await s3tables(ctx).createNamespace(tableBucketArn(ctx), resolveAnalyticsConfig(ctx).namespace);
|
|
433
|
+
recordNamespace(ctx);
|
|
434
|
+
},
|
|
435
|
+
async delete(ctx) {
|
|
436
|
+
await s3tables(ctx).deleteNamespace(tableBucketArn(ctx), resolveAnalyticsConfig(ctx).namespace);
|
|
437
|
+
},
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
/** The `page_views` table, carrying the schema and partition `schema.ts` owns. */
|
|
441
|
+
export function analyticsTableNode() {
|
|
442
|
+
return {
|
|
443
|
+
id: TABLE_NODE,
|
|
444
|
+
dependsOn: [NAMESPACE_NODE],
|
|
445
|
+
title: 'Iceberg table (us-east-1)',
|
|
446
|
+
async read(ctx) {
|
|
447
|
+
const analytics = resolveAnalyticsConfig(ctx);
|
|
448
|
+
const table = await s3tables(ctx).getTable(tableBucketArn(ctx), analytics.namespace, analytics.table);
|
|
449
|
+
if (table === undefined)
|
|
450
|
+
return false;
|
|
451
|
+
// The one identifier in this module that is genuinely unrecoverable from
|
|
452
|
+
// its inputs: a table ARN carries an opaque generated id, not a name.
|
|
453
|
+
const out = output(ctx, TABLE_NODE);
|
|
454
|
+
out.name = table.name;
|
|
455
|
+
// Guarded on the value, not just on the lookup having answered:
|
|
456
|
+
// `normalizeTable` falls back to `''` for a body carrying no `tableARN`,
|
|
457
|
+
// and an empty string recorded under `arn` reads downstream as a real
|
|
458
|
+
// one. Unreachable under the service's response model; recording nothing
|
|
459
|
+
// is the honest answer if it ever is reached.
|
|
460
|
+
if (table.arn)
|
|
461
|
+
out.arn = table.arn;
|
|
462
|
+
return true;
|
|
463
|
+
},
|
|
464
|
+
async create(ctx) {
|
|
465
|
+
const analytics = resolveAnalyticsConfig(ctx);
|
|
466
|
+
const client = s3tables(ctx);
|
|
467
|
+
const bucketArn = tableBucketArn(ctx);
|
|
468
|
+
await client.createTable(bucketArn, analytics.namespace, analytics.table, pageViewsSchema());
|
|
469
|
+
// Identity before the secondary call: `CreateTable` answers with no ARN,
|
|
470
|
+
// so hydrating one takes a second request, and a crash in between must
|
|
471
|
+
// still leave the table recorded for `destroy` to remove.
|
|
472
|
+
const out = output(ctx, TABLE_NODE);
|
|
473
|
+
out.name = analytics.table;
|
|
474
|
+
const created = await client.getTable(bucketArn, analytics.namespace, analytics.table);
|
|
475
|
+
// Same guard as `read`, for the same reason: an absent `tableARN`
|
|
476
|
+
// normalizes to `''`, which must not land in state as though it were one.
|
|
477
|
+
if (created?.arn)
|
|
478
|
+
out.arn = created.arn;
|
|
479
|
+
},
|
|
480
|
+
async delete(ctx) {
|
|
481
|
+
const analytics = resolveAnalyticsConfig(ctx);
|
|
482
|
+
await s3tables(ctx).deleteTable(tableBucketArn(ctx), analytics.namespace, analytics.table);
|
|
483
|
+
},
|
|
484
|
+
};
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* The Glue `s3tablescatalog` federation Firehose reads the `page_views` table
|
|
488
|
+
* through - **the one node in this graph that adopts shared state rather than
|
|
489
|
+
* owning it.**
|
|
490
|
+
*
|
|
491
|
+
* The integration is account-and-region scoped: a single catalog federates
|
|
492
|
+
* every S3 Tables bucket in the account and Region (see
|
|
493
|
+
* {@link federationSource}), so staging, production and anything else in the
|
|
494
|
+
* account that enabled the S3 Tables integration all read through the same one.
|
|
495
|
+
* Both halves of this node's behaviour follow from that. `read` adopts an
|
|
496
|
+
* existing federation instead of creating a second one, so a second environment
|
|
497
|
+
* converges on what is already there; `delete` removes nothing, so tearing one
|
|
498
|
+
* environment down leaves every other environment's pipeline intact.
|
|
499
|
+
*
|
|
500
|
+
* It depends on `analytics-table` rather than on the bucket even though the
|
|
501
|
+
* federation covers the account rather than any one table. Two reasons: the
|
|
502
|
+
* whole table chain is then in place before anything is federated, so
|
|
503
|
+
* `CreateCatalog`'s own `EntityNotFoundException` - which on that operation
|
|
504
|
+
* means the federated entity is missing, not the catalog - cannot fire merely
|
|
505
|
+
* because the bucket had not been created yet; and `destroyGraph` reverses the
|
|
506
|
+
* order, so this node's `delete` is reached first, before the table it was
|
|
507
|
+
* enabled for is removed.
|
|
508
|
+
*/
|
|
509
|
+
export function analyticsCatalogIntegrationNode() {
|
|
510
|
+
return {
|
|
511
|
+
id: CATALOG_NODE,
|
|
512
|
+
dependsOn: [TABLE_NODE],
|
|
513
|
+
title: `Glue ${CATALOG_NAME} federation (shared - account-and-region scoped, ${ANALYTICS_REGION})`,
|
|
514
|
+
async read(ctx) {
|
|
515
|
+
const federation = await glue(ctx).getCatalogFederation(CATALOG_NAME);
|
|
516
|
+
// Absent: `create` runs, and either creates the federation or adopts one
|
|
517
|
+
// that appeared in between. A wrongly-federated or non-federated catalog
|
|
518
|
+
// of this name is NOT absent - it fails inside `recordCatalogIntegration`
|
|
519
|
+
// rather than falling through to a create that would be swallowed as a
|
|
520
|
+
// duplicate.
|
|
521
|
+
if (federation === undefined)
|
|
522
|
+
return false;
|
|
523
|
+
recordCatalogIntegration(ctx, federation);
|
|
524
|
+
return true;
|
|
525
|
+
},
|
|
526
|
+
async create(ctx) {
|
|
527
|
+
const client = glue(ctx);
|
|
528
|
+
const source = federationSource(ctx);
|
|
529
|
+
ctx.logger.step(`enabling the ${CATALOG_NAME} Glue federation over ${source} - account-and-region scoped, shared with every other environment in this account and never removed by a teardown`);
|
|
530
|
+
await client.createCatalogFederation(CATALOG_NAME, source);
|
|
531
|
+
// Read back rather than assume. `createCatalogFederation` resolves both
|
|
532
|
+
// when it created the federation and when one already existed - which is
|
|
533
|
+
// what a second environment hits, deliberately - so its resolution says
|
|
534
|
+
// nothing about what is now in the account. And the lookup that returned
|
|
535
|
+
// "absent" just before it is exactly the shape a source-level
|
|
536
|
+
// `EntityNotFoundException` takes, so this is the only place the two can
|
|
537
|
+
// be told apart. See {@link verifiedSource}.
|
|
538
|
+
const created = await client.getCatalogFederation(CATALOG_NAME);
|
|
539
|
+
if (created === undefined) {
|
|
540
|
+
throw new Error(`the ${CATALOG_NAME} Glue federation is still not readable after CreateCatalog reported success. GetCatalog answers EntityNotFoundException both for a missing catalog and for a federation whose own source is missing, so this is a catalog that was never wired to "${source}" rather than one that was just created - check that the S3 Tables integration is enabled for this account in ${ANALYTICS_REGION}.`);
|
|
541
|
+
}
|
|
542
|
+
recordCatalogIntegration(ctx, created);
|
|
543
|
+
},
|
|
544
|
+
async delete() {
|
|
545
|
+
// Deliberately inert, and the only `delete` in this graph that is.
|
|
546
|
+
// `destroyGraph` calls every node's `delete` on teardown
|
|
547
|
+
// (`packages/cli/src/graph.ts`), so anything written here would run on
|
|
548
|
+
// every `analytics destroy`. The federation is account-and-region scoped
|
|
549
|
+
// shared state: removing it while tearing down staging would leave
|
|
550
|
+
// production's delivery stream with no catalog to write its Iceberg table
|
|
551
|
+
// through, so production would go on accepting CloudFront logs and route
|
|
552
|
+
// every record into its error bucket - with nothing in staging's output,
|
|
553
|
+
// or production's, saying what had been taken away. The rule core already
|
|
554
|
+
// states for the account-global OIDC provider it likewise never removes
|
|
555
|
+
// (`packages/core/src/aws/iam.ts`, "Account-global; never deleted here").
|
|
556
|
+
// `GlueClient` exposes no delete operation at all, which is the other half
|
|
557
|
+
// of the guard: there is nothing here to call by accident.
|
|
558
|
+
},
|
|
559
|
+
};
|
|
560
|
+
}
|
|
561
|
+
/* ------------------------------------------------------------------------- *
|
|
562
|
+
* The transform chain: the salt secret, the execution role, and the function.
|
|
563
|
+
* ------------------------------------------------------------------------- */
|
|
564
|
+
/**
|
|
565
|
+
* The suffix the transform Lambda's name carries, appended to
|
|
566
|
+
* `ctx.names.prefix`. See {@link transformFunctionName} for why the prefix is
|
|
567
|
+
* the source of the environment here and {@link resolveAnalyticsConfig} is not.
|
|
568
|
+
*/
|
|
569
|
+
const TRANSFORM_FUNCTION_SUFFIX = '-analytics-transform';
|
|
570
|
+
/** The suffix the transform Lambda's execution role carries. */
|
|
571
|
+
const TRANSFORM_ROLE_SUFFIX = '-analytics-transform-role';
|
|
572
|
+
/**
|
|
573
|
+
* The longest name IAM accepts for a role - and, separately, the longest Lambda
|
|
574
|
+
* accepts for a function. The two limits are the same number and are
|
|
575
|
+
* deliberately two constants: they belong to two services and either could
|
|
576
|
+
* move without the other.
|
|
577
|
+
*/
|
|
578
|
+
const ROLE_NAME_MAX_LENGTH = 64;
|
|
579
|
+
/** The longest name Lambda accepts for a function. See {@link ROLE_NAME_MAX_LENGTH}. */
|
|
580
|
+
const FUNCTION_NAME_MAX_LENGTH = 64;
|
|
581
|
+
/**
|
|
582
|
+
* The Lambda runtime the transform is bundled for. Node, because the bundle is
|
|
583
|
+
* this repo's own TypeScript compiled by rolldown to ESM
|
|
584
|
+
* (`transform/rolldown.config.ts`); `22.x` because the package's `engines`
|
|
585
|
+
* field declares `>=22` and the emitted bundle is checked against that
|
|
586
|
+
* assumption by `transform/write-manifest.ts` on every build, running under the
|
|
587
|
+
* developer's own Node.
|
|
588
|
+
*/
|
|
589
|
+
const TRANSFORM_RUNTIME = 'nodejs22.x';
|
|
590
|
+
/**
|
|
591
|
+
* Memory for the transform, in MB. Lambda scales CPU with memory, and the
|
|
592
|
+
* per-record work is CPU-bound rather than memory-bound - a JSON parse, a field
|
|
593
|
+
* mapping, and one HMAC-SHA256 per record for `visitor_key` - so this number is
|
|
594
|
+
* bought for the CPU share, not the heap. 256 doubles the 128 MB floor's share
|
|
595
|
+
* for a fraction of a cent per million invocations; the whole batch a Firehose
|
|
596
|
+
* transform invocation carries is bounded by Lambda's 6 MB synchronous payload
|
|
597
|
+
* limit, which no amount of it can approach on a 256 MB heap.
|
|
598
|
+
*/
|
|
599
|
+
const TRANSFORM_MEMORY_MB = 256;
|
|
600
|
+
/**
|
|
601
|
+
* The transform's timeout, in seconds. `aws/lambda.ts` records that the service
|
|
602
|
+
* caps `Timeout` at 900; this is well inside it and is not chosen for headroom
|
|
603
|
+
* against slow work but against a *stall*: Firehose invokes the transform
|
|
604
|
+
* synchronously, so a function that hangs holds a delivery buffer open. One
|
|
605
|
+
* minute is orders of magnitude more than mapping one buffer of small records
|
|
606
|
+
* takes, and far less than the time an operator would spend not noticing.
|
|
607
|
+
*/
|
|
608
|
+
const TRANSFORM_TIMEOUT_SECONDS = 60;
|
|
609
|
+
/**
|
|
610
|
+
* The largest deployment package Lambda accepts inline, in bytes.
|
|
611
|
+
*
|
|
612
|
+
* 50 MB, verified 2026-08-31 against Lambda's quotas page: "Deployment package
|
|
613
|
+
* (.zip file archive) size - 50 MB (zipped, when uploaded through the Lambda
|
|
614
|
+
* API or SDKs). Upload larger files with Amazon S3." That is the path
|
|
615
|
+
* `CreateFunction`'s `Code.ZipFile` takes (`aws/lambda.ts` base64-encodes it),
|
|
616
|
+
* and the arithmetic below is 1024-based because the same page states that
|
|
617
|
+
* Lambda's documentation writes MB for 1,024 KB.
|
|
618
|
+
*
|
|
619
|
+
* **Why the code goes inline at all**, rather than through an S3 code bucket:
|
|
620
|
+
* Lambda requires the code bucket to be in the *function's own* region. This
|
|
621
|
+
* function is pinned to {@link ANALYTICS_REGION} (see the module comment) while
|
|
622
|
+
* the site's bucket - the only bucket the CLI already owns and the only one a
|
|
623
|
+
* deploy role is already granted on - is in `config.region`, which is
|
|
624
|
+
* deliberately something else. An S3 code path would therefore mean a second
|
|
625
|
+
* bucket in us-east-1 existing only to hold one zip, a second node to create
|
|
626
|
+
* it, and a second grant; and the bundle is three orders of magnitude under
|
|
627
|
+
* this limit. `analytics-error-bucket` is not that bucket either - it is
|
|
628
|
+
* Firehose's failed-record output and is not a deploy artifact store.
|
|
629
|
+
*
|
|
630
|
+
* The guard below is a tripwire, not a budget: if the bundle ever grows past
|
|
631
|
+
* this, the fix is the S3 code path, and the raise says so with the measured
|
|
632
|
+
* size rather than letting AWS answer a 400 with the request body's length.
|
|
633
|
+
*/
|
|
634
|
+
const MAX_INLINE_ZIP_BYTES = 50 * 1024 * 1024;
|
|
635
|
+
/**
|
|
636
|
+
* The fixed timestamp every entry in the deployment package carries, so the
|
|
637
|
+
* same bundle bytes always produce the same zip bytes - `packageAndUploadAgent`
|
|
638
|
+
* uses the same one, for the same reason: a zip stamped with the current time
|
|
639
|
+
* would differ on every build, and the deployment decision has to turn on task
|
|
640
|
+
* 43's source hash rather than on whether two archives happen to compare equal.
|
|
641
|
+
*
|
|
642
|
+
* That sentence was false until this constant moved to core. The value here was
|
|
643
|
+
* `new Date('1980-01-01T00:00:00Z')`, which a zip encodes as *local* time: it
|
|
644
|
+
* threw `date not in range 1980-2099` west of Greenwich, and where it did not
|
|
645
|
+
* throw it produced different bytes per zone - the opposite of the guarantee
|
|
646
|
+
* claimed above. See {@link REPRODUCIBLE_ZIP_MTIME}.
|
|
647
|
+
*/
|
|
648
|
+
const ZIP_MTIME = REPRODUCIBLE_ZIP_MTIME;
|
|
649
|
+
/** The deflate level `packageAndUploadAgent` uses. */
|
|
650
|
+
const ZIP_LEVEL = 6;
|
|
651
|
+
/**
|
|
652
|
+
* The prefix Lambda derives a function's log group from. The group itself is
|
|
653
|
+
* created by the Lambda service on first invocation and by no node in this
|
|
654
|
+
* graph - see {@link transformLogGroupArn}.
|
|
655
|
+
*/
|
|
656
|
+
const LAMBDA_LOG_GROUP_PREFIX = '/aws/lambda/';
|
|
657
|
+
/** The name of the inline policy this plugin puts on its own transform role. */
|
|
658
|
+
const TRANSFORM_ROLE_POLICY = 'transform';
|
|
659
|
+
/** IAM's policy-language version, the only one there is. */
|
|
660
|
+
const POLICY_VERSION = '2012-10-17';
|
|
661
|
+
/**
|
|
662
|
+
* Bytes of randomness in a freshly generated salt secret: 32, so the stored
|
|
663
|
+
* seed is 256 bits - the width of the HMAC-SHA256 the transform derives each
|
|
664
|
+
* day's salt with (`transform/visitor-key.ts`), so the seed is not the weak
|
|
665
|
+
* half of that construction.
|
|
666
|
+
*/
|
|
667
|
+
const SALT_SECRET_BYTES = 32;
|
|
668
|
+
/**
|
|
669
|
+
* The description stamped on the secret when this node creates it. It is
|
|
670
|
+
* written for the operator who finds the secret in the Secrets Manager console
|
|
671
|
+
* with no other context and is deciding whether it is safe to delete: the
|
|
672
|
+
* answer is no, and the reason has to travel with the resource rather than live
|
|
673
|
+
* only in this file.
|
|
674
|
+
*/
|
|
675
|
+
const SALT_SECRET_DESCRIPTION = 'blogwright analytics: the long-lived seed the record-transform Lambda derives each day’s visitor_key salt from, as HMAC-SHA256(secret, day). Never rotate, replace or restore it from a different value - every visitor_key already written to the page_views table was derived from this seed, and a new one silently stops old rows comparing to new ones.';
|
|
676
|
+
/**
|
|
677
|
+
* The trust document the transform's execution role is created with - the shape
|
|
678
|
+
* `packages/cli/src/nodes.ts:106-115` declares as `LAMBDA_TRUST`, **restated
|
|
679
|
+
* rather than imported, deliberately**.
|
|
680
|
+
*
|
|
681
|
+
* `LAMBDA_TRUST` is CLI-private: it is a module-level `const` in
|
|
682
|
+
* `packages/cli/src/nodes.ts` with no export, and even were it exported a
|
|
683
|
+
* plugin may not reach it. Core's `plugin.ts` states the rule this package
|
|
684
|
+
* obeys - "a plugin is a package that depends on `blogwright-core` and never on
|
|
685
|
+
* the CLI - it never imports from `blogwright` (the CLI package)". Core is no
|
|
686
|
+
* home for it either: `IamClient.ensureRole` takes the document as an opaque
|
|
687
|
+
* `object`, so a shared trust constant in core would be an export with no core
|
|
688
|
+
* or CLI consumer, which `pnpm knip` reports as dead.
|
|
689
|
+
*
|
|
690
|
+
* So the two copies stand, and they are allowed to drift: this one names the
|
|
691
|
+
* principal *this* function assumes, and the CLI's names the principal its
|
|
692
|
+
* MicroVM builder assumes. Nothing reconciles one against the other, and
|
|
693
|
+
* nothing should - a change to the site's builder trust is not a change to
|
|
694
|
+
* this plugin's.
|
|
695
|
+
*/
|
|
696
|
+
const LAMBDA_TRUST = {
|
|
697
|
+
Version: POLICY_VERSION,
|
|
698
|
+
Statement: [
|
|
699
|
+
{
|
|
700
|
+
Effect: 'Allow',
|
|
701
|
+
Principal: { Service: 'lambda.amazonaws.com' },
|
|
702
|
+
Action: ['sts:AssumeRole', 'sts:TagSession'],
|
|
703
|
+
},
|
|
704
|
+
],
|
|
705
|
+
};
|
|
706
|
+
/**
|
|
707
|
+
* The plugin's own Lambda client, built the same way {@link s3tables} and
|
|
708
|
+
* {@link glue} are: core's bundle enumerates no `lambda` key at all (`microvms`
|
|
709
|
+
* is a different API on the same host - see `aws/lambda.ts`), and every client
|
|
710
|
+
* this plugin uses signs in {@link ANALYTICS_REGION}.
|
|
711
|
+
*/
|
|
712
|
+
function lambda(ctx) {
|
|
713
|
+
return createAnalyticsClients(ctx).lambda;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* The plugin's own Secrets Manager client - **not the host bundle's own copy**,
|
|
717
|
+
* which core builds over the primary-region signer
|
|
718
|
+
* (`packages/core/src/clients.ts:68`).
|
|
719
|
+
*
|
|
720
|
+
* This is the one client choice in the module where reaching for the host's
|
|
721
|
+
* copy fails silently rather than loudly. The secret would be created in
|
|
722
|
+
* `config.region`; the transform Lambda runs in {@link ANALYTICS_REGION}; and
|
|
723
|
+
* the grant the role carries names an ARN that spells its region out
|
|
724
|
+
* ({@link requireSaltSecretArn}), so the function's `GetSecretValue` would be
|
|
725
|
+
* denied against a secret that exists, in a region no other node in this graph
|
|
726
|
+
* is in. `aws/clients.ts` builds this one over `ctx.clients.signingUsEast1` for
|
|
727
|
+
* exactly that reason.
|
|
728
|
+
*/
|
|
729
|
+
function secrets(ctx) {
|
|
730
|
+
return createAnalyticsClients(ctx).secrets;
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* Reject a derived AWS name longer than the service accepts, naming the
|
|
734
|
+
* measured length rather than letting the service answer a validation error at
|
|
735
|
+
* create time. The same guard `resolveAnalyticsConfig` puts on the derived
|
|
736
|
+
* table bucket and `deriveNames` puts on the site bucket, applied where these
|
|
737
|
+
* two names are derived.
|
|
738
|
+
*/
|
|
739
|
+
function boundedName(name, limit, what) {
|
|
740
|
+
if (name.length > limit) {
|
|
741
|
+
throw new Error(`derived analytics ${what} name "${name}" is ${name.length} characters, over AWS's ${limit}-character limit; shorten env or siteName`);
|
|
742
|
+
}
|
|
743
|
+
return name;
|
|
744
|
+
}
|
|
745
|
+
/**
|
|
746
|
+
* The transform Lambda's name.
|
|
747
|
+
*
|
|
748
|
+
* Derived from `ctx.names.prefix` - core's own `<env>-<siteName>`
|
|
749
|
+
* (`packages/core/src/config.ts:388`) - and **not** from
|
|
750
|
+
* {@link resolveAnalyticsConfig}, because there is nothing there to resolve:
|
|
751
|
+
* the `analytics` block owns six settings and neither this name nor the role's
|
|
752
|
+
* is one of them (`config.ts`), so an operator has no override to honour. What
|
|
753
|
+
* `config.ts`'s seal exists to prevent is a *private, env-less* derivation - a
|
|
754
|
+
* `${ctx.config.siteName}-analytics-transform` that would make staging and
|
|
755
|
+
* production reconcile the same function. This is the opposite: the
|
|
756
|
+
* environment is the first thing in the string, and the derivation is core's
|
|
757
|
+
* own, shared with every name the site graph already uses.
|
|
758
|
+
*/
|
|
759
|
+
function transformFunctionName(ctx) {
|
|
760
|
+
return boundedName(`${ctx.names.prefix}${TRANSFORM_FUNCTION_SUFFIX}`, FUNCTION_NAME_MAX_LENGTH, 'transform function');
|
|
761
|
+
}
|
|
762
|
+
/** The transform Lambda's execution role name. See {@link transformFunctionName}. */
|
|
763
|
+
function transformRoleName(ctx) {
|
|
764
|
+
return boundedName(`${ctx.names.prefix}${TRANSFORM_ROLE_SUFFIX}`, ROLE_NAME_MAX_LENGTH, 'transform role');
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* The log group ARN the role's `logs:` grant is scoped to - the function's
|
|
768
|
+
* **own** group and nothing else, the scoping
|
|
769
|
+
* `packages/cli/src/nodes.ts:212` applies to the site's exec role.
|
|
770
|
+
*
|
|
771
|
+
* The region is {@link ANALYTICS_REGION} and not `ctx.config.region`, which is
|
|
772
|
+
* the one place this ARN differs from the CLI's `logGroupArn` helper (whose
|
|
773
|
+
* region parameter defaults to `ctx.config.region`, correctly, because the
|
|
774
|
+
* function it scopes runs there). This function is pinned to us-east-1, so its
|
|
775
|
+
* log group is too, and a grant naming the primary region would be a grant on a
|
|
776
|
+
* group that never exists.
|
|
777
|
+
*
|
|
778
|
+
* **No node creates this group.** Lambda creates it implicitly on the
|
|
779
|
+
* function's first invocation. That is worth stating because the policy below
|
|
780
|
+
* grants `logs:CreateLogStream` and `logs:PutLogEvents` and *not*
|
|
781
|
+
* `logs:CreateLogGroup`: the transform's own diagnostics therefore depend on
|
|
782
|
+
* that implicit creation succeeding, and the pipeline's real failure signal is
|
|
783
|
+
* elsewhere - a record the transform cannot map goes to Firehose's error prefix
|
|
784
|
+
* (`transform/handler.ts`), and a batch that throws raises Firehose's own error
|
|
785
|
+
* metric. Adding the group as a node of its own, with the retention the site's
|
|
786
|
+
* log groups carry, is a coherent follow-up and is outside this node set.
|
|
787
|
+
*/
|
|
788
|
+
function transformLogGroupArn(ctx) {
|
|
789
|
+
const group = `${LAMBDA_LOG_GROUP_PREFIX}${transformFunctionName(ctx)}`;
|
|
790
|
+
return `arn:aws:logs:${ANALYTICS_REGION}:${ctx.accountId}:log-group:${group}:*`;
|
|
791
|
+
}
|
|
792
|
+
/**
|
|
793
|
+
* An ARN another node recorded, or a throw naming the missing edge.
|
|
794
|
+
*
|
|
795
|
+
* Six nodes in this module interpolate an ARN they did not derive, and this is
|
|
796
|
+
* the one place that read is done. What the declared `dependsOn` buys is the
|
|
797
|
+
* ordering *as a stated fact*, not the ordering itself. `topoSort` drains its
|
|
798
|
+
* zero-indegree queue in alphabetical order
|
|
799
|
+
* (`packages/cli/src/graph.ts:46-49`), so several of these pairs would be
|
|
800
|
+
* reconciled in the right order today even with no edge declared - by the
|
|
801
|
+
* accident of how their ids sort, and by nothing else. The edge replaces the
|
|
802
|
+
* accident with the constraint that is actually true: rename either node past
|
|
803
|
+
* the other in sort order and the implicit ordering flips in silence (in
|
|
804
|
+
* teardown too, which is this same order reversed - `graph.ts:107`), whereas
|
|
805
|
+
* the declared edge either still holds or makes `topoSort` throw `depends on
|
|
806
|
+
* unknown node` (`graph.ts:40`) before a single API call is made.
|
|
807
|
+
*
|
|
808
|
+
* The throw is the runtime backstop under either regime: whatever the order, no
|
|
809
|
+
* policy is written with `undefined` interpolated into a live IAM grant and no
|
|
810
|
+
* delivery stream is created against `undefined` - a wrong permission or a
|
|
811
|
+
* misrouted stream written silently, never an error, and one nothing downstream
|
|
812
|
+
* would notice until a record was denied or lost.
|
|
813
|
+
*
|
|
814
|
+
* The empty string is rejected as hard as `undefined`: several of these ARNs
|
|
815
|
+
* are read straight off an AWS response, so a body carrying none would
|
|
816
|
+
* otherwise leave a grant on `""`.
|
|
817
|
+
*/
|
|
818
|
+
function requireRecordedArn(ctx, request) {
|
|
819
|
+
const arn = ctx.state.resources[request.node]?.arn;
|
|
820
|
+
if (typeof arn !== 'string' || arn === '') {
|
|
821
|
+
throw new Error(`the analytics ${request.what}'s ARN is not recorded in the "${ctx.env}" plugin state, so ${request.dependent} has no ${request.lack} - ${request.node} is this node's declared dependency and must be reconciled first; run \`blogwright analytics bootstrap --env ${ctx.env}\``);
|
|
822
|
+
}
|
|
823
|
+
return arn;
|
|
824
|
+
}
|
|
825
|
+
/**
|
|
826
|
+
* The salt secret's ARN as `analytics-salt-secret` recorded it. See
|
|
827
|
+
* {@link requireRecordedArn} for what the throw is doing.
|
|
828
|
+
*
|
|
829
|
+
* This ARN cannot be derived: Secrets Manager appends six random characters to
|
|
830
|
+
* the name, which is why `packages/pds/src/nodes.ts` has to glob `<name>-*` in
|
|
831
|
+
* its own grant and why this node instead depends on the node that reads the
|
|
832
|
+
* real one back.
|
|
833
|
+
*/
|
|
834
|
+
function requireSaltSecretArn(ctx) {
|
|
835
|
+
return requireRecordedArn(ctx, {
|
|
836
|
+
what: 'salt secret',
|
|
837
|
+
node: SALT_SECRET_NODE,
|
|
838
|
+
dependent: TRANSFORM_ROLE_NODE,
|
|
839
|
+
lack: 'resource to grant secretsmanager:GetSecretValue on',
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
/** The transform role's ARN as `analytics-transform-role` recorded it. See {@link requireRecordedArn}. */
|
|
843
|
+
function requireTransformRoleArn(ctx) {
|
|
844
|
+
return requireRecordedArn(ctx, {
|
|
845
|
+
what: 'transform role',
|
|
846
|
+
node: TRANSFORM_ROLE_NODE,
|
|
847
|
+
dependent: TRANSFORM_FUNCTION_NODE,
|
|
848
|
+
lack: 'execution role to run as',
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
/**
|
|
852
|
+
* Apply the transform role's inline policy. Shared by `create` and `update` -
|
|
853
|
+
* the `applyExecRolePolicy` pattern (`packages/cli/src/nodes.ts:180-216`) - so
|
|
854
|
+
* a reconcile of an existing role rewrites the same document a fresh one gets,
|
|
855
|
+
* and a changed secret ARN or a changed function name reaches the policy
|
|
856
|
+
* without a teardown.
|
|
857
|
+
*
|
|
858
|
+
* **Two statements, two concrete resources, no `*` anywhere.** The `logs`
|
|
859
|
+
* statement names the function's own log group ({@link transformLogGroupArn});
|
|
860
|
+
* the `secretsmanager` statement names the one secret this pipeline owns and
|
|
861
|
+
* nothing else. A `*` in the second would hand every secret in the account -
|
|
862
|
+
* every other environment's salt, and `blogwright-pds`'s OAuth client key and
|
|
863
|
+
* live session - to a role whose only job is to read one value, and nothing in
|
|
864
|
+
* the suite would notice, because a policy with a wildcard grants strictly more
|
|
865
|
+
* than a correct one and every functional test still passes. `nodes.test.ts`
|
|
866
|
+
* parses the document back out of the request and asserts on the `Resource`
|
|
867
|
+
* values for that reason.
|
|
868
|
+
*/
|
|
869
|
+
async function applyTransformRolePolicy(ctx) {
|
|
870
|
+
await ctx.clients.iam.putRolePolicy(transformRoleName(ctx), TRANSFORM_ROLE_POLICY, {
|
|
871
|
+
Version: POLICY_VERSION,
|
|
872
|
+
Statement: [
|
|
873
|
+
{
|
|
874
|
+
Effect: 'Allow',
|
|
875
|
+
Action: ['logs:CreateLogStream', 'logs:PutLogEvents'],
|
|
876
|
+
Resource: transformLogGroupArn(ctx),
|
|
877
|
+
},
|
|
878
|
+
{
|
|
879
|
+
Effect: 'Allow',
|
|
880
|
+
Action: ['secretsmanager:GetSecretValue'],
|
|
881
|
+
Resource: requireSaltSecretArn(ctx),
|
|
882
|
+
},
|
|
883
|
+
],
|
|
884
|
+
});
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* A fresh salt seed: {@link SALT_SECRET_BYTES} bytes from the platform CSPRNG,
|
|
888
|
+
* base64-encoded so the value is a plain string on the wire and in the console.
|
|
889
|
+
*
|
|
890
|
+
* `crypto.getRandomValues` rather than `crypto.randomUUID` (which core's
|
|
891
|
+
* `upsertSecret` uses for its idempotency token): a UUID carries 122 bits with
|
|
892
|
+
* six of its characters fixed by the version and variant, which is a fine
|
|
893
|
+
* request id and a poor key.
|
|
894
|
+
*
|
|
895
|
+
* The returned value is handed to `upsertSecret` and to nothing else. It is
|
|
896
|
+
* never logged, never recorded in `state/<env>.analytics.json`, and never read
|
|
897
|
+
* back by this module - `describeSecret` is used everywhere a value could have
|
|
898
|
+
* been, precisely because it answers with metadata and never with the secret.
|
|
899
|
+
*/
|
|
900
|
+
function newSaltSecret() {
|
|
901
|
+
const bytes = new Uint8Array(SALT_SECRET_BYTES);
|
|
902
|
+
crypto.getRandomValues(bytes);
|
|
903
|
+
return Buffer.from(bytes).toString('base64');
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Record the salt secret's **identity** - the name it is addressed by and the
|
|
907
|
+
* ARN the role's policy interpolates. Never its value: an entry in
|
|
908
|
+
* `state/<env>.analytics.json` is written to the site's S3 bucket, which is not
|
|
909
|
+
* a secret store, and the whole point of Secrets Manager holding this seed is
|
|
910
|
+
* that the digest beside `user_agent` in the table cannot be reversed by
|
|
911
|
+
* anyone who can read the analytics data.
|
|
912
|
+
*
|
|
913
|
+
* The name recorded is the resolved config's, not the one `DescribeSecret`
|
|
914
|
+
* echoes back: it is the name every call in this module addresses the secret
|
|
915
|
+
* by, and it cannot be empty. The ARN is the opposite - it is the one part that
|
|
916
|
+
* cannot be derived, so it comes from the response, guarded on its value the
|
|
917
|
+
* way `analytics-table` guards its own (an absent field would otherwise land in
|
|
918
|
+
* state as an empty string that reads downstream as a real ARN).
|
|
919
|
+
*/
|
|
920
|
+
function recordSaltSecret(ctx, name, arn) {
|
|
921
|
+
const out = output(ctx, SALT_SECRET_NODE);
|
|
922
|
+
out.name = name;
|
|
923
|
+
if (arn)
|
|
924
|
+
out.arn = arn;
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* The Secrets Manager secret holding the seed every `visitor_key` in the table
|
|
928
|
+
* is derived from - **the one resource in this graph that must outlive its own
|
|
929
|
+
* reconcile, and the one this module never overwrites or deletes.**
|
|
930
|
+
*
|
|
931
|
+
* `transform/visitor-key.ts` derives the per-day salt as
|
|
932
|
+
* `HMAC-SHA256(secret, day)` and `map-record.ts` hashes the viewer's address
|
|
933
|
+
* under it, so the stored value is the only thing standing between a row in
|
|
934
|
+
* `page_views` and the address it came from - the table keeps `user_agent` in
|
|
935
|
+
* the clear beside the key, and an unsalted SHA-256 over IPv4's 2^32 space is a
|
|
936
|
+
* lookup table, not a pseudonym. Two consequences shape every method below.
|
|
937
|
+
*
|
|
938
|
+
* **It is created once and never rewritten.** Replacing the value does not
|
|
939
|
+
* "rotate" anything: it orphans every `visitor_key` already written, because no
|
|
940
|
+
* row from before the change ever compares equal to a row from after it. The
|
|
941
|
+
* dashboard's unique-visitor figures would silently double at the boundary and
|
|
942
|
+
* `analytics backfill` - which re-derives a historical day's salt from this
|
|
943
|
+
* same seed - would produce rows that join to nothing. So `read` adopts, and
|
|
944
|
+
* `create` re-checks and adopts rather than trusting that the `read` before it
|
|
945
|
+
* is still true (see below). Daily turnover comes from the *derivation*, not
|
|
946
|
+
* from the store, which is why **no Secrets Manager rotation is configured**:
|
|
947
|
+
* managed rotation would mean a rotation Lambda, a schedule and a second
|
|
948
|
+
* execution role, to replace the one value that must not change.
|
|
949
|
+
*
|
|
950
|
+
* **Its `delete` removes nothing.** This is the second inert `delete` in the
|
|
951
|
+
* graph and it is inert for a different reason than
|
|
952
|
+
* `analytics-catalog-integration`'s, which is inert because the federation is
|
|
953
|
+
* shared. This one is inert because the act is asymmetric. Core's
|
|
954
|
+
* `deleteSecret` sends `ForceDeleteWithoutRecovery: true`, so there is no
|
|
955
|
+
* recovery window and nothing to undo with; keeping a secret costs cents a
|
|
956
|
+
* month and one command to remove by hand, while deleting one is unrecoverable
|
|
957
|
+
* and destroys the ability to interpret any `page_views` data that outlived the
|
|
958
|
+
* teardown - an exported copy, a table bucket whose own delete failed, or an
|
|
959
|
+
* environment torn down and re-bootstrapped, which is a routine recovery move
|
|
960
|
+
* and would otherwise come back with a different seed and no sign that
|
|
961
|
+
* anything had changed. The teardown says what it kept and how to remove it.
|
|
962
|
+
*/
|
|
963
|
+
export function analyticsSaltSecretNode() {
|
|
964
|
+
return {
|
|
965
|
+
id: SALT_SECRET_NODE,
|
|
966
|
+
dependsOn: [],
|
|
967
|
+
title: `visitor_key salt secret (${ANALYTICS_REGION} - created once, never replaced, kept on teardown)`,
|
|
968
|
+
async read(ctx) {
|
|
969
|
+
const name = resolveAnalyticsConfig(ctx).saltSecretName;
|
|
970
|
+
// `describeSecret`, not `getSecretValue`: existence is the question, and
|
|
971
|
+
// the value is not this process's business at any point.
|
|
972
|
+
const secret = await secrets(ctx).describeSecret(name);
|
|
973
|
+
if (secret === undefined)
|
|
974
|
+
return false;
|
|
975
|
+
recordSaltSecret(ctx, name, secret.arn);
|
|
976
|
+
return true;
|
|
977
|
+
},
|
|
978
|
+
async create(ctx) {
|
|
979
|
+
const name = resolveAnalyticsConfig(ctx).saltSecretName;
|
|
980
|
+
const client = secrets(ctx);
|
|
981
|
+
// The guard that makes "created if absent, never overwritten" true rather
|
|
982
|
+
// than merely intended. `applyGraph` calls `create` only after `read`
|
|
983
|
+
// answered false, so this lookup is redundant on the happy path - and it
|
|
984
|
+
// is here because the call below is `upsertSecret`, which falls back to
|
|
985
|
+
// `PutSecretValue` when `CreateSecret` reports the secret already exists.
|
|
986
|
+
// That fallback is right for `pds keygen`, which owns a value it means to
|
|
987
|
+
// replace, and catastrophic here. Two concurrent bootstraps of the same
|
|
988
|
+
// environment are all it takes to reach it, and the damage - every
|
|
989
|
+
// visitor_key written so far orphaned - is silent and unrepairable. One
|
|
990
|
+
// extra DescribeSecret, once in an environment's lifetime, closes all but
|
|
991
|
+
// the microseconds between these two calls.
|
|
992
|
+
const existing = await client.describeSecret(name);
|
|
993
|
+
if (existing !== undefined) {
|
|
994
|
+
ctx.logger.warn(`adopting the existing analytics salt secret "${name}" rather than creating a new one - every visitor_key already written was derived from its value, so it is never replaced`);
|
|
995
|
+
recordSaltSecret(ctx, name, existing.arn);
|
|
996
|
+
return;
|
|
997
|
+
}
|
|
998
|
+
// No rotation configuration is sent, deliberately: see this node's doc
|
|
999
|
+
// comment. `upsertSecret` sends `Name`, `SecretString`, an idempotency
|
|
1000
|
+
// token, the description and the tags, and nothing else.
|
|
1001
|
+
await client.upsertSecret(name, newSaltSecret(), SALT_SECRET_DESCRIPTION, ctx.tags);
|
|
1002
|
+
// Identity before the ARN lookup, the discipline `analytics-table`'s
|
|
1003
|
+
// `create` follows: `CreateSecret`'s response is discarded by
|
|
1004
|
+
// `upsertSecret`, so hydrating the ARN takes a second request, and a
|
|
1005
|
+
// crash in between must still leave a record that this environment now
|
|
1006
|
+
// owns a secret under this name. The role's own guard
|
|
1007
|
+
// ({@link requireSaltSecretArn}) is what stops a half-recorded entry
|
|
1008
|
+
// becoming a wrong grant: without the ARN it raises rather than
|
|
1009
|
+
// interpolating nothing.
|
|
1010
|
+
const out = output(ctx, SALT_SECRET_NODE);
|
|
1011
|
+
out.name = name;
|
|
1012
|
+
const created = await client.describeSecret(name);
|
|
1013
|
+
if (created?.arn)
|
|
1014
|
+
out.arn = created.arn;
|
|
1015
|
+
},
|
|
1016
|
+
async delete(ctx) {
|
|
1017
|
+
// Deliberately removes nothing - see this node's doc comment for why the
|
|
1018
|
+
// asymmetry between keeping and deleting decides it. Said out loud rather
|
|
1019
|
+
// than left to the title, because `destroyGraph` prints "deleted <title>"
|
|
1020
|
+
// for every node it walks and an operator tearing an environment down is
|
|
1021
|
+
// owed the one line that says what is still in the account.
|
|
1022
|
+
const name = resolveAnalyticsConfig(ctx).saltSecretName;
|
|
1023
|
+
ctx.logger.warn(`keeping the analytics salt secret "${name}" - it is the only thing that makes an already-written visitor_key meaningful, and deleting it is not reversible. Remove it by hand once no page_views data derived from it survives: aws secretsmanager delete-secret --region ${ANALYTICS_REGION} --secret-id ${name}`);
|
|
1024
|
+
},
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
/**
|
|
1028
|
+
* The transform Lambda's execution role: permission to write its own logs and
|
|
1029
|
+
* to read the one secret it needs, and nothing else.
|
|
1030
|
+
*
|
|
1031
|
+
* It declares `dependsOn: ['analytics-salt-secret']` because its policy
|
|
1032
|
+
* interpolates that node's *recorded* ARN - the implementation notes' rule that
|
|
1033
|
+
* "a node depends on every node whose recorded ARN it interpolates" - and
|
|
1034
|
+
* {@link requireSaltSecretArn} explains what an undeclared edge would produce.
|
|
1035
|
+
*/
|
|
1036
|
+
export function analyticsTransformRoleNode() {
|
|
1037
|
+
return {
|
|
1038
|
+
id: TRANSFORM_ROLE_NODE,
|
|
1039
|
+
dependsOn: [SALT_SECRET_NODE],
|
|
1040
|
+
title: `IAM transform execution role (global - IAM is not regional; it serves the ${ANALYTICS_REGION} pipeline)`,
|
|
1041
|
+
async read(ctx) {
|
|
1042
|
+
const name = transformRoleName(ctx);
|
|
1043
|
+
const arn = await ctx.clients.iam.getRoleArn(name);
|
|
1044
|
+
// Falsy rather than `=== undefined`: `getRoleArn` reads the ARN out of
|
|
1045
|
+
// the response XML, so a body without one answers `undefined` while an
|
|
1046
|
+
// empty tag would answer `""`, and neither is a role to record.
|
|
1047
|
+
if (!arn)
|
|
1048
|
+
return false;
|
|
1049
|
+
const out = output(ctx, TRANSFORM_ROLE_NODE);
|
|
1050
|
+
out.name = name;
|
|
1051
|
+
out.arn = arn;
|
|
1052
|
+
return true;
|
|
1053
|
+
},
|
|
1054
|
+
async create(ctx) {
|
|
1055
|
+
const name = transformRoleName(ctx);
|
|
1056
|
+
// `ctx.clients.iam`, the host's own client, not the plugin's bundle: IAM
|
|
1057
|
+
// is a global service (`packages/core/src/aws/endpoint.ts`'s
|
|
1058
|
+
// GLOBAL_SERVICES), so core's instance already signs us-east-1 and there
|
|
1059
|
+
// is no region for the pin to get wrong.
|
|
1060
|
+
const arn = await ctx.clients.iam.ensureRole(name, LAMBDA_TRUST, `Execution role for the ${ctx.config.siteName} analytics record-transform Lambda`, ctx.tags);
|
|
1061
|
+
// Recorded before the policy PUT - which is the opposite order to
|
|
1062
|
+
// `execRoleNode` (`packages/cli/src/nodes.ts:219-246`), and deliberately.
|
|
1063
|
+
// The role is a real IAM object the moment `ensureRole` returns; if the
|
|
1064
|
+
// policy call then fails, an entry recorded here is what tells the next
|
|
1065
|
+
// reconcile it exists. Nothing reads a role ARN as "the role is
|
|
1066
|
+
// configured": the function node reads it to *run as*, and it cannot be
|
|
1067
|
+
// reached before this node's own `update` has reapplied the policy,
|
|
1068
|
+
// because `applyGraph` reconciles this node to completion first.
|
|
1069
|
+
const out = output(ctx, TRANSFORM_ROLE_NODE);
|
|
1070
|
+
out.name = name;
|
|
1071
|
+
out.arn = arn;
|
|
1072
|
+
await applyTransformRolePolicy(ctx);
|
|
1073
|
+
},
|
|
1074
|
+
async update(ctx) {
|
|
1075
|
+
await applyTransformRolePolicy(ctx);
|
|
1076
|
+
},
|
|
1077
|
+
async delete(ctx) {
|
|
1078
|
+
// Idempotent, and removes the inline policy first - `deleteRole`
|
|
1079
|
+
// (`packages/core/src/aws/iam.ts:128`) lists and deletes them, because
|
|
1080
|
+
// IAM refuses to delete a role that still carries one, and swallows the
|
|
1081
|
+
// not-found so a half-finished teardown is re-runnable.
|
|
1082
|
+
await ctx.clients.iam.deleteRole(transformRoleName(ctx));
|
|
1083
|
+
},
|
|
1084
|
+
};
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Where this package's build put the transform's artifacts: the bundle and the
|
|
1088
|
+
* manifest stamped beside it. The package root comes from `paths.ts`, the one
|
|
1089
|
+
* module here allowed to resolve it; the directory under it is task 43's
|
|
1090
|
+
* {@link TRANSFORM_BUNDLE_DIR}, so neither half of the location is spelled
|
|
1091
|
+
* twice.
|
|
1092
|
+
*/
|
|
1093
|
+
function transformArtifactDir() {
|
|
1094
|
+
return join(ANALYTICS_PACKAGE_DIR, TRANSFORM_BUNDLE_DIR);
|
|
1095
|
+
}
|
|
1096
|
+
/** Raise for an artifact this package ships and does not have, naming what would produce it. */
|
|
1097
|
+
function missingArtifact(file, cause) {
|
|
1098
|
+
return new Error(`the analytics transform artifact "${file}" is not in ${transformArtifactDir()}, so there is no code to deploy - the package ships it, so this is an unbuilt checkout or a partial install; run \`pnpm --filter blogwright-analytics build\``, { cause });
|
|
1099
|
+
}
|
|
1100
|
+
/** Narrow parsed JSON to an object before reading a field off it - no cast, no `any`. */
|
|
1101
|
+
function isRecord(value) {
|
|
1102
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1103
|
+
}
|
|
1104
|
+
/**
|
|
1105
|
+
* Task 43's source hash and the zip key it derives, read from the manifest
|
|
1106
|
+
* beside the bundle.
|
|
1107
|
+
*
|
|
1108
|
+
* The manifest is read *beside* the zip and is never packed inside it: the
|
|
1109
|
+
* deployment package holds exactly one file (see
|
|
1110
|
+
* {@link TRANSFORM_BUNDLE_FILE}), and this is deploy-time metadata, not code.
|
|
1111
|
+
*
|
|
1112
|
+
* Only `hash` is read off the file. The manifest also carries `key`, and it is
|
|
1113
|
+
* deliberately not trusted: `transformZipKey` is the single derivation
|
|
1114
|
+
* (`transform-hash.ts`), and reading a stored key back would let the two
|
|
1115
|
+
* disagree - which is invisible, because a key that disagrees with its hash
|
|
1116
|
+
* still looks like a key. Calling the derivation is also what validates the
|
|
1117
|
+
* hash's shape, so a manifest carrying `"hash": "nope"` fails here rather than
|
|
1118
|
+
* producing a stable key that pins the deployed function at whatever code
|
|
1119
|
+
* shipped first.
|
|
1120
|
+
*
|
|
1121
|
+
* A hash is only ever compared to another hash from this same function, so the
|
|
1122
|
+
* *value* is never asserted against a literal anywhere - what matters is that
|
|
1123
|
+
* it moves when the source does, which `transform-hash.test.ts` owns.
|
|
1124
|
+
*/
|
|
1125
|
+
async function readTransformManifest(ctx) {
|
|
1126
|
+
const path = join(transformArtifactDir(), TRANSFORM_MANIFEST_FILE);
|
|
1127
|
+
let raw;
|
|
1128
|
+
try {
|
|
1129
|
+
raw = await ctx.ports.fs.readText(path);
|
|
1130
|
+
}
|
|
1131
|
+
catch (cause) {
|
|
1132
|
+
throw missingArtifact(TRANSFORM_MANIFEST_FILE, cause);
|
|
1133
|
+
}
|
|
1134
|
+
let parsed;
|
|
1135
|
+
try {
|
|
1136
|
+
parsed = JSON.parse(raw);
|
|
1137
|
+
}
|
|
1138
|
+
catch (cause) {
|
|
1139
|
+
throw new Error(`${path} is not valid JSON - rebuild the package to regenerate it`, { cause });
|
|
1140
|
+
}
|
|
1141
|
+
const hash = isRecord(parsed) ? parsed['hash'] : undefined;
|
|
1142
|
+
if (typeof hash !== 'string') {
|
|
1143
|
+
throw new Error(`${path} carries no "hash" string, so there is nothing to key the deployed function's code by - rebuild the package to regenerate it`);
|
|
1144
|
+
}
|
|
1145
|
+
return { hash, key: transformZipKey(hash) };
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* The deployment package: the bundled transform, read through the FileSystem
|
|
1149
|
+
* port and zipped the way `packageAndUploadAgent` zips the build agent
|
|
1150
|
+
* (`packages/cli/src/agent-package.ts:48-53`) - `zipSync` with a fixed `mtime`,
|
|
1151
|
+
* so identical bundle bytes always produce identical archive bytes.
|
|
1152
|
+
*
|
|
1153
|
+
* **Exactly one entry, under the bundle's own name**, never the artifact
|
|
1154
|
+
* directory wholesale. The `Handler` string the function is configured with is
|
|
1155
|
+
* `<module base name>.<export>` resolved against the root of the archive
|
|
1156
|
+
* (`TRANSFORM_LAMBDA_HANDLER`), and the manifest sitting next to the bundle on
|
|
1157
|
+
* disk is not code - packing it would put a stray file in the runtime's module
|
|
1158
|
+
* root for no reader.
|
|
1159
|
+
*
|
|
1160
|
+
* The bytes cross `ctx.ports.fs`, never Node's own filesystem module: this is a
|
|
1161
|
+
* domain module under DEVELOPMENT.md §Hexagonal architecture, and no
|
|
1162
|
+
* `packages/analytics/src/` path is in `.oxlintrc.json`'s
|
|
1163
|
+
* `no-restricted-imports` override list.
|
|
1164
|
+
*/
|
|
1165
|
+
async function packTransformBundle(ctx) {
|
|
1166
|
+
const path = join(transformArtifactDir(), TRANSFORM_BUNDLE_FILE);
|
|
1167
|
+
let bundle;
|
|
1168
|
+
try {
|
|
1169
|
+
bundle = await ctx.ports.fs.readBytes(path);
|
|
1170
|
+
}
|
|
1171
|
+
catch (cause) {
|
|
1172
|
+
throw missingArtifact(TRANSFORM_BUNDLE_FILE, cause);
|
|
1173
|
+
}
|
|
1174
|
+
const zip = zipSync({ [TRANSFORM_BUNDLE_FILE]: bundle }, { level: ZIP_LEVEL, mtime: ZIP_MTIME });
|
|
1175
|
+
if (zip.length > MAX_INLINE_ZIP_BYTES) {
|
|
1176
|
+
throw new Error(`the analytics transform's deployment package is ${zip.length} bytes, over the ${MAX_INLINE_ZIP_BYTES}-byte limit Lambda accepts for a zip sent inline - the code would have to move to an S3 code bucket in ${ANALYTICS_REGION}, which this pipeline deliberately does not have (see MAX_INLINE_ZIP_BYTES)`);
|
|
1177
|
+
}
|
|
1178
|
+
return zip;
|
|
1179
|
+
}
|
|
1180
|
+
/**
|
|
1181
|
+
* The function's version-specific settings, in the shape `CreateFunction` and
|
|
1182
|
+
* `UpdateFunctionConfiguration` share (`aws/lambda.ts`). Built in one place so
|
|
1183
|
+
* the create payload and the update payload cannot drift, and so
|
|
1184
|
+
* {@link configurationFingerprint} compares exactly what would be sent.
|
|
1185
|
+
*
|
|
1186
|
+
* Every value is a named module constant or a resolved input: the runtime, the
|
|
1187
|
+
* handler (task 43's `TRANSFORM_LAMBDA_HANDLER`, derived from the bundle's own
|
|
1188
|
+
* file name and export and spelled only there), the memory and the timeout are
|
|
1189
|
+
* constants; the role ARN comes from the node this one depends on; and the
|
|
1190
|
+
* environment carries the salt secret's **name** under
|
|
1191
|
+
* {@link SALT_SECRET_NAME_ENV} - `transform/handler.ts`'s own constant, so the
|
|
1192
|
+
* variable the function reads and the variable this node sets are one string.
|
|
1193
|
+
*
|
|
1194
|
+
* The *name* travels; the value never does. A function's configuration is
|
|
1195
|
+
* readable by anyone with `lambda:GetFunctionConfiguration`, so a secret in an
|
|
1196
|
+
* environment variable would be a secret in the console.
|
|
1197
|
+
*/
|
|
1198
|
+
function transformConfiguration(ctx) {
|
|
1199
|
+
return {
|
|
1200
|
+
roleArn: requireTransformRoleArn(ctx),
|
|
1201
|
+
runtime: TRANSFORM_RUNTIME,
|
|
1202
|
+
handler: TRANSFORM_LAMBDA_HANDLER,
|
|
1203
|
+
memoryMb: TRANSFORM_MEMORY_MB,
|
|
1204
|
+
timeoutSeconds: TRANSFORM_TIMEOUT_SECONDS,
|
|
1205
|
+
environment: { [SALT_SECRET_NAME_ENV]: resolveAnalyticsConfig(ctx).saltSecretName },
|
|
1206
|
+
};
|
|
1207
|
+
}
|
|
1208
|
+
/**
|
|
1209
|
+
* The deployed configuration, as one recorded value the next reconcile compares
|
|
1210
|
+
* against.
|
|
1211
|
+
*
|
|
1212
|
+
* One value rather than six recorded fields, because the question the update
|
|
1213
|
+
* asks is "is what is deployed what we would send now", and six fields compared
|
|
1214
|
+
* one by one is six chances to forget the seventh the day
|
|
1215
|
+
* `FunctionConfigurationInput` grows one. `JSON.stringify` over the very object
|
|
1216
|
+
* the client is handed is that question, spelled once; the key order is fixed
|
|
1217
|
+
* because {@link transformConfiguration} is the single literal that builds it.
|
|
1218
|
+
*
|
|
1219
|
+
* It is safe to write into `state/<env>.analytics.json` because the
|
|
1220
|
+
* configuration holds no secret - only the salt secret's *name*.
|
|
1221
|
+
*/
|
|
1222
|
+
function configurationFingerprint(input) {
|
|
1223
|
+
return JSON.stringify(input);
|
|
1224
|
+
}
|
|
1225
|
+
/** A string previously recorded under `key` for `nodeId`, or `undefined` if there is none. */
|
|
1226
|
+
function recordedText(ctx, nodeId, key) {
|
|
1227
|
+
const value = ctx.state.resources[nodeId]?.[key];
|
|
1228
|
+
return typeof value === 'string' ? value : undefined;
|
|
1229
|
+
}
|
|
1230
|
+
/**
|
|
1231
|
+
* Decide what an existing transform function's reconcile has to send, from what
|
|
1232
|
+
* is recorded against what the package now holds. Pure, so the decision is
|
|
1233
|
+
* testable without the AWS calls - `builderImageAction`'s shape
|
|
1234
|
+
* (`packages/cli/src/nodes.ts:311-321`) and its reason.
|
|
1235
|
+
*
|
|
1236
|
+
* **Two independent comparisons, not one.** An unchanged hash performs no code
|
|
1237
|
+
* call, which is the whole point of hashing the *source*: a rebuild on another
|
|
1238
|
+
* machine emits different bundle bytes from the same source, and keying the
|
|
1239
|
+
* decision on those bytes would redeploy the function on every platform switch
|
|
1240
|
+
* (`transform-hash.ts` argues this at length). Everything a code push could
|
|
1241
|
+
* need to accompany it is covered by that hash too, because `analytics/src` is
|
|
1242
|
+
* one of the hash's inputs - so the module constants above cannot change
|
|
1243
|
+
* without moving it.
|
|
1244
|
+
*
|
|
1245
|
+
* The configuration is compared separately because one of its inputs is *not*
|
|
1246
|
+
* in the hash and cannot be: `analytics.saltSecretName` comes from
|
|
1247
|
+
* `blogwright.config.json`, which nothing hashes. An operator who repoints it
|
|
1248
|
+
* gets a new secret, a role granted on the new ARN, and - without this
|
|
1249
|
+
* comparison - a function still reading the old name out of its environment:
|
|
1250
|
+
* every batch failing `GetSecretValue`, every record in Firehose's error
|
|
1251
|
+
* prefix. That is the same gap `builderImageAction` carries its second `logGroup`
|
|
1252
|
+
* limb for.
|
|
1253
|
+
*
|
|
1254
|
+
* Keeping them separate is also what keeps the common cases to a single call
|
|
1255
|
+
* each, which matters more than it looks: Lambda refuses a second update while
|
|
1256
|
+
* the first is still settling, so two calls in one reconcile have a window the
|
|
1257
|
+
* one-call cases do not.
|
|
1258
|
+
*/
|
|
1259
|
+
export function transformUpdate(recorded, desired) {
|
|
1260
|
+
return {
|
|
1261
|
+
code: recorded.sourceHash !== desired.sourceHash,
|
|
1262
|
+
configuration: recorded.configuration !== desired.configuration,
|
|
1263
|
+
};
|
|
1264
|
+
}
|
|
1265
|
+
/**
|
|
1266
|
+
* The record-transform Lambda: the function Firehose runs over every CloudFront
|
|
1267
|
+
* record before it reaches the `page_views` table.
|
|
1268
|
+
*
|
|
1269
|
+
* Its code is keyed by task 43's hash of the transform's **source**, recorded
|
|
1270
|
+
* in the plugin's own state, so identical source never redeploys it - and the
|
|
1271
|
+
* key that hash derives (`transformZipKey`) is recorded beside it as the
|
|
1272
|
+
* artifact's name, even though the zip travels inline rather than through a
|
|
1273
|
+
* bucket (see {@link MAX_INLINE_ZIP_BYTES} for why inline).
|
|
1274
|
+
*
|
|
1275
|
+
* It depends on `analytics-transform-role`, whose recorded ARN it runs as.
|
|
1276
|
+
*/
|
|
1277
|
+
export function analyticsTransformFunctionNode() {
|
|
1278
|
+
return {
|
|
1279
|
+
id: TRANSFORM_FUNCTION_NODE,
|
|
1280
|
+
dependsOn: [TRANSFORM_ROLE_NODE],
|
|
1281
|
+
title: `Record-transform Lambda (${ANALYTICS_REGION})`,
|
|
1282
|
+
async read(ctx) {
|
|
1283
|
+
const name = transformFunctionName(ctx);
|
|
1284
|
+
const fn = await lambda(ctx).getFunction(name);
|
|
1285
|
+
if (fn === undefined)
|
|
1286
|
+
return false;
|
|
1287
|
+
if (fn.state === 'failed') {
|
|
1288
|
+
// Not adopted as reconciled, and not reported as absent either.
|
|
1289
|
+
// Reporting absence would send `applyGraph` to `create`, whose 409 is
|
|
1290
|
+
// swallowed as "already exists" (`aws/lambda.ts`), and the reconcile
|
|
1291
|
+
// would go green over a function that cannot run - with every record
|
|
1292
|
+
// landing in Firehose's error prefix and an empty dashboard as the only
|
|
1293
|
+
// symptom.
|
|
1294
|
+
throw new Error(`the analytics transform Lambda "${name}" exists but is in the Failed state, so Firehose would route every record to the error prefix. Delete it (\`aws lambda delete-function --region ${ANALYTICS_REGION} --function-name ${name}\`) and re-run \`blogwright analytics bootstrap --env ${ctx.env}\` to recreate it.`);
|
|
1295
|
+
}
|
|
1296
|
+
const out = output(ctx, TRANSFORM_FUNCTION_NODE);
|
|
1297
|
+
out.name = name;
|
|
1298
|
+
// Guarded on the value: `normalizeFunction` falls back to `''` for a
|
|
1299
|
+
// response carrying no `Configuration.FunctionArn`, and an empty string
|
|
1300
|
+
// recorded under `arn` reads downstream as a real one.
|
|
1301
|
+
if (fn.arn)
|
|
1302
|
+
out.arn = fn.arn;
|
|
1303
|
+
// The source hash and the configuration fingerprint are deliberately NOT
|
|
1304
|
+
// hydrated here. They are this repo's record of what it deployed, and
|
|
1305
|
+
// Lambda cannot answer either: `CodeSha256` digests the built zip, which
|
|
1306
|
+
// is a different thing from a hash of the source (`aws/lambda.ts` drops
|
|
1307
|
+
// it for exactly this reason). Losing the state file therefore means the
|
|
1308
|
+
// next reconcile pushes both again - wasteful, never wrong.
|
|
1309
|
+
return true;
|
|
1310
|
+
},
|
|
1311
|
+
async create(ctx) {
|
|
1312
|
+
const name = transformFunctionName(ctx);
|
|
1313
|
+
const manifest = await readTransformManifest(ctx);
|
|
1314
|
+
const configuration = transformConfiguration(ctx);
|
|
1315
|
+
const zipFile = await packTransformBundle(ctx);
|
|
1316
|
+
const client = lambda(ctx);
|
|
1317
|
+
await client.createFunction({ name, zipFile, ...configuration });
|
|
1318
|
+
// Identity and code identity before the ARN lookup, the discipline
|
|
1319
|
+
// `analytics-table`'s `create` follows and for the same reason:
|
|
1320
|
+
// `createFunction` returns `void` by design (`aws/lambda.ts`), so the ARN
|
|
1321
|
+
// takes a second request, and a crash in between must not leave a
|
|
1322
|
+
// deployed function that state has no record of. Recording the hash here
|
|
1323
|
+
// rather than after the lookup is what stops the next reconcile
|
|
1324
|
+
// re-uploading identical code.
|
|
1325
|
+
const out = output(ctx, TRANSFORM_FUNCTION_NODE);
|
|
1326
|
+
out.name = name;
|
|
1327
|
+
out.sourceHash = manifest.hash;
|
|
1328
|
+
out.codeKey = manifest.key;
|
|
1329
|
+
out.configuration = configurationFingerprint(configuration);
|
|
1330
|
+
const created = await client.getFunction(name);
|
|
1331
|
+
if (created?.arn)
|
|
1332
|
+
out.arn = created.arn;
|
|
1333
|
+
},
|
|
1334
|
+
async update(ctx) {
|
|
1335
|
+
const manifest = await readTransformManifest(ctx);
|
|
1336
|
+
const configuration = transformConfiguration(ctx);
|
|
1337
|
+
const fingerprint = configurationFingerprint(configuration);
|
|
1338
|
+
const update = transformUpdate({
|
|
1339
|
+
sourceHash: recordedText(ctx, TRANSFORM_FUNCTION_NODE, 'sourceHash'),
|
|
1340
|
+
configuration: recordedText(ctx, TRANSFORM_FUNCTION_NODE, 'configuration'),
|
|
1341
|
+
}, { sourceHash: manifest.hash, configuration: fingerprint });
|
|
1342
|
+
// Nothing moved: no AWS call at all, which is what makes reconciling on
|
|
1343
|
+
// every deploy cheap. The manifest read above is a local file.
|
|
1344
|
+
if (!update.code && !update.configuration)
|
|
1345
|
+
return;
|
|
1346
|
+
const name = transformFunctionName(ctx);
|
|
1347
|
+
const client = lambda(ctx);
|
|
1348
|
+
const out = output(ctx, TRANSFORM_FUNCTION_NODE);
|
|
1349
|
+
// Configuration first when both moved. Lambda refuses a second update
|
|
1350
|
+
// while the first is settling (`ResourceConflictException`, which
|
|
1351
|
+
// `aws/lambda.ts` deliberately does not swallow - on this operation it
|
|
1352
|
+
// means "in progress", not "already done"), so if one of the two is going
|
|
1353
|
+
// to fail it is the second, and the survivable half-state is the one
|
|
1354
|
+
// where the old code runs under the new settings: the old code reading
|
|
1355
|
+
// the new secret name works, while new code reading the old name is
|
|
1356
|
+
// denied by the very role this graph just narrowed. Each half is recorded
|
|
1357
|
+
// as soon as it is true, so the retry after such a failure sends only the
|
|
1358
|
+
// half that did not land.
|
|
1359
|
+
if (update.configuration) {
|
|
1360
|
+
await client.updateFunctionConfiguration(name, configuration);
|
|
1361
|
+
out.configuration = fingerprint;
|
|
1362
|
+
}
|
|
1363
|
+
if (update.code) {
|
|
1364
|
+
await client.updateFunctionCode(name, await packTransformBundle(ctx));
|
|
1365
|
+
out.sourceHash = manifest.hash;
|
|
1366
|
+
out.codeKey = manifest.key;
|
|
1367
|
+
}
|
|
1368
|
+
},
|
|
1369
|
+
async delete(ctx) {
|
|
1370
|
+
// No-op when the function is already gone (`aws/lambda.ts` swallows the
|
|
1371
|
+
// 404 and nothing else), so a half-finished teardown is re-runnable.
|
|
1372
|
+
// `destroyGraph` walks the chain in reverse, so this runs before
|
|
1373
|
+
// `analytics-transform-role` removes the role it runs as.
|
|
1374
|
+
await lambda(ctx).deleteFunction(transformFunctionName(ctx));
|
|
1375
|
+
},
|
|
1376
|
+
};
|
|
1377
|
+
}
|
|
1378
|
+
/* ------------------------------------------------------------------------- *
|
|
1379
|
+
* The delivery chain: the error bucket, the delivery role, and the stream.
|
|
1380
|
+
* ------------------------------------------------------------------------- */
|
|
1381
|
+
/**
|
|
1382
|
+
* The suffix the Firehose failed-record bucket's name carries, appended to
|
|
1383
|
+
* `ctx.names.prefix`. See {@link transformFunctionName} for why the prefix is
|
|
1384
|
+
* the source of the environment here and {@link resolveAnalyticsConfig} is not:
|
|
1385
|
+
* the `analytics` block owns six settings and this is not one of them, so there
|
|
1386
|
+
* is no operator override to honour and the environment still leads the name.
|
|
1387
|
+
*/
|
|
1388
|
+
const ERROR_BUCKET_SUFFIX = '-analytics-errors';
|
|
1389
|
+
/** The suffix the Firehose delivery role carries. See {@link ERROR_BUCKET_SUFFIX}. */
|
|
1390
|
+
const FIREHOSE_ROLE_SUFFIX = '-analytics-firehose-role';
|
|
1391
|
+
/** The suffix the delivery stream carries. See {@link ERROR_BUCKET_SUFFIX}. */
|
|
1392
|
+
const FIREHOSE_STREAM_SUFFIX = '-analytics-firehose';
|
|
1393
|
+
/**
|
|
1394
|
+
* The longest name S3 accepts for a bucket - the same limit `deriveNames`
|
|
1395
|
+
* enforces on the site's own bucket (`packages/core/src/config.ts:355`) and
|
|
1396
|
+
* `resolveAnalyticsConfig` on the table bucket, restated here because this is a
|
|
1397
|
+
* third bucket name derived in a third place.
|
|
1398
|
+
*/
|
|
1399
|
+
const ERROR_BUCKET_NAME_MAX_LENGTH = 63;
|
|
1400
|
+
/** The longest name Firehose accepts for a delivery stream. */
|
|
1401
|
+
const STREAM_NAME_MAX_LENGTH = 64;
|
|
1402
|
+
/** The name of the inline policy this plugin puts on its own Firehose delivery role. */
|
|
1403
|
+
const FIREHOSE_ROLE_POLICY = 'firehose-delivery';
|
|
1404
|
+
/**
|
|
1405
|
+
* The trust document the Firehose delivery role is created with, verified
|
|
1406
|
+
* against AWS's own "Allow Firehose to assume an IAM role"
|
|
1407
|
+
* (`firehose/latest/dev/controlling-access.html`), which is one statement
|
|
1408
|
+
* granting `sts:AssumeRole` to `firehose.amazonaws.com`.
|
|
1409
|
+
*
|
|
1410
|
+
* Deliberately **not** {@link LAMBDA_TRUST} with a swapped principal, and
|
|
1411
|
+
* deliberately without that document's `sts:TagSession`: Firehose assumes this
|
|
1412
|
+
* role on its own schedule with no session tags to pass, so granting the action
|
|
1413
|
+
* would widen the trust for a caller that never uses it. The two documents are
|
|
1414
|
+
* allowed to drift for the same reason the transform's own copy is allowed to
|
|
1415
|
+
* drift from the CLI's - each one names the principal *its* resource assumes.
|
|
1416
|
+
*/
|
|
1417
|
+
const FIREHOSE_TRUST = {
|
|
1418
|
+
Version: POLICY_VERSION,
|
|
1419
|
+
Statement: [
|
|
1420
|
+
{
|
|
1421
|
+
Effect: 'Allow',
|
|
1422
|
+
Principal: { Service: 'firehose.amazonaws.com' },
|
|
1423
|
+
Action: ['sts:AssumeRole'],
|
|
1424
|
+
},
|
|
1425
|
+
],
|
|
1426
|
+
};
|
|
1427
|
+
/**
|
|
1428
|
+
* The key prefix under {@link errorBucketName} that failed records land at.
|
|
1429
|
+
*
|
|
1430
|
+
* One prefix serves both of Firehose's two distinct error surfaces - the
|
|
1431
|
+
* stream-level `S3Configuration.ErrorOutputPrefix` for a record that never
|
|
1432
|
+
* reached a table, and the table-level
|
|
1433
|
+
* `DestinationTableConfiguration.S3ErrorOutputPrefix` for one the table's
|
|
1434
|
+
* schema rejected (`aws/firehose.ts` spells out the difference). One stream
|
|
1435
|
+
* writes to one table, so separating them would sort records by a distinction
|
|
1436
|
+
* that has no second case on this side of it.
|
|
1437
|
+
*/
|
|
1438
|
+
const ERROR_OUTPUT_PREFIX = 'firehose-errors/';
|
|
1439
|
+
/**
|
|
1440
|
+
* Seconds Firehose buffers records before writing a file, and the size in MiB
|
|
1441
|
+
* that would flush one sooner. Both are sent, because the service requires the
|
|
1442
|
+
* pair when either is given (`BufferingHints`); both are at the service's
|
|
1443
|
+
* documented maximum (900 seconds, 128 MiB), and the pair is chosen together.
|
|
1444
|
+
*
|
|
1445
|
+
* At a blog's volume the size bound is unreachable, so the interval alone
|
|
1446
|
+
* governs and every flush is time-driven. The maximum interval therefore
|
|
1447
|
+
* produces the largest files this stream can produce, which is exactly what the
|
|
1448
|
+
* change spec's own cost assumption wants - "log volume for a blog is small
|
|
1449
|
+
* enough that batched Firehose delivery produces files large enough not to make
|
|
1450
|
+
* S3 Tables compaction the dominant cost". Fifteen minutes of delivery latency
|
|
1451
|
+
* is far inside what a day-partitioned dashboard needs; trading it for smaller,
|
|
1452
|
+
* more numerous Iceberg data files would buy freshness nothing here reads.
|
|
1453
|
+
*/
|
|
1454
|
+
const STREAM_BUFFER_INTERVAL_SECONDS = 900;
|
|
1455
|
+
/** See {@link STREAM_BUFFER_INTERVAL_SECONDS} - the two are chosen as a pair. */
|
|
1456
|
+
const STREAM_BUFFER_SIZE_MB = 128;
|
|
1457
|
+
/**
|
|
1458
|
+
* The plugin's own S3 client - **core's `S3Client` built over the pinned
|
|
1459
|
+
* signer, never `ctx.clients.s3`**, which core constructs over the
|
|
1460
|
+
* primary-region signer (`packages/core/src/clients.ts`). The same choice
|
|
1461
|
+
* {@link secrets} makes and for a sharper reason: an S3 bucket is created in
|
|
1462
|
+
* the region its request is signed for, so reaching for the host's copy would
|
|
1463
|
+
* put the error bucket in `config.region` while the stream writing to it is
|
|
1464
|
+
* pinned to {@link ANALYTICS_REGION}.
|
|
1465
|
+
*/
|
|
1466
|
+
function s3(ctx) {
|
|
1467
|
+
return createAnalyticsClients(ctx).s3;
|
|
1468
|
+
}
|
|
1469
|
+
/**
|
|
1470
|
+
* The plugin's own Firehose client, built the same way {@link s3tables},
|
|
1471
|
+
* {@link glue} and {@link lambda} are: core's bundle enumerates no `firehose`
|
|
1472
|
+
* key at all, and every client this plugin uses signs in
|
|
1473
|
+
* {@link ANALYTICS_REGION}.
|
|
1474
|
+
*/
|
|
1475
|
+
function firehose(ctx) {
|
|
1476
|
+
return createAnalyticsClients(ctx).firehose;
|
|
1477
|
+
}
|
|
1478
|
+
/** The Firehose failed-record bucket's name. See {@link ERROR_BUCKET_SUFFIX}. */
|
|
1479
|
+
function errorBucketName(ctx) {
|
|
1480
|
+
return boundedName(`${ctx.names.prefix}${ERROR_BUCKET_SUFFIX}`, ERROR_BUCKET_NAME_MAX_LENGTH, 'error bucket');
|
|
1481
|
+
}
|
|
1482
|
+
/** The Firehose delivery role's name. See {@link ERROR_BUCKET_SUFFIX}. */
|
|
1483
|
+
function firehoseRoleName(ctx) {
|
|
1484
|
+
return boundedName(`${ctx.names.prefix}${FIREHOSE_ROLE_SUFFIX}`, ROLE_NAME_MAX_LENGTH, 'Firehose delivery role');
|
|
1485
|
+
}
|
|
1486
|
+
/** The delivery stream's name. See {@link ERROR_BUCKET_SUFFIX}. */
|
|
1487
|
+
function streamName(ctx) {
|
|
1488
|
+
return boundedName(`${ctx.names.prefix}${FIREHOSE_STREAM_SUFFIX}`, STREAM_NAME_MAX_LENGTH, 'delivery stream');
|
|
1489
|
+
}
|
|
1490
|
+
/**
|
|
1491
|
+
* The error bucket's ARN as `analytics-error-bucket` recorded it. See
|
|
1492
|
+
* {@link requireRecordedArn}.
|
|
1493
|
+
*
|
|
1494
|
+
* Read back rather than re-derived even though an S3 bucket ARN *is* derivable
|
|
1495
|
+
* from its name, and that is the point: the read is what makes the declared
|
|
1496
|
+
* edge load-bearing. The role declares `analytics-error-bucket` directly; the
|
|
1497
|
+
* stream inherits the same ordering transitively through its edge on the role,
|
|
1498
|
+
* which is the spec's own `error-bucket -> firehose-role -> firehose-stream`
|
|
1499
|
+
* chain. Deriving here instead would let either of them name a bucket that does
|
|
1500
|
+
* not exist yet - and Firehose accepts a `BucketARN` for a bucket it cannot
|
|
1501
|
+
* write to, so the first symptom would be records failing to a bucket that was
|
|
1502
|
+
* never created.
|
|
1503
|
+
*/
|
|
1504
|
+
function requireErrorBucketArn(ctx, dependent) {
|
|
1505
|
+
return requireRecordedArn(ctx, {
|
|
1506
|
+
what: 'error bucket',
|
|
1507
|
+
node: ERROR_BUCKET_NODE,
|
|
1508
|
+
dependent,
|
|
1509
|
+
lack: "bucket to write Firehose's failed records to",
|
|
1510
|
+
});
|
|
1511
|
+
}
|
|
1512
|
+
/** The `page_views` table's ARN as `analytics-table` recorded it. See {@link requireRecordedArn}. */
|
|
1513
|
+
function requireTableArn(ctx) {
|
|
1514
|
+
return requireRecordedArn(ctx, {
|
|
1515
|
+
what: 'page_views table',
|
|
1516
|
+
node: TABLE_NODE,
|
|
1517
|
+
dependent: FIREHOSE_ROLE_NODE,
|
|
1518
|
+
lack: 'table to grant s3tables write access on',
|
|
1519
|
+
});
|
|
1520
|
+
}
|
|
1521
|
+
/**
|
|
1522
|
+
* The transform function's ARN as `analytics-transform-function` recorded it.
|
|
1523
|
+
* See {@link requireRecordedArn}. Read rather than re-derived from
|
|
1524
|
+
* {@link transformFunctionName}: the recorded value is the ARN Lambda itself
|
|
1525
|
+
* answered with, and it is what both readers need - the role grants
|
|
1526
|
+
* `lambda:InvokeFunction` on it and the stream's processor names it as
|
|
1527
|
+
* `LambdaArn`, so a derivation that drifted would produce a grant on one ARN
|
|
1528
|
+
* and an invoke of another.
|
|
1529
|
+
*/
|
|
1530
|
+
function requireTransformFunctionArn(ctx, dependent) {
|
|
1531
|
+
return requireRecordedArn(ctx, {
|
|
1532
|
+
what: 'transform function',
|
|
1533
|
+
node: TRANSFORM_FUNCTION_NODE,
|
|
1534
|
+
dependent,
|
|
1535
|
+
lack: 'record-transform Lambda to run every record through',
|
|
1536
|
+
});
|
|
1537
|
+
}
|
|
1538
|
+
/** The delivery role's ARN as `analytics-firehose-role` recorded it. See {@link requireRecordedArn}. */
|
|
1539
|
+
function requireFirehoseRoleArn(ctx) {
|
|
1540
|
+
return requireRecordedArn(ctx, {
|
|
1541
|
+
what: 'Firehose delivery role',
|
|
1542
|
+
node: FIREHOSE_ROLE_NODE,
|
|
1543
|
+
dependent: FIREHOSE_STREAM_NODE,
|
|
1544
|
+
lack: 'role for Firehose to assume',
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
/**
|
|
1548
|
+
* The Glue catalog ARN Firehose reaches this environment's Iceberg table
|
|
1549
|
+
* through: the **child** catalog the S3 Tables integration creates per table
|
|
1550
|
+
* bucket, `arn:aws:glue:<region>:<account>:catalog/s3tablescatalog/<bucket>`.
|
|
1551
|
+
*
|
|
1552
|
+
* Derived rather than read off `analytics-catalog-integration`'s recorded ARN,
|
|
1553
|
+
* which is a different string - that node adopts the account-wide federation
|
|
1554
|
+
* root (`.../catalog/s3tablescatalog`), one level above this. So the stream's
|
|
1555
|
+
* edge on that node is an *existence* dependency, not an interpolation one: the
|
|
1556
|
+
* federation has to be enabled before Firehose can resolve this child catalog,
|
|
1557
|
+
* and there is nothing recorded there to interpolate.
|
|
1558
|
+
*
|
|
1559
|
+
* The bare `arn:aws:glue:<region>:<account>:catalog` form - which is what
|
|
1560
|
+
* `CatalogConfiguration.CatalogARN`'s prose names - is the account's own Data
|
|
1561
|
+
* Catalog and holds no S3 Tables table at all. The field's pattern allows up to
|
|
1562
|
+
* two further segments precisely so this form fits; `aws/firehose.ts` records
|
|
1563
|
+
* that on the field itself.
|
|
1564
|
+
*/
|
|
1565
|
+
function federatedCatalogArn(ctx) {
|
|
1566
|
+
const bucket = resolveAnalyticsConfig(ctx).tableBucket;
|
|
1567
|
+
return `arn:aws:glue:${ANALYTICS_REGION}:${ctx.accountId}:catalog/${CATALOG_NAME}/${bucket}`;
|
|
1568
|
+
}
|
|
1569
|
+
/**
|
|
1570
|
+
* The five concrete Glue resources the delivery role's catalog grant names,
|
|
1571
|
+
* following AWS's own S3 Tables delivery policy - which writes the last three
|
|
1572
|
+
* with account-wide wildcards for the child catalog, the database and the
|
|
1573
|
+
* table, and is narrowed here to this environment's own table bucket, namespace
|
|
1574
|
+
* and table.
|
|
1575
|
+
*
|
|
1576
|
+
* Every level of the hierarchy has to be named because Glue authorises the walk
|
|
1577
|
+
* down it, not just the leaf: the account catalog, the federation root, this
|
|
1578
|
+
* table bucket's child catalog, the namespace as a database, and the table.
|
|
1579
|
+
* Dropping a level does not produce a smaller working grant - it produces a
|
|
1580
|
+
* `GetTable` that is denied, which Firehose reports by routing every record to
|
|
1581
|
+
* the error bucket.
|
|
1582
|
+
*/
|
|
1583
|
+
function glueGrantResources(ctx) {
|
|
1584
|
+
const analytics = resolveAnalyticsConfig(ctx);
|
|
1585
|
+
const glueArn = `arn:aws:glue:${ANALYTICS_REGION}:${ctx.accountId}`;
|
|
1586
|
+
const child = `${CATALOG_NAME}/${analytics.tableBucket}`;
|
|
1587
|
+
return [
|
|
1588
|
+
`${glueArn}:catalog`,
|
|
1589
|
+
`${glueArn}:catalog/${CATALOG_NAME}`,
|
|
1590
|
+
`${glueArn}:catalog/${child}`,
|
|
1591
|
+
`${glueArn}:database/${child}/${analytics.namespace}`,
|
|
1592
|
+
`${glueArn}:table/${child}/${analytics.namespace}/${analytics.table}`,
|
|
1593
|
+
];
|
|
1594
|
+
}
|
|
1595
|
+
/**
|
|
1596
|
+
* Apply the delivery role's inline policy. Shared by `create` and `update` -
|
|
1597
|
+
* the `applyExecRolePolicy` pattern (`packages/cli/src/nodes.ts:180-216`) - so
|
|
1598
|
+
* a reconcile of an existing role rewrites the same document a fresh one gets,
|
|
1599
|
+
* and a table recreated under a new generated ARN reaches the policy without a
|
|
1600
|
+
* teardown.
|
|
1601
|
+
*
|
|
1602
|
+
* **Exactly four statements, one per capability the change spec names, every
|
|
1603
|
+
* `Resource` a concrete ARN and none of them `*`.** The action lists are AWS's
|
|
1604
|
+
* own, from the "Grant Firehose access to Amazon S3 Tables" policy under IAM
|
|
1605
|
+
* access control; what is narrowed is the resources, which that policy writes
|
|
1606
|
+
* with wildcards over the whole account.
|
|
1607
|
+
*
|
|
1608
|
+
* Two of the four are easy to get subtly wrong and are worth stating:
|
|
1609
|
+
*
|
|
1610
|
+
* - the error-bucket statement names the bucket **and** `<bucket>/*`. Bucket
|
|
1611
|
+
* actions (`s3:ListBucket`, `s3:GetBucketLocation`) authorise against the
|
|
1612
|
+
* bucket ARN and object actions (`s3:PutObject`) against the key ARN, and
|
|
1613
|
+
* neither ARN matches the other. With only the bucket named, `PutObject`
|
|
1614
|
+
* would be denied and every failed record would be lost outright - which is
|
|
1615
|
+
* the one failure this whole bucket exists to make recoverable.
|
|
1616
|
+
* - the lambda statement names the transform's **unqualified** function ARN,
|
|
1617
|
+
* the one `analytics-transform-function` recorded, because that is the exact
|
|
1618
|
+
* string the stream sends as `LambdaArn`. AWS's example writes a
|
|
1619
|
+
* `:<version>`-qualified ARN; a qualified resource does not match an
|
|
1620
|
+
* unqualified invoke, so copying it would deny every transform call and send
|
|
1621
|
+
* every record to the error bucket.
|
|
1622
|
+
*
|
|
1623
|
+
* There is no fifth statement. AWS's policy carries three more - Kinesis (this
|
|
1624
|
+
* stream is `DirectPut`), KMS (no customer-managed key is configured anywhere
|
|
1625
|
+
* in this pipeline) and CloudWatch Logs (no `CloudWatchLoggingOptions` is sent,
|
|
1626
|
+
* so Firehose writes no log stream to grant on) - and each of the three is
|
|
1627
|
+
* conditional on a feature this pipeline does not use.
|
|
1628
|
+
*/
|
|
1629
|
+
async function applyFirehoseRolePolicy(ctx) {
|
|
1630
|
+
const errorBucketArn = requireErrorBucketArn(ctx, FIREHOSE_ROLE_NODE);
|
|
1631
|
+
await ctx.clients.iam.putRolePolicy(firehoseRoleName(ctx), FIREHOSE_ROLE_POLICY, {
|
|
1632
|
+
Version: POLICY_VERSION,
|
|
1633
|
+
Statement: [
|
|
1634
|
+
{
|
|
1635
|
+
Effect: 'Allow',
|
|
1636
|
+
Action: [
|
|
1637
|
+
'glue:GetDatabase',
|
|
1638
|
+
'glue:GetDatabases',
|
|
1639
|
+
'glue:GetTable',
|
|
1640
|
+
'glue:GetTables',
|
|
1641
|
+
'glue:UpdateTable',
|
|
1642
|
+
],
|
|
1643
|
+
Resource: glueGrantResources(ctx),
|
|
1644
|
+
},
|
|
1645
|
+
{
|
|
1646
|
+
Effect: 'Allow',
|
|
1647
|
+
Action: [
|
|
1648
|
+
's3tables:GetTableBucket',
|
|
1649
|
+
's3tables:GetNamespace',
|
|
1650
|
+
's3tables:GetTable',
|
|
1651
|
+
's3tables:GetTableData',
|
|
1652
|
+
's3tables:GetTableMetadataLocation',
|
|
1653
|
+
's3tables:PutTableData',
|
|
1654
|
+
's3tables:UpdateTableMetadataLocation',
|
|
1655
|
+
],
|
|
1656
|
+
Resource: [tableBucketArn(ctx), requireTableArn(ctx)],
|
|
1657
|
+
},
|
|
1658
|
+
{
|
|
1659
|
+
// `lambda:GetFunctionConfiguration` travels with the invoke in AWS's own
|
|
1660
|
+
// single statement for this capability: Firehose reads the function's
|
|
1661
|
+
// timeout before it invokes, so an invoke-only grant leaves the
|
|
1662
|
+
// processor unusable rather than merely unobservable.
|
|
1663
|
+
Effect: 'Allow',
|
|
1664
|
+
Action: ['lambda:InvokeFunction', 'lambda:GetFunctionConfiguration'],
|
|
1665
|
+
Resource: requireTransformFunctionArn(ctx, FIREHOSE_ROLE_NODE),
|
|
1666
|
+
},
|
|
1667
|
+
{
|
|
1668
|
+
// The plugin's OWN error bucket, never the site's environment bucket -
|
|
1669
|
+
// the one the CLI's `bucketNode` creates off `ctx.names`. That one sits
|
|
1670
|
+
// in `config.region` while this stream is pinned to us-east-1, and an S3
|
|
1671
|
+
// ARN carries no region, so the API can neither express nor reject the
|
|
1672
|
+
// mismatch. See `analyticsErrorBucketNode`. A schema mismatch sends
|
|
1673
|
+
// *every* affected record here, so this is a normal path, not a rare one.
|
|
1674
|
+
Effect: 'Allow',
|
|
1675
|
+
Action: [
|
|
1676
|
+
's3:AbortMultipartUpload',
|
|
1677
|
+
's3:GetBucketLocation',
|
|
1678
|
+
's3:GetObject',
|
|
1679
|
+
's3:ListBucket',
|
|
1680
|
+
's3:ListBucketMultipartUploads',
|
|
1681
|
+
's3:PutObject',
|
|
1682
|
+
],
|
|
1683
|
+
Resource: [errorBucketArn, `${errorBucketArn}/*`],
|
|
1684
|
+
},
|
|
1685
|
+
],
|
|
1686
|
+
});
|
|
1687
|
+
}
|
|
1688
|
+
/**
|
|
1689
|
+
* The Iceberg destination the stream is created and reconciled against, built
|
|
1690
|
+
* in one place so the create payload and the `UpdateDestination` payload cannot
|
|
1691
|
+
* drift - the reason {@link transformConfiguration} exists for the transform
|
|
1692
|
+
* function, and the same reason.
|
|
1693
|
+
*
|
|
1694
|
+
* Every resource in it is either a recorded output of a node this one declares
|
|
1695
|
+
* an edge to, or a derivation from the resolved analytics config; nothing is a
|
|
1696
|
+
* re-derived name.
|
|
1697
|
+
*/
|
|
1698
|
+
function firehoseDestination(ctx) {
|
|
1699
|
+
const analytics = resolveAnalyticsConfig(ctx);
|
|
1700
|
+
return {
|
|
1701
|
+
catalogArn: federatedCatalogArn(ctx),
|
|
1702
|
+
roleArn: requireFirehoseRoleArn(ctx),
|
|
1703
|
+
namespace: analytics.namespace,
|
|
1704
|
+
tableName: analytics.table,
|
|
1705
|
+
// The plugin's own bucket, in us-east-1 with the rest of the pipeline -
|
|
1706
|
+
// never the site's environment bucket, the one the CLI's `bucketNode` owns,
|
|
1707
|
+
// which lives in `config.region`. `S3DestinationConfiguration.BucketARN` matches
|
|
1708
|
+
// `arn:.*:s3:::[\w\.\-]{1,255}`: an S3 ARN carries no region, so the API can
|
|
1709
|
+
// neither express the mismatch nor reject it, and Firehose's cross-region
|
|
1710
|
+
// documentation covers only HTTP endpoint destinations. A schema mismatch
|
|
1711
|
+
// sends *every* affected record here, so this is a normal path rather than a
|
|
1712
|
+
// rare one, and resting it on undocumented behaviour is what the plugin's
|
|
1713
|
+
// own bucket exists to avoid.
|
|
1714
|
+
errorBucketArn: requireErrorBucketArn(ctx, FIREHOSE_STREAM_NODE),
|
|
1715
|
+
errorOutputPrefix: ERROR_OUTPUT_PREFIX,
|
|
1716
|
+
bufferIntervalSeconds: STREAM_BUFFER_INTERVAL_SECONDS,
|
|
1717
|
+
bufferSizeMb: STREAM_BUFFER_SIZE_MB,
|
|
1718
|
+
transformLambdaArn: requireTransformFunctionArn(ctx, FIREHOSE_STREAM_NODE),
|
|
1719
|
+
};
|
|
1720
|
+
}
|
|
1721
|
+
/**
|
|
1722
|
+
* Record `value` under `key`, or remove a stale entry when the response carried
|
|
1723
|
+
* none.
|
|
1724
|
+
*
|
|
1725
|
+
* The removal is the half that matters. {@link output} re-records rather than
|
|
1726
|
+
* replaces, so a `failure` left over from a create that failed on a KMS error
|
|
1727
|
+
* would outlive the recovery and `analytics status` would go on reporting it;
|
|
1728
|
+
* an `appendOnly` left over from a describe that stopped reporting the flag
|
|
1729
|
+
* would make the reconcile below skip work it should do. Absent in the response
|
|
1730
|
+
* has to mean absent in state, which is the same rule the ARN guards in this
|
|
1731
|
+
* module state as "never record `''` as though it were an ARN" - one direction
|
|
1732
|
+
* each of the same discipline.
|
|
1733
|
+
*/
|
|
1734
|
+
function recordOptional(out, key, value) {
|
|
1735
|
+
if (value === undefined)
|
|
1736
|
+
delete out[key];
|
|
1737
|
+
else
|
|
1738
|
+
out[key] = value;
|
|
1739
|
+
}
|
|
1740
|
+
/**
|
|
1741
|
+
* Record the delivery stream's identity and health from a `DescribeDeliveryStream`.
|
|
1742
|
+
*
|
|
1743
|
+
* `state` and `failure` are what `analytics status` reports (task 55), so the
|
|
1744
|
+
* stream's health is hydrated by the same `read` the reconcile runs and there is
|
|
1745
|
+
* no second describe path. `versionId` and `destinationId` are what
|
|
1746
|
+
* `UpdateDestination` cannot be called without, and `appendOnly` is the live
|
|
1747
|
+
* flag the reconcile compares against {@link STREAM_APPEND_ONLY}.
|
|
1748
|
+
*
|
|
1749
|
+
* The ARN is guarded on its value, the guard `analytics-table` and
|
|
1750
|
+
* `analytics-catalog-integration` both put on theirs: `describeDeliveryStream`
|
|
1751
|
+
* falls back to `''` for a body carrying no `DeliveryStreamARN`, and an empty
|
|
1752
|
+
* string recorded under `arn` reads downstream as a real one.
|
|
1753
|
+
*/
|
|
1754
|
+
function recordStream(ctx, status) {
|
|
1755
|
+
const out = output(ctx, FIREHOSE_STREAM_NODE);
|
|
1756
|
+
out.name = status.name;
|
|
1757
|
+
out.state = status.state;
|
|
1758
|
+
recordOptional(out, 'arn', status.arn === '' ? undefined : status.arn);
|
|
1759
|
+
recordOptional(out, 'versionId', status.versionId);
|
|
1760
|
+
recordOptional(out, 'destinationId', status.destinationId);
|
|
1761
|
+
recordOptional(out, 'appendOnly', status.appendOnly);
|
|
1762
|
+
recordOptional(out, 'failure', status.failure);
|
|
1763
|
+
}
|
|
1764
|
+
/**
|
|
1765
|
+
* Create the stream, record what exists as soon as it exists, then hydrate
|
|
1766
|
+
* everything only a describe can supply - and refuse to report success over a
|
|
1767
|
+
* stream that is not actually being created.
|
|
1768
|
+
*
|
|
1769
|
+
* **Record ordering**, which this module deliberately decides per node: the
|
|
1770
|
+
* name goes into state the moment `createDeliveryStream` returns, before the
|
|
1771
|
+
* describe, because `createDeliveryStream` answers with no ARN by design
|
|
1772
|
+
* (`aws/firehose.ts`) and a crash between the two calls must still leave the
|
|
1773
|
+
* stream recorded for `destroy` to remove. That is `analytics-table`'s ordering
|
|
1774
|
+
* and its reason. It is *not* `analytics-transform-function`'s, which also
|
|
1775
|
+
* records its source hash and configuration fingerprint before the lookup -
|
|
1776
|
+
* those are inputs it must not re-send, and this node has no equivalent to
|
|
1777
|
+
* protect. It is also not `analytics-catalog-integration`'s, which records
|
|
1778
|
+
* nothing at all until a check has passed, because that node adopts shared
|
|
1779
|
+
* state and this one owns what it creates.
|
|
1780
|
+
*
|
|
1781
|
+
* The guard at the end closes a hole that `createDeliveryStream`'s own
|
|
1782
|
+
* idempotency opens on the replacement path. That method swallows
|
|
1783
|
+
* `ResourceInUseException` as "already exists", which is right when a re-run
|
|
1784
|
+
* finds the stream it made last time - and wrong immediately after a delete,
|
|
1785
|
+
* where the same exception means the *old* stream is still `DELETING`. Without
|
|
1786
|
+
* this check the reconcile would report a replacement as done while the account
|
|
1787
|
+
* held a stream that was on its way out, and the first symptom would be an
|
|
1788
|
+
* empty dashboard. Re-running the bootstrap once the delete has settled is the
|
|
1789
|
+
* fix, and the message says so.
|
|
1790
|
+
*/
|
|
1791
|
+
async function createStream(ctx, client, name, destination) {
|
|
1792
|
+
await client.createDeliveryStream(name, destination, ctx.tags);
|
|
1793
|
+
output(ctx, FIREHOSE_STREAM_NODE).name = name;
|
|
1794
|
+
const created = await client.describeDeliveryStream(name);
|
|
1795
|
+
if (created !== undefined)
|
|
1796
|
+
recordStream(ctx, created);
|
|
1797
|
+
if (created === undefined || created.state === 'deleting' || created.state === 'delete-failed') {
|
|
1798
|
+
throw new Error(`the analytics delivery stream "${name}" is ${created === undefined ? 'not readable' : `still ${created.state}`} after CreateDeliveryStream reported success, so no stream is accepting records - a delete of the previous stream has not settled yet. Re-run \`blogwright analytics bootstrap --env ${ctx.env}\` in a minute.`);
|
|
1799
|
+
}
|
|
1800
|
+
}
|
|
1801
|
+
/**
|
|
1802
|
+
* The S3 bucket every record Firehose cannot deliver is written to - **the
|
|
1803
|
+
* physical place a silent pipeline failure becomes visible.**
|
|
1804
|
+
*
|
|
1805
|
+
* Firehose matches incoming JSON keys to the Iceberg column names exactly and
|
|
1806
|
+
* *errors* the records that do not match to this bucket rather than dropping
|
|
1807
|
+
* them (the change spec quotes the behaviour). So a missing column, a table
|
|
1808
|
+
* whose catalog cannot be read, a Glue grant one level too narrow - none of
|
|
1809
|
+
* them raise anything an operator sees. They fill this bucket while the
|
|
1810
|
+
* dashboard stays empty, and this bucket is the only place the records
|
|
1811
|
+
* themselves still exist. Two properties follow.
|
|
1812
|
+
*
|
|
1813
|
+
* **It is in {@link ANALYTICS_REGION}, with the rest of the pipeline**, created
|
|
1814
|
+
* through the plugin's own {@link s3} client rather than `ctx.clients.s3`,
|
|
1815
|
+
* which signs in `config.region`.
|
|
1816
|
+
*
|
|
1817
|
+
* **It is not the site's environment bucket.** The bucket the CLI's own
|
|
1818
|
+
* `bucketNode` creates lives in
|
|
1819
|
+
* `config.region` and `S3DestinationConfiguration.BucketARN` matches
|
|
1820
|
+
* `arn:.*:s3:::[\w\.\-]{1,255}` - an S3 ARN carries no region, so the API can
|
|
1821
|
+
* neither express a cross-region bucket nor reject one, and Firehose's
|
|
1822
|
+
* cross-region documentation covers only HTTP endpoint destinations. Pointing
|
|
1823
|
+
* at the site's bucket would therefore rest the pipeline's one recovery surface
|
|
1824
|
+
* on undocumented behaviour, and would put failed-record objects - which carry
|
|
1825
|
+
* the raw CloudFront fields, the viewer IP among them, precisely because the
|
|
1826
|
+
* transform did not run on them - inside a bucket the site serves from.
|
|
1827
|
+
*/
|
|
1828
|
+
export function analyticsErrorBucketNode() {
|
|
1829
|
+
return {
|
|
1830
|
+
id: ERROR_BUCKET_NODE,
|
|
1831
|
+
dependsOn: [],
|
|
1832
|
+
title: `Firehose failed-record bucket (${ANALYTICS_REGION})`,
|
|
1833
|
+
async read(ctx) {
|
|
1834
|
+
const name = errorBucketName(ctx);
|
|
1835
|
+
if (!(await s3(ctx).bucketExists(name)))
|
|
1836
|
+
return false;
|
|
1837
|
+
recordErrorBucket(ctx, name);
|
|
1838
|
+
return true;
|
|
1839
|
+
},
|
|
1840
|
+
async create(ctx) {
|
|
1841
|
+
const name = errorBucketName(ctx);
|
|
1842
|
+
const client = s3(ctx);
|
|
1843
|
+
await client.createBucket(name);
|
|
1844
|
+
// Identity before the secondary mutations, `bucketNode`'s ordering
|
|
1845
|
+
// (`packages/cli/src/nodes.ts:56-60`): a crash between CreateBucket and
|
|
1846
|
+
// the tagging/public-access calls must still leave the bucket recorded.
|
|
1847
|
+
recordErrorBucket(ctx, name);
|
|
1848
|
+
await applyErrorBucketConfiguration(ctx, name);
|
|
1849
|
+
},
|
|
1850
|
+
async update(ctx) {
|
|
1851
|
+
// Reconcile on every apply, for `bucketNode`'s reason: a bucket left by a
|
|
1852
|
+
// run that crashed before its tagging/PAB calls converges on the next one.
|
|
1853
|
+
await applyErrorBucketConfiguration(ctx, errorBucketName(ctx));
|
|
1854
|
+
},
|
|
1855
|
+
async delete(ctx) {
|
|
1856
|
+
const name = errorBucketName(ctx);
|
|
1857
|
+
const client = s3(ctx);
|
|
1858
|
+
// The existence check is what makes a re-run after a completed teardown a
|
|
1859
|
+
// no-op. `deleteBucket` swallows its own not-found, but `deletePrefix`
|
|
1860
|
+
// does not: it lists first, and `listObjects` rethrows, so a second
|
|
1861
|
+
// `analytics destroy` would fail on the half that was already done.
|
|
1862
|
+
if (!(await client.bucketExists(name)))
|
|
1863
|
+
return;
|
|
1864
|
+
// S3 refuses to delete a bucket that still holds objects, so the failed
|
|
1865
|
+
// records go first - and are counted, because they are the evidence of
|
|
1866
|
+
// whatever went wrong and an operator tearing the environment down is
|
|
1867
|
+
// owed the line that says how much of it was discarded.
|
|
1868
|
+
const removed = await client.deletePrefix(name, '');
|
|
1869
|
+
if (removed > 0) {
|
|
1870
|
+
ctx.logger.warn(`discarded ${removed} failed-record object(s) from "${name}" - these were the records Firehose could not deliver, and they are not recoverable after this`);
|
|
1871
|
+
}
|
|
1872
|
+
await client.deleteBucket(name);
|
|
1873
|
+
},
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1876
|
+
/**
|
|
1877
|
+
* Record the error bucket's identity. Shared by `read` and `create` for
|
|
1878
|
+
* {@link recordTableBucket}'s reason: both record the same two values from the
|
|
1879
|
+
* same two sources, and `bucketExists` answers with nothing to echo back.
|
|
1880
|
+
*
|
|
1881
|
+
* The ARN is derived rather than read off a response - an S3 bucket ARN carries
|
|
1882
|
+
* no region and no generated id, so it is a pure function of the name - which is
|
|
1883
|
+
* why it needs none of the "never record `''`" guards the response-derived ARNs
|
|
1884
|
+
* in this module carry. Recording it at all, rather than letting the two readers
|
|
1885
|
+
* derive it themselves, is what makes their declared edges load-bearing; see
|
|
1886
|
+
* {@link requireErrorBucketArn}.
|
|
1887
|
+
*/
|
|
1888
|
+
function recordErrorBucket(ctx, name) {
|
|
1889
|
+
const out = output(ctx, ERROR_BUCKET_NODE);
|
|
1890
|
+
out.name = name;
|
|
1891
|
+
out.arn = `arn:aws:s3:::${name}`;
|
|
1892
|
+
}
|
|
1893
|
+
/**
|
|
1894
|
+
* Tagging and the public-access block, both idempotent PUTs, shared by `create`
|
|
1895
|
+
* and `update` - `applyBucketConfiguration`'s shape
|
|
1896
|
+
* (`packages/cli/src/nodes.ts:38-42`).
|
|
1897
|
+
*
|
|
1898
|
+
* The public-access block is not decoration here. The objects in this bucket
|
|
1899
|
+
* are the records the transform Lambda did *not* successfully process, so they
|
|
1900
|
+
* carry CloudFront's raw fields - the viewer's IP address among them, the one
|
|
1901
|
+
* value the whole `visitor_key` derivation exists to keep out of storage. A
|
|
1902
|
+
* bucket that could be made public by a later policy or ACL would undo that for
|
|
1903
|
+
* exactly the records where it was never applied.
|
|
1904
|
+
*/
|
|
1905
|
+
async function applyErrorBucketConfiguration(ctx, name) {
|
|
1906
|
+
const client = s3(ctx);
|
|
1907
|
+
await client.putBucketTagging(name, ctx.tags ?? {});
|
|
1908
|
+
await client.putPublicAccessBlock(name);
|
|
1909
|
+
}
|
|
1910
|
+
/**
|
|
1911
|
+
* The role Firehose assumes to read the catalog, write the table, invoke the
|
|
1912
|
+
* transform and store what it could not deliver - four grants, four concrete
|
|
1913
|
+
* resources, no `*`.
|
|
1914
|
+
*
|
|
1915
|
+
* It declares `dependsOn` on the three nodes whose recorded ARNs those grants
|
|
1916
|
+
* interpolate. `topoSort` drains zero-indegree nodes alphabetically
|
|
1917
|
+
* (`packages/cli/src/graph.ts:35-38`), so a role declaring `dependsOn: []`
|
|
1918
|
+
* would be reconciled *before* `analytics-transform-function` - `f` sorts before
|
|
1919
|
+
* `t` - and the policy would interpolate an unrecorded output: a wrong
|
|
1920
|
+
* permission written silently, never an error. `githubOidcRoleNode`
|
|
1921
|
+
* (`packages/cli/src/nodes.ts:830`) is the precedent, declaring
|
|
1922
|
+
* `cloudfront-distribution` for exactly this reason.
|
|
1923
|
+
*/
|
|
1924
|
+
export function analyticsFirehoseRoleNode() {
|
|
1925
|
+
return {
|
|
1926
|
+
id: FIREHOSE_ROLE_NODE,
|
|
1927
|
+
dependsOn: [ERROR_BUCKET_NODE, TABLE_NODE, TRANSFORM_FUNCTION_NODE],
|
|
1928
|
+
title: `IAM Firehose delivery role (global - IAM is not regional; it serves the ${ANALYTICS_REGION} pipeline)`,
|
|
1929
|
+
async read(ctx) {
|
|
1930
|
+
const name = firehoseRoleName(ctx);
|
|
1931
|
+
const arn = await ctx.clients.iam.getRoleArn(name);
|
|
1932
|
+
// Falsy rather than `=== undefined`, `analytics-transform-role`'s guard:
|
|
1933
|
+
// `getRoleArn` reads the ARN out of the response XML, so a body without
|
|
1934
|
+
// one answers `undefined` while an empty tag would answer `""`.
|
|
1935
|
+
if (!arn)
|
|
1936
|
+
return false;
|
|
1937
|
+
const out = output(ctx, FIREHOSE_ROLE_NODE);
|
|
1938
|
+
out.name = name;
|
|
1939
|
+
out.arn = arn;
|
|
1940
|
+
return true;
|
|
1941
|
+
},
|
|
1942
|
+
async create(ctx) {
|
|
1943
|
+
const name = firehoseRoleName(ctx);
|
|
1944
|
+
// `ctx.clients.iam`, the host's own client: IAM is a global service
|
|
1945
|
+
// (`packages/core/src/aws/endpoint.ts`'s GLOBAL_SERVICES), so core's
|
|
1946
|
+
// instance already signs us-east-1 and there is no region to get wrong.
|
|
1947
|
+
const arn = await ctx.clients.iam.ensureRole(name, FIREHOSE_TRUST, `Delivery role for the ${ctx.config.siteName} analytics Firehose stream`, ctx.tags);
|
|
1948
|
+
// Recorded before the policy PUT, `analytics-transform-role`'s ordering
|
|
1949
|
+
// and its reason: the role is a real IAM object the moment `ensureRole`
|
|
1950
|
+
// returns, and if the policy call then fails, this entry is what tells the
|
|
1951
|
+
// next reconcile - and `delete` - that it exists. Nothing reads a role ARN
|
|
1952
|
+
// as "the role is configured": the stream reads it to name in its
|
|
1953
|
+
// destination, and `applyGraph` reconciles this node to completion first
|
|
1954
|
+
// (`packages/cli/src/graph.ts:74-97` rethrows, so the stream is never
|
|
1955
|
+
// reached after a failure here).
|
|
1956
|
+
const out = output(ctx, FIREHOSE_ROLE_NODE);
|
|
1957
|
+
out.name = name;
|
|
1958
|
+
out.arn = arn;
|
|
1959
|
+
await applyFirehoseRolePolicy(ctx);
|
|
1960
|
+
},
|
|
1961
|
+
async update(ctx) {
|
|
1962
|
+
await applyFirehoseRolePolicy(ctx);
|
|
1963
|
+
},
|
|
1964
|
+
async delete(ctx) {
|
|
1965
|
+
// Idempotent, and removes the inline policy first - `deleteRole`
|
|
1966
|
+
// (`packages/core/src/aws/iam.ts:128`) lists and deletes them, because IAM
|
|
1967
|
+
// refuses to delete a role that still carries one, and swallows the
|
|
1968
|
+
// not-found so a half-finished teardown is re-runnable.
|
|
1969
|
+
await ctx.clients.iam.deleteRole(firehoseRoleName(ctx));
|
|
1970
|
+
},
|
|
1971
|
+
};
|
|
1972
|
+
}
|
|
1973
|
+
/**
|
|
1974
|
+
* The delivery stream itself: CloudFront's records in, the transform Lambda in
|
|
1975
|
+
* front of the write, the `page_views` Iceberg table out, and the plugin's own
|
|
1976
|
+
* error bucket for everything that does not make it.
|
|
1977
|
+
*
|
|
1978
|
+
* Its four edges are the ones its payload actually reads. The role edge is the
|
|
1979
|
+
* spec's own rule - the `IcebergDestinationConfiguration` interpolates the
|
|
1980
|
+
* role's recorded ARN - and it also carries `analytics-error-bucket`
|
|
1981
|
+
* transitively, completing the `error-bucket -> firehose-role ->
|
|
1982
|
+
* firehose-stream` chain. Without the role edge the ordering would survive only
|
|
1983
|
+
* on `topoSort`'s alphabetical accident (`…-role` sorts before `…-stream`), the
|
|
1984
|
+
* exact coincidence-reliance the spec's implementation notes warn against.
|
|
1985
|
+
*
|
|
1986
|
+
* **The `AppendOnly` reconcile is written against neither AWS document.** The
|
|
1987
|
+
* Firehose considerations page says the flag is settable only with
|
|
1988
|
+
* `CreateDeliveryStream`; the `IcebergDestinationUpdate` API reference lists it
|
|
1989
|
+
* among the fields `UpdateDestination` accepts. They cannot both be right, and a
|
|
1990
|
+
* node written against either alone is a defect whichever one turns out to be.
|
|
1991
|
+
* So `update` attempts the in-place update first and falls back to replacing the
|
|
1992
|
+
* stream when it is refused - and only when it is refused: the re-read that
|
|
1993
|
+
* follows a successful update sits outside that `try`, because failing it is not
|
|
1994
|
+
* a rejection and replacing a stream that was updated correctly would be pure
|
|
1995
|
+
* loss. The order matters: `UpdateDestination` keeps the stream's ARN, while a
|
|
1996
|
+
* replacement gets a new one - so the CloudFront log delivery task 53 builds
|
|
1997
|
+
* would have to be repointed, and the records arriving during the gap are lost.
|
|
1998
|
+
* Which path ran is in the log line.
|
|
1999
|
+
*/
|
|
2000
|
+
export function analyticsFirehoseStreamNode() {
|
|
2001
|
+
return {
|
|
2002
|
+
id: FIREHOSE_STREAM_NODE,
|
|
2003
|
+
dependsOn: [FIREHOSE_ROLE_NODE, TABLE_NODE, CATALOG_NODE, TRANSFORM_FUNCTION_NODE],
|
|
2004
|
+
title: `Firehose delivery stream (${ANALYTICS_REGION})`,
|
|
2005
|
+
async read(ctx) {
|
|
2006
|
+
const status = await firehose(ctx).describeDeliveryStream(streamName(ctx));
|
|
2007
|
+
// Absent: `create` runs. Note that a stream in any *live* state - including
|
|
2008
|
+
// `CREATING_FAILED` and `DELETING` - is present, not absent, and is
|
|
2009
|
+
// reported so deliberately. Answering `false` for one would send
|
|
2010
|
+
// `applyGraph` to `create`, whose `ResourceInUseException` is swallowed as
|
|
2011
|
+
// "already exists", and the reconcile would go green over a stream that
|
|
2012
|
+
// accepts nothing.
|
|
2013
|
+
//
|
|
2014
|
+
// What `update` then does with such a stream is *nothing*: it branches on
|
|
2015
|
+
// the recorded `AppendOnly` flag alone, so a `CREATING_FAILED` or
|
|
2016
|
+
// `DELETING` stream whose flag already matches is reconciled with zero AWS
|
|
2017
|
+
// calls and reported done. That is stated rather than guarded because this
|
|
2018
|
+
// `read` is the hydration path - `recordStream` puts `state` and `failure`
|
|
2019
|
+
// into the plugin's scoped state, and reporting an unusable stream from
|
|
2020
|
+
// them is `analytics status`' job (task 55). Do not read this comment as a
|
|
2021
|
+
// promise that the reconcile refuses over a dead stream; it does not.
|
|
2022
|
+
if (status === undefined)
|
|
2023
|
+
return false;
|
|
2024
|
+
recordStream(ctx, status);
|
|
2025
|
+
return true;
|
|
2026
|
+
},
|
|
2027
|
+
async create(ctx) {
|
|
2028
|
+
const name = streamName(ctx);
|
|
2029
|
+
ctx.logger.step(`creating the analytics delivery stream "${name}" with AppendOnly ${STREAM_APPEND_ONLY}`);
|
|
2030
|
+
await createStream(ctx, firehose(ctx), name, firehoseDestination(ctx));
|
|
2031
|
+
},
|
|
2032
|
+
async update(ctx) {
|
|
2033
|
+
const recorded = ctx.state.resources[FIREHOSE_STREAM_NODE];
|
|
2034
|
+
const appendOnly = typeof recorded?.['appendOnly'] === 'boolean' ? recorded['appendOnly'] : undefined;
|
|
2035
|
+
// The live flag already matches what this pipeline wants, so there is
|
|
2036
|
+
// nothing to reconcile and no AWS call at all. `undefined` does NOT match:
|
|
2037
|
+
// a stream whose destination reported no flag, or a state file that lost
|
|
2038
|
+
// it, is a stream this node cannot claim is append-only, and pushing the
|
|
2039
|
+
// desired configuration is the safe direction.
|
|
2040
|
+
if (appendOnly === STREAM_APPEND_ONLY)
|
|
2041
|
+
return;
|
|
2042
|
+
const name = streamName(ctx);
|
|
2043
|
+
const client = firehose(ctx);
|
|
2044
|
+
const destination = firehoseDestination(ctx);
|
|
2045
|
+
const versionId = recordedText(ctx, FIREHOSE_STREAM_NODE, 'versionId');
|
|
2046
|
+
const destinationId = recordedText(ctx, FIREHOSE_STREAM_NODE, 'destinationId');
|
|
2047
|
+
// Falsy rather than `!== undefined`, the guard `analytics-transform-role`'s
|
|
2048
|
+
// `read` and `analytics-table`'s ARN both use: an empty recorded string is
|
|
2049
|
+
// no more a version id than a missing one, and an empty
|
|
2050
|
+
// `CurrentDeliveryStreamVersionId` fails the service's own `[0-9]+` pattern.
|
|
2051
|
+
if (versionId && destinationId) {
|
|
2052
|
+
// **Only the update call is in this `try`.** The re-read below is not,
|
|
2053
|
+
// and must never be: it runs *after* `UpdateDestination` returned 200,
|
|
2054
|
+
// so the stream is already reconfigured and still carries its ARN. A
|
|
2055
|
+
// transient failure there - `LimitExceededException`, a throttle,
|
|
2056
|
+
// anything `describeDeliveryStream` does not swallow as a not-found - is
|
|
2057
|
+
// not a refusal, and reaching the fallback on one would delete and
|
|
2058
|
+
// recreate a stream that was updated correctly: a NEW ARN, task 53's
|
|
2059
|
+
// CloudFront log delivery orphaned, the records in flight lost, and an
|
|
2060
|
+
// operator told the update was rejected when it succeeded.
|
|
2061
|
+
let refusal;
|
|
2062
|
+
try {
|
|
2063
|
+
ctx.logger.step(`updating the analytics delivery stream "${name}" in place (AppendOnly ${String(appendOnly)} -> ${STREAM_APPEND_ONLY}) - UpdateDestination keeps the stream's ARN, so the CloudFront log delivery pointed at it is untouched`);
|
|
2064
|
+
await client.updateDestination(name, destination, { versionId, destinationId });
|
|
2065
|
+
}
|
|
2066
|
+
catch (err) {
|
|
2067
|
+
// The branch the contradicting documentation makes necessary. Not
|
|
2068
|
+
// narrowed to one exception: whichever way AWS resolves it, a refused
|
|
2069
|
+
// update has to reach the fallback rather than abort the reconcile.
|
|
2070
|
+
// `String(err)` rather than the error itself, so the sentinel is set
|
|
2071
|
+
// even for a thrown `undefined`.
|
|
2072
|
+
refusal = String(err);
|
|
2073
|
+
}
|
|
2074
|
+
if (refusal === undefined) {
|
|
2075
|
+
try {
|
|
2076
|
+
// Re-read: the update bumps `VersionId`, so a state file still
|
|
2077
|
+
// holding the old one would fail the next `UpdateDestination` on a
|
|
2078
|
+
// ConcurrentModificationException it did not cause.
|
|
2079
|
+
const updated = await client.describeDeliveryStream(name);
|
|
2080
|
+
if (updated !== undefined)
|
|
2081
|
+
recordStream(ctx, updated);
|
|
2082
|
+
}
|
|
2083
|
+
catch (err) {
|
|
2084
|
+
// Warn and carry on rather than rethrow: the update is done, and the
|
|
2085
|
+
// only casualty is a recorded version id that is now one behind.
|
|
2086
|
+
// `read` re-hydrates it on the next reconcile, which is the same
|
|
2087
|
+
// path that would recover a state file that never had one.
|
|
2088
|
+
ctx.logger.warn(`the analytics delivery stream "${name}" could not be re-read after UpdateDestination succeeded (${String(err)}) - the update is applied and the stream keeps its ARN, but the recorded version id is now stale until the next reconcile refreshes it`);
|
|
2089
|
+
}
|
|
2090
|
+
ctx.logger.ok(`updated the analytics delivery stream "${name}" in place`);
|
|
2091
|
+
return;
|
|
2092
|
+
}
|
|
2093
|
+
ctx.logger.warn(`UpdateDestination was refused for the analytics delivery stream "${name}" (${refusal}) - falling back to replacing it`);
|
|
2094
|
+
}
|
|
2095
|
+
else {
|
|
2096
|
+
ctx.logger.warn(`the analytics delivery stream "${name}" has no recorded version id and destination id, which UpdateDestination requires - falling back to replacing it`);
|
|
2097
|
+
}
|
|
2098
|
+
ctx.logger.warn(`replacing the analytics delivery stream "${name}": the new stream carries a NEW ARN, so the CloudFront log delivery has to be reconciled against it, and records arriving during the gap are lost`);
|
|
2099
|
+
await client.deleteDeliveryStream(name);
|
|
2100
|
+
await createStream(ctx, client, name, destination);
|
|
2101
|
+
},
|
|
2102
|
+
async delete(ctx) {
|
|
2103
|
+
// No-op when the stream is already gone (`aws/firehose.ts` swallows the
|
|
2104
|
+
// not-found and nothing else - including `ResourceInUseException`, which on
|
|
2105
|
+
// this operation means "still CREATING", not "already deleted"), so a
|
|
2106
|
+
// half-finished teardown is re-runnable. `destroyGraph` walks the chain in
|
|
2107
|
+
// reverse, so this runs before `analytics-firehose-role` removes the role
|
|
2108
|
+
// the stream assumes.
|
|
2109
|
+
await firehose(ctx).deleteDeliveryStream(streamName(ctx));
|
|
2110
|
+
},
|
|
2111
|
+
};
|
|
2112
|
+
}
|
|
2113
|
+
/**
|
|
2114
|
+
* The suffix the plugin's own CloudWatch delivery destination carries,
|
|
2115
|
+
* appended to `ctx.names.prefix`. See {@link ERROR_BUCKET_SUFFIX} for why the
|
|
2116
|
+
* prefix rather than {@link resolveAnalyticsConfig} is the source of the
|
|
2117
|
+
* environment: the `analytics` block owns six settings and this is not one of
|
|
2118
|
+
* them, so there is no operator override to honour.
|
|
2119
|
+
*
|
|
2120
|
+
* **It must not resolve to `ctx.names.deliveryDestination`, and that single
|
|
2121
|
+
* property is what the site's own teardown guard rests on.** AWS permits
|
|
2122
|
+
* exactly one delivery source per distribution, so this plugin's delivery
|
|
2123
|
+
* necessarily hangs off the source the site created, and
|
|
2124
|
+
* `packages/cli/src/nodes.ts`'s `isOwnDelivery` tells the two apart by the one
|
|
2125
|
+
* thing that distinguishes them: the final `:`-separated segment of a
|
|
2126
|
+
* delivery's `deliveryDestinationArn`, which is the destination's name,
|
|
2127
|
+
* compared against `ctx.names.deliveryDestination` (`<env>-<siteName>-cf-dest`,
|
|
2128
|
+
* `packages/core/src/config.ts`). A suffix that made the two names equal would
|
|
2129
|
+
* make `blogwright destroy` treat this plugin's delivery as the site's own and
|
|
2130
|
+
* tear the shared source out from under it without refusing - the exact
|
|
2131
|
+
* failure task 52's two guards exist to prevent, and the reason this name is
|
|
2132
|
+
* `-analytics-cf-dest` and not `-cf-dest`.
|
|
2133
|
+
*/
|
|
2134
|
+
const LOG_DESTINATION_SUFFIX = '-analytics-cf-dest';
|
|
2135
|
+
/**
|
|
2136
|
+
* The longest name CloudWatch Logs accepts for a delivery destination
|
|
2137
|
+
* (`PutDeliveryDestination`'s `name`: 1..60 characters, `[\w-]*`). Checked
|
|
2138
|
+
* where the name is derived, the guard {@link boundedName} applies to every
|
|
2139
|
+
* other derived name in this module - `ctx.names.prefix` is bounded only by
|
|
2140
|
+
* the site bucket's 63, so an environment and site name that fit everywhere
|
|
2141
|
+
* else can still overrun this one.
|
|
2142
|
+
*/
|
|
2143
|
+
const LOG_DESTINATION_NAME_MAX_LENGTH = 60;
|
|
2144
|
+
/**
|
|
2145
|
+
* The format CloudWatch Logs renders each CloudFront record in before handing
|
|
2146
|
+
* it to the Firehose stream, and the one value here the transform Lambda makes
|
|
2147
|
+
* non-negotiable: `transform/handler.ts` base64-decodes each record and
|
|
2148
|
+
* `JSON.parse`s it, and `transform/map-record.ts` reads CloudFront field names
|
|
2149
|
+
* (`timestamp(ms)`, `c-ip`, `cs-uri-stem`) off the parsed object. `plain`,
|
|
2150
|
+
* `w3c` and `raw` all deliver delimited text that `JSON.parse` throws on, which
|
|
2151
|
+
* the handler reports as `ProcessingFailed` for every record: the error bucket
|
|
2152
|
+
* fills, the dashboard stays empty, and nothing names this constant as the
|
|
2153
|
+
* cause. `parquet` is an S3-destination format and has no meaning for a
|
|
2154
|
+
* Firehose destination at all.
|
|
2155
|
+
*
|
|
2156
|
+
* Because the format is `json`, `createDelivery`'s `fieldDelimiter` is
|
|
2157
|
+
* deliberately **not** sent - see {@link analyticsLogDeliveryNode}'s `create`.
|
|
2158
|
+
*
|
|
2159
|
+
* It is recorded beside the destination's ARN because it is immutable once the
|
|
2160
|
+
* destination exists; {@link analyticsLogDestinationNode}'s `update` is what
|
|
2161
|
+
* that recording is for.
|
|
2162
|
+
*/
|
|
2163
|
+
const DELIVERY_OUTPUT_FORMAT = 'json';
|
|
2164
|
+
/**
|
|
2165
|
+
* How often {@link requireActiveStream} re-describes a delivery stream that is
|
|
2166
|
+
* still `CREATING`, and how long it waits before refusing. Firehose brings a
|
|
2167
|
+
* `DirectPut` stream to `ACTIVE` in well under a minute, so five minutes is
|
|
2168
|
+
* generous enough that a first bootstrap does not fail on a slow account and
|
|
2169
|
+
* short enough that a stream which is never going to become active is reported
|
|
2170
|
+
* rather than waited on indefinitely.
|
|
2171
|
+
*/
|
|
2172
|
+
const STREAM_ACTIVE_POLL_INTERVAL_MS = 5_000;
|
|
2173
|
+
/** See {@link STREAM_ACTIVE_POLL_INTERVAL_MS} - the two are chosen as a pair. */
|
|
2174
|
+
const STREAM_ACTIVE_TIMEOUT_MS = 5 * 60_000;
|
|
2175
|
+
/**
|
|
2176
|
+
* The site's CloudFront distribution node, as `packages/cli/src/nodes.ts`
|
|
2177
|
+
* names it. A node id from the **site's** graph, spelled here rather than
|
|
2178
|
+
* imported because this package does not depend on `blogwright` and never
|
|
2179
|
+
* will: the site's outputs are reached read-only through
|
|
2180
|
+
* {@link requireSiteDeliverySource}.
|
|
2181
|
+
*/
|
|
2182
|
+
const SITE_DISTRIBUTION_NODE = 'cloudfront-distribution';
|
|
2183
|
+
/**
|
|
2184
|
+
* The state key holding the UTC day this plugin's delivery was **first**
|
|
2185
|
+
* created - the idempotency bound the change spec's §Backfill of historical
|
|
2186
|
+
* logs defines and task 61 reads.
|
|
2187
|
+
*
|
|
2188
|
+
* Written once and never advanced. Backfill inserts only whole days *strictly
|
|
2189
|
+
* before* it, on the reasoning that Firehose received nothing before its
|
|
2190
|
+
* delivery existed, so the two paths' row sets are disjoint. The two error
|
|
2191
|
+
* directions are not symmetric, which is why the rule is write-once rather
|
|
2192
|
+
* than keep-current: a bound that is too *early* loses at most the day at the
|
|
2193
|
+
* seam, which the spec states and accepts, while a bound that moved *later*
|
|
2194
|
+
* would let backfill insert days Firehose had already delivered and silently
|
|
2195
|
+
* double every row in them. So a second reconcile, a re-created delivery and
|
|
2196
|
+
* the destination node's Conflict retry all leave it exactly as it was.
|
|
2197
|
+
*
|
|
2198
|
+
* `read` never writes it either, even though it hydrates the rest of this
|
|
2199
|
+
* node's outputs off the live delivery: `DescribeDeliveries` reports no
|
|
2200
|
+
* creation date, so a delivery found already attached to a state file that
|
|
2201
|
+
* lost this key leaves task 61 with no bound and an actionable refusal. That
|
|
2202
|
+
* is the loud direction, and it is preferred to today's date, which would be a
|
|
2203
|
+
* bound that moved later.
|
|
2204
|
+
*
|
|
2205
|
+
* Exported since task 61, which reads it. It was deliberately module-private
|
|
2206
|
+
* while nothing consumed it - an exported constant with no consumer is what
|
|
2207
|
+
* `pnpm knip` catches - but a private constant restated in its reader is worse
|
|
2208
|
+
* than an exported one: the two spellings would have to agree and nothing
|
|
2209
|
+
* would check that they did.
|
|
2210
|
+
*/
|
|
2211
|
+
export const CREATED_DAY_KEY = 'createdDay';
|
|
2212
|
+
/** `YYYY-MM-DD` - the leading characters of an ISO-8601 timestamp that are its UTC day. */
|
|
2213
|
+
const ISO_DAY_LENGTH = 10;
|
|
2214
|
+
/**
|
|
2215
|
+
* The CloudWatch Logs client, taken off `ctx.clients` unchanged rather than
|
|
2216
|
+
* built by {@link createAnalyticsClients}, and the only client in this module
|
|
2217
|
+
* that is core's own instance. The change spec says why: `LogsClient` stays in
|
|
2218
|
+
* core because the *site* graph owns it, and `logsUsEast1` is already the
|
|
2219
|
+
* us-east-1 instance core built for CloudFront vended log delivery - the same
|
|
2220
|
+
* reason `ctx.clients.iam` is used unchanged for this plugin's two roles.
|
|
2221
|
+
* `ctx.clients.logs` would sign in `config.region`, where neither the site's
|
|
2222
|
+
* delivery source nor this plugin's stream exists.
|
|
2223
|
+
*/
|
|
2224
|
+
function logs(ctx) {
|
|
2225
|
+
return ctx.clients.logsUsEast1;
|
|
2226
|
+
}
|
|
2227
|
+
/** The plugin's own CloudWatch delivery destination name. See {@link LOG_DESTINATION_SUFFIX}. */
|
|
2228
|
+
function logDestinationName(ctx) {
|
|
2229
|
+
return boundedName(`${ctx.names.prefix}${LOG_DESTINATION_SUFFIX}`, LOG_DESTINATION_NAME_MAX_LENGTH, 'log delivery destination');
|
|
2230
|
+
}
|
|
2231
|
+
/**
|
|
2232
|
+
* True when `delivery` is the one this plugin created.
|
|
2233
|
+
*
|
|
2234
|
+
* The mirror image of `packages/cli/src/nodes.ts`'s `isOwnDelivery`, and
|
|
2235
|
+
* deliberately the same test, because the two have to partition one shared
|
|
2236
|
+
* list: the final `:`-separated segment of a `delivery-destination` ARN is the
|
|
2237
|
+
* destination's name, and the destination a delivery feeds is the only thing
|
|
2238
|
+
* that distinguishes two deliveries hanging off one source. The names the two
|
|
2239
|
+
* predicates compare against are kept distinct by
|
|
2240
|
+
* {@link LOG_DESTINATION_SUFFIX}, so each selects exactly what the other
|
|
2241
|
+
* rejects.
|
|
2242
|
+
*
|
|
2243
|
+
* Position cannot stand in for it - `findDeliveryIdBySource` returns whichever
|
|
2244
|
+
* delivery AWS lists first, which on this source may well be the site's - and
|
|
2245
|
+
* neither can this node's recorded destination ARN, which is empty precisely
|
|
2246
|
+
* when the Conflict retry needs it, because `putDeliveryDestination` threw
|
|
2247
|
+
* before anything was recorded. A delivery AWS reports without a destination
|
|
2248
|
+
* ARN matches no name and so is not this plugin's: fail-closed, which here
|
|
2249
|
+
* means this plugin deletes only what it can attribute to itself.
|
|
2250
|
+
*/
|
|
2251
|
+
function isPluginDelivery(delivery, destinationName) {
|
|
2252
|
+
return delivery.deliveryDestinationArn.split(':').pop() === destinationName;
|
|
2253
|
+
}
|
|
2254
|
+
/**
|
|
2255
|
+
* The ids of this plugin's own deliveries on the site's shared delivery
|
|
2256
|
+
* source. Every other delivery on it - the site's own CloudWatch copy above
|
|
2257
|
+
* all - is filtered out and left exactly as it was found.
|
|
2258
|
+
*
|
|
2259
|
+
* There is no refusal here, and the asymmetry with
|
|
2260
|
+
* `packages/cli/src/nodes.ts`'s `ownDeliveryIdsOrRefuse` is the point rather
|
|
2261
|
+
* than an omission. That function refuses outright when the shared source
|
|
2262
|
+
* carries a delivery the site does not own, because both of its callers go on
|
|
2263
|
+
* to delete the *source*, which AWS rejects while any delivery is still
|
|
2264
|
+
* attached - so a foreign delivery forecloses what it was about to do.
|
|
2265
|
+
* Nothing in this module ever deletes that source, so a delivery this plugin
|
|
2266
|
+
* does not own obstructs nothing here; it is simply not this plugin's to
|
|
2267
|
+
* touch, and the filter is the whole of the answer.
|
|
2268
|
+
*/
|
|
2269
|
+
async function pluginDeliveryIds(ctx) {
|
|
2270
|
+
const destinationName = logDestinationName(ctx);
|
|
2271
|
+
const deliveries = await logs(ctx).deliveriesForSource(ctx.names.deliverySource);
|
|
2272
|
+
return deliveries.filter((d) => isPluginDelivery(d, destinationName)).map((d) => d.id);
|
|
2273
|
+
}
|
|
2274
|
+
/** Remove every delivery this plugin owns off the shared source, and nothing else. */
|
|
2275
|
+
async function clearPluginDeliveries(ctx) {
|
|
2276
|
+
for (const id of await pluginDeliveryIds(ctx)) {
|
|
2277
|
+
await logs(ctx).deleteDelivery(id);
|
|
2278
|
+
}
|
|
2279
|
+
}
|
|
2280
|
+
/**
|
|
2281
|
+
* The delivery stream's ARN as `analytics-firehose-stream` recorded it. See
|
|
2282
|
+
* {@link requireRecordedArn}.
|
|
2283
|
+
*
|
|
2284
|
+
* Read back rather than derived, which is what makes this node's one declared
|
|
2285
|
+
* edge load-bearing: `analytics-firehose-stream` sorts *after*
|
|
2286
|
+
* `analytics-log-destination` alphabetically, so `topoSort`'s zero-indegree
|
|
2287
|
+
* ordering would run this node first if the edge were dropped, and the
|
|
2288
|
+
* destination would be created pointing at `undefined`.
|
|
2289
|
+
*/
|
|
2290
|
+
function requireStreamArn(ctx) {
|
|
2291
|
+
return requireRecordedArn(ctx, {
|
|
2292
|
+
what: 'delivery stream',
|
|
2293
|
+
node: FIREHOSE_STREAM_NODE,
|
|
2294
|
+
dependent: LOG_DESTINATION_NODE,
|
|
2295
|
+
lack: 'stream to point the CloudWatch delivery destination at',
|
|
2296
|
+
});
|
|
2297
|
+
}
|
|
2298
|
+
/**
|
|
2299
|
+
* The delivery destination's ARN as `analytics-log-destination` recorded it.
|
|
2300
|
+
* See {@link requireRecordedArn}.
|
|
2301
|
+
*/
|
|
2302
|
+
function requireLogDestinationArn(ctx) {
|
|
2303
|
+
return requireRecordedArn(ctx, {
|
|
2304
|
+
what: 'log delivery destination',
|
|
2305
|
+
node: LOG_DESTINATION_NODE,
|
|
2306
|
+
dependent: LOG_DELIVERY_NODE,
|
|
2307
|
+
lack: 'destination to deliver the CloudFront records to',
|
|
2308
|
+
});
|
|
2309
|
+
}
|
|
2310
|
+
/**
|
|
2311
|
+
* Wait for the delivery stream to reach `ACTIVE`, and refuse rather than build
|
|
2312
|
+
* a delivery over one that never got there.
|
|
2313
|
+
*
|
|
2314
|
+
* **This is task 51's routed finding, discharged here rather than left
|
|
2315
|
+
* implicit.** `createStream` rejects only `deleting` and `delete-failed`, so
|
|
2316
|
+
* `applyGraph` reports `analytics-firehose-stream` done over a stream that is
|
|
2317
|
+
* still `CREATING`. That is right for *that* node - the stream is being
|
|
2318
|
+
* created, and nothing it does needs the stream to accept a record - and wrong
|
|
2319
|
+
* for this one, which is the first consumer that cares. The destination and
|
|
2320
|
+
* the delivery are where CloudWatch starts pushing records at the stream:
|
|
2321
|
+
* pointed at one that is not yet accepting them, the records are refused, no
|
|
2322
|
+
* node fails, `analytics status` reports every resource present, and the only
|
|
2323
|
+
* symptom is an empty dashboard with no error anywhere.
|
|
2324
|
+
*
|
|
2325
|
+
* **Waiting rather than refusing outright is the deliberate half.** A fresh
|
|
2326
|
+
* `analytics bootstrap` creates the stream and reaches this node seconds
|
|
2327
|
+
* later, so a bare refusal would fail every first run and be re-run into
|
|
2328
|
+
* success - which teaches an operator to re-run past this message rather than
|
|
2329
|
+
* read it. `pollUntil` is the precedent task 51's contract names, and the
|
|
2330
|
+
* CLI's own (`packages/cli/src/nodes.ts:705`, the distribution's deployment
|
|
2331
|
+
* wait).
|
|
2332
|
+
*
|
|
2333
|
+
* The `done` predicate settles on anything that is no longer `creating`, not
|
|
2334
|
+
* on `active` alone, so a stream that has already failed to create is reported
|
|
2335
|
+
* at once instead of being waited out for {@link STREAM_ACTIVE_TIMEOUT_MS}.
|
|
2336
|
+
* The check after it is what turns every non-`active` outcome into one message:
|
|
2337
|
+
* a `create-failed` stream, a stream deleted from under the run, and a stream
|
|
2338
|
+
* still `creating` when the deadline passed - `pollUntil` returns its last
|
|
2339
|
+
* value rather than throwing, so without this check a timeout would fall
|
|
2340
|
+
* straight through into creating the delivery.
|
|
2341
|
+
*/
|
|
2342
|
+
async function requireActiveStream(ctx) {
|
|
2343
|
+
const name = streamName(ctx);
|
|
2344
|
+
const settled = await pollUntil(() => firehose(ctx).describeDeliveryStream(name), (status) => status === undefined || status.state !== 'creating', { intervalMs: STREAM_ACTIVE_POLL_INTERVAL_MS, timeoutMs: STREAM_ACTIVE_TIMEOUT_MS });
|
|
2345
|
+
if (settled?.state === 'active')
|
|
2346
|
+
return;
|
|
2347
|
+
throw new Error(`the analytics delivery stream "${name}" is ${settled === undefined ? 'not readable' : settled.state} rather than active, so a CloudFront log delivery pointed at it would accept no records and nothing would report it - refusing to wire one. ${settled?.state === 'creating' ? `The stream is still being created; re-run \`blogwright analytics bootstrap ${ctx.env}\` in a minute.` : `Check the stream in the Firehose console, then re-run \`blogwright analytics bootstrap ${ctx.env}\`.`}`);
|
|
2348
|
+
}
|
|
2349
|
+
/**
|
|
2350
|
+
* Create or repoint the delivery destination and record what it is.
|
|
2351
|
+
*
|
|
2352
|
+
* **Record ordering**, which this module decides per node: all three values
|
|
2353
|
+
* land after the one call returns, because there is exactly one call. The
|
|
2354
|
+
* incremental recording `packages/cli/src/nodes.ts:717-719` performs - which
|
|
2355
|
+
* `analytics-table` and `analytics-firehose-stream` both copy - exists to
|
|
2356
|
+
* survive a crash *between* two mutating calls, and this node makes no such
|
|
2357
|
+
* pair. Recording earlier would only claim a destination the service has not
|
|
2358
|
+
* confirmed; recording later is impossible, since the ARN arrives in the
|
|
2359
|
+
* response.
|
|
2360
|
+
*
|
|
2361
|
+
* The ARN carries this module's standing guard against recording `''` as
|
|
2362
|
+
* though it were an ARN (`putDeliveryDestination` falls back to the empty
|
|
2363
|
+
* string for a body carrying none). An unrecorded ARN makes `read` answer
|
|
2364
|
+
* false and the next reconcile re-put the destination, which is idempotent; an
|
|
2365
|
+
* empty one recorded under `arn` reads downstream as a real one, and
|
|
2366
|
+
* `analytics-log-delivery` would create its delivery against it.
|
|
2367
|
+
*/
|
|
2368
|
+
async function putLogDestination(ctx, name, streamArn) {
|
|
2369
|
+
const arn = await logs(ctx).putDeliveryDestination(name, streamArn, {
|
|
2370
|
+
outputFormat: DELIVERY_OUTPUT_FORMAT,
|
|
2371
|
+
});
|
|
2372
|
+
const out = output(ctx, LOG_DESTINATION_NODE);
|
|
2373
|
+
out.name = name;
|
|
2374
|
+
recordOptional(out, 'arn', arn === '' ? undefined : arn);
|
|
2375
|
+
out.outputFormat = DELIVERY_OUTPUT_FORMAT;
|
|
2376
|
+
}
|
|
2377
|
+
/**
|
|
2378
|
+
* The CloudWatch delivery destination the site's CloudFront records are
|
|
2379
|
+
* delivered to - **this plugin's own, alongside the site's and never in place
|
|
2380
|
+
* of it.**
|
|
2381
|
+
*
|
|
2382
|
+
* Its one edge is the node whose recorded ARN it points at. `PutDeliveryDestination`
|
|
2383
|
+
* accepts a `destinationResourceArn` for a resource that does not exist yet, so
|
|
2384
|
+
* without the edge the destination would be created against `undefined` and the
|
|
2385
|
+
* first symptom would be records going nowhere - see {@link requireStreamArn}.
|
|
2386
|
+
*
|
|
2387
|
+
* The output format is the one thing about a destination that cannot be
|
|
2388
|
+
* changed once it exists, which is why `update` replaces rather than mutates
|
|
2389
|
+
* and why the configured format is recorded beside the ARN in the first place.
|
|
2390
|
+
*/
|
|
2391
|
+
export function analyticsLogDestinationNode() {
|
|
2392
|
+
return {
|
|
2393
|
+
id: LOG_DESTINATION_NODE,
|
|
2394
|
+
dependsOn: [FIREHOSE_STREAM_NODE],
|
|
2395
|
+
title: `CloudWatch delivery destination (${ANALYTICS_REGION})`,
|
|
2396
|
+
async read(ctx) {
|
|
2397
|
+
// State, not AWS: core's `LogsClient` exposes no describe for a delivery
|
|
2398
|
+
// destination, and adding one is a change to core this task does not own.
|
|
2399
|
+
// `update` is what makes that safe rather than merely cheap - it re-puts
|
|
2400
|
+
// unconditionally, so a destination deleted outside this tool is restored
|
|
2401
|
+
// on the next reconcile instead of being believed present forever on the
|
|
2402
|
+
// strength of this answer. No `output()` call here: a `read` that finds
|
|
2403
|
+
// nothing must not leave an empty entry in the state file.
|
|
2404
|
+
const arn = ctx.state.resources[LOG_DESTINATION_NODE]?.arn;
|
|
2405
|
+
return typeof arn === 'string' && arn !== '';
|
|
2406
|
+
},
|
|
2407
|
+
async create(ctx) {
|
|
2408
|
+
const name = logDestinationName(ctx);
|
|
2409
|
+
// Resolved before the wait, so a missing edge fails with no AWS call at all.
|
|
2410
|
+
const streamArn = requireStreamArn(ctx);
|
|
2411
|
+
await requireActiveStream(ctx);
|
|
2412
|
+
try {
|
|
2413
|
+
await putLogDestination(ctx, name, streamArn);
|
|
2414
|
+
}
|
|
2415
|
+
catch (err) {
|
|
2416
|
+
// A destination left behind by a previous stack carries an output format
|
|
2417
|
+
// that cannot be changed, and `PutDeliveryDestination` answers a Conflict
|
|
2418
|
+
// rather than replacing it. Clear this plugin's own delivery and its own
|
|
2419
|
+
// destination and retry once - the shape of
|
|
2420
|
+
// `packages/cli/src/nodes.ts:743-761`, minus the one call in it that
|
|
2421
|
+
// would take the site down too.
|
|
2422
|
+
//
|
|
2423
|
+
// **The deliberate divergence is the absent `deleteDeliverySource`.**
|
|
2424
|
+
// The site's retry deletes the source at
|
|
2425
|
+
// `packages/cli/src/nodes.ts:758` because removing the source *is* its
|
|
2426
|
+
// retry: `PutDeliverySource` will not repoint an existing one. This
|
|
2427
|
+
// plugin never creates, repoints or deletes that source - it is the
|
|
2428
|
+
// site's, and the site's own CloudWatch delivery hangs off it. Copying
|
|
2429
|
+
// that line here would either throw (AWS rejects the delete while the
|
|
2430
|
+
// site's delivery is attached, and `deleteDeliverySource` swallows only
|
|
2431
|
+
// a not-found) or, once the site's delivery had gone with it, stop the
|
|
2432
|
+
// site's log delivery while the site's state still recorded it as
|
|
2433
|
+
// `configured`.
|
|
2434
|
+
//
|
|
2435
|
+
// The delivery is cleared before the destination for the same reason
|
|
2436
|
+
// the site's teardown deletes deliveries before the source
|
|
2437
|
+
// (`packages/cli/src/nodes.ts:763-768`): AWS rejects
|
|
2438
|
+
// `DeleteDeliveryDestination` while a delivery still points at it, and
|
|
2439
|
+
// `deleteDeliveryDestination` swallows only a not-found.
|
|
2440
|
+
if (!(err instanceof AwsError && /Conflict/i.test(err.code)))
|
|
2441
|
+
throw err;
|
|
2442
|
+
ctx.logger.step(`stale analytics delivery destination "${name}" from a previous stack - removing it and its delivery, and retrying`);
|
|
2443
|
+
await clearPluginDeliveries(ctx);
|
|
2444
|
+
await logs(ctx).deleteDeliveryDestination(name);
|
|
2445
|
+
await putLogDestination(ctx, name, streamArn);
|
|
2446
|
+
}
|
|
2447
|
+
},
|
|
2448
|
+
async update(ctx) {
|
|
2449
|
+
const name = logDestinationName(ctx);
|
|
2450
|
+
const recorded = recordedText(ctx, LOG_DESTINATION_NODE, 'outputFormat');
|
|
2451
|
+
// A state file carrying no recorded format is NOT treated as a mismatch,
|
|
2452
|
+
// which is the opposite of `analytics-firehose-stream`'s `undefined`
|
|
2453
|
+
// handling and for the opposite reason. There, pushing the desired
|
|
2454
|
+
// configuration is an in-place `UpdateDestination` that costs nothing;
|
|
2455
|
+
// here it is a delete and a re-create that drops the delivery and loses
|
|
2456
|
+
// the records arriving in the gap. Destructive on a guess is the wrong
|
|
2457
|
+
// direction, and the re-put below still converges everything mutable.
|
|
2458
|
+
if (recorded !== undefined && recorded !== DELIVERY_OUTPUT_FORMAT) {
|
|
2459
|
+
// **The output format is immutable once a destination exists**, so this
|
|
2460
|
+
// is a replacement and not an update: `PutDeliveryDestination` over a
|
|
2461
|
+
// live destination does not change the format it renders records in
|
|
2462
|
+
// (the change spec's §`LogsClient` delivery configuration says so in as
|
|
2463
|
+
// many words). Delete, then re-create.
|
|
2464
|
+
//
|
|
2465
|
+
// This plugin's own delivery has to come off first - AWS rejects
|
|
2466
|
+
// `DeleteDeliveryDestination` while a delivery points at it. That
|
|
2467
|
+
// leaves the delivery missing, which is exactly what
|
|
2468
|
+
// `analytics-log-delivery`'s `read` is written to notice: it lists the
|
|
2469
|
+
// deliveries on the site's source rather than trusting its own state,
|
|
2470
|
+
// so the same reconcile pass re-creates it. This node is its declared
|
|
2471
|
+
// dependency, so it always runs first.
|
|
2472
|
+
ctx.logger.warn(`the analytics delivery destination "${name}" was created with output format "${recorded}" and this build needs "${DELIVERY_OUTPUT_FORMAT}", which cannot be changed in place - replacing it, and the records arriving during the gap are lost`);
|
|
2473
|
+
await clearPluginDeliveries(ctx);
|
|
2474
|
+
await logs(ctx).deleteDeliveryDestination(name);
|
|
2475
|
+
}
|
|
2476
|
+
// Re-put on every reconcile, `bucketPolicyNode`'s discipline
|
|
2477
|
+
// (`packages/cli/src/nodes.ts`): `PutDeliveryDestination` is an
|
|
2478
|
+
// idempotent upsert and the resource ARN it carries is not fixed.
|
|
2479
|
+
// `analytics-firehose-stream`'s own `update` falls back to *replacing*
|
|
2480
|
+
// the stream, and a replacement carries a NEW ARN - "the CloudFront log
|
|
2481
|
+
// delivery has to be reconciled against it", in that node's own words.
|
|
2482
|
+
// This is the reconcile that does it.
|
|
2483
|
+
const streamArn = requireStreamArn(ctx);
|
|
2484
|
+
await requireActiveStream(ctx);
|
|
2485
|
+
await putLogDestination(ctx, name, streamArn);
|
|
2486
|
+
},
|
|
2487
|
+
async delete(ctx) {
|
|
2488
|
+
// Only this plugin's destination, and never the shared delivery source.
|
|
2489
|
+
// The delivery pointing at it is gone by now: `destroyGraph` walks the
|
|
2490
|
+
// topological order in reverse (`packages/cli/src/graph.ts`), so
|
|
2491
|
+
// `analytics-log-delivery` - which declares this node as its dependency -
|
|
2492
|
+
// has already run. That is the same delivery-before-destination ordering
|
|
2493
|
+
// the site's own teardown comment documents at
|
|
2494
|
+
// `packages/cli/src/nodes.ts:763-768`, expressed as an edge rather than
|
|
2495
|
+
// as two statements in one node, because here the two resources are two
|
|
2496
|
+
// nodes. `deleteDeliveryDestination` swallows a not-found, so a
|
|
2497
|
+
// half-finished teardown is re-runnable.
|
|
2498
|
+
await logs(ctx).deleteDeliveryDestination(logDestinationName(ctx));
|
|
2499
|
+
},
|
|
2500
|
+
};
|
|
2501
|
+
}
|
|
2502
|
+
/**
|
|
2503
|
+
* The site's delivery source and the distribution behind it, or a throw naming
|
|
2504
|
+
* `blogwright bootstrap` as the fix.
|
|
2505
|
+
*
|
|
2506
|
+
* **Both are read off the site and neither is written.** The name comes from
|
|
2507
|
+
* `ctx.names`, the deterministic set core derived for this environment, and
|
|
2508
|
+
* the distribution ARN from `ctx.siteState` - the read-only view of
|
|
2509
|
+
* `state/<env>.json` the SPI provides, every property `readonly` all the way
|
|
2510
|
+
* into the map values. Never from a `StateStore` constructed over the site's
|
|
2511
|
+
* key: `ctx.store`, `ctx.state` and `ctx.save()` are all scoped to
|
|
2512
|
+
* `state/<env>.analytics.json`, and `siteState` is the only route to the
|
|
2513
|
+
* site's own file. That distinction typechecks either way, so it is stated
|
|
2514
|
+
* here rather than left to be noticed in an S3 key.
|
|
2515
|
+
*
|
|
2516
|
+
* The distribution ARN is what makes this a bootstrap check rather than a
|
|
2517
|
+
* derivation. `ctx.names.deliverySource` is a pure function of the
|
|
2518
|
+
* environment, so it names a source whether or not one exists; the site's
|
|
2519
|
+
* recorded distribution ARN is the observable saying the site graph has
|
|
2520
|
+
* actually run, and the site's node creates the delivery source in the same
|
|
2521
|
+
* `wire()` that records it (`packages/cli/src/nodes.ts:713-734`).
|
|
2522
|
+
*
|
|
2523
|
+
* The source is checked too, even though `deriveNames` cannot produce an empty
|
|
2524
|
+
* one, so that the guard's name and its body agree on their own - the same
|
|
2525
|
+
* re-application `packages/cli/src/nodes.ts`'s `ownDeliveryIdsOrRefuse` makes
|
|
2526
|
+
* of its own predicate, and for the same reason.
|
|
2527
|
+
*/
|
|
2528
|
+
function requireSiteDeliverySource(ctx) {
|
|
2529
|
+
const source = ctx.names.deliverySource;
|
|
2530
|
+
const distribution = ctx.siteState.resources[SITE_DISTRIBUTION_NODE]?.arn;
|
|
2531
|
+
if (source === '' || typeof distribution !== 'string' || distribution === '') {
|
|
2532
|
+
const missing = source === ''
|
|
2533
|
+
? 'no delivery source name was derived for this environment'
|
|
2534
|
+
: `${SITE_DISTRIBUTION_NODE} has no recorded ARN in the site's state`;
|
|
2535
|
+
throw new Error(`the "${ctx.env}" site's CloudFront log delivery source is not available (${missing}), and this plugin never creates one - it hangs its delivery off the source the site already owns; run \`blogwright bootstrap ${ctx.env}\` first`);
|
|
2536
|
+
}
|
|
2537
|
+
return { source, distribution };
|
|
2538
|
+
}
|
|
2539
|
+
/**
|
|
2540
|
+
* Today's UTC day as `YYYY-MM-DD`. `Date.prototype.toISOString` renders in UTC
|
|
2541
|
+
* by definition, so the day recorded here is in the same calendar as the
|
|
2542
|
+
* table's `day` partition, which `transform/map-record.ts` derives from
|
|
2543
|
+
* CloudFront's `timestamp(ms)` - also UTC. The two have to agree, because task
|
|
2544
|
+
* 61's backfill compares partition days against this bound.
|
|
2545
|
+
*/
|
|
2546
|
+
function utcDay(now) {
|
|
2547
|
+
return now.toISOString().slice(0, ISO_DAY_LENGTH);
|
|
2548
|
+
}
|
|
2549
|
+
/**
|
|
2550
|
+
* The delivery joining the site's existing delivery source to this plugin's
|
|
2551
|
+
* destination - **a second delivery on a source the plugin reads, never
|
|
2552
|
+
* creates, never repoints and never deletes.**
|
|
2553
|
+
*
|
|
2554
|
+
* `putDeliverySource` is not called here and must never be. AWS permits one
|
|
2555
|
+
* delivery source per distribution and the site's node owns it
|
|
2556
|
+
* (`packages/cli/src/nodes.ts`'s `logDeliveryNode`); this node reads its name
|
|
2557
|
+
* off `ctx.names` and attaches a second delivery beside the site's CloudWatch
|
|
2558
|
+
* one. The site's copy is left with the field list AWS defaults to, which is
|
|
2559
|
+
* deliberate and is `schema.ts`'s to explain.
|
|
2560
|
+
*
|
|
2561
|
+
* There is no `update`. CloudWatch Logs has no `UpdateDelivery`: the record
|
|
2562
|
+
* fields a delivery selects are fixed when it is created, exactly as the
|
|
2563
|
+
* `page_views` table's schema is fixed when *it* is created
|
|
2564
|
+
* (`aws/s3tables.ts`'s `createTable` reconciles no existing schema). Changing
|
|
2565
|
+
* the column set is a rebuild of this pipeline in both places, not a
|
|
2566
|
+
* reconcile, and pretending otherwise in one of the two would be worse than
|
|
2567
|
+
* saying so in both.
|
|
2568
|
+
*/
|
|
2569
|
+
export function analyticsLogDeliveryNode() {
|
|
2570
|
+
return {
|
|
2571
|
+
id: LOG_DELIVERY_NODE,
|
|
2572
|
+
dependsOn: [LOG_DESTINATION_NODE],
|
|
2573
|
+
title: `CloudFront log delivery to the analytics stream (${ANALYTICS_REGION})`,
|
|
2574
|
+
async read(ctx) {
|
|
2575
|
+
// AWS, not state, and for a reason the state cannot cover: this
|
|
2576
|
+
// delivery lives on a source two stacks share, so it can go missing
|
|
2577
|
+
// without this plugin doing anything - the destination replacement below
|
|
2578
|
+
// detaches it on purpose, and a site re-bootstrap can reach it too.
|
|
2579
|
+
// Listing is also what `delete` has to do anyway (`createDelivery`
|
|
2580
|
+
// returns no id, so there is nothing to record and look up later), so
|
|
2581
|
+
// this costs one call that the teardown path already pays.
|
|
2582
|
+
const site = requireSiteDeliverySource(ctx);
|
|
2583
|
+
const destinationName = logDestinationName(ctx);
|
|
2584
|
+
const found = (await logs(ctx).deliveriesForSource(site.source)).find((delivery) => isPluginDelivery(delivery, destinationName));
|
|
2585
|
+
if (found === undefined)
|
|
2586
|
+
return false;
|
|
2587
|
+
const out = output(ctx, LOG_DELIVERY_NODE);
|
|
2588
|
+
out.source = site.source;
|
|
2589
|
+
out.destination = found.deliveryDestinationArn;
|
|
2590
|
+
out.distribution = site.distribution;
|
|
2591
|
+
out.delivery = 'configured';
|
|
2592
|
+
// `createdDay` is deliberately absent from this hydration - see
|
|
2593
|
+
// {@link CREATED_DAY_KEY}. `output` re-records rather than replaces, so
|
|
2594
|
+
// one already in state survives this untouched.
|
|
2595
|
+
return true;
|
|
2596
|
+
},
|
|
2597
|
+
async create(ctx) {
|
|
2598
|
+
// Both reads happen before the call, so an unbootstrapped site and a
|
|
2599
|
+
// missing destination each fail with nothing sent.
|
|
2600
|
+
const site = requireSiteDeliverySource(ctx);
|
|
2601
|
+
const destinationArn = requireLogDestinationArn(ctx);
|
|
2602
|
+
await logs(ctx).createDelivery(site.source, destinationArn, {
|
|
2603
|
+
// `CLOUDFRONT_RECORD_FIELDS`, never a list restated here: `schema.ts`
|
|
2604
|
+
// owns which CloudFront fields exist, which ones fill a column, and
|
|
2605
|
+
// which two (`cs(Cookie)`, `x-forwarded-for`) are excluded because they
|
|
2606
|
+
// carry personal data with no analytic use.
|
|
2607
|
+
//
|
|
2608
|
+
// No `fieldDelimiter`, deliberately. AWS documents `createDelivery`'s
|
|
2609
|
+
// delimiter as applying "when the final output format of a delivery is
|
|
2610
|
+
// in plain, w3c, or raw format", and {@link DELIVERY_OUTPUT_FORMAT} is
|
|
2611
|
+
// `json` because the transform Lambda parses each record with
|
|
2612
|
+
// `JSON.parse`. Sending a delimiter with a JSON delivery would be a
|
|
2613
|
+
// request field with no meaning for this delivery at best, and a
|
|
2614
|
+
// `ValidationException` that fails every bootstrap at worst.
|
|
2615
|
+
recordFields: CLOUDFRONT_RECORD_FIELDS,
|
|
2616
|
+
});
|
|
2617
|
+
// **Record ordering.** Everything lands after the one call returns, for
|
|
2618
|
+
// {@link putLogDestination}'s reason: this node makes a single mutating
|
|
2619
|
+
// call, so there is no interval between two of them for a crash to fall
|
|
2620
|
+
// into, and the incremental recording at
|
|
2621
|
+
// `packages/cli/src/nodes.ts:717-719` is answering a problem this node
|
|
2622
|
+
// does not have. `createdDay` in particular must not be written ahead of
|
|
2623
|
+
// the call: a day recorded for a delivery that was never created is a
|
|
2624
|
+
// backfill bound covering records Firehose never received.
|
|
2625
|
+
const out = output(ctx, LOG_DELIVERY_NODE);
|
|
2626
|
+
out.source = site.source;
|
|
2627
|
+
out.destination = destinationArn;
|
|
2628
|
+
out.distribution = site.distribution;
|
|
2629
|
+
out.delivery = 'configured';
|
|
2630
|
+
// Write-once, and the only place this key is ever written. See
|
|
2631
|
+
// {@link CREATED_DAY_KEY} for why moving it later is the one direction
|
|
2632
|
+
// that corrupts data rather than merely losing some.
|
|
2633
|
+
if (typeof out[CREATED_DAY_KEY] !== 'string')
|
|
2634
|
+
out[CREATED_DAY_KEY] = utcDay(new Date());
|
|
2635
|
+
},
|
|
2636
|
+
async delete(ctx) {
|
|
2637
|
+
// This plugin's own deliveries and nothing else - **never
|
|
2638
|
+
// `deleteDeliverySource`**. The site's teardown deletes the source
|
|
2639
|
+
// (`packages/cli/src/nodes.ts:773`) because the site owns it; this one
|
|
2640
|
+
// must not, and task 52's guard on that node is what stops the site's
|
|
2641
|
+
// teardown running while this delivery still exists. Removing the source
|
|
2642
|
+
// from here would take the site's own CloudWatch delivery with it.
|
|
2643
|
+
//
|
|
2644
|
+
// Looked up by destination rather than recorded at create time, because
|
|
2645
|
+
// `createDelivery` answers with nothing - there is no id to record - and
|
|
2646
|
+
// rather than by `findDeliveryIdBySource`, which returns whichever
|
|
2647
|
+
// delivery AWS lists first and on this shared source may well return the
|
|
2648
|
+
// site's. `deleteDelivery` swallows a not-found, so a half-finished
|
|
2649
|
+
// teardown is re-runnable.
|
|
2650
|
+
await clearPluginDeliveries(ctx);
|
|
2651
|
+
},
|
|
2652
|
+
};
|
|
2653
|
+
}
|
|
2654
|
+
/**
|
|
2655
|
+
* The plugin's twelve resource nodes, assembled in the order the change spec's
|
|
2656
|
+
* §Analytics pipeline → Resource nodes table lists them. This is what
|
|
2657
|
+
* `Plugin.nodes` (`plugin.ts`) hands the CLI's generic `analytics bootstrap`
|
|
2658
|
+
* and `analytics destroy` verbs, and it is the whole of what this package
|
|
2659
|
+
* contributes to a reconcile: the engine that walks them - `topoSort`,
|
|
2660
|
+
* `applyGraph`, `destroyGraph` - is the CLI's own and is never reimplemented
|
|
2661
|
+
* here.
|
|
2662
|
+
*
|
|
2663
|
+
* **The returned order is itself a topological order**, and that is a property
|
|
2664
|
+
* of this array rather than a coincidence of the table's layout: every node's
|
|
2665
|
+
* `dependsOn` names only nodes that appear EARLIER in it. That is worth stating
|
|
2666
|
+
* because it is exactly the witness `topoSort`'s two failure modes are the
|
|
2667
|
+
* absence of - a dependency naming a node outside the set, and a cycle - so a
|
|
2668
|
+
* test that checks it has proved the set passes `topoSort` without running a
|
|
2669
|
+
* second copy of `topoSort` to find out. `applyGraph` sorts the array again
|
|
2670
|
+
* regardless and does not rely on the order it arrives in; nothing here may
|
|
2671
|
+
* assume the reconcile follows this sequence, only that this sequence is a
|
|
2672
|
+
* legal one.
|
|
2673
|
+
*
|
|
2674
|
+
* **No `ctx` parameter, deliberately.** The SPI declares `nodes?(ctx)` and the
|
|
2675
|
+
* CLI calls it with one, so this function is assignable to it as written - a
|
|
2676
|
+
* zero-argument function satisfies a one-argument signature. None of the twelve
|
|
2677
|
+
* factories needs a context to be *built*: each reads `ctx` inside `read`,
|
|
2678
|
+
* `create`, `update` and `delete`, when the reconcile is actually running. A
|
|
2679
|
+
* parameter accepted and ignored here would be an unused binding and, worse, a
|
|
2680
|
+
* claim that the SET varies with the context - it does not, and `analytics
|
|
2681
|
+
* status` and `analytics destroy` both depend on it not doing so. (The plan's
|
|
2682
|
+
* task 54 spells this function `buildAnalyticsNodes(ctx)`; the argument is what
|
|
2683
|
+
* changed, not the wiring.)
|
|
2684
|
+
*
|
|
2685
|
+
* A fresh array of fresh nodes on every call, matching `buildNodes`
|
|
2686
|
+
* (`packages/cli/src/nodes.ts`): a node object carries no state between
|
|
2687
|
+
* reconciles, and two calls in one process must not share one.
|
|
2688
|
+
*/
|
|
2689
|
+
export function buildAnalyticsNodes() {
|
|
2690
|
+
return [
|
|
2691
|
+
// The table chain.
|
|
2692
|
+
analyticsTableBucketNode(),
|
|
2693
|
+
analyticsNamespaceNode(),
|
|
2694
|
+
analyticsTableNode(),
|
|
2695
|
+
analyticsCatalogIntegrationNode(),
|
|
2696
|
+
// The transform chain.
|
|
2697
|
+
analyticsSaltSecretNode(),
|
|
2698
|
+
analyticsTransformRoleNode(),
|
|
2699
|
+
analyticsTransformFunctionNode(),
|
|
2700
|
+
// The delivery chain.
|
|
2701
|
+
analyticsErrorBucketNode(),
|
|
2702
|
+
analyticsFirehoseRoleNode(),
|
|
2703
|
+
analyticsFirehoseStreamNode(),
|
|
2704
|
+
// The vended-delivery chain.
|
|
2705
|
+
analyticsLogDestinationNode(),
|
|
2706
|
+
analyticsLogDeliveryNode(),
|
|
2707
|
+
];
|
|
2708
|
+
}
|