@fjall/components-infrastructure 16.0.1 → 18.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/config/aws/scpPreset.js +103 -17
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +103 -8
- package/dist/lib/patterns/aws/clickhouseDatabase.js +51 -12
- package/dist/lib/patterns/aws/database.d.ts +1 -1
- package/dist/lib/patterns/aws/database.js +1 -1
- package/dist/lib/patterns/aws/interfaces/domain.d.ts +1 -0
- package/dist/lib/patterns/aws/storage.d.ts +8 -0
- package/dist/lib/patterns/aws/storage.js +10 -3
- package/dist/lib/patterns/aws/targets/fjallTargets.d.ts +7 -5
- package/dist/lib/patterns/aws/targets/fjallTargets.js +9 -6
- package/dist/lib/patterns/aws/targets/targetResolution.d.ts +8 -5
- package/dist/lib/patterns/aws/targets/targetResolution.js +11 -8
- package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts +103 -0
- package/dist/lib/resources/aws/database/clickhouseBackupScript.js +123 -0
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +132 -20
- package/dist/lib/resources/aws/database/clickhouseConstants.js +126 -15
- package/dist/lib/resources/aws/database/clickhouseSchemas.d.ts +33 -1
- package/dist/lib/resources/aws/database/clickhouseSchemas.js +31 -0
- package/dist/lib/resources/aws/database/clickhouseStorage.d.ts +58 -0
- package/dist/lib/resources/aws/database/clickhouseStorage.js +87 -0
- package/dist/lib/resources/aws/database/clickhouseTuning.d.ts +5 -6
- package/dist/lib/resources/aws/database/clickhouseTuning.js +39 -16
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +21 -1
- package/dist/lib/resources/aws/database/clickhouseUserData.js +22 -6
- package/dist/lib/resources/aws/database/clickhouseXmlRenderer.js +22 -1
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts +54 -5
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js +106 -10
- package/dist/lib/resources/aws/storage/s3.d.ts +19 -7
- package/dist/lib/resources/aws/storage/s3.js +15 -3
- package/package.json +3 -3
|
@@ -15,6 +15,108 @@ const EXEMPT_ROLE_PATTERNS = [
|
|
|
15
15
|
const SCP_BYTE_LIMIT = 5120;
|
|
16
16
|
const IAM_POLICY_VERSION = "2012-10-17";
|
|
17
17
|
const SCP_POLICY_TYPE = "SERVICE_CONTROL_POLICY";
|
|
18
|
+
// Region-deny exemptions: global and us-east-1-locked services evaluate
|
|
19
|
+
// aws:RequestedRegion as us-east-1, so a region deny without these blocks
|
|
20
|
+
// their consoles outright for orgs whose allowedRegions exclude us-east-1.
|
|
21
|
+
// Mirrors the NotAction list of AWS Control Tower's Region deny control
|
|
22
|
+
// (GRREGIONDENY) so future re-syncs are a mechanical diff:
|
|
23
|
+
// https://docs.aws.amazon.com/controltower/latest/controlreference/primary-region-deny-policy.html
|
|
24
|
+
// Deliberate deltas from that list:
|
|
25
|
+
// - "support-console:*" — Support Center console auth moves to this prefix
|
|
26
|
+
// on 2026-11-16 (support:* remains the API prefix); predates the CT list.
|
|
27
|
+
// - "purchase-orders:*" — the one aws-portal-successor billing prefix the CT
|
|
28
|
+
// list is missing.
|
|
29
|
+
// - "tag:*" — grandfathered superset of CT's "tag:GetResources"; narrowing
|
|
30
|
+
// a deployed deny would be a breaking tightening.
|
|
31
|
+
const GLOBAL_SERVICE_EXEMPTIONS = [
|
|
32
|
+
"a4b:*",
|
|
33
|
+
"access-analyzer:*",
|
|
34
|
+
"account:*",
|
|
35
|
+
"acm:*",
|
|
36
|
+
"activate:*",
|
|
37
|
+
"artifact:*",
|
|
38
|
+
"aws-marketplace-management:*",
|
|
39
|
+
"aws-marketplace:*",
|
|
40
|
+
"aws-portal:*",
|
|
41
|
+
"billing:*",
|
|
42
|
+
"billingconductor:*",
|
|
43
|
+
"budgets:*",
|
|
44
|
+
"ce:*",
|
|
45
|
+
"chatbot:*",
|
|
46
|
+
"chime:*",
|
|
47
|
+
"cloudfront:*",
|
|
48
|
+
"cloudtrail:LookupEvents",
|
|
49
|
+
"compute-optimizer:*",
|
|
50
|
+
"config:*",
|
|
51
|
+
"consoleapp:*",
|
|
52
|
+
"consolidatedbilling:*",
|
|
53
|
+
"cur:*",
|
|
54
|
+
"datapipeline:GetAccountLimits",
|
|
55
|
+
"devicefarm:*",
|
|
56
|
+
"directconnect:*",
|
|
57
|
+
"ec2:DescribeRegions",
|
|
58
|
+
"ec2:DescribeTransitGateways",
|
|
59
|
+
"ec2:DescribeVpnGateways",
|
|
60
|
+
"ecr-public:*",
|
|
61
|
+
"fms:*",
|
|
62
|
+
"freetier:*",
|
|
63
|
+
"globalaccelerator:*",
|
|
64
|
+
"health:*",
|
|
65
|
+
"iam:*",
|
|
66
|
+
"importexport:*",
|
|
67
|
+
"invoicing:*",
|
|
68
|
+
"iq:*",
|
|
69
|
+
"kms:*",
|
|
70
|
+
"license-manager:ListReceivedLicenses",
|
|
71
|
+
"lightsail:Get*",
|
|
72
|
+
"mobileanalytics:*",
|
|
73
|
+
"networkmanager:*",
|
|
74
|
+
"notifications-contacts:*",
|
|
75
|
+
"notifications:*",
|
|
76
|
+
"organizations:*",
|
|
77
|
+
"payments:*",
|
|
78
|
+
"pricing:*",
|
|
79
|
+
"purchase-orders:*",
|
|
80
|
+
"quicksight:DescribeAccountSubscription",
|
|
81
|
+
"resource-explorer-2:*",
|
|
82
|
+
"route53-recovery-cluster:*",
|
|
83
|
+
"route53-recovery-control-config:*",
|
|
84
|
+
"route53-recovery-readiness:*",
|
|
85
|
+
"route53:*",
|
|
86
|
+
"route53domains:*",
|
|
87
|
+
"s3:CreateMultiRegionAccessPoint",
|
|
88
|
+
"s3:DeleteMultiRegionAccessPoint",
|
|
89
|
+
"s3:DescribeMultiRegionAccessPointOperation",
|
|
90
|
+
"s3:GetAccountPublicAccessBlock",
|
|
91
|
+
"s3:GetBucketLocation",
|
|
92
|
+
"s3:GetBucketPolicyStatus",
|
|
93
|
+
"s3:GetBucketPublicAccessBlock",
|
|
94
|
+
"s3:GetMultiRegionAccessPoint",
|
|
95
|
+
"s3:GetMultiRegionAccessPointPolicy",
|
|
96
|
+
"s3:GetMultiRegionAccessPointPolicyStatus",
|
|
97
|
+
"s3:GetStorageLensConfiguration",
|
|
98
|
+
"s3:GetStorageLensDashboard",
|
|
99
|
+
"s3:ListAllMyBuckets",
|
|
100
|
+
"s3:ListMultiRegionAccessPoints",
|
|
101
|
+
"s3:ListStorageLensConfigurations",
|
|
102
|
+
"s3:PutAccountPublicAccessBlock",
|
|
103
|
+
"s3:PutMultiRegionAccessPointPolicy",
|
|
104
|
+
"savingsplans:*",
|
|
105
|
+
"shield:*",
|
|
106
|
+
"sso:*",
|
|
107
|
+
"sts:*",
|
|
108
|
+
"support-console:*",
|
|
109
|
+
"support:*",
|
|
110
|
+
"supportapp:*",
|
|
111
|
+
"supportplans:*",
|
|
112
|
+
"sustainability:*",
|
|
113
|
+
"tag:*",
|
|
114
|
+
"trustedadvisor:*",
|
|
115
|
+
"vendor-insights:ListEntitledSecurityProfiles",
|
|
116
|
+
"waf-regional:*",
|
|
117
|
+
"waf:*",
|
|
118
|
+
"wafv2:*"
|
|
119
|
+
];
|
|
18
120
|
function automationExemption() {
|
|
19
121
|
return {
|
|
20
122
|
ArnNotLike: {
|
|
@@ -62,23 +164,7 @@ function buildFoundationGuardrails(allowedRegions) {
|
|
|
62
164
|
{
|
|
63
165
|
Sid: "DenyOutsideAllowedRegions",
|
|
64
166
|
Effect: "Deny",
|
|
65
|
-
NotAction: [
|
|
66
|
-
"iam:*",
|
|
67
|
-
"sts:*",
|
|
68
|
-
"cloudfront:*",
|
|
69
|
-
"route53:*",
|
|
70
|
-
"organizations:*",
|
|
71
|
-
"support:*",
|
|
72
|
-
"budgets:*",
|
|
73
|
-
"ce:*",
|
|
74
|
-
"waf:*",
|
|
75
|
-
"wafv2:*",
|
|
76
|
-
"globalaccelerator:*",
|
|
77
|
-
"health:*",
|
|
78
|
-
"account:*",
|
|
79
|
-
"tag:*",
|
|
80
|
-
"trustedadvisor:*"
|
|
81
|
-
],
|
|
167
|
+
NotAction: [...GLOBAL_SERVICE_EXEMPTIONS],
|
|
82
168
|
Resource: "*",
|
|
83
169
|
Condition: {
|
|
84
170
|
StringNotEquals: {
|
|
@@ -29,15 +29,82 @@ export interface ClickHouseDatabaseProps {
|
|
|
29
29
|
* taking a new type live needs a launch-template refresh + instance
|
|
30
30
|
* refresh + ClickHouse restart, not a config reload.
|
|
31
31
|
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* `
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
32
|
+
* Changing this on a DEPLOYED stack keeps the data. The volume's logical
|
|
33
|
+
* ID and its `fjall:OwnerLogicalId` re-attach tag both derive from the
|
|
34
|
+
* capacity ANCHOR (`resolveCapacityAnchor`, `${Aws.STACK_NAME}/${anchor}/
|
|
35
|
+
* PersistentDataVolume`), not from the instance type, so neither moves
|
|
36
|
+
* when the type does and the re-attach chain holds. Config-keyed identity
|
|
37
|
+
* was the pre-6.0 derivation — `getEc2ConfigKey` survives only to emit
|
|
38
|
+
* the manifest's legacy alias.
|
|
39
|
+
*
|
|
40
|
+
* It is still not free: the type lives in the launch template and in
|
|
41
|
+
* user-data-baked server tuning, so it takes an instance refresh, and on
|
|
42
|
+
* this single-node cluster that refresh is a restart. The ASG terminates
|
|
43
|
+
* the instance (TERMINATING hook detaches the volume), launches a
|
|
44
|
+
* replacement, and the LAUNCHING hook re-attaches — ClickHouse is
|
|
45
|
+
* unavailable across that window. Size the change into a quiet period;
|
|
46
|
+
* do not plan a snapshot-restore migration for it. */
|
|
40
47
|
instanceType?: string;
|
|
48
|
+
/**
|
|
49
|
+
* EBS data-volume size in GiB. Default `CLICKHOUSE_EBS_VOLUME_SIZE_GB`
|
|
50
|
+
* (400). Integer, between `CLICKHOUSE_MIN_STORAGE_GB` and the gp3 ceiling;
|
|
51
|
+
* validated against `alarms.diskFreeCriticalGib` as a pair, because a
|
|
52
|
+
* volume smaller than five times that floor would deploy a critical alarm
|
|
53
|
+
* that can never leave ALARM.
|
|
54
|
+
*
|
|
55
|
+
* GROWTH ONLY, and the growth is not instantaneous. EBS `ModifyVolume`
|
|
56
|
+
* refuses to shrink a volume, so a decrease fails mid-update and rolls the
|
|
57
|
+
* stack back; it also permits one modification per volume per 6 hours, so
|
|
58
|
+
* a second bump inside that window fails until the cooldown elapses.
|
|
59
|
+
*
|
|
60
|
+
* Resizing the block device is half the job — the ext4 filesystem on it
|
|
61
|
+
* does not follow until `resize2fs` runs, which is a step in
|
|
62
|
+
* `buildClickHouseUserData` and therefore runs at BOOT. The declared size
|
|
63
|
+
* is embedded in that user data, so changing this prop versions the launch
|
|
64
|
+
* template and triggers the ASG instance refresh whose boot grows the
|
|
65
|
+
* filesystem — the increase applies in the same deploy, by the same
|
|
66
|
+
* mechanism an `instanceType` change rides. On this single-node cluster
|
|
67
|
+
* that relaunch is a restart (the TERMINATING hook detaches the volume,
|
|
68
|
+
* LAUNCHING re-attaches it): the data survives; the cluster is unavailable
|
|
69
|
+
* across the window. Size increases into a quiet period, exactly as with
|
|
70
|
+
* `instanceType`.
|
|
71
|
+
*/
|
|
72
|
+
storageGb?: number;
|
|
73
|
+
/**
|
|
74
|
+
* Provisioned IOPS on the gp3 data volume. Default `CLICKHOUSE_EBS_IOPS`
|
|
75
|
+
* (3000, the baseline the volume price includes). Integer, between
|
|
76
|
+
* `CLICKHOUSE_MIN_IOPS` and `CLICKHOUSE_MAX_IOPS`, and at most
|
|
77
|
+
* `CLICKHOUSE_MAX_IOPS_PER_GB` × `storageGb` — EBS rejects more IOPS than
|
|
78
|
+
* the volume is large enough to carry. Validated with `throughputMbps` as
|
|
79
|
+
* a pair, because throughput needs IOPS to ride on (below).
|
|
80
|
+
*
|
|
81
|
+
* The same knob RDS exposes as provisioned IOPS on gp3 storage. Applied by
|
|
82
|
+
* EBS `ModifyVolume` in place: no launch-template change, no instance
|
|
83
|
+
* relaunch, no restart — the cluster stays up. It shares the
|
|
84
|
+
* one-modification-per-volume-per-6-hours cooldown with `storageGb` and
|
|
85
|
+
* `throughputMbps`; a change to any of the three inside that window fails
|
|
86
|
+
* until it elapses, so move them together.
|
|
87
|
+
*
|
|
88
|
+
* Worth raising only once the host can use it: a volume's provisioned
|
|
89
|
+
* figure is a ceiling the instance's own EBS bandwidth may sit under, so
|
|
90
|
+
* pair a large value with an `instanceType` whose EBS bandwidth reaches it.
|
|
91
|
+
*/
|
|
92
|
+
iops?: number;
|
|
93
|
+
/**
|
|
94
|
+
* Provisioned throughput on the gp3 data volume, in MiB/s. Default
|
|
95
|
+
* `CLICKHOUSE_EBS_THROUGHPUT_MBPS` (125, the baseline the volume price
|
|
96
|
+
* includes). Integer, between `CLICKHOUSE_MIN_THROUGHPUT_MBPS` and
|
|
97
|
+
* `CLICKHOUSE_MAX_THROUGHPUT_MBPS`, and needing
|
|
98
|
+
* `CLICKHOUSE_IOPS_PER_THROUGHPUT_MBPS` (4) IOPS per MiB/s — 1000 MiB/s
|
|
99
|
+
* needs `iops: 4000` or more, and the synth error names the figure.
|
|
100
|
+
*
|
|
101
|
+
* The same knob RDS exposes as storage throughput on gp3. Applied in place
|
|
102
|
+
* by `ModifyVolume` like `iops`, under the same shared 6-hour cooldown, and
|
|
103
|
+
* likewise bounded by the instance's EBS bandwidth. The restore-verify
|
|
104
|
+
* timeout is derived from the baseline rather than this value on purpose
|
|
105
|
+
* (see `clickHouseBackupVerifyTimeoutSeconds`).
|
|
106
|
+
*/
|
|
107
|
+
throughputMbps?: number;
|
|
41
108
|
/**
|
|
42
109
|
* ECS service desired task count. Defaults to 1 (single-node, single-task
|
|
43
110
|
* by design). Override to 0 via the `clickhouseDesiredCount` CDK context
|
|
@@ -57,6 +124,12 @@ export interface ClickHouseDatabaseProps {
|
|
|
57
124
|
* - omitted/undefined: cold tier ENABLED (S3 bucket provisioned).
|
|
58
125
|
* - `false`: cold tier DISABLED (single-tier hot storage on EBS only).
|
|
59
126
|
* - `{ mode: "s3" }`: cold tier ENABLED (explicit).
|
|
127
|
+
*
|
|
128
|
+
* Interacts with the backup restore-verify: the verify restores onto the
|
|
129
|
+
* EBS volume, so a cold-heavy database can outgrow the free-space gate
|
|
130
|
+
* long before the hot tier fills — from that day the verify skips (and its
|
|
131
|
+
* alarm pages daily) with no cure but a larger `storageGb`. When that
|
|
132
|
+
* trade is deliberate, `backupVerify: false` is the sanctioned off-switch.
|
|
60
133
|
*/
|
|
61
134
|
coldTier?: false | {
|
|
62
135
|
mode: "s3";
|
|
@@ -85,10 +158,32 @@ export interface ClickHouseDatabaseProps {
|
|
|
85
158
|
* schedules sparser than ~5.8 days must disable the heartbeat.
|
|
86
159
|
*/
|
|
87
160
|
backupSchedule?: string | false;
|
|
161
|
+
/**
|
|
162
|
+
* Restore-verify on each scheduled backup. Omitted (the default): after
|
|
163
|
+
* `BACKUP DATABASE … TO S3` returns, the task restores what it just wrote
|
|
164
|
+
* into a scratch database, compares the table set, and emits a verdict
|
|
165
|
+
* marker the verify alarms read — the only signal that the backups are
|
|
166
|
+
* actually restorable. `false` disables the verify (the backup itself
|
|
167
|
+
* still runs and the backup-failure alarm + success heartbeat remain);
|
|
168
|
+
* the verify alarms are withheld with it, so disabling never reads as a
|
|
169
|
+
* silent green. Reach for it when the restore cost is structurally
|
|
170
|
+
* unaffordable on-box — a cold-heavy dataset (see `coldTier`), or a
|
|
171
|
+
* volume deliberately sized tight — and accept that `BACKUP_CREATED`
|
|
172
|
+
* becomes the only evidence a backup exists.
|
|
173
|
+
*/
|
|
174
|
+
backupVerify?: false;
|
|
88
175
|
/**
|
|
89
176
|
* Backup-bucket lifecycle expiration in days for both current and noncurrent
|
|
90
177
|
* versions. Default: `BACKUP_RETENTION_DAYS` from `clickhouseConstants.ts`.
|
|
91
178
|
* Ignored when `backupBucket` is supplied (caller-owned lifecycle).
|
|
179
|
+
* Must exceed the `backupSchedule` interval — a retention shorter than the
|
|
180
|
+
* gap between runs expires each backup before its successor exists, which
|
|
181
|
+
* is a window with no backup at all rather than a short history.
|
|
182
|
+
*
|
|
183
|
+
* An integer between `CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS` and
|
|
184
|
+
* `CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS`. This is an S3 object lifetime and
|
|
185
|
+
* deliberately does NOT share the RDS 35-day ceiling, which bounds a
|
|
186
|
+
* different mechanism (`backupRetention` on the relational types).
|
|
92
187
|
*/
|
|
93
188
|
backupRetentionDays?: number;
|
|
94
189
|
/**
|
|
@@ -23,7 +23,10 @@ import { createClickHouseAlarms, validateClickHouseAlarmThresholds, BACKUP_HEART
|
|
|
23
23
|
import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
24
24
|
import { deriveClickHouseDefaultProfiles } from "../../resources/aws/database/clickhouseTuning.js";
|
|
25
25
|
import { inferAmiHardwareType } from "../../resources/aws/compute/ecsConstants.js";
|
|
26
|
-
import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE,
|
|
26
|
+
import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
|
|
27
|
+
import { CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS, CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS } from "@fjall/util/clickhouse";
|
|
28
|
+
import { resolveClickHouseStorage, resolveClickHouseVolumePerformance } from "../../resources/aws/database/clickhouseStorage.js";
|
|
29
|
+
import { buildClickHouseBackupScript } from "../../resources/aws/database/clickhouseBackupScript.js";
|
|
27
30
|
import { TlsCertGenerator } from "../../resources/aws/utilities/tlsCertGenerator.js";
|
|
28
31
|
import { EcsCompute } from "./computeEcs.js";
|
|
29
32
|
/**
|
|
@@ -263,6 +266,7 @@ export class ClickHouseDatabase extends Construct {
|
|
|
263
266
|
const desiredCount = resolveClickHouseDesiredCount(this.node.tryGetContext("clickhouseDesiredCount"), props.desiredCount);
|
|
264
267
|
const optimiseEnabled = props.optimiseSchedule !== false;
|
|
265
268
|
const backupEnabled = props.backupSchedule !== false;
|
|
269
|
+
const backupVerifyEnabled = props.backupVerify !== false;
|
|
266
270
|
const optimiseSchedule = typeof props.optimiseSchedule === "string"
|
|
267
271
|
? props.optimiseSchedule
|
|
268
272
|
: OPTIMISE_FINAL_SCHEDULE;
|
|
@@ -270,9 +274,15 @@ export class ClickHouseDatabase extends Construct {
|
|
|
270
274
|
? props.backupSchedule
|
|
271
275
|
: BACKUP_SCHEDULE;
|
|
272
276
|
const backupRetentionDays = props.backupRetentionDays ?? BACKUP_RETENTION_DAYS;
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
277
|
+
// Checked after defaulting, so the default is held to the same bound as a
|
|
278
|
+
// caller's value. Integer-ness is part of the bound rather than a separate
|
|
279
|
+
// nicety: the value becomes `Duration.days()` on an S3 lifecycle rule,
|
|
280
|
+
// which rejects a fraction far downstream with no mention of the knob that
|
|
281
|
+
// supplied it.
|
|
282
|
+
if (!Number.isInteger(backupRetentionDays) ||
|
|
283
|
+
backupRetentionDays < CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS ||
|
|
284
|
+
backupRetentionDays > CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS) {
|
|
285
|
+
throw new Error(`ClickHouseDatabase: backupRetentionDays must be an integer between ${CLICKHOUSE_MIN_BACKUP_RETENTION_DAYS} and ${CLICKHOUSE_MAX_BACKUP_RETENTION_DAYS} days; got ${backupRetentionDays}.`);
|
|
276
286
|
}
|
|
277
287
|
const tlsOptions = props.tls ?? {
|
|
278
288
|
mode: "self-signed"
|
|
@@ -348,10 +358,29 @@ export class ClickHouseDatabase extends Construct {
|
|
|
348
358
|
retention: RetentionDays.TWO_WEEKS
|
|
349
359
|
})
|
|
350
360
|
: undefined;
|
|
361
|
+
// Resolved BEFORE the user data is built: the declared volume size is
|
|
362
|
+
// embedded into the script so a storageGb change versions the launch
|
|
363
|
+
// template (see `dataVolumeSizeGb` on BuildClickHouseUserDataOptions).
|
|
364
|
+
// The restore-verify gate is bounded by the same free-space floor the
|
|
365
|
+
// disk alarm pages on — the check that protects the data must not be
|
|
366
|
+
// able to fill the volume it is protecting. Volume size and floor are
|
|
367
|
+
// only checkable against each other, and the floor lives on the alarms
|
|
368
|
+
// knob.
|
|
369
|
+
const alarmThresholds = typeof props.alarms === "object" ? props.alarms : {};
|
|
370
|
+
validateClickHouseAlarmThresholds(alarmThresholds);
|
|
371
|
+
const { storageGb, diskFreeCriticalGib } = resolveClickHouseStorage(props.storageGb, alarmThresholds.diskFreeCriticalGib);
|
|
372
|
+
// IOPS and throughput are bounded by each other and by the size just
|
|
373
|
+
// resolved, so they resolve second, against the validated storageGb.
|
|
374
|
+
const { iops, throughputMbps } = resolveClickHouseVolumePerformance({
|
|
375
|
+
storageGb,
|
|
376
|
+
iops: props.iops,
|
|
377
|
+
throughputMbps: props.throughputMbps
|
|
378
|
+
});
|
|
351
379
|
const userData = UserData.custom(buildClickHouseUserData({
|
|
352
380
|
backupBucketName: backupBucket.bucketName,
|
|
353
381
|
backupBucketRegion: Stack.of(this).region,
|
|
354
382
|
instanceType,
|
|
383
|
+
dataVolumeSizeGb: storageGb,
|
|
355
384
|
...(coldTierBucket !== undefined && {
|
|
356
385
|
coldTier: {
|
|
357
386
|
bucketName: coldTierBucket.bucketName,
|
|
@@ -405,12 +434,15 @@ export class ClickHouseDatabase extends Construct {
|
|
|
405
434
|
// deploy-order window, or a dropped table lingering in the constants)
|
|
406
435
|
// would permanently skip every optimise after it. Failures are logged and
|
|
407
436
|
// the task still exits non-zero so they stay visible in the task history.
|
|
408
|
-
|
|
437
|
+
// Both scheduled tasks authenticate identically; the password arrives via
|
|
438
|
+
// CLICKHOUSE_PASSWORD rather than argv.
|
|
439
|
+
const clickHouseClient = `clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs}`;
|
|
409
440
|
const optimiseScript = [
|
|
410
441
|
"FAILED=0",
|
|
411
|
-
...optimiseStatements.map((stmt) => `${
|
|
442
|
+
...optimiseStatements.map((stmt) => `${clickHouseClient} --query "${stmt}" || { echo "optimise failed: ${stmt}"; FAILED=1; }`),
|
|
412
443
|
'exit "$FAILED"'
|
|
413
444
|
].join("; ");
|
|
445
|
+
// Resolved before the scheduled tasks because the backup task's
|
|
414
446
|
const scheduledTasks = [];
|
|
415
447
|
if (optimiseEnabled) {
|
|
416
448
|
scheduledTasks.push({
|
|
@@ -439,7 +471,14 @@ export class ClickHouseDatabase extends Construct {
|
|
|
439
471
|
"sh",
|
|
440
472
|
"-c",
|
|
441
473
|
// Password via CLICKHOUSE_PASSWORD env, not --password on argv (argv → /proc/<pid>/cmdline).
|
|
442
|
-
`${sidecarTlsPreamble}
|
|
474
|
+
`${sidecarTlsPreamble}${buildClickHouseBackupScript({
|
|
475
|
+
client: clickHouseClient,
|
|
476
|
+
databaseName: CLICKHOUSE_DATABASE_NAME,
|
|
477
|
+
backupDestUrl,
|
|
478
|
+
diskFreeCriticalGib,
|
|
479
|
+
storageGb,
|
|
480
|
+
verify: backupVerifyEnabled
|
|
481
|
+
})}`
|
|
443
482
|
],
|
|
444
483
|
secrets: {
|
|
445
484
|
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password"),
|
|
@@ -459,8 +498,6 @@ export class ClickHouseDatabase extends Construct {
|
|
|
459
498
|
const alarmsDisabled = props.alarms === false;
|
|
460
499
|
const applicationId = props.applicationId ??
|
|
461
500
|
(App.hasInstance() ? App.getInstance().getName() : undefined);
|
|
462
|
-
const alarmThresholds = typeof props.alarms === "object" ? props.alarms : {};
|
|
463
|
-
validateClickHouseAlarmThresholds(alarmThresholds);
|
|
464
501
|
const serviceLogAlarms = alarmsDisabled
|
|
465
502
|
? undefined
|
|
466
503
|
: buildClickHouseLogAlarmSpecs(alarmThresholds, coldTierEnabled);
|
|
@@ -520,11 +557,11 @@ export class ClickHouseDatabase extends Construct {
|
|
|
520
557
|
instanceMonitoring: Monitoring.BASIC,
|
|
521
558
|
availabilityZones: [dataAz],
|
|
522
559
|
persistentDataVolume: {
|
|
523
|
-
sizeGb:
|
|
560
|
+
sizeGb: storageGb,
|
|
524
561
|
deviceName: CLICKHOUSE_EBS_DEVICE_NAME,
|
|
525
562
|
availabilityZone: dataAz,
|
|
526
|
-
iops
|
|
527
|
-
throughputMbps
|
|
563
|
+
iops,
|
|
564
|
+
throughputMbps,
|
|
528
565
|
...(resolvedAlertsTopic !== undefined && {
|
|
529
566
|
alarmTopic: resolvedAlertsTopic
|
|
530
567
|
})
|
|
@@ -658,6 +695,8 @@ export class ClickHouseDatabase extends Construct {
|
|
|
658
695
|
asgName,
|
|
659
696
|
alarmTopic: resolvedAlertsTopic,
|
|
660
697
|
config: alarmThresholds,
|
|
698
|
+
diskFreeCriticalGib,
|
|
699
|
+
backupVerifyEnabled,
|
|
661
700
|
...(applicationId !== undefined && { applicationId }),
|
|
662
701
|
...(backupTaskLogGroup !== undefined && { backupTaskLogGroup })
|
|
663
702
|
});
|
|
@@ -332,7 +332,7 @@ export declare class RelationalDatabase extends Construct implements IRelational
|
|
|
332
332
|
grantIamConnect(grantee: IGrantable, dbUsername: string): Grant;
|
|
333
333
|
}
|
|
334
334
|
export { ClickHouseDatabase, type ClickHouseDatabaseProps, type ClickHouseAlarmThresholds };
|
|
335
|
-
export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, type ClickHouseSchemaAdmin, type ManagedPasswordName, type ProfileSpec } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
335
|
+
export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, SettingConstraintSchema, type ClickHouseSchemaAdmin, type FrozenProfileSpec, type ManagedPasswordName, type ProfileSpec, type SettingConstraint } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
336
336
|
export { renderUsersXml } from "../../resources/aws/database/clickhouseXmlRenderer.js";
|
|
337
337
|
export type { RenderUsersXmlOptions } from "../../resources/aws/database/clickhouseXmlRenderer.js";
|
|
338
338
|
export type { DynamoDBKeySchema, DynamoDBGlobalSecondaryIndex, DynamoDBTableProps } from "../../resources/aws/database/dynamodb.js";
|
|
@@ -655,6 +655,6 @@ export class RelationalDatabase extends Construct {
|
|
|
655
655
|
}
|
|
656
656
|
}
|
|
657
657
|
export { ClickHouseDatabase };
|
|
658
|
-
export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
658
|
+
export { ClickHouseDefaultProfiles, ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, SettingConstraintSchema } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
659
659
|
export { renderUsersXml } from "../../resources/aws/database/clickhouseXmlRenderer.js";
|
|
660
660
|
export { isRelationalDatabase, isDynamoDBDatabase, isClickHouseDatabase, isAuroraDatabase, isInstanceDatabase, isGlobalAuroraDatabase } from "./interfaces/database.js";
|
|
@@ -49,6 +49,14 @@ export interface S3Props {
|
|
|
49
49
|
readonly removalPolicy?: "DESTROY" | "RETAIN";
|
|
50
50
|
/** Declarative bucket-policy statements appended to the bucket's resource policy. */
|
|
51
51
|
readonly resourcePolicyStatements?: ResourcePolicyStatement[];
|
|
52
|
+
/**
|
|
53
|
+
* The owning Fjall app's name, required by website-hosting buckets — their
|
|
54
|
+
* exports are keyed on (app, bucket) so sibling apps' identically-named
|
|
55
|
+
* buckets stay deployable side by side. `StorageFactory` fills it from the
|
|
56
|
+
* App (as `CdnFactory` does); pass it only for direct `new Storage(...)`
|
|
57
|
+
* construction.
|
|
58
|
+
*/
|
|
59
|
+
readonly appName?: string;
|
|
52
60
|
}
|
|
53
61
|
export interface StorageBuildProps extends S3Props {
|
|
54
62
|
readonly stackPlacement?: "storage" | "cdn" | "compute";
|
|
@@ -74,8 +74,9 @@ export class Storage extends Construct {
|
|
|
74
74
|
this.bucket = new S3Bucket(this, `${id}Bucket`, {
|
|
75
75
|
bucketName: props.bucketName,
|
|
76
76
|
// The bucket nests one level below, so its own id is `${id}Bucket` — the
|
|
77
|
-
// author's handle for `fjallBucket(<name>)` is THIS construct's id.
|
|
77
|
+
// author's handle for `fjallBucket(<app>, <name>)` is THIS construct's id.
|
|
78
78
|
aliasTargetName: props.bucketName ?? id,
|
|
79
|
+
appName: props.appName,
|
|
79
80
|
versioned: props.versioned,
|
|
80
81
|
encryption: toBucketEncryption(props.encryption),
|
|
81
82
|
encryptionKey,
|
|
@@ -175,9 +176,15 @@ export class Storage extends Construct {
|
|
|
175
176
|
export class StorageFactory {
|
|
176
177
|
static build(id, props = {}) {
|
|
177
178
|
const { stackPlacement, ...s3Props } = props;
|
|
178
|
-
const fn = (
|
|
179
|
+
const fn = (app, scope) => {
|
|
179
180
|
validateStorageProps(s3Props);
|
|
180
|
-
|
|
181
|
+
// The App is the only place the app name is known here; website-hosting
|
|
182
|
+
// buckets need it for their (app, bucket)-keyed exports — the same fill
|
|
183
|
+
// CdnFactory performs for Cdn.
|
|
184
|
+
return new Storage(scope, id, {
|
|
185
|
+
...s3Props,
|
|
186
|
+
appName: s3Props.appName ?? app.getName()
|
|
187
|
+
});
|
|
181
188
|
};
|
|
182
189
|
if (stackPlacement) {
|
|
183
190
|
fn.stackPlacement = stackPlacement;
|
|
@@ -32,12 +32,14 @@ export declare function fjallCdn(appName: string): FjallAliasTarget;
|
|
|
32
32
|
/**
|
|
33
33
|
* Target helper for a Fjall S3 static-site bucket.
|
|
34
34
|
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
35
|
+
* `appName` names the app that owns the bucket — the first segment of every
|
|
36
|
+
* target helper's key, exactly as `fjallApp`/`fjallCdn` take it. Requires that
|
|
37
|
+
* app's stack to publish exports `${safeApp}-${safeBucket}-WebsiteEndpoint`
|
|
38
|
+
* and `${safeApp}-${safeBucket}-WebsiteHostedZoneId` — emitted by
|
|
39
|
+
* website-enabled `S3` storage. Naming a pair with no such exports fails at
|
|
40
|
+
* CloudFormation execution, exactly as `fjallApp` documents.
|
|
39
41
|
*/
|
|
40
|
-
export declare function fjallBucket(bucketName: string): FjallAliasTarget;
|
|
42
|
+
export declare function fjallBucket(appName: string, bucketName: string): FjallAliasTarget;
|
|
41
43
|
/**
|
|
42
44
|
* Target helper for an arbitrary `(dnsName, hostedZoneId)` pair. Accepts
|
|
43
45
|
* literal strings or CDK tokens (e.g. a `CfnOutput.value`). No
|
|
@@ -43,15 +43,18 @@ export function fjallCdn(appName) {
|
|
|
43
43
|
/**
|
|
44
44
|
* Target helper for a Fjall S3 static-site bucket.
|
|
45
45
|
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
46
|
+
* `appName` names the app that owns the bucket — the first segment of every
|
|
47
|
+
* target helper's key, exactly as `fjallApp`/`fjallCdn` take it. Requires that
|
|
48
|
+
* app's stack to publish exports `${safeApp}-${safeBucket}-WebsiteEndpoint`
|
|
49
|
+
* and `${safeApp}-${safeBucket}-WebsiteHostedZoneId` — emitted by
|
|
50
|
+
* website-enabled `S3` storage. Naming a pair with no such exports fails at
|
|
51
|
+
* CloudFormation execution, exactly as `fjallApp` documents.
|
|
50
52
|
*/
|
|
51
|
-
export function fjallBucket(bucketName) {
|
|
52
|
-
const resolved = resolveBucketTarget(bucketName);
|
|
53
|
+
export function fjallBucket(appName, bucketName) {
|
|
54
|
+
const resolved = resolveBucketTarget(appName, bucketName);
|
|
53
55
|
return {
|
|
54
56
|
kind: "bucket",
|
|
57
|
+
appName,
|
|
55
58
|
bucketName,
|
|
56
59
|
bind(record, zone) {
|
|
57
60
|
return resolved.bind(record, zone);
|
|
@@ -15,8 +15,8 @@ import type { FjallTarget } from "../interfaces/domain.js";
|
|
|
15
15
|
* | `ecsNetworking.ts` app stack | `resolveEcsTarget` | `<App>-<Compute>-AlbDnsName` |
|
|
16
16
|
* | `ecsNetworking.ts` app stack | `resolveEcsTarget` | `<App>-<Compute>-AlbHostedZoneId` |
|
|
17
17
|
* | `cloudFront.ts` CDN stack | `resolveCdnTarget` | `<App>CdnDistributionDomainName` |
|
|
18
|
-
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<Bucket
|
|
19
|
-
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<Bucket
|
|
18
|
+
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<App>-<Bucket>-WebsiteEndpoint` |
|
|
19
|
+
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<App>-<Bucket>-WebsiteHostedZoneId` |
|
|
20
20
|
*
|
|
21
21
|
* Both sides derive the name from `@fjall/util`'s `…ExportName` helpers, which
|
|
22
22
|
* take the RAW app or bucket name and sanitise internally. Compose these
|
|
@@ -50,10 +50,13 @@ export declare function resolveCdnTarget(appName: string): IAliasRecordTarget;
|
|
|
50
50
|
/**
|
|
51
51
|
* Resolve a Fjall S3 static-site bucket to an `IAliasRecordTarget`.
|
|
52
52
|
*
|
|
53
|
-
* `
|
|
54
|
-
*
|
|
53
|
+
* `appName` names the app whose stack exports the pair — required because the
|
|
54
|
+
* export is keyed on (app, bucket), never on the bucket alone (sibling apps'
|
|
55
|
+
* identically-named buckets must stay deployable side by side). `bucketName`
|
|
56
|
+
* is the bucket's name as written, dots and all — the helper strips what
|
|
57
|
+
* CloudFormation will not hold, and the producer strips the same.
|
|
55
58
|
*/
|
|
56
|
-
export declare function resolveBucketTarget(bucketName: string): IAliasRecordTarget;
|
|
59
|
+
export declare function resolveBucketTarget(appName: string, bucketName: string): IAliasRecordTarget;
|
|
57
60
|
/**
|
|
58
61
|
* Pure passthrough — accepts concrete DNS name + hosted-zone id (or CDK
|
|
59
62
|
* tokens) and forwards them unchanged. No `Fn.importValue` wrapping.
|
|
@@ -15,8 +15,8 @@ import { albDnsExportName, albHostedZoneIdExportName, cdnDomainExportName, bucke
|
|
|
15
15
|
* | `ecsNetworking.ts` app stack | `resolveEcsTarget` | `<App>-<Compute>-AlbDnsName` |
|
|
16
16
|
* | `ecsNetworking.ts` app stack | `resolveEcsTarget` | `<App>-<Compute>-AlbHostedZoneId` |
|
|
17
17
|
* | `cloudFront.ts` CDN stack | `resolveCdnTarget` | `<App>CdnDistributionDomainName` |
|
|
18
|
-
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<Bucket
|
|
19
|
-
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<Bucket
|
|
18
|
+
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<App>-<Bucket>-WebsiteEndpoint` |
|
|
19
|
+
* | `s3.ts` bucket stack | `resolveBucketTarget` | `<App>-<Bucket>-WebsiteHostedZoneId` |
|
|
20
20
|
*
|
|
21
21
|
* Both sides derive the name from `@fjall/util`'s `…ExportName` helpers, which
|
|
22
22
|
* take the RAW app or bucket name and sanitise internally. Compose these
|
|
@@ -69,12 +69,15 @@ export function resolveCdnTarget(appName) {
|
|
|
69
69
|
/**
|
|
70
70
|
* Resolve a Fjall S3 static-site bucket to an `IAliasRecordTarget`.
|
|
71
71
|
*
|
|
72
|
-
* `
|
|
73
|
-
*
|
|
72
|
+
* `appName` names the app whose stack exports the pair — required because the
|
|
73
|
+
* export is keyed on (app, bucket), never on the bucket alone (sibling apps'
|
|
74
|
+
* identically-named buckets must stay deployable side by side). `bucketName`
|
|
75
|
+
* is the bucket's name as written, dots and all — the helper strips what
|
|
76
|
+
* CloudFormation will not hold, and the producer strips the same.
|
|
74
77
|
*/
|
|
75
|
-
export function resolveBucketTarget(bucketName) {
|
|
76
|
-
const dnsName = Fn.importValue(bucketWebsiteEndpointExportName(bucketName));
|
|
77
|
-
const hostedZoneId = Fn.importValue(bucketWebsiteHostedZoneIdExportName(bucketName));
|
|
78
|
+
export function resolveBucketTarget(appName, bucketName) {
|
|
79
|
+
const dnsName = Fn.importValue(bucketWebsiteEndpointExportName(appName, bucketName));
|
|
80
|
+
const hostedZoneId = Fn.importValue(bucketWebsiteHostedZoneIdExportName(appName, bucketName));
|
|
78
81
|
return buildAliasTarget(dnsName, hostedZoneId);
|
|
79
82
|
}
|
|
80
83
|
/**
|
|
@@ -98,7 +101,7 @@ export function resolveTargetToDnsName(target) {
|
|
|
98
101
|
case "cdn":
|
|
99
102
|
return Fn.importValue(cdnDomainExportName(target.appName));
|
|
100
103
|
case "bucket":
|
|
101
|
-
return Fn.importValue(bucketWebsiteEndpointExportName(target.bucketName));
|
|
104
|
+
return Fn.importValue(bucketWebsiteEndpointExportName(target.appName, target.bucketName));
|
|
102
105
|
case "custom":
|
|
103
106
|
return target.dnsName;
|
|
104
107
|
default: {
|