@fjall/components-infrastructure 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/config/aws/oidcConnector.d.ts +16 -7
- package/dist/lib/config/aws/oidcConnector.js +6 -1
- package/dist/lib/patterns/aws/account.d.ts +24 -11
- package/dist/lib/patterns/aws/account.js +14 -10
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +16 -3
- package/dist/lib/patterns/aws/clickhouseDatabase.js +23 -2
- package/dist/lib/patterns/aws/computeEcsTypes.d.ts +12 -2
- package/dist/lib/patterns/aws/database.js +5 -5
- package/dist/lib/resources/aws/compute/ecs.js +1 -0
- package/dist/lib/resources/aws/compute/ecsConstants.d.ts +14 -0
- package/dist/lib/resources/aws/compute/ecsConstants.js +23 -0
- package/dist/lib/resources/aws/compute/ecsServiceFactory.d.ts +9 -0
- package/dist/lib/resources/aws/compute/ecsServiceFactory.js +71 -1
- package/dist/lib/resources/aws/compute/ecsTaskDefinition.js +2 -6
- package/dist/lib/resources/aws/compute/ecsValidation.js +9 -2
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +59 -26
- package/dist/lib/resources/aws/database/clickhouseConstants.js +90 -26
- package/dist/lib/resources/aws/database/clickhouseSchemas.js +9 -2
- package/dist/lib/resources/aws/database/clickhouseUserData.js +87 -11
- package/dist/lib/resources/aws/database/rdsAurora.js +1 -1
- package/dist/lib/resources/aws/database/rdsInstance.js +1 -1
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts +9 -5
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js +21 -12
- package/dist/lib/utils/engineCompat.d.ts +35 -13
- package/dist/lib/utils/engineCompat.js +90 -23
- package/package.json +25 -23
|
@@ -11,13 +11,18 @@ export const CLICKHOUSE_DATABASE_NAME = "analytics";
|
|
|
11
11
|
* columnar scans than Graviton2.
|
|
12
12
|
*
|
|
13
13
|
* Settings are tuned for 1 vCPU sustained (max_threads=1 across all profiles,
|
|
14
|
-
* max_concurrent_queries=
|
|
14
|
+
* max_concurrent_queries=8) — see clickhouseUserData.ts. Bumping to a larger
|
|
15
15
|
* instance MUST be paired with raising those thread/concurrency caps.
|
|
16
16
|
*
|
|
17
|
-
* Next size up
|
|
18
|
-
* `
|
|
19
|
-
*
|
|
20
|
-
*
|
|
17
|
+
* Next size up (2026-07-31 right-sizing, designs/2026-07-31-clickhouse-
|
|
18
|
+
* factory-fit-for-purpose.md): `r8g.medium` (1 vCPU Graviton4, 8 GiB,
|
|
19
|
+
* ~+30%/core over Graviton3) — CH's own doctrine floors production at
|
|
20
|
+
* 8 GiB, and the memory-bound single-tenant profile wants RAM before
|
|
21
|
+
* cores. `m7g.large` (2 vCPU, 8 GiB) only when CPU-bound. Set via
|
|
22
|
+
* `clickhouseInstanceType` CDK context or the `instanceType` prop —
|
|
23
|
+
* the container memory limit derives automatically via
|
|
24
|
+
* `clickHouseTaskMemoryMiB()`, but the thread/concurrency caps in
|
|
25
|
+
* clickhouseUserData.ts still need a lockstep re-tune. */
|
|
21
26
|
export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
|
|
22
27
|
/** ClickHouse container image. Explicit `docker.io/` prefix is required so
|
|
23
28
|
* string-form consumers in `ecsImages.ts#getContainerImage()` route through
|
|
@@ -38,17 +43,66 @@ export const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
|
|
|
38
43
|
* Image size is ~250 MB larger but irrelevant on a single-instance EC2 ASG
|
|
39
44
|
* that pulls once per launch. Upstream CH CI runs its full perf + stress
|
|
40
45
|
* matrix on the Ubuntu build; Alpine is community-tier coverage. */
|
|
41
|
-
export const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.
|
|
46
|
+
export const CLICKHOUSE_IMAGE = "docker.io/clickhouse/clickhouse-server:26.3.17.56";
|
|
42
47
|
/** EBS volume configuration. */
|
|
43
48
|
export const CLICKHOUSE_EBS_VOLUME_SIZE_GB = 80;
|
|
44
49
|
export const CLICKHOUSE_EBS_IOPS = 3000;
|
|
45
50
|
export const CLICKHOUSE_EBS_THROUGHPUT_MBPS = 125;
|
|
46
|
-
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
|
|
51
|
+
/** Host memory reserved from the ClickHouse container: kernel + ECS agent +
|
|
52
|
+
* the host-metrics timer. The ECS agent advertises MemTotal minus
|
|
53
|
+
* CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, and MemTotal itself runs below the
|
|
54
|
+
* AWS nominal figure (firmware carve-outs plus page-struct overhead that
|
|
55
|
+
* grows with RAM, ~205 MiB observed on the 4 GiB m7g.medium), so this
|
|
56
|
+
* reserve must absorb both or the task never places. 1 GiB is proven at
|
|
57
|
+
* 4–16 GiB; the ≥32 GiB rows double it (see clickHouseTaskMemoryMiB). */
|
|
58
|
+
export const CLICKHOUSE_HOST_RESERVE_MIB = 1024;
|
|
59
|
+
/** MiB the ECS agent withholds from advertised capacity — written to
|
|
60
|
+
* /etc/ecs/ecs.config as `ECS_RESERVED_MEMORY` by buildClickHouseUserData.
|
|
61
|
+
* Must stay well inside CLICKHOUSE_HOST_RESERVE_MIB: the container limit is
|
|
62
|
+
* derived from nominal memory, so the host reserve is what absorbs this
|
|
63
|
+
* withholding at task placement. */
|
|
64
|
+
export const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
|
|
65
|
+
/** Total memory (GiB) of the instance types the ClickHouse construct knows
|
|
66
|
+
* how to size a container for. Values are the AWS nominal figures. */
|
|
67
|
+
export const CLICKHOUSE_INSTANCE_MEMORY_GIB = {
|
|
68
|
+
"t4g.medium": 4,
|
|
69
|
+
"m7g.medium": 4,
|
|
70
|
+
"m8g.medium": 4,
|
|
71
|
+
"r7g.medium": 8,
|
|
72
|
+
"r8g.medium": 8,
|
|
73
|
+
"m7g.large": 8,
|
|
74
|
+
"m8g.large": 8,
|
|
75
|
+
"r7g.large": 16,
|
|
76
|
+
"r8g.large": 16,
|
|
77
|
+
"m7g.xlarge": 16,
|
|
78
|
+
"m8g.xlarge": 16,
|
|
79
|
+
"r7g.xlarge": 32,
|
|
80
|
+
"r8g.xlarge": 32
|
|
81
|
+
};
|
|
82
|
+
/** ECS container memory for the ClickHouse server task, derived from the
|
|
83
|
+
* instance type so a size change cannot leave the hand-set container cap
|
|
84
|
+
* behind (the pre-2026-07 shape: a hardcoded 3072 that silently starved —
|
|
85
|
+
* or over-committed — any non-4-GiB host). Unknown types throw at synth:
|
|
86
|
+
* a memoryLimitMiB above what the host advertises strands the task in
|
|
87
|
+
* PROVISIONING with no CloudFormation error. */
|
|
88
|
+
export function clickHouseTaskMemoryMiB(instanceType) {
|
|
89
|
+
const memoryGib = CLICKHOUSE_INSTANCE_MEMORY_GIB[instanceType];
|
|
90
|
+
if (memoryGib === undefined) {
|
|
91
|
+
throw new Error(`ClickHouseDatabase: unknown instance type '${instanceType}' — cannot ` +
|
|
92
|
+
"derive the container memory limit. Supported: " +
|
|
93
|
+
`${Object.keys(CLICKHOUSE_INSTANCE_MEMORY_GIB).join(", ")}. Add the ` +
|
|
94
|
+
"type's nominal memory to CLICKHOUSE_INSTANCE_MEMORY_GIB " +
|
|
95
|
+
"(clickhouseConstants.ts) to support it.");
|
|
96
|
+
}
|
|
97
|
+
// ≥32 GiB rows double the reserve: the kernel-invisible gap (firmware +
|
|
98
|
+
// page structs) approaches 1 GiB there, and a limit above what the ECS
|
|
99
|
+
// agent advertises (MemTotal − ECS_RESERVED_MEMORY) strands the task in
|
|
100
|
+
// PROVISIONING with no CloudFormation error.
|
|
101
|
+
const reserveMib = memoryGib >= 32
|
|
102
|
+
? CLICKHOUSE_HOST_RESERVE_MIB * 2
|
|
103
|
+
: CLICKHOUSE_HOST_RESERVE_MIB;
|
|
104
|
+
return memoryGib * 1024 - reserveMib;
|
|
105
|
+
}
|
|
52
106
|
/** ClickHouse ports. */
|
|
53
107
|
export const CLICKHOUSE_HTTP_PORT = 8123;
|
|
54
108
|
export const CLICKHOUSE_NATIVE_PORT = 9000;
|
|
@@ -58,22 +112,16 @@ export const CLICKHOUSE_PROMETHEUS_PORT = 9363;
|
|
|
58
112
|
* See aiDocs/patterns/clickhouse-tls-pattern.md § "The contract". */
|
|
59
113
|
export const CLICKHOUSE_HTTPS_PORT = 8443;
|
|
60
114
|
export const CLICKHOUSE_TCP_SECURE_PORT = 9440;
|
|
61
|
-
/** Mount path inside the
|
|
62
|
-
*
|
|
63
|
-
*
|
|
115
|
+
/** Mount path inside the ClickHouse container for the materialised TLS cert
|
|
116
|
+
* + key. The EC2 user-data bootstrap (`buildClickHouseUserData` tlsBootstrap)
|
|
117
|
+
* fetches the PEMs from Secrets Manager onto the EBS data volume at
|
|
118
|
+
* `<mount>/server-certs`, which is bind-mounted read-only into the container
|
|
119
|
+
* at this path. */
|
|
64
120
|
export const CLICKHOUSE_TLS_CERT_MOUNT_PATH = "/etc/clickhouse-server/certs";
|
|
65
|
-
/**
|
|
66
|
-
*
|
|
67
|
-
|
|
68
|
-
/** UID:GID the official ClickHouse image runs as. Init container `chown`s
|
|
69
|
-
* the materialised cert files to this UID so the server can read them. */
|
|
121
|
+
/** UID:GID the official ClickHouse image runs as. The user-data bootstrap
|
|
122
|
+
* `chown`s the materialised cert files to this UID so the server can read
|
|
123
|
+
* them. */
|
|
70
124
|
export const CLICKHOUSE_UID = 101;
|
|
71
|
-
/** Digest-pinned alpine image used by the TLS init container. The init
|
|
72
|
-
* container needs `jq` (extracts `cert`+`key` from the server-cert JSON
|
|
73
|
-
* secret); alpine ships it via `apk add` — but pinning the digest avoids
|
|
74
|
-
* fetching `:latest` on every deploy. Bump in lockstep with renovate
|
|
75
|
-
* alerts; CI verifies the digest resolves. */
|
|
76
|
-
export const ALPINE_INIT_CONTAINER_IMAGE = "public.ecr.aws/docker/library/alpine:3.20@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d";
|
|
77
125
|
/** EBS device name for the data volume (must match user data script). */
|
|
78
126
|
export const CLICKHOUSE_EBS_DEVICE_NAME = "/dev/xvdf";
|
|
79
127
|
/** EBS mount path on the EC2 host. */
|
|
@@ -129,6 +177,22 @@ export const CLICKHOUSE_SERVER_ROLE_TAG = {
|
|
|
129
177
|
key: "ClickHouseRole",
|
|
130
178
|
value: "server"
|
|
131
179
|
};
|
|
180
|
+
/** CloudWatch identity of the host metrics: namespace, metric names, and the
|
|
181
|
+
* dimension key. Three sites consume this and they MUST match: (a) the
|
|
182
|
+
* user-data put-metric-data timer (`buildClickHouseUserData`) that publishes
|
|
183
|
+
* the datapoints, (b) the host alarms (`createClickHouseAlarms`) that
|
|
184
|
+
* consume them, and (c) the instance-role PutMetricData namespace condition
|
|
185
|
+
* (`clickhouseDatabase.ts`). One-character drift re-blinds the alarms
|
|
186
|
+
* silently — and with disk-critical on TreatMissingData.BREACHING, a
|
|
187
|
+
* namespace typo pages as a phantom full disk. "CWAgent" matches what the
|
|
188
|
+
* CloudWatch Agent would publish, keeping dashboards portable if the timer
|
|
189
|
+
* is ever replaced by the real agent. */
|
|
190
|
+
export const CLICKHOUSE_HOST_METRICS = {
|
|
191
|
+
namespace: "CWAgent",
|
|
192
|
+
memoryMetric: "mem_used_percent",
|
|
193
|
+
diskMetric: "disk_used_percent",
|
|
194
|
+
asgDimension: "AutoScalingGroupName"
|
|
195
|
+
};
|
|
132
196
|
/** Shared secret generation options (all ClickHouse users share the same policy). */
|
|
133
197
|
export const CLICKHOUSE_SECRET_OPTIONS = {
|
|
134
198
|
excludePunctuation: true,
|
|
@@ -90,7 +90,14 @@ export const PROFILE_NAME_PATTERN = NAME_PATTERN;
|
|
|
90
90
|
* from the pre-Phase-4b inline `<app_writer>` / `<audit_writer>` /
|
|
91
91
|
* `<backup_reader>` / `<schema_admin>` blocks at
|
|
92
92
|
* `clickhouseUserData.ts:296-382` — zero behavioural drift from the
|
|
93
|
-
* pre-4b prod resource caps
|
|
93
|
+
* pre-4b prod resource caps, EXCEPT high_throughput_ingest's
|
|
94
|
+
* `maxConcurrentQueriesForUser`, deliberately raised 2→6 (2026-07-28):
|
|
95
|
+
* the webapp overview fires 6–10 queries per page load and the cap of 2
|
|
96
|
+
* threw TOO_MANY_SIMULTANEOUS_QUERIES instead of queueing (CH does not
|
|
97
|
+
* queue past this cap — `queue_max_wait_ms` empirically does not apply).
|
|
98
|
+
* Consumers pair it with a client-side FIFO gate
|
|
99
|
+
* (`webapp/app/.server/lib/clickhouse/appQuerySlots.ts`) whose limit
|
|
100
|
+
* MUST mirror this value — a users.xml parity test enforces the pair.
|
|
94
101
|
*
|
|
95
102
|
* Frozen via `Object.freeze` on the outer record AND each sub-object so
|
|
96
103
|
* a consumer cannot mutate the defaults in place. Pass your own
|
|
@@ -101,7 +108,7 @@ const _ClickHouseDefaultProfilesRaw = {
|
|
|
101
108
|
high_throughput_ingest: {
|
|
102
109
|
maxThreads: 1,
|
|
103
110
|
maxInsertThreads: 1,
|
|
104
|
-
maxConcurrentQueriesForUser:
|
|
111
|
+
maxConcurrentQueriesForUser: 6,
|
|
105
112
|
logQueriesMinQueryDurationMs: 100,
|
|
106
113
|
optimizeMoveToPrewhere: true,
|
|
107
114
|
useQueryConditionCache: true,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, clickHousePasswordSha256Snippet } from "./clickhouseConstants.js";
|
|
1
|
+
import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, CLICKHOUSE_ECS_RESERVED_MEMORY_MIB, CLICKHOUSE_HOST_METRICS, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_UID, clickHousePasswordSha256Snippet } from "./clickhouseConstants.js";
|
|
2
2
|
import { renderUsersXml } from "./clickhouseXmlRenderer.js";
|
|
3
3
|
export function generateServerConfigXml(options) {
|
|
4
4
|
const { backupBucketName, backupBucketRegion, coldTier } = options;
|
|
@@ -119,13 +119,16 @@ export function generateServerConfigXml(options) {
|
|
|
119
119
|
<!-- Filesystem page cache handles uncompressed-block warmth on the
|
|
120
120
|
host side; an in-CH uncompressed cache would just duplicate it. -->
|
|
121
121
|
<uncompressed_cache_size>0</uncompressed_cache_size>
|
|
122
|
-
<!--
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
122
|
+
<!-- Server-wide cap across ALL users. Raised 4→8 (2026-07-28) alongside
|
|
123
|
+
high_throughput_ingest's per-user cap (2→6): the webapp overview
|
|
124
|
+
fires 6-10 queries per page load and N app tasks share this cap, so
|
|
125
|
+
4 starved dashboard reads with TOO_MANY_SIMULTANEOUS_QUERIES (CH
|
|
126
|
+
rejects rather than queues past the cap). On the 1 vCPU m7g.medium
|
|
127
|
+
the extra slots time-slice rather than parallelise — acceptable for
|
|
128
|
+
short cache-hot dashboard reads; max_threads=1 per query keeps the
|
|
129
|
+
scheduler sane. Bump further in lockstep with instance size if
|
|
130
|
+
DEFAULT_CLICKHOUSE_INSTANCE_TYPE moves to a >=2 vCPU host. -->
|
|
131
|
+
<max_concurrent_queries>8</max_concurrent_queries>
|
|
129
132
|
<background_pool_size>2</background_pool_size>
|
|
130
133
|
<background_schedule_pool_size>2</background_schedule_pool_size>
|
|
131
134
|
<background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
|
|
@@ -317,11 +320,76 @@ cat > "$MOUNT_POINT/server-certs/client.xml" <<'CLIENTXML'
|
|
|
317
320
|
</config>
|
|
318
321
|
CLIENTXML
|
|
319
322
|
|
|
320
|
-
chown -R
|
|
323
|
+
chown -R ${CLICKHOUSE_UID}:${CLICKHOUSE_UID} "$MOUNT_POINT/server-certs"
|
|
321
324
|
chmod 600 "$MOUNT_POINT/server-certs/server.key"
|
|
322
325
|
chmod 644 "$MOUNT_POINT/server-certs/client.xml"
|
|
323
326
|
`
|
|
324
327
|
: "";
|
|
328
|
+
// The CWAgent-namespace alarms (mem_used_percent / disk_used_percent) have
|
|
329
|
+
// no producer without this — no CloudWatch Agent is installed. The ASG name
|
|
330
|
+
// (the alarms' dimension) is unknowable at synth (LaunchTemplate → ASG is a
|
|
331
|
+
// CFN cycle), so it resolves at run time: IMDS instance tags first, with a
|
|
332
|
+
// describe-auto-scaling-instances fallback. AL2023 ECS AMIs ship no cron;
|
|
333
|
+
// systemd timer is the only scheduler present. The trailing `|| echo` keeps
|
|
334
|
+
// a publisher failure from aborting the whole database bootstrap.
|
|
335
|
+
//
|
|
336
|
+
// Hang-proofing: `Type=oneshot` defaults TimeoutStartSec to infinity and a
|
|
337
|
+
// timer never re-triggers a still-activating unit, so ONE wedged run (hung
|
|
338
|
+
// IMDS or AWS call) would silence the publisher forever — every curl carries
|
|
339
|
+
// --max-time and the unit a TimeoutStartSec under the 60s cadence. The disk
|
|
340
|
+
// metric publishes only when the data volume is actually mounted: `df` on
|
|
341
|
+
// an unmounted path reports the ROOT filesystem, a false-healthy number
|
|
342
|
+
// that would defeat the disk-critical alarm's missing-data=BREACHING flip.
|
|
343
|
+
const { namespace, memoryMetric, diskMetric, asgDimension } = CLICKHOUSE_HOST_METRICS;
|
|
344
|
+
const metricsPublisher = `
|
|
345
|
+
cat > /usr/local/bin/fjall-ch-metrics.sh << 'METRICSEOF'
|
|
346
|
+
#!/bin/bash
|
|
347
|
+
set -euo pipefail
|
|
348
|
+
IMDS_BASE="http://169.254.169.254/latest"
|
|
349
|
+
TOKEN=$(curl -sf --max-time 5 -X PUT "$IMDS_BASE/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
|
|
350
|
+
imds() { curl -sf --max-time 5 -H "X-aws-ec2-metadata-token: $TOKEN" "$IMDS_BASE/meta-data/$1"; }
|
|
351
|
+
REGION=$(imds placement/region)
|
|
352
|
+
ASG=$(imds tags/instance/aws:autoscaling:groupName || true)
|
|
353
|
+
if [ -z "$ASG" ]; then
|
|
354
|
+
INSTANCE_ID=$(imds instance-id)
|
|
355
|
+
ASG=$(aws autoscaling describe-auto-scaling-instances --instance-ids "$INSTANCE_ID" --region "$REGION" --query "AutoScalingInstances[0].AutoScalingGroupName" --output text)
|
|
356
|
+
fi
|
|
357
|
+
if [ -z "$ASG" ] || [ "$ASG" = "None" ]; then
|
|
358
|
+
echo "fjall-ch-metrics: could not resolve ASG name" >&2
|
|
359
|
+
exit 1
|
|
360
|
+
fi
|
|
361
|
+
MEM=$(awk '/MemTotal/{t=$2} /MemAvailable/{a=$2} END{printf "%.1f", (t-a)*100/t}' /proc/meminfo)
|
|
362
|
+
METRICS=("MetricName=${memoryMetric},Dimensions=[{Name=${asgDimension},Value=$ASG}],Value=$MEM,Unit=Percent")
|
|
363
|
+
if mountpoint -q ${CLICKHOUSE_DATA_MOUNT_PATH}; then
|
|
364
|
+
DISK=$(df --output=pcent ${CLICKHOUSE_DATA_MOUNT_PATH} | tail -n 1 | tr -dc '0-9')
|
|
365
|
+
METRICS+=("MetricName=${diskMetric},Dimensions=[{Name=${asgDimension},Value=$ASG}],Value=$DISK,Unit=Percent")
|
|
366
|
+
else
|
|
367
|
+
echo "fjall-ch-metrics: ${CLICKHOUSE_DATA_MOUNT_PATH} not mounted, skipping disk metric" >&2
|
|
368
|
+
fi
|
|
369
|
+
aws cloudwatch put-metric-data --region "$REGION" --namespace ${namespace} --metric-data "\${METRICS[@]}"
|
|
370
|
+
METRICSEOF
|
|
371
|
+
chmod 755 /usr/local/bin/fjall-ch-metrics.sh
|
|
372
|
+
cat > /etc/systemd/system/fjall-ch-metrics.service << 'SVCEOF'
|
|
373
|
+
[Unit]
|
|
374
|
+
Description=Publish ClickHouse host metrics to CloudWatch
|
|
375
|
+
[Service]
|
|
376
|
+
Type=oneshot
|
|
377
|
+
TimeoutStartSec=50
|
|
378
|
+
ExecStart=/usr/local/bin/fjall-ch-metrics.sh
|
|
379
|
+
SVCEOF
|
|
380
|
+
cat > /etc/systemd/system/fjall-ch-metrics.timer << 'TIMEREOF'
|
|
381
|
+
[Unit]
|
|
382
|
+
Description=Run fjall-ch-metrics every minute
|
|
383
|
+
[Timer]
|
|
384
|
+
OnBootSec=60s
|
|
385
|
+
OnUnitActiveSec=60s
|
|
386
|
+
RandomizedDelaySec=5s
|
|
387
|
+
[Install]
|
|
388
|
+
WantedBy=timers.target
|
|
389
|
+
TIMEREOF
|
|
390
|
+
systemctl daemon-reload
|
|
391
|
+
systemctl enable --now fjall-ch-metrics.timer || echo "WARN: fjall-ch-metrics timer enable failed" >&2
|
|
392
|
+
`;
|
|
325
393
|
const script = `#!/bin/bash
|
|
326
394
|
set -euo pipefail
|
|
327
395
|
|
|
@@ -354,6 +422,14 @@ DOCKEROVERRIDE
|
|
|
354
422
|
systemctl daemon-reload
|
|
355
423
|
systemctl restart docker || true
|
|
356
424
|
|
|
425
|
+
# 3. Tell the ECS agent to withhold 256 MiB from advertised capacity so the
|
|
426
|
+
# scheduler accounts for kernel + agent + metrics-timer overhead instead of
|
|
427
|
+
# offering the container the host's full nominal memory. CDK appends its own
|
|
428
|
+
# 'echo ECS_CLUSTER=... >> /etc/ecs/ecs.config' after this script, so >> is
|
|
429
|
+
# required here (a > would be clobber-safe only by ordering luck).
|
|
430
|
+
mkdir -p /etc/ecs
|
|
431
|
+
echo "ECS_RESERVED_MEMORY=${CLICKHOUSE_ECS_RESERVED_MEMORY_MIB}" >> /etc/ecs/ecs.config
|
|
432
|
+
|
|
357
433
|
# Wait for the EBS volume to be attached (up to 60 seconds)
|
|
358
434
|
for i in $(seq 1 60); do
|
|
359
435
|
if [ -b "$DEVICE" ]; then
|
|
@@ -407,8 +483,8 @@ if [ "$USERS_CONFIG_RETRIEVED" != "1" ]; then
|
|
|
407
483
|
exit 1
|
|
408
484
|
fi
|
|
409
485
|
|
|
410
|
-
chown -R
|
|
411
|
-
${tlsBootstrap}`;
|
|
486
|
+
chown -R ${CLICKHOUSE_UID}:${CLICKHOUSE_UID} "$MOUNT_POINT"
|
|
487
|
+
${metricsPublisher}${tlsBootstrap}`;
|
|
412
488
|
return compactUserDataScript(script);
|
|
413
489
|
}
|
|
414
490
|
/**
|
|
@@ -152,7 +152,7 @@ export class RdsAurora extends Construct {
|
|
|
152
152
|
const readers = this.buildReaders(props, piEnabled, performanceInsightsKey, performanceInsightsRetention);
|
|
153
153
|
const engine = props.engine ||
|
|
154
154
|
DatabaseClusterEngine.auroraPostgres({
|
|
155
|
-
version: AuroraPostgresEngineVersion.of("
|
|
155
|
+
version: AuroraPostgresEngineVersion.of("18.4", "18")
|
|
156
156
|
});
|
|
157
157
|
const parameterGroup = new ParameterGroup(this, `${this.databaseNameValue}ParameterGroup`, {
|
|
158
158
|
engine,
|
|
@@ -110,7 +110,7 @@ export class RdsInstance extends Construct {
|
|
|
110
110
|
: undefined;
|
|
111
111
|
const engine = props.engine ||
|
|
112
112
|
DatabaseInstanceEngine.postgres({
|
|
113
|
-
version: PostgresEngineVersion.
|
|
113
|
+
version: PostgresEngineVersion.VER_17_9
|
|
114
114
|
});
|
|
115
115
|
const parameterGroup = new ParameterGroup(this, `${this.databaseNameValue}ParameterGroup`, {
|
|
116
116
|
engine,
|
|
@@ -6,11 +6,11 @@ import type { Construct } from "constructs";
|
|
|
6
6
|
export interface ClickHouseAlarmThresholds {
|
|
7
7
|
/** EC2 host CPU % over 5 min. Default 90. */
|
|
8
8
|
cpuThreshold?: number;
|
|
9
|
-
/** EC2 host memory % over 5 min (
|
|
9
|
+
/** EC2 host memory % over 5 min (CWAgent namespace). Default 80. */
|
|
10
10
|
memoryThreshold?: number;
|
|
11
|
-
/**
|
|
11
|
+
/** ClickHouse data volume disk % used. Default 70 (warn) — paired with critical at 85. */
|
|
12
12
|
diskWarnThreshold?: number;
|
|
13
|
-
/**
|
|
13
|
+
/** ClickHouse data volume disk % used. Default 85. */
|
|
14
14
|
diskCriticalThreshold?: number;
|
|
15
15
|
}
|
|
16
16
|
export interface ClickHouseAlarmsProps {
|
|
@@ -39,8 +39,12 @@ export interface ClickHouseAlarmsProps {
|
|
|
39
39
|
config?: ClickHouseAlarmThresholds;
|
|
40
40
|
}
|
|
41
41
|
/**
|
|
42
|
-
* Single-node ClickHouse host-posture alarms. Covers host-level CPU
|
|
43
|
-
* memory and disk
|
|
42
|
+
* Single-node ClickHouse host-posture alarms. Covers host-level CPU (AWS/EC2)
|
|
43
|
+
* plus memory and disk in the `CWAgent` namespace — fed NOT by the CloudWatch
|
|
44
|
+
* Agent (never installed on these hosts) but by the lightweight put-metric-data
|
|
45
|
+
* timer the user-data installs (see `buildClickHouseUserData`), which publishes
|
|
46
|
+
* `mem_used_percent` / `disk_used_percent` under the same names and
|
|
47
|
+
* `AutoScalingGroupName` dimension the agent would use — plus the
|
|
44
48
|
* backup-failure log alarm when a backup-task log group is supplied:
|
|
45
49
|
*
|
|
46
50
|
* - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
|
|
@@ -5,9 +5,14 @@ import { Metric } from "aws-cdk-lib/aws-cloudwatch";
|
|
|
5
5
|
import { FilterPattern, MetricFilter } from "aws-cdk-lib/aws-logs";
|
|
6
6
|
import { ALARM_DEFAULTS, registerAlarm, buildAlarmDescription } from "./alarmDefaults.js";
|
|
7
7
|
import { METRIC_NAMESPACE } from "./metricNamespaces.js";
|
|
8
|
+
import { CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
|
|
8
9
|
/**
|
|
9
|
-
* Single-node ClickHouse host-posture alarms. Covers host-level CPU
|
|
10
|
-
* memory and disk
|
|
10
|
+
* Single-node ClickHouse host-posture alarms. Covers host-level CPU (AWS/EC2)
|
|
11
|
+
* plus memory and disk in the `CWAgent` namespace — fed NOT by the CloudWatch
|
|
12
|
+
* Agent (never installed on these hosts) but by the lightweight put-metric-data
|
|
13
|
+
* timer the user-data installs (see `buildClickHouseUserData`), which publishes
|
|
14
|
+
* `mem_used_percent` / `disk_used_percent` under the same names and
|
|
15
|
+
* `AutoScalingGroupName` dimension the agent would use — plus the
|
|
11
16
|
* backup-failure log alarm when a backup-task log group is supplied:
|
|
12
17
|
*
|
|
13
18
|
* - **Backup failures** — `AccessDenied` or `S3Exception` from the backup
|
|
@@ -42,9 +47,9 @@ export function createClickHouseAlarms(props) {
|
|
|
42
47
|
const memoryAlarm = new Alarm(scope, "ClickHouseMemoryAlarm", {
|
|
43
48
|
alarmDescription: buildAlarmDescription("ClickHouse host memory utilisation exceeds threshold (CWAgent)", undefined),
|
|
44
49
|
metric: new Metric({
|
|
45
|
-
namespace:
|
|
46
|
-
metricName:
|
|
47
|
-
dimensionsMap: {
|
|
50
|
+
namespace: CLICKHOUSE_HOST_METRICS.namespace,
|
|
51
|
+
metricName: CLICKHOUSE_HOST_METRICS.memoryMetric,
|
|
52
|
+
dimensionsMap: { [CLICKHOUSE_HOST_METRICS.asgDimension]: asgName },
|
|
48
53
|
period: ALARM_DEFAULTS.EVALUATION_PERIOD,
|
|
49
54
|
statistic: "Average"
|
|
50
55
|
}),
|
|
@@ -58,9 +63,9 @@ export function createClickHouseAlarms(props) {
|
|
|
58
63
|
const diskWarnAlarm = new Alarm(scope, "ClickHouseDiskWarnAlarm", {
|
|
59
64
|
alarmDescription: buildAlarmDescription("ClickHouse data volume above 70% used — plan growth response", undefined),
|
|
60
65
|
metric: new Metric({
|
|
61
|
-
namespace:
|
|
62
|
-
metricName:
|
|
63
|
-
dimensionsMap: {
|
|
66
|
+
namespace: CLICKHOUSE_HOST_METRICS.namespace,
|
|
67
|
+
metricName: CLICKHOUSE_HOST_METRICS.diskMetric,
|
|
68
|
+
dimensionsMap: { [CLICKHOUSE_HOST_METRICS.asgDimension]: asgName },
|
|
64
69
|
period: Duration.minutes(15),
|
|
65
70
|
statistic: "Average"
|
|
66
71
|
}),
|
|
@@ -74,9 +79,9 @@ export function createClickHouseAlarms(props) {
|
|
|
74
79
|
const diskCriticalAlarm = new Alarm(scope, "ClickHouseDiskCriticalAlarm", {
|
|
75
80
|
alarmDescription: buildAlarmDescription("ClickHouse data volume above 85% used — imminent insert failures", undefined),
|
|
76
81
|
metric: new Metric({
|
|
77
|
-
namespace:
|
|
78
|
-
metricName:
|
|
79
|
-
dimensionsMap: {
|
|
82
|
+
namespace: CLICKHOUSE_HOST_METRICS.namespace,
|
|
83
|
+
metricName: CLICKHOUSE_HOST_METRICS.diskMetric,
|
|
84
|
+
dimensionsMap: { [CLICKHOUSE_HOST_METRICS.asgDimension]: asgName },
|
|
80
85
|
period: Duration.minutes(5),
|
|
81
86
|
statistic: "Average"
|
|
82
87
|
}),
|
|
@@ -84,7 +89,11 @@ export function createClickHouseAlarms(props) {
|
|
|
84
89
|
evaluationPeriods: 2,
|
|
85
90
|
datapointsToAlarm: 2,
|
|
86
91
|
comparisonOperator: ComparisonOperator.GREATER_THAN_THRESHOLD,
|
|
87
|
-
|
|
92
|
+
// BREACHING, unlike the other host alarms: a dead metrics publisher looks
|
|
93
|
+
// identical to a full disk (no datapoints either way), and this alarm's
|
|
94
|
+
// whole job is to fire before inserts start failing. The 2×5min window
|
|
95
|
+
// absorbs the boot gap on instance replacement.
|
|
96
|
+
treatMissingData: TreatMissingData.BREACHING
|
|
88
97
|
});
|
|
89
98
|
registerAlarm(diskCriticalAlarm, snsAction, alarms);
|
|
90
99
|
if (backupTaskLogGroup !== undefined) {
|
|
@@ -1,21 +1,41 @@
|
|
|
1
1
|
import { type EngineCompat } from "@fjall/util/manifest/schemas";
|
|
2
|
+
/**
|
|
3
|
+
* Normalise an npm dependency RANGE to its minimum X.Y.Z floor: strip a leading
|
|
4
|
+
* range operator (`^`, `~`, `>=`, `>`, `=`, `v`) and any prerelease/build
|
|
5
|
+
* metadata, then reconstruct `${major}.${minor}.${patch}` (missing minor/patch
|
|
6
|
+
* default to 0). Returns undefined for anything without a valid integer major —
|
|
7
|
+
* so a non-semver range (a git URL, `*`) yields no floor and the caller omits
|
|
8
|
+
* the field rather than emitting garbage.
|
|
9
|
+
*/
|
|
10
|
+
export declare function rangeFloor(range: string | undefined): string | undefined;
|
|
2
11
|
/**
|
|
3
12
|
* The constructs' own package version — the assembly's `synthesisedBy`.
|
|
4
13
|
* `undefined` only when the package layout is broken enough that the upward
|
|
5
14
|
* walk finds no matching `package.json`.
|
|
6
15
|
*/
|
|
7
16
|
export declare const CONSTRUCTS_VERSION: string | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* The minimum `aws-cdk` CLI floor these constructs were blessed against, derived
|
|
19
|
+
* from their own `peerDependencies["aws-cdk"]` — the FM1 `minimumAwsCdkCli` emit
|
|
20
|
+
* value. `undefined` when the peer is absent or non-semver (the field is then
|
|
21
|
+
* omitted from the stamped block).
|
|
22
|
+
*/
|
|
23
|
+
export declare const AWS_CDK_CLI_FLOOR: string | undefined;
|
|
8
24
|
/**
|
|
9
25
|
* Pure core of the emitter: derive the `engineCompat` block from a given
|
|
10
|
-
* version string (or undefined)
|
|
11
|
-
* `${major}.0.0` non-integer-major derivation live
|
|
12
|
-
* the derivation, delegated to by both
|
|
13
|
-
* `buildEngineCompat` — so the fail-safe and
|
|
14
|
-
* unit-testable directly, without mocking
|
|
15
|
-
* when the version is absent, empty, or has
|
|
16
|
-
* the caller stamps no block).
|
|
26
|
+
* version string (or undefined) and an optional aws-cdk CLI floor. Both the
|
|
27
|
+
* absent/empty guard and the `${major}.0.0` non-integer-major derivation live
|
|
28
|
+
* here — the single source of the derivation, delegated to by both
|
|
29
|
+
* `MINIMUM_ENGINE_VERSION` and `buildEngineCompat` — so the fail-safe and
|
|
30
|
+
* malformed-version paths are unit-testable directly, without mocking
|
|
31
|
+
* module-load state. Returns undefined when the version is absent, empty, or has
|
|
32
|
+
* no valid integer major (fail-safe: the caller stamps no block).
|
|
33
|
+
*
|
|
34
|
+
* `awsCdkCliFloor` is conditional-spread: an absent/empty floor omits
|
|
35
|
+
* `minimumAwsCdkCli` entirely (Zod-pitfall-2 safe — never `undefined`), so the
|
|
36
|
+
* FM1 check degrades to no-op rather than shipping a malformed constraint.
|
|
17
37
|
*/
|
|
18
|
-
export declare function buildEngineCompatFrom(version: string | undefined): EngineCompat | undefined;
|
|
38
|
+
export declare function buildEngineCompatFrom(version: string | undefined, awsCdkCliFloor?: string | undefined): EngineCompat | undefined;
|
|
19
39
|
/**
|
|
20
40
|
* The compatible deploy-engine floor for assemblies this constructs version
|
|
21
41
|
* synthesises: `${major}.0.0`, derived from `CONSTRUCTS_VERSION`. `undefined`
|
|
@@ -28,10 +48,12 @@ export declare const MINIMUM_ENGINE_VERSION: string | undefined;
|
|
|
28
48
|
* when the constructs' own version cannot be resolved (fail-safe: the caller
|
|
29
49
|
* stamps no block and the deploy degrades to a constraint-free proceed).
|
|
30
50
|
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
51
|
+
* Emits the two required fields plus `minimumAwsCdkCli` (Phase 2 FM1) whenever the
|
|
52
|
+
* aws-cdk peer floor resolves. The verdict classification of the emitted floors is
|
|
53
|
+
* proven in `@fjall/deploy-core`'s
|
|
54
|
+
* `orchestration/application/__tests__/engineCompat.test.ts` (a `4.0.0` engine
|
|
55
|
+
* floor refuses a 3.x engine; a `minimumAwsCdkCli` floor refuses an older bundled
|
|
56
|
+
* CLI); this module's tests prove the emitted shape matches those floors and reads
|
|
57
|
+
* back through the gate's reader.
|
|
36
58
|
*/
|
|
37
59
|
export declare function buildEngineCompat(): EngineCompat | undefined;
|