@fjall/components-infrastructure 20.0.0 → 21.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +50 -0
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +23 -3
- package/dist/lib/patterns/aws/clickhouseDatabase.js +116 -26
- package/dist/lib/patterns/aws/computeEcs.d.ts +14 -16
- package/dist/lib/patterns/aws/computeEcs.js +30 -74
- package/dist/lib/patterns/aws/database.js +2 -2
- package/dist/lib/patterns/aws/interfaces/database.d.ts +14 -13
- package/dist/lib/resources/aws/compute/ecs.js +2 -1
- package/dist/lib/resources/aws/compute/ecsTypes.d.ts +5 -3
- package/dist/lib/resources/aws/database/clickhouseBackupScript.d.ts +12 -0
- package/dist/lib/resources/aws/database/clickhouseBackupScript.js +34 -14
- package/dist/lib/resources/aws/database/clickhouseConstants.d.ts +30 -2
- package/dist/lib/resources/aws/database/clickhouseConstants.js +30 -2
- package/dist/lib/resources/aws/database/clickhouseMaintenanceScript.d.ts +69 -0
- package/dist/lib/resources/aws/database/clickhouseMaintenanceScript.js +131 -0
- package/dist/lib/resources/aws/database/rdsAurora.js +3 -0
- package/dist/lib/resources/aws/database/rdsInstance.js +4 -0
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.d.ts +12 -0
- package/dist/lib/resources/aws/monitoring/clickhouseAlarms.js +79 -3
- package/dist/lib/resources/aws/monitoring/ecsAlarms.d.ts +10 -1
- package/dist/lib/resources/aws/monitoring/ecsAlarms.js +14 -4
- package/dist/lib/resources/aws/networking/vpc.js +6 -2
- package/dist/lib/resources/aws/storage/s3.js +13 -3
- package/dist/lib/utils/validationLogger.js +20 -9
- package/package.json +3 -3
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shell fragments shared by the two maintenance sidecars (OPTIMIZE FINAL and
|
|
3
|
+
* BACKUP), which both authenticate as `fjall_maintenance` under a
|
|
4
|
+
* `max_concurrent_queries_for_user = 1` cap.
|
|
5
|
+
*
|
|
6
|
+
* Two things the cap forces on every statement the sidecars issue:
|
|
7
|
+
*
|
|
8
|
+
* 1. **The migrate ↔ maintenance mutex.** A migration holds the DDL slot for
|
|
9
|
+
* minutes; a maintenance job that started underneath it would either
|
|
10
|
+
* contend for parts the migration is rewriting or — worse — make the
|
|
11
|
+
* migration's own probes lose the slot. So each job opens with an activity
|
|
12
|
+
* pre-flight (`buildControlPlaneActivityQuery` against the schema admin)
|
|
13
|
+
* and exits `deferred` (0, a metric but no page) when a migration is under
|
|
14
|
+
* way. The runner asks the mirror question about `fjall_maintenance`.
|
|
15
|
+
*
|
|
16
|
+
* 2. **`Code: 202` is a wait, not a failure.** With a cap of one, the second
|
|
17
|
+
* query from the same identity is refused with TOO_MANY_SIMULTANEOUS_
|
|
18
|
+
* QUERIES. That is the sibling sidecar (or this job's own previous
|
|
19
|
+
* statement still winding down), so the statement runner sleeps with
|
|
20
|
+
* bounded backoff and retries until `MAINTENANCE_BUSY_BUDGET_SECONDS` is
|
|
21
|
+
* spent — only then is it a failure, and it says `reason=busy` so the
|
|
22
|
+
* responder knows to look for the holder rather than the statement.
|
|
23
|
+
*
|
|
24
|
+
* Every marker goes to stderr. The backup script captures several statement
|
|
25
|
+
* results with `$(…)`, and a marker on stdout would land in the captured
|
|
26
|
+
* value; ECS awslogs carries both streams to the same log group, which is
|
|
27
|
+
* where the metric filters read them.
|
|
28
|
+
*
|
|
29
|
+
* `sh` only (the ClickHouse image's `/bin/sh` is dash): no `$RANDOM`, no
|
|
30
|
+
* arrays, no `pipefail`. Jitter comes from the nanosecond clock instead —
|
|
31
|
+
* `%s%N` rather than `%N`, because a bare `%N` is zero-padded and a leading
|
|
32
|
+
* zero makes dash read the number as octal.
|
|
33
|
+
*
|
|
34
|
+
* The TLS preamble these fragments follow sets `set -eu` ONLY when TLS is
|
|
35
|
+
* active, so nothing here relies on `-e`: every failure path is an explicit
|
|
36
|
+
* `||` / `if` branch and the function bodies use `… && return 0` shapes that
|
|
37
|
+
* behave identically with or without it.
|
|
38
|
+
*/
|
|
39
|
+
import { buildControlPlaneActivityQuery } from "@fjall/util/migration";
|
|
40
|
+
import { MAINTENANCE_BUSY_BUDGET_SECONDS, MAINTENANCE_STATUS, maintenanceStatusMarker } from "./clickhouseConstants.js";
|
|
41
|
+
/** Backoff between `Code: 202` retries: 5s doubling to a 60s cap, plus up to
|
|
42
|
+
* `jitterSeconds` so two sidecars refused together do not retry together. */
|
|
43
|
+
const BUSY_BACKOFF = {
|
|
44
|
+
baseSeconds: 5,
|
|
45
|
+
capSeconds: 60,
|
|
46
|
+
jitterSeconds: 5
|
|
47
|
+
};
|
|
48
|
+
/** ClickHouse TOO_MANY_SIMULTANEOUS_QUERIES as it appears on the client's
|
|
49
|
+
* stderr. Matched as a substring of the captured stderr. */
|
|
50
|
+
const BUSY_ERROR_TOKEN = "Code: 202";
|
|
51
|
+
const STDERR_CAPTURE_FILE = "/tmp/fjall-ch-err";
|
|
52
|
+
/** Runs a statement; on failure emits `status=failed reason=<label>` (or
|
|
53
|
+
* `reason=busy label=<label>` when the busy budget ran out) and returns 1.
|
|
54
|
+
* For statements whose failure the job treats as fatal. */
|
|
55
|
+
export const MAINTENANCE_RUN_FN = "fjall_ch_run";
|
|
56
|
+
/** Runs a statement with the same `Code: 202` retry loop but emits NO
|
|
57
|
+
* failed-status marker on a non-busy failure — the caller owns that branch
|
|
58
|
+
* (a tolerated read, or a statement with its own marker such as the backup
|
|
59
|
+
* restore). The busy-exhausted marker is still emitted: a statement that
|
|
60
|
+
* never got a slot is always a job failure. */
|
|
61
|
+
export const MAINTENANCE_TRY_FN = "fjall_ch_try";
|
|
62
|
+
const LABEL_PATTERN = /^[a-z][a-z0-9_]*$/;
|
|
63
|
+
function markerLine(status, detail) {
|
|
64
|
+
return `echo "${maintenanceStatusMarker(status)} ${detail}" >&2`;
|
|
65
|
+
}
|
|
66
|
+
function assertLabel(label) {
|
|
67
|
+
if (!LABEL_PATTERN.test(label)) {
|
|
68
|
+
throw new Error(`ClickHouse maintenance statement label must match ${LABEL_PATTERN}, got '${label}'`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Newline-joined preamble: sets `FJALL_JOB`, defines the two statement
|
|
73
|
+
* runners, then runs the activity pre-flight. Exits the script itself on
|
|
74
|
+
* `deferred` (0) or a probe failure (1), so the job body that follows only
|
|
75
|
+
* ever runs when no migration is active.
|
|
76
|
+
*/
|
|
77
|
+
export function buildMaintenancePreamble(params) {
|
|
78
|
+
const { client, job, schemaAdminName } = params;
|
|
79
|
+
const activityQuery = buildControlPlaneActivityQuery({
|
|
80
|
+
users: [schemaAdminName]
|
|
81
|
+
}).replace(/\s+/g, " ");
|
|
82
|
+
const tryFn = [
|
|
83
|
+
`${MAINTENANCE_TRY_FN}() {`,
|
|
84
|
+
` _label="$1"; _sql="$2"; _pre=""; [ -n "\${3:-}" ] && _pre="timeout $3"`,
|
|
85
|
+
` _deadline=$(( $(date +%s) + ${MAINTENANCE_BUSY_BUDGET_SECONDS} )); _delay=${BUSY_BACKOFF.baseSeconds}`,
|
|
86
|
+
` while :; do`,
|
|
87
|
+
` $_pre ${client} --query "$_sql" 2>"${STDERR_CAPTURE_FILE}" && return 0`,
|
|
88
|
+
` _rc=$?`,
|
|
89
|
+
` if ! grep -q "${BUSY_ERROR_TOKEN}" "${STDERR_CAPTURE_FILE}"; then cat "${STDERR_CAPTURE_FILE}" >&2; return "$_rc"; fi`,
|
|
90
|
+
` if [ "$(date +%s)" -ge "$_deadline" ]; then cat "${STDERR_CAPTURE_FILE}" >&2; FJALL_CH_BUSY=1; ${markerLine(MAINTENANCE_STATUS.failed, "reason=busy label=$_label job=$FJALL_JOB")}; return 1; fi`,
|
|
91
|
+
` echo "fjall:maintenance:busy label=$_label retry_in=\${_delay}s job=$FJALL_JOB" >&2`,
|
|
92
|
+
` sleep $(( _delay + $(date +%s%N) % ${BUSY_BACKOFF.jitterSeconds} ))`,
|
|
93
|
+
` _delay=$(( _delay * 2 )); [ "$_delay" -gt ${BUSY_BACKOFF.capSeconds} ] && _delay=${BUSY_BACKOFF.capSeconds}`,
|
|
94
|
+
` done`,
|
|
95
|
+
`}`
|
|
96
|
+
];
|
|
97
|
+
const runFn = [
|
|
98
|
+
`${MAINTENANCE_RUN_FN}() {`,
|
|
99
|
+
` FJALL_CH_BUSY=0`,
|
|
100
|
+
` ${MAINTENANCE_TRY_FN} "$@" && return 0`,
|
|
101
|
+
` [ "$FJALL_CH_BUSY" = 1 ] || ${markerLine(MAINTENANCE_STATUS.failed, "reason=$1 job=$FJALL_JOB")}`,
|
|
102
|
+
` return 1`,
|
|
103
|
+
`}`
|
|
104
|
+
];
|
|
105
|
+
const probeFailed = markerLine(MAINTENANCE_STATUS.failed, "reason=probe_error job=$FJALL_JOB");
|
|
106
|
+
const deferred = markerLine(MAINTENANCE_STATUS.deferred, "reason=migration_active job=$FJALL_JOB");
|
|
107
|
+
return [
|
|
108
|
+
`FJALL_JOB=${job}`,
|
|
109
|
+
`FJALL_CH_BUSY=0`,
|
|
110
|
+
...tryFn,
|
|
111
|
+
...runFn,
|
|
112
|
+
`FJALL_ACTIVE=$(${MAINTENANCE_TRY_FN} activity_probe "${activityQuery}") || { ${probeFailed}; exit 1; }`,
|
|
113
|
+
`case "$FJALL_ACTIVE" in 0) ;; 1) ${deferred}; exit 0 ;; *) ${probeFailed}; exit 1 ;; esac`
|
|
114
|
+
].join("\n");
|
|
115
|
+
}
|
|
116
|
+
/** `fjall_ch_run <label> "<sql>" [timeoutSeconds]` — see `MAINTENANCE_RUN_FN`. */
|
|
117
|
+
export function maintenanceRun(label, sql, timeoutSeconds) {
|
|
118
|
+
assertLabel(label);
|
|
119
|
+
const timeout = timeoutSeconds !== undefined ? ` ${timeoutSeconds}` : "";
|
|
120
|
+
return `${MAINTENANCE_RUN_FN} ${label} "${sql}"${timeout}`;
|
|
121
|
+
}
|
|
122
|
+
/** `fjall_ch_try <label> "<sql>" [timeoutSeconds]` — see `MAINTENANCE_TRY_FN`. */
|
|
123
|
+
export function maintenanceTry(label, sql, timeoutSeconds) {
|
|
124
|
+
assertLabel(label);
|
|
125
|
+
const timeout = timeoutSeconds !== undefined ? ` ${timeoutSeconds}` : "";
|
|
126
|
+
return `${MAINTENANCE_TRY_FN} ${label} "${sql}"${timeout}`;
|
|
127
|
+
}
|
|
128
|
+
/** The job's terminal success line. */
|
|
129
|
+
export function maintenanceOkMarker(job) {
|
|
130
|
+
return markerLine(MAINTENANCE_STATUS.ok, `job=${job}`);
|
|
131
|
+
}
|
|
@@ -175,6 +175,9 @@ export class RdsAurora extends Construct {
|
|
|
175
175
|
},
|
|
176
176
|
storageEncrypted: true,
|
|
177
177
|
...(storageEncryptionKey && { storageEncryptionKey }),
|
|
178
|
+
// Without this the RDS API default (false) leaves every automated and
|
|
179
|
+
// manual snapshot untagged — untagged-asset posture findings fleet-wide.
|
|
180
|
+
copyTagsToSnapshot: true,
|
|
178
181
|
clusterIdentifier: props.clusterIdentifier?.toLowerCase() || ResourceNaming.clusterId(id),
|
|
179
182
|
monitoringInterval: props.monitoringInterval ?? RDS_DEFAULTS.MONITORING_INTERVAL,
|
|
180
183
|
preferredMaintenanceWindow: props.preferredMaintenanceWindow ??
|
|
@@ -139,6 +139,9 @@ export class RdsInstance extends Construct {
|
|
|
139
139
|
caCertificate: CaCertificate.RDS_CA_RSA4096_G1,
|
|
140
140
|
removalPolicy: RemovalPolicy.SNAPSHOT,
|
|
141
141
|
deleteAutomatedBackups: false,
|
|
142
|
+
// Without this the RDS API default (false) leaves every automated and
|
|
143
|
+
// manual snapshot untagged — untagged-asset posture findings fleet-wide.
|
|
144
|
+
copyTagsToSnapshot: true,
|
|
142
145
|
enablePerformanceInsights: piEnabled,
|
|
143
146
|
performanceInsightEncryptionKey: performanceInsightsEncryptionKey,
|
|
144
147
|
performanceInsightRetention: performanceInsightsRetention,
|
|
@@ -314,6 +317,7 @@ exports.handler = async (event) => {
|
|
|
314
317
|
caCertificate: CaCertificate.RDS_CA_RSA4096_G1,
|
|
315
318
|
removalPolicy: RemovalPolicy.DESTROY,
|
|
316
319
|
deleteAutomatedBackups: false,
|
|
320
|
+
copyTagsToSnapshot: true,
|
|
317
321
|
enablePerformanceInsights: piEnabled,
|
|
318
322
|
performanceInsightEncryptionKey: readReplicaPerformanceInsightsKey,
|
|
319
323
|
performanceInsightRetention: piEnabled
|
|
@@ -3,6 +3,7 @@ import type { IRole } from "aws-cdk-lib/aws-iam";
|
|
|
3
3
|
import type { ITopic } from "aws-cdk-lib/aws-sns";
|
|
4
4
|
import type { ILogGroup } from "aws-cdk-lib/aws-logs";
|
|
5
5
|
import type { Construct } from "constructs";
|
|
6
|
+
import { type ClickHouseMaintenanceJob } from "../database/clickhouseConstants.js";
|
|
6
7
|
/**
|
|
7
8
|
* The single alarm-tuning knob on `ClickHouseDatabaseProps.alarms`. The
|
|
8
9
|
* ClickHouseDatabase pattern fans it out across three mechanisms: host-level
|
|
@@ -135,6 +136,17 @@ export interface ClickHouseAlarmsProps {
|
|
|
135
136
|
* alarm and the success heartbeat stay: they monitor the backup itself.
|
|
136
137
|
*/
|
|
137
138
|
backupVerifyEnabled?: boolean;
|
|
139
|
+
/**
|
|
140
|
+
* Log group per scheduled maintenance job (OPTIMIZE FINAL, BACKUP), each
|
|
141
|
+
* running as `fjall_maintenance`. Every group gets a metric filter for the
|
|
142
|
+
* `fjall:maintenance:status=failed` marker (one shared alarm) and one for
|
|
143
|
+
* `status=deferred` (metric only — a job that stood aside for a migration
|
|
144
|
+
* is the mutex working, not a page). Empty when both jobs are disabled.
|
|
145
|
+
*/
|
|
146
|
+
maintenanceLogGroups?: ReadonlyArray<{
|
|
147
|
+
job: ClickHouseMaintenanceJob;
|
|
148
|
+
logGroup: ILogGroup;
|
|
149
|
+
}>;
|
|
138
150
|
config?: ClickHouseHostAlarmThresholds;
|
|
139
151
|
/**
|
|
140
152
|
* Critical free-space floor in GiB, ALREADY RESOLVED. Not read off `config`
|
|
@@ -5,7 +5,8 @@ import { Metric } from "aws-cdk-lib/aws-cloudwatch";
|
|
|
5
5
|
import { FilterPattern, MetricFilter } from "aws-cdk-lib/aws-logs";
|
|
6
6
|
import { ALARM_DEFAULTS, registerAlarm, tagAlarmsWithApplicationId, buildAlarmDescription } from "./alarmDefaults.js";
|
|
7
7
|
import { METRIC_NAMESPACE, stackScopedMetricNamespace } from "./metricNamespaces.js";
|
|
8
|
-
import { CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, CLICKHOUSE_HOST_METRICS } from "../database/clickhouseConstants.js";
|
|
8
|
+
import { CLICKHOUSE_BACKUP_VERIFY_FAILED_MARKER, CLICKHOUSE_BACKUP_VERIFY_SKIPPED_MARKER, CLICKHOUSE_HOST_METRICS, MAINTENANCE_STATUS, maintenanceStatusMarker } from "../database/clickhouseConstants.js";
|
|
9
|
+
import { toPascalCase } from "../../../utils/capitaliseString.js";
|
|
9
10
|
export const BACKUP_HEARTBEAT_DEFAULT_WINDOW_HOURS = 26;
|
|
10
11
|
/**
|
|
11
12
|
* Ceiling derivation — an ORDERING bound, not CloudWatch expressibility.
|
|
@@ -81,10 +82,11 @@ export function validateClickHouseAlarmThresholds(config) {
|
|
|
81
82
|
* see (the EC2 host and the backup task).
|
|
82
83
|
*/
|
|
83
84
|
export function createClickHouseAlarms(props) {
|
|
84
|
-
const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, backupVerifyEnabled = true, config = {}, diskFreeCriticalGib, applicationId } = props;
|
|
85
|
+
const { scope, instanceRole, asgName, alarmTopic, backupTaskLogGroup, backupVerifyEnabled = true, maintenanceLogGroups = [], config = {}, diskFreeCriticalGib, applicationId } = props;
|
|
85
86
|
validateClickHouseAlarmThresholds(config);
|
|
86
87
|
const alarms = [];
|
|
87
88
|
const snsAction = new SnsAction(alarmTopic);
|
|
89
|
+
const clickHouseNamespace = stackScopedMetricNamespace(METRIC_NAMESPACE.CLICKHOUSE);
|
|
88
90
|
const cpuAlarm = new Alarm(scope, "ClickHouseCpuAlarm", {
|
|
89
91
|
alarmDescription: buildAlarmDescription("ClickHouse host CPU utilisation exceeds threshold", applicationId),
|
|
90
92
|
metric: new Metric({
|
|
@@ -145,7 +147,7 @@ export function createClickHouseAlarms(props) {
|
|
|
145
147
|
registerAlarm(diskCriticalAlarm, snsAction, alarms);
|
|
146
148
|
if (backupTaskLogGroup !== undefined) {
|
|
147
149
|
const backupFailureMetricName = "ClickHouseBackupFailureCount";
|
|
148
|
-
const backupFailureNamespace =
|
|
150
|
+
const backupFailureNamespace = clickHouseNamespace;
|
|
149
151
|
new MetricFilter(scope, "ClickHouseBackupFailureMetricFilter", {
|
|
150
152
|
logGroup: backupTaskLogGroup,
|
|
151
153
|
metricNamespace: backupFailureNamespace,
|
|
@@ -189,9 +191,83 @@ export function createClickHouseAlarms(props) {
|
|
|
189
191
|
alarms
|
|
190
192
|
});
|
|
191
193
|
}
|
|
194
|
+
if (maintenanceLogGroups.length > 0) {
|
|
195
|
+
createMaintenanceStatusAlarms({
|
|
196
|
+
scope,
|
|
197
|
+
maintenanceLogGroups,
|
|
198
|
+
namespace: clickHouseNamespace,
|
|
199
|
+
snsAction,
|
|
200
|
+
applicationId,
|
|
201
|
+
alarms
|
|
202
|
+
});
|
|
203
|
+
}
|
|
192
204
|
tagAlarmsWithApplicationId(alarms, applicationId);
|
|
193
205
|
return alarms;
|
|
194
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Maintenance-job status alarms — the sidecars' own verdict on each run.
|
|
209
|
+
*
|
|
210
|
+
* Both scheduled jobs authenticate as `fjall_maintenance` under a
|
|
211
|
+
* one-query-per-user cap and open with a migrate ↔ maintenance activity
|
|
212
|
+
* pre-flight, so each run ends in exactly one of three markers
|
|
213
|
+
* (`clickhouseMaintenanceScript`):
|
|
214
|
+
*
|
|
215
|
+
* - **failed** (`fjall:maintenance:status=failed`, Sum >= 1 per hour, one
|
|
216
|
+
* alarm fed by every job's log group) — a statement raised, the busy
|
|
217
|
+
* budget ran out waiting for the sibling job (`reason=busy`), or the
|
|
218
|
+
* activity probe itself could not be answered (`reason=probe_error`).
|
|
219
|
+
* The `reason=` / `label=` fields on the line name the statement.
|
|
220
|
+
* - **deferred** (`status=deferred`) — metric filter only, no alarm. The job
|
|
221
|
+
* stood aside because a migration was active; that is the mutex doing its
|
|
222
|
+
* job. The metric exists so a job that defers on EVERY run (a wedged
|
|
223
|
+
* migration holder) is visible on a dashboard, and can be alarmed on later
|
|
224
|
+
* without touching the scripts.
|
|
225
|
+
* - **ok** — nothing to count; the backup heartbeat already covers absence.
|
|
226
|
+
*
|
|
227
|
+
* The backup job's verify markers (`BACKUP_VERIFY_*`) keep their own alarms
|
|
228
|
+
* above; the maintenance runners deliberately emit no failed-status marker
|
|
229
|
+
* on the statements those branches already report, so one bad restore does
|
|
230
|
+
* not page twice.
|
|
231
|
+
*/
|
|
232
|
+
function createMaintenanceStatusAlarms(props) {
|
|
233
|
+
const { scope, maintenanceLogGroups, namespace, snsAction, applicationId, alarms } = props;
|
|
234
|
+
const failedMetricName = "ClickHouseMaintenanceFailedCount";
|
|
235
|
+
const deferredMetricName = "ClickHouseMaintenanceDeferredCount";
|
|
236
|
+
for (const { job, logGroup } of maintenanceLogGroups) {
|
|
237
|
+
const jobId = toPascalCase(job);
|
|
238
|
+
new MetricFilter(scope, `ClickHouseMaintenance${jobId}FailedMetricFilter`, {
|
|
239
|
+
logGroup,
|
|
240
|
+
metricNamespace: namespace,
|
|
241
|
+
metricName: failedMetricName,
|
|
242
|
+
filterPattern: FilterPattern.allTerms(maintenanceStatusMarker(MAINTENANCE_STATUS.failed)),
|
|
243
|
+
metricValue: "1",
|
|
244
|
+
defaultValue: 0
|
|
245
|
+
});
|
|
246
|
+
new MetricFilter(scope, `ClickHouseMaintenance${jobId}DeferredMetricFilter`, {
|
|
247
|
+
logGroup,
|
|
248
|
+
metricNamespace: namespace,
|
|
249
|
+
metricName: deferredMetricName,
|
|
250
|
+
filterPattern: FilterPattern.allTerms(maintenanceStatusMarker(MAINTENANCE_STATUS.deferred)),
|
|
251
|
+
metricValue: "1",
|
|
252
|
+
defaultValue: 0
|
|
253
|
+
});
|
|
254
|
+
}
|
|
255
|
+
const failedAlarm = new Alarm(scope, "ClickHouseMaintenanceFailedAlarm", {
|
|
256
|
+
alarmDescription: buildAlarmDescription("ClickHouse maintenance job (OPTIMIZE FINAL or BACKUP) reported status=failed — read the reason= field on the marker line; reason=busy means the sibling job or a stuck query is holding the fjall_maintenance slot", applicationId),
|
|
257
|
+
metric: new Metric({
|
|
258
|
+
namespace,
|
|
259
|
+
metricName: failedMetricName,
|
|
260
|
+
period: Duration.hours(1),
|
|
261
|
+
statistic: "Sum"
|
|
262
|
+
}),
|
|
263
|
+
threshold: 1,
|
|
264
|
+
evaluationPeriods: 1,
|
|
265
|
+
datapointsToAlarm: 1,
|
|
266
|
+
comparisonOperator: ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
|
|
267
|
+
treatMissingData: TreatMissingData.NOT_BREACHING
|
|
268
|
+
});
|
|
269
|
+
registerAlarm(failedAlarm, snsAction, alarms);
|
|
270
|
+
}
|
|
195
271
|
/**
|
|
196
272
|
* Restore-verify alarms — the half of the backup story `BACKUP_CREATED`
|
|
197
273
|
* cannot tell.
|
|
@@ -11,7 +11,11 @@ export interface EcsServiceAlarmThresholds {
|
|
|
11
11
|
cpuThreshold?: number | false;
|
|
12
12
|
/** Memory utilisation % threshold. `false` disables the memory alarm. */
|
|
13
13
|
memoryThreshold?: number | false;
|
|
14
|
-
/** Minimum running tasks; `0` disables the running-tasks alarm.
|
|
14
|
+
/** Minimum running tasks; `0` disables the running-tasks alarm. The metric
|
|
15
|
+
* (`RunningTaskCount`) exists only in the `ECS/ContainerInsights`
|
|
16
|
+
* namespace, so the alarm is also skipped entirely when Container
|
|
17
|
+
* Insights is not enabled on the cluster — an alarm with no possible
|
|
18
|
+
* witness would sit in INSUFFICIENT_DATA forever. */
|
|
15
19
|
runningTasksMinimum?: number;
|
|
16
20
|
/** 5xx error-rate % threshold (ALB services only). `false` disables the
|
|
17
21
|
* 5xx alarm; the p99 response-time alarm is unaffected. */
|
|
@@ -30,5 +34,10 @@ export interface EcsServiceAlarmsProps {
|
|
|
30
34
|
config: EcsServiceAlarmThresholds;
|
|
31
35
|
alarmTopic: ITopic;
|
|
32
36
|
applicationId?: string;
|
|
37
|
+
/** Whether Container Insights is enabled on the service's cluster. The
|
|
38
|
+
* running-tasks alarm consumes `ECS/ContainerInsights RunningTaskCount`,
|
|
39
|
+
* which does not exist without insights — absent/false skips that alarm
|
|
40
|
+
* rather than synthesising one that can never observe its metric. */
|
|
41
|
+
containerInsightsEnabled?: boolean;
|
|
33
42
|
}
|
|
34
43
|
export declare function createEcsServiceAlarms(props: EcsServiceAlarmsProps): Alarm[];
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Duration } from "aws-cdk-lib";
|
|
2
|
-
import { Alarm, ComparisonOperator, MathExpression, TreatMissingData } from "aws-cdk-lib/aws-cloudwatch";
|
|
2
|
+
import { Alarm, ComparisonOperator, MathExpression, Metric, TreatMissingData } from "aws-cdk-lib/aws-cloudwatch";
|
|
3
3
|
import { SnsAction } from "aws-cdk-lib/aws-cloudwatch-actions";
|
|
4
4
|
import { HttpCodeTarget } from "aws-cdk-lib/aws-elasticloadbalancingv2";
|
|
5
5
|
import { ALARM_DEFAULTS, registerAlarm, tagAlarmsWithApplicationId, buildAlarmDescription } from "./alarmDefaults.js";
|
|
6
6
|
export function createEcsServiceAlarms(props) {
|
|
7
|
-
const { scope, serviceName, service, targetGroup, config, alarmTopic, applicationId } = props;
|
|
7
|
+
const { scope, serviceName, service, targetGroup, config, alarmTopic, applicationId, containerInsightsEnabled = false } = props;
|
|
8
8
|
const alarms = [];
|
|
9
9
|
const snsAction = new SnsAction(alarmTopic);
|
|
10
10
|
if (config.cpuThreshold !== false) {
|
|
@@ -36,10 +36,20 @@ export function createEcsServiceAlarms(props) {
|
|
|
36
36
|
registerAlarm(memoryAlarm, snsAction, alarms);
|
|
37
37
|
}
|
|
38
38
|
const runningMin = config.runningTasksMinimum ?? ALARM_DEFAULTS.ECS.RUNNING_TASKS_MIN;
|
|
39
|
-
|
|
39
|
+
// Skipped without Container Insights: `RunningTaskCount` exists only in the
|
|
40
|
+
// `ECS/ContainerInsights` namespace (BaseService.metric() pins `AWS/ECS`,
|
|
41
|
+
// where it never publishes), so an alarm on it would sit in
|
|
42
|
+
// INSUFFICIENT_DATA forever — a silent dead alarm is worse than no alarm.
|
|
43
|
+
if (runningMin > 0 && containerInsightsEnabled) {
|
|
40
44
|
const runningTasksAlarm = new Alarm(scope, `${serviceName}RunningTasksAlarm`, {
|
|
41
45
|
alarmDescription: buildAlarmDescription(`ECS service ${serviceName} running tasks below minimum`, applicationId),
|
|
42
|
-
metric:
|
|
46
|
+
metric: new Metric({
|
|
47
|
+
namespace: "ECS/ContainerInsights",
|
|
48
|
+
metricName: "RunningTaskCount",
|
|
49
|
+
dimensionsMap: {
|
|
50
|
+
ClusterName: service.cluster.clusterName,
|
|
51
|
+
ServiceName: service.serviceName
|
|
52
|
+
},
|
|
43
53
|
period: Duration.minutes(1),
|
|
44
54
|
statistic: "Average"
|
|
45
55
|
}),
|
|
@@ -7,6 +7,9 @@ import { ResourceNaming } from "../../../utils/resourceNaming.js";
|
|
|
7
7
|
export const FLOW_LOG_TRAFFIC_TYPES = ["ALL", "ACCEPT", "REJECT"];
|
|
8
8
|
// Read at two sites (resources-layer constructor default + patterns-layer natGateways guard) that must agree.
|
|
9
9
|
export const DEFAULT_MAX_AZS = 3;
|
|
10
|
+
// Shared by the default-path and explicit-config CloudWatch flow-log groups —
|
|
11
|
+
// drift would give the two paths different retention for the same log stream.
|
|
12
|
+
const DEFAULT_FLOW_LOG_RETENTION_DAYS = 14;
|
|
10
13
|
export class Vpc extends ec2.Vpc {
|
|
11
14
|
gatewayEndpoints = [];
|
|
12
15
|
interfaceEndpoints = [];
|
|
@@ -168,7 +171,8 @@ export class Vpc extends ec2.Vpc {
|
|
|
168
171
|
[`${id}VpcFlowLogs`]: {
|
|
169
172
|
destination: ec2.FlowLogDestination.toCloudWatchLogs(new LogGroup(scope, `${id}FlowLogGroup`, {
|
|
170
173
|
logGroupName: `/vpc/flowlogs/vpc-${id}/`,
|
|
171
|
-
removalPolicy: RemovalPolicy.DESTROY
|
|
174
|
+
removalPolicy: RemovalPolicy.DESTROY,
|
|
175
|
+
retention: Vpc.daysToRetention(DEFAULT_FLOW_LOG_RETENTION_DAYS)
|
|
172
176
|
}))
|
|
173
177
|
}
|
|
174
178
|
};
|
|
@@ -190,7 +194,7 @@ export class Vpc extends ec2.Vpc {
|
|
|
190
194
|
}
|
|
191
195
|
};
|
|
192
196
|
}
|
|
193
|
-
const retentionDays = config.retentionDays ??
|
|
197
|
+
const retentionDays = config.retentionDays ?? DEFAULT_FLOW_LOG_RETENTION_DAYS;
|
|
194
198
|
return {
|
|
195
199
|
[`${id}VpcFlowLogs`]: {
|
|
196
200
|
destination: ec2.FlowLogDestination.toCloudWatchLogs(new LogGroup(scope, `${id}FlowLogGroup`, {
|
|
@@ -44,9 +44,19 @@ export class S3Bucket extends Bucket {
|
|
|
44
44
|
websiteErrorDocument: websiteHosting.errorDocument ?? "error.html"
|
|
45
45
|
}),
|
|
46
46
|
versioned,
|
|
47
|
-
lifecycleRules:
|
|
48
|
-
|
|
49
|
-
|
|
47
|
+
lifecycleRules: [
|
|
48
|
+
// Unconditional multipart hygiene: without it, abandoned multipart
|
|
49
|
+
// uploads accrue invisible storage cost forever (CDK bootstrap does
|
|
50
|
+
// the same at 1 day).
|
|
51
|
+
{
|
|
52
|
+
id: "AbortIncompleteMultipartUploads",
|
|
53
|
+
abortIncompleteMultipartUploadAfter: Duration.days(7),
|
|
54
|
+
enabled: true
|
|
55
|
+
},
|
|
56
|
+
...(versioned && !props.lifecycleRules
|
|
57
|
+
? [{ noncurrentVersionExpiration: Duration.days(30), enabled: true }]
|
|
58
|
+
: (props.lifecycleRules ?? []))
|
|
59
|
+
]
|
|
50
60
|
});
|
|
51
61
|
this.backupVaultTier = backupVaultTier;
|
|
52
62
|
for (const statement of resourcePolicyStatements ?? []) {
|
|
@@ -4,20 +4,29 @@
|
|
|
4
4
|
* Also writes to JSONL file for debugging and support.
|
|
5
5
|
*/
|
|
6
6
|
import { appendFileSync, existsSync, mkdirSync } from "fs";
|
|
7
|
-
import { homedir } from "os";
|
|
8
7
|
import { join } from "path";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
8
|
+
import { fjallLogDir } from "@fjall/util";
|
|
9
|
+
import { FileRotator, LOG_ROTATION_DEFAULTS } from "@fjall/util/logRotation";
|
|
10
|
+
const LOG_FILENAME = "infrastructure.jsonl";
|
|
11
|
+
/**
|
|
12
|
+
* Shared with the CLI's own logs, so this file cannot outgrow the budget
|
|
13
|
+
* the rest of Fjall's logging keeps to. Without it this log had no bound of
|
|
14
|
+
* any kind: it is appended to on every CDK synth, and one developer machine
|
|
15
|
+
* reached 267MB of it inside a single release cycle.
|
|
16
|
+
*/
|
|
17
|
+
const rotator = new FileRotator({
|
|
18
|
+
maxFiles: LOG_ROTATION_DEFAULTS.MAX_FILES
|
|
19
|
+
});
|
|
14
20
|
/**
|
|
15
21
|
* Write a log entry to the infrastructure JSONL file
|
|
16
22
|
*/
|
|
17
23
|
function writeToLog(level, message) {
|
|
18
24
|
try {
|
|
19
|
-
|
|
20
|
-
|
|
25
|
+
// Resolved per write, not at module load: the log dir is an env var and
|
|
26
|
+
// CDK imports this module long before a command has read its environment.
|
|
27
|
+
const logDir = fjallLogDir();
|
|
28
|
+
if (!existsSync(logDir)) {
|
|
29
|
+
mkdirSync(logDir, { recursive: true });
|
|
21
30
|
}
|
|
22
31
|
const entry = JSON.stringify({
|
|
23
32
|
timestamp: new Date().toISOString(),
|
|
@@ -26,7 +35,9 @@ function writeToLog(level, message) {
|
|
|
26
35
|
message,
|
|
27
36
|
source: "cdk-subprocess"
|
|
28
37
|
});
|
|
29
|
-
|
|
38
|
+
const logPath = join(logDir, LOG_FILENAME);
|
|
39
|
+
rotator.rotateIfNeeded(logPath, LOG_ROTATION_DEFAULTS.MAX_FILE_SIZE);
|
|
40
|
+
appendFileSync(logPath, entry + "\n");
|
|
30
41
|
}
|
|
31
42
|
catch (err) {
|
|
32
43
|
console.debug("[Fjall] writeToLog failed:", err instanceof Error ? err.message : String(err));
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fjall/components-infrastructure",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "21.1.0",
|
|
4
4
|
"repository": {
|
|
5
5
|
"type": "git",
|
|
6
6
|
"url": "git+https://github.com/fjall-tech/fjall.git",
|
|
@@ -80,8 +80,8 @@
|
|
|
80
80
|
},
|
|
81
81
|
"dependencies": {
|
|
82
82
|
"@aws-sdk/client-organizations": "^3.1098.0",
|
|
83
|
-
"@fjall/generator": "^
|
|
84
|
-
"@fjall/util": "^
|
|
83
|
+
"@fjall/generator": "^21.1.0",
|
|
84
|
+
"@fjall/util": "^21.1.0",
|
|
85
85
|
"constructs": "^10.7.2"
|
|
86
86
|
},
|
|
87
87
|
"overrides": {
|