@fjall/components-infrastructure 10.1.2 → 11.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,10 +20,13 @@ export interface ClickHouseDatabaseProps {
20
20
  /** EC2 instance type. Default: `m7g.medium` (1 vCPU sustained, 4 GiB);
21
21
  * recommended step-up `r8g.medium` (8 GiB — see
22
22
  * `DEFAULT_CLICKHOUSE_INSTANCE_TYPE`). Must appear in
23
- * `CLICKHOUSE_INSTANCE_MEMORY_GIB` — the container memory limit derives
24
- * from it via `clickHouseTaskMemoryMiB()` and unknown types throw at
25
- * synth. CH user-data is tuned for 1 vCPU; raise per-user `max_threads`
26
- * and `max_concurrent_queries` if bumping to a >=2 vCPU host.
23
+ * `CLICKHOUSE_INSTANCE_SPECS` — the container memory limit
24
+ * (`clickHouseTaskMemoryMiB()`), the server pools/caches
25
+ * (`deriveClickHouseServerTuning()`) and the default profiles
26
+ * (`deriveClickHouseDefaultProfiles()`) ALL derive from it, and unknown
27
+ * types throw at synth. Server-side values are baked into EC2 user-data:
28
+ * taking a new type live needs a launch-template refresh + instance
29
+ * refresh + ClickHouse restart, not a config reload.
27
30
  *
28
31
  * WARNING — changing this on a DEPLOYED stack REPLACES the persistent
29
32
  * data volume, not just the instance. The instance type is embedded in
@@ -103,9 +106,12 @@ export interface ClickHouseDatabaseProps {
103
106
  managedPasswords?: string[];
104
107
  /**
105
108
  * Per-profile resource caps. Profile keys MUST match lowercase snake_case
106
- * (`^[a-z][a-z0-9_]*$`). Defaults to `ClickHouseDefaultProfiles` when
107
- * omitted four workload-class profiles
108
- * (`high_throughput_ingest`, `audit_append`, `read_only`, `ddl_admin`).
109
+ * (`^[a-z][a-z0-9_]*$`). Defaults to the four workload-class profiles
110
+ * (`high_throughput_ingest`, `audit_append`, `read_only`, `ddl_admin`)
111
+ * derived from the RESOLVED instance type via
112
+ * `deriveClickHouseDefaultProfiles()` — memory caps and thread counts
113
+ * track the host. Supplying this prop replaces the derived defaults
114
+ * wholesale (no merge), so an explicit map opts out of instance scaling.
109
115
  *
110
116
  * Throws `Error` on validation failure — unknown profile fields,
111
117
  * non-snake_case names.
@@ -20,9 +20,10 @@ import { buildClickHouseEntrypointWrapper, buildClickHouseUserData, generateUser
20
20
  import { toPascalCase } from "../../utils/capitaliseString.js";
21
21
  import { resolveAlertsTopic } from "../../utils/resolveAlertsTopic.js";
22
22
  import { createClickHouseAlarms } from "../../resources/aws/monitoring/index.js";
23
- import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, ClickHouseDefaultProfiles, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
23
+ import { ClickHouseSchemaAdminSchema, ManagedPasswordNameSchema, ProfileSpecSchema, PROFILE_NAME_PATTERN } from "../../resources/aws/database/clickhouseSchemas.js";
24
+ import { deriveClickHouseDefaultProfiles } from "../../resources/aws/database/clickhouseTuning.js";
24
25
  import { inferAmiHardwareType } from "../../resources/aws/compute/ecsConstants.js";
25
- import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
26
+ import { CLICKHOUSE_DATABASE_NAME, DEFAULT_CLICKHOUSE_INSTANCE_TYPE, CLICKHOUSE_IMAGE, CLICKHOUSE_EBS_VOLUME_SIZE_GB, CLICKHOUSE_EBS_IOPS, CLICKHOUSE_EBS_THROUGHPUT_MBPS, clickHouseTaskMemoryMiB, CLICKHOUSE_HTTP_PORT, CLICKHOUSE_HTTPS_PORT, CLICKHOUSE_NATIVE_PORT, CLICKHOUSE_TCP_SECURE_PORT, CLICKHOUSE_TLS_CERT_MOUNT_PATH, CLICKHOUSE_PROMETHEUS_PORT, CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_SECRET_OPTIONS, CLICKHOUSE_SERVER_ROLE_TAG, CLICKHOUSE_HOST_METRICS, clickHouseUserSecretName, CLICKHOUSE_HEALTH_CHECK, CLICKHOUSE_STOP_TIMEOUT_SECONDS, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONFIG_SUBDIR, CLICKHOUSE_USERS_SUBDIR, userPasswordEnvName, OPTIMISE_FINAL_SCHEDULE, REPLACING_MERGE_TREE_TABLES, OPTIMISE_MV_TABLES, CLICKHOUSE_CLOUDMAP_SERVICE_NAME, CLICKHOUSE_SERVER_CONTAINER_NAME, CLICKHOUSE_SERVICE_NAME, OPTIMISE_TASK_MEMORY_MIB, OPTIMISE_TASK_CPU_UNITS, BACKUP_SCHEDULE, BACKUP_TASK_MEMORY_MIB, BACKUP_TASK_CPU_UNITS, BACKUP_RETENTION_DAYS } from "../../resources/aws/database/clickhouseConstants.js";
26
27
  import { TlsCertGenerator } from "../../resources/aws/utilities/tlsCertGenerator.js";
27
28
  import { EcsCompute } from "./computeEcs.js";
28
29
  /**
@@ -134,7 +135,20 @@ export class ClickHouseDatabase extends Construct {
134
135
  throw new Error(`ClickHouseDatabase: invalid managedPasswords: prop — ${managedPasswordsParse.error.message}`);
135
136
  }
136
137
  const managedPasswords = managedPasswordsParse.data;
137
- const profiles = props.profiles ?? ClickHouseDefaultProfiles;
138
+ const contextValue = this.node.tryGetContext("clickhouseInstanceType");
139
+ const contextInstanceType = typeof contextValue === "string" && contextValue !== ""
140
+ ? contextValue
141
+ : undefined;
142
+ const propInstanceType = props.instanceType !== undefined && props.instanceType !== ""
143
+ ? props.instanceType
144
+ : undefined;
145
+ const instanceType = contextInstanceType ??
146
+ propInstanceType ??
147
+ DEFAULT_CLICKHOUSE_INSTANCE_TYPE;
148
+ // Default profiles derive from the RESOLVED instance type so memory caps
149
+ // and thread counts track the actual host — a caller-supplied `profiles:`
150
+ // map opts out of scaling entirely (their values, their responsibility).
151
+ const profiles = props.profiles ?? deriveClickHouseDefaultProfiles(instanceType);
138
152
  const ProfilesRecordSchema = z.record(z
139
153
  .string()
140
154
  .regex(PROFILE_NAME_PATTERN, "Profile name must be lowercase snake_case"), ProfileSpecSchema);
@@ -157,16 +171,6 @@ export class ClickHouseDatabase extends Construct {
157
171
  schemaAdmin.name,
158
172
  ...managedPasswords
159
173
  ];
160
- const contextValue = this.node.tryGetContext("clickhouseInstanceType");
161
- const contextInstanceType = typeof contextValue === "string" && contextValue !== ""
162
- ? contextValue
163
- : undefined;
164
- const propInstanceType = props.instanceType !== undefined && props.instanceType !== ""
165
- ? props.instanceType
166
- : undefined;
167
- const instanceType = contextInstanceType ??
168
- propInstanceType ??
169
- DEFAULT_CLICKHOUSE_INSTANCE_TYPE;
170
174
  const desiredCount = resolveClickHouseDesiredCount(this.node.tryGetContext("clickhouseDesiredCount"), props.desiredCount);
171
175
  const optimiseEnabled = props.optimiseSchedule !== false;
172
176
  const backupEnabled = props.backupSchedule !== false;
@@ -258,6 +262,7 @@ export class ClickHouseDatabase extends Construct {
258
262
  const userData = UserData.custom(buildClickHouseUserData({
259
263
  backupBucketName: backupBucket.bucketName,
260
264
  backupBucketRegion: Stack.of(this).region,
265
+ instanceType,
261
266
  ...(coldTierBucket !== undefined && {
262
267
  coldTier: {
263
268
  bucketName: coldTierBucket.bucketName,
@@ -372,7 +377,7 @@ export class ClickHouseDatabase extends Construct {
372
377
  },
373
378
  services: [
374
379
  {
375
- name: "ClickHouseService",
380
+ name: CLICKHOUSE_SERVICE_NAME,
376
381
  capacityProvider: "EC2",
377
382
  desiredCount,
378
383
  // Omitting `scaling` attaches default CPU target tracking. A scaled-out
@@ -484,9 +489,9 @@ export class ClickHouseDatabase extends Construct {
484
489
  }
485
490
  ]
486
491
  });
487
- const clickHouseTaskDef = ecsCompute.getTaskDefinition("ClickHouseService");
492
+ const clickHouseTaskDef = ecsCompute.getTaskDefinition(CLICKHOUSE_SERVICE_NAME);
488
493
  if (!clickHouseTaskDef) {
489
- throw new Error("ClickHouseDatabase: EcsCompute did not expose a ClickHouseService task definition — expected the service to be registered.");
494
+ throw new Error(`ClickHouseDatabase: EcsCompute did not expose a ${CLICKHOUSE_SERVICE_NAME} task definition — expected the service to be registered.`);
490
495
  }
491
496
  backupBucket.grantReadWrite(clickHouseTaskDef.taskRole);
492
497
  coldTierBucket?.grantReadWrite(clickHouseTaskDef.taskRole);
@@ -28,7 +28,7 @@ import { evaluateBakeGuard } from "@fjall/util/docker";
28
28
  import { toKebab, buildParameterPath } from "@fjall/util";
29
29
  import { SCHEMA_GATE_CH_CA_CERT_ENV, SCHEMA_GATE_CH_DATABASE_ENV, SCHEMA_GATE_CH_URL_ENV, SCHEMA_GATE_CONTAINER_NAME, SCHEMA_GATE_DB_URL_BASE_ENV, SCHEMA_GATE_ECR_REPO_NAME } from "@fjall/util/migration";
30
30
  import { CONSTRUCTS_VERSION } from "../../utils/engineCompat.js";
31
- import { validateEc2ServiceSizing, validateEcsDomainConfig, validateSecretName } from "../../resources/aws/compute/ecsValidation.js";
31
+ import { validateEc2ServiceSizing, validateEc2TaskMemoryFit, validateEcsDomainConfig, validateSecretName } from "../../resources/aws/compute/ecsValidation.js";
32
32
  import { validateSharedEc2CapacityConfig } from "../../resources/aws/compute/ecsCapacityConfig.js";
33
33
  import { COMPUTE_DEFAULTS, collectImportedSecretNames } from "./compute.js";
34
34
  import { isHookMigrations, normaliseSchemaGate } from "./computeEcsTypes.js";
@@ -183,6 +183,12 @@ export function validateEcsProps(props) {
183
183
  "The ec2Config will be ignored unless capacityProvider is set to 'EC2'.");
184
184
  }
185
185
  validateEc2ServiceSizing(service);
186
+ // Throw-only at this layer: the gate container is not yet injected here
187
+ // (wireSchemaGate prepends it later), so the demand under-counts by the
188
+ // gate's 512 MiB — a conservative early throw. The resources-layer call
189
+ // in validateEcsClusterProps sees the final wired containers and also
190
+ // emits the non-fatal warnings.
191
+ validateEc2TaskMemoryFit(service);
186
192
  if (typeof service.circuitBreaker === "object") {
187
193
  const threshold = service.circuitBreaker.threshold;
188
194
  if (threshold !== undefined &&
@@ -94,7 +94,7 @@ export default class EcsCluster extends Construct {
94
94
  this.directAccessEnabled = props.cluster?.directAccess === true;
95
95
  this.loadBalancerDisabled =
96
96
  props.cluster?.loadBalancer === false || this.directAccessEnabled;
97
- validateEcsClusterProps(props);
97
+ validateEcsClusterProps(props, this);
98
98
  this.cluster = this.addCluster(props);
99
99
  for (const serviceProps of props.services) {
100
100
  if (serviceProps.capacityProvider === "EC2") {
@@ -41,6 +41,19 @@ export declare const DEFAULT_ROLLING_UPDATE_PAUSE_SECONDS = 300;
41
41
  export declare function resolveEc2ContainerMemoryMiB(ec2Config: {
42
42
  readonly memoryLimitMiB?: number;
43
43
  } | undefined): number;
44
+ /**
45
+ * Hard memory limit for the synthetic `fjall-schema-gate` container on
46
+ * EC2-capacity services. The gate is a run-to-completion Node probe (a
47
+ * handful of DB queries, then exit) — giving it the service's full
48
+ * `ec2Config.memoryLimitMiB` doubles the task's placement requirement and
49
+ * can exceed the instance's registered memory entirely: with the 2048 MiB
50
+ * deploy-worker limit the task demanded 4096 MiB against a t4g.medium's
51
+ * 3835, making every task structurally unplaceable (2026-08-12 production
52
+ * wedge — armed silently at desiredCount 0, detonated on the first
53
+ * queue-driven scale-up mid-CFN-update). Fargate services are unaffected:
54
+ * their memory is task-level, so the gate shares the task envelope.
55
+ */
56
+ export declare const SCHEMA_GATE_CONTAINER_MEMORY_MIB = 512;
44
57
  export declare const DEFAULT_ECS_FALLBACK_IMAGE = "amazon/amazon-ecs-sample";
45
58
  export declare const DEFAULT_CUSTOM_RESOURCE_TIMEOUT_SECONDS = 300;
46
59
  export declare const DEFAULT_HEALTH_CHECK_GRACE_SECONDS = 120;
@@ -62,3 +75,26 @@ export declare const ARM_INSTANCE_PREFIXES: string[];
62
75
  * @returns AmiHardwareType.ARM for Graviton instances, AmiHardwareType.STANDARD for Intel/AMD
63
76
  */
64
77
  export declare function inferAmiHardwareType(instanceType: string): AmiHardwareType;
78
+ /**
79
+ * The nominal (advertised) memory of an EC2 instance type, or `undefined` when
80
+ * the family or size is outside the allow-list (metal sizes, network variants,
81
+ * free-form strings) — callers MUST treat `undefined` as "cannot judge" and
82
+ * skip, never as zero.
83
+ *
84
+ * Nominal is an upper bound on what ECS can place against: the agent, kernel
85
+ * and system reserve consume a slice before the instance registers with the
86
+ * cluster (a t4g.medium registers 3835 of its nominal 4096 — the measured
87
+ * ground truth from the 2026-08-12 deploy-worker wedge). So a task demand
88
+ * `>= nominal` can NEVER be placed, and {@link estimatedEc2MemoryReserveMiB}
89
+ * approximates the slice for the near-miss warning band.
90
+ */
91
+ export declare function nominalEc2InstanceMemoryMiB(instanceType: string): number | undefined;
92
+ /**
93
+ * Estimated MiB the ECS agent + OS reserve consume before an instance
94
+ * registers its memory with the cluster. 6.5% is calibrated on the one
95
+ * measured point we own (t4g.medium: 4096 nominal − 3835 registered = 261,
96
+ * 6.4%), rounded up because under-estimating the reserve turns the warning
97
+ * band into a false pass; the 128 floor covers the smallest sizes where a
98
+ * percentage underestimates the fixed kernel cost.
99
+ */
100
+ export declare function estimatedEc2MemoryReserveMiB(nominalMiB: number): number;
@@ -54,6 +54,19 @@ export const DEFAULT_ROLLING_UPDATE_PAUSE_SECONDS = 300;
54
54
  export function resolveEc2ContainerMemoryMiB(ec2Config) {
55
55
  return ec2Config?.memoryLimitMiB ?? DEFAULT_EC2_CONTAINER_MEMORY_MIB;
56
56
  }
57
+ /**
58
+ * Hard memory limit for the synthetic `fjall-schema-gate` container on
59
+ * EC2-capacity services. The gate is a run-to-completion Node probe (a
60
+ * handful of DB queries, then exit) — giving it the service's full
61
+ * `ec2Config.memoryLimitMiB` doubles the task's placement requirement and
62
+ * can exceed the instance's registered memory entirely: with the 2048 MiB
63
+ * deploy-worker limit the task demanded 4096 MiB against a t4g.medium's
64
+ * 3835, making every task structurally unplaceable (2026-08-12 production
65
+ * wedge — armed silently at desiredCount 0, detonated on the first
66
+ * queue-driven scale-up mid-CFN-update). Fargate services are unaffected:
67
+ * their memory is task-level, so the gate shares the task envelope.
68
+ */
69
+ export const SCHEMA_GATE_CONTAINER_MEMORY_MIB = 512;
57
70
  // AWS sample image used when no ECR repository is provided. Consumed by both
58
71
  // the resources/ image resolver and the patterns/ defaults block — keep them
59
72
  // in lockstep via this single export.
@@ -117,3 +130,103 @@ export function inferAmiHardwareType(instanceType) {
117
130
  ? AmiHardwareType.ARM
118
131
  : AmiHardwareType.STANDARD;
119
132
  }
133
+ /**
134
+ * Nominal (advertised) memory at the `.medium` size for the EC2 families the
135
+ * memory-fit guard recognises. Sizes scale linearly from this base (nano ×1/8,
136
+ * micro ×1/4, small ×1/2, medium ×1, large ×2, xlarge ×4, Nxlarge ×4N), so one
137
+ * number per family covers the whole size ladder.
138
+ *
139
+ * Deliberately an allow-list, not a parser: network/local-NVMe variants
140
+ * (c5n, c6gn, m5zn, …) and exotic families carry different memory-per-size
141
+ * ratios, and a wrong nominal would turn the fit guard's verdict into a false
142
+ * rejection or a false pass. Unknown families make the guard skip instead —
143
+ * add a family here only with its ratio verified against the EC2 sizing table.
144
+ */
145
+ const EC2_FAMILY_MEDIUM_MEMORY_MIB = {
146
+ // 4 GiB at .medium — burstable and general-purpose
147
+ t2: 4096,
148
+ t3: 4096,
149
+ t3a: 4096,
150
+ t4g: 4096,
151
+ m5: 4096,
152
+ m5a: 4096,
153
+ m6g: 4096,
154
+ m6gd: 4096,
155
+ m6i: 4096,
156
+ m7g: 4096,
157
+ m7gd: 4096,
158
+ m7i: 4096,
159
+ m8g: 4096,
160
+ m8gd: 4096,
161
+ // 2 GiB at .medium — compute-optimised
162
+ c5: 2048,
163
+ c6g: 2048,
164
+ c6gd: 2048,
165
+ c6i: 2048,
166
+ c7g: 2048,
167
+ c7gd: 2048,
168
+ c7i: 2048,
169
+ c8g: 2048,
170
+ c8gd: 2048,
171
+ // 8 GiB at .medium — memory-optimised
172
+ r5: 8192,
173
+ r6g: 8192,
174
+ r6gd: 8192,
175
+ r6i: 8192,
176
+ r7g: 8192,
177
+ r7gd: 8192,
178
+ r8g: 8192,
179
+ r8gd: 8192
180
+ };
181
+ const EC2_SIZE_MEMORY_MULTIPLIER = {
182
+ nano: 0.125,
183
+ micro: 0.25,
184
+ small: 0.5,
185
+ medium: 1,
186
+ large: 2,
187
+ xlarge: 4
188
+ };
189
+ /**
190
+ * The nominal (advertised) memory of an EC2 instance type, or `undefined` when
191
+ * the family or size is outside the allow-list (metal sizes, network variants,
192
+ * free-form strings) — callers MUST treat `undefined` as "cannot judge" and
193
+ * skip, never as zero.
194
+ *
195
+ * Nominal is an upper bound on what ECS can place against: the agent, kernel
196
+ * and system reserve consume a slice before the instance registers with the
197
+ * cluster (a t4g.medium registers 3835 of its nominal 4096 — the measured
198
+ * ground truth from the 2026-08-12 deploy-worker wedge). So a task demand
199
+ * `>= nominal` can NEVER be placed, and {@link estimatedEc2MemoryReserveMiB}
200
+ * approximates the slice for the near-miss warning band.
201
+ */
202
+ export function nominalEc2InstanceMemoryMiB(instanceType) {
203
+ const parts = instanceType.split(".");
204
+ if (parts.length !== 2)
205
+ return undefined;
206
+ const family = parts[0];
207
+ const size = parts[1];
208
+ if (family === undefined || size === undefined)
209
+ return undefined;
210
+ const mediumMemoryMiB = EC2_FAMILY_MEDIUM_MEMORY_MIB[family];
211
+ if (mediumMemoryMiB === undefined)
212
+ return undefined;
213
+ const fixedMultiplier = EC2_SIZE_MEMORY_MULTIPLIER[size];
214
+ if (fixedMultiplier !== undefined)
215
+ return mediumMemoryMiB * fixedMultiplier;
216
+ const scaledSize = /^(\d+)xlarge$/.exec(size);
217
+ if (scaledSize?.[1] !== undefined) {
218
+ return mediumMemoryMiB * 4 * parseInt(scaledSize[1], 10);
219
+ }
220
+ return undefined;
221
+ }
222
+ /**
223
+ * Estimated MiB the ECS agent + OS reserve consume before an instance
224
+ * registers its memory with the cluster. 6.5% is calibrated on the one
225
+ * measured point we own (t4g.medium: 4096 nominal − 3835 registered = 261,
226
+ * 6.4%), rounded up because under-estimating the reserve turns the warning
227
+ * band into a false pass; the 128 floor covers the smallest sizes where a
228
+ * percentage underestimates the fixed kernel cost.
229
+ */
230
+ export function estimatedEc2MemoryReserveMiB(nominalMiB) {
231
+ return Math.max(128, Math.ceil(nominalMiB * 0.065));
232
+ }
@@ -4,9 +4,10 @@ import { Secret as EcsSecret } from "aws-cdk-lib/aws-ecs";
4
4
  import { StringParameter } from "aws-cdk-lib/aws-ssm";
5
5
  import { buildParameterPath } from "@fjall/util";
6
6
  import { DEFAULT_ALB_IDLE_TIMEOUT_SECONDS, FJALL_ALB_IDLE_TIMEOUT_ENV_VAR } from "@fjall/util/httpKeepAlive";
7
+ import { SCHEMA_GATE_CONTAINER_NAME } from "@fjall/util/migration";
7
8
  import { resolveOrgId } from "../../../utils/cdkContext.js";
8
9
  import { validateSsmPathComponent, validateSecretName } from "./ecsValidation.js";
9
- import { DEFAULT_LOG_RETENTION, DEFAULT_FARGATE_CPU, DEFAULT_FARGATE_MEMORY_MIB, resolveEc2ContainerMemoryMiB } from "./ecsConstants.js";
10
+ import { DEFAULT_LOG_RETENTION, DEFAULT_FARGATE_CPU, DEFAULT_FARGATE_MEMORY_MIB, resolveEc2ContainerMemoryMiB, SCHEMA_GATE_CONTAINER_MEMORY_MIB } from "./ecsConstants.js";
10
11
  import { LogGroup } from "../logging/logGroup.js";
11
12
  import { getContainerImage } from "./ecsImages.js";
12
13
  import { resolveRemoteConnections } from "./ecsRemoteConnections.js";
@@ -230,8 +231,13 @@ export function addContainersToTask(ctx, serviceName, serviceProps, taskDefiniti
230
231
  startTimeout: containerConfig.startTimeout !== undefined
231
232
  ? Duration.seconds(containerConfig.startTimeout)
232
233
  : undefined,
234
+ // EC2 placement demands the SUM of container hard limits — the gate
235
+ // probe must not inherit the service's full per-container limit (see
236
+ // SCHEMA_GATE_CONTAINER_MEMORY_MIB; the gate name is synth-reserved).
233
237
  ...(isServiceEc2(serviceProps) && {
234
- memoryLimitMiB: resolveEc2ContainerMemoryMiB(serviceProps.ec2Config)
238
+ memoryLimitMiB: containerConfig.name === SCHEMA_GATE_CONTAINER_NAME
239
+ ? SCHEMA_GATE_CONTAINER_MEMORY_MIB
240
+ : resolveEc2ContainerMemoryMiB(serviceProps.ec2Config)
235
241
  })
236
242
  });
237
243
  if (containerConfig.port !== undefined &&
@@ -1,3 +1,4 @@
1
+ import type { IConstruct } from "constructs";
1
2
  import type { DomainConfig, EcsCapacityProvider, EcsClusterProps } from "./ecsTypes.js";
2
3
  /**
3
4
  * Validates ECS cluster props before construction.
@@ -26,9 +27,14 @@ import type { DomainConfig, EcsCapacityProvider, EcsClusterProps } from "./ecsTy
26
27
  * cannot pass it and there is no resources-layer code path to validate.
27
28
  *
28
29
  * @param props - The cluster props to validate
30
+ * @param annotationsScope - When provided (the `EcsCluster` constructor passes
31
+ * `this`), non-fatal findings from {@link validateEc2TaskMemoryFit} are
32
+ * emitted as CDK synth warnings; without a scope (pure validation contexts:
33
+ * the contract corpus, unit tests) warnings are dropped and only throws
34
+ * surface
29
35
  * @throws Error if validation fails
30
36
  */
31
- export declare function validateEcsClusterProps(props: EcsClusterProps): void;
37
+ export declare function validateEcsClusterProps(props: EcsClusterProps, annotationsScope?: IConstruct): void;
32
38
  /**
33
39
  * Minimal structural input for {@link validateEc2ServiceSizing} — exactly the
34
40
  * fields the guard reads. Both the resources-layer `EcsServiceProps` and the
@@ -83,6 +89,68 @@ interface Ec2ServiceSizingInput {
83
89
  * `ec2Config.memoryLimitMiB`
84
90
  */
85
91
  export declare function validateEc2ServiceSizing(service: Ec2ServiceSizingInput): void;
92
+ /**
93
+ * Minimal structural input for {@link validateEc2TaskMemoryFit} — the fields
94
+ * the guard reads and nothing wider, so the resources-layer `EcsServiceProps`
95
+ * and the patterns-layer `EcsServiceConfig` both satisfy it (same seam
96
+ * discipline as {@link validateEc2ServiceSizing} above).
97
+ */
98
+ interface Ec2MemoryFitInput {
99
+ name: string;
100
+ capacityProvider?: EcsCapacityProvider;
101
+ containers?: ReadonlyArray<{
102
+ name?: string;
103
+ }>;
104
+ ec2Config?: {
105
+ instanceType?: string;
106
+ memoryLimitMiB?: number;
107
+ };
108
+ }
109
+ /**
110
+ * Rejects an EC2 service whose task can never be PLACED: on the EC2 launch
111
+ * type every container carries a hard memory limit (the gate container gets
112
+ * {@link SCHEMA_GATE_CONTAINER_MEMORY_MIB}, every other container gets
113
+ * `ec2Config.memoryLimitMiB ?? DEFAULT_EC2_CONTAINER_MEMORY_MIB` — see
114
+ * `addContainersToTask` in `ecsTaskDefinition.ts`, whose derivation this guard
115
+ * mirrors), and ECS places a task only if the SUM of those limits fits the
116
+ * instance's registered memory. Registered memory is always strictly below the
117
+ * type's nominal memory (agent + OS reserve), so a demand `>= nominal` is
118
+ * structurally unplaceable on ANY instance count — the stack deploys cleanly
119
+ * while desiredCount is 0, then wedges at the first scale-up. That is the
120
+ * 2026-08-12 production wedge (deploy-worker 2048 + gate 2048 = 4096 demanded
121
+ * of a t4g.medium's 3835 registered), which this guard converts into a synth
122
+ * error.
123
+ *
124
+ * Three verdicts:
125
+ *
126
+ * - demand >= nominal AND the sizing is explicit (`ec2Config.instanceType` or
127
+ * `ec2Config.memoryLimitMiB` set) → THROW. Zero false positives by
128
+ * construction: registered < nominal always, so nothing that deploys today
129
+ * can trip it.
130
+ * - demand >= nominal on an all-defaults `ec2Config` → WARNING only. The
131
+ * default instanceType/memory combo (t4g.micro + 1024) is itself
132
+ * oversubscribed, so throwing would reject every bare `ec2Config: {}`
133
+ * consumer; fixing the defaults is a separate decision.
134
+ * - demand > nominal − {@link estimatedEc2MemoryReserveMiB} → WARNING. The
135
+ * demand fits nominal but sits inside the estimated agent/OS reserve band,
136
+ * where placement depends on the AMI's exact registered memory.
137
+ *
138
+ * Skips (returns no findings) for Fargate services and for instance types
139
+ * outside the `nominalEc2InstanceMemoryMiB` allow-list.
140
+ *
141
+ * Called from BOTH validation layers, like {@link validateEc2ServiceSizing}:
142
+ * patterns `validateEcsProps` (throw-only — at that layer the gate container
143
+ * is not yet injected, so its demand under-counts by the gate's 512 MiB and
144
+ * warnings would be premature) and resources `validateEcsClusterProps` (the
145
+ * containers list is final there — gate and migration containers included —
146
+ * so both throws and warnings are exact).
147
+ *
148
+ * @param service - The service props to validate (no-op unless EC2 capacity)
149
+ * @returns Warning messages for the non-fatal verdicts (caller decides the
150
+ * sink — `validateEcsClusterProps` emits them as CDK synth warnings)
151
+ * @throws Error when the task demand cannot fit the explicitly chosen sizing
152
+ */
153
+ export declare function validateEc2TaskMemoryFit(service: Ec2MemoryFitInput): string[];
86
154
  /**
87
155
  * Domain-config constraints shared by the resources-layer hook
88
156
  * (`validateEcsClusterProps`) and the patterns-layer mirror
@@ -1,8 +1,10 @@
1
+ import { Annotations } from "aws-cdk-lib";
1
2
  import { NetworkMode } from "aws-cdk-lib/aws-ecs";
2
3
  import { evaluateBakeGuard } from "@fjall/util/docker";
4
+ import { SCHEMA_GATE_CONTAINER_NAME } from "@fjall/util/migration";
3
5
  import { toKebab, SSM_COMPONENT_PATTERN, SSM_COMPONENT_ERROR, SECRET_NAME_PATTERN, SECRET_NAME_ERROR } from "@fjall/util";
4
6
  import { ScalingType } from "./ecsTypes.js";
5
- import { DEFAULT_EC2_CONTAINER_MEMORY_MIB } from "./ecsConstants.js";
7
+ import { DEFAULT_EC2_CONTAINER_MEMORY_MIB, DEFAULT_EC2_INSTANCE_TYPE, SCHEMA_GATE_CONTAINER_MEMORY_MIB, estimatedEc2MemoryReserveMiB, nominalEc2InstanceMemoryMiB, resolveEc2ContainerMemoryMiB } from "./ecsConstants.js";
6
8
  import { validateSharedEc2CapacityConfig } from "./ecsCapacityConfig.js";
7
9
  /**
8
10
  * Validates ECS cluster props before construction.
@@ -31,9 +33,14 @@ import { validateSharedEc2CapacityConfig } from "./ecsCapacityConfig.js";
31
33
  * cannot pass it and there is no resources-layer code path to validate.
32
34
  *
33
35
  * @param props - The cluster props to validate
36
+ * @param annotationsScope - When provided (the `EcsCluster` constructor passes
37
+ * `this`), non-fatal findings from {@link validateEc2TaskMemoryFit} are
38
+ * emitted as CDK synth warnings; without a scope (pure validation contexts:
39
+ * the contract corpus, unit tests) warnings are dropped and only throws
40
+ * surface
34
41
  * @throws Error if validation fails
35
42
  */
36
- export function validateEcsClusterProps(props) {
43
+ export function validateEcsClusterProps(props, annotationsScope) {
37
44
  validateEcsDomainConfig(props.cluster?.domainConfig, props.clusterName);
38
45
  const loadBalancerDisabled = props.cluster?.loadBalancer === false ||
39
46
  props.cluster?.directAccess === true;
@@ -130,6 +137,12 @@ export function validateEcsClusterProps(props) {
130
137
  "Provide ec2Config on the service.");
131
138
  }
132
139
  validateEc2ServiceSizing(service);
140
+ const memoryFitWarnings = validateEc2TaskMemoryFit(service);
141
+ if (annotationsScope !== undefined) {
142
+ for (const warning of memoryFitWarnings) {
143
+ Annotations.of(annotationsScope).addWarningV2("@fjall/components-infrastructure:ec2TaskMemoryFit", warning);
144
+ }
145
+ }
133
146
  if (typeof service.circuitBreaker === "object") {
134
147
  const threshold = service.circuitBreaker.threshold;
135
148
  if (threshold !== undefined &&
@@ -280,6 +293,104 @@ export function validateEc2ServiceSizing(service) {
280
293
  "(Service-level cpu only applies to Fargate services.)");
281
294
  }
282
295
  }
296
+ /**
297
+ * Rejects an EC2 service whose task can never be PLACED: on the EC2 launch
298
+ * type every container carries a hard memory limit (the gate container gets
299
+ * {@link SCHEMA_GATE_CONTAINER_MEMORY_MIB}, every other container gets
300
+ * `ec2Config.memoryLimitMiB ?? DEFAULT_EC2_CONTAINER_MEMORY_MIB` — see
301
+ * `addContainersToTask` in `ecsTaskDefinition.ts`, whose derivation this guard
302
+ * mirrors), and ECS places a task only if the SUM of those limits fits the
303
+ * instance's registered memory. Registered memory is always strictly below the
304
+ * type's nominal memory (agent + OS reserve), so a demand `>= nominal` is
305
+ * structurally unplaceable on ANY instance count — the stack deploys cleanly
306
+ * while desiredCount is 0, then wedges at the first scale-up. That is the
307
+ * 2026-08-12 production wedge (deploy-worker 2048 + gate 2048 = 4096 demanded
308
+ * of a t4g.medium's 3835 registered), which this guard converts into a synth
309
+ * error.
310
+ *
311
+ * Three verdicts:
312
+ *
313
+ * - demand >= nominal AND the sizing is explicit (`ec2Config.instanceType` or
314
+ * `ec2Config.memoryLimitMiB` set) → THROW. Zero false positives by
315
+ * construction: registered < nominal always, so nothing that deploys today
316
+ * can trip it.
317
+ * - demand >= nominal on an all-defaults `ec2Config` → WARNING only. The
318
+ * default instanceType/memory combo (t4g.micro + 1024) is itself
319
+ * oversubscribed, so throwing would reject every bare `ec2Config: {}`
320
+ * consumer; fixing the defaults is a separate decision.
321
+ * - demand > nominal − {@link estimatedEc2MemoryReserveMiB} → WARNING. The
322
+ * demand fits nominal but sits inside the estimated agent/OS reserve band,
323
+ * where placement depends on the AMI's exact registered memory.
324
+ *
325
+ * Skips (returns no findings) for Fargate services and for instance types
326
+ * outside the `nominalEc2InstanceMemoryMiB` allow-list.
327
+ *
328
+ * Called from BOTH validation layers, like {@link validateEc2ServiceSizing}:
329
+ * patterns `validateEcsProps` (throw-only — at that layer the gate container
330
+ * is not yet injected, so its demand under-counts by the gate's 512 MiB and
331
+ * warnings would be premature) and resources `validateEcsClusterProps` (the
332
+ * containers list is final there — gate and migration containers included —
333
+ * so both throws and warnings are exact).
334
+ *
335
+ * @param service - The service props to validate (no-op unless EC2 capacity)
336
+ * @returns Warning messages for the non-fatal verdicts (caller decides the
337
+ * sink — `validateEcsClusterProps` emits them as CDK synth warnings)
338
+ * @throws Error when the task demand cannot fit the explicitly chosen sizing
339
+ */
340
+ export function validateEc2TaskMemoryFit(service) {
341
+ if (service.capacityProvider !== "EC2")
342
+ return [];
343
+ const instanceType = service.ec2Config?.instanceType ?? DEFAULT_EC2_INSTANCE_TYPE;
344
+ const nominal = nominalEc2InstanceMemoryMiB(instanceType);
345
+ if (nominal === undefined)
346
+ return [];
347
+ const perContainerMemory = resolveEc2ContainerMemoryMiB(service.ec2Config);
348
+ const containers = service.containers !== undefined && service.containers.length > 0
349
+ ? service.containers
350
+ : [{ name: service.name }];
351
+ const accounting = [];
352
+ let demand = 0;
353
+ for (const container of containers) {
354
+ const memory = container.name === SCHEMA_GATE_CONTAINER_NAME
355
+ ? SCHEMA_GATE_CONTAINER_MEMORY_MIB
356
+ : perContainerMemory;
357
+ demand += memory;
358
+ accounting.push(`${container.name ?? "(default)"}=${memory}`);
359
+ }
360
+ const reserve = estimatedEc2MemoryReserveMiB(nominal);
361
+ const explicitlySized = service.ec2Config?.instanceType !== undefined ||
362
+ service.ec2Config?.memoryLimitMiB !== undefined;
363
+ if (demand >= nominal) {
364
+ if (explicitlySized) {
365
+ throw new Error(`Service '${service.name}': container hard memory limits total ${demand} MiB ` +
366
+ `(${accounting.join(" + ")}), which cannot fit a ${instanceType} — ECS ` +
367
+ `registers less than the nominal ${nominal} MiB per instance (the ECS agent ` +
368
+ `and OS reserve roughly ${reserve} MiB), so no task of this shape can ever ` +
369
+ `be placed, on any instance count. The stack deploys cleanly while ` +
370
+ `desiredCount is 0, then wedges at the first scale-up. Lower ` +
371
+ `ec2Config.memoryLimitMiB (every app container takes that hard limit), ` +
372
+ `choose a larger ec2Config.instanceType, or run fewer containers in the task.`);
373
+ }
374
+ return [
375
+ `Service '${service.name}': container hard memory limits total ${demand} MiB ` +
376
+ `(${accounting.join(" + ")}), which cannot fit the default ${instanceType} ` +
377
+ `(nominal ${nominal} MiB, less after the ECS agent/OS reserve) — no task of ` +
378
+ `this shape can be placed at runtime. Set ec2Config.instanceType to a ` +
379
+ `larger type or lower ec2Config.memoryLimitMiB.`
380
+ ];
381
+ }
382
+ if (demand > nominal - reserve) {
383
+ return [
384
+ `Service '${service.name}': container hard memory limits total ${demand} MiB ` +
385
+ `of a ${instanceType}'s nominal ${nominal} MiB — inside the ECS agent/OS ` +
386
+ `reserve band (estimated usable ~${nominal - reserve} MiB). Placement can ` +
387
+ `fail depending on the AMI's exact registered memory. Leave ~${reserve} MiB ` +
388
+ `headroom: lower ec2Config.memoryLimitMiB or choose a larger ` +
389
+ `ec2Config.instanceType.`
390
+ ];
391
+ }
392
+ return [];
393
+ }
283
394
  /**
284
395
  * Domain-config constraints shared by the resources-layer hook
285
396
  * (`validateEcsClusterProps`) and the patterns-layer mirror
@@ -10,19 +10,17 @@ export declare const CLICKHOUSE_DATABASE_NAME = "analytics";
10
10
  * of 2 vCPU) for the rest of the hour. Graviton3 is also ~25% faster on
11
11
  * columnar scans than Graviton2.
12
12
  *
13
- * Settings are tuned for 1 vCPU sustained (max_threads=1 across all profiles,
14
- * max_concurrent_queries=8) — see clickhouseUserData.ts. Bumping to a larger
15
- * instance MUST be paired with raising those thread/concurrency caps.
16
- *
17
13
  * Next size up (2026-07-31 right-sizing, designs/2026-07-31-clickhouse-
18
14
  * factory-fit-for-purpose.md): `r8g.medium` (1 vCPU Graviton4, 8 GiB,
19
15
  * ~+30%/core over Graviton3) — CH's own doctrine floors production at
20
16
  * 8 GiB, and the memory-bound single-tenant profile wants RAM before
21
17
  * cores. `m7g.large` (2 vCPU, 8 GiB) only when CPU-bound. Set via
22
18
  * `clickhouseInstanceType` CDK context or the `instanceType` prop —
23
- * the container memory limit derives automatically via
24
- * `clickHouseTaskMemoryMiB()`, but the thread/concurrency caps in
25
- * clickhouseUserData.ts still need a lockstep re-tune. */
19
+ * the container memory limit, server pools/caches, and profile resource
20
+ * caps ALL derive from the type automatically (`clickHouseTaskMemoryMiB()`,
21
+ * `deriveClickHouseServerTuning()`, `deriveClickHouseDefaultProfiles()`).
22
+ * A size change needs no manual re-tune — but it DOES need an instance
23
+ * refresh + server restart to take effect (config is baked into user-data). */
26
24
  export declare const DEFAULT_CLICKHOUSE_INSTANCE_TYPE = "m7g.medium";
27
25
  /** ClickHouse container image. Explicit `docker.io/` prefix is required so
28
26
  * string-form consumers in `ecsImages.ts#getContainerImage()` route through
@@ -62,8 +60,17 @@ export declare const CLICKHOUSE_HOST_RESERVE_MIB = 1024;
62
60
  * derived from nominal memory, so the host reserve is what absorbs this
63
61
  * withholding at task placement. */
64
62
  export declare const CLICKHOUSE_ECS_RESERVED_MEMORY_MIB = 256;
65
- /** Total memory (GiB) of the instance types the ClickHouse construct knows
66
- * how to size a container for. Values are the AWS nominal figures. */
63
+ /** Hardware spec of every instance type the ClickHouse construct supports.
64
+ * Values are the AWS nominal figures. Single source of truth for BOTH the
65
+ * container memory limit (`clickHouseTaskMemoryMiB`) and the derived server
66
+ * tuning + default profiles (`clickhouseTuning.ts`) — adding a type here is
67
+ * the only step needed to support it end to end. */
68
+ export interface ClickHouseInstanceSpec {
69
+ vcpus: number;
70
+ memoryGib: number;
71
+ }
72
+ export declare const CLICKHOUSE_INSTANCE_SPECS: Record<string, ClickHouseInstanceSpec>;
73
+ /** Derived memory-only view of CLICKHOUSE_INSTANCE_SPECS. */
67
74
  export declare const CLICKHOUSE_INSTANCE_MEMORY_GIB: Record<string, number>;
68
75
  /** ECS container memory for the ClickHouse server task, derived from the
69
76
  * instance type so a size change cannot leave the hand-set container cap
@@ -197,6 +204,14 @@ export declare const CLICKHOUSE_CLOUDMAP_SERVICE_NAME = "clickhouse";
197
204
  * live-reload path into a silent no-op without any compile/test signal,
198
205
  * so this lives as a single source of truth. */
199
206
  export declare const CLICKHOUSE_SERVER_CONTAINER_NAME = "clickhouse";
207
+ /** ECS service name for the ClickHouse server. Coupled cross-repo value:
208
+ * the webapp log collector excludes this service from customer log
209
+ * collection by exact name (`CLICKHOUSE_SERVER_SERVICE_NAME` at
210
+ * `webapp/app/.server/services/monitoring/logGroupResolution.ts`) —
211
+ * deliberately duplicated rather than shared, so a webapp deploy is never
212
+ * release-coupled to a constructs mint. Renaming this service breaks that
213
+ * exclusion silently; update both sites together. */
214
+ export declare const CLICKHOUSE_SERVICE_NAME = "ClickHouseService";
200
215
  /** Materialised views that benefit from periodic OPTIMIZE to reduce part count at read time.
201
216
  * These are not ReplacingMergeTree (no dedup needed) but un-merged parts force
202
217
  * read-time aggregation which degrades query performance.