@fjall/components-infrastructure 1.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/lambda-assets/cert-generator/asset/index.js +27 -5
- package/dist/lib/patterns/aws/clickhouseDatabase.d.ts +13 -3
- package/dist/lib/patterns/aws/clickhouseDatabase.js +66 -27
- package/dist/lib/patterns/aws/clickhouseTls/types.d.ts +8 -0
- package/dist/lib/patterns/aws/clickhouseTls/types.js +11 -1
- package/dist/lib/patterns/aws/computeEcs.d.ts +12 -4
- package/dist/lib/patterns/aws/computeEcs.js +54 -15
- package/dist/lib/patterns/aws/index.d.ts +1 -0
- package/dist/lib/patterns/aws/index.js +1 -0
- package/dist/lib/patterns/aws/interfaces/database.d.ts +28 -7
- package/dist/lib/patterns/aws/interfaces/database.js +1 -1
- package/dist/lib/resources/aws/compute/ecsNetworking.js +3 -1
- package/dist/lib/resources/aws/database/clickhouseUserData.d.ts +23 -19
- package/dist/lib/resources/aws/database/clickhouseUserData.js +42 -12
- package/dist/lib/resources/aws/organisation/costAllocationTagActivator.js +3 -0
- package/dist/lib/resources/aws/secrets/tlsCaSecret.d.ts +5 -2
- package/dist/lib/resources/aws/secrets/tlsCaSecret.js +5 -4
- package/dist/lib/resources/aws/secrets/tlsServerSecret.d.ts +6 -4
- package/dist/lib/resources/aws/secrets/tlsServerSecret.js +6 -6
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.d.ts +7 -0
- package/dist/lib/resources/aws/utilities/tlsCertGenerator.js +22 -3
- package/dist/lib/utils/manifestWriter.js +6 -13
- package/package.json +5 -5
|
@@ -3436,7 +3436,7 @@ ${values.join("\n")}` : `${blockName} :`;
|
|
|
3436
3436
|
};
|
|
3437
3437
|
LocalBooleanValueBlock.NAME = "BooleanValueBlock";
|
|
3438
3438
|
var _a$s;
|
|
3439
|
-
var
|
|
3439
|
+
var Boolean2 = class extends BaseBlock {
|
|
3440
3440
|
getValue() {
|
|
3441
3441
|
return this.valueBlock.value;
|
|
3442
3442
|
}
|
|
@@ -3452,11 +3452,11 @@ ${values.join("\n")}` : `${blockName} :`;
|
|
|
3452
3452
|
return `${this.constructor.NAME} : ${this.getValue}`;
|
|
3453
3453
|
}
|
|
3454
3454
|
};
|
|
3455
|
-
_a$s =
|
|
3455
|
+
_a$s = Boolean2;
|
|
3456
3456
|
(() => {
|
|
3457
3457
|
typeStore.Boolean = _a$s;
|
|
3458
3458
|
})();
|
|
3459
|
-
|
|
3459
|
+
Boolean2.NAME = "BOOLEAN";
|
|
3460
3460
|
var LocalOctetStringValueBlock = class extends HexBlock(LocalConstructedValueBlock) {
|
|
3461
3461
|
constructor({ isConstructed = false, ...parameters } = {}) {
|
|
3462
3462
|
super(parameters);
|
|
@@ -5420,7 +5420,7 @@ ${values.join("\n")}` : `${blockName} :`;
|
|
|
5420
5420
|
exports2.BaseStringBlock = BaseStringBlock;
|
|
5421
5421
|
exports2.BitString = BitString;
|
|
5422
5422
|
exports2.BmpString = BmpString;
|
|
5423
|
-
exports2.Boolean =
|
|
5423
|
+
exports2.Boolean = Boolean2;
|
|
5424
5424
|
exports2.CharacterString = CharacterString;
|
|
5425
5425
|
exports2.Choice = Choice;
|
|
5426
5426
|
exports2.Constructed = Constructed;
|
|
@@ -17814,7 +17814,8 @@ var { webcrypto: crypto2, createHash } = require("node:crypto");
|
|
|
17814
17814
|
var x509 = require_x509_cjs();
|
|
17815
17815
|
var {
|
|
17816
17816
|
SecretsManagerClient,
|
|
17817
|
-
PutSecretValueCommand
|
|
17817
|
+
PutSecretValueCommand,
|
|
17818
|
+
DeleteSecretCommand
|
|
17818
17819
|
} = require("@aws-sdk/client-secrets-manager");
|
|
17819
17820
|
x509.cryptoProvider.set(crypto2);
|
|
17820
17821
|
var ECDSA_P256 = Object.freeze({
|
|
@@ -17898,6 +17899,27 @@ ${body}
|
|
|
17898
17899
|
}
|
|
17899
17900
|
exports.handler = async (event) => {
|
|
17900
17901
|
if (event.RequestType === "Delete") {
|
|
17902
|
+
const props2 = event.ResourceProperties || {};
|
|
17903
|
+
const arns = [props2.caSecretArn, props2.serverSecretArn].filter(Boolean);
|
|
17904
|
+
if (arns.length > 0) {
|
|
17905
|
+
const sm2 = new SecretsManagerClient({});
|
|
17906
|
+
for (const SecretId of arns) {
|
|
17907
|
+
try {
|
|
17908
|
+
await sm2.send(
|
|
17909
|
+
new DeleteSecretCommand({
|
|
17910
|
+
SecretId,
|
|
17911
|
+
ForceDeleteWithoutRecovery: true
|
|
17912
|
+
})
|
|
17913
|
+
);
|
|
17914
|
+
} catch (err) {
|
|
17915
|
+
console.warn("TlsCertGenerator: force-DeleteSecret failed", {
|
|
17916
|
+
SecretId,
|
|
17917
|
+
name: err && err.name,
|
|
17918
|
+
message: err && err.message
|
|
17919
|
+
});
|
|
17920
|
+
}
|
|
17921
|
+
}
|
|
17922
|
+
}
|
|
17901
17923
|
return { PhysicalResourceId: event.PhysicalResourceId || "tls-cert" };
|
|
17902
17924
|
}
|
|
17903
17925
|
const props = event.ResourceProperties || {};
|
|
@@ -2,7 +2,7 @@ import { Connections, type IConnectable, type IVpc } from "aws-cdk-lib/aws-ec2";
|
|
|
2
2
|
import { type TaskDefinition } from "aws-cdk-lib/aws-ecs";
|
|
3
3
|
import { type IBucket } from "aws-cdk-lib/aws-s3";
|
|
4
4
|
import { Construct } from "constructs";
|
|
5
|
-
import { Secret } from "../../resources/aws/secrets/secret.js";
|
|
5
|
+
import { Secret, type SecretImport } from "../../resources/aws/secrets/secret.js";
|
|
6
6
|
import { type ClickHouseSchemaAdmin, type ProfileSpec } from "../../resources/aws/database/clickhouseSchemas.js";
|
|
7
7
|
import { type ISecret } from "aws-cdk-lib/aws-secretsmanager";
|
|
8
8
|
import type { ClickHouseTlsOptions } from "./clickhouseTls/index.js";
|
|
@@ -174,8 +174,14 @@ export declare class ClickHouseDatabase extends Construct implements IClickHouse
|
|
|
174
174
|
* Port matches: HTTPS (8443) under TLS, HTTP (8123) otherwise.
|
|
175
175
|
*/
|
|
176
176
|
getUrl(): string;
|
|
177
|
-
/**
|
|
178
|
-
|
|
177
|
+
/**
|
|
178
|
+
* SecretImport for the CA certificate secret, or `undefined` when TLS is
|
|
179
|
+
* disabled (`tls: { mode: "none", … }`). Consumers wire this into their
|
|
180
|
+
* `secretsImport:` block under `CLICKHOUSE_CA_CERT` so `@fjall/clickhouse`
|
|
181
|
+
* `createFjallClickHouseClient()` picks it up from `process.env`. The secret
|
|
182
|
+
* holds raw PEM (no JSON field) so `field` is `undefined`.
|
|
183
|
+
*/
|
|
184
|
+
getTlsCaCertImport(): SecretImport | undefined;
|
|
179
185
|
getNativeUrl(): string;
|
|
180
186
|
getDatabaseName(): string;
|
|
181
187
|
getBackupBucket(): IBucket;
|
|
@@ -205,6 +211,10 @@ export declare class ClickHouseDatabase extends Construct implements IClickHouse
|
|
|
205
211
|
* carries the IAM grant via the standard `secretsImport` framework path.
|
|
206
212
|
*/
|
|
207
213
|
getMigrationContributions(): MigrationContributions;
|
|
214
|
+
getSchemaGateContribution(): {
|
|
215
|
+
readonly environment: Record<string, string>;
|
|
216
|
+
readonly secretsImport: Record<string, SecretImport>;
|
|
217
|
+
} | undefined;
|
|
208
218
|
/**
|
|
209
219
|
* Wires a task definition to connect to ClickHouse as `opts.user` (if
|
|
210
220
|
* supplied). Adds `CLICKHOUSE_URL` / `CLICKHOUSE_DATABASE` env vars to the
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CLICKHOUSE_MANAGED_USERS_ENV, pickLatestClickHouseMigration } from "@fjall/util/migration";
|
|
1
|
+
import { CLICKHOUSE_MANAGED_USERS_ENV, pickLatestClickHouseMigration, SCHEMA_ADMIN_PASSWORD_ENV, SCHEMA_ADMIN_USER_ENV } from "@fjall/util/migration";
|
|
2
2
|
import { Annotations, CfnOutput, Duration, Fn, Stack } from "aws-cdk-lib";
|
|
3
3
|
import { Connections, Port, UserData } from "aws-cdk-lib/aws-ec2";
|
|
4
4
|
import { Monitoring } from "aws-cdk-lib/aws-autoscaling";
|
|
@@ -262,12 +262,13 @@ export class ClickHouseDatabase extends Construct {
|
|
|
262
262
|
region: Stack.of(this).region
|
|
263
263
|
}
|
|
264
264
|
}),
|
|
265
|
-
tlsActive,
|
|
266
265
|
...(tlsActive &&
|
|
267
266
|
tlsCaSecret !== undefined &&
|
|
268
267
|
tlsServerSecret !== undefined && {
|
|
269
|
-
|
|
270
|
-
|
|
268
|
+
tls: {
|
|
269
|
+
caSecretArn: tlsCaSecret.secretArn,
|
|
270
|
+
serverSecretArn: tlsServerSecret.secretArn
|
|
271
|
+
}
|
|
271
272
|
})
|
|
272
273
|
}));
|
|
273
274
|
// Source.data wraps the content in a zip; the default `extract: true`
|
|
@@ -294,6 +295,15 @@ export class ClickHouseDatabase extends Construct {
|
|
|
294
295
|
...OPTIMISE_MV_TABLES.map((table) => `OPTIMIZE TABLE ${CLICKHOUSE_DATABASE_NAME}.${table}`)
|
|
295
296
|
].join("; ");
|
|
296
297
|
const adminSecret = expectDefined(userSecrets.get(schemaAdmin.name), `schemaAdmin '${schemaAdmin.name}' secret not minted.`);
|
|
298
|
+
const sidecarTlsPreamble = tlsActive
|
|
299
|
+
? `set -eu && printf '%s\\n' "$CLICKHOUSE_CA_CERT" > /tmp/ca.crt && printf '%s' '<?xml version="1.0"?><config><openSSL><client><caConfig>/tmp/ca.crt</caConfig><verificationMode>strict</verificationMode><loadDefaultCAFile>false</loadDefaultCAFile><invalidCertificateHandler><name>RejectCertificateHandler</name></invalidCertificateHandler></client></openSSL></config>' > /tmp/clickhouse-client.xml && `
|
|
300
|
+
: "";
|
|
301
|
+
const sidecarTlsClientArgs = tlsActive
|
|
302
|
+
? " --config-file=/tmp/clickhouse-client.xml --secure"
|
|
303
|
+
: "";
|
|
304
|
+
const sidecarTlsSecrets = tlsActive && tlsCaSecret !== undefined
|
|
305
|
+
? { CLICKHOUSE_CA_CERT: EcsSecret.fromSecretsManager(tlsCaSecret) }
|
|
306
|
+
: {};
|
|
297
307
|
const scheduledTasks = [];
|
|
298
308
|
if (optimiseEnabled) {
|
|
299
309
|
scheduledTasks.push({
|
|
@@ -303,19 +313,13 @@ export class ClickHouseDatabase extends Construct {
|
|
|
303
313
|
cpu: OPTIMISE_TASK_CPU_UNITS,
|
|
304
314
|
memoryLimitMiB: OPTIMISE_TASK_MEMORY_MIB,
|
|
305
315
|
command: [
|
|
306
|
-
"
|
|
307
|
-
"
|
|
308
|
-
clickHouseHost
|
|
309
|
-
"--port",
|
|
310
|
-
String(nativePort),
|
|
311
|
-
"--user",
|
|
312
|
-
schemaAdmin.name,
|
|
313
|
-
...(tlsActive ? ["--secure", "--accept-invalid-certificate"] : []),
|
|
314
|
-
"--query",
|
|
315
|
-
`${optimiseQuery};`
|
|
316
|
+
"sh",
|
|
317
|
+
"-c",
|
|
318
|
+
`${sidecarTlsPreamble}clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs} --query "${optimiseQuery};"`
|
|
316
319
|
],
|
|
317
320
|
secrets: {
|
|
318
|
-
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password")
|
|
321
|
+
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password"),
|
|
322
|
+
...sidecarTlsSecrets
|
|
319
323
|
},
|
|
320
324
|
logRetention: RetentionDays.ONE_WEEK,
|
|
321
325
|
securityGroups: [securityGroup]
|
|
@@ -332,10 +336,11 @@ export class ClickHouseDatabase extends Construct {
|
|
|
332
336
|
"sh",
|
|
333
337
|
"-c",
|
|
334
338
|
// Password via CLICKHOUSE_PASSWORD env, not --password on argv (argv → /proc/<pid>/cmdline).
|
|
335
|
-
|
|
339
|
+
`${sidecarTlsPreamble}STAMP=$(date +%Y%m%d-%H%M%S) && clickhouse-client --host ${clickHouseHost} --port ${nativePort} --user ${schemaAdmin.name}${sidecarTlsClientArgs} --query "BACKUP DATABASE ${CLICKHOUSE_DATABASE_NAME} TO S3('${backupDestUrl}weekly-$STAMP/')"`
|
|
336
340
|
],
|
|
337
341
|
secrets: {
|
|
338
|
-
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password")
|
|
342
|
+
CLICKHOUSE_PASSWORD: EcsSecret.fromSecretsManager(adminSecret.secret, "password"),
|
|
343
|
+
...sidecarTlsSecrets
|
|
339
344
|
},
|
|
340
345
|
logGroup: backupTaskLogGroup,
|
|
341
346
|
securityGroups: [securityGroup]
|
|
@@ -445,7 +450,7 @@ export class ClickHouseDatabase extends Construct {
|
|
|
445
450
|
command: [
|
|
446
451
|
"CMD-SHELL",
|
|
447
452
|
tlsActive
|
|
448
|
-
? `wget -q --
|
|
453
|
+
? `wget -q --ca-certificate=${CLICKHOUSE_TLS_CERT_MOUNT_PATH}/ca.crt -O /dev/null https://127.0.0.1:${httpPort}/ping || exit 1`
|
|
449
454
|
: `wget -q -O /dev/null http://127.0.0.1:${httpPort}/ping || exit 1`
|
|
450
455
|
],
|
|
451
456
|
interval: CLICKHOUSE_HEALTH_CHECK.INTERVAL_SECONDS,
|
|
@@ -489,7 +494,7 @@ export class ClickHouseDatabase extends Construct {
|
|
|
489
494
|
`ADMIN_PASSWORD=$(aws secretsmanager get-secret-value --secret-id "${adminSecretName}" --query SecretString --output text | jq -r .password)`,
|
|
490
495
|
`if [ -z "$ADMIN_PASSWORD" ] || [ "$ADMIN_PASSWORD" = "null" ]; then echo "fjall:reload:status=failed reason=password-fetch" >&2; exit 1; fi`,
|
|
491
496
|
`mv /var/lib/clickhouse/users.d/fjall.xml.new /var/lib/clickhouse/users.d/fjall.xml`,
|
|
492
|
-
`docker exec -i -e CLICKHOUSE_CLIENT_PASSWORD="$ADMIN_PASSWORD" "$CONTAINER" clickhouse-client${tlsActive ?
|
|
497
|
+
`docker exec -i -e CLICKHOUSE_CLIENT_PASSWORD="$ADMIN_PASSWORD" "$CONTAINER" clickhouse-client${tlsActive ? ` --config-file=${CLICKHOUSE_TLS_CERT_MOUNT_PATH}/client.xml --secure --host localhost` : ""} --port ${nativePort} --user ${schemaAdmin.name} -q "SYSTEM RELOAD USERS"`
|
|
493
498
|
].join("\n");
|
|
494
499
|
const region = Stack.of(this).region;
|
|
495
500
|
const account = Stack.of(this).account;
|
|
@@ -605,9 +610,21 @@ export class ClickHouseDatabase extends Construct {
|
|
|
605
610
|
const scheme = this.tlsCaSecret !== undefined ? "https" : "http";
|
|
606
611
|
return `${scheme}://${this.getHostEndpoint()}:${this.#httpPort}`;
|
|
607
612
|
}
|
|
608
|
-
/**
|
|
609
|
-
|
|
610
|
-
|
|
613
|
+
/**
|
|
614
|
+
* SecretImport for the CA certificate secret, or `undefined` when TLS is
|
|
615
|
+
* disabled (`tls: { mode: "none", … }`). Consumers wire this into their
|
|
616
|
+
* `secretsImport:` block under `CLICKHOUSE_CA_CERT` so `@fjall/clickhouse`
|
|
617
|
+
* `createFjallClickHouseClient()` picks it up from `process.env`. The secret
|
|
618
|
+
* holds raw PEM (no JSON field) so `field` is `undefined`.
|
|
619
|
+
*/
|
|
620
|
+
getTlsCaCertImport() {
|
|
621
|
+
if (this.tlsCaSecret === undefined)
|
|
622
|
+
return undefined;
|
|
623
|
+
return {
|
|
624
|
+
id: `${this.node.id}TlsCaCertImport`,
|
|
625
|
+
name: this.tlsCaSecret.secretName,
|
|
626
|
+
field: undefined
|
|
627
|
+
};
|
|
611
628
|
}
|
|
612
629
|
getNativeUrl() {
|
|
613
630
|
return `tcp://${this.getHostEndpoint()}:${this.#nativePort}`;
|
|
@@ -637,7 +654,7 @@ export class ClickHouseDatabase extends Construct {
|
|
|
637
654
|
}
|
|
638
655
|
grantConnect(grantee) {
|
|
639
656
|
this.connections.allowDefaultPortFrom(grantee);
|
|
640
|
-
this.connections.allowFrom(grantee, Port.tcp(
|
|
657
|
+
this.connections.allowFrom(grantee, Port.tcp(this.#nativePort));
|
|
641
658
|
}
|
|
642
659
|
/**
|
|
643
660
|
* Migration contributions for a task connecting to this ClickHouse cluster
|
|
@@ -662,12 +679,21 @@ export class ClickHouseDatabase extends Construct {
|
|
|
662
679
|
*/
|
|
663
680
|
getMigrationContributions() {
|
|
664
681
|
const environment = {
|
|
665
|
-
CLICKHOUSE_URL: this.
|
|
666
|
-
CLICKHOUSE_DATABASE: this.getDatabaseName()
|
|
682
|
+
CLICKHOUSE_URL: this.getUrl(),
|
|
683
|
+
CLICKHOUSE_DATABASE: this.getDatabaseName(),
|
|
684
|
+
[SCHEMA_ADMIN_USER_ENV]: this.#schemaAdmin.name
|
|
667
685
|
};
|
|
668
686
|
const secretsImport = {};
|
|
669
687
|
const adminSecret = this.getUser(this.#schemaAdmin.name);
|
|
670
|
-
secretsImport
|
|
688
|
+
secretsImport[SCHEMA_ADMIN_PASSWORD_ENV] =
|
|
689
|
+
adminSecret.getImport("password");
|
|
690
|
+
const caCertImport = this.getTlsCaCertImport();
|
|
691
|
+
if (caCertImport !== undefined) {
|
|
692
|
+
secretsImport.CLICKHOUSE_CA_CERT = caCertImport;
|
|
693
|
+
}
|
|
694
|
+
if (this.caCertSha256 !== undefined) {
|
|
695
|
+
environment.CLICKHOUSE_CA_CERT_SHA256 = this.caCertSha256;
|
|
696
|
+
}
|
|
671
697
|
for (const name of this.#managedPasswordNames) {
|
|
672
698
|
const userSecret = this.getUser(name);
|
|
673
699
|
secretsImport[userPasswordEnvName(name)] =
|
|
@@ -692,6 +718,19 @@ export class ClickHouseDatabase extends Construct {
|
|
|
692
718
|
}
|
|
693
719
|
return { environment, secretsImport, egress };
|
|
694
720
|
}
|
|
721
|
+
getSchemaGateContribution() {
|
|
722
|
+
if (this.#migrationsConfig === undefined)
|
|
723
|
+
return undefined;
|
|
724
|
+
const adminSecret = this.getUser(this.#schemaAdmin.name);
|
|
725
|
+
return {
|
|
726
|
+
environment: {
|
|
727
|
+
[SCHEMA_ADMIN_USER_ENV]: this.#schemaAdmin.name
|
|
728
|
+
},
|
|
729
|
+
secretsImport: {
|
|
730
|
+
[SCHEMA_ADMIN_PASSWORD_ENV]: adminSecret.getImport("password")
|
|
731
|
+
}
|
|
732
|
+
};
|
|
733
|
+
}
|
|
695
734
|
/**
|
|
696
735
|
* Wires a task definition to connect to ClickHouse as `opts.user` (if
|
|
697
736
|
* supplied). Adds `CLICKHOUSE_URL` / `CLICKHOUSE_DATABASE` env vars to the
|
|
@@ -707,7 +746,7 @@ export class ClickHouseDatabase extends Construct {
|
|
|
707
746
|
if (container === undefined) {
|
|
708
747
|
throw new Error("ClickHouseDatabase.connectFromTask: task has no default container");
|
|
709
748
|
}
|
|
710
|
-
container.addEnvironment("CLICKHOUSE_URL", this.
|
|
749
|
+
container.addEnvironment("CLICKHOUSE_URL", this.getUrl());
|
|
711
750
|
container.addEnvironment("CLICKHOUSE_DATABASE", CLICKHOUSE_DATABASE_NAME);
|
|
712
751
|
if (opts?.user !== undefined) {
|
|
713
752
|
const userSecret = this.getUser(opts.user);
|
|
@@ -35,6 +35,14 @@ export interface ClickHouseNoTls {
|
|
|
35
35
|
readonly acknowledgement: "I understand plaintext-only ClickHouse fails SOC2 DP5";
|
|
36
36
|
}
|
|
37
37
|
export type ClickHouseTlsOptions = ClickHouseSelfSignedTls | ClickHouseImportedTls | ClickHouseNoTls;
|
|
38
|
+
/**
|
|
39
|
+
* Canonical fixture for the plaintext-only escape hatch. Spelling the
|
|
40
|
+
* acknowledgement literal once and importing it everywhere prevents drift
|
|
41
|
+
* between the type definition and consumers. Pass via `tls:` to opt out of
|
|
42
|
+
* TLS-by-default; the construct still requires the explicit acknowledgement
|
|
43
|
+
* for the same reason the type does — silent plaintext is the SOC2 trap.
|
|
44
|
+
*/
|
|
45
|
+
export declare const CLICKHOUSE_TLS_NONE: ClickHouseNoTls;
|
|
38
46
|
/**
|
|
39
47
|
* The resolved TLS surface a `ClickHouseDatabase` carries after dispatch.
|
|
40
48
|
* Every field is `undefined` in `mode: "none"`. `certGeneratorFunction`
|
|
@@ -1 +1,11 @@
|
|
|
1
|
-
|
|
1
|
+
/**
|
|
2
|
+
* Canonical fixture for the plaintext-only escape hatch. Spelling the
|
|
3
|
+
* acknowledgement literal once and importing it everywhere prevents drift
|
|
4
|
+
* between the type definition and consumers. Pass via `tls:` to opt out of
|
|
5
|
+
* TLS-by-default; the construct still requires the explicit acknowledgement
|
|
6
|
+
* for the same reason the type does — silent plaintext is the SOC2 trap.
|
|
7
|
+
*/
|
|
8
|
+
export const CLICKHOUSE_TLS_NONE = {
|
|
9
|
+
mode: "none",
|
|
10
|
+
acknowledgement: "I understand plaintext-only ClickHouse fails SOC2 DP5"
|
|
11
|
+
};
|
|
@@ -4,6 +4,7 @@ import { type IGrantable, type IRole, Grant } from "aws-cdk-lib/aws-iam";
|
|
|
4
4
|
import { type IApplicationLoadBalancer, type ApplicationListener } from "aws-cdk-lib/aws-elasticloadbalancingv2";
|
|
5
5
|
import { Construct } from "constructs";
|
|
6
6
|
import { type IEcsCompute } from "./interfaces/compute.js";
|
|
7
|
+
import { type SecretImport } from "../../resources/aws/secrets/index.js";
|
|
7
8
|
import EcsCluster, { type EcsClusterProps } from "../../resources/aws/compute/ecs.js";
|
|
8
9
|
export { ScalingType } from "./computeEcsTypes.js";
|
|
9
10
|
export type { EcsCapacityProvider, Ec2CapacityConfig, RemoteConnectionSpec, EcsCapacityProviderConfig, EcsContainerConfig, ContainerDependency, ContainerVolume, EcsScheduledTaskConfig, EcsLifecycleHookMigrationsConfig, EcsPostDeployMigrationsConfig, EcsHookMigrationsConfig, EcsMigrationsConfig, EcsMigrationsMode, EcsCircuitBreakerConfig, EcsScalingConfig, EcsClusterConfig, EcsRoutingConfig, EcsServiceConfig, EcsComputeProps } from "./computeEcsTypes.js";
|
|
@@ -48,7 +49,10 @@ export declare function expandMigrationsSugar(service: EcsServiceConfig, userCon
|
|
|
48
49
|
* themselves and the resolved value differs.
|
|
49
50
|
* @internal Exported for testing only
|
|
50
51
|
*/
|
|
51
|
-
export declare function buildContainerConfigs(service: EcsServiceConfig, schemaVersionEnv?: Record<string, string>, annotationsScope?: Construct,
|
|
52
|
+
export declare function buildContainerConfigs(service: EcsServiceConfig, schemaVersionEnv?: Record<string, string>, annotationsScope?: Construct, chGate?: {
|
|
53
|
+
environment: Record<string, string>;
|
|
54
|
+
secretsImport: Record<string, SecretImport>;
|
|
55
|
+
}): EcsClusterProps["services"][number]["containers"];
|
|
52
56
|
/**
|
|
53
57
|
* Resolved scaling configuration for an ECS service.
|
|
54
58
|
* @internal Exported for testing only
|
|
@@ -94,13 +98,17 @@ export declare class EcsCompute extends Construct implements IEcsCompute {
|
|
|
94
98
|
private resolveSchemaVersionEnv;
|
|
95
99
|
/**
|
|
96
100
|
* ClickHouse mirror of `resolveSchemaVersionEnv`. Returns the
|
|
97
|
-
* `EXPECTED_CH_SCHEMA_VERSION` env entry
|
|
98
|
-
*
|
|
101
|
+
* `EXPECTED_CH_SCHEMA_VERSION` env entry alongside the schema-admin
|
|
102
|
+
* credentials the boot gate needs to authenticate against
|
|
103
|
+
* `_schema_migrations`, or `undefined` when the service is not gated.
|
|
99
104
|
*
|
|
100
105
|
* - `schemaGate: false` → returns `undefined` (auditable opt-out — same flag
|
|
101
106
|
* covers BOTH PG and CH gates)
|
|
102
107
|
* - No migrated CH in connections → returns `undefined`
|
|
103
|
-
* - Exactly one migrated CH → returns `
|
|
108
|
+
* - Exactly one migrated CH → returns env `EXPECTED_CH_SCHEMA_VERSION` +
|
|
109
|
+
* `SCHEMA_ADMIN_USER`, plus `secretsImport.SCHEMA_ADMIN_PASSWORD`. The
|
|
110
|
+
* creds piggyback on the same gate resolution so adding the gate to a
|
|
111
|
+
* service can't ship without the creds it needs to use it.
|
|
104
112
|
* - Two or more migrated CHs → throws via `resolveClickHouseDatabaseForService`
|
|
105
113
|
*/
|
|
106
114
|
private resolveClickHouseSchemaVersionEnv;
|
|
@@ -16,6 +16,7 @@ import { createScheduledTaskDefinition, createMigrationTaskDefinition } from "..
|
|
|
16
16
|
import { EcsLifecycleHookMigration } from "../../resources/aws/compute/ecsLifecycleHookMigration.js";
|
|
17
17
|
import { Role } from "../../resources/aws/iam/role.js";
|
|
18
18
|
import { LogGroup } from "../../resources/aws/logging/logGroup.js";
|
|
19
|
+
import { Schedule } from "../../resources/aws/messaging/schedule.js";
|
|
19
20
|
import { SecurityGroup } from "../../resources/aws/networking/securityGroup.js";
|
|
20
21
|
import { vpcHasNatGateways } from "../../utils/vpcUtils.js";
|
|
21
22
|
import { toPascalCase } from "../../utils/capitaliseString.js";
|
|
@@ -467,13 +468,15 @@ function mergeContributionsIntoSeparateTaskDef(separateTaskDef, contributions) {
|
|
|
467
468
|
* themselves and the resolved value differs.
|
|
468
469
|
* @internal Exported for testing only
|
|
469
470
|
*/
|
|
470
|
-
export function buildContainerConfigs(service, schemaVersionEnv, annotationsScope,
|
|
471
|
+
export function buildContainerConfigs(service, schemaVersionEnv, annotationsScope, chGate) {
|
|
471
472
|
const userContainers = service.containers && service.containers.length > 0
|
|
472
473
|
? service.containers
|
|
473
474
|
: undefined;
|
|
474
475
|
const expanded = service.migrations
|
|
475
476
|
? expandMigrationsSugar(service, userContainers)
|
|
476
477
|
: userContainers;
|
|
478
|
+
const chSchemaVersionEnv = chGate?.environment;
|
|
479
|
+
const chGateSecretsImport = chGate?.secretsImport;
|
|
477
480
|
const mergeSchemaEnv = (authored) => {
|
|
478
481
|
if (schemaVersionEnv === undefined)
|
|
479
482
|
return authored;
|
|
@@ -517,12 +520,29 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
|
|
|
517
520
|
`ClickHouse database's \`migrations:\` config ('${resolvedValue}'). The ` +
|
|
518
521
|
`author value wins; set \`schemaGate: false\` to silence this warning.`);
|
|
519
522
|
}
|
|
523
|
+
return mergeAuthorWins(authored, chSchemaVersionEnv);
|
|
524
|
+
}
|
|
525
|
+
return mergeAuthorWins(authored, chSchemaVersionEnv);
|
|
526
|
+
};
|
|
527
|
+
const mergeAuthorWins = (authored, framework) => {
|
|
528
|
+
const merged = { ...(authored ?? {}) };
|
|
529
|
+
for (const [k, v] of Object.entries(framework)) {
|
|
530
|
+
if (merged[k] === undefined)
|
|
531
|
+
merged[k] = v;
|
|
532
|
+
}
|
|
533
|
+
return merged;
|
|
534
|
+
};
|
|
535
|
+
const mergeChGateSecretsImport = (authored) => {
|
|
536
|
+
if (chGateSecretsImport === undefined)
|
|
520
537
|
return authored;
|
|
538
|
+
if (Object.keys(chGateSecretsImport).length === 0)
|
|
539
|
+
return authored;
|
|
540
|
+
const merged = { ...(authored ?? {}) };
|
|
541
|
+
for (const [k, v] of Object.entries(chGateSecretsImport)) {
|
|
542
|
+
if (merged[k] === undefined)
|
|
543
|
+
merged[k] = v;
|
|
521
544
|
}
|
|
522
|
-
return
|
|
523
|
-
...(authored ?? {}),
|
|
524
|
-
[EXPECTED_CH_SCHEMA_VERSION_ENV]: resolvedValue
|
|
525
|
-
};
|
|
545
|
+
return merged;
|
|
526
546
|
};
|
|
527
547
|
const mergeAllSchemaEnv = (authored) => mergeChSchemaEnv(mergeSchemaEnv(authored));
|
|
528
548
|
if (expanded) {
|
|
@@ -533,7 +553,7 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
|
|
|
533
553
|
port: c.port,
|
|
534
554
|
environment: mergeAllSchemaEnv(c.environment),
|
|
535
555
|
secrets: c.secrets,
|
|
536
|
-
secretsImport: c.secretsImport,
|
|
556
|
+
secretsImport: mergeChGateSecretsImport(c.secretsImport),
|
|
537
557
|
command: c.command,
|
|
538
558
|
entryPoint: c.entryPoint,
|
|
539
559
|
essential: c.essential,
|
|
@@ -546,10 +566,14 @@ export function buildContainerConfigs(service, schemaVersionEnv, annotationsScop
|
|
|
546
566
|
});
|
|
547
567
|
}
|
|
548
568
|
const fallbackEnv = mergeAllSchemaEnv(undefined);
|
|
569
|
+
const fallbackSecretsImport = mergeChGateSecretsImport(undefined);
|
|
549
570
|
return [
|
|
550
571
|
{
|
|
551
572
|
name: `${service.name}Container`,
|
|
552
|
-
...(fallbackEnv !== undefined && { environment: fallbackEnv })
|
|
573
|
+
...(fallbackEnv !== undefined && { environment: fallbackEnv }),
|
|
574
|
+
...(fallbackSecretsImport !== undefined && {
|
|
575
|
+
secretsImport: fallbackSecretsImport
|
|
576
|
+
})
|
|
553
577
|
}
|
|
554
578
|
];
|
|
555
579
|
}
|
|
@@ -617,8 +641,8 @@ export class EcsCompute extends Construct {
|
|
|
617
641
|
this.appName = props.appName;
|
|
618
642
|
const services = props.services.map((service) => {
|
|
619
643
|
const schemaVersionEnv = this.resolveSchemaVersionEnv(service);
|
|
620
|
-
const
|
|
621
|
-
const containers = buildContainerConfigs(service, schemaVersionEnv, this,
|
|
644
|
+
const chGate = this.resolveClickHouseSchemaVersionEnv(service);
|
|
645
|
+
const containers = buildContainerConfigs(service, schemaVersionEnv, this, chGate);
|
|
622
646
|
const { scalingType, minCapacity, maxCapacity } = resolveScalingConfig(service.scaling);
|
|
623
647
|
const cloudMapService = service.serviceDiscovery !== undefined
|
|
624
648
|
? App.getInstance().registerService({
|
|
@@ -728,13 +752,17 @@ export class EcsCompute extends Construct {
|
|
|
728
752
|
}
|
|
729
753
|
/**
|
|
730
754
|
* ClickHouse mirror of `resolveSchemaVersionEnv`. Returns the
|
|
731
|
-
* `EXPECTED_CH_SCHEMA_VERSION` env entry
|
|
732
|
-
*
|
|
755
|
+
* `EXPECTED_CH_SCHEMA_VERSION` env entry alongside the schema-admin
|
|
756
|
+
* credentials the boot gate needs to authenticate against
|
|
757
|
+
* `_schema_migrations`, or `undefined` when the service is not gated.
|
|
733
758
|
*
|
|
734
759
|
* - `schemaGate: false` → returns `undefined` (auditable opt-out — same flag
|
|
735
760
|
* covers BOTH PG and CH gates)
|
|
736
761
|
* - No migrated CH in connections → returns `undefined`
|
|
737
|
-
* - Exactly one migrated CH → returns `
|
|
762
|
+
* - Exactly one migrated CH → returns env `EXPECTED_CH_SCHEMA_VERSION` +
|
|
763
|
+
* `SCHEMA_ADMIN_USER`, plus `secretsImport.SCHEMA_ADMIN_PASSWORD`. The
|
|
764
|
+
* creds piggyback on the same gate resolution so adding the gate to a
|
|
765
|
+
* service can't ship without the creds it needs to use it.
|
|
738
766
|
* - Two or more migrated CHs → throws via `resolveClickHouseDatabaseForService`
|
|
739
767
|
*/
|
|
740
768
|
resolveClickHouseSchemaVersionEnv(service) {
|
|
@@ -746,9 +774,16 @@ export class EcsCompute extends Construct {
|
|
|
746
774
|
const version = db.getExpectedSchemaVersion();
|
|
747
775
|
if (version === undefined)
|
|
748
776
|
return undefined;
|
|
749
|
-
|
|
777
|
+
const environment = {
|
|
750
778
|
[EXPECTED_CH_SCHEMA_VERSION_ENV]: version
|
|
751
779
|
};
|
|
780
|
+
const secretsImport = {};
|
|
781
|
+
const gateContribution = db.getSchemaGateContribution();
|
|
782
|
+
if (gateContribution !== undefined) {
|
|
783
|
+
Object.assign(environment, gateContribution.environment);
|
|
784
|
+
Object.assign(secretsImport, gateContribution.secretsImport);
|
|
785
|
+
}
|
|
786
|
+
return { environment, secretsImport };
|
|
752
787
|
}
|
|
753
788
|
materialiseScheduledTasks(id, props) {
|
|
754
789
|
const scheduledTasks = props.cluster?.scheduledTasks;
|
|
@@ -758,14 +793,18 @@ export class EcsCompute extends Construct {
|
|
|
758
793
|
for (const entry of scheduledTasks) {
|
|
759
794
|
const taskDef = this.buildScheduledTaskDefinition(id, entry);
|
|
760
795
|
this.ecsCluster.registerScheduledTaskDefinition(entry.name, taskDef);
|
|
761
|
-
app.addSchedule
|
|
796
|
+
// Construct under `this` (NOT app.addSchedule) so the schedule lives
|
|
797
|
+
// in EcsCompute's stack. Crossing stacks (e.g. when this EcsCompute is
|
|
798
|
+
// nested in ClickHouseDatabase) forces a CFN export-update freeze on
|
|
799
|
+
// task-def replacement.
|
|
800
|
+
new Schedule(this, `${id}${toPascalCase(entry.name)}Schedule`, {
|
|
762
801
|
schedule: entry.schedule,
|
|
763
802
|
target: {
|
|
764
803
|
ecs: this,
|
|
765
804
|
serviceName: entry.name,
|
|
766
805
|
taskCount: 1
|
|
767
806
|
},
|
|
768
|
-
|
|
807
|
+
applicationId: app.getName()
|
|
769
808
|
});
|
|
770
809
|
}
|
|
771
810
|
}
|
|
@@ -19,11 +19,11 @@ import { type TaskDefinition } from "aws-cdk-lib/aws-ecs";
|
|
|
19
19
|
import { type IGrantable, type Grant, type PolicyStatement } from "aws-cdk-lib/aws-iam";
|
|
20
20
|
import { type IBucket } from "aws-cdk-lib/aws-s3";
|
|
21
21
|
import { type Construct } from "constructs";
|
|
22
|
-
import { type Secret } from "../../../resources/aws/secrets/index.js";
|
|
22
|
+
import { type Secret, type SecretImport } from "../../../resources/aws/secrets/index.js";
|
|
23
23
|
import { type SnapshotTarget } from "../../../utils/databaseTypes.js";
|
|
24
24
|
import { type IMigrationContributor } from "./migrationContributor.js";
|
|
25
25
|
export { type SnapshotTarget } from "../../../utils/databaseTypes.js";
|
|
26
|
-
export { MIGRATION_SNAPSHOT_NAME_PREFIX, EXPECTED_SCHEMA_VERSION_ENV, EXPECTED_SCHEMA_VERSION_TOOL_ENV, EXPECTED_CH_SCHEMA_VERSION_ENV } from "@fjall/util";
|
|
26
|
+
export { MIGRATION_SNAPSHOT_NAME_PREFIX, EXPECTED_SCHEMA_VERSION_ENV, EXPECTED_SCHEMA_VERSION_TOOL_ENV, EXPECTED_CH_SCHEMA_VERSION_ENV, SCHEMA_ADMIN_USER_ENV, SCHEMA_ADMIN_PASSWORD_ENV } from "@fjall/util";
|
|
27
27
|
/**
|
|
28
28
|
* Declarative migration configuration on a relational database. Drives:
|
|
29
29
|
* - the schema-version resolver at synth (filesystem scan of `dir`)
|
|
@@ -263,12 +263,14 @@ export interface IClickHouseDatabase extends IDatabase, IConnectable, IMigration
|
|
|
263
263
|
/** Native TCP port (typically 9000). */
|
|
264
264
|
getNativePort(): number;
|
|
265
265
|
/**
|
|
266
|
-
*
|
|
267
|
-
*
|
|
268
|
-
*
|
|
269
|
-
*
|
|
266
|
+
* Canonical URL for the HTTP(S) interface. Scheme is `https` when TLS is
|
|
267
|
+
* active (default), `http` only when `tls: { mode: "none", … }` is set.
|
|
268
|
+
* Port matches: HTTPS (8443) under TLS, HTTP (8123) otherwise. No
|
|
269
|
+
* credentials in the URL — ClickHouse HTTP uses Basic Auth headers or
|
|
270
|
+
* `X-ClickHouse-User`/`X-ClickHouse-Key` headers. Compose creds at
|
|
271
|
+
* runtime via `getUser(name).getImport(...)`.
|
|
270
272
|
*/
|
|
271
|
-
|
|
273
|
+
getUrl(): string;
|
|
272
274
|
/**
|
|
273
275
|
* Native TCP URL (`tcp://<host>:<nativePort>`).
|
|
274
276
|
* No credentials in the URL — the native protocol handshake carries them.
|
|
@@ -298,6 +300,25 @@ export interface IClickHouseDatabase extends IDatabase, IConnectable, IMigration
|
|
|
298
300
|
* loud, never silent.
|
|
299
301
|
*/
|
|
300
302
|
getExpectedSchemaVersion(): string | undefined;
|
|
303
|
+
/**
|
|
304
|
+
* Schema-admin credentials for the boot-time schema-version gate run by
|
|
305
|
+
* every container that connects to this database. Returns the username
|
|
306
|
+
* (plaintext env) and the password-secret import (Secrets-Manager-backed
|
|
307
|
+
* env) so callers can authenticate against `_schema_migrations`. Returns
|
|
308
|
+
* `undefined` when no `migrations:` config is declared on this database —
|
|
309
|
+
* the gate is the only consumer, so without migrations there's nothing
|
|
310
|
+
* for it to verify.
|
|
311
|
+
*
|
|
312
|
+
* Distinct from `getMigrationContributions()`: that's the per-migration
|
|
313
|
+
* task contract (env + secrets + IAM + egress for running migrations);
|
|
314
|
+
* this is the narrower per-runtime-service contract (env + secret only)
|
|
315
|
+
* needed by the boot gate. Both contributions name the same secret —
|
|
316
|
+
* synth-time merging is the caller's concern.
|
|
317
|
+
*/
|
|
318
|
+
getSchemaGateContribution(): {
|
|
319
|
+
readonly environment: Record<string, string>;
|
|
320
|
+
readonly secretsImport: Record<string, SecretImport>;
|
|
321
|
+
} | undefined;
|
|
301
322
|
/**
|
|
302
323
|
* Grant connect permissions to a grantee.
|
|
303
324
|
* Adds the grantee to the database security group on both ports.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* dynamo.getTableName(); // ✓ Available on IDynamoDBDatabase
|
|
14
14
|
* dynamo.getHostEndpoint(); // ✗ Compile error - not on IDynamoDBDatabase
|
|
15
15
|
*/
|
|
16
|
-
export { MIGRATION_SNAPSHOT_NAME_PREFIX, EXPECTED_SCHEMA_VERSION_ENV, EXPECTED_SCHEMA_VERSION_TOOL_ENV, EXPECTED_CH_SCHEMA_VERSION_ENV } from "@fjall/util";
|
|
16
|
+
export { MIGRATION_SNAPSHOT_NAME_PREFIX, EXPECTED_SCHEMA_VERSION_ENV, EXPECTED_SCHEMA_VERSION_TOOL_ENV, EXPECTED_CH_SCHEMA_VERSION_ENV, SCHEMA_ADMIN_USER_ENV, SCHEMA_ADMIN_PASSWORD_ENV } from "@fjall/util";
|
|
17
17
|
/**
|
|
18
18
|
* Relational database type discriminator.
|
|
19
19
|
*/
|
|
@@ -110,7 +110,9 @@ export function addLoadBalancerListener(ctx, loadBalancer, certificate) {
|
|
|
110
110
|
: [];
|
|
111
111
|
return rules.length > 1;
|
|
112
112
|
});
|
|
113
|
-
|
|
113
|
+
// CDK rejects listeners with neither a default action nor target groups.
|
|
114
|
+
const noServicePorts = servicesWithPorts.length === 0;
|
|
115
|
+
const defaultAction = willHaveMultipleRoutes || noServicePorts
|
|
114
116
|
? ListenerAction.fixedResponse(404, {
|
|
115
117
|
contentType: "text/plain",
|
|
116
118
|
messageBody: "Not Found"
|
|
@@ -20,26 +20,30 @@ export interface BuildClickHouseUserDataOptions {
|
|
|
20
20
|
region: string;
|
|
21
21
|
};
|
|
22
22
|
/**
|
|
23
|
-
* When
|
|
24
|
-
* and an `<openSSL><server>` block
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
23
|
+
* TLS bootstrap configuration. When provided, the server config emits
|
|
24
|
+
* `<https_port>`, `<tcp_port_secure>`, and an `<openSSL><server>` block
|
|
25
|
+
* referencing the cert files materialised at `CLICKHOUSE_TLS_CERT_MOUNT_PATH`
|
|
26
|
+
* by the user-data bootstrap step (plaintext `<http_port>` is omitted to
|
|
27
|
+
* avoid mixed-protocol exposure). When absent, plaintext `<http_port>` is
|
|
28
|
+
* emitted and no cert materialisation runs.
|
|
29
|
+
*
|
|
30
|
+
* Both ARNs are required together (CA + server) — there is no half-TLS
|
|
31
|
+
* mode. The discriminated shape makes the "TLS-on but one ARN missing"
|
|
32
|
+
* misuse type-unrepresentable.
|
|
29
33
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
34
|
+
tls?: {
|
|
35
|
+
/**
|
|
36
|
+
* Secrets Manager ARN holding the CA cert as raw PEM. Fetched at instance
|
|
37
|
+
* boot via the EC2 instance role and written to `${dataMount}/server-certs/ca.crt`.
|
|
38
|
+
*/
|
|
39
|
+
caSecretArn: string;
|
|
40
|
+
/**
|
|
41
|
+
* Secrets Manager ARN holding the server cert + key as JSON
|
|
42
|
+
* `{ "cert": "<pem>", "key": "<pem>" }`. Fetched at instance boot and
|
|
43
|
+
* written to `${dataMount}/server-certs/server.{crt,key}`.
|
|
44
|
+
*/
|
|
45
|
+
serverSecretArn: string;
|
|
46
|
+
};
|
|
43
47
|
}
|
|
44
48
|
export declare function generateServerConfigXml(options: BuildClickHouseUserDataOptions): string;
|
|
45
49
|
export interface GenerateUsersConfigXmlOptions {
|
|
@@ -2,7 +2,7 @@ import { CLICKHOUSE_DATA_MOUNT_PATH, CLICKHOUSE_EBS_DEVICE_NAME, CLICKHOUSE_CONF
|
|
|
2
2
|
import { renderUsersXml } from "./clickhouseXmlRenderer.js";
|
|
3
3
|
export function generateServerConfigXml(options) {
|
|
4
4
|
const { backupBucketName, backupBucketRegion, coldTier } = options;
|
|
5
|
-
const tlsActive = options.
|
|
5
|
+
const tlsActive = options.tls !== undefined;
|
|
6
6
|
const storageBlock = coldTier !== undefined
|
|
7
7
|
? ` <storage_configuration>
|
|
8
8
|
<!-- Same CH 26 rule as the no-cold-tier branch: a <local_ssd> disk
|
|
@@ -260,12 +260,8 @@ function compactUserDataScript(script) {
|
|
|
260
260
|
*/
|
|
261
261
|
export function buildClickHouseUserData(options) {
|
|
262
262
|
const serverConfigXml = generateServerConfigXml(options);
|
|
263
|
-
const
|
|
264
|
-
|
|
265
|
-
(options.caSecretArn === undefined || options.serverSecretArn === undefined)) {
|
|
266
|
-
throw new Error("buildClickHouseUserData: tlsActive=true requires both caSecretArn and serverSecretArn");
|
|
267
|
-
}
|
|
268
|
-
const tlsBootstrap = tlsActive
|
|
263
|
+
const tls = options.tls;
|
|
264
|
+
const tlsBootstrap = tls !== undefined
|
|
269
265
|
? `
|
|
270
266
|
# Materialise TLS certs from Secrets Manager. The EC2 instance role carries
|
|
271
267
|
# the secretsmanager:GetSecretValue grant; certs land on the EBS-backed mount
|
|
@@ -280,16 +276,50 @@ mkdir -p "$MOUNT_POINT/server-certs"
|
|
|
280
276
|
command -v jq >/dev/null 2>&1 || dnf install -y jq || yum install -y jq
|
|
281
277
|
|
|
282
278
|
# Fetch CA (raw PEM SecretString). Quoting prevents word-splitting on multi-line PEM.
|
|
283
|
-
|
|
284
|
-
|
|
279
|
+
# tmp + mv matches the in-file users.d/fjall.xml pattern below. Even though
|
|
280
|
+
# pipefail plus the container starting only after this script exits means
|
|
281
|
+
# there is no concurrent reader at write time today, a future change that
|
|
282
|
+
# introduces a watcher or moves cert materialisation closer to container
|
|
283
|
+
# start would silently regress to partial-write exposure. Atomic-replace
|
|
284
|
+
# preserves the invariant by construction.
|
|
285
|
+
CA_PEM=$(aws secretsmanager get-secret-value --secret-id "${tls.caSecretArn}" --query SecretString --output text)
|
|
286
|
+
printf '%s\\n' "$CA_PEM" > "$MOUNT_POINT/server-certs/ca.crt.new"
|
|
287
|
+
mv "$MOUNT_POINT/server-certs/ca.crt.new" "$MOUNT_POINT/server-certs/ca.crt"
|
|
285
288
|
|
|
286
289
|
# Fetch server bundle (JSON { "cert": "<pem>", "key": "<pem>" }).
|
|
287
|
-
SERVER_JSON=$(aws secretsmanager get-secret-value --secret-id "${
|
|
288
|
-
printf '%s' "$SERVER_JSON" | jq -r .cert > "$MOUNT_POINT/server-certs/server.crt"
|
|
289
|
-
|
|
290
|
+
SERVER_JSON=$(aws secretsmanager get-secret-value --secret-id "${tls.serverSecretArn}" --query SecretString --output text)
|
|
291
|
+
printf '%s' "$SERVER_JSON" | jq -r .cert > "$MOUNT_POINT/server-certs/server.crt.new"
|
|
292
|
+
mv "$MOUNT_POINT/server-certs/server.crt.new" "$MOUNT_POINT/server-certs/server.crt"
|
|
293
|
+
# chmod BEFORE mv so the private key is 0600 the instant it lands at its
|
|
294
|
+
# final path. Without this, server.key briefly carries the umask-default mode
|
|
295
|
+
# (0644) between mv and the trailing \`chmod 600\` below — short window but
|
|
296
|
+
# observable to any non-root reader on the host during user-data execution.
|
|
297
|
+
printf '%s' "$SERVER_JSON" | jq -r .key > "$MOUNT_POINT/server-certs/server.key.new"
|
|
298
|
+
chmod 600 "$MOUNT_POINT/server-certs/server.key.new"
|
|
299
|
+
mv "$MOUNT_POINT/server-certs/server.key.new" "$MOUNT_POINT/server-certs/server.key"
|
|
300
|
+
|
|
301
|
+
# Strict-verify client config consumed by docker-exec reload SSM script and any
|
|
302
|
+
# in-container clickhouse-client invocation. RejectCertificateHandler + strict
|
|
303
|
+
# verificationMode together close the trust-anchor-skipped class.
|
|
304
|
+
cat > "$MOUNT_POINT/server-certs/client.xml" <<'CLIENTXML'
|
|
305
|
+
<?xml version="1.0"?>
|
|
306
|
+
<config>
|
|
307
|
+
<openSSL>
|
|
308
|
+
<client>
|
|
309
|
+
<caConfig>/etc/clickhouse-server/certs/ca.crt</caConfig>
|
|
310
|
+
<verificationMode>strict</verificationMode>
|
|
311
|
+
<loadDefaultCAFile>false</loadDefaultCAFile>
|
|
312
|
+
<invalidCertificateHandler>
|
|
313
|
+
<name>RejectCertificateHandler</name>
|
|
314
|
+
</invalidCertificateHandler>
|
|
315
|
+
</client>
|
|
316
|
+
</openSSL>
|
|
317
|
+
</config>
|
|
318
|
+
CLIENTXML
|
|
290
319
|
|
|
291
320
|
chown -R 101:101 "$MOUNT_POINT/server-certs"
|
|
292
321
|
chmod 600 "$MOUNT_POINT/server-certs/server.key"
|
|
322
|
+
chmod 644 "$MOUNT_POINT/server-certs/client.xml"
|
|
293
323
|
`
|
|
294
324
|
: "";
|
|
295
325
|
const script = `#!/bin/bash
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Construct } from "constructs";
|
|
2
|
+
import { PhysicalName } from "aws-cdk-lib";
|
|
2
3
|
import { Code, Runtime } from "aws-cdk-lib/aws-lambda";
|
|
3
4
|
import { PolicyStatement, Effect } from "aws-cdk-lib/aws-iam";
|
|
4
5
|
import { LambdaFunction } from "../compute/lambda.js";
|
|
@@ -23,6 +24,8 @@ export class CostAllocationTagActivator extends Construct {
|
|
|
23
24
|
constructor(scope, id) {
|
|
24
25
|
super(scope, id);
|
|
25
26
|
this.lambda = new LambdaFunction(this, "TagActivatorFunction", {
|
|
27
|
+
// Referenced cross-environment by the App messaging stack's schedule rule.
|
|
28
|
+
functionName: PhysicalName.GENERATE_IF_NEEDED,
|
|
26
29
|
runtime: Runtime.NODEJS_22_X,
|
|
27
30
|
handler: "index.handler",
|
|
28
31
|
code: Code.fromInline(HANDLER_CODE),
|
|
@@ -5,8 +5,11 @@ export interface TlsCaSecretProps {
|
|
|
5
5
|
readonly appName: string;
|
|
6
6
|
}
|
|
7
7
|
/**
|
|
8
|
-
* ClickHouse TLS CA cert (public — clients pin against this).
|
|
9
|
-
*
|
|
8
|
+
* ClickHouse TLS CA cert (public — clients pin against this). The cert-generator
|
|
9
|
+
* Lambda mints a fresh CA on each create and force-deletes on stack-delete
|
|
10
|
+
* (bypassing the AWS recovery window) so the deterministic name is reusable
|
|
11
|
+
* immediately on redeploy. Use CFN stack termination protection to guard
|
|
12
|
+
* production teardown.
|
|
10
13
|
*/
|
|
11
14
|
export declare class TlsCaSecret extends Secret {
|
|
12
15
|
constructor(scope: Construct, id: string, props: TlsCaSecretProps);
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
-
import { RemovalPolicy } from "aws-cdk-lib";
|
|
2
1
|
import { Secret } from "./secret.js";
|
|
3
2
|
/**
|
|
4
|
-
* ClickHouse TLS CA cert (public — clients pin against this).
|
|
5
|
-
*
|
|
3
|
+
* ClickHouse TLS CA cert (public — clients pin against this). The cert-generator
|
|
4
|
+
* Lambda mints a fresh CA on each create and force-deletes on stack-delete
|
|
5
|
+
* (bypassing the AWS recovery window) so the deterministic name is reusable
|
|
6
|
+
* immediately on redeploy. Use CFN stack termination protection to guard
|
|
7
|
+
* production teardown.
|
|
6
8
|
*/
|
|
7
9
|
export class TlsCaSecret extends Secret {
|
|
8
10
|
constructor(scope, id, props) {
|
|
@@ -10,6 +12,5 @@ export class TlsCaSecret extends Secret {
|
|
|
10
12
|
secretName: `fjall-${props.appName}-clickhouse-tls-ca`,
|
|
11
13
|
description: "ClickHouse TLS CA cert (public — clients pin against this)"
|
|
12
14
|
});
|
|
13
|
-
this.secret.applyRemovalPolicy(RemovalPolicy.RETAIN);
|
|
14
15
|
}
|
|
15
16
|
}
|
|
@@ -5,10 +5,12 @@ export interface TlsServerSecretProps {
|
|
|
5
5
|
readonly appName: string;
|
|
6
6
|
}
|
|
7
7
|
/**
|
|
8
|
-
* ClickHouse TLS server cert + key (JSON: `{ cert, key }`).
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* ClickHouse TLS server cert + key (JSON: `{ cert, key }`). The cert-generator
|
|
9
|
+
* Lambda mints a fresh leaf on each create and force-deletes on stack-delete
|
|
10
|
+
* (bypassing the AWS recovery window) so the deterministic name is reusable
|
|
11
|
+
* immediately on redeploy. The private key only leaves Secrets Manager via
|
|
12
|
+
* ECS executionRole + the TLS init container that materialises it onto the
|
|
13
|
+
* shared volume.
|
|
12
14
|
*/
|
|
13
15
|
export declare class TlsServerSecret extends Secret {
|
|
14
16
|
constructor(scope: Construct, id: string, props: TlsServerSecretProps);
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { RemovalPolicy } from "aws-cdk-lib";
|
|
2
1
|
import { Secret } from "./secret.js";
|
|
3
2
|
/**
|
|
4
|
-
* ClickHouse TLS server cert + key (JSON: `{ cert, key }`).
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
3
|
+
* ClickHouse TLS server cert + key (JSON: `{ cert, key }`). The cert-generator
|
|
4
|
+
* Lambda mints a fresh leaf on each create and force-deletes on stack-delete
|
|
5
|
+
* (bypassing the AWS recovery window) so the deterministic name is reusable
|
|
6
|
+
* immediately on redeploy. The private key only leaves Secrets Manager via
|
|
7
|
+
* ECS executionRole + the TLS init container that materialises it onto the
|
|
8
|
+
* shared volume.
|
|
8
9
|
*/
|
|
9
10
|
export class TlsServerSecret extends Secret {
|
|
10
11
|
constructor(scope, id, props) {
|
|
@@ -12,6 +13,5 @@ export class TlsServerSecret extends Secret {
|
|
|
12
13
|
secretName: `fjall-${props.appName}-clickhouse-tls-server`,
|
|
13
14
|
description: "ClickHouse TLS server cert + key (JSON: { cert, key })"
|
|
14
15
|
});
|
|
15
|
-
this.secret.applyRemovalPolicy(RemovalPolicy.RETAIN);
|
|
16
16
|
}
|
|
17
17
|
}
|
|
@@ -20,6 +20,13 @@ export interface TlsCertGeneratorProps {
|
|
|
20
20
|
* raw PEM (public — clients pin against it); the server secret carries a
|
|
21
21
|
* JSON `{ cert, key }` payload consumed by the TLS init container.
|
|
22
22
|
*
|
|
23
|
+
* On stack-delete, the Lambda force-deletes both secrets via
|
|
24
|
+
* `DeleteSecret(ForceDeleteWithoutRecovery=true)` so the deterministic names
|
|
25
|
+
* (`fjall-<app>-clickhouse-tls-{ca,server}`) become reusable immediately on
|
|
26
|
+
* redeploy. AWS's default 7-30 day recovery window would otherwise block
|
|
27
|
+
* recreation under the same name. CFN exposes no lever for this — the
|
|
28
|
+
* AWS::SecretsManager::Secret resource has no `RecoveryWindowInDays` field.
|
|
29
|
+
*
|
|
23
30
|
* `caCertSha256` is exposed for downstream consumers to wire into their
|
|
24
31
|
* container `environment` block as `CA_CERT_SHA256` — its CFN value
|
|
25
32
|
* changes whenever the cert regenerates, forcing task-def replacement so
|
|
@@ -23,6 +23,13 @@ const LAMBDA_TIMEOUT = Duration.minutes(2);
|
|
|
23
23
|
* raw PEM (public — clients pin against it); the server secret carries a
|
|
24
24
|
* JSON `{ cert, key }` payload consumed by the TLS init container.
|
|
25
25
|
*
|
|
26
|
+
* On stack-delete, the Lambda force-deletes both secrets via
|
|
27
|
+
* `DeleteSecret(ForceDeleteWithoutRecovery=true)` so the deterministic names
|
|
28
|
+
* (`fjall-<app>-clickhouse-tls-{ca,server}`) become reusable immediately on
|
|
29
|
+
* redeploy. AWS's default 7-30 day recovery window would otherwise block
|
|
30
|
+
* recreation under the same name. CFN exposes no lever for this — the
|
|
31
|
+
* AWS::SecretsManager::Secret resource has no `RecoveryWindowInDays` field.
|
|
32
|
+
*
|
|
26
33
|
* `caCertSha256` is exposed for downstream consumers to wire into their
|
|
27
34
|
* container `environment` block as `CA_CERT_SHA256` — its CFN value
|
|
28
35
|
* changes whenever the cert regenerates, forcing task-def replacement so
|
|
@@ -43,17 +50,29 @@ export class TlsCertGenerator extends Construct {
|
|
|
43
50
|
const serverSecretArnPattern = `arn:${stack.partition}:secretsmanager:${stack.region}:${stack.account}:secret:fjall-${props.appName}-clickhouse-tls-server-*`;
|
|
44
51
|
const writePolicy = new PolicyStatement({
|
|
45
52
|
effect: Effect.ALLOW,
|
|
46
|
-
actions: ["secretsmanager:PutSecretValue"],
|
|
53
|
+
actions: ["secretsmanager:PutSecretValue", "secretsmanager:DeleteSecret"],
|
|
47
54
|
resources: [caSecretArnPattern, serverSecretArnPattern]
|
|
48
55
|
});
|
|
56
|
+
// PutSecretValue against a CMK-encrypted secret needs kms:GenerateDataKey
|
|
57
|
+
// on the CMK — Secrets Manager calls KMS server-side as the caller.
|
|
58
|
+
const caCmk = this.caSecret.secretsCustomerManagedKey;
|
|
59
|
+
const serverCmk = this.serverSecret.secretsCustomerManagedKey;
|
|
60
|
+
if (caCmk === undefined || serverCmk === undefined) {
|
|
61
|
+
throw new Error("TlsCertGenerator: TlsCaSecret/TlsServerSecret must expose their CMK; importExisting=true is not supported here");
|
|
62
|
+
}
|
|
63
|
+
const kmsPolicy = new PolicyStatement({
|
|
64
|
+
effect: Effect.ALLOW,
|
|
65
|
+
actions: ["kms:GenerateDataKey", "kms:Decrypt"],
|
|
66
|
+
resources: [caCmk.key.keyArn, serverCmk.key.keyArn]
|
|
67
|
+
});
|
|
49
68
|
const cr = new CustomResource(this, "Generator", {
|
|
50
69
|
runtime: Runtime.NODEJS_22_X,
|
|
51
70
|
timeout: LAMBDA_TIMEOUT,
|
|
52
71
|
codePath: LAMBDA_ASSET_DIR,
|
|
53
72
|
handler: "index.handler",
|
|
54
73
|
lambdaDescription: `${id} ClickHouse TLS cert generator`,
|
|
55
|
-
roleDescription: `Execution role for ${id} cert generator (PutSecretValue on two cert secrets)`,
|
|
56
|
-
inlinePolicy: [writePolicy],
|
|
74
|
+
roleDescription: `Execution role for ${id} cert generator (PutSecretValue + force-DeleteSecret on two cert secrets)`,
|
|
75
|
+
inlinePolicy: [writePolicy, kmsPolicy],
|
|
57
76
|
properties: {
|
|
58
77
|
caSecretArn: this.caSecret.secret.secretArn,
|
|
59
78
|
serverSecretArn: this.serverSecret.secret.secretArn,
|
|
@@ -17,7 +17,7 @@ import { writeFileSync, readFileSync, existsSync, readdirSync, renameSync, unlin
|
|
|
17
17
|
import { join, basename } from "path";
|
|
18
18
|
import { createHash } from "crypto";
|
|
19
19
|
import { buildConstructMap, constructMapToRecord, ACCOUNT_CONSTRUCT_GROUPS } from "@fjall/util/constructMap";
|
|
20
|
-
import { maskSensitiveOutput } from "@fjall/util";
|
|
20
|
+
import { getErrorMessage, maskSensitiveOutput } from "@fjall/util";
|
|
21
21
|
import { FJALL_MANIFEST_FILENAME, MANIFEST_SCHEMA_VERSION, FjallManifestSchema } from "@fjall/util/manifest/schemas";
|
|
22
22
|
import { FjallLogger } from "./validationLogger.js";
|
|
23
23
|
/**
|
|
@@ -117,7 +117,7 @@ function computeTemplateHash(templatePath) {
|
|
|
117
117
|
return createHash("sha256").update(normalised).digest("hex");
|
|
118
118
|
}
|
|
119
119
|
catch (error) {
|
|
120
|
-
const maskedMessage = maskSensitiveOutput(
|
|
120
|
+
const maskedMessage = maskSensitiveOutput(getErrorMessage(error));
|
|
121
121
|
FjallLogger.warn(`Failed to compute hash for ${templatePath}: ${maskedMessage}`);
|
|
122
122
|
return null;
|
|
123
123
|
}
|
|
@@ -144,7 +144,7 @@ function computeStackHashes(cdkOutPath) {
|
|
|
144
144
|
}
|
|
145
145
|
}
|
|
146
146
|
catch (error) {
|
|
147
|
-
const maskedMessage = maskSensitiveOutput(
|
|
147
|
+
const maskedMessage = maskSensitiveOutput(getErrorMessage(error));
|
|
148
148
|
FjallLogger.warn(`Failed to read cdk.out for hash computation: ${maskedMessage}`);
|
|
149
149
|
}
|
|
150
150
|
return stacks;
|
|
@@ -194,16 +194,10 @@ export function writeManifest(assembly, collector) {
|
|
|
194
194
|
unlinkSync(tmpPath);
|
|
195
195
|
}
|
|
196
196
|
catch (cleanupError) {
|
|
197
|
-
|
|
198
|
-
? cleanupError.message
|
|
199
|
-
: String(cleanupError);
|
|
200
|
-
FjallLogger.warn(`Failed to clean up tmp manifest at ${tmpPath}: ${maskSensitiveOutput(cleanupMessage)}`);
|
|
197
|
+
FjallLogger.warn(`Failed to clean up tmp manifest at ${tmpPath}: ${maskSensitiveOutput(getErrorMessage(cleanupError))}`);
|
|
201
198
|
}
|
|
202
199
|
}
|
|
203
|
-
|
|
204
|
-
throw new Error(`Failed to write Fjall manifest: ${errorMessage}`, {
|
|
205
|
-
cause: error
|
|
206
|
-
});
|
|
200
|
+
throw new Error(`Failed to write Fjall manifest: ${getErrorMessage(error)}`, { cause: error });
|
|
207
201
|
}
|
|
208
202
|
}
|
|
209
203
|
/**
|
|
@@ -224,8 +218,7 @@ export function readExistingManifest(cdkOutPath) {
|
|
|
224
218
|
return result.data;
|
|
225
219
|
}
|
|
226
220
|
catch (error) {
|
|
227
|
-
|
|
228
|
-
FjallLogger.warn(`Failed to parse existing manifest: ${maskSensitiveOutput(message)}`);
|
|
221
|
+
FjallLogger.warn(`Failed to parse existing manifest: ${maskSensitiveOutput(getErrorMessage(error))}`);
|
|
229
222
|
return null;
|
|
230
223
|
}
|
|
231
224
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fjall/components-infrastructure",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "2.2.0",
|
|
4
4
|
"license": "SEE LICENSE IN LICENSE",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -59,12 +59,12 @@
|
|
|
59
59
|
"eslint": "^10.2.1",
|
|
60
60
|
"prettier": "^3.8.3",
|
|
61
61
|
"typescript": "^6.0.3",
|
|
62
|
-
"vitest": "^4.1.
|
|
62
|
+
"vitest": "^4.1.7"
|
|
63
63
|
},
|
|
64
64
|
"dependencies": {
|
|
65
65
|
"@aws-sdk/client-organizations": "^3.1038.0",
|
|
66
|
-
"@fjall/generator": "^
|
|
67
|
-
"@fjall/util": "^
|
|
66
|
+
"@fjall/generator": "^2.2.0",
|
|
67
|
+
"@fjall/util": "^2.2.0",
|
|
68
68
|
"constructs": "^10.0.0",
|
|
69
69
|
"uuid": "^14.0.0"
|
|
70
70
|
},
|
|
@@ -79,5 +79,5 @@
|
|
|
79
79
|
"engines": {
|
|
80
80
|
"node": ">=18.0.0"
|
|
81
81
|
},
|
|
82
|
-
"gitHead": "
|
|
82
|
+
"gitHead": "cd83079d6ed53b131e4c97b5966dc0d8715c8735"
|
|
83
83
|
}
|