@fntio/infra-client 0.0.0-stage → 0.5.0

package/dist/lib/network-stack.js

@@ -0,0 +1,776 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NetworkStack = void 0;
+const cdk = __importStar(require("aws-cdk-lib"));
+const ec2 = __importStar(require("aws-cdk-lib/aws-ec2"));
+const ecs = __importStar(require("aws-cdk-lib/aws-ecs"));
+const efs = __importStar(require("aws-cdk-lib/aws-efs"));
+const elbv2 = __importStar(require("aws-cdk-lib/aws-elasticloadbalancingv2"));
+const s3 = __importStar(require("aws-cdk-lib/aws-s3"));
+const acm = __importStar(require("aws-cdk-lib/aws-certificatemanager"));
+const route53 = __importStar(require("aws-cdk-lib/aws-route53"));
+const targets = __importStar(require("aws-cdk-lib/aws-route53-targets"));
+const logs = __importStar(require("aws-cdk-lib/aws-logs"));
+const iam = __importStar(require("aws-cdk-lib/aws-iam"));
+const events = __importStar(require("aws-cdk-lib/aws-events"));
+const eventsTargets = __importStar(require("aws-cdk-lib/aws-events-targets"));
+const cloudwatch = __importStar(require("aws-cdk-lib/aws-cloudwatch"));
+const cwactions = __importStar(require("aws-cdk-lib/aws-cloudwatch-actions"));
+const cr = __importStar(require("aws-cdk-lib/custom-resources"));
+const app_function_1 = require("./constructs/app-function");
+const scratch_tmpfs_1 = require("./constructs/scratch-tmpfs");
+const mirrored_image_1 = require("./constructs/mirrored-image");
+const naming_1 = require("../lambdacode/network/naming");
+const audit_log_1 = require("./audit-log");
+/** The /api/* block MUST outrank every site's forward rule or Headscale's
+ *  admin API goes public. Site priorities: stable per-id hashes in 100–989. */
+const BLOCK_ADMIN_PRIORITY = 10;
+const MAX_SITES = 90; // stays inside the ALB's 100-rules-per-listener quota
+/** Self-hosted Headscale, ONE PER SITE, sharing one cluster/EFS/ALB. Control
+ *  planes never carry user traffic; admin ops via ECS Exec — no admin port. */
+class NetworkStack extends cdk.Stack {
+    sites;
+    /** Mints DEVICE keys for any site (siteId argument — its only caller, the
+     *  vending API role, binds userName to the verified JWT). */
+    deviceKeyMinter;
+    /** Per-site CONNECTOR minters; each ignores input and is pinned to its
+     *  site's control plane by env. Safe to grant cross-account. */
+    connectorKeyMinters = {};
+    constructor(scope, id, props) {
+        super(scope, id, props);
+        const prefix = props.resourcePrefix;
+        this.sites = props.sites;
+        if (props.sites.length === 0)
+            throw new Error("NetworkStack needs ≥1 site.");
+        if (props.sites.length > MAX_SITES) {
+            throw new Error(`NetworkStack supports at most ${MAX_SITES} sites.`);
+        }
+        // Public-subnets-only: outbound internet without NAT gateways to pay for.
+        const vpc = new ec2.Vpc(this, "Vpc", {
+            maxAzs: 2,
+            natGateways: 0,
+            // Tasks request their public IP explicitly (assignPublicIp); the subnet
+            // itself hands none out (Security Hub EC2.15).
+            subnetConfiguration: [
+                { name: "public", subnetType: ec2.SubnetType.PUBLIC, mapPublicIpOnLaunch: false },
+            ],
+            // Explicit, so it never depends on the consumer's cdk.json flags (EC2.2).
+            restrictDefaultSecurityGroup: true,
+        });
+        // REJECT-only keeps the volume small; Security Hub EC2.6 asks for no more.
+        vpc.addFlowLog("RejectFlowLog", {
+            trafficType: ec2.FlowLogTrafficType.REJECT,
+            destination: ec2.FlowLogDestination.toCloudWatchLogs(new logs.LogGroup(this, "VpcFlowLogs", {
+                retention: logs.RetentionDays.THREE_MONTHS,
+            })),
+        });
+        // Headscale state is SQLite: one shared EFS, one ACCESS POINT PER SITE.
+        // Each site's directory is precious — losing it re-enrolls that site.
+        const fileSystem = new efs.FileSystem(this, "StateFs", {
+            vpc,
+            encrypted: true,
+            // Security Hub EFS.2/EFS.7 (the state is megabytes). Not for sandboxes:
+            // recovery points would outlive the destroyed stack for 35 days.
+            enableAutomaticBackups: props.prod,
+            removalPolicy: props.prod
+                ? cdk.RemovalPolicy.RETAIN
+                : cdk.RemovalPolicy.DESTROY,
+        });
+        const cluster = new ecs.Cluster(this, "Cluster", {
+            vpc,
+            // Explicit either way, so turning the knob off actually disables it.
+            containerInsightsV2: (props.containerInsights ?? props.prod)
+                ? ecs.ContainerInsights.ENABLED
+                : ecs.ContainerInsights.DISABLED,
+        });
+        // Private DNS so the key minters/reconciler reach each Headscale WITHOUT
+        // the public ALB (where /api/* is 403-blocked below).
+        const namespace = cluster.addDefaultCloudMapNamespace({
+            name: "tailnet.internal",
+        });
+        const zone = route53.HostedZone.fromHostedZoneAttributes(this, "Zone", {
+            hostedZoneId: props.hostedZoneId,
+            zoneName: props.zoneName,
+        });
+        // One wildcard covers every site — adding one changes no cert, no DNS.
+        const certificate = new acm.Certificate(this, "WildcardCert", {
+            domainName: `*.${props.subdomain}.${props.zoneName}`,
+            validation: acm.CertificateValidation.fromDns(zone),
+        });
+        // The access-log bucket policy names the Region's ELB delivery account, and
+        // a foreign-region connector VPC is placed by comparing Regions.
+        if (cdk.Token.isUnresolved(this.region)) {
+            throw new Error("NetworkStack needs an explicit env.region (ALB access logs, connector VPC placement).");
+        }
+        const alb = new elbv2.ApplicationLoadBalancer(this, "Alb", {
+            vpc,
+            internetFacing: true,
+            // Headscale clients long-poll; the default 60s idle would churn them.
+            idleTimeout: cdk.Duration.seconds(3600),
+            dropInvalidHeaderFields: true, // Security Hub ELB.4
+            deletionProtection: props.prod, // ELB.6 — disable before a prod teardown
+        });
+        // ELB.5. SSE-S3 only: the ELB log-delivery principal cannot use a KMS key.
+        const albLogs = new s3.Bucket(this, "AlbAccessLogs", {
+            blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+            enforceSSL: true,
+            encryption: s3.BucketEncryption.S3_MANAGED,
+            objectOwnership: s3.ObjectOwnership.BUCKET_OWNER_ENFORCED,
+            lifecycleRules: [
+                { expiration: cdk.Duration.days(90), abortIncompleteMultipartUploadAfter: cdk.Duration.days(7) },
+            ],
+            removalPolicy: props.prod ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY,
+            autoDeleteObjects: !props.prod,
+            // ACL-based delivery into the hub's log bucket: no policy that would
+            // reference this bucket from DataStack (a cross-stack cycle).
+            serverAccessLogsBucket: props.accessLogsBucket,
+            serverAccessLogsPrefix: props.accessLogsBucket && "s3/alb-logs/",
+        });
+        alb.logAccessLogs(albLogs, "alb");
+        // No port-80 listener at all: Security Hub ELB.18 counts even a redirect
+        // listener as plaintext, and every client is handed an https URL.
+        const listener = alb.addListener("Https", {
+            port: 443,
+            certificates: [certificate],
+            sslPolicy: elbv2.SslPolicy.TLS13_RES, // ELB.17's accepted list; TLS 1.2+ only
+            open: true,
+            // The wildcard record sends EVERY name here — an explicit 404 turns a
+            // mistyped --login-server into a message instead of a hang.
+            defaultAction: elbv2.ListenerAction.fixedResponse(404, {
+                contentType: "text/plain",
+                messageBody: "No site exists at this hostname.",
+            }),
+        });
+        // The admin API must not exist on the internet, for ANY site — admin
+        // traffic goes Lambda → Cloud Map → task. Must outrank every site rule.
+        listener.addAction("BlockAdminApi", {
+            priority: BLOCK_ADMIN_PRIORITY,
+            conditions: [elbv2.ListenerCondition.pathPatterns(["/api/*"])],
+            action: elbv2.ListenerAction.fixedResponse(403, {
+                contentType: "text/plain",
+                messageBody: "Forbidden",
+            }),
+        });
+        new route53.ARecord(this, "WildcardAlias", {
+            zone,
+            recordName: `*.${props.subdomain}`,
+            target: route53.RecordTarget.fromAlias(new targets.LoadBalancerTarget(alb)),
+        });
+        /** siteId → VPC-internal admin API URL (Cloud Map, never the ALB). */
+        const apiHostOf = (siteId) => `headscale-${siteId}.${namespace.namespaceName}`;
+        const apiUrlOf = (siteId) => `https://${apiHostOf(siteId)}:8080`;
+        /** siteId → the SSM parameter the site's task publishes its current TLS certificate to;
+         *  the Lambdas verify the admin API against it (the pair is self-signed, re-minted per task start). */
+        const tlsParamOf = (siteId) => `/${prefix}/headscale-tls-${siteId}`;
+        const tlsParamArnOf = (siteId) => this.formatArn({ service: "ssm", resource: "parameter", resourceName: `${prefix}/headscale-tls-${siteId}` });
+        const grantTlsRead = (fn, siteIds) => fn.addToRolePolicy(new iam.PolicyStatement({ actions: ["ssm:GetParameter"], resources: siteIds.map(tlsParamArnOf) }));
+        /** siteId → the site's Headscale admin API key (Secrets Manager), minted/
+         *  adopted by this stack's Custom::HeadscaleApiKey. */
+        const apiKeyOf = (siteId) => cdk.SecretValue.secretsManager(`${prefix}/headscale-apikey-${siteId}`).unsafeUnwrap();
+        const keyRotation = String(this.node.tryGetContext("headscaleKeyRotation") ?? "0");
+        const services = {};
+        const bootstrapInputs = {};
+        for (const site of props.sites) {
+            if (site.priority <= BLOCK_ADMIN_PRIORITY) {
+                throw new Error(`Site "${site.id}" priority ${site.priority} would outrank the ` +
+                    `admin-API block rule (${BLOCK_ADMIN_PRIORITY}).`);
+            }
+        }
+        // Pinned: we ship control-plane updates deliberately, never :latest. One mirrored copy serves every site.
+        // The -debug variant adds busybox (/busybox/sh): the server container runs
+        // its own bootstrap, so no sidecar needs to reach its Unix socket.
+        const HEADSCALE = "ghcr.io/juanfont/headscale:v0.26.1-debug";
+        // Every container is non-root (distroless `nonroot`) on a read-only root.
+        // No shared task volume (a Fargate task volume is root-owned whatever the
+        // image says): config and TLS pair sit on the EFS mount, whose
+        // access point acts as uid 0 for any container user; the socket on tmpfs.
+        // The CLI finds the config through the cwd, so ECS Exec needs no flag.
+        const NONROOT = "65532";
+        const STATE = "/var/lib/headscale";
+        const SCRATCH = "/tmp";
+        // Debian slim ships the openssl CLI (via ca-certificates); busybox has none.
+        const INIT_IMAGE = "public.ecr.aws/docker/library/python:3.13-slim";
+        const headscaleMirror = props.mirrorImages ? (0, mirrored_image_1.mirroredImage)(this, "HeadscaleImage", HEADSCALE) : undefined;
+        const headscaleImage = (taskDef) => {
+            if (!headscaleMirror)
+                return ecs.ContainerImage.fromRegistry(HEADSCALE);
+            taskDef.node.addDependency(headscaleMirror.copy);
+            return headscaleMirror.image;
+        };
+        props.sites.forEach((site) => {
+            const accessPoint = fileSystem.addAccessPoint(`StateAp-${site.id}`, {
+                path: `/headscale-${site.id}`,
+                createAcl: { ownerUid: "0", ownerGid: "0", permissions: "755" },
+                posixUser: { uid: "0", gid: "0" },
+            });
+            const taskDef = new ecs.FargateTaskDefinition(this, `TaskDef-${site.id}`, {
+                cpu: 256,
+                memoryLimitMiB: 512,
+            });
+            fileSystem.grantRootAccess(taskDef.taskRole);
+            taskDef.taskRole.addToPrincipalPolicy(new iam.PolicyStatement({ actions: ["ssm:PutParameter"], resources: [tlsParamArnOf(site.id)] }));
+            taskDef.addVolume({
+                name: "state",
+                efsVolumeConfiguration: {
+                    fileSystemId: fileSystem.fileSystemId,
+                    transitEncryption: "ENABLED",
+                    authorizationConfig: {
+                        accessPointId: accessPoint.accessPointId,
+                        iam: "ENABLED",
+                    },
+                },
+            });
+            // Rendered by an init container at task start — keeps us on the stock
+            // image (Headscale has no complete env-var interface).
+            const configYaml = [
+                `server_url: ${site.controlUrl}`,
+                `listen_addr: 0.0.0.0:8080`,
+                `metrics_listen_addr: 127.0.0.1:9090`,
+                `grpc_listen_addr: 127.0.0.1:50443`,
+                `noise:`,
+                `  private_key_path: /var/lib/headscale/noise_private.key`,
+                `prefixes:`,
+                `  v4: 100.64.0.0/10`,
+                `  v6: fd7a:115c:a1e0::/48`,
+                // Embedded DERP stays DISABLED: it needs UDP/3478 (STUN), which an
+                // ALB cannot serve — enabling it would force an NLB redesign.
+                `derp:`,
+                `  server:`,
+                `    enabled: false`,
+                `  urls:`,
+                `    - https://controlplane.tailscale.com/derpmap/default`,
+                `  auto_update_enabled: true`,
+                `  update_frequency: 24h`,
+                `disable_check_updates: true`,
+                `ephemeral_node_inactivity_timeout: 30m`,
+                `database:`,
+                `  type: sqlite`,
+                `  sqlite:`,
+                `    path: /var/lib/headscale/db.sqlite`,
+                `log:`,
+                `  level: info`,
+                // ACLs live in Headscale's DB and are PUT by the reconciler — never a
+                // hand-edited policy file. Deny-by-default once a policy is set.
+                `policy:`,
+                `  mode: database`,
+                `dns:`,
+                `  magic_dns: true`,
+                // Distinct per site — also what lets the desktop shell tell its saved
+                // tailscale profiles apart when switching sites.
+                `  base_domain: ${site.id}.tail.internal`,
+                `  nameservers:`,
+                `    global:`,
+                `      - 1.1.1.1`,
+                // Split DNS: PER SITE — overlapping VPCs collide on resolver IPs too.
+                ...(Object.keys(site.splitDns).length
+                    ? [
+                        `    split:`,
+                        ...Object.entries(site.splitDns).flatMap(([domain, servers]) => [
+                            `      ${domain}:`,
+                            ...servers.map((ip) => `        - ${ip}`),
+                        ]),
+                    ]
+                    : []),
+                `unix_socket: ${SCRATCH}/headscale.sock`,
+                // Self-signed per task start: the ALB encrypts the hop but never
+                // validates a target's certificate (ELB.21/.22); the Lambdas verify
+                // against the copy the task publishes to SSM.
+                `tls_cert_path: ${STATE}/tls.crt`,
+                `tls_key_path: ${STATE}/tls.key`,
+            ].join("\n");
+            const initContainer = taskDef.addContainer("write-config", {
+                image: ecs.ContainerImage.fromRegistry(INIT_IMAGE),
+                essential: false,
+                command: [
+                    "sh",
+                    "-ec",
+                    `cat > ${STATE}/config.yaml <<'EOF'\n${configYaml}\nEOF\n` +
+                        `openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes -days 3650 ` +
+                        `-subj "/CN=${apiHostOf(site.id)}" -addext "subjectAltName=DNS:${apiHostOf(site.id)},DNS:${site.fqdn}" ` +
+                        `-keyout ${STATE}/tls.key -out ${STATE}/tls.crt`,
+                ],
+                user: NONROOT,
+                readonlyRootFilesystem: true,
+                logging: ecs.LogDrivers.awsLogs({
+                    streamPrefix: `headscale-${site.id}-init`,
+                    logRetention: logs.RetentionDays.ONE_MONTH,
+                }),
+            });
+            initContainer.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false });
+            // Serve, then — once the socket answers — the fresh-site CLI steps the
+            // Custom::HeadscaleApiKey resource reads from this container's log stream
+            // ("already exists" on every later start is fine). The trap hands the
+            // task's SIGTERM to the server so SQLite closes cleanly.
+            const serveScript = [
+                "headscale serve &",
+                "pid=$!",
+                // Re-wait after forwarding: the interrupted wait returns at once and
+                // PID 1 exiting would SIGKILL the server before SQLite closes.
+                "trap 'kill -TERM $pid; wait $pid' TERM INT",
+                "n=0",
+                "until headscale users list >/dev/null 2>&1; do",
+                '  n=$((n+1)); if [ $n -ge 150 ]; then echo "headscale did not come up"; kill -TERM $pid; wait $pid; exit 1; fi',
+                "  sleep 2",
+                "done",
+                "headscale users create infra || true",
+                // Marker line the Custom::HeadscaleApiKey handler matches on.
+                'echo "APIKEY=$(headscale apikeys create --expiration 87600h -o json)"',
+                "wait $pid",
+            ].join("\n");
+            const headscale = taskDef.addContainer("headscale", {
+                image: headscaleImage(taskDef),
+                entryPoint: ["/busybox/sh", "-c", serveScript],
+                workingDirectory: STATE,
+                user: NONROOT,
+                readonlyRootFilesystem: true,
+                linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-headscale`, 64),
+                // A label (not an env var — cdk-nag ECS2) so the rotation stamp
+                // still forces a new task revision, hence a fresh key in the log.
+                dockerLabels: { rotation: keyRotation },
+                portMappings: [{ containerPort: 8080 }],
+                // The CLI answers only once the server serves (no curl in the image);
+                // generous so a slow start never trips a restart loop.
+                healthCheck: {
+                    command: ["CMD", "headscale", "users", "list"],
+                    interval: cdk.Duration.seconds(60),
+                    timeout: cdk.Duration.seconds(15),
+                    retries: 5,
+                    startPeriod: cdk.Duration.seconds(120),
+                },
+                logging: ecs.LogDrivers.awsLogs({
+                    streamPrefix: `headscale-${site.id}`,
+                    logRetention: logs.RetentionDays.ONE_MONTH,
+                }),
+            });
+            headscale.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: false });
+            // Publishes the fresh certificate before the server takes traffic, so a
+            // Lambda's next call verifies against the pair this task actually serves.
+            const publishCert = taskDef.addContainer("publish-cert", {
+                image: ecs.ContainerImage.fromRegistry("public.ecr.aws/aws-cli/aws-cli:2.15.17"),
+                essential: false,
+                // Region as an argument, not an env var (cdk-nag ECS2); the uid has no
+                // passwd entry here, which the CLI tolerates (no ~ to read or write).
+                command: [
+                    "--region", this.region,
+                    "ssm", "put-parameter",
+                    "--name", tlsParamOf(site.id),
+                    "--type", "String",
+                    "--overwrite",
+                    "--value", `file://${STATE}/tls.crt`,
+                ],
+                user: NONROOT,
+                readonlyRootFilesystem: true,
+                linuxParameters: (0, scratch_tmpfs_1.scratchTmpfs)(this, `Tmpfs-${site.id}-publish`, 16),
+                logging: ecs.LogDrivers.awsLogs({
+                    streamPrefix: `headscale-${site.id}-init`,
+                    logRetention: logs.RetentionDays.ONE_MONTH,
+                }),
+            });
+            publishCert.addMountPoints({ containerPath: STATE, sourceVolume: "state", readOnly: true });
+            publishCert.addContainerDependencies({
+                container: initContainer,
+                condition: ecs.ContainerDependencyCondition.SUCCESS,
+            });
+            headscale.addContainerDependencies({
+                container: publishCert,
+                condition: ecs.ContainerDependencyCondition.SUCCESS,
+            });
+            const service = new ecs.FargateService(this, `Service-${site.id}`, {
+                cluster,
+                taskDefinition: taskDef,
+                desiredCount: 1,
+                // SQLite on EFS = exactly one writer, ever: replace, don't overlap.
+                minHealthyPercent: 0,
+                maxHealthyPercent: 100,
+                assignPublicIp: true, // public subnets, no NAT — needed for image pull
+                vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
+                circuitBreaker: { rollback: true },
+                enableExecuteCommand: true, // admin CLI access — no admin port exposed
+                cloudMapOptions: { name: `headscale-${site.id}` },
+            });
+            fileSystem.connections.allowDefaultPortFrom(service);
+            services[site.id] = service;
+            bootstrapInputs[site.id] = {
+                logGroup: headscale.logDriverConfig.options["awslogs-group"],
+                streamPrefix: `headscale-${site.id}`,
+            };
+            listener.addTargets(`Headscale-${site.id}`, {
+                priority: site.priority,
+                conditions: [elbv2.ListenerCondition.hostHeaders([site.fqdn])],
+                port: 8080,
+                protocol: elbv2.ApplicationProtocol.HTTPS,
+                targets: [service],
+                healthCheck: { path: "/health", healthyHttpCodes: "200", protocol: elbv2.Protocol.HTTPS },
+                deregistrationDelay: cdk.Duration.seconds(30),
+            });
+            new cdk.CfnOutput(this, `ControlUrl-${site.id}`, {
+                value: site.controlUrl,
+            });
+        });
+        const minterBase = {
+            timeout: cdk.Duration.seconds(10),
+            vpc,
+            vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
+            allowPublicSubnet: true, // no NAT here; the fns need no internet, only VPC DNS
+        };
+        // CFN re-resolves the secret only when the consuming RESOURCE changes —
+        // rotate with `-c headscaleKeyRotation=$(date +%s)` or env keeps the old key.
+        /** siteId → {url, key} map the device minter and reconciler read. */
+        const sitesEnv = JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, { url: apiUrlOf(s.id), certParam: tlsParamOf(s.id), key: apiKeyOf(s.id) }])));
+        // Stores each site's server-minted admin API key (adopting a hand-made
+        // secret); everything that resolves the secret into env waits on this.
+        const bootstrapFn = new app_function_1.AppFunction(this, "HeadscaleApiKeyFn", {
+            description: "Deploy-time helper: stores each VPN site's control-server admin API key in Secrets Manager.",
+            asset: "headscale-apikey",
+            timeout: cdk.Duration.minutes(14),
+        });
+        const secretArn = (siteId) => `arn:aws:secretsmanager:${this.region}:${this.account}:secret:${prefix}/headscale-apikey-${siteId}-*`;
+        bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
+            actions: ["ecs:DescribeTasks", "ecs:DescribeServices", "ecs:ListTasks"],
+            resources: ["*"],
+        }));
+        bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
+            actions: ["logs:GetLogEvents"],
+            resources: props.sites.map((s) => `arn:aws:logs:${this.region}:${this.account}:log-group:${bootstrapInputs[s.id].logGroup}:*`),
+        }));
+        bootstrapFn.addToRolePolicy(new iam.PolicyStatement({
+            actions: [
+                "secretsmanager:GetSecretValue",
+                "secretsmanager:CreateSecret",
+                "secretsmanager:PutSecretValue",
+            ],
+            resources: props.sites.map((s) => secretArn(s.id)),
+        }));
+        const bootstrapProvider = new cr.Provider(this, "HeadscaleApiKeyProvider", {
+            onEventHandler: bootstrapFn,
+        });
+        const bootstrapBySite = {};
+        for (const site of props.sites) {
+            const { logGroup, streamPrefix } = bootstrapInputs[site.id];
+            const service = services[site.id];
+            const resource = new cdk.CustomResource(this, `HeadscaleBootstrap-${site.id}`, {
+                serviceToken: bootstrapProvider.serviceToken,
+                resourceType: "Custom::HeadscaleApiKey",
+                properties: {
+                    SiteId: site.id,
+                    SecretName: `${prefix}/headscale-apikey-${site.id}`,
+                    ClusterArn: cluster.clusterArn,
+                    ServiceName: service.serviceName,
+                    LogGroupName: logGroup,
+                    StreamPrefix: streamPrefix,
+                    Rotation: keyRotation,
+                },
+            });
+            // Resource-level edge on purpose: a construct-level one pulls the
+            // service SG's ingress (referencing the minters' SGs) into a cycle.
+            resource.node.addDependency(service.node.defaultChild);
+            bootstrapBySite[site.id] = resource;
+            new cdk.CfnOutput(this, `HeadscaleApiKey-${site.id}`, {
+                value: resource.getAttString("Source"),
+            });
+        }
+        const bootstraps = Object.values(bootstrapBySite);
+        /** Make only a function's own CloudFormation resource wait for the key. */
+        const waitForKey = (fn, ...on) => {
+            const cfn = fn.node.defaultChild;
+            for (const r of on)
+                cfn.addResourceDependency(r.node.defaultChild);
+        };
+        // ONE device minter for all sites: its only caller (the vending API role)
+        // binds userName to the verified JWT, and users enroll wherever granted.
+        this.deviceKeyMinter = new app_function_1.AppFunction(this, "DeviceKeyMinter", {
+            ...minterBase,
+            functionName: (0, naming_1.deviceKeyMinterFnName)(prefix),
+            asset: "keyminter",
+            description: "Mints a one-time VPN enrollment key for a signed-in user's device.",
+            handler: "deviceHandler",
+            environment: { HEADSCALE_SITES: sitesEnv, HEADSCALE_KEY_ROTATION: keyRotation },
+        });
+        waitForKey(this.deviceKeyMinter, ...bootstraps);
+        grantTlsRead(this.deviceKeyMinter, props.sites.map((s) => s.id));
+        // One connector minter PER SITE, env-pinned, input ignored — which site a
+        // connector joins = which minter its account may invoke.
+        for (const site of props.sites) {
+            const minter = new app_function_1.AppFunction(this, `ConnectorKeyMinter-${site.id}`, {
+                ...minterBase,
+                functionName: (0, naming_1.connectorKeyMinterFnName)(prefix, site.id),
+                asset: "keyminter",
+                description: `Mints VPN enrollment keys for connectors joining site ${site.id}.`,
+                handler: "connectorHandler",
+                environment: {
+                    HEADSCALE_API_URL: apiUrlOf(site.id),
+                    HEADSCALE_API_KEY: apiKeyOf(site.id),
+                    HEADSCALE_TLS_PARAM: tlsParamOf(site.id),
+                    HEADSCALE_USER_ID: "1", // the `infra` machine-owner user
+                    HEADSCALE_KEY_ROTATION: keyRotation,
+                },
+            });
+            waitForKey(minter, bootstrapBySite[site.id]);
+            grantTlsRead(minter, [site.id]);
+            this.connectorKeyMinters[site.id] = minter;
+            services[site.id].connections.allowFrom(minter, ec2.Port.tcp(8080));
+            services[site.id].connections.allowFrom(this.deviceKeyMinter, ec2.Port.tcp(8080));
+            new cdk.CfnOutput(this, `ConnectorKeyMinterArn-${site.id}`, {
+                value: minter.functionArn,
+            });
+        }
+        const reconcilerRole = new iam.Role(this, "AclReconcilerRole", {
+            roleName: (0, naming_1.aclReconcilerRoleName)(prefix),
+            assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
+            managedPolicies: [
+                iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaVPCAccessExecutionRole"),
+            ],
+        });
+        // Every account's connector VPCs by name and (foreign) region — the reconciler's fan-out targets, and
+        // the (account, region) pairs the out-of-VPC relay may address (why a relay: wiki Private Network,
+        // "Connector VPCs in another region").
+        const connectorVpcs = {};
+        // `<account>#<name or empty>` → the routes config declares for that VPC: all the reconciler will approve for it.
+        const connectorRoutes = {};
+        for (const s of props.sites) {
+            for (const [account, list] of Object.entries(s.connectorVpcs)) {
+                for (const vpc of list)
+                    connectorRoutes[`${account}#${vpc.name ?? ""}`] = vpc.routes;
+                connectorVpcs[account] = list.map((vpc) => {
+                    // JSON.stringify drops the undefineds: an unnamed hub-region VPC travels as `{}`.
+                    return {
+                        name: vpc.name,
+                        region: vpc.region && vpc.region !== this.region ? vpc.region : undefined,
+                    };
+                });
+            }
+        }
+        const relayRegions = [
+            ...new Set(Object.values(connectorVpcs)
+                .flat()
+                .map((c) => c.region)
+                .filter((r) => Boolean(r))),
+        ].sort();
+        const reconciler = new app_function_1.AppFunction(this, "AclReconciler", {
+            functionName: (0, naming_1.aclReconcilerFnName)(prefix),
+            description: "Keeps each VPN site's access rules, routes and security-group rules in line with the portal's grants.",
+            asset: "acl-reconciler",
+            handler: "handler",
+            role: reconcilerRole,
+            // Per-site budget: the pass runs the SG-manager AND tailnet-resolver
+            // fan-outs (each per-account invoke aborts at 10s) plus up to ~6s of
+            // sequential local DNS fallbacks per unresolvable dnsName resource.
+            timeout: cdk.Duration.seconds(Math.min(90 * props.sites.length, 900)),
+            // One pass at a time: the API kicks a pass on every dev-share create/stop, and two passes racing on
+            // the whole-policy PUT could re-publish a rule the later one just removed.
+            reservedConcurrentExecutions: 1,
+            vpc,
+            vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC },
+            allowPublicSubnet: true, // like the key minters: needs VPC DNS, not internet
+            environment: {
+                RESOURCE_PREFIX: prefix, // by-name SG-manager target derivation
+                HEADSCALE_SITES: sitesEnv,
+                HEADSCALE_KEY_ROTATION: keyRotation,
+                // Only this account's names may be resolved from inside this VPC (Cloud Map, plain DNS):
+                // another account's private zone can shadow a label of the hub's.
+                HUB_ACCOUNT: this.account,
+                // accountNumber → siteId: THE site-membership source of truth.
+                SITE_ACCOUNTS: JSON.stringify(Object.fromEntries(props.sites.flatMap((s) => s.accounts.map((a) => [a, s.id])))),
+                // accountNumber → its connector VPCs ({ name?, region? }; region only outside this region).
+                CONNECTOR_VPCS: JSON.stringify(connectorVpcs),
+                CONNECTOR_ROUTES: JSON.stringify(connectorRoutes),
+                // siteId → displayName: how the activity log names a site.
+                SITE_NAMES: JSON.stringify(Object.fromEntries(props.sites.map((s) => [s.id, s.displayName]))),
+                RESOURCES_TABLE_NAME: props.resourcesTable.tableName,
+                ACCESS_TABLE_NAME: props.accessTable.tableName,
+                GROUPS_TABLE_NAME: props.groupsTable.tableName,
+                // Resolvers stay reachable on :53 for everyone ON THAT SITE, or
+                // name-based access dies before any granted port is tried.
+                SPLIT_DNS_BY_SITE: JSON.stringify(Object.fromEntries(props.sites.map((s) => [
+                    s.id,
+                    [...new Set(Object.values(s.splitDns).flat())],
+                ]))),
+                ...(props.auditLog ? (0, audit_log_1.auditLogEnv)(props.auditLog) : {}),
+            },
+        });
+        if (props.auditLog)
+            (0, audit_log_1.grantAuditLogWrite)(reconciler, props.auditLog);
+        waitForKey(reconciler, ...bootstraps);
+        grantTlsRead(reconciler, props.sites.map((s) => s.id));
+        for (const site of props.sites) {
+            services[site.id].connections.allowFrom(reconciler, ec2.Port.tcp(8080));
+        }
+        // ReadWrite: the reconciler writes per-site status sentinels into Resources.
+        props.resourcesTable.grantReadWriteData(reconciler);
+        props.accessTable.grantReadData(reconciler);
+        props.groupsTable.grantReadData(reconciler);
+        // Cloud Map re-resolution of resource dnsNames (Fargate IP drift):
+        // DiscoverInstances has no resource-level scoping.
+        reconciler.addToRolePolicy(new iam.PolicyStatement({
+            actions: ["servicediscovery:DiscoverInstances"],
+            resources: ["*"],
+        }));
+        // SG-manager and tailnet-resolver invokes by well-known name in ANY account —
+        // reaching one still requires that account's resource policy (pinned to this role).
+        reconciler.addToRolePolicy(new iam.PolicyStatement({
+            actions: ["lambda:InvokeFunction"],
+            resources: [
+                `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`,
+                // Named connector VPCs suffix the manager; per-site (and per-name)
+                // resolvers — the wildcards cover every one.
+                `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`,
+                `arn:aws:lambda:${this.region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`,
+            ],
+        }));
+        // The relay's invoke-only role. Connector accounts in another region pin their trust to it
+        // by name, so it exists even without a relay (Lambda validates the principal at permission time).
+        const relayRole = new iam.Role(this, "AclReconcilerRelayRole", {
+            roleName: (0, naming_1.aclReconcilerRelayRoleName)(prefix),
+            assumedBy: new iam.ServicePrincipal("lambda.amazonaws.com"),
+            managedPolicies: [
+                iam.ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"),
+            ],
+        });
+        if (relayRegions.length) {
+            relayRole.addToPolicy(new iam.PolicyStatement({
+                actions: ["lambda:InvokeFunction"],
+                resources: relayRegions.flatMap((region) => [
+                    `arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix)}`,
+                    `arn:aws:lambda:${region}:*:function:${(0, naming_1.sgManagerFnName)(prefix, "*")}`,
+                    `arn:aws:lambda:${region}:*:function:${(0, naming_1.tailnetResolverFnName)(prefix, "*")}`,
+                ]),
+            }));
+            const relay = new app_function_1.AppFunction(this, "InvokeRelay", {
+                functionName: (0, naming_1.aclReconcilerRelayFnName)(prefix),
+                description: "Forwards the VPN reconciler's calls to connector functions in other regions.",
+                asset: "invoke-relay",
+                handler: "handler",
+                role: relayRole,
+                // The reconciler's per-target budget (INVOKE_TIMEOUT_MS) plus this function's own start.
+                timeout: cdk.Duration.seconds(12),
+                environment: {
+                    RESOURCE_PREFIX: prefix,
+                    CONNECTOR_VPCS: JSON.stringify(connectorVpcs),
+                },
+            });
+            reconciler.addToRolePolicy(new iam.PolicyStatement({
+                actions: ["lambda:InvokeFunction"],
+                resources: [relay.functionArn],
+            }));
+        }
+        // The Lambdas have no internet path — DiscoverInstances needs Cloud Map's
+        // data endpoint. One subnet = one ENI (~$8/mo) serving the whole VPC.
+        vpc.addInterfaceEndpoint("CloudMapDataEndpoint", {
+            service: new ec2.InterfaceVpcEndpointAwsService("data-servicediscovery"),
+            subnets: { subnets: [vpc.publicSubnets[0]] },
+        });
+        // Lambda invokes (the SG-managers) need Lambda's API from this
+        // no-internet VPC — same one-ENI pattern as the Cloud Map endpoint.
+        vpc.addInterfaceEndpoint("LambdaEndpoint", {
+            service: ec2.InterfaceVpcEndpointAwsService.LAMBDA,
+            subnets: { subnets: [vpc.publicSubnets[0]] },
+        });
+        // The minters and the reconciler read each site's TLS certificate from
+        // SSM before calling Headscale; without this endpoint that read hangs.
+        vpc.addInterfaceEndpoint("SsmEndpoint", {
+            service: ec2.InterfaceVpcEndpointAwsService.SSM,
+            subnets: { subnets: [vpc.publicSubnets[0]] },
+        });
+        // DynamoDB from inside the VPC without internet: free gateway endpoint.
+        vpc.addGatewayEndpoint("DynamoDbEndpoint", {
+            service: ec2.GatewayVpcEndpointAwsService.DYNAMODB,
+        });
+        new events.Rule(this, "ReconcileSchedule", {
+            schedule: events.Schedule.rate(cdk.Duration.minutes(5)),
+            targets: [new eventsTargets.LambdaFunction(reconciler)],
+        });
+        // A broken reconciler freezes reachability INCLUDING revocations; one
+        // failure self-heals next pass, so alarm on two consecutive.
+        if (props.alertTopic) {
+            const notify = new cwactions.SnsAction(props.alertTopic);
+            const alarm = new cloudwatch.Alarm(this, "ReconcilerErrors", {
+                metric: reconciler.metricErrors({ period: cdk.Duration.minutes(15) }),
+                threshold: 2,
+                evaluationPeriods: 1,
+                comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+                treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+                alarmDescription: "The ACL reconciler has failed 2+ consecutive passes — tailnet " +
+                    "reachability is frozen at the last synced state, so grant changes " +
+                    "AND revocations are not taking effect. Check its CloudWatch logs " +
+                    "and the Headscale services.",
+            });
+            alarm.addAlarmAction(notify);
+            alarm.addOkAction(notify);
+            // Zombie: "online" but silent — forwards on STALE rules and never
+            // learns new device keys. EMF metric from the reconciler, summed.
+            const staleAlarm = new cloudwatch.Alarm(this, "StaleConnectors", {
+                metric: new cloudwatch.Metric({
+                    namespace: "InfraClient/Network",
+                    metricName: "StaleConnectors",
+                    statistic: "Maximum",
+                    period: cdk.Duration.minutes(15),
+                }),
+                threshold: 1,
+                evaluationPeriods: 1,
+                comparisonOperator: cloudwatch.ComparisonOperator.GREATER_THAN_OR_EQUAL_TO_THRESHOLD,
+                treatMissingData: cloudwatch.TreatMissingData.NOT_BREACHING,
+                alarmDescription: "A connector node with approved routes has been OFFLINE 15+ " +
+                    "minutes (zombie): it enforces stale rules and won't learn " +
+                    "re-enrolled devices. Restart it: aws ecs update-service " +
+                    "--force-new-deployment on the connector service; the reconciler " +
+                    "auto-approves the replacement's routes.",
+            });
+            staleAlarm.addAlarmAction(notify);
+            staleAlarm.addOkAction(notify);
+        }
+        new cdk.CfnOutput(this, "ClusterName", { value: cluster.clusterName });
+        new cdk.CfnOutput(this, "DeviceKeyMinterArn", {
+            value: this.deviceKeyMinter.functionArn,
+        });
+    }
+    /** Grant invoke on ONE SITE's connector minter (machine keys only — never
+     *  the device minter). Prefer a task-role ARN over a whole-account id. */
+    grantConnectorAccount(siteId, connectorPrincipal) {
+        const minter = this.connectorKeyMinters[siteId];
+        if (!minter)
+            throw new Error(`No such site: ${siteId}`);
+        const principal = connectorPrincipal.includes(":")
+            ? new iam.ArnPrincipal(connectorPrincipal) // specific role ARN (preferred)
+            : new iam.AccountPrincipal(connectorPrincipal); // whole account (coarser)
+        // Construct ids must be alphanumeric; keep them stable + unique per grant.
+        const suffix = connectorPrincipal.replace(/[^a-zA-Z0-9]/g, "").slice(-24);
+        minter.addPermission(`ConnectorInvoke${suffix}`, {
+            principal,
+            action: "lambda:InvokeFunction",
+        });
+    }
+}
+exports.NetworkStack = NetworkStack;