@go-to-k/cdkd 0.284.41 → 0.284.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{asg-provider-D0lenV1M.js → asg-provider-DIKQ3APM.js} +2 -2
- package/dist/{asg-provider-D0lenV1M.js.map → asg-provider-DIKQ3APM.js.map} +1 -1
- package/dist/cli.js +33 -11
- package/dist/cli.js.map +1 -1
- package/dist/{deploy-engine-CCWl_tAH.js → deploy-engine-bFTrzeeu.js} +595 -41
- package/dist/deploy-engine-bFTrzeeu.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/deploy-engine-CCWl_tAH.js.map +0 -1
|
@@ -5102,6 +5102,184 @@ function formatDockerLoginError(stderr, endpoint) {
|
|
|
5102
5102
|
if (trimmed.includes("already exists in the keychain") || trimmed.includes("Error saving credentials")) return `docker's credential helper (osxkeychain on macOS / wincred on Windows / pass / secretservice on Linux) failed to persist the ECR auth token. The "already exists in the keychain" / "Error saving credentials" output is a known docker-credential-helpers issue — unrelated to cdkd, AWS credentials, or IAM perms. Quick fix: run \`docker logout ${endpoint}\` to clear the stale entry, then retry the cdkd command. Permanent fix: edit ~/.docker/config.json and remove (or empty) the platform-specific "credsStore" entry (e.g. "osxkeychain" → "" or "desktop" on macOS Docker Desktop). Original docker stderr: ${trimmed}`;
|
|
5103
5103
|
return trimmed;
|
|
5104
5104
|
}
|
|
5105
|
+
/**
|
|
5106
|
+
* Env vars the docker CLI itself reads to decide how / where to run. A resolved
|
|
5107
|
+
* ECS secret (or SecureString) whose NAME collides with one of these must NOT
|
|
5108
|
+
* override it in the docker client's own process environment: a secret named
|
|
5109
|
+
* `DOCKER_HOST` would redirect the client to a different daemon, and `PATH`
|
|
5110
|
+
* would break locating the docker binary. See issue
|
|
5111
|
+
* https://github.com/go-to-k/cdkd/issues/2183.
|
|
5112
|
+
*/
|
|
5113
|
+
const DOCKER_CLIENT_ENV_KEYS = /* @__PURE__ */ new Set([
|
|
5114
|
+
"PATH",
|
|
5115
|
+
"PATHEXT",
|
|
5116
|
+
"HOME",
|
|
5117
|
+
"USERPROFILE",
|
|
5118
|
+
"HOMEDRIVE",
|
|
5119
|
+
"HOMEPATH",
|
|
5120
|
+
"DOCKER_HOST",
|
|
5121
|
+
"DOCKER_CONTEXT",
|
|
5122
|
+
"DOCKER_CONFIG",
|
|
5123
|
+
"DOCKER_CERT_PATH",
|
|
5124
|
+
"DOCKER_TLS",
|
|
5125
|
+
"DOCKER_TLS_VERIFY",
|
|
5126
|
+
"DOCKER_API_VERSION",
|
|
5127
|
+
"DOCKER_AUTH_CONFIG",
|
|
5128
|
+
"DOCKER_DEFAULT_PLATFORM",
|
|
5129
|
+
"DOCKER_CUSTOM_HEADERS",
|
|
5130
|
+
"DOCKER_CONTENT_TRUST",
|
|
5131
|
+
"DOCKER_CONTENT_TRUST_SERVER",
|
|
5132
|
+
"DOCKER_HIDE_LEGACY_COMMANDS",
|
|
5133
|
+
"BUILDKIT_PROGRESS",
|
|
5134
|
+
"GLIBC_TUNABLES",
|
|
5135
|
+
"GCONV_PATH",
|
|
5136
|
+
"BASH_ENV",
|
|
5137
|
+
"SSH_AUTH_SOCK",
|
|
5138
|
+
"SSH_ASKPASS",
|
|
5139
|
+
"SSH_ASKPASS_REQUIRE",
|
|
5140
|
+
"SSH_SK_HELPER",
|
|
5141
|
+
"SSH_SK_PROVIDER",
|
|
5142
|
+
"SSH_PKCS11_HELPER",
|
|
5143
|
+
"SSH_AGENT_PID",
|
|
5144
|
+
"AWS_ENDPOINT_URL",
|
|
5145
|
+
"AWS_CA_BUNDLE",
|
|
5146
|
+
"AWS_PROFILE",
|
|
5147
|
+
"AWS_CONFIG_FILE",
|
|
5148
|
+
"AWS_SHARED_CREDENTIALS_FILE",
|
|
5149
|
+
"AWS_WEB_IDENTITY_TOKEN_FILE",
|
|
5150
|
+
"AWS_CONTAINER_CREDENTIALS_FULL_URI",
|
|
5151
|
+
"AWS_ROLE_ARN",
|
|
5152
|
+
"AWS_EC2_METADATA_SERVICE_ENDPOINT",
|
|
5153
|
+
"SSL_CERT_FILE",
|
|
5154
|
+
"SSL_CERT_DIR",
|
|
5155
|
+
"GODEBUG",
|
|
5156
|
+
"HTTP_PROXY",
|
|
5157
|
+
"HTTPS_PROXY",
|
|
5158
|
+
"NO_PROXY",
|
|
5159
|
+
"FTP_PROXY",
|
|
5160
|
+
"ALL_PROXY"
|
|
5161
|
+
]);
|
|
5162
|
+
const DOCKER_CLIENT_ENV_KEYS_UPPER = new Set([...DOCKER_CLIENT_ENV_KEYS].map((k) => k.toUpperCase()));
|
|
5163
|
+
/**
|
|
5164
|
+
* Env-var prefixes whose WHOLE family the docker client (or a helper it execs)
|
|
5165
|
+
* reads, so a NAMED list is always one release behind and a colliding secret in
|
|
5166
|
+
* ANY member is a hazard. `LD_*` / `DYLD_*` are the dynamic loader (code
|
|
5167
|
+
* injection, glibc + macOS); `AWS_ENDPOINT_URL_*` is the per-service endpoint
|
|
5168
|
+
* family aws-sdk-go-v2 (and so `docker-credential-ecr-login`) honours — a
|
|
5169
|
+
* secret named `AWS_ENDPOINT_URL_ECR` walks around the exact
|
|
5170
|
+
* `AWS_ENDPOINT_URL` entry and redirects a request signed with the operator's
|
|
5171
|
+
* real credentials (#2186 round 4). No plausible secret name collides with
|
|
5172
|
+
* any of the three. Matched by prefix rather than enumerated (issue #2183
|
|
5173
|
+
* review). `SSH_` was a prefix here and was demoted to an EXACT enumeration
|
|
5174
|
+
* in {@link DOCKER_CLIENT_ENV_KEYS} (#2186 review round 3): the family is not
|
|
5175
|
+
* uniformly dangerous and is not growing, while the prefix broke realistic,
|
|
5176
|
+
* currently-working secrets (`SSH_PRIVATE_KEY`, GitLab CI's canonical
|
|
5177
|
+
* deploy-key spelling). Exported so the test fence can assert the EXACT
|
|
5178
|
+
* contents — a hardcoded copy in the test made the anti-shadowing fence
|
|
5179
|
+
* one-directional (#2186 round 4 finding 2).
|
|
5180
|
+
*/
|
|
5181
|
+
const DOCKER_CLIENT_ENV_PREFIXES = [
|
|
5182
|
+
"LD_",
|
|
5183
|
+
"DYLD_",
|
|
5184
|
+
"AWS_ENDPOINT_URL_"
|
|
5185
|
+
];
|
|
5186
|
+
/**
|
|
5187
|
+
* Is `key` the name of a var the docker client reads? Case-INSENSITIVE, because
|
|
5188
|
+
* Windows environment lookups are, so a lowercase `docker_host` must be caught
|
|
5189
|
+
* too (issue #2183). Matches the exact denylist OR a prefixed family — the
|
|
5190
|
+
* prefix families are fail-closed on the whole prefix, so an unlisted `LD_*` /
|
|
5191
|
+
* `DYLD_*` / `AWS_ENDPOINT_URL_*` secret is dropped (with a rename warning)
|
|
5192
|
+
* rather than reaching the client.
|
|
5193
|
+
*/
|
|
5194
|
+
function isDockerClientEnvKey(key) {
|
|
5195
|
+
const upper = key.toUpperCase();
|
|
5196
|
+
if (DOCKER_CLIENT_ENV_KEYS_UPPER.has(upper)) return true;
|
|
5197
|
+
return DOCKER_CLIENT_ENV_PREFIXES.some((prefix) => upper.startsWith(prefix));
|
|
5198
|
+
}
|
|
5199
|
+
/**
|
|
5200
|
+
* Is `key` a shape that cannot be a well-formed `docker run -e` variable NAME?
|
|
5201
|
+
* Defined POSITIVELY as the good shape's complement (#2186 round 5): a valid
|
|
5202
|
+
* name is non-empty and contains neither `=` nor NUL. Enumerating the bad
|
|
5203
|
+
* spellings one at a time closed `=` in round 4 and left the empty key (`-e ''`
|
|
5204
|
+
* — docker rejects it with an opaque error naming no secret) and a NUL-bearing
|
|
5205
|
+
* key still open. A sensitive key matching this takes the same fail-closed
|
|
5206
|
+
* collision path as a docker-client-var name: no `-e` flag, no spawn-env entry,
|
|
5207
|
+
* reported in `collisions`. (This is the NAME only; a secret VALUE containing a
|
|
5208
|
+
* NUL is a separate pre-existing leak tracked in issue #2189.)
|
|
5209
|
+
*/
|
|
5210
|
+
function isMalformedEnvKey(key) {
|
|
5211
|
+
return key.length === 0 || key.includes("=") || key.includes("\0");
|
|
5212
|
+
}
|
|
5213
|
+
/**
|
|
5214
|
+
* Split a container's environment into `docker run` `-e` flags and the values
|
|
5215
|
+
* that must travel through the spawn env instead of the argv.
|
|
5216
|
+
*
|
|
5217
|
+
* - a NON-sensitive key becomes `-e KEY=value` on the argv (unchanged);
|
|
5218
|
+
* - a sensitive key becomes a value-less `-e KEY`, its value returned in
|
|
5219
|
+
* `sensitiveEnv` for {@link dockerSpawnEnvWithSensitive};
|
|
5220
|
+
* - a sensitive key that NAMES a docker-client var ({@link isDockerClientEnvKey}
|
|
5221
|
+
* — the exact denylist plus the prefix families, so an external caller must
|
|
5222
|
+
* use the predicate, not `DOCKER_CLIENT_ENV_KEYS.has`) gets NO flag at all,
|
|
5223
|
+
* its value is dropped, and the key is reported in `collisions` so the
|
|
5224
|
+
* caller can warn;
|
|
5225
|
+
* - a sensitive key of a MALFORMED shape ({@link isMalformedEnvKey} — empty,
|
|
5226
|
+
* or containing `=` / NUL) takes the same fail-closed path (#2186 rounds
|
|
5227
|
+
* 4-5). The denylist matches the WHOLE key string, but Node serialises env
|
|
5228
|
+
* as `key=value` and the OS parses the variable NAME as everything before
|
|
5229
|
+
* the FIRST `=` — so a secret named `PATH=/tmp/evil:` is not a denylist
|
|
5230
|
+
* match while the environ it produces (`PATH=/tmp/evil:=<secret>`) POISONS
|
|
5231
|
+
* the client's `PATH`, and the poisoned duplicate wins (measured). The
|
|
5232
|
+
* docker CLI execs `docker-credential-*` helpers off `PATH`, so that is code
|
|
5233
|
+
* execution as the operator. Defining the GOOD shape positively also catches
|
|
5234
|
+
* the empty key (`-e ''`, an opaque docker rejection) in one predicate.
|
|
5235
|
+
*
|
|
5236
|
+
* The collision case is why the argv half lives here beside the env half.
|
|
5237
|
+
* Emitting `-e KEY` for a key that `dockerSpawnEnvWithSensitive` refuses to
|
|
5238
|
+
* set makes docker resolve the flag against the CLIENT's own environment, so
|
|
5239
|
+
* the container silently receives the HOST's value for that var (issue #2183)
|
|
5240
|
+
* -- e.g. the host's `HTTPS_PROXY` credential, or a macOS `PATH` inside a
|
|
5241
|
+
* Linux image. Dropping the flag is what makes "not forwarded" literally true.
|
|
5242
|
+
*/
|
|
5243
|
+
function partitionSensitiveEnv(env, sensitiveKeys) {
|
|
5244
|
+
const flags = [];
|
|
5245
|
+
const sensitiveEnv = {};
|
|
5246
|
+
const collisions = [];
|
|
5247
|
+
for (const [k, v] of Object.entries(env)) {
|
|
5248
|
+
if (!sensitiveKeys.has(k)) {
|
|
5249
|
+
flags.push("-e", `${k}=${v}`);
|
|
5250
|
+
continue;
|
|
5251
|
+
}
|
|
5252
|
+
if (isMalformedEnvKey(k) || isDockerClientEnvKey(k)) {
|
|
5253
|
+
collisions.push(k);
|
|
5254
|
+
continue;
|
|
5255
|
+
}
|
|
5256
|
+
flags.push("-e", k);
|
|
5257
|
+
sensitiveEnv[k] = v;
|
|
5258
|
+
}
|
|
5259
|
+
return {
|
|
5260
|
+
flags,
|
|
5261
|
+
sensitiveEnv,
|
|
5262
|
+
collisions
|
|
5263
|
+
};
|
|
5264
|
+
}
|
|
5265
|
+
/**
|
|
5266
|
+
* Build the environment for a `docker run` that forwards value-less `-e KEY`
|
|
5267
|
+
* flags (the pattern that keeps secret VALUES off the argv / `/proc/<pid>/cmdline`).
|
|
5268
|
+
* The child gets the full parent env plus the sensitive passthrough, but the
|
|
5269
|
+
* docker client's own critical vars ({@link isDockerClientEnvKey}) are kept
|
|
5270
|
+
* authoritative from `process.env`, so a user-controlled secret NAME cannot
|
|
5271
|
+
* hijack the client (issue #2183). Callers should partition through
|
|
5272
|
+
* {@link partitionSensitiveEnv}, which never puts a colliding key in
|
|
5273
|
+
* `sensitiveEnv`; the guard here is defence in depth.
|
|
5274
|
+
*/
|
|
5275
|
+
function dockerSpawnEnvWithSensitive(sensitiveEnv) {
|
|
5276
|
+
const env = { ...process.env };
|
|
5277
|
+
for (const [k, v] of Object.entries(sensitiveEnv)) {
|
|
5278
|
+
if (isMalformedEnvKey(k) || isDockerClientEnvKey(k)) continue;
|
|
5279
|
+
env[k] = v;
|
|
5280
|
+
}
|
|
5281
|
+
return env;
|
|
5282
|
+
}
|
|
5105
5283
|
function mergeEnv(overrides) {
|
|
5106
5284
|
const merged = { ...process.env };
|
|
5107
5285
|
for (const [k, v] of Object.entries(overrides)) if (v === void 0) delete merged[k];
|
|
@@ -7594,6 +7772,26 @@ function displaySafe(value, opts) {
|
|
|
7594
7772
|
//#endregion
|
|
7595
7773
|
//#region src/state/lock-manager.ts
|
|
7596
7774
|
/**
|
|
7775
|
+
* Upper bound on the gap between lock renewals.
|
|
7776
|
+
*
|
|
7777
|
+
* Deliberately a fixed wall-clock ceiling rather than a pure fraction of the
|
|
7778
|
+
* TTL. A fraction alone (TTL/4 = 7.5 min at the default) makes renewal
|
|
7779
|
+
* unobservable in any deploy shorter than that, so nothing short of a
|
|
7780
|
+
* multi-hour fixture could ever prove the loop runs -- and an untestable
|
|
7781
|
+
* heartbeat is one that goes dead silently. At 2 min the default 30 min TTL
|
|
7782
|
+
* tolerates FOURTEEN consecutive missed renewals before it lapses, and an
|
|
7783
|
+
* ordinary broad-set integ (~8 min) performs three.
|
|
7784
|
+
*/
|
|
7785
|
+
const MAX_RENEWAL_INTERVAL_MS = 120 * 1e3;
|
|
7786
|
+
/**
|
|
7787
|
+
* Renew after at most a quarter of the TTL, so a short TTL still gets three
|
|
7788
|
+
* chances to renew before it lapses. This is what binds the two numbers
|
|
7789
|
+
* together for a caller that shortens `ttlMinutes`.
|
|
7790
|
+
*/
|
|
7791
|
+
const RENEWAL_TTL_FRACTION = 4;
|
|
7792
|
+
/** Floor, so a pathologically small TTL cannot spin the event loop. */
|
|
7793
|
+
const MIN_RENEWAL_INTERVAL_MS = 1e3;
|
|
7794
|
+
/**
|
|
7597
7795
|
* S3-based lock manager using conditional writes (If-None-Match)
|
|
7598
7796
|
*
|
|
7599
7797
|
* Implements distributed locking using S3's If-None-Match: "*" condition
|
|
@@ -7615,13 +7813,20 @@ var LockManager = class {
|
|
|
7615
7813
|
s3Client;
|
|
7616
7814
|
config;
|
|
7617
7815
|
ttlMs;
|
|
7816
|
+
renewalIntervalMs;
|
|
7817
|
+
renewalDisabled;
|
|
7818
|
+
/** Locks held by THIS process, keyed by S3 lock key. */
|
|
7819
|
+
heldLocks = /* @__PURE__ */ new Map();
|
|
7618
7820
|
clientResolved = false;
|
|
7619
7821
|
resolveInFlight = null;
|
|
7620
7822
|
constructor(s3Client, config, options) {
|
|
7621
7823
|
this.s3Client = s3Client;
|
|
7622
7824
|
this.config = config;
|
|
7623
7825
|
const ttlMinutes = options?.ttlMinutes ?? 30;
|
|
7826
|
+
if (!(ttlMinutes > 0) || !Number.isFinite(ttlMinutes)) throw new LockError(`Lock TTL must be a positive number of minutes (got ${String(ttlMinutes)}). A non-positive TTL makes every lock born expired, and the renewal interval clamps to its floor, so the heartbeat would spin one request per second for the life of the process.`);
|
|
7624
7827
|
this.ttlMs = ttlMinutes * 60 * 1e3;
|
|
7828
|
+
this.renewalDisabled = options?.disableRenewal === true;
|
|
7829
|
+
this.renewalIntervalMs = Math.max(MIN_RENEWAL_INTERVAL_MS, Math.min(MAX_RENEWAL_INTERVAL_MS, Math.floor(this.ttlMs / RENEWAL_TTL_FRACTION)));
|
|
7625
7830
|
}
|
|
7626
7831
|
/**
|
|
7627
7832
|
* Resolve the state bucket's actual region and, if it differs from the
|
|
@@ -7702,6 +7907,7 @@ var LockManager = class {
|
|
|
7702
7907
|
* Check if a lock is expired based on its expiresAt field
|
|
7703
7908
|
*/
|
|
7704
7909
|
isLockExpired(lockInfo) {
|
|
7910
|
+
if (!Number.isFinite(lockInfo.expiresAt)) return true;
|
|
7705
7911
|
return Date.now() >= lockInfo.expiresAt;
|
|
7706
7912
|
}
|
|
7707
7913
|
/**
|
|
@@ -7736,40 +7942,50 @@ var LockManager = class {
|
|
|
7736
7942
|
};
|
|
7737
7943
|
try {
|
|
7738
7944
|
this.logger.debug(`Attempting to acquire lock for stack: ${stackName} (${region})`);
|
|
7739
|
-
const
|
|
7740
|
-
await this.s3Client.send(new PutObjectCommand({
|
|
7741
|
-
Bucket: this.config.bucket,
|
|
7742
|
-
...await this.ownerParam(),
|
|
7743
|
-
Key: key,
|
|
7744
|
-
Body: lockBody,
|
|
7745
|
-
ContentLength: Buffer.byteLength(lockBody),
|
|
7746
|
-
ContentType: "application/json",
|
|
7747
|
-
IfNoneMatch: "*"
|
|
7748
|
-
}));
|
|
7945
|
+
const etag = await this.putLockObject(key, lockInfo);
|
|
7749
7946
|
this.logger.debug(`Lock acquired for stack: ${stackName} (${region}), owner: ${lockOwner}`);
|
|
7947
|
+
this.trackHeldLock({
|
|
7948
|
+
stackName,
|
|
7949
|
+
region,
|
|
7950
|
+
key,
|
|
7951
|
+
info: lockInfo,
|
|
7952
|
+
etag
|
|
7953
|
+
});
|
|
7750
7954
|
return true;
|
|
7751
7955
|
} catch (error) {
|
|
7752
|
-
if (error
|
|
7956
|
+
if (this.isForeignLockError(error)) {
|
|
7753
7957
|
this.logger.debug(`Lock already exists for stack: ${stackName} (${region})`);
|
|
7754
|
-
const
|
|
7755
|
-
if (
|
|
7756
|
-
|
|
7757
|
-
|
|
7958
|
+
const existing = await this.getLockRecord(stackName, region);
|
|
7959
|
+
if (existing && this.isLockExpired(existing.info)) {
|
|
7960
|
+
if (existing.etag === void 0) {
|
|
7961
|
+
this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): its current version could not be identified, so removing it could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
|
|
7962
|
+
return false;
|
|
7963
|
+
}
|
|
7964
|
+
try {
|
|
7965
|
+
await this.deleteLock(stackName, region, existing.etag);
|
|
7966
|
+
} catch (deleteError) {
|
|
7967
|
+
if (this.isConditionUnsupportedError(deleteError)) {
|
|
7968
|
+
this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): this endpoint or policy will not evaluate a conditional delete, and removing it unconditionally could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
|
|
7969
|
+
return false;
|
|
7970
|
+
} else if (this.isNotOursError(deleteError)) {
|
|
7971
|
+
this.logger.debug(`Expired lock for stack ${stackName} (${region}) changed before takeover; treating as contended`);
|
|
7972
|
+
return false;
|
|
7973
|
+
} else throw deleteError;
|
|
7974
|
+
}
|
|
7975
|
+
this.logger.warn(`Took over an EXPIRED lock for stack: ${stackName} (${region}, owner: ${existing.info.owner}, expired ${this.formatDuration(now - existing.info.expiresAt)} ago). A live cdkd process renews its lock well inside the TTL, so this normally means the previous owner crashed or was suspended -- if it is in fact still running, both processes are now writing to the same stack.`);
|
|
7758
7976
|
try {
|
|
7759
|
-
const
|
|
7760
|
-
await this.s3Client.send(new PutObjectCommand({
|
|
7761
|
-
Bucket: this.config.bucket,
|
|
7762
|
-
...await this.ownerParam(),
|
|
7763
|
-
Key: key,
|
|
7764
|
-
Body: retryBody,
|
|
7765
|
-
ContentLength: Buffer.byteLength(retryBody),
|
|
7766
|
-
ContentType: "application/json",
|
|
7767
|
-
IfNoneMatch: "*"
|
|
7768
|
-
}));
|
|
7977
|
+
const retryEtag = await this.putLockObject(key, lockInfo);
|
|
7769
7978
|
this.logger.debug(`Lock acquired for stack: ${stackName} (${region}) after expired lock cleanup, owner: ${lockOwner}`);
|
|
7979
|
+
this.trackHeldLock({
|
|
7980
|
+
stackName,
|
|
7981
|
+
region,
|
|
7982
|
+
key,
|
|
7983
|
+
info: lockInfo,
|
|
7984
|
+
etag: retryEtag
|
|
7985
|
+
});
|
|
7770
7986
|
return true;
|
|
7771
7987
|
} catch (retryError) {
|
|
7772
|
-
if (retryError
|
|
7988
|
+
if (this.isForeignLockError(retryError)) {
|
|
7773
7989
|
this.logger.debug(`Lock was acquired by another process during expired lock cleanup for stack: ${stackName} (${region})`);
|
|
7774
7990
|
return false;
|
|
7775
7991
|
}
|
|
@@ -7782,6 +7998,24 @@ var LockManager = class {
|
|
|
7782
7998
|
}
|
|
7783
7999
|
}
|
|
7784
8000
|
/**
|
|
8001
|
+
* Write the lock object and return the ETag S3 assigned it.
|
|
8002
|
+
*
|
|
8003
|
+
* `condition` defaults to `IfNoneMatch: '*'` (acquisition -- succeed only if
|
|
8004
|
+
* no current version exists). Renewal passes `IfMatch: <etag>` instead.
|
|
8005
|
+
*/
|
|
8006
|
+
async putLockObject(key, lockInfo, condition = { IfNoneMatch: "*" }) {
|
|
8007
|
+
const body = JSON.stringify(lockInfo, null, 2);
|
|
8008
|
+
return (await this.s3Client.send(new PutObjectCommand({
|
|
8009
|
+
Bucket: this.config.bucket,
|
|
8010
|
+
...await this.ownerParam(),
|
|
8011
|
+
Key: key,
|
|
8012
|
+
Body: body,
|
|
8013
|
+
ContentLength: Buffer.byteLength(body),
|
|
8014
|
+
ContentType: "application/json",
|
|
8015
|
+
...condition
|
|
8016
|
+
}))).ETag;
|
|
8017
|
+
}
|
|
8018
|
+
/**
|
|
7785
8019
|
* Get current lock information.
|
|
7786
8020
|
*
|
|
7787
8021
|
* `region` is required for the new region-scoped lock layout. Pass
|
|
@@ -7789,6 +8023,18 @@ var LockManager = class {
|
|
|
7789
8023
|
* file (e.g. for state-listing tools that don't yet know the region).
|
|
7790
8024
|
*/
|
|
7791
8025
|
async getLockInfo(stackName, region) {
|
|
8026
|
+
const record = await this.getLockRecord(stackName, region);
|
|
8027
|
+
return record ? record.info : null;
|
|
8028
|
+
}
|
|
8029
|
+
/**
|
|
8030
|
+
* `getLockInfo` plus the S3 ETag of the object the info came from.
|
|
8031
|
+
*
|
|
8032
|
+
* Internal because the ETag is only meaningful to code that then issues a
|
|
8033
|
+
* CONDITIONAL write or delete against the exact bytes it read (issue #2168):
|
|
8034
|
+
* the expired-lock takeover in `acquireLock`. Every public reader wants the
|
|
8035
|
+
* body alone.
|
|
8036
|
+
*/
|
|
8037
|
+
async getLockRecord(stackName, region) {
|
|
7792
8038
|
await this.ensureClientForBucket();
|
|
7793
8039
|
const key = this.getLockKey(stackName, region);
|
|
7794
8040
|
try {
|
|
@@ -7812,7 +8058,10 @@ var LockManager = class {
|
|
|
7812
8058
|
...parsed.operation !== void 0 && { operation: displaySafe(parsed.operation) }
|
|
7813
8059
|
};
|
|
7814
8060
|
this.logger.debug(`Lock info for stack: ${stackName}:`, lockInfo);
|
|
7815
|
-
return
|
|
8061
|
+
return {
|
|
8062
|
+
info: lockInfo,
|
|
8063
|
+
etag: response.ETag
|
|
8064
|
+
};
|
|
7816
8065
|
} catch (error) {
|
|
7817
8066
|
if (error instanceof NoSuchKey) {
|
|
7818
8067
|
this.logger.debug(`No lock exists for stack: ${stackName}`);
|
|
@@ -7834,24 +8083,137 @@ var LockManager = class {
|
|
|
7834
8083
|
return await this.getLockInfo(stackName, region) !== null;
|
|
7835
8084
|
}
|
|
7836
8085
|
/**
|
|
7837
|
-
* Release a lock for a stack
|
|
8086
|
+
* Release a lock for a stack.
|
|
8087
|
+
*
|
|
8088
|
+
* The DELETE is CONDITIONAL on the ETag this process last wrote (issue
|
|
8089
|
+
* #2168). Before that it was an owner-blind unconditional delete, which is
|
|
8090
|
+
* what turned a single lost lock into a cascade: an operation that outlived
|
|
8091
|
+
* its TTL had its lock taken over by a second process, then deleted the
|
|
8092
|
+
* SECOND process's lock on its way out, freeing the stack for a third.
|
|
8093
|
+
*
|
|
8094
|
+
* A `PreconditionFailed` therefore means "the object here is not the one I
|
|
8095
|
+
* wrote", and the correct response is to leave it alone -- not to raise, as
|
|
8096
|
+
* the operation itself has already finished and its caller has nothing to do
|
|
8097
|
+
* about it.
|
|
8098
|
+
*
|
|
8099
|
+
* The condition is dropped for exactly ONE class of failure: the endpoint or
|
|
8100
|
+
* the policy will not EVALUATE it (403 / 501 -- see
|
|
8101
|
+
* `isConditionUnsupportedError`), and even then only after an ownership
|
|
8102
|
+
* re-check. Everything else RAISES, as this method always has. In particular
|
|
8103
|
+
* a 409 (S3's answer to a concurrent operation on the key) and a 503 are not
|
|
8104
|
+
* fallback-worthy: the first IS the contended case, and the second may mean
|
|
8105
|
+
* the delete already landed with the response lost. The heartbeat is stopped
|
|
8106
|
+
* by then, so the worst outcome of raising is a lock that lapses at its TTL
|
|
8107
|
+
* -- recoverable, unlike one deleted out from under a live writer.
|
|
8108
|
+
*
|
|
8109
|
+
* Callers must therefore tolerate a throw here, and all sixteen do: four
|
|
8110
|
+
* wrap it in `try`/`catch` (`deploy-engine.ts` plus three in
|
|
8111
|
+
* `destroy-runner.ts`) and twelve attach a `.catch()`: a
|
|
8112
|
+
* failed release is a warning, never the error a command reports.
|
|
7838
8113
|
*/
|
|
7839
8114
|
async releaseLock(stackName, region) {
|
|
7840
|
-
await this.ensureClientForBucket();
|
|
7841
8115
|
const key = this.getLockKey(stackName, region);
|
|
8116
|
+
const held = this.heldLocks.get(key);
|
|
8117
|
+
if (held?.releasing) {
|
|
8118
|
+
this.logger.debug(`Release already in flight (or done) for stack ${stackName} (${region})`);
|
|
8119
|
+
return held.releasing;
|
|
8120
|
+
}
|
|
8121
|
+
if (!held) return this.doReleaseLock(stackName, region, void 0);
|
|
8122
|
+
this.stopRenewal(held);
|
|
8123
|
+
const releasing = this.doReleaseLock(stackName, region, held);
|
|
8124
|
+
held.releasing = releasing;
|
|
8125
|
+
return releasing;
|
|
8126
|
+
}
|
|
8127
|
+
async doReleaseLock(stackName, region, held) {
|
|
8128
|
+
await this.ensureClientForBucket();
|
|
8129
|
+
if (held) await held.renewing?.catch(() => void 0);
|
|
8130
|
+
if (held?.lost) {
|
|
8131
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process lost it while the operation was still running, so the lock present now belongs to someone else.`);
|
|
8132
|
+
return;
|
|
8133
|
+
}
|
|
8134
|
+
if (held && held.etag === void 0 && !await this.stillOursByBody(held)) {
|
|
8135
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process never learned which version of the lock it wrote, and could not confirm the one present is its own. It expires on its own, or clear it with cdkd force-unlock.`);
|
|
8136
|
+
return;
|
|
8137
|
+
}
|
|
7842
8138
|
try {
|
|
7843
8139
|
this.logger.debug(`Releasing lock for stack: ${stackName} (${region})`);
|
|
7844
|
-
await this.
|
|
7845
|
-
Bucket: this.config.bucket,
|
|
7846
|
-
...await this.ownerParam(),
|
|
7847
|
-
Key: key
|
|
7848
|
-
}));
|
|
8140
|
+
await this.deleteLock(stackName, region, held?.etag);
|
|
7849
8141
|
this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
|
|
7850
8142
|
} catch (error) {
|
|
8143
|
+
if (this.isForeignLockError(error)) {
|
|
8144
|
+
this.logger.warn(held?.etagUncertain ? `Not releasing the lock for stack '${stackName}' (${region}): this process could not confirm which version of the lock it last wrote, so it will not delete one it may not own. The lock clears on its own at ${new Date(held.info.expiresAt).toISOString()}, or immediately with cdkd force-unlock.` : `Not releasing the lock for stack '${stackName}' (${region}): it has been replaced since this process acquired it, so another cdkd process now holds it. Leaving it in place.`);
|
|
8145
|
+
return;
|
|
8146
|
+
}
|
|
8147
|
+
if (this.isGoneError(error)) {
|
|
8148
|
+
this.logger.debug(`Lock for stack ${stackName} (${region}) was already gone`);
|
|
8149
|
+
return;
|
|
8150
|
+
}
|
|
8151
|
+
if (held?.etag !== void 0 && this.isConditionUnsupportedError(error)) {
|
|
8152
|
+
if (!await this.stillOursByBody(held)) {
|
|
8153
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): the conditional delete could not be evaluated here, and this process could not confirm the lock is still its own. Leaving it in place rather than risk deleting another process's lock. It expires on its own, or clear it with cdkd force-unlock. If this repeats, grant s3:GetObject on the lock key -- a delete conditioned on an ETag requires it.`);
|
|
8154
|
+
return;
|
|
8155
|
+
}
|
|
8156
|
+
this.logger.debug(`Conditional lock release for stack ${stackName} (${region}) is not supported here (${error instanceof Error ? error.message : String(error)}); retrying unconditionally`);
|
|
8157
|
+
try {
|
|
8158
|
+
await this.deleteLock(stackName, region);
|
|
8159
|
+
this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
|
|
8160
|
+
return;
|
|
8161
|
+
} catch (fallbackError) {
|
|
8162
|
+
throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`, fallbackError instanceof Error ? fallbackError : void 0);
|
|
8163
|
+
}
|
|
8164
|
+
}
|
|
7851
8165
|
throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${error instanceof Error ? error.message : String(error)}`, error instanceof Error ? error : void 0);
|
|
7852
8166
|
}
|
|
7853
8167
|
}
|
|
7854
8168
|
/**
|
|
8169
|
+
* Is the stored lock still the one this process acquired?
|
|
8170
|
+
*
|
|
8171
|
+
* Used only where the ETag condition could not be EVALUATED, to decide
|
|
8172
|
+
* whether dropping it is safe. Two earlier cuts of this were wrong in
|
|
8173
|
+
* opposite directions and both are worth stating, because the shape recurs:
|
|
8174
|
+
*
|
|
8175
|
+
* - It short-circuited on `Date.now() < held.info.expiresAt`, reasoning that
|
|
8176
|
+
* nobody could have taken over while our own deadline was ahead. But
|
|
8177
|
+
* `forceReleaseLock` deletes regardless of expiry -- that is its whole
|
|
8178
|
+
* contract -- so a user running `cdkd force-unlock` mid-operation is a
|
|
8179
|
+
* LEGITIMATE takeover the shortcut cannot see. Cross-machine clock skew
|
|
8180
|
+
* reaches the same state without anyone running anything. The shortcut is
|
|
8181
|
+
* gone; the read costs one GetObject on an already-failed path.
|
|
8182
|
+
*
|
|
8183
|
+
* - It answered TRUE when the read FAILED, to avoid stranding a lock. That
|
|
8184
|
+
* made it inert in exactly the case it exists for: the documented trigger
|
|
8185
|
+
* for the 403 is a policy granting `s3:DeleteObject` without the
|
|
8186
|
+
* `s3:GetObject` a conditional delete needs -- under which this read fails
|
|
8187
|
+
* too. So the guard would wave through precisely the situation it was
|
|
8188
|
+
* added to catch. A failed read now REFUSES.
|
|
8189
|
+
*
|
|
8190
|
+
* The remaining asymmetry is deliberate and is this method's stated rule: a
|
|
8191
|
+
* stranded lock is bounded by the TTL and clearable with `force-unlock`,
|
|
8192
|
+
* while a lock deleted out from under a live writer is neither.
|
|
8193
|
+
*/
|
|
8194
|
+
async stillOursByBody(held) {
|
|
8195
|
+
let record;
|
|
8196
|
+
try {
|
|
8197
|
+
record = await this.getLockRecord(held.stackName, held.region);
|
|
8198
|
+
} catch {
|
|
8199
|
+
return false;
|
|
8200
|
+
}
|
|
8201
|
+
if (!record) return false;
|
|
8202
|
+
return this.sameLockIdentity(record.info, held.info);
|
|
8203
|
+
}
|
|
8204
|
+
/**
|
|
8205
|
+
* Whether two lock bodies name the same acquisition.
|
|
8206
|
+
*
|
|
8207
|
+
* `owner` is compared through `displaySafe` on BOTH sides: a body read back
|
|
8208
|
+
* has been sanitized by `getLockRecord` while the one this process
|
|
8209
|
+
* constructed has not, so an unsanitized comparison would never match for a
|
|
8210
|
+
* `$USER` or hostname containing a stripped codepoint -- and the process
|
|
8211
|
+
* would then disown its own lock.
|
|
8212
|
+
*/
|
|
8213
|
+
sameLockIdentity(a, b) {
|
|
8214
|
+
return displaySafe(a.owner) === displaySafe(b.owner) && a.timestamp === b.timestamp;
|
|
8215
|
+
}
|
|
8216
|
+
/**
|
|
7855
8217
|
* Force release a lock regardless of owner or expiry status
|
|
7856
8218
|
*
|
|
7857
8219
|
* This is intended for CLI usage (e.g., --force-unlock flag) when a lock
|
|
@@ -7864,21 +8226,213 @@ var LockManager = class {
|
|
|
7864
8226
|
const where = `${stackName}${region ? ` (${region})` : ""}`;
|
|
7865
8227
|
const lockInfo = await this.getLockInfo(stackName, region).catch(() => null);
|
|
7866
8228
|
this.logger.warn(lockInfo ? `Force releasing lock for stack: ${where}, owner: ${lockInfo.owner}${lockInfo.operation ? `, operation: ${lockInfo.operation}` : ""}, expired: ${this.isLockExpired(lockInfo)}` : `Force releasing lock for stack: ${where} (no lock body read — absent or unparseable; deleting the object either way, since a lock cdkd cannot read is a lock nothing else can clear)`);
|
|
8229
|
+
const held = this.heldLocks.get(this.getLockKey(stackName, region));
|
|
8230
|
+
if (held) {
|
|
8231
|
+
this.stopRenewal(held);
|
|
8232
|
+
held.releasing = Promise.resolve();
|
|
8233
|
+
}
|
|
7867
8234
|
await this.deleteLock(stackName, region);
|
|
7868
8235
|
}
|
|
7869
8236
|
/**
|
|
7870
|
-
* Internal method to delete the lock file from S3
|
|
8237
|
+
* Internal method to delete the lock file from S3.
|
|
8238
|
+
*
|
|
8239
|
+
* When `etag` is supplied the delete is conditional (`IfMatch`), so it
|
|
8240
|
+
* removes the object ONLY while it is still byte-identical to the one the
|
|
8241
|
+
* caller read or wrote. S3 evaluates the condition against the CURRENT
|
|
8242
|
+
* version, which is the right unit here even on the versioned state bucket:
|
|
8243
|
+
* the current version IS the live lock.
|
|
7871
8244
|
*/
|
|
7872
|
-
async deleteLock(stackName, region) {
|
|
8245
|
+
async deleteLock(stackName, region, etag) {
|
|
7873
8246
|
await this.ensureClientForBucket();
|
|
7874
8247
|
const key = this.getLockKey(stackName, region);
|
|
7875
8248
|
await this.s3Client.send(new DeleteObjectCommand({
|
|
7876
8249
|
Bucket: this.config.bucket,
|
|
7877
8250
|
...await this.ownerParam(),
|
|
7878
|
-
Key: key
|
|
8251
|
+
Key: key,
|
|
8252
|
+
...etag !== void 0 && { IfMatch: etag }
|
|
7879
8253
|
}));
|
|
7880
8254
|
}
|
|
7881
8255
|
/**
|
|
8256
|
+
* Record a lock this process now holds and start renewing it.
|
|
8257
|
+
*/
|
|
8258
|
+
trackHeldLock(args) {
|
|
8259
|
+
const previous = this.heldLocks.get(args.key);
|
|
8260
|
+
if (previous) this.stopRenewal(previous);
|
|
8261
|
+
const held = {
|
|
8262
|
+
stackName: args.stackName,
|
|
8263
|
+
region: args.region,
|
|
8264
|
+
key: args.key,
|
|
8265
|
+
info: args.info,
|
|
8266
|
+
etag: args.etag,
|
|
8267
|
+
timer: void 0,
|
|
8268
|
+
renewing: void 0,
|
|
8269
|
+
lost: false,
|
|
8270
|
+
releasing: void 0,
|
|
8271
|
+
etagUncertain: false,
|
|
8272
|
+
warnedPastExpiry: false
|
|
8273
|
+
};
|
|
8274
|
+
this.heldLocks.set(args.key, held);
|
|
8275
|
+
if (this.renewalDisabled) {
|
|
8276
|
+
this.logger.debug(`Lock renewal is disabled; the lock for ${args.stackName} (${args.region}) will lapse at its TTL`);
|
|
8277
|
+
return;
|
|
8278
|
+
}
|
|
8279
|
+
if (args.etag === void 0) {
|
|
8280
|
+
this.logger.debug(`No ETag returned when acquiring the lock for ${args.stackName} (${args.region}); renewal disabled for this lock`);
|
|
8281
|
+
return;
|
|
8282
|
+
}
|
|
8283
|
+
this.startRenewal(held);
|
|
8284
|
+
}
|
|
8285
|
+
startRenewal(held) {
|
|
8286
|
+
const timer = setInterval(() => {
|
|
8287
|
+
this.renewLock(held);
|
|
8288
|
+
}, this.renewalIntervalMs);
|
|
8289
|
+
timer.unref?.();
|
|
8290
|
+
held.timer = timer;
|
|
8291
|
+
}
|
|
8292
|
+
stopRenewal(held) {
|
|
8293
|
+
if (held.timer !== void 0) {
|
|
8294
|
+
clearInterval(held.timer);
|
|
8295
|
+
held.timer = void 0;
|
|
8296
|
+
}
|
|
8297
|
+
}
|
|
8298
|
+
/**
|
|
8299
|
+
* Push the lock's `expiresAt` forward, conditional on still owning it.
|
|
8300
|
+
*
|
|
8301
|
+
* This is what makes the TTL mean "the owner has been silent for TTL"
|
|
8302
|
+
* instead of "the operation has been running for TTL". Before issue #2168
|
|
8303
|
+
* there was no renewal at all, so any operation slower than the TTL -- an
|
|
8304
|
+
* FSx or EMR resource waiting up to an hour, or simply a large stack --
|
|
8305
|
+
* had its lock treated as free by the next process while it was still
|
|
8306
|
+
* running.
|
|
8307
|
+
*/
|
|
8308
|
+
renewLock(held) {
|
|
8309
|
+
if (held.lost || held.renewing !== void 0 || held.etag === void 0) return Promise.resolve();
|
|
8310
|
+
const inFlight = this.doRenewLock(held).finally(() => {
|
|
8311
|
+
held.renewing = void 0;
|
|
8312
|
+
});
|
|
8313
|
+
held.renewing = inFlight;
|
|
8314
|
+
return inFlight;
|
|
8315
|
+
}
|
|
8316
|
+
async doRenewLock(held) {
|
|
8317
|
+
const currentEtag = held.etag;
|
|
8318
|
+
if (currentEtag === void 0) return;
|
|
8319
|
+
const renewed = {
|
|
8320
|
+
...held.info,
|
|
8321
|
+
expiresAt: Date.now() + this.ttlMs
|
|
8322
|
+
};
|
|
8323
|
+
try {
|
|
8324
|
+
const etag = await this.putLockObject(held.key, renewed, { IfMatch: currentEtag });
|
|
8325
|
+
if (etag === void 0) {
|
|
8326
|
+
if (await this.adoptOwnWrite(held, renewed)) return;
|
|
8327
|
+
held.info = renewed;
|
|
8328
|
+
held.etagUncertain = true;
|
|
8329
|
+
this.stopRenewal(held);
|
|
8330
|
+
this.logger.debug(`Lock renewal for ${held.stackName} (${held.region}) returned no ETag and the object could not be re-read; stopping renewal and keeping the previous ETag`);
|
|
8331
|
+
return;
|
|
8332
|
+
}
|
|
8333
|
+
held.info = renewed;
|
|
8334
|
+
held.etag = etag;
|
|
8335
|
+
held.warnedPastExpiry = false;
|
|
8336
|
+
this.logger.debug(`Renewed lock for stack: ${held.stackName} (${held.region}) until ${new Date(renewed.expiresAt).toISOString()}`);
|
|
8337
|
+
} catch (error) {
|
|
8338
|
+
if (this.isForeignLockError(error) && await this.adoptOwnWrite(held, renewed)) return;
|
|
8339
|
+
if (this.isNotOursError(error)) {
|
|
8340
|
+
held.lost = true;
|
|
8341
|
+
this.stopRenewal(held);
|
|
8342
|
+
this.logger.warn(`Lost the lock for stack '${held.stackName}' (${held.region}) while the operation was still running: the lock object has been replaced or removed. Another cdkd process may now be writing to this stack concurrently. This process will not delete the current lock when it finishes.`);
|
|
8343
|
+
return;
|
|
8344
|
+
}
|
|
8345
|
+
this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) failed, will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
8346
|
+
if (!held.warnedPastExpiry && Date.now() >= held.info.expiresAt) {
|
|
8347
|
+
held.warnedPastExpiry = true;
|
|
8348
|
+
this.logger.warn(`Lock renewal for stack '${held.stackName}' (${held.region}) has been failing long enough that the lock expired at ${new Date(held.info.expiresAt).toISOString()}. Another cdkd process can now take it while this operation is still running. Renewal keeps retrying.`);
|
|
8349
|
+
}
|
|
8350
|
+
}
|
|
8351
|
+
}
|
|
8352
|
+
/**
|
|
8353
|
+
* Recover from a 412 that this process itself caused.
|
|
8354
|
+
*
|
|
8355
|
+
* A conditional PUT that S3 APPLIED but whose response never arrived -- a
|
|
8356
|
+
* dropped connection, or an SDK-internal retry of the same request -- leaves
|
|
8357
|
+
* `held.etag` pointing at the PREVIOUS version while the object already
|
|
8358
|
+
* holds the renewal. The retry then carries the stale `IfMatch` and S3
|
|
8359
|
+
* correctly answers 412. Reading that as "someone took my lock" is wrong in
|
|
8360
|
+
* both directions: it stops the heartbeat on a lock this process still owns,
|
|
8361
|
+
* it prints a warning naming a concurrent writer that does not exist, and
|
|
8362
|
+
* `releaseLock` then refuses to remove the process's OWN lock -- stranding
|
|
8363
|
+
* the stack for up to a full TTL, which is strictly worse than the
|
|
8364
|
+
* unconditional delete this change replaced.
|
|
8365
|
+
*
|
|
8366
|
+
* So a 412 is disambiguated by READING the object once. `renewed` is a body
|
|
8367
|
+
* only this process could have produced: its `owner` identifies the process
|
|
8368
|
+
* and its `expiresAt` is a millisecond timestamp this specific PUT chose. An
|
|
8369
|
+
* exact match on both (plus `timestamp`, which is fixed at acquisition) means
|
|
8370
|
+
* the write landed and only the answer was lost, so the renewal is adopted
|
|
8371
|
+
* rather than mourned.
|
|
8372
|
+
*
|
|
8373
|
+
* The read is on the 412 path only, which is rare. If it fails, the caller
|
|
8374
|
+
* falls through to the pessimistic branch -- an unreadable lock is not
|
|
8375
|
+
* evidence of ownership.
|
|
8376
|
+
*/
|
|
8377
|
+
async adoptOwnWrite(held, renewed) {
|
|
8378
|
+
const record = await this.getLockRecord(held.stackName, held.region).catch(() => null);
|
|
8379
|
+
if (!record || record.etag === void 0 || record.info.expiresAt !== renewed.expiresAt || !this.sameLockIdentity(record.info, renewed)) return false;
|
|
8380
|
+
held.info = renewed;
|
|
8381
|
+
held.etag = record.etag;
|
|
8382
|
+
held.warnedPastExpiry = false;
|
|
8383
|
+
this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) reported a conflict, but the object holds this process's own renewal -- adopting it and continuing`);
|
|
8384
|
+
return true;
|
|
8385
|
+
}
|
|
8386
|
+
/**
|
|
8387
|
+
* S3 told us the object is not the one we wrote (someone else replaced it).
|
|
8388
|
+
*
|
|
8389
|
+
* Matched on the status as well as the name: a 412 arriving as a bare
|
|
8390
|
+
* `S3ServiceException`, or under a renamed code, would otherwise skip the
|
|
8391
|
+
* "leave it alone" branch and reach the destructive fallback below it.
|
|
8392
|
+
*/
|
|
8393
|
+
isForeignLockError(error) {
|
|
8394
|
+
return error instanceof S3ServiceException && (error.name === "PreconditionFailed" || error.$metadata.httpStatusCode === 412);
|
|
8395
|
+
}
|
|
8396
|
+
/**
|
|
8397
|
+
* S3 told us this OBJECT is not there at all.
|
|
8398
|
+
*
|
|
8399
|
+
* `NoSuchBucket` is also a 404 and must NOT count: it says nothing about the
|
|
8400
|
+
* lock. Read as "gone" it would make `releaseLock` resolve where it used to
|
|
8401
|
+
* raise, and -- worse -- make one bucket-level 404 during renewal set
|
|
8402
|
+
* `lost`, killing the heartbeat and refusing to release a lock this process
|
|
8403
|
+
* still holds.
|
|
8404
|
+
*/
|
|
8405
|
+
isGoneError(error) {
|
|
8406
|
+
if (error instanceof NoSuchKey) return true;
|
|
8407
|
+
if (!(error instanceof S3ServiceException)) return false;
|
|
8408
|
+
if (error.name === "NoSuchBucket") return false;
|
|
8409
|
+
return error.name === "NoSuchKey" || error.$metadata.httpStatusCode === 404;
|
|
8410
|
+
}
|
|
8411
|
+
/**
|
|
8412
|
+
* The endpoint or the policy will not evaluate a conditional delete at all.
|
|
8413
|
+
*
|
|
8414
|
+
* Deliberately narrow, because this is the one predicate that authorises
|
|
8415
|
+
* dropping the ownership check. A conditional delete with a specific ETag
|
|
8416
|
+
* additionally requires `s3:GetObject` (AWS documents this for its
|
|
8417
|
+
* enforce-conditional-deletes bucket policy), so a policy granting only
|
|
8418
|
+
* `s3:DeleteObject` answers 403 -- and an S3-compatible endpoint that has
|
|
8419
|
+
* not implemented the header answers 501. Neither says anything about who
|
|
8420
|
+
* owns the lock, whereas 409 / 412 / 5xx all do.
|
|
8421
|
+
*/
|
|
8422
|
+
isConditionUnsupportedError(error) {
|
|
8423
|
+
if (!(error instanceof S3ServiceException)) return false;
|
|
8424
|
+
const status = error.$metadata.httpStatusCode;
|
|
8425
|
+
return status === 403 || status === 501 || error.name === "AccessDenied" || error.name === "Forbidden" || error.name === "NotImplemented";
|
|
8426
|
+
}
|
|
8427
|
+
/**
|
|
8428
|
+
* Either of the two ways S3 can tell us the lock we thought we held is no
|
|
8429
|
+
* longer ours: replaced by someone else (412) or deleted outright (404, e.g.
|
|
8430
|
+
* by `cdkd force-unlock`). Both mean stop renewing and do not delete.
|
|
8431
|
+
*/
|
|
8432
|
+
isNotOursError(error) {
|
|
8433
|
+
return this.isForeignLockError(error) || this.isGoneError(error);
|
|
8434
|
+
}
|
|
8435
|
+
/**
|
|
7882
8436
|
* Acquire lock with retry logic
|
|
7883
8437
|
*
|
|
7884
8438
|
* Retries up to maxRetries times with retryDelay between attempts.
|
|
@@ -17609,7 +18163,7 @@ var CloudControlProvider = class {
|
|
|
17609
18163
|
if (context?.finalSnapshotIdentifier !== void 0) throw new ProvisioningError(`${logicalId} (${resourceType}) requires a final snapshot (DeletionPolicy: Snapshot), but the Cloud Control API delete route has no final-snapshot parameter. Re-run with --skip-final-snapshot after snapshotting manually, or retain the resource.`, resourceType, logicalId, physicalId);
|
|
17610
18164
|
if (context?.removeProtection === true && resourceType === "AWS::AutoScaling::AutoScalingGroup") {
|
|
17611
18165
|
this.logger.debug(`Delegating protected AutoScalingGroup ${logicalId} delete to the SDK ASGProvider (Cloud Control cannot force-delete a protected ASG)`);
|
|
17612
|
-
const { ASGProvider } = await import("./asg-provider-
|
|
18166
|
+
const { ASGProvider } = await import("./asg-provider-DIKQ3APM.js").then((n) => n.n);
|
|
17613
18167
|
return await new ASGProvider().delete(logicalId, physicalId, resourceType, _properties, context);
|
|
17614
18168
|
}
|
|
17615
18169
|
const isProtectedEc2Instance = context?.removeProtection === true && resourceType === "AWS::EC2::Instance";
|
|
@@ -26305,7 +26859,7 @@ const FLUSH_INTERVAL_MS = 2e3;
|
|
|
26305
26859
|
const FLUSH_EVENT_THRESHOLD = 50;
|
|
26306
26860
|
/** Build-time cdkd version, with a dev fallback for non-built contexts. */
|
|
26307
26861
|
function getCdkdVersion() {
|
|
26308
|
-
return "0.284.
|
|
26862
|
+
return "0.284.43";
|
|
26309
26863
|
}
|
|
26310
26864
|
/**
|
|
26311
26865
|
* Generate a time-sortable unique run id, e.g.
|
|
@@ -28927,5 +29481,5 @@ var DeployEngine = class {
|
|
|
28927
29481
|
};
|
|
28928
29482
|
|
|
28929
29483
|
//#endregion
|
|
28930
|
-
export { startInterruptWatch as $,
|
|
28931
|
-
//# sourceMappingURL=deploy-engine-
|
|
29484
|
+
export { startInterruptWatch as $, setAwsClients as $n, WorkGraph as $t, renderStatefulReason as A, resolveCaptureObservedState as An, redactSecretsForState as At, exportAliasCollisionScrubWarning as B, findLargeInlineResources as Bn, DiffCalculator as Bt, isFinalSnapshotError as C, getDockerImageBySourceHash as Cn, normalizeAwsError as Cr, TEMPLATE_SOURCED_RULES as Ct, extractDeploymentEventError as D, getLegacyStateBucketName as Dn, isThrottlingError as Dr, isSingleDynamicReferenceToken as Dt, makeCanonicalizePropertiesFn as E, getDefaultStateBucketName as En, isRetryableTransientError as Er, errorCauseChain as Et, green as F, stateBucketExistenceConfirmed as Fn, s3BucketDomainName as Ft, collectInlinePolicyNamesManagedBySiblings as G, derivePartitionAndUrlSuffix as Gn, TemplateParser as Gt, secretBearingStateKeyWarning as H, expectedOwnerParam as Hn, describeTypeWithThrottleRetry as Ht, red as I, warnDeprecatedNoPrefixCliFlag as In, s3BucketDualStackDomainName as It, findActionableSilentDrops as J, clearBucketRegionCache as Jn, S3StateBackend as Jt, clearOnUpdateRemoval as K, AssemblyReader as Kn, LockManager as Kt, yellow as L, CFN_TEMPLATE_BODY_LIMIT as Ln, s3BucketRegionalDomainName as Lt, bold as M, resolveStateBucketWithDefault as Mn, classifyReplaySecretRegion as Mt, cyan as N, resolveStateBucketWithDefaultAndSource as Nn, producerRegionsFromState as Nt, MULTI_REGION_RECREATE_BLOCKED_TYPES as O, resolveApp as On, markNonRetryable as Or, maskSecretsInError as Ot, gray as P, resolveUseCdkBootstrapAssets as Pn, s3BucketArn as Pt, isInterruptedWaitError as Q, resetAwsClients as Qn, stringifyValue as Qt, collectDeclaredOutputNames as R, CFN_TEMPLATE_URL_LIMIT as Rn, s3BucketWebsiteUrl as Rt, createPreDeleteFinalSnapshot as S, AssetManifestLoader as Sn, isCdkdError as Sr, STATE_SOURCED_READBACK_RULES as St, unsupportedFinalSnapshotError as T, synthesisStatusMessage as Tn, isMarkedNonRetryable as Tr, dynamicReferenceTokens as Tt, stateKeySecretExposure as U, PARTITION_TABLE as Un, withRetry as Ut, isExportAliasCollision as V, uploadCfnTemplate as Vn, INTRINSIC_KEYS as Vt, IAMRoleProvider as W, canonicalizeRegion as Wn, DagBuilder as Wt, beginCommandInterruptScope as X, AwsClients as Xn, shouldRetainResource as Xt, findSilentDropProperties as Y, resolveBucketRegion as Yn, rebuildClientForBucketRegion as Yt, endCommandInterruptScope as Z, getAwsClients as Zn, AssetPublisher as Zt, computeImplicitDeleteEdges as _, formatDockerLoginError as _n, StackHasActiveImportsError as _r, replayWarn as _t, DeploymentEventsStore as a, stripControlChars as an, DeployCancelledError as ar, carriesDynamicReference as at, buildFinalSnapshotIdentifier as b, runDockerForeground as bn, SynthesisError as br, requireConfigString as bt, replayFailedOperations as c, ensureAssetStorage as cn, LocalMigrateError as cr, refStateLookupFromResource as ct, updatePartialReason as d, readBootstrapMarkerBody as dn, MissingCdkCliError as dr, resolveExplicitPhysicalId as dt, buildAssetRedirectMap as en, AssetError as er, CloudControlProvider as et, UNSPECIFIED_SKIP_REASON as f, validateAssetBucketName as fn, NestedStackChildDirectDestroyError as fr, assertRegionMatch as ft, IMPLICIT_DELETE_DEPENDENCIES as g, dockerSpawnEnvWithSensitive as gn, ResourceUpdateNotSupportedError as gr, readConfigString as gt, maskingRetryLogger as h, buildDockerImage as hn, ResourceTimeoutError as hr, configStringRefusal as ht, DeploymentEventsReader as i, escapeRegExp$1 as in, DependencyError as ir, IntrinsicFunctionResolver as it, formatResourceLine as j, resolveSkipPrefix as jn, scrubResourceRecord as jt, isStatefulRecreateTargetSync as k, resolveAutoAssetStorage as kn, __exportAll as kr, maskSecretsInText as kt, replayRollback as l, getBootstrapMarkerKey as ln, LocalStartServiceError as lr, WAFv2WebACLProvider as lt, withResourceDeadline as m, buildDenyExternalAccessPolicy as mn, ProvisioningError as mr, configBooleanRefusal as mt, DEFAULT_RESOURCE_WARN_AFTER_MS as n, loadPublishableAssetManifest as nn, ConfigError as nr, disableInstanceApiTermination as nt, planFailedOps as o, AssetModeResolver as on, DynamicReferenceRegionAmbiguousError as or, cfnRefValueFromPhysicalId as ot, deleteSkipReason as p, validateContainerRepoName as pn, PartialFailureError as pr, coerceCfnBoolean as pt, ProviderRegistry as q, processStackMessages as qn, displaySafe as qt, DeployEngine as r, rewriteTemplateAssetReferences as rn, CrossAccountSecretRefusalError as rr, isTerminationProtectionPropagationError as rt, planRollback as s, BOOTSTRAP_MARKER_PREFIX as sn, LocalInvokeBuildError as sr, getAccountInfo as st, DEFAULT_RESOURCE_TIMEOUT_MS as t, createAssetRedirectResolver as tn, CdkdError as tr, slowCcOperationTimeoutMs as tt, updatePartialMessage as u, parseBootstrapMarker as un, LockError as ur, normalizeAwsTagsToCfn as ut, ATOMIC_FINAL_SNAPSHOT_TYPES as v, getDockerCmd as vn, StackTerminationProtectionError as vr, requireConfigArray as vt, refusesFinalSnapshot as w, Synthesizer as wn, withErrorHandling as wr, createSecretMasker as wt, ccRoutedFinalSnapshotError as x, runDockerStreaming as xn, formatError as xr, STATE_SOURCED_CROSS_GENERATION_RULES as xt, PRE_DELETE_SNAPSHOT_TYPES as y, partitionSensitiveEnv as yn, StateError as yr, requireConfigObject as yt, collectPublishedOutputNames as z, MIGRATE_TMP_PREFIX as zn, applyRoleArnIfSet as zt };
|
|
29485
|
+
//# sourceMappingURL=deploy-engine-bFTrzeeu.js.map
|