@go-to-k/cdkd 0.284.41 → 0.284.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{asg-provider-D0lenV1M.js → asg-provider-Dn9oXi9c.js} +2 -2
- package/dist/{asg-provider-D0lenV1M.js.map → asg-provider-Dn9oXi9c.js.map} +1 -1
- package/dist/cli.js +8 -4
- package/dist/cli.js.map +1 -1
- package/dist/{deploy-engine-CCWl_tAH.js → deploy-engine-8ygMMbQ8.js} +416 -40
- package/dist/deploy-engine-8ygMMbQ8.js.map +1 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/deploy-engine-CCWl_tAH.js.map +0 -1
|
@@ -7594,6 +7594,26 @@ function displaySafe(value, opts) {
|
|
|
7594
7594
|
//#endregion
|
|
7595
7595
|
//#region src/state/lock-manager.ts
|
|
7596
7596
|
/**
|
|
7597
|
+
* Upper bound on the gap between lock renewals.
|
|
7598
|
+
*
|
|
7599
|
+
* Deliberately a fixed wall-clock ceiling rather than a pure fraction of the
|
|
7600
|
+
* TTL. A fraction alone (TTL/4 = 7.5 min at the default) makes renewal
|
|
7601
|
+
* unobservable in any deploy shorter than that, so nothing short of a
|
|
7602
|
+
* multi-hour fixture could ever prove the loop runs -- and an untestable
|
|
7603
|
+
* heartbeat is one that goes dead silently. At 2 min the default 30 min TTL
|
|
7604
|
+
* tolerates FOURTEEN consecutive missed renewals before it lapses, and an
|
|
7605
|
+
* ordinary broad-set integ (~8 min) performs three.
|
|
7606
|
+
*/
|
|
7607
|
+
const MAX_RENEWAL_INTERVAL_MS = 120 * 1e3;
|
|
7608
|
+
/**
|
|
7609
|
+
* Renew after at most a quarter of the TTL, so a short TTL still gets three
|
|
7610
|
+
* chances to renew before it lapses. This is what binds the two numbers
|
|
7611
|
+
* together for a caller that shortens `ttlMinutes`.
|
|
7612
|
+
*/
|
|
7613
|
+
const RENEWAL_TTL_FRACTION = 4;
|
|
7614
|
+
/** Floor, so a pathologically small TTL cannot spin the event loop. */
|
|
7615
|
+
const MIN_RENEWAL_INTERVAL_MS = 1e3;
|
|
7616
|
+
/**
|
|
7597
7617
|
* S3-based lock manager using conditional writes (If-None-Match)
|
|
7598
7618
|
*
|
|
7599
7619
|
* Implements distributed locking using S3's If-None-Match: "*" condition
|
|
@@ -7615,13 +7635,20 @@ var LockManager = class {
|
|
|
7615
7635
|
s3Client;
|
|
7616
7636
|
config;
|
|
7617
7637
|
ttlMs;
|
|
7638
|
+
renewalIntervalMs;
|
|
7639
|
+
renewalDisabled;
|
|
7640
|
+
/** Locks held by THIS process, keyed by S3 lock key. */
|
|
7641
|
+
heldLocks = /* @__PURE__ */ new Map();
|
|
7618
7642
|
clientResolved = false;
|
|
7619
7643
|
resolveInFlight = null;
|
|
7620
7644
|
constructor(s3Client, config, options) {
|
|
7621
7645
|
this.s3Client = s3Client;
|
|
7622
7646
|
this.config = config;
|
|
7623
7647
|
const ttlMinutes = options?.ttlMinutes ?? 30;
|
|
7648
|
+
if (!(ttlMinutes > 0) || !Number.isFinite(ttlMinutes)) throw new LockError(`Lock TTL must be a positive number of minutes (got ${String(ttlMinutes)}). A non-positive TTL makes every lock born expired, and the renewal interval clamps to its floor, so the heartbeat would spin one request per second for the life of the process.`);
|
|
7624
7649
|
this.ttlMs = ttlMinutes * 60 * 1e3;
|
|
7650
|
+
this.renewalDisabled = options?.disableRenewal === true;
|
|
7651
|
+
this.renewalIntervalMs = Math.max(MIN_RENEWAL_INTERVAL_MS, Math.min(MAX_RENEWAL_INTERVAL_MS, Math.floor(this.ttlMs / RENEWAL_TTL_FRACTION)));
|
|
7625
7652
|
}
|
|
7626
7653
|
/**
|
|
7627
7654
|
* Resolve the state bucket's actual region and, if it differs from the
|
|
@@ -7702,6 +7729,7 @@ var LockManager = class {
|
|
|
7702
7729
|
* Check if a lock is expired based on its expiresAt field
|
|
7703
7730
|
*/
|
|
7704
7731
|
isLockExpired(lockInfo) {
|
|
7732
|
+
if (!Number.isFinite(lockInfo.expiresAt)) return true;
|
|
7705
7733
|
return Date.now() >= lockInfo.expiresAt;
|
|
7706
7734
|
}
|
|
7707
7735
|
/**
|
|
@@ -7736,40 +7764,50 @@ var LockManager = class {
|
|
|
7736
7764
|
};
|
|
7737
7765
|
try {
|
|
7738
7766
|
this.logger.debug(`Attempting to acquire lock for stack: ${stackName} (${region})`);
|
|
7739
|
-
const
|
|
7740
|
-
await this.s3Client.send(new PutObjectCommand({
|
|
7741
|
-
Bucket: this.config.bucket,
|
|
7742
|
-
...await this.ownerParam(),
|
|
7743
|
-
Key: key,
|
|
7744
|
-
Body: lockBody,
|
|
7745
|
-
ContentLength: Buffer.byteLength(lockBody),
|
|
7746
|
-
ContentType: "application/json",
|
|
7747
|
-
IfNoneMatch: "*"
|
|
7748
|
-
}));
|
|
7767
|
+
const etag = await this.putLockObject(key, lockInfo);
|
|
7749
7768
|
this.logger.debug(`Lock acquired for stack: ${stackName} (${region}), owner: ${lockOwner}`);
|
|
7769
|
+
this.trackHeldLock({
|
|
7770
|
+
stackName,
|
|
7771
|
+
region,
|
|
7772
|
+
key,
|
|
7773
|
+
info: lockInfo,
|
|
7774
|
+
etag
|
|
7775
|
+
});
|
|
7750
7776
|
return true;
|
|
7751
7777
|
} catch (error) {
|
|
7752
|
-
if (error
|
|
7778
|
+
if (this.isForeignLockError(error)) {
|
|
7753
7779
|
this.logger.debug(`Lock already exists for stack: ${stackName} (${region})`);
|
|
7754
|
-
const
|
|
7755
|
-
if (
|
|
7756
|
-
|
|
7757
|
-
|
|
7780
|
+
const existing = await this.getLockRecord(stackName, region);
|
|
7781
|
+
if (existing && this.isLockExpired(existing.info)) {
|
|
7782
|
+
if (existing.etag === void 0) {
|
|
7783
|
+
this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): its current version could not be identified, so removing it could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
|
|
7784
|
+
return false;
|
|
7785
|
+
}
|
|
7758
7786
|
try {
|
|
7759
|
-
|
|
7760
|
-
|
|
7761
|
-
|
|
7762
|
-
|
|
7763
|
-
|
|
7764
|
-
|
|
7765
|
-
|
|
7766
|
-
|
|
7767
|
-
|
|
7768
|
-
|
|
7787
|
+
await this.deleteLock(stackName, region, existing.etag);
|
|
7788
|
+
} catch (deleteError) {
|
|
7789
|
+
if (this.isConditionUnsupportedError(deleteError)) {
|
|
7790
|
+
this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): this endpoint or policy will not evaluate a conditional delete, and removing it unconditionally could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
|
|
7791
|
+
return false;
|
|
7792
|
+
} else if (this.isNotOursError(deleteError)) {
|
|
7793
|
+
this.logger.debug(`Expired lock for stack ${stackName} (${region}) changed before takeover; treating as contended`);
|
|
7794
|
+
return false;
|
|
7795
|
+
} else throw deleteError;
|
|
7796
|
+
}
|
|
7797
|
+
this.logger.warn(`Took over an EXPIRED lock for stack: ${stackName} (${region}, owner: ${existing.info.owner}, expired ${this.formatDuration(now - existing.info.expiresAt)} ago). A live cdkd process renews its lock well inside the TTL, so this normally means the previous owner crashed or was suspended -- if it is in fact still running, both processes are now writing to the same stack.`);
|
|
7798
|
+
try {
|
|
7799
|
+
const retryEtag = await this.putLockObject(key, lockInfo);
|
|
7769
7800
|
this.logger.debug(`Lock acquired for stack: ${stackName} (${region}) after expired lock cleanup, owner: ${lockOwner}`);
|
|
7801
|
+
this.trackHeldLock({
|
|
7802
|
+
stackName,
|
|
7803
|
+
region,
|
|
7804
|
+
key,
|
|
7805
|
+
info: lockInfo,
|
|
7806
|
+
etag: retryEtag
|
|
7807
|
+
});
|
|
7770
7808
|
return true;
|
|
7771
7809
|
} catch (retryError) {
|
|
7772
|
-
if (retryError
|
|
7810
|
+
if (this.isForeignLockError(retryError)) {
|
|
7773
7811
|
this.logger.debug(`Lock was acquired by another process during expired lock cleanup for stack: ${stackName} (${region})`);
|
|
7774
7812
|
return false;
|
|
7775
7813
|
}
|
|
@@ -7782,6 +7820,24 @@ var LockManager = class {
|
|
|
7782
7820
|
}
|
|
7783
7821
|
}
|
|
7784
7822
|
/**
|
|
7823
|
+
* Write the lock object and return the ETag S3 assigned it.
|
|
7824
|
+
*
|
|
7825
|
+
* `condition` defaults to `IfNoneMatch: '*'` (acquisition -- succeed only if
|
|
7826
|
+
* no current version exists). Renewal passes `IfMatch: <etag>` instead.
|
|
7827
|
+
*/
|
|
7828
|
+
async putLockObject(key, lockInfo, condition = { IfNoneMatch: "*" }) {
|
|
7829
|
+
const body = JSON.stringify(lockInfo, null, 2);
|
|
7830
|
+
return (await this.s3Client.send(new PutObjectCommand({
|
|
7831
|
+
Bucket: this.config.bucket,
|
|
7832
|
+
...await this.ownerParam(),
|
|
7833
|
+
Key: key,
|
|
7834
|
+
Body: body,
|
|
7835
|
+
ContentLength: Buffer.byteLength(body),
|
|
7836
|
+
ContentType: "application/json",
|
|
7837
|
+
...condition
|
|
7838
|
+
}))).ETag;
|
|
7839
|
+
}
|
|
7840
|
+
/**
|
|
7785
7841
|
* Get current lock information.
|
|
7786
7842
|
*
|
|
7787
7843
|
* `region` is required for the new region-scoped lock layout. Pass
|
|
@@ -7789,6 +7845,18 @@ var LockManager = class {
|
|
|
7789
7845
|
* file (e.g. for state-listing tools that don't yet know the region).
|
|
7790
7846
|
*/
|
|
7791
7847
|
async getLockInfo(stackName, region) {
|
|
7848
|
+
const record = await this.getLockRecord(stackName, region);
|
|
7849
|
+
return record ? record.info : null;
|
|
7850
|
+
}
|
|
7851
|
+
/**
|
|
7852
|
+
* `getLockInfo` plus the S3 ETag of the object the info came from.
|
|
7853
|
+
*
|
|
7854
|
+
* Internal because the ETag is only meaningful to code that then issues a
|
|
7855
|
+
* CONDITIONAL write or delete against the exact bytes it read (issue #2168):
|
|
7856
|
+
* the expired-lock takeover in `acquireLock`. Every public reader wants the
|
|
7857
|
+
* body alone.
|
|
7858
|
+
*/
|
|
7859
|
+
async getLockRecord(stackName, region) {
|
|
7792
7860
|
await this.ensureClientForBucket();
|
|
7793
7861
|
const key = this.getLockKey(stackName, region);
|
|
7794
7862
|
try {
|
|
@@ -7812,7 +7880,10 @@ var LockManager = class {
|
|
|
7812
7880
|
...parsed.operation !== void 0 && { operation: displaySafe(parsed.operation) }
|
|
7813
7881
|
};
|
|
7814
7882
|
this.logger.debug(`Lock info for stack: ${stackName}:`, lockInfo);
|
|
7815
|
-
return
|
|
7883
|
+
return {
|
|
7884
|
+
info: lockInfo,
|
|
7885
|
+
etag: response.ETag
|
|
7886
|
+
};
|
|
7816
7887
|
} catch (error) {
|
|
7817
7888
|
if (error instanceof NoSuchKey) {
|
|
7818
7889
|
this.logger.debug(`No lock exists for stack: ${stackName}`);
|
|
@@ -7834,24 +7905,137 @@ var LockManager = class {
|
|
|
7834
7905
|
return await this.getLockInfo(stackName, region) !== null;
|
|
7835
7906
|
}
|
|
7836
7907
|
/**
|
|
7837
|
-
* Release a lock for a stack
|
|
7908
|
+
* Release a lock for a stack.
|
|
7909
|
+
*
|
|
7910
|
+
* The DELETE is CONDITIONAL on the ETag this process last wrote (issue
|
|
7911
|
+
* #2168). Before that it was an owner-blind unconditional delete, which is
|
|
7912
|
+
* what turned a single lost lock into a cascade: an operation that outlived
|
|
7913
|
+
* its TTL had its lock taken over by a second process, then deleted the
|
|
7914
|
+
* SECOND process's lock on its way out, freeing the stack for a third.
|
|
7915
|
+
*
|
|
7916
|
+
* A `PreconditionFailed` therefore means "the object here is not the one I
|
|
7917
|
+
* wrote", and the correct response is to leave it alone -- not to raise, as
|
|
7918
|
+
* the operation itself has already finished and its caller has nothing to do
|
|
7919
|
+
* about it.
|
|
7920
|
+
*
|
|
7921
|
+
* The condition is dropped for exactly ONE class of failure: the endpoint or
|
|
7922
|
+
* the policy will not EVALUATE it (403 / 501 -- see
|
|
7923
|
+
* `isConditionUnsupportedError`), and even then only after an ownership
|
|
7924
|
+
* re-check. Everything else RAISES, as this method always has. In particular
|
|
7925
|
+
* a 409 (S3's answer to a concurrent operation on the key) and a 503 are not
|
|
7926
|
+
* fallback-worthy: the first IS the contended case, and the second may mean
|
|
7927
|
+
* the delete already landed with the response lost. The heartbeat is stopped
|
|
7928
|
+
* by then, so the worst outcome of raising is a lock that lapses at its TTL
|
|
7929
|
+
* -- recoverable, unlike one deleted out from under a live writer.
|
|
7930
|
+
*
|
|
7931
|
+
* Callers must therefore tolerate a throw here, and all sixteen do: four
|
|
7932
|
+
* wrap it in `try`/`catch` (`deploy-engine.ts` plus three in
|
|
7933
|
+
* `destroy-runner.ts`) and twelve attach a `.catch()`: a
|
|
7934
|
+
* failed release is a warning, never the error a command reports.
|
|
7838
7935
|
*/
|
|
7839
7936
|
async releaseLock(stackName, region) {
|
|
7840
|
-
await this.ensureClientForBucket();
|
|
7841
7937
|
const key = this.getLockKey(stackName, region);
|
|
7938
|
+
const held = this.heldLocks.get(key);
|
|
7939
|
+
if (held?.releasing) {
|
|
7940
|
+
this.logger.debug(`Release already in flight (or done) for stack ${stackName} (${region})`);
|
|
7941
|
+
return held.releasing;
|
|
7942
|
+
}
|
|
7943
|
+
if (!held) return this.doReleaseLock(stackName, region, void 0);
|
|
7944
|
+
this.stopRenewal(held);
|
|
7945
|
+
const releasing = this.doReleaseLock(stackName, region, held);
|
|
7946
|
+
held.releasing = releasing;
|
|
7947
|
+
return releasing;
|
|
7948
|
+
}
|
|
7949
|
+
async doReleaseLock(stackName, region, held) {
|
|
7950
|
+
await this.ensureClientForBucket();
|
|
7951
|
+
if (held) await held.renewing?.catch(() => void 0);
|
|
7952
|
+
if (held?.lost) {
|
|
7953
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process lost it while the operation was still running, so the lock present now belongs to someone else.`);
|
|
7954
|
+
return;
|
|
7955
|
+
}
|
|
7956
|
+
if (held && held.etag === void 0 && !await this.stillOursByBody(held)) {
|
|
7957
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process never learned which version of the lock it wrote, and could not confirm the one present is its own. It expires on its own, or clear it with cdkd force-unlock.`);
|
|
7958
|
+
return;
|
|
7959
|
+
}
|
|
7842
7960
|
try {
|
|
7843
7961
|
this.logger.debug(`Releasing lock for stack: ${stackName} (${region})`);
|
|
7844
|
-
await this.
|
|
7845
|
-
Bucket: this.config.bucket,
|
|
7846
|
-
...await this.ownerParam(),
|
|
7847
|
-
Key: key
|
|
7848
|
-
}));
|
|
7962
|
+
await this.deleteLock(stackName, region, held?.etag);
|
|
7849
7963
|
this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
|
|
7850
7964
|
} catch (error) {
|
|
7965
|
+
if (this.isForeignLockError(error)) {
|
|
7966
|
+
this.logger.warn(held?.etagUncertain ? `Not releasing the lock for stack '${stackName}' (${region}): this process could not confirm which version of the lock it last wrote, so it will not delete one it may not own. The lock clears on its own at ${new Date(held.info.expiresAt).toISOString()}, or immediately with cdkd force-unlock.` : `Not releasing the lock for stack '${stackName}' (${region}): it has been replaced since this process acquired it, so another cdkd process now holds it. Leaving it in place.`);
|
|
7967
|
+
return;
|
|
7968
|
+
}
|
|
7969
|
+
if (this.isGoneError(error)) {
|
|
7970
|
+
this.logger.debug(`Lock for stack ${stackName} (${region}) was already gone`);
|
|
7971
|
+
return;
|
|
7972
|
+
}
|
|
7973
|
+
if (held?.etag !== void 0 && this.isConditionUnsupportedError(error)) {
|
|
7974
|
+
if (!await this.stillOursByBody(held)) {
|
|
7975
|
+
this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): the conditional delete could not be evaluated here, and this process could not confirm the lock is still its own. Leaving it in place rather than risk deleting another process's lock. It expires on its own, or clear it with cdkd force-unlock. If this repeats, grant s3:GetObject on the lock key -- a delete conditioned on an ETag requires it.`);
|
|
7976
|
+
return;
|
|
7977
|
+
}
|
|
7978
|
+
this.logger.debug(`Conditional lock release for stack ${stackName} (${region}) is not supported here (${error instanceof Error ? error.message : String(error)}); retrying unconditionally`);
|
|
7979
|
+
try {
|
|
7980
|
+
await this.deleteLock(stackName, region);
|
|
7981
|
+
this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
|
|
7982
|
+
return;
|
|
7983
|
+
} catch (fallbackError) {
|
|
7984
|
+
throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`, fallbackError instanceof Error ? fallbackError : void 0);
|
|
7985
|
+
}
|
|
7986
|
+
}
|
|
7851
7987
|
throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${error instanceof Error ? error.message : String(error)}`, error instanceof Error ? error : void 0);
|
|
7852
7988
|
}
|
|
7853
7989
|
}
|
|
7854
7990
|
/**
|
|
7991
|
+
* Is the stored lock still the one this process acquired?
|
|
7992
|
+
*
|
|
7993
|
+
* Used only where the ETag condition could not be EVALUATED, to decide
|
|
7994
|
+
* whether dropping it is safe. Two earlier cuts of this were wrong in
|
|
7995
|
+
* opposite directions and both are worth stating, because the shape recurs:
|
|
7996
|
+
*
|
|
7997
|
+
* - It short-circuited on `Date.now() < held.info.expiresAt`, reasoning that
|
|
7998
|
+
* nobody could have taken over while our own deadline was ahead. But
|
|
7999
|
+
* `forceReleaseLock` deletes regardless of expiry -- that is its whole
|
|
8000
|
+
* contract -- so a user running `cdkd force-unlock` mid-operation is a
|
|
8001
|
+
* LEGITIMATE takeover the shortcut cannot see. Cross-machine clock skew
|
|
8002
|
+
* reaches the same state without anyone running anything. The shortcut is
|
|
8003
|
+
* gone; the read costs one GetObject on an already-failed path.
|
|
8004
|
+
*
|
|
8005
|
+
* - It answered TRUE when the read FAILED, to avoid stranding a lock. That
|
|
8006
|
+
* made it inert in exactly the case it exists for: the documented trigger
|
|
8007
|
+
* for the 403 is a policy granting `s3:DeleteObject` without the
|
|
8008
|
+
* `s3:GetObject` a conditional delete needs -- under which this read fails
|
|
8009
|
+
* too. So the guard would wave through precisely the situation it was
|
|
8010
|
+
* added to catch. A failed read now REFUSES.
|
|
8011
|
+
*
|
|
8012
|
+
* The remaining asymmetry is deliberate and is this method's stated rule: a
|
|
8013
|
+
* stranded lock is bounded by the TTL and clearable with `force-unlock`,
|
|
8014
|
+
* while a lock deleted out from under a live writer is neither.
|
|
8015
|
+
*/
|
|
8016
|
+
async stillOursByBody(held) {
|
|
8017
|
+
let record;
|
|
8018
|
+
try {
|
|
8019
|
+
record = await this.getLockRecord(held.stackName, held.region);
|
|
8020
|
+
} catch {
|
|
8021
|
+
return false;
|
|
8022
|
+
}
|
|
8023
|
+
if (!record) return false;
|
|
8024
|
+
return this.sameLockIdentity(record.info, held.info);
|
|
8025
|
+
}
|
|
8026
|
+
/**
|
|
8027
|
+
* Whether two lock bodies name the same acquisition.
|
|
8028
|
+
*
|
|
8029
|
+
* `owner` is compared through `displaySafe` on BOTH sides: a body read back
|
|
8030
|
+
* has been sanitized by `getLockRecord` while the one this process
|
|
8031
|
+
* constructed has not, so an unsanitized comparison would never match for a
|
|
8032
|
+
* `$USER` or hostname containing a stripped codepoint -- and the process
|
|
8033
|
+
* would then disown its own lock.
|
|
8034
|
+
*/
|
|
8035
|
+
sameLockIdentity(a, b) {
|
|
8036
|
+
return displaySafe(a.owner) === displaySafe(b.owner) && a.timestamp === b.timestamp;
|
|
8037
|
+
}
|
|
8038
|
+
/**
|
|
7855
8039
|
* Force release a lock regardless of owner or expiry status
|
|
7856
8040
|
*
|
|
7857
8041
|
* This is intended for CLI usage (e.g., --force-unlock flag) when a lock
|
|
@@ -7864,21 +8048,213 @@ var LockManager = class {
|
|
|
7864
8048
|
const where = `${stackName}${region ? ` (${region})` : ""}`;
|
|
7865
8049
|
const lockInfo = await this.getLockInfo(stackName, region).catch(() => null);
|
|
7866
8050
|
this.logger.warn(lockInfo ? `Force releasing lock for stack: ${where}, owner: ${lockInfo.owner}${lockInfo.operation ? `, operation: ${lockInfo.operation}` : ""}, expired: ${this.isLockExpired(lockInfo)}` : `Force releasing lock for stack: ${where} (no lock body read — absent or unparseable; deleting the object either way, since a lock cdkd cannot read is a lock nothing else can clear)`);
|
|
8051
|
+
const held = this.heldLocks.get(this.getLockKey(stackName, region));
|
|
8052
|
+
if (held) {
|
|
8053
|
+
this.stopRenewal(held);
|
|
8054
|
+
held.releasing = Promise.resolve();
|
|
8055
|
+
}
|
|
7867
8056
|
await this.deleteLock(stackName, region);
|
|
7868
8057
|
}
|
|
7869
8058
|
/**
|
|
7870
|
-
* Internal method to delete the lock file from S3
|
|
8059
|
+
* Internal method to delete the lock file from S3.
|
|
8060
|
+
*
|
|
8061
|
+
* When `etag` is supplied the delete is conditional (`IfMatch`), so it
|
|
8062
|
+
* removes the object ONLY while it is still byte-identical to the one the
|
|
8063
|
+
* caller read or wrote. S3 evaluates the condition against the CURRENT
|
|
8064
|
+
* version, which is the right unit here even on the versioned state bucket:
|
|
8065
|
+
* the current version IS the live lock.
|
|
7871
8066
|
*/
|
|
7872
|
-
async deleteLock(stackName, region) {
|
|
8067
|
+
async deleteLock(stackName, region, etag) {
|
|
7873
8068
|
await this.ensureClientForBucket();
|
|
7874
8069
|
const key = this.getLockKey(stackName, region);
|
|
7875
8070
|
await this.s3Client.send(new DeleteObjectCommand({
|
|
7876
8071
|
Bucket: this.config.bucket,
|
|
7877
8072
|
...await this.ownerParam(),
|
|
7878
|
-
Key: key
|
|
8073
|
+
Key: key,
|
|
8074
|
+
...etag !== void 0 && { IfMatch: etag }
|
|
7879
8075
|
}));
|
|
7880
8076
|
}
|
|
7881
8077
|
/**
|
|
8078
|
+
* Record a lock this process now holds and start renewing it.
|
|
8079
|
+
*/
|
|
8080
|
+
trackHeldLock(args) {
|
|
8081
|
+
const previous = this.heldLocks.get(args.key);
|
|
8082
|
+
if (previous) this.stopRenewal(previous);
|
|
8083
|
+
const held = {
|
|
8084
|
+
stackName: args.stackName,
|
|
8085
|
+
region: args.region,
|
|
8086
|
+
key: args.key,
|
|
8087
|
+
info: args.info,
|
|
8088
|
+
etag: args.etag,
|
|
8089
|
+
timer: void 0,
|
|
8090
|
+
renewing: void 0,
|
|
8091
|
+
lost: false,
|
|
8092
|
+
releasing: void 0,
|
|
8093
|
+
etagUncertain: false,
|
|
8094
|
+
warnedPastExpiry: false
|
|
8095
|
+
};
|
|
8096
|
+
this.heldLocks.set(args.key, held);
|
|
8097
|
+
if (this.renewalDisabled) {
|
|
8098
|
+
this.logger.debug(`Lock renewal is disabled; the lock for ${args.stackName} (${args.region}) will lapse at its TTL`);
|
|
8099
|
+
return;
|
|
8100
|
+
}
|
|
8101
|
+
if (args.etag === void 0) {
|
|
8102
|
+
this.logger.debug(`No ETag returned when acquiring the lock for ${args.stackName} (${args.region}); renewal disabled for this lock`);
|
|
8103
|
+
return;
|
|
8104
|
+
}
|
|
8105
|
+
this.startRenewal(held);
|
|
8106
|
+
}
|
|
8107
|
+
startRenewal(held) {
|
|
8108
|
+
const timer = setInterval(() => {
|
|
8109
|
+
this.renewLock(held);
|
|
8110
|
+
}, this.renewalIntervalMs);
|
|
8111
|
+
timer.unref?.();
|
|
8112
|
+
held.timer = timer;
|
|
8113
|
+
}
|
|
8114
|
+
stopRenewal(held) {
|
|
8115
|
+
if (held.timer !== void 0) {
|
|
8116
|
+
clearInterval(held.timer);
|
|
8117
|
+
held.timer = void 0;
|
|
8118
|
+
}
|
|
8119
|
+
}
|
|
8120
|
+
/**
|
|
8121
|
+
* Push the lock's `expiresAt` forward, conditional on still owning it.
|
|
8122
|
+
*
|
|
8123
|
+
* This is what makes the TTL mean "the owner has been silent for TTL"
|
|
8124
|
+
* instead of "the operation has been running for TTL". Before issue #2168
|
|
8125
|
+
* there was no renewal at all, so any operation slower than the TTL -- an
|
|
8126
|
+
* FSx or EMR resource waiting up to an hour, or simply a large stack --
|
|
8127
|
+
* had its lock treated as free by the next process while it was still
|
|
8128
|
+
* running.
|
|
8129
|
+
*/
|
|
8130
|
+
renewLock(held) {
|
|
8131
|
+
if (held.lost || held.renewing !== void 0 || held.etag === void 0) return Promise.resolve();
|
|
8132
|
+
const inFlight = this.doRenewLock(held).finally(() => {
|
|
8133
|
+
held.renewing = void 0;
|
|
8134
|
+
});
|
|
8135
|
+
held.renewing = inFlight;
|
|
8136
|
+
return inFlight;
|
|
8137
|
+
}
|
|
8138
|
+
async doRenewLock(held) {
|
|
8139
|
+
const currentEtag = held.etag;
|
|
8140
|
+
if (currentEtag === void 0) return;
|
|
8141
|
+
const renewed = {
|
|
8142
|
+
...held.info,
|
|
8143
|
+
expiresAt: Date.now() + this.ttlMs
|
|
8144
|
+
};
|
|
8145
|
+
try {
|
|
8146
|
+
const etag = await this.putLockObject(held.key, renewed, { IfMatch: currentEtag });
|
|
8147
|
+
if (etag === void 0) {
|
|
8148
|
+
if (await this.adoptOwnWrite(held, renewed)) return;
|
|
8149
|
+
held.info = renewed;
|
|
8150
|
+
held.etagUncertain = true;
|
|
8151
|
+
this.stopRenewal(held);
|
|
8152
|
+
this.logger.debug(`Lock renewal for ${held.stackName} (${held.region}) returned no ETag and the object could not be re-read; stopping renewal and keeping the previous ETag`);
|
|
8153
|
+
return;
|
|
8154
|
+
}
|
|
8155
|
+
held.info = renewed;
|
|
8156
|
+
held.etag = etag;
|
|
8157
|
+
held.warnedPastExpiry = false;
|
|
8158
|
+
this.logger.debug(`Renewed lock for stack: ${held.stackName} (${held.region}) until ${new Date(renewed.expiresAt).toISOString()}`);
|
|
8159
|
+
} catch (error) {
|
|
8160
|
+
if (this.isForeignLockError(error) && await this.adoptOwnWrite(held, renewed)) return;
|
|
8161
|
+
if (this.isNotOursError(error)) {
|
|
8162
|
+
held.lost = true;
|
|
8163
|
+
this.stopRenewal(held);
|
|
8164
|
+
this.logger.warn(`Lost the lock for stack '${held.stackName}' (${held.region}) while the operation was still running: the lock object has been replaced or removed. Another cdkd process may now be writing to this stack concurrently. This process will not delete the current lock when it finishes.`);
|
|
8165
|
+
return;
|
|
8166
|
+
}
|
|
8167
|
+
this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) failed, will retry: ${error instanceof Error ? error.message : String(error)}`);
|
|
8168
|
+
if (!held.warnedPastExpiry && Date.now() >= held.info.expiresAt) {
|
|
8169
|
+
held.warnedPastExpiry = true;
|
|
8170
|
+
this.logger.warn(`Lock renewal for stack '${held.stackName}' (${held.region}) has been failing long enough that the lock expired at ${new Date(held.info.expiresAt).toISOString()}. Another cdkd process can now take it while this operation is still running. Renewal keeps retrying.`);
|
|
8171
|
+
}
|
|
8172
|
+
}
|
|
8173
|
+
}
|
|
8174
|
+
/**
|
|
8175
|
+
* Recover from a 412 that this process itself caused.
|
|
8176
|
+
*
|
|
8177
|
+
* A conditional PUT that S3 APPLIED but whose response never arrived -- a
|
|
8178
|
+
* dropped connection, or an SDK-internal retry of the same request -- leaves
|
|
8179
|
+
* `held.etag` pointing at the PREVIOUS version while the object already
|
|
8180
|
+
* holds the renewal. The retry then carries the stale `IfMatch` and S3
|
|
8181
|
+
* correctly answers 412. Reading that as "someone took my lock" is wrong in
|
|
8182
|
+
* both directions: it stops the heartbeat on a lock this process still owns,
|
|
8183
|
+
* it prints a warning naming a concurrent writer that does not exist, and
|
|
8184
|
+
* `releaseLock` then refuses to remove the process's OWN lock -- stranding
|
|
8185
|
+
* the stack for up to a full TTL, which is strictly worse than the
|
|
8186
|
+
* unconditional delete this change replaced.
|
|
8187
|
+
*
|
|
8188
|
+
* So a 412 is disambiguated by READING the object once. `renewed` is a body
|
|
8189
|
+
* only this process could have produced: its `owner` identifies the process
|
|
8190
|
+
* and its `expiresAt` is a millisecond timestamp this specific PUT chose. An
|
|
8191
|
+
* exact match on both (plus `timestamp`, which is fixed at acquisition) means
|
|
8192
|
+
* the write landed and only the answer was lost, so the renewal is adopted
|
|
8193
|
+
* rather than mourned.
|
|
8194
|
+
*
|
|
8195
|
+
* The read is on the 412 path only, which is rare. If it fails, the caller
|
|
8196
|
+
* falls through to the pessimistic branch -- an unreadable lock is not
|
|
8197
|
+
* evidence of ownership.
|
|
8198
|
+
*/
|
|
8199
|
+
async adoptOwnWrite(held, renewed) {
|
|
8200
|
+
const record = await this.getLockRecord(held.stackName, held.region).catch(() => null);
|
|
8201
|
+
if (!record || record.etag === void 0 || record.info.expiresAt !== renewed.expiresAt || !this.sameLockIdentity(record.info, renewed)) return false;
|
|
8202
|
+
held.info = renewed;
|
|
8203
|
+
held.etag = record.etag;
|
|
8204
|
+
held.warnedPastExpiry = false;
|
|
8205
|
+
this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) reported a conflict, but the object holds this process's own renewal -- adopting it and continuing`);
|
|
8206
|
+
return true;
|
|
8207
|
+
}
|
|
8208
|
+
/**
|
|
8209
|
+
* S3 told us the object is not the one we wrote (someone else replaced it).
|
|
8210
|
+
*
|
|
8211
|
+
* Matched on the status as well as the name: a 412 arriving as a bare
|
|
8212
|
+
* `S3ServiceException`, or under a renamed code, would otherwise skip the
|
|
8213
|
+
* "leave it alone" branch and reach the destructive fallback below it.
|
|
8214
|
+
*/
|
|
8215
|
+
isForeignLockError(error) {
|
|
8216
|
+
return error instanceof S3ServiceException && (error.name === "PreconditionFailed" || error.$metadata.httpStatusCode === 412);
|
|
8217
|
+
}
|
|
8218
|
+
/**
|
|
8219
|
+
* S3 told us this OBJECT is not there at all.
|
|
8220
|
+
*
|
|
8221
|
+
* `NoSuchBucket` is also a 404 and must NOT count: it says nothing about the
|
|
8222
|
+
* lock. Read as "gone" it would make `releaseLock` resolve where it used to
|
|
8223
|
+
* raise, and -- worse -- make one bucket-level 404 during renewal set
|
|
8224
|
+
* `lost`, killing the heartbeat and refusing to release a lock this process
|
|
8225
|
+
* still holds.
|
|
8226
|
+
*/
|
|
8227
|
+
isGoneError(error) {
|
|
8228
|
+
if (error instanceof NoSuchKey) return true;
|
|
8229
|
+
if (!(error instanceof S3ServiceException)) return false;
|
|
8230
|
+
if (error.name === "NoSuchBucket") return false;
|
|
8231
|
+
return error.name === "NoSuchKey" || error.$metadata.httpStatusCode === 404;
|
|
8232
|
+
}
|
|
8233
|
+
/**
|
|
8234
|
+
* The endpoint or the policy will not evaluate a conditional delete at all.
|
|
8235
|
+
*
|
|
8236
|
+
* Deliberately narrow, because this is the one predicate that authorises
|
|
8237
|
+
* dropping the ownership check. A conditional delete with a specific ETag
|
|
8238
|
+
* additionally requires `s3:GetObject` (AWS documents this for its
|
|
8239
|
+
* enforce-conditional-deletes bucket policy), so a policy granting only
|
|
8240
|
+
* `s3:DeleteObject` answers 403 -- and an S3-compatible endpoint that has
|
|
8241
|
+
* not implemented the header answers 501. Neither says anything about who
|
|
8242
|
+
* owns the lock, whereas 409 / 412 / 5xx all do.
|
|
8243
|
+
*/
|
|
8244
|
+
isConditionUnsupportedError(error) {
|
|
8245
|
+
if (!(error instanceof S3ServiceException)) return false;
|
|
8246
|
+
const status = error.$metadata.httpStatusCode;
|
|
8247
|
+
return status === 403 || status === 501 || error.name === "AccessDenied" || error.name === "Forbidden" || error.name === "NotImplemented";
|
|
8248
|
+
}
|
|
8249
|
+
/**
|
|
8250
|
+
* Either of the two ways S3 can tell us the lock we thought we held is no
|
|
8251
|
+
* longer ours: replaced by someone else (412) or deleted outright (404, e.g.
|
|
8252
|
+
* by `cdkd force-unlock`). Both mean stop renewing and do not delete.
|
|
8253
|
+
*/
|
|
8254
|
+
isNotOursError(error) {
|
|
8255
|
+
return this.isForeignLockError(error) || this.isGoneError(error);
|
|
8256
|
+
}
|
|
8257
|
+
/**
|
|
7882
8258
|
* Acquire lock with retry logic
|
|
7883
8259
|
*
|
|
7884
8260
|
* Retries up to maxRetries times with retryDelay between attempts.
|
|
@@ -17609,7 +17985,7 @@ var CloudControlProvider = class {
|
|
|
17609
17985
|
if (context?.finalSnapshotIdentifier !== void 0) throw new ProvisioningError(`${logicalId} (${resourceType}) requires a final snapshot (DeletionPolicy: Snapshot), but the Cloud Control API delete route has no final-snapshot parameter. Re-run with --skip-final-snapshot after snapshotting manually, or retain the resource.`, resourceType, logicalId, physicalId);
|
|
17610
17986
|
if (context?.removeProtection === true && resourceType === "AWS::AutoScaling::AutoScalingGroup") {
|
|
17611
17987
|
this.logger.debug(`Delegating protected AutoScalingGroup ${logicalId} delete to the SDK ASGProvider (Cloud Control cannot force-delete a protected ASG)`);
|
|
17612
|
-
const { ASGProvider } = await import("./asg-provider-
|
|
17988
|
+
const { ASGProvider } = await import("./asg-provider-Dn9oXi9c.js").then((n) => n.n);
|
|
17613
17989
|
return await new ASGProvider().delete(logicalId, physicalId, resourceType, _properties, context);
|
|
17614
17990
|
}
|
|
17615
17991
|
const isProtectedEc2Instance = context?.removeProtection === true && resourceType === "AWS::EC2::Instance";
|
|
@@ -26305,7 +26681,7 @@ const FLUSH_INTERVAL_MS = 2e3;
|
|
|
26305
26681
|
const FLUSH_EVENT_THRESHOLD = 50;
|
|
26306
26682
|
/** Build-time cdkd version, with a dev fallback for non-built contexts. */
|
|
26307
26683
|
function getCdkdVersion() {
|
|
26308
|
-
return "0.284.
|
|
26684
|
+
return "0.284.42";
|
|
26309
26685
|
}
|
|
26310
26686
|
/**
|
|
26311
26687
|
* Generate a time-sortable unique run id, e.g.
|
|
@@ -28928,4 +29304,4 @@ var DeployEngine = class {
|
|
|
28928
29304
|
|
|
28929
29305
|
//#endregion
|
|
28930
29306
|
export { startInterruptWatch as $, CdkdError as $n, WorkGraph as $t, renderStatefulReason as A, resolveStateBucketWithDefault as An, redactSecretsForState as At, exportAliasCollisionScrubWarning as B, expectedOwnerParam as Bn, DiffCalculator as Bt, isFinalSnapshotError as C, synthesisStatusMessage as Cn, isMarkedNonRetryable as Cr, TEMPLATE_SOURCED_RULES as Ct, extractDeploymentEventError as D, resolveAutoAssetStorage as Dn, __exportAll as Dr, isSingleDynamicReferenceToken as Dt, makeCanonicalizePropertiesFn as E, resolveApp as En, markNonRetryable as Er, errorCauseChain as Et, green as F, CFN_TEMPLATE_BODY_LIMIT as Fn, s3BucketDomainName as Ft, collectInlinePolicyNamesManagedBySiblings as G, processStackMessages as Gn, TemplateParser as Gt, secretBearingStateKeyWarning as H, canonicalizeRegion as Hn, describeTypeWithThrottleRetry as Ht, red as I, CFN_TEMPLATE_URL_LIMIT as In, s3BucketDualStackDomainName as It, findActionableSilentDrops as J, AwsClients as Jn, S3StateBackend as Jt, clearOnUpdateRemoval as K, clearBucketRegionCache as Kn, LockManager as Kt, yellow as L, MIGRATE_TMP_PREFIX as Ln, s3BucketRegionalDomainName as Lt, bold as M, resolveUseCdkBootstrapAssets as Mn, classifyReplaySecretRegion as Mt, cyan as N, stateBucketExistenceConfirmed as Nn, producerRegionsFromState as Nt, MULTI_REGION_RECREATE_BLOCKED_TYPES as O, resolveCaptureObservedState as On, maskSecretsInError as Ot, gray as P, warnDeprecatedNoPrefixCliFlag as Pn, s3BucketArn as Pt, isInterruptedWaitError as Q, AssetError as Qn, stringifyValue as Qt, collectDeclaredOutputNames as R, findLargeInlineResources as Rn, s3BucketWebsiteUrl as Rt, createPreDeleteFinalSnapshot as S, Synthesizer as Sn, withErrorHandling as Sr, STATE_SOURCED_READBACK_RULES as St, unsupportedFinalSnapshotError as T, getLegacyStateBucketName as Tn, isThrottlingError as Tr, dynamicReferenceTokens as Tt, stateKeySecretExposure as U, derivePartitionAndUrlSuffix as Un, withRetry as Ut, isExportAliasCollision as V, PARTITION_TABLE as Vn, INTRINSIC_KEYS as Vt, IAMRoleProvider as W, AssemblyReader as Wn, DagBuilder as Wt, beginCommandInterruptScope as X, resetAwsClients as Xn, shouldRetainResource as Xt, findSilentDropProperties as Y, getAwsClients as Yn, rebuildClientForBucketRegion as Yt, endCommandInterruptScope as Z, setAwsClients as Zn, AssetPublisher as Zt, computeImplicitDeleteEdges as _, getDockerCmd as _n, StateError as _r, replayWarn as _t, DeploymentEventsStore as a, stripControlChars as an, LocalInvokeBuildError as ar, carriesDynamicReference as at, buildFinalSnapshotIdentifier as b, AssetManifestLoader as bn, isCdkdError as br, requireConfigString as bt, replayFailedOperations as c, ensureAssetStorage as cn, LockError as cr, refStateLookupFromResource as ct, updatePartialReason as d, readBootstrapMarkerBody as dn, PartialFailureError as dr, resolveExplicitPhysicalId as dt, buildAssetRedirectMap as en, ConfigError as er, CloudControlProvider as et, UNSPECIFIED_SKIP_REASON as f, validateAssetBucketName as fn, ProvisioningError as fr, assertRegionMatch as ft, IMPLICIT_DELETE_DEPENDENCIES as g, formatDockerLoginError as gn, StackTerminationProtectionError as gr, readConfigString as gt, maskingRetryLogger as h, buildDockerImage as hn, StackHasActiveImportsError as hr, configStringRefusal as ht, DeploymentEventsReader as i, escapeRegExp$1 as in, DynamicReferenceRegionAmbiguousError as ir, IntrinsicFunctionResolver as it, formatResourceLine as j, resolveStateBucketWithDefaultAndSource as jn, scrubResourceRecord as jt, isStatefulRecreateTargetSync as k, resolveSkipPrefix as kn, maskSecretsInText as kt, replayRollback as l, getBootstrapMarkerKey as ln, MissingCdkCliError as lr, WAFv2WebACLProvider as lt, withResourceDeadline as m, buildDenyExternalAccessPolicy as mn, ResourceUpdateNotSupportedError as mr, configBooleanRefusal as mt, DEFAULT_RESOURCE_WARN_AFTER_MS as n, loadPublishableAssetManifest as nn, DependencyError as nr, disableInstanceApiTermination as nt, planFailedOps as o, AssetModeResolver as on, LocalMigrateError as or, cfnRefValueFromPhysicalId as ot, deleteSkipReason as p, validateContainerRepoName as pn, ResourceTimeoutError as pr, coerceCfnBoolean as pt, ProviderRegistry as q, resolveBucketRegion as qn, displaySafe as qt, DeployEngine as r, rewriteTemplateAssetReferences as rn, DeployCancelledError as rr, isTerminationProtectionPropagationError as rt, planRollback as s, BOOTSTRAP_MARKER_PREFIX as sn, LocalStartServiceError as sr, getAccountInfo as st, DEFAULT_RESOURCE_TIMEOUT_MS as t, createAssetRedirectResolver as tn, CrossAccountSecretRefusalError as tr, slowCcOperationTimeoutMs as tt, updatePartialMessage as u, parseBootstrapMarker as un, NestedStackChildDirectDestroyError as ur, normalizeAwsTagsToCfn as ut, ATOMIC_FINAL_SNAPSHOT_TYPES as v, runDockerForeground as vn, SynthesisError as vr, requireConfigArray as vt, refusesFinalSnapshot as w, getDefaultStateBucketName as wn, isRetryableTransientError as wr, createSecretMasker as wt, ccRoutedFinalSnapshotError as x, getDockerImageBySourceHash as xn, normalizeAwsError as xr, STATE_SOURCED_CROSS_GENERATION_RULES as xt, PRE_DELETE_SNAPSHOT_TYPES as y, runDockerStreaming as yn, formatError as yr, requireConfigObject as yt, collectPublishedOutputNames as z, uploadCfnTemplate as zn, applyRoleArnIfSet as zt };
|
|
28931
|
-
//# sourceMappingURL=deploy-engine-
|
|
29307
|
+
//# sourceMappingURL=deploy-engine-8ygMMbQ8.js.map
|