@go-to-k/cdkd 0.284.41 → 0.284.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7594,6 +7594,26 @@ function displaySafe(value, opts) {
7594
7594
  //#endregion
7595
7595
  //#region src/state/lock-manager.ts
7596
7596
  /**
7597
+ * Upper bound on the gap between lock renewals.
7598
+ *
7599
+ * Deliberately a fixed wall-clock ceiling rather than a pure fraction of the
7600
+ * TTL. A fraction alone (TTL/4 = 7.5 min at the default) makes renewal
7601
+ * unobservable in any deploy shorter than that, so nothing short of a
7602
+ * multi-hour fixture could ever prove the loop runs -- and an untestable
7603
+ * heartbeat is one that goes dead silently. At 2 min the default 30 min TTL
7604
+ * tolerates FOURTEEN consecutive missed renewals before it lapses, and an
7605
+ * ordinary broad-set integ (~8 min) performs three.
7606
+ */
7607
+ const MAX_RENEWAL_INTERVAL_MS = 120 * 1e3;
7608
+ /**
7609
+ * Renew after at most a quarter of the TTL, so a short TTL still gets three
7610
+ * chances to renew before it lapses. This is what binds the two numbers
7611
+ * together for a caller that shortens `ttlMinutes`.
7612
+ */
7613
+ const RENEWAL_TTL_FRACTION = 4;
7614
+ /** Floor, so a pathologically small TTL cannot spin the event loop. */
7615
+ const MIN_RENEWAL_INTERVAL_MS = 1e3;
7616
+ /**
7597
7617
  * S3-based lock manager using conditional writes (If-None-Match)
7598
7618
  *
7599
7619
  * Implements distributed locking using S3's If-None-Match: "*" condition
@@ -7615,13 +7635,20 @@ var LockManager = class {
7615
7635
  s3Client;
7616
7636
  config;
7617
7637
  ttlMs;
7638
+ renewalIntervalMs;
7639
+ renewalDisabled;
7640
+ /** Locks held by THIS process, keyed by S3 lock key. */
7641
+ heldLocks = /* @__PURE__ */ new Map();
7618
7642
  clientResolved = false;
7619
7643
  resolveInFlight = null;
7620
7644
  constructor(s3Client, config, options) {
7621
7645
  this.s3Client = s3Client;
7622
7646
  this.config = config;
7623
7647
  const ttlMinutes = options?.ttlMinutes ?? 30;
7648
+ if (!(ttlMinutes > 0) || !Number.isFinite(ttlMinutes)) throw new LockError(`Lock TTL must be a positive number of minutes (got ${String(ttlMinutes)}). A non-positive TTL makes every lock born expired, and the renewal interval clamps to its floor, so the heartbeat would spin one request per second for the life of the process.`);
7624
7649
  this.ttlMs = ttlMinutes * 60 * 1e3;
7650
+ this.renewalDisabled = options?.disableRenewal === true;
7651
+ this.renewalIntervalMs = Math.max(MIN_RENEWAL_INTERVAL_MS, Math.min(MAX_RENEWAL_INTERVAL_MS, Math.floor(this.ttlMs / RENEWAL_TTL_FRACTION)));
7625
7652
  }
7626
7653
  /**
7627
7654
  * Resolve the state bucket's actual region and, if it differs from the
@@ -7702,6 +7729,7 @@ var LockManager = class {
7702
7729
  * Check if a lock is expired based on its expiresAt field
7703
7730
  */
7704
7731
  isLockExpired(lockInfo) {
7732
+ if (!Number.isFinite(lockInfo.expiresAt)) return true;
7705
7733
  return Date.now() >= lockInfo.expiresAt;
7706
7734
  }
7707
7735
  /**
@@ -7736,40 +7764,50 @@ var LockManager = class {
7736
7764
  };
7737
7765
  try {
7738
7766
  this.logger.debug(`Attempting to acquire lock for stack: ${stackName} (${region})`);
7739
- const lockBody = JSON.stringify(lockInfo, null, 2);
7740
- await this.s3Client.send(new PutObjectCommand({
7741
- Bucket: this.config.bucket,
7742
- ...await this.ownerParam(),
7743
- Key: key,
7744
- Body: lockBody,
7745
- ContentLength: Buffer.byteLength(lockBody),
7746
- ContentType: "application/json",
7747
- IfNoneMatch: "*"
7748
- }));
7767
+ const etag = await this.putLockObject(key, lockInfo);
7749
7768
  this.logger.debug(`Lock acquired for stack: ${stackName} (${region}), owner: ${lockOwner}`);
7769
+ this.trackHeldLock({
7770
+ stackName,
7771
+ region,
7772
+ key,
7773
+ info: lockInfo,
7774
+ etag
7775
+ });
7750
7776
  return true;
7751
7777
  } catch (error) {
7752
- if (error instanceof S3ServiceException && error.name === "PreconditionFailed") {
7778
+ if (this.isForeignLockError(error)) {
7753
7779
  this.logger.debug(`Lock already exists for stack: ${stackName} (${region})`);
7754
- const existingLock = await this.getLockInfo(stackName, region);
7755
- if (existingLock && this.isLockExpired(existingLock)) {
7756
- this.logger.info(`Expired lock detected for stack: ${stackName} (${region}, owner: ${existingLock.owner}, expired ${this.formatDuration(now - existingLock.expiresAt)} ago). Cleaning up...`);
7757
- await this.deleteLock(stackName, region);
7780
+ const existing = await this.getLockRecord(stackName, region);
7781
+ if (existing && this.isLockExpired(existing.info)) {
7782
+ if (existing.etag === void 0) {
7783
+ this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): its current version could not be identified, so removing it could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
7784
+ return false;
7785
+ }
7758
7786
  try {
7759
- const retryBody = JSON.stringify(lockInfo, null, 2);
7760
- await this.s3Client.send(new PutObjectCommand({
7761
- Bucket: this.config.bucket,
7762
- ...await this.ownerParam(),
7763
- Key: key,
7764
- Body: retryBody,
7765
- ContentLength: Buffer.byteLength(retryBody),
7766
- ContentType: "application/json",
7767
- IfNoneMatch: "*"
7768
- }));
7787
+ await this.deleteLock(stackName, region, existing.etag);
7788
+ } catch (deleteError) {
7789
+ if (this.isConditionUnsupportedError(deleteError)) {
7790
+ this.logger.warn(`Cannot take over the expired lock for stack '${stackName}' (${region}): this endpoint or policy will not evaluate a conditional delete, and removing it unconditionally could delete a lock another process has since taken. Clear it with cdkd force-unlock.`);
7791
+ return false;
7792
+ } else if (this.isNotOursError(deleteError)) {
7793
+ this.logger.debug(`Expired lock for stack ${stackName} (${region}) changed before takeover; treating as contended`);
7794
+ return false;
7795
+ } else throw deleteError;
7796
+ }
7797
+ this.logger.warn(`Took over an EXPIRED lock for stack: ${stackName} (${region}, owner: ${existing.info.owner}, expired ${this.formatDuration(now - existing.info.expiresAt)} ago). A live cdkd process renews its lock well inside the TTL, so this normally means the previous owner crashed or was suspended -- if it is in fact still running, both processes are now writing to the same stack.`);
7798
+ try {
7799
+ const retryEtag = await this.putLockObject(key, lockInfo);
7769
7800
  this.logger.debug(`Lock acquired for stack: ${stackName} (${region}) after expired lock cleanup, owner: ${lockOwner}`);
7801
+ this.trackHeldLock({
7802
+ stackName,
7803
+ region,
7804
+ key,
7805
+ info: lockInfo,
7806
+ etag: retryEtag
7807
+ });
7770
7808
  return true;
7771
7809
  } catch (retryError) {
7772
- if (retryError instanceof S3ServiceException && retryError.name === "PreconditionFailed") {
7810
+ if (this.isForeignLockError(retryError)) {
7773
7811
  this.logger.debug(`Lock was acquired by another process during expired lock cleanup for stack: ${stackName} (${region})`);
7774
7812
  return false;
7775
7813
  }
@@ -7782,6 +7820,24 @@ var LockManager = class {
7782
7820
  }
7783
7821
  }
7784
7822
  /**
7823
+ * Write the lock object and return the ETag S3 assigned it.
7824
+ *
7825
+ * `condition` defaults to `IfNoneMatch: '*'` (acquisition -- succeed only if
7826
+ * no current version exists). Renewal passes `IfMatch: <etag>` instead.
7827
+ */
7828
+ async putLockObject(key, lockInfo, condition = { IfNoneMatch: "*" }) {
7829
+ const body = JSON.stringify(lockInfo, null, 2);
7830
+ return (await this.s3Client.send(new PutObjectCommand({
7831
+ Bucket: this.config.bucket,
7832
+ ...await this.ownerParam(),
7833
+ Key: key,
7834
+ Body: body,
7835
+ ContentLength: Buffer.byteLength(body),
7836
+ ContentType: "application/json",
7837
+ ...condition
7838
+ }))).ETag;
7839
+ }
7840
+ /**
7785
7841
  * Get current lock information.
7786
7842
  *
7787
7843
  * `region` is required for the new region-scoped lock layout. Pass
@@ -7789,6 +7845,18 @@ var LockManager = class {
7789
7845
  * file (e.g. for state-listing tools that don't yet know the region).
7790
7846
  */
7791
7847
  async getLockInfo(stackName, region) {
7848
+ const record = await this.getLockRecord(stackName, region);
7849
+ return record ? record.info : null;
7850
+ }
7851
+ /**
7852
+ * `getLockInfo` plus the S3 ETag of the object the info came from.
7853
+ *
7854
+ * Internal because the ETag is only meaningful to code that then issues a
7855
+ * CONDITIONAL write or delete against the exact bytes it read (issue #2168):
7856
+ * the expired-lock takeover in `acquireLock`. Every public reader wants the
7857
+ * body alone.
7858
+ */
7859
+ async getLockRecord(stackName, region) {
7792
7860
  await this.ensureClientForBucket();
7793
7861
  const key = this.getLockKey(stackName, region);
7794
7862
  try {
@@ -7812,7 +7880,10 @@ var LockManager = class {
7812
7880
  ...parsed.operation !== void 0 && { operation: displaySafe(parsed.operation) }
7813
7881
  };
7814
7882
  this.logger.debug(`Lock info for stack: ${stackName}:`, lockInfo);
7815
- return lockInfo;
7883
+ return {
7884
+ info: lockInfo,
7885
+ etag: response.ETag
7886
+ };
7816
7887
  } catch (error) {
7817
7888
  if (error instanceof NoSuchKey) {
7818
7889
  this.logger.debug(`No lock exists for stack: ${stackName}`);
@@ -7834,24 +7905,137 @@ var LockManager = class {
7834
7905
  return await this.getLockInfo(stackName, region) !== null;
7835
7906
  }
7836
7907
  /**
7837
- * Release a lock for a stack
7908
+ * Release a lock for a stack.
7909
+ *
7910
+ * The DELETE is CONDITIONAL on the ETag this process last wrote (issue
7911
+ * #2168). Before that it was an owner-blind unconditional delete, which is
7912
+ * what turned a single lost lock into a cascade: an operation that outlived
7913
+ * its TTL had its lock taken over by a second process, then deleted the
7914
+ * SECOND process's lock on its way out, freeing the stack for a third.
7915
+ *
7916
+ * A `PreconditionFailed` therefore means "the object here is not the one I
7917
+ * wrote", and the correct response is to leave it alone -- not to raise, as
7918
+ * the operation itself has already finished and its caller has nothing to do
7919
+ * about it.
7920
+ *
7921
+ * The condition is dropped for exactly ONE class of failure: the endpoint or
7922
+ * the policy will not EVALUATE it (403 / 501 -- see
7923
+ * `isConditionUnsupportedError`), and even then only after an ownership
7924
+ * re-check. Everything else RAISES, as this method always has. In particular
7925
+ * a 409 (S3's answer to a concurrent operation on the key) and a 503 are not
7926
+ * fallback-worthy: the first IS the contended case, and the second may mean
7927
+ * the delete already landed with the response lost. The heartbeat is stopped
7928
+ * by then, so the worst outcome of raising is a lock that lapses at its TTL
7929
+ * -- recoverable, unlike one deleted out from under a live writer.
7930
+ *
7931
+ * Callers must therefore tolerate a throw here, and all sixteen do: four
7932
+ * wrap it in `try`/`catch` (`deploy-engine.ts` plus three in
7933
+ * `destroy-runner.ts`) and twelve attach a `.catch()`: a
7934
+ * failed release is a warning, never the error a command reports.
7838
7935
  */
7839
7936
  async releaseLock(stackName, region) {
7840
- await this.ensureClientForBucket();
7841
7937
  const key = this.getLockKey(stackName, region);
7938
+ const held = this.heldLocks.get(key);
7939
+ if (held?.releasing) {
7940
+ this.logger.debug(`Release already in flight (or done) for stack ${stackName} (${region})`);
7941
+ return held.releasing;
7942
+ }
7943
+ if (!held) return this.doReleaseLock(stackName, region, void 0);
7944
+ this.stopRenewal(held);
7945
+ const releasing = this.doReleaseLock(stackName, region, held);
7946
+ held.releasing = releasing;
7947
+ return releasing;
7948
+ }
7949
+ async doReleaseLock(stackName, region, held) {
7950
+ await this.ensureClientForBucket();
7951
+ if (held) await held.renewing?.catch(() => void 0);
7952
+ if (held?.lost) {
7953
+ this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process lost it while the operation was still running, so the lock present now belongs to someone else.`);
7954
+ return;
7955
+ }
7956
+ if (held && held.etag === void 0 && !await this.stillOursByBody(held)) {
7957
+ this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): this process never learned which version of the lock it wrote, and could not confirm the one present is its own. It expires on its own, or clear it with cdkd force-unlock.`);
7958
+ return;
7959
+ }
7842
7960
  try {
7843
7961
  this.logger.debug(`Releasing lock for stack: ${stackName} (${region})`);
7844
- await this.s3Client.send(new DeleteObjectCommand({
7845
- Bucket: this.config.bucket,
7846
- ...await this.ownerParam(),
7847
- Key: key
7848
- }));
7962
+ await this.deleteLock(stackName, region, held?.etag);
7849
7963
  this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
7850
7964
  } catch (error) {
7965
+ if (this.isForeignLockError(error)) {
7966
+ this.logger.warn(held?.etagUncertain ? `Not releasing the lock for stack '${stackName}' (${region}): this process could not confirm which version of the lock it last wrote, so it will not delete one it may not own. The lock clears on its own at ${new Date(held.info.expiresAt).toISOString()}, or immediately with cdkd force-unlock.` : `Not releasing the lock for stack '${stackName}' (${region}): it has been replaced since this process acquired it, so another cdkd process now holds it. Leaving it in place.`);
7967
+ return;
7968
+ }
7969
+ if (this.isGoneError(error)) {
7970
+ this.logger.debug(`Lock for stack ${stackName} (${region}) was already gone`);
7971
+ return;
7972
+ }
7973
+ if (held?.etag !== void 0 && this.isConditionUnsupportedError(error)) {
7974
+ if (!await this.stillOursByBody(held)) {
7975
+ this.logger.warn(`Not releasing the lock for stack '${stackName}' (${region}): the conditional delete could not be evaluated here, and this process could not confirm the lock is still its own. Leaving it in place rather than risk deleting another process's lock. It expires on its own, or clear it with cdkd force-unlock. If this repeats, grant s3:GetObject on the lock key -- a delete conditioned on an ETag requires it.`);
7976
+ return;
7977
+ }
7978
+ this.logger.debug(`Conditional lock release for stack ${stackName} (${region}) is not supported here (${error instanceof Error ? error.message : String(error)}); retrying unconditionally`);
7979
+ try {
7980
+ await this.deleteLock(stackName, region);
7981
+ this.logger.debug(`Lock released for stack: ${stackName} (${region})`);
7982
+ return;
7983
+ } catch (fallbackError) {
7984
+ throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${fallbackError instanceof Error ? fallbackError.message : String(fallbackError)}`, fallbackError instanceof Error ? fallbackError : void 0);
7985
+ }
7986
+ }
7851
7987
  throw new LockError(`Failed to release lock for stack '${stackName}' (${region}): ${error instanceof Error ? error.message : String(error)}`, error instanceof Error ? error : void 0);
7852
7988
  }
7853
7989
  }
7854
7990
  /**
7991
+ * Is the stored lock still the one this process acquired?
7992
+ *
7993
+ * Used only where the ETag condition could not be EVALUATED, to decide
7994
+ * whether dropping it is safe. Two earlier cuts of this were wrong in
7995
+ * opposite directions and both are worth stating, because the shape recurs:
7996
+ *
7997
+ * - It short-circuited on `Date.now() < held.info.expiresAt`, reasoning that
7998
+ * nobody could have taken over while our own deadline was ahead. But
7999
+ * `forceReleaseLock` deletes regardless of expiry -- that is its whole
8000
+ * contract -- so a user running `cdkd force-unlock` mid-operation is a
8001
+ * LEGITIMATE takeover the shortcut cannot see. Cross-machine clock skew
8002
+ * reaches the same state without anyone running anything. The shortcut is
8003
+ * gone; the read costs one GetObject on an already-failed path.
8004
+ *
8005
+ * - It answered TRUE when the read FAILED, to avoid stranding a lock. That
8006
+ * made it inert in exactly the case it exists for: the documented trigger
8007
+ * for the 403 is a policy granting `s3:DeleteObject` without the
8008
+ * `s3:GetObject` a conditional delete needs -- under which this read fails
8009
+ * too. So the guard would wave through precisely the situation it was
8010
+ * added to catch. A failed read now REFUSES.
8011
+ *
8012
+ * The remaining asymmetry is deliberate and is this method's stated rule: a
8013
+ * stranded lock is bounded by the TTL and clearable with `force-unlock`,
8014
+ * while a lock deleted out from under a live writer is neither.
8015
+ */
8016
+ async stillOursByBody(held) {
8017
+ let record;
8018
+ try {
8019
+ record = await this.getLockRecord(held.stackName, held.region);
8020
+ } catch {
8021
+ return false;
8022
+ }
8023
+ if (!record) return false;
8024
+ return this.sameLockIdentity(record.info, held.info);
8025
+ }
8026
+ /**
8027
+ * Whether two lock bodies name the same acquisition.
8028
+ *
8029
+ * `owner` is compared through `displaySafe` on BOTH sides: a body read back
8030
+ * has been sanitized by `getLockRecord` while the one this process
8031
+ * constructed has not, so an unsanitized comparison would never match for a
8032
+ * `$USER` or hostname containing a stripped codepoint -- and the process
8033
+ * would then disown its own lock.
8034
+ */
8035
+ sameLockIdentity(a, b) {
8036
+ return displaySafe(a.owner) === displaySafe(b.owner) && a.timestamp === b.timestamp;
8037
+ }
8038
+ /**
7855
8039
  * Force release a lock regardless of owner or expiry status
7856
8040
  *
7857
8041
  * This is intended for CLI usage (e.g., --force-unlock flag) when a lock
@@ -7864,21 +8048,213 @@ var LockManager = class {
7864
8048
  const where = `${stackName}${region ? ` (${region})` : ""}`;
7865
8049
  const lockInfo = await this.getLockInfo(stackName, region).catch(() => null);
7866
8050
  this.logger.warn(lockInfo ? `Force releasing lock for stack: ${where}, owner: ${lockInfo.owner}${lockInfo.operation ? `, operation: ${lockInfo.operation}` : ""}, expired: ${this.isLockExpired(lockInfo)}` : `Force releasing lock for stack: ${where} (no lock body read — absent or unparseable; deleting the object either way, since a lock cdkd cannot read is a lock nothing else can clear)`);
8051
+ const held = this.heldLocks.get(this.getLockKey(stackName, region));
8052
+ if (held) {
8053
+ this.stopRenewal(held);
8054
+ held.releasing = Promise.resolve();
8055
+ }
7867
8056
  await this.deleteLock(stackName, region);
7868
8057
  }
7869
8058
  /**
7870
- * Internal method to delete the lock file from S3
8059
+ * Internal method to delete the lock file from S3.
8060
+ *
8061
+ * When `etag` is supplied the delete is conditional (`IfMatch`), so it
8062
+ * removes the object ONLY while it is still byte-identical to the one the
8063
+ * caller read or wrote. S3 evaluates the condition against the CURRENT
8064
+ * version, which is the right unit here even on the versioned state bucket:
8065
+ * the current version IS the live lock.
7871
8066
  */
7872
- async deleteLock(stackName, region) {
8067
+ async deleteLock(stackName, region, etag) {
7873
8068
  await this.ensureClientForBucket();
7874
8069
  const key = this.getLockKey(stackName, region);
7875
8070
  await this.s3Client.send(new DeleteObjectCommand({
7876
8071
  Bucket: this.config.bucket,
7877
8072
  ...await this.ownerParam(),
7878
- Key: key
8073
+ Key: key,
8074
+ ...etag !== void 0 && { IfMatch: etag }
7879
8075
  }));
7880
8076
  }
7881
8077
  /**
8078
+ * Record a lock this process now holds and start renewing it.
8079
+ */
8080
+ trackHeldLock(args) {
8081
+ const previous = this.heldLocks.get(args.key);
8082
+ if (previous) this.stopRenewal(previous);
8083
+ const held = {
8084
+ stackName: args.stackName,
8085
+ region: args.region,
8086
+ key: args.key,
8087
+ info: args.info,
8088
+ etag: args.etag,
8089
+ timer: void 0,
8090
+ renewing: void 0,
8091
+ lost: false,
8092
+ releasing: void 0,
8093
+ etagUncertain: false,
8094
+ warnedPastExpiry: false
8095
+ };
8096
+ this.heldLocks.set(args.key, held);
8097
+ if (this.renewalDisabled) {
8098
+ this.logger.debug(`Lock renewal is disabled; the lock for ${args.stackName} (${args.region}) will lapse at its TTL`);
8099
+ return;
8100
+ }
8101
+ if (args.etag === void 0) {
8102
+ this.logger.debug(`No ETag returned when acquiring the lock for ${args.stackName} (${args.region}); renewal disabled for this lock`);
8103
+ return;
8104
+ }
8105
+ this.startRenewal(held);
8106
+ }
8107
+ startRenewal(held) {
8108
+ const timer = setInterval(() => {
8109
+ this.renewLock(held);
8110
+ }, this.renewalIntervalMs);
8111
+ timer.unref?.();
8112
+ held.timer = timer;
8113
+ }
8114
+ stopRenewal(held) {
8115
+ if (held.timer !== void 0) {
8116
+ clearInterval(held.timer);
8117
+ held.timer = void 0;
8118
+ }
8119
+ }
8120
+ /**
8121
+ * Push the lock's `expiresAt` forward, conditional on still owning it.
8122
+ *
8123
+ * This is what makes the TTL mean "the owner has been silent for TTL"
8124
+ * instead of "the operation has been running for TTL". Before issue #2168
8125
+ * there was no renewal at all, so any operation slower than the TTL -- an
8126
+ * FSx or EMR resource waiting up to an hour, or simply a large stack --
8127
+ * had its lock treated as free by the next process while it was still
8128
+ * running.
8129
+ */
8130
+ renewLock(held) {
8131
+ if (held.lost || held.renewing !== void 0 || held.etag === void 0) return Promise.resolve();
8132
+ const inFlight = this.doRenewLock(held).finally(() => {
8133
+ held.renewing = void 0;
8134
+ });
8135
+ held.renewing = inFlight;
8136
+ return inFlight;
8137
+ }
8138
+ async doRenewLock(held) {
8139
+ const currentEtag = held.etag;
8140
+ if (currentEtag === void 0) return;
8141
+ const renewed = {
8142
+ ...held.info,
8143
+ expiresAt: Date.now() + this.ttlMs
8144
+ };
8145
+ try {
8146
+ const etag = await this.putLockObject(held.key, renewed, { IfMatch: currentEtag });
8147
+ if (etag === void 0) {
8148
+ if (await this.adoptOwnWrite(held, renewed)) return;
8149
+ held.info = renewed;
8150
+ held.etagUncertain = true;
8151
+ this.stopRenewal(held);
8152
+ this.logger.debug(`Lock renewal for ${held.stackName} (${held.region}) returned no ETag and the object could not be re-read; stopping renewal and keeping the previous ETag`);
8153
+ return;
8154
+ }
8155
+ held.info = renewed;
8156
+ held.etag = etag;
8157
+ held.warnedPastExpiry = false;
8158
+ this.logger.debug(`Renewed lock for stack: ${held.stackName} (${held.region}) until ${new Date(renewed.expiresAt).toISOString()}`);
8159
+ } catch (error) {
8160
+ if (this.isForeignLockError(error) && await this.adoptOwnWrite(held, renewed)) return;
8161
+ if (this.isNotOursError(error)) {
8162
+ held.lost = true;
8163
+ this.stopRenewal(held);
8164
+ this.logger.warn(`Lost the lock for stack '${held.stackName}' (${held.region}) while the operation was still running: the lock object has been replaced or removed. Another cdkd process may now be writing to this stack concurrently. This process will not delete the current lock when it finishes.`);
8165
+ return;
8166
+ }
8167
+ this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) failed, will retry: ${error instanceof Error ? error.message : String(error)}`);
8168
+ if (!held.warnedPastExpiry && Date.now() >= held.info.expiresAt) {
8169
+ held.warnedPastExpiry = true;
8170
+ this.logger.warn(`Lock renewal for stack '${held.stackName}' (${held.region}) has been failing long enough that the lock expired at ${new Date(held.info.expiresAt).toISOString()}. Another cdkd process can now take it while this operation is still running. Renewal keeps retrying.`);
8171
+ }
8172
+ }
8173
+ }
8174
+ /**
8175
+ * Recover from a 412 that this process itself caused.
8176
+ *
8177
+ * A conditional PUT that S3 APPLIED but whose response never arrived -- a
8178
+ * dropped connection, or an SDK-internal retry of the same request -- leaves
8179
+ * `held.etag` pointing at the PREVIOUS version while the object already
8180
+ * holds the renewal. The retry then carries the stale `IfMatch` and S3
8181
+ * correctly answers 412. Reading that as "someone took my lock" is wrong in
8182
+ * both directions: it stops the heartbeat on a lock this process still owns,
8183
+ * it prints a warning naming a concurrent writer that does not exist, and
8184
+ * `releaseLock` then refuses to remove the process's OWN lock -- stranding
8185
+ * the stack for up to a full TTL, which is strictly worse than the
8186
+ * unconditional delete this change replaced.
8187
+ *
8188
+ * So a 412 is disambiguated by READING the object once. `renewed` is a body
8189
+ * only this process could have produced: its `owner` identifies the process
8190
+ * and its `expiresAt` is a millisecond timestamp this specific PUT chose. An
8191
+ * exact match on both (plus `timestamp`, which is fixed at acquisition) means
8192
+ * the write landed and only the answer was lost, so the renewal is adopted
8193
+ * rather than mourned.
8194
+ *
8195
+ * The read is on the 412 path only, which is rare. If it fails, the caller
8196
+ * falls through to the pessimistic branch -- an unreadable lock is not
8197
+ * evidence of ownership.
8198
+ */
8199
+ async adoptOwnWrite(held, renewed) {
8200
+ const record = await this.getLockRecord(held.stackName, held.region).catch(() => null);
8201
+ if (!record || record.etag === void 0 || record.info.expiresAt !== renewed.expiresAt || !this.sameLockIdentity(record.info, renewed)) return false;
8202
+ held.info = renewed;
8203
+ held.etag = record.etag;
8204
+ held.warnedPastExpiry = false;
8205
+ this.logger.debug(`Lock renewal for stack ${held.stackName} (${held.region}) reported a conflict, but the object holds this process's own renewal -- adopting it and continuing`);
8206
+ return true;
8207
+ }
8208
+ /**
8209
+ * S3 told us the object is not the one we wrote (someone else replaced it).
8210
+ *
8211
+ * Matched on the status as well as the name: a 412 arriving as a bare
8212
+ * `S3ServiceException`, or under a renamed code, would otherwise skip the
8213
+ * "leave it alone" branch and reach the destructive fallback below it.
8214
+ */
8215
+ isForeignLockError(error) {
8216
+ return error instanceof S3ServiceException && (error.name === "PreconditionFailed" || error.$metadata.httpStatusCode === 412);
8217
+ }
8218
+ /**
8219
+ * S3 told us this OBJECT is not there at all.
8220
+ *
8221
+ * `NoSuchBucket` is also a 404 and must NOT count: it says nothing about the
8222
+ * lock. Read as "gone" it would make `releaseLock` resolve where it used to
8223
+ * raise, and -- worse -- make one bucket-level 404 during renewal set
8224
+ * `lost`, killing the heartbeat and refusing to release a lock this process
8225
+ * still holds.
8226
+ */
8227
+ isGoneError(error) {
8228
+ if (error instanceof NoSuchKey) return true;
8229
+ if (!(error instanceof S3ServiceException)) return false;
8230
+ if (error.name === "NoSuchBucket") return false;
8231
+ return error.name === "NoSuchKey" || error.$metadata.httpStatusCode === 404;
8232
+ }
8233
+ /**
8234
+ * The endpoint or the policy will not evaluate a conditional delete at all.
8235
+ *
8236
+ * Deliberately narrow, because this is the one predicate that authorises
8237
+ * dropping the ownership check. A conditional delete with a specific ETag
8238
+ * additionally requires `s3:GetObject` (AWS documents this for its
8239
+ * enforce-conditional-deletes bucket policy), so a policy granting only
8240
+ * `s3:DeleteObject` answers 403 -- and an S3-compatible endpoint that has
8241
+ * not implemented the header answers 501. Neither says anything about who
8242
+ * owns the lock, whereas 409 / 412 / 5xx all do.
8243
+ */
8244
+ isConditionUnsupportedError(error) {
8245
+ if (!(error instanceof S3ServiceException)) return false;
8246
+ const status = error.$metadata.httpStatusCode;
8247
+ return status === 403 || status === 501 || error.name === "AccessDenied" || error.name === "Forbidden" || error.name === "NotImplemented";
8248
+ }
8249
+ /**
8250
+ * Either of the two ways S3 can tell us the lock we thought we held is no
8251
+ * longer ours: replaced by someone else (412) or deleted outright (404, e.g.
8252
+ * by `cdkd force-unlock`). Both mean stop renewing and do not delete.
8253
+ */
8254
+ isNotOursError(error) {
8255
+ return this.isForeignLockError(error) || this.isGoneError(error);
8256
+ }
8257
+ /**
7882
8258
  * Acquire lock with retry logic
7883
8259
  *
7884
8260
  * Retries up to maxRetries times with retryDelay between attempts.
@@ -17609,7 +17985,7 @@ var CloudControlProvider = class {
17609
17985
  if (context?.finalSnapshotIdentifier !== void 0) throw new ProvisioningError(`${logicalId} (${resourceType}) requires a final snapshot (DeletionPolicy: Snapshot), but the Cloud Control API delete route has no final-snapshot parameter. Re-run with --skip-final-snapshot after snapshotting manually, or retain the resource.`, resourceType, logicalId, physicalId);
17610
17986
  if (context?.removeProtection === true && resourceType === "AWS::AutoScaling::AutoScalingGroup") {
17611
17987
  this.logger.debug(`Delegating protected AutoScalingGroup ${logicalId} delete to the SDK ASGProvider (Cloud Control cannot force-delete a protected ASG)`);
17612
- const { ASGProvider } = await import("./asg-provider-D0lenV1M.js").then((n) => n.n);
17988
+ const { ASGProvider } = await import("./asg-provider-Dn9oXi9c.js").then((n) => n.n);
17613
17989
  return await new ASGProvider().delete(logicalId, physicalId, resourceType, _properties, context);
17614
17990
  }
17615
17991
  const isProtectedEc2Instance = context?.removeProtection === true && resourceType === "AWS::EC2::Instance";
@@ -26305,7 +26681,7 @@ const FLUSH_INTERVAL_MS = 2e3;
26305
26681
  const FLUSH_EVENT_THRESHOLD = 50;
26306
26682
  /** Build-time cdkd version, with a dev fallback for non-built contexts. */
26307
26683
  function getCdkdVersion() {
26308
- return "0.284.41";
26684
+ return "0.284.42";
26309
26685
  }
26310
26686
  /**
26311
26687
  * Generate a time-sortable unique run id, e.g.
@@ -28928,4 +29304,4 @@ var DeployEngine = class {
28928
29304
 
28929
29305
  //#endregion
28930
29306
  export { startInterruptWatch as $, CdkdError as $n, WorkGraph as $t, renderStatefulReason as A, resolveStateBucketWithDefault as An, redactSecretsForState as At, exportAliasCollisionScrubWarning as B, expectedOwnerParam as Bn, DiffCalculator as Bt, isFinalSnapshotError as C, synthesisStatusMessage as Cn, isMarkedNonRetryable as Cr, TEMPLATE_SOURCED_RULES as Ct, extractDeploymentEventError as D, resolveAutoAssetStorage as Dn, __exportAll as Dr, isSingleDynamicReferenceToken as Dt, makeCanonicalizePropertiesFn as E, resolveApp as En, markNonRetryable as Er, errorCauseChain as Et, green as F, CFN_TEMPLATE_BODY_LIMIT as Fn, s3BucketDomainName as Ft, collectInlinePolicyNamesManagedBySiblings as G, processStackMessages as Gn, TemplateParser as Gt, secretBearingStateKeyWarning as H, canonicalizeRegion as Hn, describeTypeWithThrottleRetry as Ht, red as I, CFN_TEMPLATE_URL_LIMIT as In, s3BucketDualStackDomainName as It, findActionableSilentDrops as J, AwsClients as Jn, S3StateBackend as Jt, clearOnUpdateRemoval as K, clearBucketRegionCache as Kn, LockManager as Kt, yellow as L, MIGRATE_TMP_PREFIX as Ln, s3BucketRegionalDomainName as Lt, bold as M, resolveUseCdkBootstrapAssets as Mn, classifyReplaySecretRegion as Mt, cyan as N, stateBucketExistenceConfirmed as Nn, producerRegionsFromState as Nt, MULTI_REGION_RECREATE_BLOCKED_TYPES as O, resolveCaptureObservedState as On, maskSecretsInError as Ot, gray as P, warnDeprecatedNoPrefixCliFlag as Pn, s3BucketArn as Pt, isInterruptedWaitError as Q, AssetError as Qn, stringifyValue as Qt, collectDeclaredOutputNames as R, findLargeInlineResources as Rn, s3BucketWebsiteUrl as Rt, createPreDeleteFinalSnapshot as S, Synthesizer as Sn, withErrorHandling as Sr, STATE_SOURCED_READBACK_RULES as St, unsupportedFinalSnapshotError as T, getLegacyStateBucketName as Tn, isThrottlingError as Tr, dynamicReferenceTokens as Tt, stateKeySecretExposure as U, derivePartitionAndUrlSuffix as Un, withRetry as Ut, isExportAliasCollision as V, PARTITION_TABLE as Vn, INTRINSIC_KEYS as Vt, IAMRoleProvider as W, AssemblyReader as Wn, DagBuilder as Wt, beginCommandInterruptScope as X, resetAwsClients as Xn, shouldRetainResource as Xt, findSilentDropProperties as Y, getAwsClients as Yn, rebuildClientForBucketRegion as Yt, endCommandInterruptScope as Z, setAwsClients as Zn, AssetPublisher as Zt, computeImplicitDeleteEdges as _, getDockerCmd as _n, StateError as _r, replayWarn as _t, DeploymentEventsStore as a, stripControlChars as an, LocalInvokeBuildError as ar, carriesDynamicReference as at, buildFinalSnapshotIdentifier as b, AssetManifestLoader as bn, isCdkdError as br, requireConfigString as bt, replayFailedOperations as c, ensureAssetStorage as cn, LockError as cr, refStateLookupFromResource as ct, updatePartialReason as d, readBootstrapMarkerBody as dn, PartialFailureError as dr, resolveExplicitPhysicalId as dt, buildAssetRedirectMap as en, ConfigError as er, CloudControlProvider as et, UNSPECIFIED_SKIP_REASON as f, validateAssetBucketName as fn, ProvisioningError as fr, assertRegionMatch as ft, IMPLICIT_DELETE_DEPENDENCIES as g, formatDockerLoginError as gn, StackTerminationProtectionError as gr, readConfigString as gt, maskingRetryLogger as h, buildDockerImage as hn, StackHasActiveImportsError as hr, configStringRefusal as ht, DeploymentEventsReader as i, escapeRegExp$1 as in, DynamicReferenceRegionAmbiguousError as ir, IntrinsicFunctionResolver as it, formatResourceLine as j, resolveStateBucketWithDefaultAndSource as jn, scrubResourceRecord as jt, isStatefulRecreateTargetSync as k, resolveSkipPrefix as kn, maskSecretsInText as kt, replayRollback as l, getBootstrapMarkerKey as ln, MissingCdkCliError as lr, WAFv2WebACLProvider as lt, withResourceDeadline as m, buildDenyExternalAccessPolicy as mn, ResourceUpdateNotSupportedError as mr, configBooleanRefusal as mt, DEFAULT_RESOURCE_WARN_AFTER_MS as n, loadPublishableAssetManifest as nn, DependencyError as nr, disableInstanceApiTermination as nt, planFailedOps as o, AssetModeResolver as on, LocalMigrateError as or, cfnRefValueFromPhysicalId as ot, deleteSkipReason as p, validateContainerRepoName as pn, ResourceTimeoutError as pr, coerceCfnBoolean as pt, ProviderRegistry as q, resolveBucketRegion as qn, displaySafe as qt, DeployEngine as r, rewriteTemplateAssetReferences as rn, DeployCancelledError as rr, isTerminationProtectionPropagationError as rt, planRollback as s, BOOTSTRAP_MARKER_PREFIX as sn, LocalStartServiceError as sr, getAccountInfo as st, DEFAULT_RESOURCE_TIMEOUT_MS as t, createAssetRedirectResolver as tn, CrossAccountSecretRefusalError as tr, slowCcOperationTimeoutMs as tt, updatePartialMessage as u, parseBootstrapMarker as un, NestedStackChildDirectDestroyError as ur, normalizeAwsTagsToCfn as ut, ATOMIC_FINAL_SNAPSHOT_TYPES as v, runDockerForeground as vn, SynthesisError as vr, requireConfigArray as vt, refusesFinalSnapshot as w, getDefaultStateBucketName as wn, isRetryableTransientError as wr, createSecretMasker as wt, ccRoutedFinalSnapshotError as x, getDockerImageBySourceHash as xn, normalizeAwsError as xr, STATE_SOURCED_CROSS_GENERATION_RULES as xt, PRE_DELETE_SNAPSHOT_TYPES as y, runDockerStreaming as yn, formatError as yr, requireConfigObject as yt, collectPublishedOutputNames as z, uploadCfnTemplate as zn, applyRoleArnIfSet as zt };
28931
- //# sourceMappingURL=deploy-engine-CCWl_tAH.js.map
29307
+ //# sourceMappingURL=deploy-engine-8ygMMbQ8.js.map