@go-to-k/cdkd 0.284.1 → 0.284.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{asg-provider-DDEsD6uA.js → asg-provider-CMuiarRD.js} +2 -2
- package/dist/{asg-provider-DDEsD6uA.js.map → asg-provider-CMuiarRD.js.map} +1 -1
- package/dist/cli.js +669 -45
- package/dist/cli.js.map +1 -1
- package/dist/{deploy-engine-CruC7Qxn.js → deploy-engine-NER3C-7K.js} +3 -3
- package/dist/{deploy-engine-CruC7Qxn.js.map → deploy-engine-NER3C-7K.js.map} +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { $ as IntrinsicFunctionResolver, $n as StackTerminationProtectionError, $t as buildDenyExternalAccessPolicy, A as formatResourceLine, An as processStackMessages, At as withRetry, B as isExportAliasCollision, Bt as buildAssetRedirectMap, C as refusesFinalSnapshot, Cn as findLargeInlineResources, Ct as s3BucketDualStackDomainName, D as MULTI_REGION_RECREATE_BLOCKED_TYPES, Dn as canonicalizeRegion, Dt as DiffCalculator, E as extractDeploymentEventError, En as PARTITION_TABLE, Et as applyRoleArnIfSet, F as red, Fn as resetAwsClients, Ft as rebuildClientForBucketRegion, G as clearOnUpdateRemoval, Gt as stripControlChars, H as stateKeySecretExposure, Hn as LocalInvokeBuildError$1, Ht as loadPublishableAssetManifest, I as yellow, In as setAwsClients, It as shouldRetainResource, J as findSilentDropProperties, Jn as PartialFailureError, Jt as ensureAssetStorage, K as ProviderRegistry, Kn as MissingCdkCliError, Kt as AssetModeResolver, L as collectDeclaredOutputNames, Lt as AssetPublisher, M as cyan, Mn as resolveBucketRegion, Mt as TemplateParser, N as gray, Nn as AwsClients, Nt as LockManager, O as isStatefulRecreateTargetSync, On as derivePartitionAndUrlSuffix, Ot as INTRINSIC_KEYS, P as green, Pn as getAwsClients, Pt as S3StateBackend, Q as isTerminationProtectionPropagationError, Qn as StackHasActiveImportsError, Qt as validateContainerRepoName, R as collectPublishedOutputNames, Rn as CdkdError, Rt as stringifyValue, S as isFinalSnapshotError, Sn as MIGRATE_TMP_PREFIX, St as s3BucketDomainName, T as makeCanonicalizePropertiesFn, Tn as expectedOwnerParam, Tt as s3BucketWebsiteUrl, U as IAMRoleProvider, Un as LocalMigrateError, Ut as rewriteTemplateAssetReferences, V as secretBearingStateKeyWarning, Vn as DeployCancelledError, Vt as createAssetRedirectResolver, W as collectInlinePolicyNamesManagedBySiblings, Wn as LocalStartServiceError, Wt as escapeRegExp, X as slowCcOperationTimeoutMs, Xn as ResourceTimeoutError, Xt as parseBootstrapMarker, Y as CloudControlProvider, Yn as ProvisioningError, Yt as getBootstrapMarkerKey, Z as disableInstanceApiTermination, Zn as ResourceUpdateNotSupportedError, Zt as validateAssetBucketName, _ as ATOMIC_FINAL_SNAPSHOT_TYPES, _n as resolveUseCdkBootstrapAssets, _t as TEMPLATE_SOURCED_RULES, a as DeploymentEventsStore, an as AssetManifestLoader, ar as withErrorHandling, at as resolveExplicitPhysicalId, b as ccRoutedFinalSnapshotError, bn as CFN_TEMPLATE_BODY_LIMIT, bt as scrubResourceRecord, c as replayFailedOperations, cn as synthesisStatusMessage, cr as isThrottlingError, ct as configBooleanRefusal, d as updatePartialReason, dn as resolveApp, dt as replayWarn, en as buildDockerImage, et as cfnRefValueFromPhysicalId, f as UNSPECIFIED_SKIP_REASON, fn as resolveAutoAssetStorage, ft as requireConfigArray, g as computeImplicitDeleteEdges, gn as resolveStateBucketWithDefaultAndSource, gt as STATE_SOURCED_READBACK_RULES, h as IMPLICIT_DELETE_DEPENDENCIES, hn as resolveStateBucketWithDefault, ht as STATE_SOURCED_CROSS_GENERATION_RULES, i as DeploymentEventsReader, in as runDockerStreaming, ir as normalizeAwsError, it as normalizeAwsTagsToCfn, j as bold, jt as DagBuilder, k as renderStatefulReason, kn as AssemblyReader, kt as describeTypeWithThrottleRetry, l as replayRollback, ln as getDefaultStateBucketName, lr as markNonRetryable, lt as configStringRefusal, m as withResourceDeadline, mn as resolveSkipPrefix, mt as requireConfigString, n as DEFAULT_RESOURCE_WARN_AFTER_MS, nn as getDockerCmd, nt as refStateLookupFromResource, o as planFailedOps, on as getDockerImageBySourceHash, or as isMarkedNonRetryable, ot as assertRegionMatch, p as deleteSkipReason, pn as resolveCaptureObservedState, pt as requireConfigObject, q as findActionableSilentDrops, qn as NestedStackChildDirectDestroyError, qt as BOOTSTRAP_MARKER_PREFIX, r as DeployEngine, rn as runDockerForeground, rt as WAFv2WebACLProvider, s as planRollback, sn as Synthesizer, sr as isRetryableTransientError, st as coerceCfnBoolean, t as DEFAULT_RESOURCE_TIMEOUT_MS, tn as formatDockerLoginError, tt as getAccountInfo, u as updatePartialMessage, un as getLegacyStateBucketName, ut as readConfigString, v as PRE_DELETE_SNAPSHOT_TYPES, vn as stateBucketExistenceConfirmed, vt as maskSecretsInText, w as unsupportedFinalSnapshotError, wn as uploadCfnTemplate, wt as s3BucketRegionalDomainName, x as createPreDeleteFinalSnapshot, xn as CFN_TEMPLATE_URL_LIMIT, xt as s3BucketArn, y as buildFinalSnapshotIdentifier, yn as warnDeprecatedNoPrefixCliFlag, yt as redactSecretsForState, z as exportAliasCollisionScrubWarning, zt as WorkGraph } from "./deploy-engine-
|
|
2
|
+
import { $ as IntrinsicFunctionResolver, $n as StackTerminationProtectionError, $t as buildDenyExternalAccessPolicy, A as formatResourceLine, An as processStackMessages, At as withRetry, B as isExportAliasCollision, Bt as buildAssetRedirectMap, C as refusesFinalSnapshot, Cn as findLargeInlineResources, Ct as s3BucketDualStackDomainName, D as MULTI_REGION_RECREATE_BLOCKED_TYPES, Dn as canonicalizeRegion, Dt as DiffCalculator, E as extractDeploymentEventError, En as PARTITION_TABLE, Et as applyRoleArnIfSet, F as red, Fn as resetAwsClients, Ft as rebuildClientForBucketRegion, G as clearOnUpdateRemoval, Gt as stripControlChars, H as stateKeySecretExposure, Hn as LocalInvokeBuildError$1, Ht as loadPublishableAssetManifest, I as yellow, In as setAwsClients, It as shouldRetainResource, J as findSilentDropProperties, Jn as PartialFailureError, Jt as ensureAssetStorage, K as ProviderRegistry, Kn as MissingCdkCliError, Kt as AssetModeResolver, L as collectDeclaredOutputNames, Lt as AssetPublisher, M as cyan, Mn as resolveBucketRegion, Mt as TemplateParser, N as gray, Nn as AwsClients, Nt as LockManager, O as isStatefulRecreateTargetSync, On as derivePartitionAndUrlSuffix, Ot as INTRINSIC_KEYS, P as green, Pn as getAwsClients, Pt as S3StateBackend, Q as isTerminationProtectionPropagationError, Qn as StackHasActiveImportsError, Qt as validateContainerRepoName, R as collectPublishedOutputNames, Rn as CdkdError, Rt as stringifyValue, S as isFinalSnapshotError, Sn as MIGRATE_TMP_PREFIX, St as s3BucketDomainName, T as makeCanonicalizePropertiesFn, Tn as expectedOwnerParam, Tt as s3BucketWebsiteUrl, U as IAMRoleProvider, Un as LocalMigrateError, Ut as rewriteTemplateAssetReferences, V as secretBearingStateKeyWarning, Vn as DeployCancelledError, Vt as createAssetRedirectResolver, W as collectInlinePolicyNamesManagedBySiblings, Wn as LocalStartServiceError, Wt as escapeRegExp, X as slowCcOperationTimeoutMs, Xn as ResourceTimeoutError, Xt as parseBootstrapMarker, Y as CloudControlProvider, Yn as ProvisioningError, Yt as getBootstrapMarkerKey, Z as disableInstanceApiTermination, Zn as ResourceUpdateNotSupportedError, Zt as validateAssetBucketName, _ as ATOMIC_FINAL_SNAPSHOT_TYPES, _n as resolveUseCdkBootstrapAssets, _t as TEMPLATE_SOURCED_RULES, a as DeploymentEventsStore, an as AssetManifestLoader, ar as withErrorHandling, at as resolveExplicitPhysicalId, b as ccRoutedFinalSnapshotError, bn as CFN_TEMPLATE_BODY_LIMIT, bt as scrubResourceRecord, c as replayFailedOperations, cn as synthesisStatusMessage, cr as isThrottlingError, ct as configBooleanRefusal, d as updatePartialReason, dn as resolveApp, dt as replayWarn, en as buildDockerImage, et as cfnRefValueFromPhysicalId, f as UNSPECIFIED_SKIP_REASON, fn as resolveAutoAssetStorage, ft as requireConfigArray, g as computeImplicitDeleteEdges, gn as resolveStateBucketWithDefaultAndSource, gt as STATE_SOURCED_READBACK_RULES, h as IMPLICIT_DELETE_DEPENDENCIES, hn as resolveStateBucketWithDefault, ht as STATE_SOURCED_CROSS_GENERATION_RULES, i as DeploymentEventsReader, in as runDockerStreaming, ir as normalizeAwsError, it as normalizeAwsTagsToCfn, j as bold, jt as DagBuilder, k as renderStatefulReason, kn as AssemblyReader, kt as describeTypeWithThrottleRetry, l as replayRollback, ln as getDefaultStateBucketName, lr as markNonRetryable, lt as configStringRefusal, m as withResourceDeadline, mn as resolveSkipPrefix, mt as requireConfigString, n as DEFAULT_RESOURCE_WARN_AFTER_MS, nn as getDockerCmd, nt as refStateLookupFromResource, o as planFailedOps, on as getDockerImageBySourceHash, or as isMarkedNonRetryable, ot as assertRegionMatch, p as deleteSkipReason, pn as resolveCaptureObservedState, pt as requireConfigObject, q as findActionableSilentDrops, qn as NestedStackChildDirectDestroyError, qt as BOOTSTRAP_MARKER_PREFIX, r as DeployEngine, rn as runDockerForeground, rt as WAFv2WebACLProvider, s as planRollback, sn as Synthesizer, sr as isRetryableTransientError, st as coerceCfnBoolean, t as DEFAULT_RESOURCE_TIMEOUT_MS, tn as formatDockerLoginError, tt as getAccountInfo, u as updatePartialMessage, un as getLegacyStateBucketName, ut as readConfigString, v as PRE_DELETE_SNAPSHOT_TYPES, vn as stateBucketExistenceConfirmed, vt as maskSecretsInText, w as unsupportedFinalSnapshotError, wn as uploadCfnTemplate, wt as s3BucketRegionalDomainName, x as createPreDeleteFinalSnapshot, xn as CFN_TEMPLATE_URL_LIMIT, xt as s3BucketArn, y as buildFinalSnapshotIdentifier, yn as warnDeprecatedNoPrefixCliFlag, yt as redactSecretsForState, z as exportAliasCollisionScrubWarning, zt as WorkGraph } from "./deploy-engine-NER3C-7K.js";
|
|
3
3
|
import { a as getLiveRenderer, c as PATTERN_B_RESOURCE_TYPES, d as generateResourceNameWithFallback, h as withStackName, i as runStackBuffered, m as withSkipPrefix, n as getLogger, o as PATTERN_B_NAME_OPTIONS, s as PATTERN_B_NAME_PROPERTIES, u as generateResourceName } from "./logger-zRrlbaQt.js";
|
|
4
|
-
import { t as ASGProvider } from "./asg-provider-
|
|
4
|
+
import { t as ASGProvider } from "./asg-provider-CMuiarRD.js";
|
|
5
5
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
6
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
7
7
|
import { CopyObjectCommand, CreateBucketCommand, DeleteBucketAnalyticsConfigurationCommand, DeleteBucketCommand, DeleteBucketCorsCommand, DeleteBucketEncryptionCommand, DeleteBucketIntelligentTieringConfigurationCommand, DeleteBucketInventoryConfigurationCommand, DeleteBucketLifecycleCommand, DeleteBucketMetricsConfigurationCommand, DeleteBucketOwnershipControlsCommand, DeleteBucketPolicyCommand, DeleteBucketReplicationCommand, DeleteBucketTaggingCommand, DeleteBucketWebsiteCommand, DeleteObjectsCommand, GetBucketAccelerateConfigurationCommand, GetBucketCorsCommand, GetBucketEncryptionCommand, GetBucketLifecycleConfigurationCommand, GetBucketLocationCommand, GetBucketLoggingCommand, GetBucketNotificationConfigurationCommand, GetBucketOwnershipControlsCommand, GetBucketPolicyCommand, GetBucketReplicationCommand, GetBucketTaggingCommand, GetBucketVersioningCommand, GetBucketWebsiteCommand, GetObjectCommand, GetObjectLockConfigurationCommand, GetPublicAccessBlockCommand, HeadBucketCommand, ListBucketAnalyticsConfigurationsCommand, ListBucketIntelligentTieringConfigurationsCommand, ListBucketInventoryConfigurationsCommand, ListBucketMetricsConfigurationsCommand, ListObjectVersionsCommand, ListObjectsV2Command, NoSuchBucket, PutBucketAccelerateConfigurationCommand, PutBucketAnalyticsConfigurationCommand, PutBucketCorsCommand, PutBucketEncryptionCommand, PutBucketIntelligentTieringConfigurationCommand, PutBucketInventoryConfigurationCommand, PutBucketLifecycleConfigurationCommand, PutBucketLoggingCommand, PutBucketMetricsConfigurationCommand, PutBucketNotificationConfigurationCommand, PutBucketOwnershipControlsCommand, PutBucketPolicyCommand, PutBucketReplicationCommand, PutBucketTaggingCommand, PutBucketVersioningCommand, PutBucketWebsiteCommand, PutObjectCommand, PutObjectLockConfigurationCommand, PutPublicAccessBlockCommand, S3Client, S3ServiceException } from "@aws-sdk/client-s3";
|
|
@@ -14308,8 +14308,10 @@ function isIndexBusyDeleteError(message) {
|
|
|
14308
14308
|
* gate uses, because THAT default is sized for a wait that runs once while
|
|
14309
14309
|
* this one runs per retry: the caller's wall clock is
|
|
14310
14310
|
* `its retry budget x this x ~1.2s per poll + withRetry's backoff`, and `destroy-runner.ts` runs the delete under a per-resource
|
|
14311
|
-
* deadline (30 min by default;
|
|
14312
|
-
*
|
|
14311
|
+
* deadline (30 min by default; since issue #1955 both DynamoDB providers
|
|
14312
|
+
* self-report that same 30 min through `getMinResourceTimeoutMs`, so a lowered
|
|
14313
|
+
* `--resource-timeout` cannot shrink it below what the shared delete budget in
|
|
14314
|
+
* `providers/dynamodb-delete-budget.ts` assumes). At 900 polls the product would be
|
|
14313
14315
|
* ~4.2h, so a genuinely stuck index would produce a 30-minute wait ending in a
|
|
14314
14316
|
* generic `ResourceTimeoutError` that never mentions indexes. At 60 the LOOP's
|
|
14315
14317
|
* worst case is ~18.4 min on `AWS::DynamoDB::Table` and ~10.4 min on
|
|
@@ -14388,7 +14390,10 @@ const DELETE_INDEX_WAIT_PROCEED_NOTE = `the DeleteTable goes ahead anyway and AW
|
|
|
14388
14390
|
*/
|
|
14389
14391
|
async function waitForIndexesSettled(opts) {
|
|
14390
14392
|
const { tableName, logicalId, logger, describeTable, maxAttempts, proceedNote } = opts;
|
|
14391
|
-
|
|
14393
|
+
const budgetWait = opts.budgetWait;
|
|
14394
|
+
let attempt = 0;
|
|
14395
|
+
while (budgetWait ? budgetWait.nextPoll() : attempt < maxAttempts) {
|
|
14396
|
+
attempt += 1;
|
|
14392
14397
|
try {
|
|
14393
14398
|
if (!hasTransitionalIndex((await describeTable()).Table?.GlobalSecondaryIndexes ?? [])) return;
|
|
14394
14399
|
} catch (err) {
|
|
@@ -14410,7 +14415,8 @@ async function waitForIndexesSettled(opts) {
|
|
|
14410
14415
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
14411
14416
|
}
|
|
14412
14417
|
const pollSeconds = INDEX_SETTLE_POLL_INTERVAL_MS / 1e3;
|
|
14413
|
-
|
|
14418
|
+
const pollsRun = budgetWait ? budgetWait.pollsRun : maxAttempts;
|
|
14419
|
+
logger.warn(`Indexes on ${tableName} (${logicalId}) did not all reach ACTIVE within ${pollsRun} DescribeTable polls (~${pollSeconds}s apart, so a little over ${pollsRun * pollSeconds}s of wall clock).${budgetWait?.note() ?? ""} ${proceedNote}`);
|
|
14414
14420
|
}
|
|
14415
14421
|
/**
|
|
14416
14422
|
* The ONE line the retry prints at default verbosity, for both providers.
|
|
@@ -14525,12 +14531,486 @@ async function deleteTableWithIndexBusyRetry(opts) {
|
|
|
14525
14531
|
await opts.deleteTable();
|
|
14526
14532
|
}, opts.logicalId, {
|
|
14527
14533
|
maxRetries: opts.maxRetries,
|
|
14528
|
-
isRetryable: (message) => isIndexBusyDeleteError(message),
|
|
14534
|
+
isRetryable: (message) => isIndexBusyDeleteError(message) && (opts.shouldKeepRetrying?.() ?? true),
|
|
14529
14535
|
logger: opts.logger,
|
|
14530
14536
|
sleep: (ms) => opts.sleepSeam.sleep ? opts.sleepSeam.sleep(ms) : new Promise((resolve) => setTimeout(resolve, ms))
|
|
14531
14537
|
});
|
|
14532
14538
|
}
|
|
14533
14539
|
|
|
14540
|
+
//#endregion
|
|
14541
|
+
//#region src/utils/elapsed-budget.ts
|
|
14542
|
+
/**
|
|
14543
|
+
* ONE wall-clock allowance shared by a sequence of independently capped waits.
|
|
14544
|
+
*
|
|
14545
|
+
* **The problem it exists for.** A provider operation is run under a
|
|
14546
|
+
* per-resource deadline (`withResourceDeadline`, applied by
|
|
14547
|
+
* `src/cli/commands/destroy-runner.ts` and the deploy engine). Inside that
|
|
14548
|
+
* deadline a single `delete()` can stack several polling waits, each with its
|
|
14549
|
+
* own justified cap. Every cap is defensible on its own and the SUM is not:
|
|
14550
|
+
* three ~12-18 minute waits inside one 30-minute deadline overshoot it, and
|
|
14551
|
+
* `withResourceDeadline` does NOT cancel what it wraps — it rejects with a
|
|
14552
|
+
* generic timeout while the polling loop keeps running behind the run that has
|
|
14553
|
+
* already reported failure (issue #1955).
|
|
14554
|
+
*
|
|
14555
|
+
* A shared budget converts "each wait is capped" into "the PATH is capped": a
|
|
14556
|
+
* wait asks how many polls still fit rather than how many its own constant
|
|
14557
|
+
* allows, so the path spends the allowance ONCE and stops itself while the
|
|
14558
|
+
* deadline still has margin. Stopping ourselves is what preserves the
|
|
14559
|
+
* operation's own actionable error (AWS's "Cannot delete table while indexes
|
|
14560
|
+
* are being ..." rather than `ResourceTimeoutError`) and what stops work from
|
|
14561
|
+
* outliving the run.
|
|
14562
|
+
*
|
|
14563
|
+
* Deliberately NOT an `AbortController`: nothing here interrupts an in-flight
|
|
14564
|
+
* SDK call. It bounds how many MORE polls a loop will start, which is the whole
|
|
14565
|
+
* of the cost for a wait built out of `describe`-then-sleep — and it needs no
|
|
14566
|
+
* plumbing through every provider call.
|
|
14567
|
+
*
|
|
14568
|
+
* The clock is injectable so a test can drive elapsed time without faking
|
|
14569
|
+
* timers globally (the polling loops under test are real `async` loops; a
|
|
14570
|
+
* global fake clock would freeze them too).
|
|
14571
|
+
*/
|
|
14572
|
+
/**
|
|
14573
|
+
* The default clock: MONOTONIC, not `Date.now()`.
|
|
14574
|
+
*
|
|
14575
|
+
* `Date.now()` follows wall clock, so an NTP correction moves it in either
|
|
14576
|
+
* direction mid-operation — and the two directions fail differently. Backwards
|
|
14577
|
+
* would hand the path MORE than its total (guarded below by clamping elapsed at
|
|
14578
|
+
* zero); FORWARDS instantly drains the allowance, which cuts every remaining
|
|
14579
|
+
* wait to its one-poll floor and makes the whole path report "cdkd's allowance
|
|
14580
|
+
* ran out" for something that never happened. A 30-minute delete is long enough
|
|
14581
|
+
* to sit across a correction.
|
|
14582
|
+
*
|
|
14583
|
+
* `performance.now()` is monotonic and millisecond-denominated (a float, which
|
|
14584
|
+
* only matters to a consumer dividing by a poll cost — flooring already handles
|
|
14585
|
+
* it). Guarded rather than assumed present, so a runtime without it degrades to
|
|
14586
|
+
* the old behaviour instead of throwing.
|
|
14587
|
+
*/
|
|
14588
|
+
const monotonicNowMs = typeof globalThis.performance?.now === "function" ? () => globalThis.performance.now() : () => Date.now();
|
|
14589
|
+
/** A monotonic wall-clock allowance, started when the object is created. */
|
|
14590
|
+
var ElapsedBudget = class {
|
|
14591
|
+
/** Total wall clock this budget may spend, in milliseconds. */
|
|
14592
|
+
totalMs;
|
|
14593
|
+
clock;
|
|
14594
|
+
startedAt;
|
|
14595
|
+
constructor(totalMs, clock = monotonicNowMs) {
|
|
14596
|
+
if (!Number.isFinite(totalMs) || totalMs <= 0) throw new RangeError(`ElapsedBudget: totalMs must be a positive finite number (got ${totalMs})`);
|
|
14597
|
+
this.totalMs = totalMs;
|
|
14598
|
+
this.clock = clock;
|
|
14599
|
+
this.startedAt = clock();
|
|
14600
|
+
}
|
|
14601
|
+
/** Wall clock spent since the budget started. */
|
|
14602
|
+
elapsedMs() {
|
|
14603
|
+
return Math.max(0, this.clock() - this.startedAt);
|
|
14604
|
+
}
|
|
14605
|
+
/** Wall clock still available, never negative. */
|
|
14606
|
+
remainingMs() {
|
|
14607
|
+
return Math.max(0, this.totalMs - this.elapsedMs());
|
|
14608
|
+
}
|
|
14609
|
+
isExhausted() {
|
|
14610
|
+
return this.remainingMs() <= 0;
|
|
14611
|
+
}
|
|
14612
|
+
/**
|
|
14613
|
+
* How many polls of `perAttemptMs` still fit, clamped into
|
|
14614
|
+
* `[minAttempts, cap]`.
|
|
14615
|
+
*
|
|
14616
|
+
* `minAttempts` defaults to 1 rather than 0 on purpose: every caller here is
|
|
14617
|
+
* a `describe`-then-decide loop, and a loop granted ZERO attempts reports
|
|
14618
|
+
* "did not settle" without ever having looked — a verdict it did not earn,
|
|
14619
|
+
* and one that reads as a bug in the wait rather than as a spent budget. One
|
|
14620
|
+
* poll costs ~1s and is the difference between a guess and an observation.
|
|
14621
|
+
* The overshoot it can add is therefore bounded by one poll per wait, which
|
|
14622
|
+
* the caller's margin against its deadline absorbs.
|
|
14623
|
+
*/
|
|
14624
|
+
attemptsWithin(cap, perAttemptMs, minAttempts = 1) {
|
|
14625
|
+
if (!Number.isFinite(cap) || cap <= 0) return 0;
|
|
14626
|
+
if (!Number.isFinite(perAttemptMs) || perAttemptMs <= 0) return Math.min(minAttempts, cap);
|
|
14627
|
+
const affordable = Math.floor(this.remainingMs() / perAttemptMs);
|
|
14628
|
+
return Math.max(Math.min(minAttempts, cap), Math.min(cap, affordable));
|
|
14629
|
+
}
|
|
14630
|
+
};
|
|
14631
|
+
/**
|
|
14632
|
+
* Budgets keyed by the thing they bound, so an operation that is RE-ENTERED
|
|
14633
|
+
* inside one deadline keeps spending the same allowance.
|
|
14634
|
+
*
|
|
14635
|
+
* That re-entry is the case a per-call budget silently misses. `destroy-runner`
|
|
14636
|
+
* wraps its whole outer retry loop — up to four `delete()` calls — in ONE
|
|
14637
|
+
* deadline, so a `delete()` that fails on something the outer loop classes as
|
|
14638
|
+
* retryable (a throttle) starts again from the top. A budget created per call
|
|
14639
|
+
* would reset there and the same path could be paid twice inside one deadline;
|
|
14640
|
+
* a budget looked up by key does not.
|
|
14641
|
+
*
|
|
14642
|
+
* Entries are RETAINED on failure (that is the point) and released by the
|
|
14643
|
+
* caller on a terminal outcome, so the map holds at most one entry per
|
|
14644
|
+
* in-flight resource.
|
|
14645
|
+
*
|
|
14646
|
+
* **The key must identify the resource, not just name it.** A registry holding
|
|
14647
|
+
* bare names collides whenever two distinct resources share one — for AWS that
|
|
14648
|
+
* is any name-scoped-by-region identifier — and the collision is silent: the
|
|
14649
|
+
* second resource inherits the first's spent allowance. Callers are expected to
|
|
14650
|
+
* qualify the key themselves (see `providers/dynamodb-delete-budget.ts`'s
|
|
14651
|
+
* `deleteBudgetKey`), because only the caller knows what the name is scoped by.
|
|
14652
|
+
*/
|
|
14653
|
+
var ElapsedBudgetRegistry = class {
|
|
14654
|
+
budgets = /* @__PURE__ */ new Map();
|
|
14655
|
+
/**
|
|
14656
|
+
* The budget for `key`, creating it on first use and REUSING it on re-entry.
|
|
14657
|
+
* `totalMs` and `clock` are only read when the entry is CREATED — a re-entry
|
|
14658
|
+
* must not be able to grant itself a fresh allowance, nor a fresh clock.
|
|
14659
|
+
*
|
|
14660
|
+
* `reuseWithinMs` bounds how long an entry may be reused for, and exists
|
|
14661
|
+
* because entries are deliberately RETAINED on failure: without it, a spent
|
|
14662
|
+
* allowance would be inherited by any later operation that happens to reach
|
|
14663
|
+
* the same key in the same process. The caller passes the wall clock of the
|
|
14664
|
+
* DEADLINE the allowance is sized against — past that, the deadline has
|
|
14665
|
+
* certainly fired and whoever is calling now belongs to a new operation with
|
|
14666
|
+
* a new deadline, so reuse would be wrong rather than conservative.
|
|
14667
|
+
*/
|
|
14668
|
+
acquire(key, totalMs, clock = monotonicNowMs, reuseWithinMs) {
|
|
14669
|
+
const existing = this.budgets.get(key);
|
|
14670
|
+
if (existing) {
|
|
14671
|
+
if (!(reuseWithinMs !== void 0 && existing.elapsedMs() > reuseWithinMs)) return existing;
|
|
14672
|
+
this.budgets.delete(key);
|
|
14673
|
+
}
|
|
14674
|
+
const created = new ElapsedBudget(totalMs, clock);
|
|
14675
|
+
this.budgets.set(key, created);
|
|
14676
|
+
return created;
|
|
14677
|
+
}
|
|
14678
|
+
/** Drop `key`'s budget — call on a TERMINAL outcome, never between retries. */
|
|
14679
|
+
release(key) {
|
|
14680
|
+
this.budgets.delete(key);
|
|
14681
|
+
}
|
|
14682
|
+
/** Live entries; exists so a test can prove release actually released. */
|
|
14683
|
+
get size() {
|
|
14684
|
+
return this.budgets.size;
|
|
14685
|
+
}
|
|
14686
|
+
clear() {
|
|
14687
|
+
this.budgets.clear();
|
|
14688
|
+
}
|
|
14689
|
+
};
|
|
14690
|
+
|
|
14691
|
+
//#endregion
|
|
14692
|
+
//#region src/provisioning/providers/dynamodb-delete-budget.ts
|
|
14693
|
+
/**
|
|
14694
|
+
* The ONE wall-clock allowance the DynamoDB DELETE path spends, for both
|
|
14695
|
+
* `AWS::DynamoDB::Table` and `AWS::DynamoDB::GlobalTable` (issue #1955).
|
|
14696
|
+
*
|
|
14697
|
+
* **What was wrong.** Every wait on that path is individually capped and
|
|
14698
|
+
* individually justified, and the caps do not know about each other:
|
|
14699
|
+
*
|
|
14700
|
+
* ```
|
|
14701
|
+
* --remove-protection 600 polls ~12 min GlobalTable, ACTIVE wait, FIRST
|
|
14702
|
+
* ACTIVE wait 60 polls ~72s Table, ACTIVE wait, FIRST
|
|
14703
|
+
* auto-scaling teardown see below GlobalTable, table + per GSI,
|
|
14704
|
+
* per replica, BEFORE the first poll
|
|
14705
|
+
* waitForReplicaGone 600 polls ~12 min per NON-LOCAL replica
|
|
14706
|
+
* #1521 pre-delete gate 900 polls ~18 min (issue #1521)
|
|
14707
|
+
* index-busy retry loop ~10.4 min at 8 retries / ~18.4 at 14 (#1830/#1931/#1950)
|
|
14708
|
+
* waitForTableGone 600 polls ~12 min (only on a late SUCCESS)
|
|
14709
|
+
* ```
|
|
14710
|
+
*
|
|
14711
|
+
* Every poll figure is `polls x ~1.2s`, the MEASURED cost of a sleep plus a
|
|
14712
|
+
* `DescribeTable` round trip — so a 600-poll wait is ~12 min, not the ~10 min
|
|
14713
|
+
* its `1 poll = 1s` era comments used to say.
|
|
14714
|
+
*
|
|
14715
|
+
* The `--remove-protection` term is the sharpest of the ones a first pass at
|
|
14716
|
+
* this fix missed: it runs FIRST, its predicate is `ACTIVE` *and* no
|
|
14717
|
+
* transitional index — the very condition the rest of the path is about — so on
|
|
14718
|
+
* a `cdkd destroy --remove-protection` against a table with a building GSI it
|
|
14719
|
+
* spends its full ~12 min at t=0 and leaves the gate and the loop the
|
|
14720
|
+
* remainder.
|
|
14721
|
+
*
|
|
14722
|
+
* The auto-scaling teardown is not poll-shaped at all, and its story changed in
|
|
14723
|
+
* review. It was believed to inherit `withRetry`'s default schedule; it did
|
|
14724
|
+
* not — the two `withRetry` sites in that method are both in the REGISTER
|
|
14725
|
+
* branch, and every delete-path caller passes `newSettings: undefined`, which
|
|
14726
|
+
* takes the bare-`send` teardown branch. So the term was single-attempt and
|
|
14727
|
+
* fast, and a budget clamp on it was inert. What the correction exposed is a
|
|
14728
|
+
* half-applied twin: the register branch retries throttles because an
|
|
14729
|
+
* un-retried `ThrottlingException` leaves a target silently UNregistered, and
|
|
14730
|
+
* the teardown has the exact symmetric hazard — a silently RETAINED target that
|
|
14731
|
+
* a future table of the same name inherits (PR #403) — with no retry at all.
|
|
14732
|
+
* The teardown is now wrapped in the same throttle-only retry, bounded by
|
|
14733
|
+
* {@link autoScalingRetriesWithinDeleteBudget}, so the term is real, bounded,
|
|
14734
|
+
* and no longer asymmetric.
|
|
14735
|
+
*
|
|
14736
|
+
* `destroy-runner.ts` runs the whole thing under ONE per-resource deadline
|
|
14737
|
+
* (`DEFAULT_RESOURCE_TIMEOUT_MS`, 30 min). A replicated GlobalTable whose index
|
|
14738
|
+
* is transitioning reached ~40 min, a single-region one reached ~40.4 min on
|
|
14739
|
+
* the late-success path, and a single-region `--remove-protection` destroy
|
|
14740
|
+
* reached 12 + 18 + 10.4 ~= ~40.4 min on the EXHAUSTED-budget path — all over
|
|
14741
|
+
* it. Three consequences, all of them the
|
|
14742
|
+
* issue's:
|
|
14743
|
+
*
|
|
14744
|
+
* 1. **Stacked waits.** The deadline was spent three or four times over,
|
|
14745
|
+
* because each cap was sized as though it were the only one.
|
|
14746
|
+
* 2. **A non-cancelling overshoot.** `withResourceDeadline` rejects but does
|
|
14747
|
+
* not cancel, so the run reported a generic `ResourceTimeoutError` that
|
|
14748
|
+
* never mentions indexes — replacing AWS's own actionable sentence, which
|
|
14749
|
+
* the bounded re-arm exists to surface — WHILE the polling loop kept
|
|
14750
|
+
* running behind the failed run.
|
|
14751
|
+
* 3. **Throttle compounding.** `deleteTableWithIndexBusyRetry`'s inner
|
|
14752
|
+
* `isRetryable` is index-busy-only by design, so a THROTTLED `DeleteTable`
|
|
14753
|
+
* escapes to `destroy-runner.ts`'s outer loop, which DOES class throttles
|
|
14754
|
+
* as retryable and re-enters `delete()` from the top — paying the whole
|
|
14755
|
+
* settle-and-retry sequence again inside the SAME deadline (2 x ~19.6 min
|
|
14756
|
+
* ~= ~39 min on the `Table` path after issue #1950).
|
|
14757
|
+
*
|
|
14758
|
+
* **What replaces it.** A budget acquired once per physical table and shared by
|
|
14759
|
+
* every wait on the path. Each wait asks how many polls still fit rather than
|
|
14760
|
+
* how many its own constant allows, so the path spends the allowance ONCE
|
|
14761
|
+
* ((1)), stops itself while the deadline still has minutes of margin — leaving
|
|
14762
|
+
* AWS's own error as the failure and nothing polling behind it ((2)) — and,
|
|
14763
|
+
* because the budget is keyed by physical id in a registry rather than created
|
|
14764
|
+
* per call, a re-entry from the outer retry loop CONTINUES the same allowance
|
|
14765
|
+
* instead of restarting it ((3)).
|
|
14766
|
+
*
|
|
14767
|
+
* The per-type retry budgets (`TABLE_DELETE_INDEX_BUSY_MAX_RETRIES` /
|
|
14768
|
+
* `GLOBAL_TABLE_DELETE_INDEX_BUSY_MAX_RETRIES`) are unchanged and still
|
|
14769
|
+
* meaningful: they shape HOW the allowance is spent — how many fresh
|
|
14770
|
+
* `DeleteTable` probes it buys — while this bounds how much there is. What
|
|
14771
|
+
* changes is that neither of them, nor any wait cap, is load-bearing for
|
|
14772
|
+
* staying under the deadline any more.
|
|
14773
|
+
*/
|
|
14774
|
+
/**
|
|
14775
|
+
* The `DescribeTable` round trip each poll pays ON TOP of its sleep.
|
|
14776
|
+
*
|
|
14777
|
+
* MEASURED, not assumed: the live issue-#1950 run's 60-poll re-arm took ~72s
|
|
14778
|
+
* and its per-attempt gaps were 73 / 74 / 77 / 80 / 80 / 80s once `withRetry`'s
|
|
14779
|
+
* backoff is subtracted, i.e. ~1.2s per poll against a 1s sleep. Pricing a poll
|
|
14780
|
+
* at the bare sleep interval is what once let a ~10.4 min loop be quoted as
|
|
14781
|
+
* ~8.8 min.
|
|
14782
|
+
*/
|
|
14783
|
+
const DYNAMODB_DELETE_POLL_RTT_MS = 200;
|
|
14784
|
+
/**
|
|
14785
|
+
* What ONE poll of a delete-path wait costs in wall clock. Every budget->polls
|
|
14786
|
+
* conversion below goes through this, and it READS
|
|
14787
|
+
* {@link INDEX_SETTLE_POLL_INTERVAL_MS} rather than restating 1000 so the two
|
|
14788
|
+
* cannot drift while this arithmetic claims they agree.
|
|
14789
|
+
*/
|
|
14790
|
+
const DYNAMODB_DELETE_POLL_COST_MS = INDEX_SETTLE_POLL_INTERVAL_MS + 200;
|
|
14791
|
+
/**
|
|
14792
|
+
* The allowance the WHOLE delete path gets, however many waits it contains.
|
|
14793
|
+
*
|
|
14794
|
+
* **Why 26 minutes.** It has to sit under the per-resource deadline with enough
|
|
14795
|
+
* margin that the deadline never fires first — because the whole point is to
|
|
14796
|
+
* fail with the operation's own error rather than with a generic timeout — and
|
|
14797
|
+
* it has to be large enough that no shape which completes TODAY starts failing:
|
|
14798
|
+
*
|
|
14799
|
+
* ```
|
|
14800
|
+
* Table, single pass ACTIVE wait ~72s + loop(14) ~18.4 min = ~19.6 min fits
|
|
14801
|
+
* GlobalTable, single-region #1521 gate ~18 min + loop(8) ~10.4 min = ~28.4 min clamped by ~2.4 min
|
|
14802
|
+
* ```
|
|
14803
|
+
*
|
|
14804
|
+
* The one shape it shortens is the GlobalTable single-region EXHAUSTED-budget
|
|
14805
|
+
* case, and shortening it costs nothing: that path ends by throwing AWS's own
|
|
14806
|
+
* index-busy sentence either way, ~2.4 min sooner now.
|
|
14807
|
+
*
|
|
14808
|
+
* **Every OTHER shape was already past the 30-minute deadline before this
|
|
14809
|
+
* change**, which is why counting the two newly-added terms does not move the
|
|
14810
|
+
* number. `--remove-protection` on a single-region GlobalTable is
|
|
14811
|
+
* 12 + 18 + 10.4 ~= ~40.4 min; on a one-replica table 12 + 12 + 18 + 10.4 ~=
|
|
14812
|
+
* ~52.4 min; the late-success gone-wait adds ~12 min to whichever of those it
|
|
14813
|
+
* follows; the auto-scaling teardown is unbounded. All of them previously ended
|
|
14814
|
+
* in the generic `ResourceTimeoutError` with work still running, so a bounded
|
|
14815
|
+
* stop is a strict improvement there rather than a trade — there is no shape
|
|
14816
|
+
* between 26 and 30 minutes that this budget takes away except the ~2.4 min
|
|
14817
|
+
* above, because 30 minutes was already the ceiling and everything past it
|
|
14818
|
+
* failed.
|
|
14819
|
+
*
|
|
14820
|
+
* It is deliberately ONE number for both types rather than a per-type pair. The
|
|
14821
|
+
* two per-type RETRY budgets exist because the two types arrive at the loop
|
|
14822
|
+
* having spent different amounts; this is the ceiling on the total, and a total
|
|
14823
|
+
* does not need to know which terms produced it.
|
|
14824
|
+
*/
|
|
14825
|
+
const DYNAMODB_DELETE_BUDGET_MS = 26 * 6e4;
|
|
14826
|
+
/**
|
|
14827
|
+
* What both DynamoDB providers self-report through `getMinResourceTimeoutMs()`.
|
|
14828
|
+
*
|
|
14829
|
+
* The engine resolves the per-resource deadline as
|
|
14830
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), slowTypeFloor,
|
|
14831
|
+
* globalCliDefault)`, so declaring this makes "the budget fits inside the
|
|
14832
|
+
* deadline" a GUARANTEE rather than an assumption about the CLI default: a user
|
|
14833
|
+
* who lowers `--resource-timeout` globally can no longer re-create the crossing
|
|
14834
|
+
* this module exists to remove. It equals today's `DEFAULT_RESOURCE_TIMEOUT_MS`
|
|
14835
|
+
* (30 min), so at default settings nothing moves — `max(30m, 30m)` is 30m.
|
|
14836
|
+
*
|
|
14837
|
+
* A per-type override (`--resource-timeout AWS::DynamoDB::Table=5m`) still
|
|
14838
|
+
* wins; that is the documented escape hatch, and it is the only way to make
|
|
14839
|
+
* this type abort sooner than its self-report.
|
|
14840
|
+
*/
|
|
14841
|
+
const DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS = 18e5;
|
|
14842
|
+
/**
|
|
14843
|
+
* Test seam for the budget, mirroring the providers' `deleteTableRetryDelays`
|
|
14844
|
+
* sleep seams. Production leaves BOTH fields unset, so
|
|
14845
|
+
* {@link DYNAMODB_DELETE_BUDGET_MS} and the real wall clock apply.
|
|
14846
|
+
*
|
|
14847
|
+
* `clock` is here rather than only `totalMs` because the behaviour worth
|
|
14848
|
+
* testing is what happens as the budget DRAINS, and a test cannot drain a
|
|
14849
|
+
* 26-minute allowance by waiting. Shrinking `totalMs` alone reaches the
|
|
14850
|
+
* clamped-poll-count arm but never the EXHAUSTED one, which is the arm that
|
|
14851
|
+
* stops the retry loop and preserves AWS's own message — the whole point of
|
|
14852
|
+
* issue #1955. A controllable clock reaches both, deterministically, without
|
|
14853
|
+
* `vi.useFakeTimers()` freezing the provider's own polling loops.
|
|
14854
|
+
*/
|
|
14855
|
+
const dynamoDbDeleteBudgetOverride = {};
|
|
14856
|
+
function resolveDynamoDbDeleteBudgetMs() {
|
|
14857
|
+
const override = dynamoDbDeleteBudgetOverride.totalMs;
|
|
14858
|
+
return override !== void 0 && override > 0 ? override : DYNAMODB_DELETE_BUDGET_MS;
|
|
14859
|
+
}
|
|
14860
|
+
function resolveDynamoDbDeleteBudgetClock() {
|
|
14861
|
+
return dynamoDbDeleteBudgetOverride.clock ?? monotonicNowMs;
|
|
14862
|
+
}
|
|
14863
|
+
/**
|
|
14864
|
+
* The registry key for one table's delete allowance.
|
|
14865
|
+
*
|
|
14866
|
+
* A DynamoDB table NAME is unique per region, not globally, so a bare
|
|
14867
|
+
* `physicalId` collides across regions and the second table silently inherits
|
|
14868
|
+
* the first's spent allowance. In practice `destroy-runner.ts` builds a fresh
|
|
14869
|
+
* provider (and therefore a fresh registry) per region, so the collision is not
|
|
14870
|
+
* reachable today — but that is an invariant of the CALLER, enforced nowhere,
|
|
14871
|
+
* and the fix costs one string concat. `\0` cannot appear in either component.
|
|
14872
|
+
*
|
|
14873
|
+
* `region` is the STATE's region (`DeleteContext.expectedRegion`), which is
|
|
14874
|
+
* available synchronously at the top of `delete()` — unlike the client's own
|
|
14875
|
+
* region, which needs an await and would push the acquire past the
|
|
14876
|
+
* `--remove-protection` flip. Pre-region state records leave it undefined,
|
|
14877
|
+
* which degrades to the old bare-name key rather than failing.
|
|
14878
|
+
*/
|
|
14879
|
+
function deleteBudgetKey(physicalId, region) {
|
|
14880
|
+
return `${region ?? ""}\u0000${physicalId}`;
|
|
14881
|
+
}
|
|
14882
|
+
/**
|
|
14883
|
+
* The poll cap a wait may actually use: its own constant, or what the shared
|
|
14884
|
+
* budget can still afford, whichever is smaller.
|
|
14885
|
+
*
|
|
14886
|
+
* `budget` is optional and an absent one means "unbounded by this mechanism" —
|
|
14887
|
+
* the CREATE / UPDATE callers of these same waits keep their original caps
|
|
14888
|
+
* verbatim, since the deadline arithmetic this fixes is the delete path's.
|
|
14889
|
+
*/
|
|
14890
|
+
function pollsWithinDeleteBudget(cap, budget) {
|
|
14891
|
+
if (!budget) return {
|
|
14892
|
+
attempts: cap,
|
|
14893
|
+
cap,
|
|
14894
|
+
clampedByBudget: false
|
|
14895
|
+
};
|
|
14896
|
+
const attempts = budget.attemptsWithin(cap, DYNAMODB_DELETE_POLL_COST_MS);
|
|
14897
|
+
return {
|
|
14898
|
+
attempts,
|
|
14899
|
+
cap,
|
|
14900
|
+
clampedByBudget: attempts < cap
|
|
14901
|
+
};
|
|
14902
|
+
}
|
|
14903
|
+
/**
|
|
14904
|
+
* Polls below which a wait is not EVIDENCE about AWS, only evidence that cdkd
|
|
14905
|
+
* stopped asking.
|
|
14906
|
+
*
|
|
14907
|
+
* The warn-versus-throw split on `waitForTableGone`, and the strong wording of
|
|
14908
|
+
* the exhaustion note, both hang off this, and before it existed both hung off
|
|
14909
|
+
* `attempts < cap` — which is arithmetic, not evidence. With ~14 min of the
|
|
14910
|
+
* allowance already spent a 600-poll wait is still granted ~500 polls, i.e.
|
|
14911
|
+
* ten real minutes of `DescribeTable`; calling that "cdkd's allowance ran out,
|
|
14912
|
+
* AWS was not given a real wait" is false, and downgrading its failure to a
|
|
14913
|
+
* warning is a decision made by two minutes of wall clock rather than by
|
|
14914
|
+
* anything the user can act on.
|
|
14915
|
+
*
|
|
14916
|
+
* 60 polls (~72s) because the provider's own gone-wait note records that a
|
|
14917
|
+
* typical small-table delete completes in 5-30s: past roughly twice the top of
|
|
14918
|
+
* that range, cdkd HAS given AWS a real wait and a table still present is a
|
|
14919
|
+
* signal about AWS. Below it, cdkd barely looked, and the honest report is
|
|
14920
|
+
* about cdkd rather than about the table.
|
|
14921
|
+
*/
|
|
14922
|
+
const DELETE_SHORT_WAIT_POLLS = 60;
|
|
14923
|
+
var DeleteWaitImpl = class {
|
|
14924
|
+
cap;
|
|
14925
|
+
grantedPolls;
|
|
14926
|
+
ran = 0;
|
|
14927
|
+
brokeEarly = false;
|
|
14928
|
+
entryClamped;
|
|
14929
|
+
budget;
|
|
14930
|
+
constructor(cap, budget) {
|
|
14931
|
+
const polls = pollsWithinDeleteBudget(cap, budget);
|
|
14932
|
+
this.cap = polls.cap;
|
|
14933
|
+
this.grantedPolls = polls.attempts;
|
|
14934
|
+
this.entryClamped = polls.clampedByBudget;
|
|
14935
|
+
this.budget = budget;
|
|
14936
|
+
}
|
|
14937
|
+
nextPoll() {
|
|
14938
|
+
if (this.ran >= this.grantedPolls) return false;
|
|
14939
|
+
if (this.ran >= 1 && this.budget?.isExhausted() === true) {
|
|
14940
|
+
this.brokeEarly = true;
|
|
14941
|
+
return false;
|
|
14942
|
+
}
|
|
14943
|
+
this.ran += 1;
|
|
14944
|
+
return true;
|
|
14945
|
+
}
|
|
14946
|
+
get pollsRun() {
|
|
14947
|
+
return this.ran;
|
|
14948
|
+
}
|
|
14949
|
+
get endedByBudget() {
|
|
14950
|
+
return this.brokeEarly || this.entryClamped;
|
|
14951
|
+
}
|
|
14952
|
+
get cutShort() {
|
|
14953
|
+
return this.endedByBudget && this.ran < 60;
|
|
14954
|
+
}
|
|
14955
|
+
note() {
|
|
14956
|
+
if (!this.endedByBudget) return "";
|
|
14957
|
+
const minutes = Math.max(1, Math.round((this.budget?.totalMs ?? 0) / 6e4));
|
|
14958
|
+
if (this.cutShort) return ` — cdkd's shared ~${minutes}-minute delete allowance was already spent, so this wait made only ${this.ran} of its ${this.cap} polls and AWS was not given a real wait. Nothing is necessarily wrong with the table; re-run the destroy.`;
|
|
14959
|
+
return ` (cdkd's shared ~${minutes}-minute delete allowance capped this wait at ${this.ran} of its ${this.cap} polls.)`;
|
|
14960
|
+
}
|
|
14961
|
+
};
|
|
14962
|
+
/** Begin a delete-path wait that draws on the shared allowance. */
|
|
14963
|
+
function beginDeleteWait(cap, budget) {
|
|
14964
|
+
return new DeleteWaitImpl(cap, budget);
|
|
14965
|
+
}
|
|
14966
|
+
/**
|
|
14967
|
+
* `withRetry` retries the delete-path auto-scaling teardown may still fund.
|
|
14968
|
+
*
|
|
14969
|
+
* The teardown runs BEFORE the first budgeted poll — table-level plus one call
|
|
14970
|
+
* per GSI, per non-local replica, then again locally — and until this change it
|
|
14971
|
+
* had NO retry at all: the two `withRetry` sites in `applyAutoScalingDiff` are
|
|
14972
|
+
* both in the REGISTER branch, and every delete-path caller passes
|
|
14973
|
+
* `newSettings: undefined`, which takes the bare-`send` teardown branch. (An
|
|
14974
|
+
* earlier revision of this file claimed the teardown inherited `withRetry`'s
|
|
14975
|
+
* default schedule and would therefore spend ~47s per call under account-wide
|
|
14976
|
+
* throttling; that was false, and the clamp built on it was inert. The claim is
|
|
14977
|
+
* recorded here because it is the kind of arithmetic a later reader would
|
|
14978
|
+
* otherwise trust.)
|
|
14979
|
+
*
|
|
14980
|
+
* The teardown is now wrapped in the same throttle-only retry the register
|
|
14981
|
+
* branch uses, for the symmetric reason: that branch retries because an
|
|
14982
|
+
* un-retried `ThrottlingException` leaves a target silently UNregistered, and
|
|
14983
|
+
* an un-retried throttle here leaves one silently REGISTERED — the PR #403 leak
|
|
14984
|
+
* a future table of the same name inherits. Retrying is what makes a bound
|
|
14985
|
+
* necessary, and the bound is what makes retrying safe on a path that shares
|
|
14986
|
+
* one deadline with everything after it.
|
|
14987
|
+
*
|
|
14988
|
+
* Retries are bounded rather than the CALL being skipped, deliberately. The
|
|
14989
|
+
* teardown exists because a surviving scalable target is silently inherited by
|
|
14990
|
+
* a future table of the same name (PR #403), and skipping re-introduces that
|
|
14991
|
+
* leak. So a drained allowance still issues every call once —
|
|
14992
|
+
* `withRetry` with `maxRetries: 0` runs the operation exactly once — and still
|
|
14993
|
+
* warns actionably on each failure; it just stops paying to retry them.
|
|
14994
|
+
*/
|
|
14995
|
+
function autoScalingRetriesWithinDeleteBudget(budget, defaultMaxRetries) {
|
|
14996
|
+
if (!budget) return defaultMaxRetries;
|
|
14997
|
+
const surplusMs = budget.remainingMs() - AUTO_SCALING_TEARDOWN_RESERVE_MS;
|
|
14998
|
+
if (surplusMs <= 0) return 0;
|
|
14999
|
+
return Math.max(0, Math.min(defaultMaxRetries, Math.floor(surplusMs / AUTO_SCALING_RETRY_STEP_MS)));
|
|
15000
|
+
}
|
|
15001
|
+
/**
|
|
15002
|
+
* The part of the allowance the auto-scaling teardown may NOT spend on retries.
|
|
15003
|
+
*
|
|
15004
|
+
* Sized so the terms that actually delete the table — the #1521 gate, the
|
|
15005
|
+
* index-busy loop and the gone-wait — keep the bulk of the allowance no matter
|
|
15006
|
+
* how throttled the teardown is. 20 of the 26 minutes, leaving the teardown up
|
|
15007
|
+
* to ~6 min of retry backoff, which at ~8s a retry is ~45 retries spread across
|
|
15008
|
+
* however many calls the table's index and replica count produce.
|
|
15009
|
+
*/
|
|
15010
|
+
const AUTO_SCALING_TEARDOWN_RESERVE_MS = 20 * 6e4;
|
|
15011
|
+
/** `withRetry`'s capped backoff step (`maxDelayMs`), the cost of one retry. */
|
|
15012
|
+
const AUTO_SCALING_RETRY_STEP_MS = 8e3;
|
|
15013
|
+
|
|
14534
15014
|
//#endregion
|
|
14535
15015
|
//#region src/provisioning/providers/dynamodb-table-provider.ts
|
|
14536
15016
|
/**
|
|
@@ -15097,11 +15577,47 @@ var DynamoDBTableProvider = class {
|
|
|
15097
15577
|
"ContributorInsightsSpecification"
|
|
15098
15578
|
])]]);
|
|
15099
15579
|
unhandledByDesign = /* @__PURE__ */ new Map([["AWS::DynamoDB::Table", /* @__PURE__ */ new Map([["ImportSourceSpecification", "S3 import uses the separate ImportTable API (not CreateTable) and is create-only with no readback; deferred to a dedicated import-from-S3 PR"]])]]);
|
|
15580
|
+
/**
|
|
15581
|
+
* The ONE wall-clock allowance each `delete()` spends, keyed by physical id
|
|
15582
|
+
* (issue #1955). Keyed rather than per-call because `destroy-runner.ts` can
|
|
15583
|
+
* RE-ENTER `delete()` from the top: a THROTTLED `DeleteTable` is not retried
|
|
15584
|
+
* by the index-busy loop below — that classifier is message-scoped by design
|
|
15585
|
+
* — so it escapes to the outer loop, which DOES class throttles as retryable
|
|
15586
|
+
* and pays the whole settle-and-retry sequence a second time inside the same
|
|
15587
|
+
* per-resource deadline (2 x ~19.6 min ~= ~39 min after issue #1950 raised
|
|
15588
|
+
* this type's retry budget to 14). A per-call budget would hand that second
|
|
15589
|
+
* pass a fresh allowance; this one does not. Keyed rather than per-instance
|
|
15590
|
+
* because a destroy level deletes its resources CONCURRENTLY through one
|
|
15591
|
+
* provider instance.
|
|
15592
|
+
*/
|
|
15593
|
+
deleteBudgets = new ElapsedBudgetRegistry();
|
|
15100
15594
|
constructor() {
|
|
15101
15595
|
const awsClients = getAwsClients();
|
|
15102
15596
|
this.dynamoDBClient = awsClients.dynamoDB;
|
|
15103
15597
|
}
|
|
15104
15598
|
/**
|
|
15599
|
+
* Self-reported minimum per-resource deadline (issue #1955).
|
|
15600
|
+
*
|
|
15601
|
+
* The engine resolves the deadline as
|
|
15602
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), globalCliDefault)`,
|
|
15603
|
+
* so declaring this is what turns "the shared delete budget fits inside the
|
|
15604
|
+
* deadline" from an assumption about the CLI default into a guarantee. It
|
|
15605
|
+
* equals today's `DEFAULT_RESOURCE_TIMEOUT_MS`, so at default settings this
|
|
15606
|
+
* changes nothing.
|
|
15607
|
+
*
|
|
15608
|
+
* **It is TYPE-level, not operation-level**, and that is a real consequence
|
|
15609
|
+
* rather than a detail: `deploy-engine.ts` and `destroy-runner.ts` both fold
|
|
15610
|
+
* it in through the same `Math.max(...)`, so a user running
|
|
15611
|
+
* `--resource-timeout 5m` for fail-fast CI gets 30 minutes for this type's
|
|
15612
|
+
* CREATE and UPDATE too, not only for the DELETE the budget is about. The
|
|
15613
|
+
* interface has no per-operation form, and the escape hatch for anyone who
|
|
15614
|
+
* wants the shorter deadline back is the per-type override
|
|
15615
|
+
* (`--resource-timeout AWS::DynamoDB::Table=5m`), which still wins.
|
|
15616
|
+
*/
|
|
15617
|
+
getMinResourceTimeoutMs() {
|
|
15618
|
+
return DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS;
|
|
15619
|
+
}
|
|
15620
|
+
/**
|
|
15105
15621
|
* Create a DynamoDB table
|
|
15106
15622
|
*/
|
|
15107
15623
|
async create(logicalId, resourceType, properties, context) {
|
|
@@ -15391,6 +15907,7 @@ var DynamoDBTableProvider = class {
|
|
|
15391
15907
|
*/
|
|
15392
15908
|
async delete(logicalId, physicalId, resourceType, _properties, context) {
|
|
15393
15909
|
this.logger.debug(`Deleting DynamoDB table ${logicalId}: ${physicalId}`);
|
|
15910
|
+
const budget = this.deleteBudgets.acquire(deleteBudgetKey(physicalId, context?.expectedRegion), resolveDynamoDbDeleteBudgetMs(), resolveDynamoDbDeleteBudgetClock(), DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS);
|
|
15394
15911
|
if (context?.removeProtection === true) try {
|
|
15395
15912
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
15396
15913
|
TableName: physicalId,
|
|
@@ -15398,7 +15915,7 @@ var DynamoDBTableProvider = class {
|
|
|
15398
15915
|
}));
|
|
15399
15916
|
this.logger.debug(`Disabled DeletionProtectionEnabled on DynamoDB table ${logicalId}, waiting for ACTIVE`);
|
|
15400
15917
|
try {
|
|
15401
|
-
await this.waitForTableActiveAfterUpdate(physicalId);
|
|
15918
|
+
await this.waitForTableActiveAfterUpdate(physicalId, 60, budget);
|
|
15402
15919
|
} catch (waitErr) {
|
|
15403
15920
|
this.logger.debug(`Could not wait for table ${physicalId} ACTIVE after disabling protection: ${waitErr instanceof Error ? waitErr.message : String(waitErr)}`);
|
|
15404
15921
|
}
|
|
@@ -15406,6 +15923,13 @@ var DynamoDBTableProvider = class {
|
|
|
15406
15923
|
if (!(flipError instanceof ResourceNotFoundException$1)) this.logger.debug(`Could not disable DeletionProtectionEnabled on ${physicalId}: ${flipError instanceof Error ? flipError.message : String(flipError)}`);
|
|
15407
15924
|
}
|
|
15408
15925
|
try {
|
|
15926
|
+
const reArmPolls = () => {
|
|
15927
|
+
const wait = beginDeleteWait(60, budget);
|
|
15928
|
+
return {
|
|
15929
|
+
maxAttempts: wait.grantedPolls,
|
|
15930
|
+
budgetWait: wait
|
|
15931
|
+
};
|
|
15932
|
+
};
|
|
15409
15933
|
await deleteTableWithIndexBusyRetry({
|
|
15410
15934
|
logicalId,
|
|
15411
15935
|
physicalId,
|
|
@@ -15420,16 +15944,23 @@ var DynamoDBTableProvider = class {
|
|
|
15420
15944
|
logicalId,
|
|
15421
15945
|
logger: this.logger,
|
|
15422
15946
|
describeTable: () => this.dynamoDBClient.send(new DescribeTableCommand({ TableName: physicalId })),
|
|
15423
|
-
|
|
15947
|
+
...reArmPolls(),
|
|
15424
15948
|
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE
|
|
15425
15949
|
}),
|
|
15950
|
+
shouldKeepRetrying: () => {
|
|
15951
|
+
if (!budget.isExhausted()) return true;
|
|
15952
|
+
this.logger.warn(`DynamoDB table ${logicalId}: giving up the index-busy DeleteTable retry on ${physicalId} — the delete path has spent its whole ${Math.round(budget.totalMs / 6e4)}-minute budget. AWS's own message follows; re-running the destroy succeeds once the index is ACTIVE.`);
|
|
15953
|
+
return false;
|
|
15954
|
+
},
|
|
15426
15955
|
sleepSeam: deleteTableRetryDelays$1
|
|
15427
15956
|
});
|
|
15428
15957
|
this.logger.debug(`Successfully deleted DynamoDB table ${logicalId}`);
|
|
15958
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
15429
15959
|
} catch (error) {
|
|
15430
15960
|
if (error instanceof ResourceNotFoundException$1) {
|
|
15431
15961
|
assertRegionMatch(await this.dynamoDBClient.config.region(), context?.expectedRegion, resourceType, logicalId, physicalId);
|
|
15432
15962
|
this.logger.debug(`DynamoDB table ${physicalId} does not exist, skipping deletion`);
|
|
15963
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
15433
15964
|
return;
|
|
15434
15965
|
}
|
|
15435
15966
|
const cause = error instanceof Error ? error : void 0;
|
|
@@ -15770,12 +16301,13 @@ var DynamoDBTableProvider = class {
|
|
|
15770
16301
|
* blanket raise would turn every genuinely wedged capacity edit into a
|
|
15771
16302
|
* 10-minute hang.
|
|
15772
16303
|
*/
|
|
15773
|
-
async waitForTableActiveAfterUpdate(tableName, maxAttempts = 60) {
|
|
15774
|
-
|
|
16304
|
+
async waitForTableActiveAfterUpdate(tableName, maxAttempts = 60, budget) {
|
|
16305
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
16306
|
+
while (wait.nextPoll()) {
|
|
15775
16307
|
if ((await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }))).Table?.TableStatus === "ACTIVE") return;
|
|
15776
16308
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
15777
16309
|
}
|
|
15778
|
-
throw new Error(`Table ${tableName} did not reach ACTIVE status within ${
|
|
16310
|
+
throw new Error(`Table ${tableName} did not reach ACTIVE status within ${wait.pollsRun} seconds after UpdateTable${wait.note()}`);
|
|
15779
16311
|
}
|
|
15780
16312
|
/**
|
|
15781
16313
|
* Poll DescribeTable until the table is ACTIVE AND every Global Secondary
|
|
@@ -16581,6 +17113,19 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
16581
17113
|
* UpdateTable success.
|
|
16582
17114
|
*/
|
|
16583
17115
|
attributeCache = /* @__PURE__ */ new Map();
|
|
17116
|
+
/**
|
|
17117
|
+
* The ONE wall-clock allowance each `delete()` spends, keyed by physical id
|
|
17118
|
+
* (issue #1955). Keyed rather than per-call because `destroy-runner.ts` can
|
|
17119
|
+
* RE-ENTER `delete()` from the top — its outer retry loop classes a throttled
|
|
17120
|
+
* `DeleteTable` as retryable — inside the SAME per-resource deadline, and a
|
|
17121
|
+
* per-call budget would hand that second pass a fresh allowance. Keyed rather
|
|
17122
|
+
* than per-instance because a destroy level deletes its resources
|
|
17123
|
+
* CONCURRENTLY through this one provider instance.
|
|
17124
|
+
*
|
|
17125
|
+
* See `./dynamodb-delete-budget.ts` for the sizing and for the three failure
|
|
17126
|
+
* terms it closes.
|
|
17127
|
+
*/
|
|
17128
|
+
deleteBudgets = new ElapsedBudgetRegistry();
|
|
16584
17129
|
handledProperties = /* @__PURE__ */ new Map([["AWS::DynamoDB::GlobalTable", /* @__PURE__ */ new Set([
|
|
16585
17130
|
"TableName",
|
|
16586
17131
|
"KeySchema",
|
|
@@ -16602,6 +17147,29 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
16602
17147
|
this.dynamoDBClient = awsClients.dynamoDB;
|
|
16603
17148
|
}
|
|
16604
17149
|
/**
|
|
17150
|
+
* Self-reported minimum per-resource deadline (issue #1955).
|
|
17151
|
+
*
|
|
17152
|
+
* The engine resolves the deadline as
|
|
17153
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), globalCliDefault)`,
|
|
17154
|
+
* so declaring this is what turns "the shared delete budget fits inside the
|
|
17155
|
+
* deadline" from an assumption about the CLI default into a guarantee: a user
|
|
17156
|
+
* who lowers `--resource-timeout` globally can no longer re-create the
|
|
17157
|
+
* crossing the budget exists to remove. It equals today's
|
|
17158
|
+
* `DEFAULT_RESOURCE_TIMEOUT_MS`, so at default settings this changes nothing.
|
|
17159
|
+
*
|
|
17160
|
+
* **It is TYPE-level, not operation-level**, and that is a real consequence
|
|
17161
|
+
* rather than a detail: `deploy-engine.ts` and `destroy-runner.ts` both fold
|
|
17162
|
+
* it in through the same `Math.max(...)`, so a user running
|
|
17163
|
+
* `--resource-timeout 5m` for fail-fast CI gets 30 minutes for this type's
|
|
17164
|
+
* CREATE and UPDATE too, not only for the DELETE the budget is about. The
|
|
17165
|
+
* interface has no per-operation form, and the escape hatch for anyone who
|
|
17166
|
+
* wants the shorter deadline back is the per-type override
|
|
17167
|
+
* (`--resource-timeout AWS::DynamoDB::GlobalTable=5m`), which still wins.
|
|
17168
|
+
*/
|
|
17169
|
+
getMinResourceTimeoutMs() {
|
|
17170
|
+
return DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS;
|
|
17171
|
+
}
|
|
17172
|
+
/**
|
|
16605
17173
|
* Return a `DynamoDBClient` pinned to the given region, caching per
|
|
16606
17174
|
* region for the lifetime of this provider instance. Uses the default
|
|
16607
17175
|
* credential chain (env / shared config / IAM role) — no explicit
|
|
@@ -17561,7 +18129,8 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17561
18129
|
* create-only field with no corresponding `RegisterScalableTarget`
|
|
17562
18130
|
* surface; `readAutoScalingSettings` also explicitly skips it.
|
|
17563
18131
|
*/
|
|
17564
|
-
async applyAutoScalingDiff(tableName, dimension, oldSettings, newSettings, client, indexName) {
|
|
18132
|
+
async applyAutoScalingDiff(tableName, dimension, oldSettings, newSettings, client, indexName, budget) {
|
|
18133
|
+
const asMaxRetries = autoScalingRetriesWithinDeleteBudget(budget, 8);
|
|
17565
18134
|
const metricType = dimension.endsWith("WriteCapacityUnits") ? "DynamoDBWriteCapacityUtilization" : "DynamoDBReadCapacityUtilization";
|
|
17566
18135
|
const resourceId = autoScalingResourceId(tableName, indexName);
|
|
17567
18136
|
const policyName = `${metricType}:${resourceId}`;
|
|
@@ -17585,6 +18154,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17585
18154
|
MaxCapacity: maxCapacity
|
|
17586
18155
|
})), `${resourceId} (${dimension})`, {
|
|
17587
18156
|
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18157
|
+
maxRetries: asMaxRetries,
|
|
17588
18158
|
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
17589
18159
|
});
|
|
17590
18160
|
} catch (err) {
|
|
@@ -17614,6 +18184,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17614
18184
|
TargetTrackingScalingPolicyConfiguration: targetTrackingConfig
|
|
17615
18185
|
})), `${policyName}`, {
|
|
17616
18186
|
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18187
|
+
maxRetries: asMaxRetries,
|
|
17617
18188
|
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
17618
18189
|
});
|
|
17619
18190
|
this.logger.debug(`Upserted auto-scaling policy ${policyName} on ${tableName} (${dimension})`);
|
|
@@ -17629,21 +18200,29 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17629
18200
|
return name === "ObjectNotFoundException" || msg.includes("No scaling policy found") || msg.includes("No scalable target found");
|
|
17630
18201
|
};
|
|
17631
18202
|
try {
|
|
17632
|
-
await asClient.send(new DeleteScalingPolicyCommand({
|
|
18203
|
+
await withRetry(() => asClient.send(new DeleteScalingPolicyCommand({
|
|
17633
18204
|
PolicyName: policyName,
|
|
17634
18205
|
ServiceNamespace: "dynamodb",
|
|
17635
18206
|
ResourceId: resourceId,
|
|
17636
18207
|
ScalableDimension: dimension
|
|
17637
|
-
}))
|
|
18208
|
+
})), `${resourceId} (${dimension})`, {
|
|
18209
|
+
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18210
|
+
maxRetries: asMaxRetries,
|
|
18211
|
+
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
18212
|
+
});
|
|
17638
18213
|
} catch (err) {
|
|
17639
18214
|
if (!isObjectNotFound(err)) this.logger.warn(`Could not delete auto-scaling policy on ${tableName} (${dimension}): ${err instanceof Error ? err.message : String(err)}. Run: aws application-autoscaling delete-scaling-policy --policy-name ${policyName} --service-namespace dynamodb --resource-id ${resourceId} --scalable-dimension ${dimension}`);
|
|
17640
18215
|
}
|
|
17641
18216
|
try {
|
|
17642
|
-
await asClient.send(new DeregisterScalableTargetCommand({
|
|
18217
|
+
await withRetry(() => asClient.send(new DeregisterScalableTargetCommand({
|
|
17643
18218
|
ServiceNamespace: "dynamodb",
|
|
17644
18219
|
ResourceId: resourceId,
|
|
17645
18220
|
ScalableDimension: dimension
|
|
17646
|
-
}))
|
|
18221
|
+
})), `${resourceId} (${dimension})`, {
|
|
18222
|
+
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18223
|
+
maxRetries: asMaxRetries,
|
|
18224
|
+
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
18225
|
+
});
|
|
17647
18226
|
this.logger.debug(`Deregistered auto-scaling target ${resourceId} (${dimension})`);
|
|
17648
18227
|
} catch (err) {
|
|
17649
18228
|
if (!isObjectNotFound(err)) this.logger.warn(`Could not deregister auto-scaling target on ${tableName} (${dimension}): ${err instanceof Error ? err.message : String(err)}. Run: aws application-autoscaling deregister-scalable-target --service-namespace dynamodb --resource-id ${resourceId} --scalable-dimension ${dimension}`);
|
|
@@ -17792,6 +18371,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17792
18371
|
*/
|
|
17793
18372
|
async delete(logicalId, physicalId, resourceType, _properties, context) {
|
|
17794
18373
|
this.logger.debug(`Deleting DynamoDB GlobalTable ${logicalId}: ${physicalId}`);
|
|
18374
|
+
const budget = this.deleteBudgets.acquire(deleteBudgetKey(physicalId, context?.expectedRegion), resolveDynamoDbDeleteBudgetMs(), resolveDynamoDbDeleteBudgetClock(), DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS);
|
|
17795
18375
|
if (context?.removeProtection === true) try {
|
|
17796
18376
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
17797
18377
|
TableName: physicalId,
|
|
@@ -17799,7 +18379,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17799
18379
|
}));
|
|
17800
18380
|
this.logger.debug(`Disabled DeletionProtectionEnabled on ${logicalId}, waiting for ACTIVE`);
|
|
17801
18381
|
try {
|
|
17802
|
-
await this.waitForTableActiveAfterUpdate(physicalId, logicalId);
|
|
18382
|
+
await this.waitForTableActiveAfterUpdate(physicalId, logicalId, 600, budget);
|
|
17803
18383
|
} catch (waitErr) {
|
|
17804
18384
|
this.logger.debug(`Could not wait for table ${physicalId} ACTIVE after protection flip: ${waitErr instanceof Error ? waitErr.message : String(waitErr)}`);
|
|
17805
18385
|
}
|
|
@@ -17823,25 +18403,26 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17823
18403
|
const region = replica.RegionName;
|
|
17824
18404
|
if (!region || region === currentRegion) continue;
|
|
17825
18405
|
try {
|
|
17826
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region));
|
|
17827
|
-
for (const indexName of tableIndexNames) await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), indexName);
|
|
18406
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), void 0, budget);
|
|
18407
|
+
for (const indexName of tableIndexNames) await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), indexName, budget);
|
|
17828
18408
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
17829
18409
|
TableName: physicalId,
|
|
17830
18410
|
ReplicaUpdates: [{ Delete: { RegionName: region } }]
|
|
17831
18411
|
}));
|
|
17832
|
-
await this.waitForReplicaGone(physicalId, region, logicalId);
|
|
18412
|
+
await this.waitForReplicaGone(physicalId, region, logicalId, REPLICA_GONE_WAIT_ATTEMPTS, budget);
|
|
17833
18413
|
} catch (replicaErr) {
|
|
17834
18414
|
if (!(replicaErr instanceof ResourceNotFoundException$1)) throw replicaErr;
|
|
17835
18415
|
}
|
|
17836
18416
|
}
|
|
17837
18417
|
const localAsClient = await this.getLocalAutoScalingClient();
|
|
17838
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, localAsClient);
|
|
17839
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:WriteCapacityUnits", {}, void 0, localAsClient);
|
|
18418
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, localAsClient, void 0, budget);
|
|
18419
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:WriteCapacityUnits", {}, void 0, localAsClient, void 0, budget);
|
|
17840
18420
|
for (const indexName of tableIndexNames) {
|
|
17841
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:WriteCapacityUnits", {}, void 0, localAsClient, indexName);
|
|
17842
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, localAsClient, indexName);
|
|
18421
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:WriteCapacityUnits", {}, void 0, localAsClient, indexName, budget);
|
|
18422
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, localAsClient, indexName, budget);
|
|
17843
18423
|
}
|
|
17844
18424
|
} catch (describeErr) {
|
|
18425
|
+
if (describeErr instanceof ProvisioningError) throw describeErr;
|
|
17845
18426
|
if (!(describeErr instanceof ResourceNotFoundException$1)) {
|
|
17846
18427
|
const cause = describeErr instanceof Error ? describeErr : void 0;
|
|
17847
18428
|
throw new ProvisioningError(`Failed to describe DynamoDB GlobalTable ${logicalId} before delete: ${describeErr instanceof Error ? describeErr.message : String(describeErr)}`, resourceType, logicalId, physicalId, cause);
|
|
@@ -17850,7 +18431,10 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17850
18431
|
try {
|
|
17851
18432
|
if (hasTransitionalIndex(preDeleteIndexes)) {
|
|
17852
18433
|
this.logger.debug(`Waiting for indexes on ${physicalId} to settle before DeleteTable (issue #1521)`);
|
|
17853
|
-
await this.waitForIndexesActive(physicalId, logicalId, {
|
|
18434
|
+
await this.waitForIndexesActive(physicalId, logicalId, {
|
|
18435
|
+
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE,
|
|
18436
|
+
budget
|
|
18437
|
+
});
|
|
17854
18438
|
}
|
|
17855
18439
|
await deleteTableWithIndexBusyRetry({
|
|
17856
18440
|
logicalId,
|
|
@@ -17863,16 +18447,24 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17863
18447
|
},
|
|
17864
18448
|
reArm: () => this.waitForIndexesActive(physicalId, logicalId, {
|
|
17865
18449
|
maxAttempts: 60,
|
|
17866
|
-
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE
|
|
18450
|
+
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE,
|
|
18451
|
+
budget
|
|
17867
18452
|
}),
|
|
18453
|
+
shouldKeepRetrying: () => {
|
|
18454
|
+
if (!budget.isExhausted()) return true;
|
|
18455
|
+
this.logger.warn(`DynamoDB GlobalTable ${logicalId}: giving up the index-busy DeleteTable retry on ${physicalId} — the delete path has spent its whole ${Math.round(budget.totalMs / 6e4)}-minute budget. AWS's own message follows; re-running the destroy succeeds once the index is ACTIVE.`);
|
|
18456
|
+
return false;
|
|
18457
|
+
},
|
|
17868
18458
|
sleepSeam: deleteTableRetryDelays
|
|
17869
18459
|
});
|
|
17870
|
-
await this.waitForTableGone(physicalId, logicalId);
|
|
18460
|
+
await this.waitForTableGone(physicalId, logicalId, 600, budget);
|
|
17871
18461
|
this.logger.debug(`Successfully deleted DynamoDB GlobalTable ${logicalId}`);
|
|
18462
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
17872
18463
|
} catch (error) {
|
|
17873
18464
|
if (error instanceof ResourceNotFoundException$1) {
|
|
17874
18465
|
assertRegionMatch(await this.dynamoDBClient.config.region(), context?.expectedRegion, resourceType, logicalId, physicalId);
|
|
17875
18466
|
this.logger.debug(`DynamoDB GlobalTable ${physicalId} does not exist, skipping`);
|
|
18467
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
17876
18468
|
return;
|
|
17877
18469
|
}
|
|
17878
18470
|
const cause = error instanceof Error ? error : void 0;
|
|
@@ -18444,9 +19036,10 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18444
19036
|
* rejection as the backstop, which is the behavior it had before #1521 for
|
|
18445
19037
|
* every case except the one the wait exists to fix.
|
|
18446
19038
|
*/
|
|
18447
|
-
async waitForTableActiveAfterUpdate(tableName, logicalId, maxAttempts = 600) {
|
|
19039
|
+
async waitForTableActiveAfterUpdate(tableName, logicalId, maxAttempts = 600, budget) {
|
|
19040
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
18448
19041
|
let tableReachedActive = false;
|
|
18449
|
-
|
|
19042
|
+
while (wait.nextPoll()) {
|
|
18450
19043
|
const response = await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }));
|
|
18451
19044
|
if (response.Table?.TableStatus === "ACTIVE") {
|
|
18452
19045
|
tableReachedActive = true;
|
|
@@ -18454,11 +19047,12 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18454
19047
|
}
|
|
18455
19048
|
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
18456
19049
|
}
|
|
19050
|
+
const budgetNote = wait.note();
|
|
18457
19051
|
if (tableReachedActive) {
|
|
18458
|
-
this.logger.warn(`Indexes on ${tableName} (${logicalId}) were still transitioning after ${
|
|
19052
|
+
this.logger.warn(`Indexes on ${tableName} (${logicalId}) were still transitioning after ${wait.pollsRun}s — a large index backfill can outlive this wait. Proceeding; AWS rejects the next call if it is still too early.${budgetNote}`);
|
|
18459
19053
|
return;
|
|
18460
19054
|
}
|
|
18461
|
-
throw new ProvisioningError(`Table ${tableName} did not reach ACTIVE within ${
|
|
19055
|
+
throw new ProvisioningError(`Table ${tableName} did not reach ACTIVE within ${wait.pollsRun}s after UpdateTable${budgetNote}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18462
19056
|
}
|
|
18463
19057
|
/**
|
|
18464
19058
|
* Wait until every GSI reports `IndexStatus: ACTIVE` (Issue #1419).
|
|
@@ -18503,18 +19097,25 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18503
19097
|
* passes {@link DELETE_INDEX_WAIT_PROCEED_NOTE}.
|
|
18504
19098
|
*/
|
|
18505
19099
|
async waitForIndexesActive(tableName, logicalId, opts) {
|
|
19100
|
+
const gateWait = beginDeleteWait(opts?.maxAttempts ?? 900, opts?.budget);
|
|
18506
19101
|
await waitForIndexesSettled({
|
|
18507
19102
|
tableName,
|
|
18508
19103
|
logicalId,
|
|
18509
19104
|
logger: this.logger,
|
|
18510
19105
|
describeTable: () => this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName })),
|
|
18511
|
-
maxAttempts:
|
|
19106
|
+
maxAttempts: gateWait.grantedPolls,
|
|
19107
|
+
budgetWait: gateWait,
|
|
18512
19108
|
proceedNote: opts?.proceedNote ?? "auto-scaling registration for a still-building index may have been skipped. The next deploy re-asserts it."
|
|
18513
19109
|
});
|
|
18514
19110
|
}
|
|
18515
19111
|
/**
|
|
18516
19112
|
* Wait until a specific replica's `ReplicaStatus` flips to ACTIVE.
|
|
18517
|
-
* Replica provisioning typically takes 1–5 min;
|
|
19113
|
+
* Replica provisioning typically takes 1–5 min; capped at
|
|
19114
|
+
* {@link REPLICA_GONE_WAIT_ATTEMPTS} polls — ~12 min at the MEASURED ~1.2s
|
|
19115
|
+
* per poll (a sleep plus a `DescribeTable` round trip), not the ~10 min the
|
|
19116
|
+
* `1 poll = 1s` reading gives. The delete-path arithmetic in
|
|
19117
|
+
* `./dynamodb-delete-budget.ts` prices it at the measured figure, so the two
|
|
19118
|
+
* have to agree.
|
|
18518
19119
|
*/
|
|
18519
19120
|
async waitForReplicaActive(tableName, region, logicalId, maxAttempts = 600) {
|
|
18520
19121
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
@@ -18528,17 +19129,19 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18528
19129
|
/**
|
|
18529
19130
|
* Wait until a specific replica disappears from `Replicas[]` after a
|
|
18530
19131
|
* Delete replica update. Replica deletion typically takes 1–5 min;
|
|
18531
|
-
*
|
|
19132
|
+
* capped at {@link REPLICA_GONE_WAIT_ATTEMPTS} polls (~12 min at the
|
|
19133
|
+
* measured ~1.2s per poll).
|
|
18532
19134
|
*/
|
|
18533
|
-
async waitForReplicaGone(tableName, region, logicalId, maxAttempts = REPLICA_GONE_WAIT_ATTEMPTS) {
|
|
18534
|
-
|
|
19135
|
+
async waitForReplicaGone(tableName, region, logicalId, maxAttempts = REPLICA_GONE_WAIT_ATTEMPTS, budget) {
|
|
19136
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
19137
|
+
while (wait.nextPoll()) try {
|
|
18535
19138
|
if (!(await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }))).Table?.Replicas?.find((r) => r.RegionName === region)) return;
|
|
18536
19139
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
18537
19140
|
} catch (err) {
|
|
18538
19141
|
if (err instanceof ResourceNotFoundException$1) return;
|
|
18539
19142
|
throw err;
|
|
18540
19143
|
}
|
|
18541
|
-
throw new ProvisioningError(`Replica ${region} for table ${tableName} did not disappear within ${
|
|
19144
|
+
throw new ProvisioningError(`Replica ${region} for table ${tableName} did not disappear within ${wait.pollsRun}s${wait.note()}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18542
19145
|
}
|
|
18543
19146
|
/**
|
|
18544
19147
|
* Wait for `DescribeTable` to return `ResourceNotFoundException`,
|
|
@@ -18548,18 +19151,26 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18548
19151
|
* wait, downstream observers (siblings deleted in the same destroy
|
|
18549
19152
|
* run, integ scripts that re-check via `aws dynamodb describe-table`)
|
|
18550
19153
|
* see "destroy succeeded" but the table is still listed by AWS.
|
|
18551
|
-
* Typical small-table delete completes in 5–30s;
|
|
18552
|
-
*
|
|
18553
|
-
|
|
18554
|
-
|
|
18555
|
-
|
|
19154
|
+
* Typical small-table delete completes in 5–30s; capped at
|
|
19155
|
+
* {@link TABLE_GONE_WAIT_ATTEMPTS} polls (~12 min at the measured ~1.2s per
|
|
19156
|
+
* poll) for worst-case large-table / replica-cascade scenarios. That 5-30s
|
|
19157
|
+
* figure is what {@link DELETE_SHORT_WAIT_POLLS} is calibrated against: past
|
|
19158
|
+
* roughly twice the top of it, cdkd HAS given AWS a real wait.
|
|
19159
|
+
*/
|
|
19160
|
+
async waitForTableGone(tableName, logicalId, maxAttempts = 600, budget) {
|
|
19161
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
19162
|
+
while (wait.nextPoll()) try {
|
|
18556
19163
|
await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }));
|
|
18557
19164
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
18558
19165
|
} catch (err) {
|
|
18559
19166
|
if (err instanceof ResourceNotFoundException$1) return;
|
|
18560
19167
|
throw err;
|
|
18561
19168
|
}
|
|
18562
|
-
|
|
19169
|
+
if (wait.cutShort) {
|
|
19170
|
+
this.logger.warn(`DynamoDB GlobalTable ${logicalId}: DeleteTable on ${tableName} was ACCEPTED by AWS but cdkd stopped waiting for the table to disappear${wait.note()} The delete completes on AWS's side; a later describe (or a re-run of this destroy) sees it gone.`);
|
|
19171
|
+
return;
|
|
19172
|
+
}
|
|
19173
|
+
throw new ProvisioningError(`Table ${tableName} did not disappear within ${wait.pollsRun}s${wait.note()}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18563
19174
|
}
|
|
18564
19175
|
};
|
|
18565
19176
|
/**
|
|
@@ -18660,6 +19271,19 @@ const autoScalingRetryDelays = {};
|
|
|
18660
19271
|
*/
|
|
18661
19272
|
const REPLICA_GONE_WAIT_ATTEMPTS = 600;
|
|
18662
19273
|
/**
|
|
19274
|
+
* Polls the `--remove-protection` flip's ACTIVE wait spends
|
|
19275
|
+
* (`waitForTableActiveAfterUpdate`).
|
|
19276
|
+
*
|
|
19277
|
+
* EXPORTED and named rather than left as an inline default argument because it
|
|
19278
|
+
* is a TERM of the delete-path deadline arithmetic (see
|
|
19279
|
+
* `./dynamodb-delete-budget.ts`): on `cdkd destroy --remove-protection` it runs
|
|
19280
|
+
* FIRST, ahead of the #1521 gate, and the wait it performs is the same
|
|
19281
|
+
* indexes-settled one the rest of the path is about. The budget fence in the
|
|
19282
|
+
* unit suite reads THIS rather than a copied literal, after the sibling
|
|
19283
|
+
* arithmetic was twice found asserting a cap the provider no longer had.
|
|
19284
|
+
*/
|
|
19285
|
+
const GLOBAL_TABLE_ACTIVE_WAIT_ATTEMPTS = 600;
|
|
19286
|
+
/**
|
|
18663
19287
|
* Polls `waitForTableGone` spends confirming the table is actually gone.
|
|
18664
19288
|
*
|
|
18665
19289
|
* EXPORTED, unlike its replica sibling, because the delete-path budget fence
|
|
@@ -81373,7 +81997,7 @@ function createMigrateCommand() {
|
|
|
81373
81997
|
*/
|
|
81374
81998
|
function buildProgram() {
|
|
81375
81999
|
const program = new Command();
|
|
81376
|
-
program.name("cdkd").description("CDK Direct - Deploy AWS CDK apps directly via SDK/Cloud Control API").version("0.284.
|
|
82000
|
+
program.name("cdkd").description("CDK Direct - Deploy AWS CDK apps directly via SDK/Cloud Control API").version("0.284.2");
|
|
81377
82001
|
program.hook("preAction", (_thisCommand, actionCommand) => {
|
|
81378
82002
|
const { profile } = actionCommand.optsWithGlobals();
|
|
81379
82003
|
if (profile !== void 0) process.env["AWS_PROFILE"] = profile;
|