@go-to-k/cdkd 0.284.1 → 0.284.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -1
- package/dist/{asg-provider-DDEsD6uA.js → asg-provider-iHWEv4gN.js} +2 -2
- package/dist/{asg-provider-DDEsD6uA.js.map → asg-provider-iHWEv4gN.js.map} +1 -1
- package/dist/cli.js +734 -52
- package/dist/cli.js.map +1 -1
- package/dist/{deploy-engine-CruC7Qxn.js → deploy-engine-DJ-CgH80.js} +3 -3
- package/dist/{deploy-engine-CruC7Qxn.js.map → deploy-engine-DJ-CgH80.js.map} +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { $ as IntrinsicFunctionResolver, $n as StackTerminationProtectionError, $t as buildDenyExternalAccessPolicy, A as formatResourceLine, An as processStackMessages, At as withRetry, B as isExportAliasCollision, Bt as buildAssetRedirectMap, C as refusesFinalSnapshot, Cn as findLargeInlineResources, Ct as s3BucketDualStackDomainName, D as MULTI_REGION_RECREATE_BLOCKED_TYPES, Dn as canonicalizeRegion, Dt as DiffCalculator, E as extractDeploymentEventError, En as PARTITION_TABLE, Et as applyRoleArnIfSet, F as red, Fn as resetAwsClients, Ft as rebuildClientForBucketRegion, G as clearOnUpdateRemoval, Gt as stripControlChars, H as stateKeySecretExposure, Hn as LocalInvokeBuildError$1, Ht as loadPublishableAssetManifest, I as yellow, In as setAwsClients, It as shouldRetainResource, J as findSilentDropProperties, Jn as PartialFailureError, Jt as ensureAssetStorage, K as ProviderRegistry, Kn as MissingCdkCliError, Kt as AssetModeResolver, L as collectDeclaredOutputNames, Lt as AssetPublisher, M as cyan, Mn as resolveBucketRegion, Mt as TemplateParser, N as gray, Nn as AwsClients, Nt as LockManager, O as isStatefulRecreateTargetSync, On as derivePartitionAndUrlSuffix, Ot as INTRINSIC_KEYS, P as green, Pn as getAwsClients, Pt as S3StateBackend, Q as isTerminationProtectionPropagationError, Qn as StackHasActiveImportsError, Qt as validateContainerRepoName, R as collectPublishedOutputNames, Rn as CdkdError, Rt as stringifyValue, S as isFinalSnapshotError, Sn as MIGRATE_TMP_PREFIX, St as s3BucketDomainName, T as makeCanonicalizePropertiesFn, Tn as expectedOwnerParam, Tt as s3BucketWebsiteUrl, U as IAMRoleProvider, Un as LocalMigrateError, Ut as rewriteTemplateAssetReferences, V as secretBearingStateKeyWarning, Vn as DeployCancelledError, Vt as createAssetRedirectResolver, W as collectInlinePolicyNamesManagedBySiblings, Wn as LocalStartServiceError, Wt as escapeRegExp, X as slowCcOperationTimeoutMs, Xn as ResourceTimeoutError, Xt as parseBootstrapMarker, Y as CloudControlProvider, Yn as ProvisioningError, Yt as getBootstrapMarkerKey, Z as disableInstanceApiTermination, Zn as ResourceUpdateNotSupportedError, Zt as validateAssetBucketName, _ as ATOMIC_FINAL_SNAPSHOT_TYPES, _n as resolveUseCdkBootstrapAssets, _t as TEMPLATE_SOURCED_RULES, a as DeploymentEventsStore, an as AssetManifestLoader, ar as withErrorHandling, at as resolveExplicitPhysicalId, b as ccRoutedFinalSnapshotError, bn as CFN_TEMPLATE_BODY_LIMIT, bt as scrubResourceRecord, c as replayFailedOperations, cn as synthesisStatusMessage, cr as isThrottlingError, ct as configBooleanRefusal, d as updatePartialReason, dn as resolveApp, dt as replayWarn, en as buildDockerImage, et as cfnRefValueFromPhysicalId, f as UNSPECIFIED_SKIP_REASON, fn as resolveAutoAssetStorage, ft as requireConfigArray, g as computeImplicitDeleteEdges, gn as resolveStateBucketWithDefaultAndSource, gt as STATE_SOURCED_READBACK_RULES, h as IMPLICIT_DELETE_DEPENDENCIES, hn as resolveStateBucketWithDefault, ht as STATE_SOURCED_CROSS_GENERATION_RULES, i as DeploymentEventsReader, in as runDockerStreaming, ir as normalizeAwsError, it as normalizeAwsTagsToCfn, j as bold, jt as DagBuilder, k as renderStatefulReason, kn as AssemblyReader, kt as describeTypeWithThrottleRetry, l as replayRollback, ln as getDefaultStateBucketName, lr as markNonRetryable, lt as configStringRefusal, m as withResourceDeadline, mn as resolveSkipPrefix, mt as requireConfigString, n as DEFAULT_RESOURCE_WARN_AFTER_MS, nn as getDockerCmd, nt as refStateLookupFromResource, o as planFailedOps, on as getDockerImageBySourceHash, or as isMarkedNonRetryable, ot as assertRegionMatch, p as deleteSkipReason, pn as resolveCaptureObservedState, pt as requireConfigObject, q as findActionableSilentDrops, qn as NestedStackChildDirectDestroyError, qt as BOOTSTRAP_MARKER_PREFIX, r as DeployEngine, rn as runDockerForeground, rt as WAFv2WebACLProvider, s as planRollback, sn as Synthesizer, sr as isRetryableTransientError, st as coerceCfnBoolean, t as DEFAULT_RESOURCE_TIMEOUT_MS, tn as formatDockerLoginError, tt as getAccountInfo, u as updatePartialMessage, un as getLegacyStateBucketName, ut as readConfigString, v as PRE_DELETE_SNAPSHOT_TYPES, vn as stateBucketExistenceConfirmed, vt as maskSecretsInText, w as unsupportedFinalSnapshotError, wn as uploadCfnTemplate, wt as s3BucketRegionalDomainName, x as createPreDeleteFinalSnapshot, xn as CFN_TEMPLATE_URL_LIMIT, xt as s3BucketArn, y as buildFinalSnapshotIdentifier, yn as warnDeprecatedNoPrefixCliFlag, yt as redactSecretsForState, z as exportAliasCollisionScrubWarning, zt as WorkGraph } from "./deploy-engine-
|
|
2
|
+
import { $ as IntrinsicFunctionResolver, $n as StackTerminationProtectionError, $t as buildDenyExternalAccessPolicy, A as formatResourceLine, An as processStackMessages, At as withRetry, B as isExportAliasCollision, Bt as buildAssetRedirectMap, C as refusesFinalSnapshot, Cn as findLargeInlineResources, Ct as s3BucketDualStackDomainName, D as MULTI_REGION_RECREATE_BLOCKED_TYPES, Dn as canonicalizeRegion, Dt as DiffCalculator, E as extractDeploymentEventError, En as PARTITION_TABLE, Et as applyRoleArnIfSet, F as red, Fn as resetAwsClients, Ft as rebuildClientForBucketRegion, G as clearOnUpdateRemoval, Gt as stripControlChars, H as stateKeySecretExposure, Hn as LocalInvokeBuildError$1, Ht as loadPublishableAssetManifest, I as yellow, In as setAwsClients, It as shouldRetainResource, J as findSilentDropProperties, Jn as PartialFailureError, Jt as ensureAssetStorage, K as ProviderRegistry, Kn as MissingCdkCliError, Kt as AssetModeResolver, L as collectDeclaredOutputNames, Lt as AssetPublisher, M as cyan, Mn as resolveBucketRegion, Mt as TemplateParser, N as gray, Nn as AwsClients, Nt as LockManager, O as isStatefulRecreateTargetSync, On as derivePartitionAndUrlSuffix, Ot as INTRINSIC_KEYS, P as green, Pn as getAwsClients, Pt as S3StateBackend, Q as isTerminationProtectionPropagationError, Qn as StackHasActiveImportsError, Qt as validateContainerRepoName, R as collectPublishedOutputNames, Rn as CdkdError, Rt as stringifyValue, S as isFinalSnapshotError, Sn as MIGRATE_TMP_PREFIX, St as s3BucketDomainName, T as makeCanonicalizePropertiesFn, Tn as expectedOwnerParam, Tt as s3BucketWebsiteUrl, U as IAMRoleProvider, Un as LocalMigrateError, Ut as rewriteTemplateAssetReferences, V as secretBearingStateKeyWarning, Vn as DeployCancelledError, Vt as createAssetRedirectResolver, W as collectInlinePolicyNamesManagedBySiblings, Wn as LocalStartServiceError, Wt as escapeRegExp, X as slowCcOperationTimeoutMs, Xn as ResourceTimeoutError, Xt as parseBootstrapMarker, Y as CloudControlProvider, Yn as ProvisioningError, Yt as getBootstrapMarkerKey, Z as disableInstanceApiTermination, Zn as ResourceUpdateNotSupportedError, Zt as validateAssetBucketName, _ as ATOMIC_FINAL_SNAPSHOT_TYPES, _n as resolveUseCdkBootstrapAssets, _t as TEMPLATE_SOURCED_RULES, a as DeploymentEventsStore, an as AssetManifestLoader, ar as withErrorHandling, at as resolveExplicitPhysicalId, b as ccRoutedFinalSnapshotError, bn as CFN_TEMPLATE_BODY_LIMIT, bt as scrubResourceRecord, c as replayFailedOperations, cn as synthesisStatusMessage, cr as isThrottlingError, ct as configBooleanRefusal, d as updatePartialReason, dn as resolveApp, dt as replayWarn, en as buildDockerImage, et as cfnRefValueFromPhysicalId, f as UNSPECIFIED_SKIP_REASON, fn as resolveAutoAssetStorage, ft as requireConfigArray, g as computeImplicitDeleteEdges, gn as resolveStateBucketWithDefaultAndSource, gt as STATE_SOURCED_READBACK_RULES, h as IMPLICIT_DELETE_DEPENDENCIES, hn as resolveStateBucketWithDefault, ht as STATE_SOURCED_CROSS_GENERATION_RULES, i as DeploymentEventsReader, in as runDockerStreaming, ir as normalizeAwsError, it as normalizeAwsTagsToCfn, j as bold, jt as DagBuilder, k as renderStatefulReason, kn as AssemblyReader, kt as describeTypeWithThrottleRetry, l as replayRollback, ln as getDefaultStateBucketName, lr as markNonRetryable, lt as configStringRefusal, m as withResourceDeadline, mn as resolveSkipPrefix, mt as requireConfigString, n as DEFAULT_RESOURCE_WARN_AFTER_MS, nn as getDockerCmd, nt as refStateLookupFromResource, o as planFailedOps, on as getDockerImageBySourceHash, or as isMarkedNonRetryable, ot as assertRegionMatch, p as deleteSkipReason, pn as resolveCaptureObservedState, pt as requireConfigObject, q as findActionableSilentDrops, qn as NestedStackChildDirectDestroyError, qt as BOOTSTRAP_MARKER_PREFIX, r as DeployEngine, rn as runDockerForeground, rt as WAFv2WebACLProvider, s as planRollback, sn as Synthesizer, sr as isRetryableTransientError, st as coerceCfnBoolean, t as DEFAULT_RESOURCE_TIMEOUT_MS, tn as formatDockerLoginError, tt as getAccountInfo, u as updatePartialMessage, un as getLegacyStateBucketName, ut as readConfigString, v as PRE_DELETE_SNAPSHOT_TYPES, vn as stateBucketExistenceConfirmed, vt as maskSecretsInText, w as unsupportedFinalSnapshotError, wn as uploadCfnTemplate, wt as s3BucketRegionalDomainName, x as createPreDeleteFinalSnapshot, xn as CFN_TEMPLATE_URL_LIMIT, xt as s3BucketArn, y as buildFinalSnapshotIdentifier, yn as warnDeprecatedNoPrefixCliFlag, yt as redactSecretsForState, z as exportAliasCollisionScrubWarning, zt as WorkGraph } from "./deploy-engine-DJ-CgH80.js";
|
|
3
3
|
import { a as getLiveRenderer, c as PATTERN_B_RESOURCE_TYPES, d as generateResourceNameWithFallback, h as withStackName, i as runStackBuffered, m as withSkipPrefix, n as getLogger, o as PATTERN_B_NAME_OPTIONS, s as PATTERN_B_NAME_PROPERTIES, u as generateResourceName } from "./logger-zRrlbaQt.js";
|
|
4
|
-
import { t as ASGProvider } from "./asg-provider-
|
|
4
|
+
import { t as ASGProvider } from "./asg-provider-iHWEv4gN.js";
|
|
5
5
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
6
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
7
7
|
import { CopyObjectCommand, CreateBucketCommand, DeleteBucketAnalyticsConfigurationCommand, DeleteBucketCommand, DeleteBucketCorsCommand, DeleteBucketEncryptionCommand, DeleteBucketIntelligentTieringConfigurationCommand, DeleteBucketInventoryConfigurationCommand, DeleteBucketLifecycleCommand, DeleteBucketMetricsConfigurationCommand, DeleteBucketOwnershipControlsCommand, DeleteBucketPolicyCommand, DeleteBucketReplicationCommand, DeleteBucketTaggingCommand, DeleteBucketWebsiteCommand, DeleteObjectsCommand, GetBucketAccelerateConfigurationCommand, GetBucketCorsCommand, GetBucketEncryptionCommand, GetBucketLifecycleConfigurationCommand, GetBucketLocationCommand, GetBucketLoggingCommand, GetBucketNotificationConfigurationCommand, GetBucketOwnershipControlsCommand, GetBucketPolicyCommand, GetBucketReplicationCommand, GetBucketTaggingCommand, GetBucketVersioningCommand, GetBucketWebsiteCommand, GetObjectCommand, GetObjectLockConfigurationCommand, GetPublicAccessBlockCommand, HeadBucketCommand, ListBucketAnalyticsConfigurationsCommand, ListBucketIntelligentTieringConfigurationsCommand, ListBucketInventoryConfigurationsCommand, ListBucketMetricsConfigurationsCommand, ListObjectVersionsCommand, ListObjectsV2Command, NoSuchBucket, PutBucketAccelerateConfigurationCommand, PutBucketAnalyticsConfigurationCommand, PutBucketCorsCommand, PutBucketEncryptionCommand, PutBucketIntelligentTieringConfigurationCommand, PutBucketInventoryConfigurationCommand, PutBucketLifecycleConfigurationCommand, PutBucketLoggingCommand, PutBucketMetricsConfigurationCommand, PutBucketNotificationConfigurationCommand, PutBucketOwnershipControlsCommand, PutBucketPolicyCommand, PutBucketReplicationCommand, PutBucketTaggingCommand, PutBucketVersioningCommand, PutBucketWebsiteCommand, PutObjectCommand, PutObjectLockConfigurationCommand, PutPublicAccessBlockCommand, S3Client, S3ServiceException } from "@aws-sdk/client-s3";
|
|
@@ -591,6 +591,37 @@ const useCdkBootstrapAssetsOption = new Option("--use-cdk-bootstrap-assets", "Pu
|
|
|
591
591
|
*/
|
|
592
592
|
const strictGetattOption = new Option("--strict-getatt", "Fail the deploy on ANY Fn::GetAtt that falls back to the physical ID because cdkd cannot construct the attribute (default: warn and use the physical ID, hard-failing only when the fallback is knowably wrong — an *Arn attribute with a non-ARN value or a *Url attribute with a non-http(s) value). Also fails the deploy when a stack Output cannot be resolved (default: warn and store no value).").default(false);
|
|
593
593
|
/**
|
|
594
|
+
* Issue [#1960](https://github.com/go-to-k/cdkd/issues/1960) —
|
|
595
|
+
* `--allow-unaddressed` (deploy only). A deploy that finishes having left a
|
|
596
|
+
* resource cdkd was responsible for ALIVE in AWS and no longer in state exits
|
|
597
|
+
* 2 (partial failure), matching what `cdkd destroy` has done for the identical
|
|
598
|
+
* outcome since issue #1752. Two cases produce it:
|
|
599
|
+
*
|
|
600
|
+
* - a skipped DELETE (issue #1762) — a resource removed from the template
|
|
601
|
+
* whose provider could not issue the delete, typically because the state
|
|
602
|
+
* record is malformed (see `src/provisioning/composite-id.ts`);
|
|
603
|
+
* - a partial UPDATE (issue #1819) — a replacement whose new resource was
|
|
604
|
+
* created and whose OLD one survives untracked, most commonly an ACM
|
|
605
|
+
* certificate a consumer still references.
|
|
606
|
+
*
|
|
607
|
+
* This flag forces exit 0 for both. It exists because the partial-UPDATE case
|
|
608
|
+
* has a legitimate not-yet-fixable window: an ACM replacement blocked on
|
|
609
|
+
* `DescribeCertificate.InUseBy` clears itself once the consumer (e.g. a
|
|
610
|
+
* CloudFront distribution in another stack) finishes updating, so a pipeline
|
|
611
|
+
* can be red for a cause it cannot act on. Wrapping the command in a shell
|
|
612
|
+
* exit-code test is NOT an equivalent workaround — `cdkd deploy` also exits 2
|
|
613
|
+
* for `MacroExpansionError` and `ResourceUpdateNotSupportedError`, so a
|
|
614
|
+
* `|| [ $? -eq 2 ]` wrapper silences unrelated real failures; this flag is
|
|
615
|
+
* scoped to the one cause.
|
|
616
|
+
*
|
|
617
|
+
* The flag suppresses only the EXIT CODE. The summary rows
|
|
618
|
+
* (`Skipped (not deleted)` / `of which left an orphaned predecessor`), the
|
|
619
|
+
* per-resource warnings, and the `RunCounts.skipped` figure in `cdkd events`
|
|
620
|
+
* are all emitted unchanged, so a run that used it still says in its log that
|
|
621
|
+
* a resource survived.
|
|
622
|
+
*/
|
|
623
|
+
const allowUnaddressedOption = new Option("--allow-unaddressed", "Exit 0 even when the deploy left a resource unaddressed — a DELETE the provider could not issue, or a replacement whose old resource survives untracked (default: exit 2, matching cdkd destroy). The summary rows and warnings are printed either way; only the exit code changes.").default(false);
|
|
624
|
+
/**
|
|
594
625
|
* Issue #1697 — `--no-cfn-fallback` (deploy + diff). By default, a
|
|
595
626
|
* cross-stack reference (`Fn::ImportValue` / `Fn::GetStackOutput`) that is
|
|
596
627
|
* not found in cdkd state falls back to CloudFormation (`ListExports` /
|
|
@@ -639,6 +670,7 @@ const deployOptions = [
|
|
|
639
670
|
useCdkBootstrapAssetsOption,
|
|
640
671
|
strictGetattOption,
|
|
641
672
|
noCfnFallbackOption,
|
|
673
|
+
allowUnaddressedOption,
|
|
642
674
|
...resourceTimeoutOptions
|
|
643
675
|
];
|
|
644
676
|
/**
|
|
@@ -2069,15 +2101,31 @@ function startRunRecorder(args) {
|
|
|
2069
2101
|
return recorder;
|
|
2070
2102
|
}
|
|
2071
2103
|
/**
|
|
2072
|
-
* Emit a
|
|
2073
|
-
* (dry-run / older state). `durationMs` is
|
|
2074
|
-
* carry one at the run level).
|
|
2104
|
+
* Emit a `RUN_FINISHED` carrying COUNTS under a caller-chosen result. No-op
|
|
2105
|
+
* when `recorder` is `undefined` (dry-run / older state). `durationMs` is
|
|
2106
|
+
* optional (destroy does not carry one at the run level).
|
|
2107
|
+
*
|
|
2108
|
+
* Replaced a `recordRunSucceeded` that hard-coded `'SUCCEEDED'`, because
|
|
2109
|
+
* deploy has an outcome that is neither a clean success nor a thrown failure:
|
|
2110
|
+
* a run that finished, failed no resource, and yet left one
|
|
2111
|
+
* cdkd was responsible for alive in AWS (issue
|
|
2112
|
+
* [#1960](https://github.com/go-to-k/cdkd/issues/1960)). That run exits 2, so
|
|
2113
|
+
* recording it as `SUCCEEDED` would put the durable post-mortem at odds with
|
|
2114
|
+
* the exit code the same run returned.
|
|
2115
|
+
*
|
|
2116
|
+
* {@link recordRunFailed} cannot serve it: that one carries error metadata and
|
|
2117
|
+
* NO counts, and the `skipped` count is the only thing in the summary that says
|
|
2118
|
+
* a resource survived — dropping it would leave `cdkd events` showing a failed
|
|
2119
|
+
* run naming nothing that failed, which is the exact shape the destroy side
|
|
2120
|
+
* added `skipped` to avoid. `cdkd destroy` already records `FAILED` for this
|
|
2121
|
+
* outcome (see the `skippedCount > 0` arm in src/cli/commands/destroy.ts), so
|
|
2122
|
+
* this is parity rather than a new convention.
|
|
2075
2123
|
*/
|
|
2076
|
-
function
|
|
2124
|
+
function recordRunOutcome(recorder, stackName, result, counts, durationMs) {
|
|
2077
2125
|
recorder?.record({
|
|
2078
2126
|
eventType: "RUN_FINISHED",
|
|
2079
2127
|
stackName,
|
|
2080
|
-
result
|
|
2128
|
+
result,
|
|
2081
2129
|
...durationMs !== void 0 && { durationMs },
|
|
2082
2130
|
counts
|
|
2083
2131
|
});
|
|
@@ -14308,8 +14356,10 @@ function isIndexBusyDeleteError(message) {
|
|
|
14308
14356
|
* gate uses, because THAT default is sized for a wait that runs once while
|
|
14309
14357
|
* this one runs per retry: the caller's wall clock is
|
|
14310
14358
|
* `its retry budget x this x ~1.2s per poll + withRetry's backoff`, and `destroy-runner.ts` runs the delete under a per-resource
|
|
14311
|
-
* deadline (30 min by default;
|
|
14312
|
-
*
|
|
14359
|
+
* deadline (30 min by default; since issue #1955 both DynamoDB providers
|
|
14360
|
+
* self-report that same 30 min through `getMinResourceTimeoutMs`, so a lowered
|
|
14361
|
+
* `--resource-timeout` cannot shrink it below what the shared delete budget in
|
|
14362
|
+
* `providers/dynamodb-delete-budget.ts` assumes). At 900 polls the product would be
|
|
14313
14363
|
* ~4.2h, so a genuinely stuck index would produce a 30-minute wait ending in a
|
|
14314
14364
|
* generic `ResourceTimeoutError` that never mentions indexes. At 60 the LOOP's
|
|
14315
14365
|
* worst case is ~18.4 min on `AWS::DynamoDB::Table` and ~10.4 min on
|
|
@@ -14388,7 +14438,10 @@ const DELETE_INDEX_WAIT_PROCEED_NOTE = `the DeleteTable goes ahead anyway and AW
|
|
|
14388
14438
|
*/
|
|
14389
14439
|
async function waitForIndexesSettled(opts) {
|
|
14390
14440
|
const { tableName, logicalId, logger, describeTable, maxAttempts, proceedNote } = opts;
|
|
14391
|
-
|
|
14441
|
+
const budgetWait = opts.budgetWait;
|
|
14442
|
+
let attempt = 0;
|
|
14443
|
+
while (budgetWait ? budgetWait.nextPoll() : attempt < maxAttempts) {
|
|
14444
|
+
attempt += 1;
|
|
14392
14445
|
try {
|
|
14393
14446
|
if (!hasTransitionalIndex((await describeTable()).Table?.GlobalSecondaryIndexes ?? [])) return;
|
|
14394
14447
|
} catch (err) {
|
|
@@ -14410,7 +14463,8 @@ async function waitForIndexesSettled(opts) {
|
|
|
14410
14463
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
14411
14464
|
}
|
|
14412
14465
|
const pollSeconds = INDEX_SETTLE_POLL_INTERVAL_MS / 1e3;
|
|
14413
|
-
|
|
14466
|
+
const pollsRun = budgetWait ? budgetWait.pollsRun : maxAttempts;
|
|
14467
|
+
logger.warn(`Indexes on ${tableName} (${logicalId}) did not all reach ACTIVE within ${pollsRun} DescribeTable polls (~${pollSeconds}s apart, so a little over ${pollsRun * pollSeconds}s of wall clock).${budgetWait?.note() ?? ""} ${proceedNote}`);
|
|
14414
14468
|
}
|
|
14415
14469
|
/**
|
|
14416
14470
|
* The ONE line the retry prints at default verbosity, for both providers.
|
|
@@ -14525,12 +14579,486 @@ async function deleteTableWithIndexBusyRetry(opts) {
|
|
|
14525
14579
|
await opts.deleteTable();
|
|
14526
14580
|
}, opts.logicalId, {
|
|
14527
14581
|
maxRetries: opts.maxRetries,
|
|
14528
|
-
isRetryable: (message) => isIndexBusyDeleteError(message),
|
|
14582
|
+
isRetryable: (message) => isIndexBusyDeleteError(message) && (opts.shouldKeepRetrying?.() ?? true),
|
|
14529
14583
|
logger: opts.logger,
|
|
14530
14584
|
sleep: (ms) => opts.sleepSeam.sleep ? opts.sleepSeam.sleep(ms) : new Promise((resolve) => setTimeout(resolve, ms))
|
|
14531
14585
|
});
|
|
14532
14586
|
}
|
|
14533
14587
|
|
|
14588
|
+
//#endregion
|
|
14589
|
+
//#region src/utils/elapsed-budget.ts
|
|
14590
|
+
/**
|
|
14591
|
+
* ONE wall-clock allowance shared by a sequence of independently capped waits.
|
|
14592
|
+
*
|
|
14593
|
+
* **The problem it exists for.** A provider operation is run under a
|
|
14594
|
+
* per-resource deadline (`withResourceDeadline`, applied by
|
|
14595
|
+
* `src/cli/commands/destroy-runner.ts` and the deploy engine). Inside that
|
|
14596
|
+
* deadline a single `delete()` can stack several polling waits, each with its
|
|
14597
|
+
* own justified cap. Every cap is defensible on its own and the SUM is not:
|
|
14598
|
+
* three ~12-18 minute waits inside one 30-minute deadline overshoot it, and
|
|
14599
|
+
* `withResourceDeadline` does NOT cancel what it wraps — it rejects with a
|
|
14600
|
+
* generic timeout while the polling loop keeps running behind the run that has
|
|
14601
|
+
* already reported failure (issue #1955).
|
|
14602
|
+
*
|
|
14603
|
+
* A shared budget converts "each wait is capped" into "the PATH is capped": a
|
|
14604
|
+
* wait asks how many polls still fit rather than how many its own constant
|
|
14605
|
+
* allows, so the path spends the allowance ONCE and stops itself while the
|
|
14606
|
+
* deadline still has margin. Stopping ourselves is what preserves the
|
|
14607
|
+
* operation's own actionable error (AWS's "Cannot delete table while indexes
|
|
14608
|
+
* are being ..." rather than `ResourceTimeoutError`) and what stops work from
|
|
14609
|
+
* outliving the run.
|
|
14610
|
+
*
|
|
14611
|
+
* Deliberately NOT an `AbortController`: nothing here interrupts an in-flight
|
|
14612
|
+
* SDK call. It bounds how many MORE polls a loop will start, which is the whole
|
|
14613
|
+
* of the cost for a wait built out of `describe`-then-sleep — and it needs no
|
|
14614
|
+
* plumbing through every provider call.
|
|
14615
|
+
*
|
|
14616
|
+
* The clock is injectable so a test can drive elapsed time without faking
|
|
14617
|
+
* timers globally (the polling loops under test are real `async` loops; a
|
|
14618
|
+
* global fake clock would freeze them too).
|
|
14619
|
+
*/
|
|
14620
|
+
/**
|
|
14621
|
+
* The default clock: MONOTONIC, not `Date.now()`.
|
|
14622
|
+
*
|
|
14623
|
+
* `Date.now()` follows wall clock, so an NTP correction moves it in either
|
|
14624
|
+
* direction mid-operation — and the two directions fail differently. Backwards
|
|
14625
|
+
* would hand the path MORE than its total (guarded below by clamping elapsed at
|
|
14626
|
+
* zero); FORWARDS instantly drains the allowance, which cuts every remaining
|
|
14627
|
+
* wait to its one-poll floor and makes the whole path report "cdkd's allowance
|
|
14628
|
+
* ran out" for something that never happened. A 30-minute delete is long enough
|
|
14629
|
+
* to sit across a correction.
|
|
14630
|
+
*
|
|
14631
|
+
* `performance.now()` is monotonic and millisecond-denominated (a float, which
|
|
14632
|
+
* only matters to a consumer dividing by a poll cost — flooring already handles
|
|
14633
|
+
* it). Guarded rather than assumed present, so a runtime without it degrades to
|
|
14634
|
+
* the old behaviour instead of throwing.
|
|
14635
|
+
*/
|
|
14636
|
+
const monotonicNowMs = typeof globalThis.performance?.now === "function" ? () => globalThis.performance.now() : () => Date.now();
|
|
14637
|
+
/** A monotonic wall-clock allowance, started when the object is created. */
|
|
14638
|
+
var ElapsedBudget = class {
|
|
14639
|
+
/** Total wall clock this budget may spend, in milliseconds. */
|
|
14640
|
+
totalMs;
|
|
14641
|
+
clock;
|
|
14642
|
+
startedAt;
|
|
14643
|
+
constructor(totalMs, clock = monotonicNowMs) {
|
|
14644
|
+
if (!Number.isFinite(totalMs) || totalMs <= 0) throw new RangeError(`ElapsedBudget: totalMs must be a positive finite number (got ${totalMs})`);
|
|
14645
|
+
this.totalMs = totalMs;
|
|
14646
|
+
this.clock = clock;
|
|
14647
|
+
this.startedAt = clock();
|
|
14648
|
+
}
|
|
14649
|
+
/** Wall clock spent since the budget started. */
|
|
14650
|
+
elapsedMs() {
|
|
14651
|
+
return Math.max(0, this.clock() - this.startedAt);
|
|
14652
|
+
}
|
|
14653
|
+
/** Wall clock still available, never negative. */
|
|
14654
|
+
remainingMs() {
|
|
14655
|
+
return Math.max(0, this.totalMs - this.elapsedMs());
|
|
14656
|
+
}
|
|
14657
|
+
isExhausted() {
|
|
14658
|
+
return this.remainingMs() <= 0;
|
|
14659
|
+
}
|
|
14660
|
+
/**
|
|
14661
|
+
* How many polls of `perAttemptMs` still fit, clamped into
|
|
14662
|
+
* `[minAttempts, cap]`.
|
|
14663
|
+
*
|
|
14664
|
+
* `minAttempts` defaults to 1 rather than 0 on purpose: every caller here is
|
|
14665
|
+
* a `describe`-then-decide loop, and a loop granted ZERO attempts reports
|
|
14666
|
+
* "did not settle" without ever having looked — a verdict it did not earn,
|
|
14667
|
+
* and one that reads as a bug in the wait rather than as a spent budget. One
|
|
14668
|
+
* poll costs ~1s and is the difference between a guess and an observation.
|
|
14669
|
+
* The overshoot it can add is therefore bounded by one poll per wait, which
|
|
14670
|
+
* the caller's margin against its deadline absorbs.
|
|
14671
|
+
*/
|
|
14672
|
+
attemptsWithin(cap, perAttemptMs, minAttempts = 1) {
|
|
14673
|
+
if (!Number.isFinite(cap) || cap <= 0) return 0;
|
|
14674
|
+
if (!Number.isFinite(perAttemptMs) || perAttemptMs <= 0) return Math.min(minAttempts, cap);
|
|
14675
|
+
const affordable = Math.floor(this.remainingMs() / perAttemptMs);
|
|
14676
|
+
return Math.max(Math.min(minAttempts, cap), Math.min(cap, affordable));
|
|
14677
|
+
}
|
|
14678
|
+
};
|
|
14679
|
+
/**
|
|
14680
|
+
* Budgets keyed by the thing they bound, so an operation that is RE-ENTERED
|
|
14681
|
+
* inside one deadline keeps spending the same allowance.
|
|
14682
|
+
*
|
|
14683
|
+
* That re-entry is the case a per-call budget silently misses. `destroy-runner`
|
|
14684
|
+
* wraps its whole outer retry loop — up to four `delete()` calls — in ONE
|
|
14685
|
+
* deadline, so a `delete()` that fails on something the outer loop classes as
|
|
14686
|
+
* retryable (a throttle) starts again from the top. A budget created per call
|
|
14687
|
+
* would reset there and the same path could be paid twice inside one deadline;
|
|
14688
|
+
* a budget looked up by key does not.
|
|
14689
|
+
*
|
|
14690
|
+
* Entries are RETAINED on failure (that is the point) and released by the
|
|
14691
|
+
* caller on a terminal outcome, so the map holds at most one entry per
|
|
14692
|
+
* in-flight resource.
|
|
14693
|
+
*
|
|
14694
|
+
* **The key must identify the resource, not just name it.** A registry holding
|
|
14695
|
+
* bare names collides whenever two distinct resources share one — for AWS that
|
|
14696
|
+
* is any name-scoped-by-region identifier — and the collision is silent: the
|
|
14697
|
+
* second resource inherits the first's spent allowance. Callers are expected to
|
|
14698
|
+
* qualify the key themselves (see `providers/dynamodb-delete-budget.ts`'s
|
|
14699
|
+
* `deleteBudgetKey`), because only the caller knows what the name is scoped by.
|
|
14700
|
+
*/
|
|
14701
|
+
var ElapsedBudgetRegistry = class {
|
|
14702
|
+
budgets = /* @__PURE__ */ new Map();
|
|
14703
|
+
/**
|
|
14704
|
+
* The budget for `key`, creating it on first use and REUSING it on re-entry.
|
|
14705
|
+
* `totalMs` and `clock` are only read when the entry is CREATED — a re-entry
|
|
14706
|
+
* must not be able to grant itself a fresh allowance, nor a fresh clock.
|
|
14707
|
+
*
|
|
14708
|
+
* `reuseWithinMs` bounds how long an entry may be reused for, and exists
|
|
14709
|
+
* because entries are deliberately RETAINED on failure: without it, a spent
|
|
14710
|
+
* allowance would be inherited by any later operation that happens to reach
|
|
14711
|
+
* the same key in the same process. The caller passes the wall clock of the
|
|
14712
|
+
* DEADLINE the allowance is sized against — past that, the deadline has
|
|
14713
|
+
* certainly fired and whoever is calling now belongs to a new operation with
|
|
14714
|
+
* a new deadline, so reuse would be wrong rather than conservative.
|
|
14715
|
+
*/
|
|
14716
|
+
acquire(key, totalMs, clock = monotonicNowMs, reuseWithinMs) {
|
|
14717
|
+
const existing = this.budgets.get(key);
|
|
14718
|
+
if (existing) {
|
|
14719
|
+
if (!(reuseWithinMs !== void 0 && existing.elapsedMs() > reuseWithinMs)) return existing;
|
|
14720
|
+
this.budgets.delete(key);
|
|
14721
|
+
}
|
|
14722
|
+
const created = new ElapsedBudget(totalMs, clock);
|
|
14723
|
+
this.budgets.set(key, created);
|
|
14724
|
+
return created;
|
|
14725
|
+
}
|
|
14726
|
+
/** Drop `key`'s budget — call on a TERMINAL outcome, never between retries. */
|
|
14727
|
+
release(key) {
|
|
14728
|
+
this.budgets.delete(key);
|
|
14729
|
+
}
|
|
14730
|
+
/** Live entries; exists so a test can prove release actually released. */
|
|
14731
|
+
get size() {
|
|
14732
|
+
return this.budgets.size;
|
|
14733
|
+
}
|
|
14734
|
+
clear() {
|
|
14735
|
+
this.budgets.clear();
|
|
14736
|
+
}
|
|
14737
|
+
};
|
|
14738
|
+
|
|
14739
|
+
//#endregion
|
|
14740
|
+
//#region src/provisioning/providers/dynamodb-delete-budget.ts
|
|
14741
|
+
/**
|
|
14742
|
+
* The ONE wall-clock allowance the DynamoDB DELETE path spends, for both
|
|
14743
|
+
* `AWS::DynamoDB::Table` and `AWS::DynamoDB::GlobalTable` (issue #1955).
|
|
14744
|
+
*
|
|
14745
|
+
* **What was wrong.** Every wait on that path is individually capped and
|
|
14746
|
+
* individually justified, and the caps do not know about each other:
|
|
14747
|
+
*
|
|
14748
|
+
* ```
|
|
14749
|
+
* --remove-protection 600 polls ~12 min GlobalTable, ACTIVE wait, FIRST
|
|
14750
|
+
* ACTIVE wait 60 polls ~72s Table, ACTIVE wait, FIRST
|
|
14751
|
+
* auto-scaling teardown see below GlobalTable, table + per GSI,
|
|
14752
|
+
* per replica, BEFORE the first poll
|
|
14753
|
+
* waitForReplicaGone 600 polls ~12 min per NON-LOCAL replica
|
|
14754
|
+
* #1521 pre-delete gate 900 polls ~18 min (issue #1521)
|
|
14755
|
+
* index-busy retry loop ~10.4 min at 8 retries / ~18.4 at 14 (#1830/#1931/#1950)
|
|
14756
|
+
* waitForTableGone 600 polls ~12 min (only on a late SUCCESS)
|
|
14757
|
+
* ```
|
|
14758
|
+
*
|
|
14759
|
+
* Every poll figure is `polls x ~1.2s`, the MEASURED cost of a sleep plus a
|
|
14760
|
+
* `DescribeTable` round trip — so a 600-poll wait is ~12 min, not the ~10 min
|
|
14761
|
+
* its `1 poll = 1s` era comments used to say.
|
|
14762
|
+
*
|
|
14763
|
+
* The `--remove-protection` term is the sharpest of the ones a first pass at
|
|
14764
|
+
* this fix missed: it runs FIRST, its predicate is `ACTIVE` *and* no
|
|
14765
|
+
* transitional index — the very condition the rest of the path is about — so on
|
|
14766
|
+
* a `cdkd destroy --remove-protection` against a table with a building GSI it
|
|
14767
|
+
* spends its full ~12 min at t=0 and leaves the gate and the loop the
|
|
14768
|
+
* remainder.
|
|
14769
|
+
*
|
|
14770
|
+
* The auto-scaling teardown is not poll-shaped at all, and its story changed in
|
|
14771
|
+
* review. It was believed to inherit `withRetry`'s default schedule; it did
|
|
14772
|
+
* not — the two `withRetry` sites in that method are both in the REGISTER
|
|
14773
|
+
* branch, and every delete-path caller passes `newSettings: undefined`, which
|
|
14774
|
+
* takes the bare-`send` teardown branch. So the term was single-attempt and
|
|
14775
|
+
* fast, and a budget clamp on it was inert. What the correction exposed is a
|
|
14776
|
+
* half-applied twin: the register branch retries throttles because an
|
|
14777
|
+
* un-retried `ThrottlingException` leaves a target silently UNregistered, and
|
|
14778
|
+
* the teardown has the exact symmetric hazard — a silently RETAINED target that
|
|
14779
|
+
* a future table of the same name inherits (PR #403) — with no retry at all.
|
|
14780
|
+
* The teardown is now wrapped in the same throttle-only retry, bounded by
|
|
14781
|
+
* {@link autoScalingRetriesWithinDeleteBudget}, so the term is real, bounded,
|
|
14782
|
+
* and no longer asymmetric.
|
|
14783
|
+
*
|
|
14784
|
+
* `destroy-runner.ts` runs the whole thing under ONE per-resource deadline
|
|
14785
|
+
* (`DEFAULT_RESOURCE_TIMEOUT_MS`, 30 min). A replicated GlobalTable whose index
|
|
14786
|
+
* is transitioning reached ~40 min, a single-region one reached ~40.4 min on
|
|
14787
|
+
* the late-success path, and a single-region `--remove-protection` destroy
|
|
14788
|
+
* reached 12 + 18 + 10.4 ~= ~40.4 min on the EXHAUSTED-budget path — all over
|
|
14789
|
+
* it. Three consequences, all of them the
|
|
14790
|
+
* issue's:
|
|
14791
|
+
*
|
|
14792
|
+
* 1. **Stacked waits.** The deadline was spent three or four times over,
|
|
14793
|
+
* because each cap was sized as though it were the only one.
|
|
14794
|
+
* 2. **A non-cancelling overshoot.** `withResourceDeadline` rejects but does
|
|
14795
|
+
* not cancel, so the run reported a generic `ResourceTimeoutError` that
|
|
14796
|
+
* never mentions indexes — replacing AWS's own actionable sentence, which
|
|
14797
|
+
* the bounded re-arm exists to surface — WHILE the polling loop kept
|
|
14798
|
+
* running behind the failed run.
|
|
14799
|
+
* 3. **Throttle compounding.** `deleteTableWithIndexBusyRetry`'s inner
|
|
14800
|
+
* `isRetryable` is index-busy-only by design, so a THROTTLED `DeleteTable`
|
|
14801
|
+
* escapes to `destroy-runner.ts`'s outer loop, which DOES class throttles
|
|
14802
|
+
* as retryable and re-enters `delete()` from the top — paying the whole
|
|
14803
|
+
* settle-and-retry sequence again inside the SAME deadline (2 x ~19.6 min
|
|
14804
|
+
* ~= ~39 min on the `Table` path after issue #1950).
|
|
14805
|
+
*
|
|
14806
|
+
* **What replaces it.** A budget acquired once per physical table and shared by
|
|
14807
|
+
* every wait on the path. Each wait asks how many polls still fit rather than
|
|
14808
|
+
* how many its own constant allows, so the path spends the allowance ONCE
|
|
14809
|
+
* ((1)), stops itself while the deadline still has minutes of margin — leaving
|
|
14810
|
+
* AWS's own error as the failure and nothing polling behind it ((2)) — and,
|
|
14811
|
+
* because the budget is keyed by physical id in a registry rather than created
|
|
14812
|
+
* per call, a re-entry from the outer retry loop CONTINUES the same allowance
|
|
14813
|
+
* instead of restarting it ((3)).
|
|
14814
|
+
*
|
|
14815
|
+
* The per-type retry budgets (`TABLE_DELETE_INDEX_BUSY_MAX_RETRIES` /
|
|
14816
|
+
* `GLOBAL_TABLE_DELETE_INDEX_BUSY_MAX_RETRIES`) are unchanged and still
|
|
14817
|
+
* meaningful: they shape HOW the allowance is spent — how many fresh
|
|
14818
|
+
* `DeleteTable` probes it buys — while this bounds how much there is. What
|
|
14819
|
+
* changes is that neither of them, nor any wait cap, is load-bearing for
|
|
14820
|
+
* staying under the deadline any more.
|
|
14821
|
+
*/
|
|
14822
|
+
/**
|
|
14823
|
+
* The `DescribeTable` round trip each poll pays ON TOP of its sleep.
|
|
14824
|
+
*
|
|
14825
|
+
* MEASURED, not assumed: the live issue-#1950 run's 60-poll re-arm took ~72s
|
|
14826
|
+
* and its per-attempt gaps were 73 / 74 / 77 / 80 / 80 / 80s once `withRetry`'s
|
|
14827
|
+
* backoff is subtracted, i.e. ~1.2s per poll against a 1s sleep. Pricing a poll
|
|
14828
|
+
* at the bare sleep interval is what once let a ~10.4 min loop be quoted as
|
|
14829
|
+
* ~8.8 min.
|
|
14830
|
+
*/
|
|
14831
|
+
const DYNAMODB_DELETE_POLL_RTT_MS = 200;
|
|
14832
|
+
/**
|
|
14833
|
+
* What ONE poll of a delete-path wait costs in wall clock. Every budget->polls
|
|
14834
|
+
* conversion below goes through this, and it READS
|
|
14835
|
+
* {@link INDEX_SETTLE_POLL_INTERVAL_MS} rather than restating 1000 so the two
|
|
14836
|
+
* cannot drift while this arithmetic claims they agree.
|
|
14837
|
+
*/
|
|
14838
|
+
const DYNAMODB_DELETE_POLL_COST_MS = INDEX_SETTLE_POLL_INTERVAL_MS + 200;
|
|
14839
|
+
/**
|
|
14840
|
+
* The allowance the WHOLE delete path gets, however many waits it contains.
|
|
14841
|
+
*
|
|
14842
|
+
* **Why 26 minutes.** It has to sit under the per-resource deadline with enough
|
|
14843
|
+
* margin that the deadline never fires first — because the whole point is to
|
|
14844
|
+
* fail with the operation's own error rather than with a generic timeout — and
|
|
14845
|
+
* it has to be large enough that no shape which completes TODAY starts failing:
|
|
14846
|
+
*
|
|
14847
|
+
* ```
|
|
14848
|
+
* Table, single pass ACTIVE wait ~72s + loop(14) ~18.4 min = ~19.6 min fits
|
|
14849
|
+
* GlobalTable, single-region #1521 gate ~18 min + loop(8) ~10.4 min = ~28.4 min clamped by ~2.4 min
|
|
14850
|
+
* ```
|
|
14851
|
+
*
|
|
14852
|
+
* The one shape it shortens is the GlobalTable single-region EXHAUSTED-budget
|
|
14853
|
+
* case, and shortening it costs nothing: that path ends by throwing AWS's own
|
|
14854
|
+
* index-busy sentence either way, ~2.4 min sooner now.
|
|
14855
|
+
*
|
|
14856
|
+
* **Every OTHER shape was already past the 30-minute deadline before this
|
|
14857
|
+
* change**, which is why counting the two newly-added terms does not move the
|
|
14858
|
+
* number. `--remove-protection` on a single-region GlobalTable is
|
|
14859
|
+
* 12 + 18 + 10.4 ~= ~40.4 min; on a one-replica table 12 + 12 + 18 + 10.4 ~=
|
|
14860
|
+
* ~52.4 min; the late-success gone-wait adds ~12 min to whichever of those it
|
|
14861
|
+
* follows; the auto-scaling teardown is unbounded. All of them previously ended
|
|
14862
|
+
* in the generic `ResourceTimeoutError` with work still running, so a bounded
|
|
14863
|
+
* stop is a strict improvement there rather than a trade — there is no shape
|
|
14864
|
+
* between 26 and 30 minutes that this budget takes away except the ~2.4 min
|
|
14865
|
+
* above, because 30 minutes was already the ceiling and everything past it
|
|
14866
|
+
* failed.
|
|
14867
|
+
*
|
|
14868
|
+
* It is deliberately ONE number for both types rather than a per-type pair. The
|
|
14869
|
+
* two per-type RETRY budgets exist because the two types arrive at the loop
|
|
14870
|
+
* having spent different amounts; this is the ceiling on the total, and a total
|
|
14871
|
+
* does not need to know which terms produced it.
|
|
14872
|
+
*/
|
|
14873
|
+
const DYNAMODB_DELETE_BUDGET_MS = 26 * 6e4;
|
|
14874
|
+
/**
|
|
14875
|
+
* What both DynamoDB providers self-report through `getMinResourceTimeoutMs()`.
|
|
14876
|
+
*
|
|
14877
|
+
* The engine resolves the per-resource deadline as
|
|
14878
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), slowTypeFloor,
|
|
14879
|
+
* globalCliDefault)`, so declaring this makes "the budget fits inside the
|
|
14880
|
+
* deadline" a GUARANTEE rather than an assumption about the CLI default: a user
|
|
14881
|
+
* who lowers `--resource-timeout` globally can no longer re-create the crossing
|
|
14882
|
+
* this module exists to remove. It equals today's `DEFAULT_RESOURCE_TIMEOUT_MS`
|
|
14883
|
+
* (30 min), so at default settings nothing moves — `max(30m, 30m)` is 30m.
|
|
14884
|
+
*
|
|
14885
|
+
* A per-type override (`--resource-timeout AWS::DynamoDB::Table=5m`) still
|
|
14886
|
+
* wins; that is the documented escape hatch, and it is the only way to make
|
|
14887
|
+
* this type abort sooner than its self-report.
|
|
14888
|
+
*/
|
|
14889
|
+
const DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS = 18e5;
|
|
14890
|
+
/**
|
|
14891
|
+
* Test seam for the budget, mirroring the providers' `deleteTableRetryDelays`
|
|
14892
|
+
* sleep seams. Production leaves BOTH fields unset, so
|
|
14893
|
+
* {@link DYNAMODB_DELETE_BUDGET_MS} and the real wall clock apply.
|
|
14894
|
+
*
|
|
14895
|
+
* `clock` is here rather than only `totalMs` because the behaviour worth
|
|
14896
|
+
* testing is what happens as the budget DRAINS, and a test cannot drain a
|
|
14897
|
+
* 26-minute allowance by waiting. Shrinking `totalMs` alone reaches the
|
|
14898
|
+
* clamped-poll-count arm but never the EXHAUSTED one, which is the arm that
|
|
14899
|
+
* stops the retry loop and preserves AWS's own message — the whole point of
|
|
14900
|
+
* issue #1955. A controllable clock reaches both, deterministically, without
|
|
14901
|
+
* `vi.useFakeTimers()` freezing the provider's own polling loops.
|
|
14902
|
+
*/
|
|
14903
|
+
const dynamoDbDeleteBudgetOverride = {};
|
|
14904
|
+
function resolveDynamoDbDeleteBudgetMs() {
|
|
14905
|
+
const override = dynamoDbDeleteBudgetOverride.totalMs;
|
|
14906
|
+
return override !== void 0 && override > 0 ? override : DYNAMODB_DELETE_BUDGET_MS;
|
|
14907
|
+
}
|
|
14908
|
+
function resolveDynamoDbDeleteBudgetClock() {
|
|
14909
|
+
return dynamoDbDeleteBudgetOverride.clock ?? monotonicNowMs;
|
|
14910
|
+
}
|
|
14911
|
+
/**
|
|
14912
|
+
* The registry key for one table's delete allowance.
|
|
14913
|
+
*
|
|
14914
|
+
* A DynamoDB table NAME is unique per region, not globally, so a bare
|
|
14915
|
+
* `physicalId` collides across regions and the second table silently inherits
|
|
14916
|
+
* the first's spent allowance. In practice `destroy-runner.ts` builds a fresh
|
|
14917
|
+
* provider (and therefore a fresh registry) per region, so the collision is not
|
|
14918
|
+
* reachable today — but that is an invariant of the CALLER, enforced nowhere,
|
|
14919
|
+
* and the fix costs one string concat. `\0` cannot appear in either component.
|
|
14920
|
+
*
|
|
14921
|
+
* `region` is the STATE's region (`DeleteContext.expectedRegion`), which is
|
|
14922
|
+
* available synchronously at the top of `delete()` — unlike the client's own
|
|
14923
|
+
* region, which needs an await and would push the acquire past the
|
|
14924
|
+
* `--remove-protection` flip. Pre-region state records leave it undefined,
|
|
14925
|
+
* which degrades to the old bare-name key rather than failing.
|
|
14926
|
+
*/
|
|
14927
|
+
function deleteBudgetKey(physicalId, region) {
|
|
14928
|
+
return `${region ?? ""}\u0000${physicalId}`;
|
|
14929
|
+
}
|
|
14930
|
+
/**
|
|
14931
|
+
* The poll cap a wait may actually use: its own constant, or what the shared
|
|
14932
|
+
* budget can still afford, whichever is smaller.
|
|
14933
|
+
*
|
|
14934
|
+
* `budget` is optional and an absent one means "unbounded by this mechanism" —
|
|
14935
|
+
* the CREATE / UPDATE callers of these same waits keep their original caps
|
|
14936
|
+
* verbatim, since the deadline arithmetic this fixes is the delete path's.
|
|
14937
|
+
*/
|
|
14938
|
+
function pollsWithinDeleteBudget(cap, budget) {
|
|
14939
|
+
if (!budget) return {
|
|
14940
|
+
attempts: cap,
|
|
14941
|
+
cap,
|
|
14942
|
+
clampedByBudget: false
|
|
14943
|
+
};
|
|
14944
|
+
const attempts = budget.attemptsWithin(cap, DYNAMODB_DELETE_POLL_COST_MS);
|
|
14945
|
+
return {
|
|
14946
|
+
attempts,
|
|
14947
|
+
cap,
|
|
14948
|
+
clampedByBudget: attempts < cap
|
|
14949
|
+
};
|
|
14950
|
+
}
|
|
14951
|
+
/**
|
|
14952
|
+
* Polls below which a wait is not EVIDENCE about AWS, only evidence that cdkd
|
|
14953
|
+
* stopped asking.
|
|
14954
|
+
*
|
|
14955
|
+
* The warn-versus-throw split on `waitForTableGone`, and the strong wording of
|
|
14956
|
+
* the exhaustion note, both hang off this, and before it existed both hung off
|
|
14957
|
+
* `attempts < cap` — which is arithmetic, not evidence. With ~14 min of the
|
|
14958
|
+
* allowance already spent a 600-poll wait is still granted ~500 polls, i.e.
|
|
14959
|
+
* ten real minutes of `DescribeTable`; calling that "cdkd's allowance ran out,
|
|
14960
|
+
* AWS was not given a real wait" is false, and downgrading its failure to a
|
|
14961
|
+
* warning is a decision made by two minutes of wall clock rather than by
|
|
14962
|
+
* anything the user can act on.
|
|
14963
|
+
*
|
|
14964
|
+
* 60 polls (~72s) because the provider's own gone-wait note records that a
|
|
14965
|
+
* typical small-table delete completes in 5-30s: past roughly twice the top of
|
|
14966
|
+
* that range, cdkd HAS given AWS a real wait and a table still present is a
|
|
14967
|
+
* signal about AWS. Below it, cdkd barely looked, and the honest report is
|
|
14968
|
+
* about cdkd rather than about the table.
|
|
14969
|
+
*/
|
|
14970
|
+
const DELETE_SHORT_WAIT_POLLS = 60;
|
|
14971
|
+
var DeleteWaitImpl = class {
|
|
14972
|
+
cap;
|
|
14973
|
+
grantedPolls;
|
|
14974
|
+
ran = 0;
|
|
14975
|
+
brokeEarly = false;
|
|
14976
|
+
entryClamped;
|
|
14977
|
+
budget;
|
|
14978
|
+
constructor(cap, budget) {
|
|
14979
|
+
const polls = pollsWithinDeleteBudget(cap, budget);
|
|
14980
|
+
this.cap = polls.cap;
|
|
14981
|
+
this.grantedPolls = polls.attempts;
|
|
14982
|
+
this.entryClamped = polls.clampedByBudget;
|
|
14983
|
+
this.budget = budget;
|
|
14984
|
+
}
|
|
14985
|
+
nextPoll() {
|
|
14986
|
+
if (this.ran >= this.grantedPolls) return false;
|
|
14987
|
+
if (this.ran >= 1 && this.budget?.isExhausted() === true) {
|
|
14988
|
+
this.brokeEarly = true;
|
|
14989
|
+
return false;
|
|
14990
|
+
}
|
|
14991
|
+
this.ran += 1;
|
|
14992
|
+
return true;
|
|
14993
|
+
}
|
|
14994
|
+
get pollsRun() {
|
|
14995
|
+
return this.ran;
|
|
14996
|
+
}
|
|
14997
|
+
get endedByBudget() {
|
|
14998
|
+
return this.brokeEarly || this.entryClamped;
|
|
14999
|
+
}
|
|
15000
|
+
get cutShort() {
|
|
15001
|
+
return this.endedByBudget && this.ran < 60;
|
|
15002
|
+
}
|
|
15003
|
+
note() {
|
|
15004
|
+
if (!this.endedByBudget) return "";
|
|
15005
|
+
const minutes = Math.max(1, Math.round((this.budget?.totalMs ?? 0) / 6e4));
|
|
15006
|
+
if (this.cutShort) return ` — cdkd's shared ~${minutes}-minute delete allowance was already spent, so this wait made only ${this.ran} of its ${this.cap} polls and AWS was not given a real wait. Nothing is necessarily wrong with the table; re-run the destroy.`;
|
|
15007
|
+
return ` (cdkd's shared ~${minutes}-minute delete allowance capped this wait at ${this.ran} of its ${this.cap} polls.)`;
|
|
15008
|
+
}
|
|
15009
|
+
};
|
|
15010
|
+
/** Begin a delete-path wait that draws on the shared allowance. */
|
|
15011
|
+
function beginDeleteWait(cap, budget) {
|
|
15012
|
+
return new DeleteWaitImpl(cap, budget);
|
|
15013
|
+
}
|
|
15014
|
+
/**
|
|
15015
|
+
* `withRetry` retries the delete-path auto-scaling teardown may still fund.
|
|
15016
|
+
*
|
|
15017
|
+
* The teardown runs BEFORE the first budgeted poll — table-level plus one call
|
|
15018
|
+
* per GSI, per non-local replica, then again locally — and until this change it
|
|
15019
|
+
* had NO retry at all: the two `withRetry` sites in `applyAutoScalingDiff` are
|
|
15020
|
+
* both in the REGISTER branch, and every delete-path caller passes
|
|
15021
|
+
* `newSettings: undefined`, which takes the bare-`send` teardown branch. (An
|
|
15022
|
+
* earlier revision of this file claimed the teardown inherited `withRetry`'s
|
|
15023
|
+
* default schedule and would therefore spend ~47s per call under account-wide
|
|
15024
|
+
* throttling; that was false, and the clamp built on it was inert. The claim is
|
|
15025
|
+
* recorded here because it is the kind of arithmetic a later reader would
|
|
15026
|
+
* otherwise trust.)
|
|
15027
|
+
*
|
|
15028
|
+
* The teardown is now wrapped in the same throttle-only retry the register
|
|
15029
|
+
* branch uses, for the symmetric reason: that branch retries because an
|
|
15030
|
+
* un-retried `ThrottlingException` leaves a target silently UNregistered, and
|
|
15031
|
+
* an un-retried throttle here leaves one silently REGISTERED — the PR #403 leak
|
|
15032
|
+
* a future table of the same name inherits. Retrying is what makes a bound
|
|
15033
|
+
* necessary, and the bound is what makes retrying safe on a path that shares
|
|
15034
|
+
* one deadline with everything after it.
|
|
15035
|
+
*
|
|
15036
|
+
* Retries are bounded rather than the CALL being skipped, deliberately. The
|
|
15037
|
+
* teardown exists because a surviving scalable target is silently inherited by
|
|
15038
|
+
* a future table of the same name (PR #403), and skipping re-introduces that
|
|
15039
|
+
* leak. So a drained allowance still issues every call once —
|
|
15040
|
+
* `withRetry` with `maxRetries: 0` runs the operation exactly once — and still
|
|
15041
|
+
* warns actionably on each failure; it just stops paying to retry them.
|
|
15042
|
+
*/
|
|
15043
|
+
function autoScalingRetriesWithinDeleteBudget(budget, defaultMaxRetries) {
|
|
15044
|
+
if (!budget) return defaultMaxRetries;
|
|
15045
|
+
const surplusMs = budget.remainingMs() - AUTO_SCALING_TEARDOWN_RESERVE_MS;
|
|
15046
|
+
if (surplusMs <= 0) return 0;
|
|
15047
|
+
return Math.max(0, Math.min(defaultMaxRetries, Math.floor(surplusMs / AUTO_SCALING_RETRY_STEP_MS)));
|
|
15048
|
+
}
|
|
15049
|
+
/**
|
|
15050
|
+
* The part of the allowance the auto-scaling teardown may NOT spend on retries.
|
|
15051
|
+
*
|
|
15052
|
+
* Sized so the terms that actually delete the table — the #1521 gate, the
|
|
15053
|
+
* index-busy loop and the gone-wait — keep the bulk of the allowance no matter
|
|
15054
|
+
* how throttled the teardown is. 20 of the 26 minutes, leaving the teardown up
|
|
15055
|
+
* to ~6 min of retry backoff, which at ~8s a retry is ~45 retries spread across
|
|
15056
|
+
* however many calls the table's index and replica count produce.
|
|
15057
|
+
*/
|
|
15058
|
+
const AUTO_SCALING_TEARDOWN_RESERVE_MS = 20 * 6e4;
|
|
15059
|
+
/** `withRetry`'s capped backoff step (`maxDelayMs`), the cost of one retry. */
|
|
15060
|
+
const AUTO_SCALING_RETRY_STEP_MS = 8e3;
|
|
15061
|
+
|
|
14534
15062
|
//#endregion
|
|
14535
15063
|
//#region src/provisioning/providers/dynamodb-table-provider.ts
|
|
14536
15064
|
/**
|
|
@@ -15097,11 +15625,47 @@ var DynamoDBTableProvider = class {
|
|
|
15097
15625
|
"ContributorInsightsSpecification"
|
|
15098
15626
|
])]]);
|
|
15099
15627
|
unhandledByDesign = /* @__PURE__ */ new Map([["AWS::DynamoDB::Table", /* @__PURE__ */ new Map([["ImportSourceSpecification", "S3 import uses the separate ImportTable API (not CreateTable) and is create-only with no readback; deferred to a dedicated import-from-S3 PR"]])]]);
|
|
15628
|
+
/**
|
|
15629
|
+
* The ONE wall-clock allowance each `delete()` spends, keyed by physical id
|
|
15630
|
+
* (issue #1955). Keyed rather than per-call because `destroy-runner.ts` can
|
|
15631
|
+
* RE-ENTER `delete()` from the top: a THROTTLED `DeleteTable` is not retried
|
|
15632
|
+
* by the index-busy loop below — that classifier is message-scoped by design
|
|
15633
|
+
* — so it escapes to the outer loop, which DOES class throttles as retryable
|
|
15634
|
+
* and pays the whole settle-and-retry sequence a second time inside the same
|
|
15635
|
+
* per-resource deadline (2 x ~19.6 min ~= ~39 min after issue #1950 raised
|
|
15636
|
+
* this type's retry budget to 14). A per-call budget would hand that second
|
|
15637
|
+
* pass a fresh allowance; this one does not. Keyed rather than per-instance
|
|
15638
|
+
* because a destroy level deletes its resources CONCURRENTLY through one
|
|
15639
|
+
* provider instance.
|
|
15640
|
+
*/
|
|
15641
|
+
deleteBudgets = new ElapsedBudgetRegistry();
|
|
15100
15642
|
constructor() {
|
|
15101
15643
|
const awsClients = getAwsClients();
|
|
15102
15644
|
this.dynamoDBClient = awsClients.dynamoDB;
|
|
15103
15645
|
}
|
|
15104
15646
|
/**
|
|
15647
|
+
* Self-reported minimum per-resource deadline (issue #1955).
|
|
15648
|
+
*
|
|
15649
|
+
* The engine resolves the deadline as
|
|
15650
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), globalCliDefault)`,
|
|
15651
|
+
* so declaring this is what turns "the shared delete budget fits inside the
|
|
15652
|
+
* deadline" from an assumption about the CLI default into a guarantee. It
|
|
15653
|
+
* equals today's `DEFAULT_RESOURCE_TIMEOUT_MS`, so at default settings this
|
|
15654
|
+
* changes nothing.
|
|
15655
|
+
*
|
|
15656
|
+
* **It is TYPE-level, not operation-level**, and that is a real consequence
|
|
15657
|
+
* rather than a detail: `deploy-engine.ts` and `destroy-runner.ts` both fold
|
|
15658
|
+
* it in through the same `Math.max(...)`, so a user running
|
|
15659
|
+
* `--resource-timeout 5m` for fail-fast CI gets 30 minutes for this type's
|
|
15660
|
+
* CREATE and UPDATE too, not only for the DELETE the budget is about. The
|
|
15661
|
+
* interface has no per-operation form, and the escape hatch for anyone who
|
|
15662
|
+
* wants the shorter deadline back is the per-type override
|
|
15663
|
+
* (`--resource-timeout AWS::DynamoDB::Table=5m`), which still wins.
|
|
15664
|
+
*/
|
|
15665
|
+
getMinResourceTimeoutMs() {
|
|
15666
|
+
return DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS;
|
|
15667
|
+
}
|
|
15668
|
+
/**
|
|
15105
15669
|
* Create a DynamoDB table
|
|
15106
15670
|
*/
|
|
15107
15671
|
async create(logicalId, resourceType, properties, context) {
|
|
@@ -15391,6 +15955,7 @@ var DynamoDBTableProvider = class {
|
|
|
15391
15955
|
*/
|
|
15392
15956
|
async delete(logicalId, physicalId, resourceType, _properties, context) {
|
|
15393
15957
|
this.logger.debug(`Deleting DynamoDB table ${logicalId}: ${physicalId}`);
|
|
15958
|
+
const budget = this.deleteBudgets.acquire(deleteBudgetKey(physicalId, context?.expectedRegion), resolveDynamoDbDeleteBudgetMs(), resolveDynamoDbDeleteBudgetClock(), DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS);
|
|
15394
15959
|
if (context?.removeProtection === true) try {
|
|
15395
15960
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
15396
15961
|
TableName: physicalId,
|
|
@@ -15398,7 +15963,7 @@ var DynamoDBTableProvider = class {
|
|
|
15398
15963
|
}));
|
|
15399
15964
|
this.logger.debug(`Disabled DeletionProtectionEnabled on DynamoDB table ${logicalId}, waiting for ACTIVE`);
|
|
15400
15965
|
try {
|
|
15401
|
-
await this.waitForTableActiveAfterUpdate(physicalId);
|
|
15966
|
+
await this.waitForTableActiveAfterUpdate(physicalId, 60, budget);
|
|
15402
15967
|
} catch (waitErr) {
|
|
15403
15968
|
this.logger.debug(`Could not wait for table ${physicalId} ACTIVE after disabling protection: ${waitErr instanceof Error ? waitErr.message : String(waitErr)}`);
|
|
15404
15969
|
}
|
|
@@ -15406,6 +15971,13 @@ var DynamoDBTableProvider = class {
|
|
|
15406
15971
|
if (!(flipError instanceof ResourceNotFoundException$1)) this.logger.debug(`Could not disable DeletionProtectionEnabled on ${physicalId}: ${flipError instanceof Error ? flipError.message : String(flipError)}`);
|
|
15407
15972
|
}
|
|
15408
15973
|
try {
|
|
15974
|
+
const reArmPolls = () => {
|
|
15975
|
+
const wait = beginDeleteWait(60, budget);
|
|
15976
|
+
return {
|
|
15977
|
+
maxAttempts: wait.grantedPolls,
|
|
15978
|
+
budgetWait: wait
|
|
15979
|
+
};
|
|
15980
|
+
};
|
|
15409
15981
|
await deleteTableWithIndexBusyRetry({
|
|
15410
15982
|
logicalId,
|
|
15411
15983
|
physicalId,
|
|
@@ -15420,16 +15992,23 @@ var DynamoDBTableProvider = class {
|
|
|
15420
15992
|
logicalId,
|
|
15421
15993
|
logger: this.logger,
|
|
15422
15994
|
describeTable: () => this.dynamoDBClient.send(new DescribeTableCommand({ TableName: physicalId })),
|
|
15423
|
-
|
|
15995
|
+
...reArmPolls(),
|
|
15424
15996
|
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE
|
|
15425
15997
|
}),
|
|
15998
|
+
shouldKeepRetrying: () => {
|
|
15999
|
+
if (!budget.isExhausted()) return true;
|
|
16000
|
+
this.logger.warn(`DynamoDB table ${logicalId}: giving up the index-busy DeleteTable retry on ${physicalId} — the delete path has spent its whole ${Math.round(budget.totalMs / 6e4)}-minute budget. AWS's own message follows; re-running the destroy succeeds once the index is ACTIVE.`);
|
|
16001
|
+
return false;
|
|
16002
|
+
},
|
|
15426
16003
|
sleepSeam: deleteTableRetryDelays$1
|
|
15427
16004
|
});
|
|
15428
16005
|
this.logger.debug(`Successfully deleted DynamoDB table ${logicalId}`);
|
|
16006
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
15429
16007
|
} catch (error) {
|
|
15430
16008
|
if (error instanceof ResourceNotFoundException$1) {
|
|
15431
16009
|
assertRegionMatch(await this.dynamoDBClient.config.region(), context?.expectedRegion, resourceType, logicalId, physicalId);
|
|
15432
16010
|
this.logger.debug(`DynamoDB table ${physicalId} does not exist, skipping deletion`);
|
|
16011
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
15433
16012
|
return;
|
|
15434
16013
|
}
|
|
15435
16014
|
const cause = error instanceof Error ? error : void 0;
|
|
@@ -15770,12 +16349,13 @@ var DynamoDBTableProvider = class {
|
|
|
15770
16349
|
* blanket raise would turn every genuinely wedged capacity edit into a
|
|
15771
16350
|
* 10-minute hang.
|
|
15772
16351
|
*/
|
|
15773
|
-
async waitForTableActiveAfterUpdate(tableName, maxAttempts = 60) {
|
|
15774
|
-
|
|
16352
|
+
async waitForTableActiveAfterUpdate(tableName, maxAttempts = 60, budget) {
|
|
16353
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
16354
|
+
while (wait.nextPoll()) {
|
|
15775
16355
|
if ((await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }))).Table?.TableStatus === "ACTIVE") return;
|
|
15776
16356
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
15777
16357
|
}
|
|
15778
|
-
throw new Error(`Table ${tableName} did not reach ACTIVE status within ${
|
|
16358
|
+
throw new Error(`Table ${tableName} did not reach ACTIVE status within ${wait.pollsRun} seconds after UpdateTable${wait.note()}`);
|
|
15779
16359
|
}
|
|
15780
16360
|
/**
|
|
15781
16361
|
* Poll DescribeTable until the table is ACTIVE AND every Global Secondary
|
|
@@ -16581,6 +17161,19 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
16581
17161
|
* UpdateTable success.
|
|
16582
17162
|
*/
|
|
16583
17163
|
attributeCache = /* @__PURE__ */ new Map();
|
|
17164
|
+
/**
|
|
17165
|
+
* The ONE wall-clock allowance each `delete()` spends, keyed by physical id
|
|
17166
|
+
* (issue #1955). Keyed rather than per-call because `destroy-runner.ts` can
|
|
17167
|
+
* RE-ENTER `delete()` from the top — its outer retry loop classes a throttled
|
|
17168
|
+
* `DeleteTable` as retryable — inside the SAME per-resource deadline, and a
|
|
17169
|
+
* per-call budget would hand that second pass a fresh allowance. Keyed rather
|
|
17170
|
+
* than per-instance because a destroy level deletes its resources
|
|
17171
|
+
* CONCURRENTLY through this one provider instance.
|
|
17172
|
+
*
|
|
17173
|
+
* See `./dynamodb-delete-budget.ts` for the sizing and for the three failure
|
|
17174
|
+
* terms it closes.
|
|
17175
|
+
*/
|
|
17176
|
+
deleteBudgets = new ElapsedBudgetRegistry();
|
|
16584
17177
|
handledProperties = /* @__PURE__ */ new Map([["AWS::DynamoDB::GlobalTable", /* @__PURE__ */ new Set([
|
|
16585
17178
|
"TableName",
|
|
16586
17179
|
"KeySchema",
|
|
@@ -16602,6 +17195,29 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
16602
17195
|
this.dynamoDBClient = awsClients.dynamoDB;
|
|
16603
17196
|
}
|
|
16604
17197
|
/**
|
|
17198
|
+
* Self-reported minimum per-resource deadline (issue #1955).
|
|
17199
|
+
*
|
|
17200
|
+
* The engine resolves the deadline as
|
|
17201
|
+
* `perTypeCliOverride ?? max(getMinResourceTimeoutMs(), globalCliDefault)`,
|
|
17202
|
+
* so declaring this is what turns "the shared delete budget fits inside the
|
|
17203
|
+
* deadline" from an assumption about the CLI default into a guarantee: a user
|
|
17204
|
+
* who lowers `--resource-timeout` globally can no longer re-create the
|
|
17205
|
+
* crossing the budget exists to remove. It equals today's
|
|
17206
|
+
* `DEFAULT_RESOURCE_TIMEOUT_MS`, so at default settings this changes nothing.
|
|
17207
|
+
*
|
|
17208
|
+
* **It is TYPE-level, not operation-level**, and that is a real consequence
|
|
17209
|
+
* rather than a detail: `deploy-engine.ts` and `destroy-runner.ts` both fold
|
|
17210
|
+
* it in through the same `Math.max(...)`, so a user running
|
|
17211
|
+
* `--resource-timeout 5m` for fail-fast CI gets 30 minutes for this type's
|
|
17212
|
+
* CREATE and UPDATE too, not only for the DELETE the budget is about. The
|
|
17213
|
+
* interface has no per-operation form, and the escape hatch for anyone who
|
|
17214
|
+
* wants the shorter deadline back is the per-type override
|
|
17215
|
+
* (`--resource-timeout AWS::DynamoDB::GlobalTable=5m`), which still wins.
|
|
17216
|
+
*/
|
|
17217
|
+
getMinResourceTimeoutMs() {
|
|
17218
|
+
return DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS;
|
|
17219
|
+
}
|
|
17220
|
+
/**
|
|
16605
17221
|
* Return a `DynamoDBClient` pinned to the given region, caching per
|
|
16606
17222
|
* region for the lifetime of this provider instance. Uses the default
|
|
16607
17223
|
* credential chain (env / shared config / IAM role) — no explicit
|
|
@@ -17561,7 +18177,8 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17561
18177
|
* create-only field with no corresponding `RegisterScalableTarget`
|
|
17562
18178
|
* surface; `readAutoScalingSettings` also explicitly skips it.
|
|
17563
18179
|
*/
|
|
17564
|
-
async applyAutoScalingDiff(tableName, dimension, oldSettings, newSettings, client, indexName) {
|
|
18180
|
+
async applyAutoScalingDiff(tableName, dimension, oldSettings, newSettings, client, indexName, budget) {
|
|
18181
|
+
const asMaxRetries = autoScalingRetriesWithinDeleteBudget(budget, 8);
|
|
17565
18182
|
const metricType = dimension.endsWith("WriteCapacityUnits") ? "DynamoDBWriteCapacityUtilization" : "DynamoDBReadCapacityUtilization";
|
|
17566
18183
|
const resourceId = autoScalingResourceId(tableName, indexName);
|
|
17567
18184
|
const policyName = `${metricType}:${resourceId}`;
|
|
@@ -17585,6 +18202,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17585
18202
|
MaxCapacity: maxCapacity
|
|
17586
18203
|
})), `${resourceId} (${dimension})`, {
|
|
17587
18204
|
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18205
|
+
maxRetries: asMaxRetries,
|
|
17588
18206
|
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
17589
18207
|
});
|
|
17590
18208
|
} catch (err) {
|
|
@@ -17614,6 +18232,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17614
18232
|
TargetTrackingScalingPolicyConfiguration: targetTrackingConfig
|
|
17615
18233
|
})), `${policyName}`, {
|
|
17616
18234
|
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18235
|
+
maxRetries: asMaxRetries,
|
|
17617
18236
|
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
17618
18237
|
});
|
|
17619
18238
|
this.logger.debug(`Upserted auto-scaling policy ${policyName} on ${tableName} (${dimension})`);
|
|
@@ -17629,21 +18248,29 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17629
18248
|
return name === "ObjectNotFoundException" || msg.includes("No scaling policy found") || msg.includes("No scalable target found");
|
|
17630
18249
|
};
|
|
17631
18250
|
try {
|
|
17632
|
-
await asClient.send(new DeleteScalingPolicyCommand({
|
|
18251
|
+
await withRetry(() => asClient.send(new DeleteScalingPolicyCommand({
|
|
17633
18252
|
PolicyName: policyName,
|
|
17634
18253
|
ServiceNamespace: "dynamodb",
|
|
17635
18254
|
ResourceId: resourceId,
|
|
17636
18255
|
ScalableDimension: dimension
|
|
17637
|
-
}))
|
|
18256
|
+
})), `${resourceId} (${dimension})`, {
|
|
18257
|
+
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18258
|
+
maxRetries: asMaxRetries,
|
|
18259
|
+
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
18260
|
+
});
|
|
17638
18261
|
} catch (err) {
|
|
17639
18262
|
if (!isObjectNotFound(err)) this.logger.warn(`Could not delete auto-scaling policy on ${tableName} (${dimension}): ${err instanceof Error ? err.message : String(err)}. Run: aws application-autoscaling delete-scaling-policy --policy-name ${policyName} --service-namespace dynamodb --resource-id ${resourceId} --scalable-dimension ${dimension}`);
|
|
17640
18263
|
}
|
|
17641
18264
|
try {
|
|
17642
|
-
await asClient.send(new DeregisterScalableTargetCommand({
|
|
18265
|
+
await withRetry(() => asClient.send(new DeregisterScalableTargetCommand({
|
|
17643
18266
|
ServiceNamespace: "dynamodb",
|
|
17644
18267
|
ResourceId: resourceId,
|
|
17645
18268
|
ScalableDimension: dimension
|
|
17646
|
-
}))
|
|
18269
|
+
})), `${resourceId} (${dimension})`, {
|
|
18270
|
+
isRetryable: (_message, error) => isThrottlingError(error),
|
|
18271
|
+
maxRetries: asMaxRetries,
|
|
18272
|
+
...autoScalingRetryDelays.sleep ? { sleep: autoScalingRetryDelays.sleep } : {}
|
|
18273
|
+
});
|
|
17647
18274
|
this.logger.debug(`Deregistered auto-scaling target ${resourceId} (${dimension})`);
|
|
17648
18275
|
} catch (err) {
|
|
17649
18276
|
if (!isObjectNotFound(err)) this.logger.warn(`Could not deregister auto-scaling target on ${tableName} (${dimension}): ${err instanceof Error ? err.message : String(err)}. Run: aws application-autoscaling deregister-scalable-target --service-namespace dynamodb --resource-id ${resourceId} --scalable-dimension ${dimension}`);
|
|
@@ -17792,6 +18419,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17792
18419
|
*/
|
|
17793
18420
|
async delete(logicalId, physicalId, resourceType, _properties, context) {
|
|
17794
18421
|
this.logger.debug(`Deleting DynamoDB GlobalTable ${logicalId}: ${physicalId}`);
|
|
18422
|
+
const budget = this.deleteBudgets.acquire(deleteBudgetKey(physicalId, context?.expectedRegion), resolveDynamoDbDeleteBudgetMs(), resolveDynamoDbDeleteBudgetClock(), DYNAMODB_DELETE_MIN_RESOURCE_TIMEOUT_MS);
|
|
17795
18423
|
if (context?.removeProtection === true) try {
|
|
17796
18424
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
17797
18425
|
TableName: physicalId,
|
|
@@ -17799,7 +18427,7 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17799
18427
|
}));
|
|
17800
18428
|
this.logger.debug(`Disabled DeletionProtectionEnabled on ${logicalId}, waiting for ACTIVE`);
|
|
17801
18429
|
try {
|
|
17802
|
-
await this.waitForTableActiveAfterUpdate(physicalId, logicalId);
|
|
18430
|
+
await this.waitForTableActiveAfterUpdate(physicalId, logicalId, 600, budget);
|
|
17803
18431
|
} catch (waitErr) {
|
|
17804
18432
|
this.logger.debug(`Could not wait for table ${physicalId} ACTIVE after protection flip: ${waitErr instanceof Error ? waitErr.message : String(waitErr)}`);
|
|
17805
18433
|
}
|
|
@@ -17823,25 +18451,26 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17823
18451
|
const region = replica.RegionName;
|
|
17824
18452
|
if (!region || region === currentRegion) continue;
|
|
17825
18453
|
try {
|
|
17826
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region));
|
|
17827
|
-
for (const indexName of tableIndexNames) await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), indexName);
|
|
18454
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), void 0, budget);
|
|
18455
|
+
for (const indexName of tableIndexNames) await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, this.getRegionalAutoScalingClient(region), indexName, budget);
|
|
17828
18456
|
await this.dynamoDBClient.send(new UpdateTableCommand({
|
|
17829
18457
|
TableName: physicalId,
|
|
17830
18458
|
ReplicaUpdates: [{ Delete: { RegionName: region } }]
|
|
17831
18459
|
}));
|
|
17832
|
-
await this.waitForReplicaGone(physicalId, region, logicalId);
|
|
18460
|
+
await this.waitForReplicaGone(physicalId, region, logicalId, REPLICA_GONE_WAIT_ATTEMPTS, budget);
|
|
17833
18461
|
} catch (replicaErr) {
|
|
17834
18462
|
if (!(replicaErr instanceof ResourceNotFoundException$1)) throw replicaErr;
|
|
17835
18463
|
}
|
|
17836
18464
|
}
|
|
17837
18465
|
const localAsClient = await this.getLocalAutoScalingClient();
|
|
17838
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, localAsClient);
|
|
17839
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:WriteCapacityUnits", {}, void 0, localAsClient);
|
|
18466
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:ReadCapacityUnits", {}, void 0, localAsClient, void 0, budget);
|
|
18467
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:table:WriteCapacityUnits", {}, void 0, localAsClient, void 0, budget);
|
|
17840
18468
|
for (const indexName of tableIndexNames) {
|
|
17841
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:WriteCapacityUnits", {}, void 0, localAsClient, indexName);
|
|
17842
|
-
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, localAsClient, indexName);
|
|
18469
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:WriteCapacityUnits", {}, void 0, localAsClient, indexName, budget);
|
|
18470
|
+
await this.applyAutoScalingDiff(physicalId, "dynamodb:index:ReadCapacityUnits", {}, void 0, localAsClient, indexName, budget);
|
|
17843
18471
|
}
|
|
17844
18472
|
} catch (describeErr) {
|
|
18473
|
+
if (describeErr instanceof ProvisioningError) throw describeErr;
|
|
17845
18474
|
if (!(describeErr instanceof ResourceNotFoundException$1)) {
|
|
17846
18475
|
const cause = describeErr instanceof Error ? describeErr : void 0;
|
|
17847
18476
|
throw new ProvisioningError(`Failed to describe DynamoDB GlobalTable ${logicalId} before delete: ${describeErr instanceof Error ? describeErr.message : String(describeErr)}`, resourceType, logicalId, physicalId, cause);
|
|
@@ -17850,7 +18479,10 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17850
18479
|
try {
|
|
17851
18480
|
if (hasTransitionalIndex(preDeleteIndexes)) {
|
|
17852
18481
|
this.logger.debug(`Waiting for indexes on ${physicalId} to settle before DeleteTable (issue #1521)`);
|
|
17853
|
-
await this.waitForIndexesActive(physicalId, logicalId, {
|
|
18482
|
+
await this.waitForIndexesActive(physicalId, logicalId, {
|
|
18483
|
+
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE,
|
|
18484
|
+
budget
|
|
18485
|
+
});
|
|
17854
18486
|
}
|
|
17855
18487
|
await deleteTableWithIndexBusyRetry({
|
|
17856
18488
|
logicalId,
|
|
@@ -17863,16 +18495,24 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
17863
18495
|
},
|
|
17864
18496
|
reArm: () => this.waitForIndexesActive(physicalId, logicalId, {
|
|
17865
18497
|
maxAttempts: 60,
|
|
17866
|
-
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE
|
|
18498
|
+
proceedNote: DELETE_INDEX_WAIT_PROCEED_NOTE,
|
|
18499
|
+
budget
|
|
17867
18500
|
}),
|
|
18501
|
+
shouldKeepRetrying: () => {
|
|
18502
|
+
if (!budget.isExhausted()) return true;
|
|
18503
|
+
this.logger.warn(`DynamoDB GlobalTable ${logicalId}: giving up the index-busy DeleteTable retry on ${physicalId} — the delete path has spent its whole ${Math.round(budget.totalMs / 6e4)}-minute budget. AWS's own message follows; re-running the destroy succeeds once the index is ACTIVE.`);
|
|
18504
|
+
return false;
|
|
18505
|
+
},
|
|
17868
18506
|
sleepSeam: deleteTableRetryDelays
|
|
17869
18507
|
});
|
|
17870
|
-
await this.waitForTableGone(physicalId, logicalId);
|
|
18508
|
+
await this.waitForTableGone(physicalId, logicalId, 600, budget);
|
|
17871
18509
|
this.logger.debug(`Successfully deleted DynamoDB GlobalTable ${logicalId}`);
|
|
18510
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
17872
18511
|
} catch (error) {
|
|
17873
18512
|
if (error instanceof ResourceNotFoundException$1) {
|
|
17874
18513
|
assertRegionMatch(await this.dynamoDBClient.config.region(), context?.expectedRegion, resourceType, logicalId, physicalId);
|
|
17875
18514
|
this.logger.debug(`DynamoDB GlobalTable ${physicalId} does not exist, skipping`);
|
|
18515
|
+
this.deleteBudgets.release(deleteBudgetKey(physicalId, context?.expectedRegion));
|
|
17876
18516
|
return;
|
|
17877
18517
|
}
|
|
17878
18518
|
const cause = error instanceof Error ? error : void 0;
|
|
@@ -18444,9 +19084,10 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18444
19084
|
* rejection as the backstop, which is the behavior it had before #1521 for
|
|
18445
19085
|
* every case except the one the wait exists to fix.
|
|
18446
19086
|
*/
|
|
18447
|
-
async waitForTableActiveAfterUpdate(tableName, logicalId, maxAttempts = 600) {
|
|
19087
|
+
async waitForTableActiveAfterUpdate(tableName, logicalId, maxAttempts = 600, budget) {
|
|
19088
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
18448
19089
|
let tableReachedActive = false;
|
|
18449
|
-
|
|
19090
|
+
while (wait.nextPoll()) {
|
|
18450
19091
|
const response = await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }));
|
|
18451
19092
|
if (response.Table?.TableStatus === "ACTIVE") {
|
|
18452
19093
|
tableReachedActive = true;
|
|
@@ -18454,11 +19095,12 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18454
19095
|
}
|
|
18455
19096
|
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
18456
19097
|
}
|
|
19098
|
+
const budgetNote = wait.note();
|
|
18457
19099
|
if (tableReachedActive) {
|
|
18458
|
-
this.logger.warn(`Indexes on ${tableName} (${logicalId}) were still transitioning after ${
|
|
19100
|
+
this.logger.warn(`Indexes on ${tableName} (${logicalId}) were still transitioning after ${wait.pollsRun}s — a large index backfill can outlive this wait. Proceeding; AWS rejects the next call if it is still too early.${budgetNote}`);
|
|
18459
19101
|
return;
|
|
18460
19102
|
}
|
|
18461
|
-
throw new ProvisioningError(`Table ${tableName} did not reach ACTIVE within ${
|
|
19103
|
+
throw new ProvisioningError(`Table ${tableName} did not reach ACTIVE within ${wait.pollsRun}s after UpdateTable${budgetNote}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18462
19104
|
}
|
|
18463
19105
|
/**
|
|
18464
19106
|
* Wait until every GSI reports `IndexStatus: ACTIVE` (Issue #1419).
|
|
@@ -18503,18 +19145,25 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18503
19145
|
* passes {@link DELETE_INDEX_WAIT_PROCEED_NOTE}.
|
|
18504
19146
|
*/
|
|
18505
19147
|
async waitForIndexesActive(tableName, logicalId, opts) {
|
|
19148
|
+
const gateWait = beginDeleteWait(opts?.maxAttempts ?? 900, opts?.budget);
|
|
18506
19149
|
await waitForIndexesSettled({
|
|
18507
19150
|
tableName,
|
|
18508
19151
|
logicalId,
|
|
18509
19152
|
logger: this.logger,
|
|
18510
19153
|
describeTable: () => this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName })),
|
|
18511
|
-
maxAttempts:
|
|
19154
|
+
maxAttempts: gateWait.grantedPolls,
|
|
19155
|
+
budgetWait: gateWait,
|
|
18512
19156
|
proceedNote: opts?.proceedNote ?? "auto-scaling registration for a still-building index may have been skipped. The next deploy re-asserts it."
|
|
18513
19157
|
});
|
|
18514
19158
|
}
|
|
18515
19159
|
/**
|
|
18516
19160
|
* Wait until a specific replica's `ReplicaStatus` flips to ACTIVE.
|
|
18517
|
-
* Replica provisioning typically takes 1–5 min;
|
|
19161
|
+
* Replica provisioning typically takes 1–5 min; capped at
|
|
19162
|
+
* {@link REPLICA_GONE_WAIT_ATTEMPTS} polls — ~12 min at the MEASURED ~1.2s
|
|
19163
|
+
* per poll (a sleep plus a `DescribeTable` round trip), not the ~10 min the
|
|
19164
|
+
* `1 poll = 1s` reading gives. The delete-path arithmetic in
|
|
19165
|
+
* `./dynamodb-delete-budget.ts` prices it at the measured figure, so the two
|
|
19166
|
+
* have to agree.
|
|
18518
19167
|
*/
|
|
18519
19168
|
async waitForReplicaActive(tableName, region, logicalId, maxAttempts = 600) {
|
|
18520
19169
|
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
@@ -18528,17 +19177,19 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18528
19177
|
/**
|
|
18529
19178
|
* Wait until a specific replica disappears from `Replicas[]` after a
|
|
18530
19179
|
* Delete replica update. Replica deletion typically takes 1–5 min;
|
|
18531
|
-
*
|
|
19180
|
+
* capped at {@link REPLICA_GONE_WAIT_ATTEMPTS} polls (~12 min at the
|
|
19181
|
+
* measured ~1.2s per poll).
|
|
18532
19182
|
*/
|
|
18533
|
-
async waitForReplicaGone(tableName, region, logicalId, maxAttempts = REPLICA_GONE_WAIT_ATTEMPTS) {
|
|
18534
|
-
|
|
19183
|
+
async waitForReplicaGone(tableName, region, logicalId, maxAttempts = REPLICA_GONE_WAIT_ATTEMPTS, budget) {
|
|
19184
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
19185
|
+
while (wait.nextPoll()) try {
|
|
18535
19186
|
if (!(await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }))).Table?.Replicas?.find((r) => r.RegionName === region)) return;
|
|
18536
19187
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
18537
19188
|
} catch (err) {
|
|
18538
19189
|
if (err instanceof ResourceNotFoundException$1) return;
|
|
18539
19190
|
throw err;
|
|
18540
19191
|
}
|
|
18541
|
-
throw new ProvisioningError(`Replica ${region} for table ${tableName} did not disappear within ${
|
|
19192
|
+
throw new ProvisioningError(`Replica ${region} for table ${tableName} did not disappear within ${wait.pollsRun}s${wait.note()}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18542
19193
|
}
|
|
18543
19194
|
/**
|
|
18544
19195
|
* Wait for `DescribeTable` to return `ResourceNotFoundException`,
|
|
@@ -18548,18 +19199,26 @@ var DynamoDBGlobalTableProvider = class {
|
|
|
18548
19199
|
* wait, downstream observers (siblings deleted in the same destroy
|
|
18549
19200
|
* run, integ scripts that re-check via `aws dynamodb describe-table`)
|
|
18550
19201
|
* see "destroy succeeded" but the table is still listed by AWS.
|
|
18551
|
-
* Typical small-table delete completes in 5–30s;
|
|
18552
|
-
*
|
|
18553
|
-
|
|
18554
|
-
|
|
18555
|
-
|
|
19202
|
+
* Typical small-table delete completes in 5–30s; capped at
|
|
19203
|
+
* {@link TABLE_GONE_WAIT_ATTEMPTS} polls (~12 min at the measured ~1.2s per
|
|
19204
|
+
* poll) for worst-case large-table / replica-cascade scenarios. That 5-30s
|
|
19205
|
+
* figure is what {@link DELETE_SHORT_WAIT_POLLS} is calibrated against: past
|
|
19206
|
+
* roughly twice the top of it, cdkd HAS given AWS a real wait.
|
|
19207
|
+
*/
|
|
19208
|
+
async waitForTableGone(tableName, logicalId, maxAttempts = 600, budget) {
|
|
19209
|
+
const wait = beginDeleteWait(maxAttempts, budget);
|
|
19210
|
+
while (wait.nextPoll()) try {
|
|
18556
19211
|
await this.dynamoDBClient.send(new DescribeTableCommand({ TableName: tableName }));
|
|
18557
19212
|
await new Promise((resolve) => setTimeout(resolve, INDEX_SETTLE_POLL_INTERVAL_MS));
|
|
18558
19213
|
} catch (err) {
|
|
18559
19214
|
if (err instanceof ResourceNotFoundException$1) return;
|
|
18560
19215
|
throw err;
|
|
18561
19216
|
}
|
|
18562
|
-
|
|
19217
|
+
if (wait.cutShort) {
|
|
19218
|
+
this.logger.warn(`DynamoDB GlobalTable ${logicalId}: DeleteTable on ${tableName} was ACCEPTED by AWS but cdkd stopped waiting for the table to disappear${wait.note()} The delete completes on AWS's side; a later describe (or a re-run of this destroy) sees it gone.`);
|
|
19219
|
+
return;
|
|
19220
|
+
}
|
|
19221
|
+
throw new ProvisioningError(`Table ${tableName} did not disappear within ${wait.pollsRun}s${wait.note()}`, "AWS::DynamoDB::GlobalTable", logicalId, tableName);
|
|
18563
19222
|
}
|
|
18564
19223
|
};
|
|
18565
19224
|
/**
|
|
@@ -18660,6 +19319,19 @@ const autoScalingRetryDelays = {};
|
|
|
18660
19319
|
*/
|
|
18661
19320
|
const REPLICA_GONE_WAIT_ATTEMPTS = 600;
|
|
18662
19321
|
/**
|
|
19322
|
+
* Polls the `--remove-protection` flip's ACTIVE wait spends
|
|
19323
|
+
* (`waitForTableActiveAfterUpdate`).
|
|
19324
|
+
*
|
|
19325
|
+
* EXPORTED and named rather than left as an inline default argument because it
|
|
19326
|
+
* is a TERM of the delete-path deadline arithmetic (see
|
|
19327
|
+
* `./dynamodb-delete-budget.ts`): on `cdkd destroy --remove-protection` it runs
|
|
19328
|
+
* FIRST, ahead of the #1521 gate, and the wait it performs is the same
|
|
19329
|
+
* indexes-settled one the rest of the path is about. The budget fence in the
|
|
19330
|
+
* unit suite reads THIS rather than a copied literal, after the sibling
|
|
19331
|
+
* arithmetic was twice found asserting a cap the provider no longer had.
|
|
19332
|
+
*/
|
|
19333
|
+
const GLOBAL_TABLE_ACTIVE_WAIT_ATTEMPTS = 600;
|
|
19334
|
+
/**
|
|
18663
19335
|
* Polls `waitForTableGone` spends confirming the table is actually gone.
|
|
18664
19336
|
*
|
|
18665
19337
|
* EXPORTED, unlike its replica sibling, because the delete-path budget fence
|
|
@@ -57586,6 +58258,8 @@ async function deployCommand(stacks, options) {
|
|
|
57586
58258
|
const summary = workGraph.summary();
|
|
57587
58259
|
logger.debug(`Work graph: ${summary["asset-publish"]} asset(s), ${summary["stack"]} stack(s)`);
|
|
57588
58260
|
const bufferStackOutput = targetStacks.length > 1;
|
|
58261
|
+
let totalUnaddressed = 0;
|
|
58262
|
+
let cancelledStacks = 0;
|
|
57589
58263
|
const runStack = async (stackInfo) => {
|
|
57590
58264
|
return withSkipPrefix(skipPrefix, () => runStackInner(stackInfo));
|
|
57591
58265
|
};
|
|
@@ -57718,6 +58392,8 @@ async function deployCommand(stacks, options) {
|
|
|
57718
58392
|
logger.info(` Deleted: ${deployResult.deleted > 0 ? red(deployResult.deleted) : gray(deployResult.deleted)}`);
|
|
57719
58393
|
if (deployResult.deleteSkipped > 0) logger.info(` Skipped (not deleted): ${yellow(deployResult.deleteSkipped)}`);
|
|
57720
58394
|
if (deployResult.updatePartial > 0) logger.info(` of which left an orphaned predecessor: ${yellow(deployResult.updatePartial)}`);
|
|
58395
|
+
const stackUnaddressed = deployResult.deleteSkipped + deployResult.updatePartial;
|
|
58396
|
+
if (!options.dryRun) totalUnaddressed += stackUnaddressed;
|
|
57721
58397
|
logger.info(` Unchanged: ${gray(deployResult.unchanged)}`);
|
|
57722
58398
|
logger.info(` Duration: ${cyan((deployResult.durationMs / 1e3).toFixed(2) + "s")}`);
|
|
57723
58399
|
if (deployResult.attributeFallbackCount > 0) logger.warn(`${deployResult.attributeFallbackCount} attribute resolution(s) fell back to the physical ID (potentially wrong values); re-run with --strict-getatt to fail on these`);
|
|
@@ -57726,15 +58402,20 @@ async function deployCommand(stacks, options) {
|
|
|
57726
58402
|
for (const [key, value] of Object.entries(deployResult.outputs)) logger.info(` ${deployResult.stackName}.${key} = ${String(value)}`);
|
|
57727
58403
|
}
|
|
57728
58404
|
if (options.dryRun) logger.info(`\n${green("✓")} ${bold("Dry run completed")} - no actual changes made`);
|
|
58405
|
+
else if (stackUnaddressed > 0) logger.warn(`\n⚠ ${bold(`Stack ${deployResult.stackName} deployed, but ${stackUnaddressed} resource(s) were left unaddressed`)} — they may still exist in AWS. ` + (options.allowUnaddressed ? `Exiting 0 because --allow-unaddressed was passed.` : "This counts toward a non-zero exit (2 unless something else fails; pass --allow-unaddressed to exit 0)."));
|
|
57729
58406
|
else logger.info(`\n${green("✓")} ${bold("Deployment completed successfully")}`);
|
|
57730
|
-
|
|
58407
|
+
if (!options.dryRun && stackUnaddressed > 0) runResult = "FAILED";
|
|
58408
|
+
recordRunOutcome(eventRecorder, stackInfo.stackName, runResult, {
|
|
57731
58409
|
created: deployResult.created,
|
|
57732
58410
|
updated: deployResult.updated + deployResult.updatePartial,
|
|
57733
58411
|
deleted: deployResult.deleted,
|
|
57734
58412
|
...deployResult.deleteSkipped + deployResult.updatePartial > 0 && { skipped: deployResult.deleteSkipped + deployResult.updatePartial }
|
|
57735
58413
|
}, deployResult.durationMs);
|
|
57736
58414
|
} catch (deployError) {
|
|
57737
|
-
if (deployError instanceof DeployCancelledError)
|
|
58415
|
+
if (deployError instanceof DeployCancelledError) {
|
|
58416
|
+
cancelledStacks++;
|
|
58417
|
+
return;
|
|
58418
|
+
}
|
|
57738
58419
|
runResult = "FAILED";
|
|
57739
58420
|
recordRunFailed(eventRecorder, stackInfo.stackName, deployError);
|
|
57740
58421
|
throw deployError;
|
|
@@ -57763,6 +58444,7 @@ async function deployCommand(stacks, options) {
|
|
|
57763
58444
|
if (!outcome.ok) throw outcome.error;
|
|
57764
58445
|
}
|
|
57765
58446
|
});
|
|
58447
|
+
if (totalUnaddressed > 0 && !options.allowUnaddressed) throw new PartialFailureError(`Deploy left ${totalUnaddressed} resource(s) unaddressed, so they may still exist in AWS. The two cases differ in what happens next: a DELETE the provider could not issue KEEPS its state record, so the next 'cdkd deploy' re-attempts it, while a replacement's surviving predecessor is NOT tracked and will never be retried — delete it by hand. The per-stack summaries above give the breakdown, and each resource's own warning names its cause and remedy. ` + (cancelledStacks > 0 ? `Note ${cancelledStacks} stack(s) were also cancelled and never deployed, so the template is not fully applied beyond the resources named above. ` : "") + `Pass --allow-unaddressed to exit 0 instead.`);
|
|
57766
58448
|
} finally {
|
|
57767
58449
|
unforwardSigterm();
|
|
57768
58450
|
process.removeListener("SIGINT", topLevelSigintHandler);
|
|
@@ -81373,7 +82055,7 @@ function createMigrateCommand() {
|
|
|
81373
82055
|
*/
|
|
81374
82056
|
function buildProgram() {
|
|
81375
82057
|
const program = new Command();
|
|
81376
|
-
program.name("cdkd").description("CDK Direct - Deploy AWS CDK apps directly via SDK/Cloud Control API").version("0.284.
|
|
82058
|
+
program.name("cdkd").description("CDK Direct - Deploy AWS CDK apps directly via SDK/Cloud Control API").version("0.284.3");
|
|
81377
82059
|
program.hook("preAction", (_thisCommand, actionCommand) => {
|
|
81378
82060
|
const { profile } = actionCommand.optsWithGlobals();
|
|
81379
82061
|
if (profile !== void 0) process.env["AWS_PROFILE"] = profile;
|