@fjall/deploy-core 0.89.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/src/aws/AwsProvider.d.ts +39 -0
- package/dist/src/aws/AwsProvider.js +1 -0
- package/dist/src/aws/SimpleAwsProvider.d.ts +22 -0
- package/dist/src/aws/SimpleAwsProvider.js +73 -0
- package/dist/src/aws/index.d.ts +4 -0
- package/dist/src/aws/index.js +3 -0
- package/dist/src/aws/organisations/accounts.d.ts +21 -0
- package/dist/src/aws/organisations/accounts.js +99 -0
- package/dist/src/aws/organisations/backup.d.ts +12 -0
- package/dist/src/aws/organisations/backup.js +28 -0
- package/dist/src/aws/organisations/costAllocation.d.ts +12 -0
- package/dist/src/aws/organisations/costAllocation.js +26 -0
- package/dist/src/aws/organisations/identityCentre.d.ts +8 -0
- package/dist/src/aws/organisations/identityCentre.js +19 -0
- package/dist/src/aws/organisations/index.d.ts +16 -0
- package/dist/src/aws/organisations/index.js +12 -0
- package/dist/src/aws/organisations/ipam.d.ts +7 -0
- package/dist/src/aws/organisations/ipam.js +18 -0
- package/dist/src/aws/organisations/organisation.d.ts +12 -0
- package/dist/src/aws/organisations/organisation.js +94 -0
- package/dist/src/aws/organisations/organisationalUnits.d.ts +19 -0
- package/dist/src/aws/organisations/organisationalUnits.js +125 -0
- package/dist/src/aws/organisations/policies.d.ts +7 -0
- package/dist/src/aws/organisations/policies.js +36 -0
- package/dist/src/aws/organisations/ram.d.ts +7 -0
- package/dist/src/aws/organisations/ram.js +15 -0
- package/dist/src/aws/organisations/serviceAccess.d.ts +7 -0
- package/dist/src/aws/organisations/serviceAccess.js +38 -0
- package/dist/src/aws/organisations/trustedAccess.d.ts +7 -0
- package/dist/src/aws/organisations/trustedAccess.js +15 -0
- package/dist/src/aws/organisations/types.d.ts +29 -0
- package/dist/src/aws/organisations/types.js +16 -0
- package/dist/src/aws/utils/CloudFormationFailureAnalyser.d.ts +32 -0
- package/dist/src/aws/utils/CloudFormationFailureAnalyser.js +228 -0
- package/dist/src/aws/utils/cloudformationEvents.d.ts +98 -0
- package/dist/src/aws/utils/cloudformationEvents.js +596 -0
- package/dist/src/aws/utils/errors.d.ts +26 -0
- package/dist/src/aws/utils/errors.js +59 -0
- package/dist/src/aws/utils/regions.d.ts +1 -0
- package/dist/src/aws/utils/regions.js +1 -0
- package/dist/src/aws/utils/stackStatus.d.ts +23 -0
- package/dist/src/aws/utils/stackStatus.js +90 -0
- package/dist/src/index.d.ts +35 -0
- package/dist/src/index.js +45 -0
- package/dist/src/orchestration/applicationDeploy.d.ts +11 -0
- package/dist/src/orchestration/applicationDeploy.js +327 -0
- package/dist/src/orchestration/contextHelpers.d.ts +9 -0
- package/dist/src/orchestration/contextHelpers.js +14 -0
- package/dist/src/orchestration/deploy.d.ts +10 -0
- package/dist/src/orchestration/deploy.js +42 -0
- package/dist/src/orchestration/detectionPipeline.d.ts +23 -0
- package/dist/src/orchestration/detectionPipeline.js +65 -0
- package/dist/src/orchestration/dockerInterface.d.ts +56 -0
- package/dist/src/orchestration/dockerInterface.js +1 -0
- package/dist/src/orchestration/domainInterface.d.ts +37 -0
- package/dist/src/orchestration/domainInterface.js +1 -0
- package/dist/src/orchestration/index.d.ts +8 -0
- package/dist/src/orchestration/index.js +3 -0
- package/dist/src/orchestration/organisationDeploy.d.ts +16 -0
- package/dist/src/orchestration/organisationDeploy.js +382 -0
- package/dist/src/orchestration/organisationSetup.d.ts +42 -0
- package/dist/src/orchestration/organisationSetup.js +227 -0
- package/dist/src/orchestration/resolveOperation.d.ts +10 -0
- package/dist/src/orchestration/resolveOperation.js +53 -0
- package/dist/src/orchestration/serviceFactory.d.ts +15 -0
- package/dist/src/orchestration/serviceFactory.js +16 -0
- package/dist/src/services/application/ApplicationStackService.d.ts +93 -0
- package/dist/src/services/application/ApplicationStackService.js +436 -0
- package/dist/src/services/application/index.d.ts +1 -0
- package/dist/src/services/application/index.js +1 -0
- package/dist/src/services/infrastructure/CdkArgumentBuilder.d.ts +12 -0
- package/dist/src/services/infrastructure/CdkArgumentBuilder.js +67 -0
- package/dist/src/services/infrastructure/CdkCommandRunner.d.ts +30 -0
- package/dist/src/services/infrastructure/CdkCommandRunner.js +241 -0
- package/dist/src/services/infrastructure/CdkErrorFormatter.d.ts +4 -0
- package/dist/src/services/infrastructure/CdkErrorFormatter.js +194 -0
- package/dist/src/services/infrastructure/CdkEventMonitoring.d.ts +19 -0
- package/dist/src/services/infrastructure/CdkEventMonitoring.js +41 -0
- package/dist/src/services/infrastructure/CdkOutputAnalyser.d.ts +43 -0
- package/dist/src/services/infrastructure/CdkOutputAnalyser.js +125 -0
- package/dist/src/services/infrastructure/CdkOutputParser.d.ts +8 -0
- package/dist/src/services/infrastructure/CdkOutputParser.js +33 -0
- package/dist/src/services/infrastructure/CdkProcessManager.d.ts +20 -0
- package/dist/src/services/infrastructure/CdkProcessManager.js +244 -0
- package/dist/src/services/infrastructure/CdkService.d.ts +71 -0
- package/dist/src/services/infrastructure/CdkService.js +254 -0
- package/dist/src/services/infrastructure/CloudFormationService.d.ts +79 -0
- package/dist/src/services/infrastructure/CloudFormationService.js +249 -0
- package/dist/src/services/infrastructure/index.d.ts +8 -0
- package/dist/src/services/infrastructure/index.js +7 -0
- package/dist/src/services/supporting/CdkContextBuilder.d.ts +49 -0
- package/dist/src/services/supporting/CdkContextBuilder.js +44 -0
- package/dist/src/services/supporting/TemplateHashService.d.ts +67 -0
- package/dist/src/services/supporting/TemplateHashService.js +152 -0
- package/dist/src/services/supporting/helpers.d.ts +46 -0
- package/dist/src/services/supporting/helpers.js +81 -0
- package/dist/src/services/supporting/index.d.ts +3 -0
- package/dist/src/services/supporting/index.js +3 -0
- package/dist/src/types/FjallState.d.ts +50 -0
- package/dist/src/types/FjallState.js +118 -0
- package/dist/src/types/ProgressEvent.d.ts +35 -0
- package/dist/src/types/ProgressEvent.js +48 -0
- package/dist/src/types/apiClient.d.ts +34 -0
- package/dist/src/types/apiClient.js +1 -0
- package/dist/src/types/application/ApplicationServiceTypes.d.ts +56 -0
- package/dist/src/types/application/ApplicationServiceTypes.js +30 -0
- package/dist/src/types/application/index.d.ts +1 -0
- package/dist/src/types/application/index.js +1 -0
- package/dist/src/types/callbacks.d.ts +36 -0
- package/dist/src/types/callbacks.js +1 -0
- package/dist/src/types/constants.d.ts +6 -0
- package/dist/src/types/constants.js +6 -0
- package/dist/src/types/credentials.d.ts +30 -0
- package/dist/src/types/credentials.js +1 -0
- package/dist/src/types/deployment/DeploymentServiceTypes.d.ts +23 -0
- package/dist/src/types/deployment/DeploymentServiceTypes.js +1 -0
- package/dist/src/types/deployment/DeploymentTypes.d.ts +29 -0
- package/dist/src/types/deployment/DeploymentTypes.js +1 -0
- package/dist/src/types/deployment/cloudformation.d.ts +14 -0
- package/dist/src/types/deployment/cloudformation.js +1 -0
- package/dist/src/types/deployment/index.d.ts +5 -0
- package/dist/src/types/deployment/index.js +1 -0
- package/dist/src/types/deployment/parallel.d.ts +46 -0
- package/dist/src/types/deployment/parallel.js +10 -0
- package/dist/src/types/errors/CdkError.d.ts +14 -0
- package/dist/src/types/errors/CdkError.js +20 -0
- package/dist/src/types/errors/ServiceError.d.ts +86 -0
- package/dist/src/types/errors/ServiceError.js +119 -0
- package/dist/src/types/events.d.ts +40 -0
- package/dist/src/types/events.js +5 -0
- package/dist/src/types/index.d.ts +20 -0
- package/dist/src/types/index.js +9 -0
- package/dist/src/types/operations.d.ts +193 -0
- package/dist/src/types/operations.js +285 -0
- package/dist/src/types/orgConfig.d.ts +28 -0
- package/dist/src/types/orgConfig.js +11 -0
- package/dist/src/types/params.d.ts +74 -0
- package/dist/src/types/params.js +1 -0
- package/dist/src/types/patternDetection.d.ts +43 -0
- package/dist/src/types/patternDetection.js +92 -0
- package/dist/src/types/validation.d.ts +12 -0
- package/dist/src/types/validation.js +1 -0
- package/dist/src/util/fsHelpers.d.ts +4 -0
- package/dist/src/util/fsHelpers.js +16 -0
- package/dist/src/util/index.d.ts +3 -0
- package/dist/src/util/index.js +3 -0
- package/dist/src/util/securityHelpers.d.ts +31 -0
- package/dist/src/util/securityHelpers.js +124 -0
- package/dist/src/util/singleton.d.ts +2 -0
- package/dist/src/util/singleton.js +9 -0
- package/dist/src/util/sleep.d.ts +4 -0
- package/dist/src/util/sleep.js +4 -0
- package/package.json +42 -0
|
@@ -0,0 +1,596 @@
|
|
|
1
|
+
import { CloudFormationClient, DescribeStackEventsCommand, DescribeStacksCommand } from "@aws-sdk/client-cloudformation";
|
|
2
|
+
import { logger } from "@fjall/util";
|
|
3
|
+
import { getErrorMessage } from "@fjall/util";
|
|
4
|
+
import { maskSensitiveOutput } from "../../util/securityHelpers.js";
|
|
5
|
+
import { sleep } from "../../util/sleep.js";
|
|
6
|
+
// CloudFormation error pattern for stacks that haven't been created yet
|
|
7
|
+
export const STACK_NOT_FOUND_PATTERN = "does not exist";
|
|
8
|
+
// CDK error string indicating the targeted stack does not exist in the synthesised output
|
|
9
|
+
export const CDK_NO_STACKS_MATCH = "No stacks match the name(s)";
|
|
10
|
+
export function isResourceEvent(event) {
|
|
11
|
+
return (typeof event === "object" &&
|
|
12
|
+
event !== null &&
|
|
13
|
+
"logicalId" in event &&
|
|
14
|
+
"resourceType" in event &&
|
|
15
|
+
"status" in event);
|
|
16
|
+
}
|
|
17
|
+
export class CloudFormationEventMonitor {
|
|
18
|
+
aws;
|
|
19
|
+
seenEventIds = new Set();
|
|
20
|
+
isMonitoring = false;
|
|
21
|
+
pollInterval = null;
|
|
22
|
+
activeNestedStacks = new Map(); // Track nested stack states across polls
|
|
23
|
+
failureReasons = new Map(); // Track failure reasons for resources
|
|
24
|
+
eventHistory = new Map(); // Complete event history per resource
|
|
25
|
+
eventLogger = null;
|
|
26
|
+
failureAnalyser;
|
|
27
|
+
eventLogWriterFactory;
|
|
28
|
+
lastAnalysis = null;
|
|
29
|
+
deploymentStartTime = null;
|
|
30
|
+
// 1 000 events ≈ a large CDK stack; caps memory during multi-hour deploys
|
|
31
|
+
maxHistorySize = 1000;
|
|
32
|
+
// 10 000 IDs ≈ 10 large stacks; prevents unbounded Set growth across retries
|
|
33
|
+
maxSeenEventIds = 10000;
|
|
34
|
+
// Track in-progress IPAM resources for concurrency diagnostics
|
|
35
|
+
ipamInProgress = new Map();
|
|
36
|
+
constructor(aws, deps) {
|
|
37
|
+
this.aws = aws;
|
|
38
|
+
this.failureAnalyser = deps?.failureAnalyser ?? null;
|
|
39
|
+
this.eventLogWriterFactory = deps?.eventLogWriterFactory;
|
|
40
|
+
}
|
|
41
|
+
enableLogging(deploymentId, stackName, region, deploymentName) {
|
|
42
|
+
if (this.eventLogWriterFactory) {
|
|
43
|
+
this.eventLogger = this.eventLogWriterFactory(deploymentId, stackName, region, deploymentName);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async startMonitoring(stackName, onResourceUpdate, onStackComplete) {
|
|
47
|
+
if (this.isMonitoring) {
|
|
48
|
+
logger.debug("CloudFormation", "startMonitoring SKIPPED - already monitoring", { stackName });
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
logger.debug("CloudFormation", "startMonitoring STARTED", { stackName });
|
|
52
|
+
this.isMonitoring = true;
|
|
53
|
+
this.seenEventIds.clear();
|
|
54
|
+
this.eventHistory.clear();
|
|
55
|
+
this.deploymentStartTime = new Date();
|
|
56
|
+
try {
|
|
57
|
+
await this.pollEvents(stackName, () => { });
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
// Stack might not exist yet — only log unexpected errors
|
|
61
|
+
const msg = getErrorMessage(error);
|
|
62
|
+
if (!msg.includes(STACK_NOT_FOUND_PATTERN)) {
|
|
63
|
+
logger.debug("CloudFormation", "Initial poll failed", { error: msg });
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
// AWS CDK uses 5 second intervals, but we'll use a more conservative approach
|
|
67
|
+
// to avoid rate limiting, especially for accounts with many concurrent operations
|
|
68
|
+
let pollDelay = 5000; // Start at 5 seconds (matching CDK default)
|
|
69
|
+
const maxPollDelay = 10000; // Max 10 seconds between polls
|
|
70
|
+
let pollCount = 0;
|
|
71
|
+
let throttleCount = 0;
|
|
72
|
+
const poll = async () => {
|
|
73
|
+
if (!this.isMonitoring) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
try {
|
|
77
|
+
const result = await this.pollEvents(stackName, onResourceUpdate);
|
|
78
|
+
pollCount++;
|
|
79
|
+
// If we're being throttled, implement exponential backoff
|
|
80
|
+
if (result === "throttled") {
|
|
81
|
+
throttleCount++;
|
|
82
|
+
// Exponential backoff: 5s, 10s, 20s, 40s (capped at 30s)
|
|
83
|
+
pollDelay = Math.min(30000, 5000 * Math.pow(2, throttleCount - 1));
|
|
84
|
+
}
|
|
85
|
+
else {
|
|
86
|
+
// Reset throttle count on successful poll
|
|
87
|
+
throttleCount = 0;
|
|
88
|
+
// After initial polls, gradually increase interval to reduce API calls
|
|
89
|
+
// This helps with long-running operations (10+ minutes)
|
|
90
|
+
if (pollCount > 20 && pollDelay < maxPollDelay) {
|
|
91
|
+
pollDelay = Math.min(maxPollDelay, pollDelay + 1000);
|
|
92
|
+
}
|
|
93
|
+
else if (pollCount <= 20) {
|
|
94
|
+
// Keep at 5s for first ~100 seconds to maintain good visibility
|
|
95
|
+
pollDelay = 5000;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
if (result === true) {
|
|
99
|
+
this.stopMonitoring();
|
|
100
|
+
// Get final stack status
|
|
101
|
+
const finalStatusObj = await this.getStackStatus(stackName);
|
|
102
|
+
const finalStatus = finalStatusObj?.status || "UNKNOWN";
|
|
103
|
+
const success = finalStatus.endsWith("_COMPLETE") &&
|
|
104
|
+
!finalStatus.includes("ROLLBACK") &&
|
|
105
|
+
!finalStatus.includes("FAILED");
|
|
106
|
+
// Get first failure reason if deployment failed
|
|
107
|
+
let failureMessage = finalStatusObj?.statusReason || undefined;
|
|
108
|
+
if (!success && this.failureReasons.size > 0) {
|
|
109
|
+
const firstFailure = Array.from(this.failureReasons.values())[0];
|
|
110
|
+
failureMessage = firstFailure || failureMessage;
|
|
111
|
+
}
|
|
112
|
+
// Log deployment completion
|
|
113
|
+
if (this.eventLogger) {
|
|
114
|
+
this.eventLogger.writeDeploymentEnd(success || false, finalStatus, failureMessage);
|
|
115
|
+
// If failed, log all failure reasons collected
|
|
116
|
+
if (!success && this.failureReasons.size > 0) {
|
|
117
|
+
this.eventLogger.writeFailureSummary(Array.from(this.failureReasons.entries()).map(([logicalId, reason]) => ({
|
|
118
|
+
logicalId,
|
|
119
|
+
reason
|
|
120
|
+
})));
|
|
121
|
+
}
|
|
122
|
+
// Also log failure analysis if available
|
|
123
|
+
if (!success && this.failureAnalyser) {
|
|
124
|
+
const analysis = this.failureAnalyser.analyseFailure(this.eventHistory);
|
|
125
|
+
this.lastAnalysis = analysis;
|
|
126
|
+
if (analysis) {
|
|
127
|
+
this.eventLogger.writeFailureAnalysis({
|
|
128
|
+
rootCause: analysis.rootCause.reason,
|
|
129
|
+
failedResources: analysis.affectedResources.map((r) => ({
|
|
130
|
+
logicalId: r.logicalId,
|
|
131
|
+
resourceType: r.resourceType,
|
|
132
|
+
reason: r.statusReason || "Unknown reason"
|
|
133
|
+
})),
|
|
134
|
+
dependencyChain: analysis.dependencyChain,
|
|
135
|
+
remediation: analysis.remediation
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
if (onStackComplete) {
|
|
141
|
+
onStackComplete(success || false, failureMessage);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
// Schedule next poll
|
|
146
|
+
this.pollInterval = setTimeout(poll, pollDelay);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
catch (error) {
|
|
150
|
+
logger.debug("CloudFormation", "Polling iteration error (continuing)", {
|
|
151
|
+
error: getErrorMessage(error)
|
|
152
|
+
});
|
|
153
|
+
this.pollInterval = setTimeout(poll, pollDelay);
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
// Start polling
|
|
157
|
+
this.pollInterval = setTimeout(poll, pollDelay);
|
|
158
|
+
}
|
|
159
|
+
stopMonitoring() {
|
|
160
|
+
logger.debug("CloudFormation", "stopMonitoring called", {
|
|
161
|
+
wasMonitoring: this.isMonitoring,
|
|
162
|
+
seenEventCount: this.seenEventIds.size
|
|
163
|
+
});
|
|
164
|
+
this.isMonitoring = false;
|
|
165
|
+
if (this.pollInterval) {
|
|
166
|
+
clearTimeout(this.pollInterval);
|
|
167
|
+
this.pollInterval = null;
|
|
168
|
+
}
|
|
169
|
+
this.cleanup();
|
|
170
|
+
}
|
|
171
|
+
cleanup() {
|
|
172
|
+
this.activeNestedStacks.clear();
|
|
173
|
+
this.failureReasons.clear();
|
|
174
|
+
this.eventHistory.clear();
|
|
175
|
+
this.ipamInProgress.clear();
|
|
176
|
+
if (this.seenEventIds.size > this.maxSeenEventIds) {
|
|
177
|
+
const idsArray = Array.from(this.seenEventIds);
|
|
178
|
+
const keepCount = Math.floor(this.maxSeenEventIds / 2);
|
|
179
|
+
this.seenEventIds = new Set(idsArray.slice(-keepCount));
|
|
180
|
+
}
|
|
181
|
+
if (this.eventLogger) {
|
|
182
|
+
this.eventLogger = null;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
getResourceHistory(logicalId) {
|
|
186
|
+
return this.eventHistory.get(logicalId) || [];
|
|
187
|
+
}
|
|
188
|
+
getEventHistory() {
|
|
189
|
+
return new Map(this.eventHistory);
|
|
190
|
+
}
|
|
191
|
+
getFailureAnalysis() {
|
|
192
|
+
return this.lastAnalysis;
|
|
193
|
+
}
|
|
194
|
+
getFirstFailureReason() {
|
|
195
|
+
if (this.failureReasons.size > 0) {
|
|
196
|
+
return Array.from(this.failureReasons.values())[0];
|
|
197
|
+
}
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
getEventLogger() {
|
|
201
|
+
return this.eventLogger;
|
|
202
|
+
}
|
|
203
|
+
getEventLogPath() {
|
|
204
|
+
return this.eventLogger?.getLogPath() || null;
|
|
205
|
+
}
|
|
206
|
+
getLogSummary() {
|
|
207
|
+
return this.eventLogger?.getLogSummary() || null;
|
|
208
|
+
}
|
|
209
|
+
async waitForStackComplete(stackName, options = {}) {
|
|
210
|
+
const { timeout = 30 * 60 * 1000, // 30 minutes default
|
|
211
|
+
pollInterval = 2000, // 2 seconds
|
|
212
|
+
onResourceUpdate, onStackComplete } = options;
|
|
213
|
+
const startTime = Date.now();
|
|
214
|
+
let lastStatus;
|
|
215
|
+
let stackComplete = false;
|
|
216
|
+
let stackSuccess = false;
|
|
217
|
+
let failureReason;
|
|
218
|
+
logger.debug("CloudFormation", "waitForStackComplete called", {
|
|
219
|
+
stackName,
|
|
220
|
+
timeout,
|
|
221
|
+
pollInterval,
|
|
222
|
+
hasOnResourceUpdate: !!onResourceUpdate
|
|
223
|
+
});
|
|
224
|
+
await this.startMonitoring(stackName, (event) => {
|
|
225
|
+
if (onResourceUpdate) {
|
|
226
|
+
onResourceUpdate(event);
|
|
227
|
+
}
|
|
228
|
+
if (event.resourceType === "AWS::CloudFormation::Stack" &&
|
|
229
|
+
event.logicalId === stackName) {
|
|
230
|
+
lastStatus = event.status;
|
|
231
|
+
if (this.isTerminalState(event.status)) {
|
|
232
|
+
stackComplete = true;
|
|
233
|
+
stackSuccess = this.isSuccessState(event.status);
|
|
234
|
+
if (!stackSuccess) {
|
|
235
|
+
failureReason =
|
|
236
|
+
this.getFirstFailureReason() ||
|
|
237
|
+
event.statusReason ||
|
|
238
|
+
"Stack operation failed";
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
}, (success, message) => {
|
|
243
|
+
if (onStackComplete) {
|
|
244
|
+
onStackComplete(success, message);
|
|
245
|
+
}
|
|
246
|
+
});
|
|
247
|
+
try {
|
|
248
|
+
let hasSeenStackEvents = false;
|
|
249
|
+
let noEventWarningShown = false;
|
|
250
|
+
while (!stackComplete &&
|
|
251
|
+
this.isMonitoring &&
|
|
252
|
+
Date.now() - startTime < timeout) {
|
|
253
|
+
await sleep(pollInterval);
|
|
254
|
+
if (!hasSeenStackEvents &&
|
|
255
|
+
Date.now() - startTime > 30000 &&
|
|
256
|
+
!noEventWarningShown) {
|
|
257
|
+
noEventWarningShown = true;
|
|
258
|
+
const stackStatus = await this.getStackStatus(stackName);
|
|
259
|
+
if (!stackStatus) {
|
|
260
|
+
// Stack doesn't exist yet - CDK may still be uploading assets
|
|
261
|
+
// (e.g. Lambda deployment packages). Don't break - CDK result is
|
|
262
|
+
// the source of truth; keep waiting for the stack to appear.
|
|
263
|
+
logger.debug("CloudFormation", "Stack not found after 30s, continuing to wait (CDK may be uploading assets)", { stackName });
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (lastStatus) {
|
|
267
|
+
hasSeenStackEvents = true;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
this.stopMonitoring();
|
|
271
|
+
if (!stackComplete) {
|
|
272
|
+
if (failureReason) {
|
|
273
|
+
return {
|
|
274
|
+
success: false,
|
|
275
|
+
status: "FAILED",
|
|
276
|
+
failureReason: failureReason,
|
|
277
|
+
logPath: this.getLogSummary() || undefined
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
const finalStatus = await this.getStackStatus(stackName);
|
|
281
|
+
return {
|
|
282
|
+
success: false,
|
|
283
|
+
status: finalStatus?.status || "UNKNOWN",
|
|
284
|
+
failureReason: `Deployment timed out after ${timeout / 1000} seconds. Stack status: ${finalStatus?.status || "UNKNOWN"}`,
|
|
285
|
+
logPath: this.getLogSummary() || undefined
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
return {
|
|
289
|
+
success: stackSuccess,
|
|
290
|
+
status: lastStatus,
|
|
291
|
+
failureReason: failureReason,
|
|
292
|
+
logPath: this.getLogSummary() || undefined
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
this.stopMonitoring();
|
|
297
|
+
return {
|
|
298
|
+
success: false,
|
|
299
|
+
failureReason: `Monitoring error: ${getErrorMessage(error)}`,
|
|
300
|
+
logPath: this.getLogSummary() || undefined
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
isTerminalState(status) {
|
|
305
|
+
return [
|
|
306
|
+
"CREATE_COMPLETE",
|
|
307
|
+
"CREATE_FAILED",
|
|
308
|
+
"DELETE_COMPLETE",
|
|
309
|
+
"DELETE_FAILED",
|
|
310
|
+
"UPDATE_COMPLETE",
|
|
311
|
+
"UPDATE_FAILED",
|
|
312
|
+
"UPDATE_ROLLBACK_COMPLETE",
|
|
313
|
+
"UPDATE_ROLLBACK_FAILED",
|
|
314
|
+
"ROLLBACK_COMPLETE",
|
|
315
|
+
"ROLLBACK_FAILED",
|
|
316
|
+
"IMPORT_COMPLETE",
|
|
317
|
+
"IMPORT_ROLLBACK_COMPLETE",
|
|
318
|
+
"IMPORT_ROLLBACK_FAILED"
|
|
319
|
+
].includes(status);
|
|
320
|
+
}
|
|
321
|
+
isSuccessState(status) {
|
|
322
|
+
return [
|
|
323
|
+
"CREATE_COMPLETE",
|
|
324
|
+
"UPDATE_COMPLETE",
|
|
325
|
+
"DELETE_COMPLETE",
|
|
326
|
+
"IMPORT_COMPLETE"
|
|
327
|
+
].includes(status);
|
|
328
|
+
}
|
|
329
|
+
async pollEvents(stackName, onResourceUpdate) {
|
|
330
|
+
try {
|
|
331
|
+
const client = this.aws.getClient(CloudFormationClient);
|
|
332
|
+
let nextToken;
|
|
333
|
+
let stackComplete = false;
|
|
334
|
+
let reachedSeenEvents = false;
|
|
335
|
+
do {
|
|
336
|
+
const command = new DescribeStackEventsCommand({
|
|
337
|
+
StackName: stackName,
|
|
338
|
+
...(nextToken ? { NextToken: nextToken } : {})
|
|
339
|
+
});
|
|
340
|
+
const response = await client.send(command);
|
|
341
|
+
if (!response.StackEvents) {
|
|
342
|
+
break;
|
|
343
|
+
}
|
|
344
|
+
for (const event of response.StackEvents) {
|
|
345
|
+
const eventId = `${event.EventId}`;
|
|
346
|
+
if (this.seenEventIds.has(eventId)) {
|
|
347
|
+
// Events are returned newest-first; a seen event means all
|
|
348
|
+
// remaining events on this page (and subsequent pages) are older
|
|
349
|
+
// and already processed — stop paginating after this page.
|
|
350
|
+
// Skip processing (including terminal-state checks) for already-seen events.
|
|
351
|
+
reachedSeenEvents = true;
|
|
352
|
+
continue;
|
|
353
|
+
}
|
|
354
|
+
this.seenEventIds.add(eventId);
|
|
355
|
+
if (event.ResourceType === "AWS::CloudFormation::Stack" &&
|
|
356
|
+
event.LogicalResourceId !== stackName) {
|
|
357
|
+
const status = event.ResourceStatus || "";
|
|
358
|
+
const nestedStackId = event.LogicalResourceId || "";
|
|
359
|
+
if (status.includes("IN_PROGRESS")) {
|
|
360
|
+
this.activeNestedStacks.set(nestedStackId, status);
|
|
361
|
+
}
|
|
362
|
+
else if (status.includes("COMPLETE") ||
|
|
363
|
+
status.includes("FAILED")) {
|
|
364
|
+
this.activeNestedStacks.delete(nestedStackId);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
if (event.LogicalResourceId === stackName &&
|
|
368
|
+
event.ResourceType === "AWS::CloudFormation::Stack") {
|
|
369
|
+
const status = event.ResourceStatus || "";
|
|
370
|
+
if (status === "DELETE_COMPLETE") {
|
|
371
|
+
stackComplete = true;
|
|
372
|
+
}
|
|
373
|
+
else if (status === "CREATE_COMPLETE" ||
|
|
374
|
+
status === "UPDATE_COMPLETE") {
|
|
375
|
+
stackComplete = true;
|
|
376
|
+
}
|
|
377
|
+
else if (status === "CREATE_FAILED" ||
|
|
378
|
+
status === "UPDATE_FAILED" ||
|
|
379
|
+
status === "DELETE_FAILED" ||
|
|
380
|
+
status === "ROLLBACK_COMPLETE" ||
|
|
381
|
+
status === "UPDATE_ROLLBACK_COMPLETE") {
|
|
382
|
+
stackComplete = true;
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
if (event.ResourceStatus &&
|
|
386
|
+
event.ResourceStatus.includes("FAILED") &&
|
|
387
|
+
event.ResourceStatusReason) {
|
|
388
|
+
this.failureReasons.set(event.LogicalResourceId || "unknown", event.ResourceStatusReason);
|
|
389
|
+
// Log failure details for debugging
|
|
390
|
+
logger.debug("CloudFormation", `Captured failure: ${event.LogicalResourceId} - ${event.ResourceStatusReason}`);
|
|
391
|
+
}
|
|
392
|
+
const resourceEvent = {
|
|
393
|
+
logicalId: event.LogicalResourceId || "",
|
|
394
|
+
physicalId: event.PhysicalResourceId,
|
|
395
|
+
resourceType: event.ResourceType || "",
|
|
396
|
+
status: event.ResourceStatus || "",
|
|
397
|
+
statusReason: event.ResourceStatusReason,
|
|
398
|
+
timestamp: event.Timestamp || new Date()
|
|
399
|
+
};
|
|
400
|
+
const logicalId = resourceEvent.logicalId;
|
|
401
|
+
const history = this.eventHistory.get(logicalId) ?? [];
|
|
402
|
+
history.push(resourceEvent);
|
|
403
|
+
this.eventHistory.set(logicalId, history);
|
|
404
|
+
// Enforce maxHistorySize to prevent memory leaks during long deployments
|
|
405
|
+
if (this.eventHistory.size > this.maxHistorySize) {
|
|
406
|
+
const keysToDelete = [...this.eventHistory.keys()].slice(0, this.eventHistory.size - this.maxHistorySize);
|
|
407
|
+
for (const key of keysToDelete) {
|
|
408
|
+
this.eventHistory.delete(key);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
// IPAM concurrency tracking — logs which IPAM pools/CIDRs are in-flight
|
|
412
|
+
// simultaneously, critical for diagnosing "too many concurrent mutations"
|
|
413
|
+
if (resourceEvent.resourceType === "AWS::EC2::IPAMPool" ||
|
|
414
|
+
resourceEvent.resourceType === "AWS::EC2::IPAMPoolCidr" ||
|
|
415
|
+
resourceEvent.resourceType === "AWS::EC2::IPAM") {
|
|
416
|
+
if (resourceEvent.status.includes("IN_PROGRESS")) {
|
|
417
|
+
this.ipamInProgress.set(resourceEvent.logicalId, {
|
|
418
|
+
resourceType: resourceEvent.resourceType,
|
|
419
|
+
startTime: resourceEvent.timestamp
|
|
420
|
+
});
|
|
421
|
+
const concurrent = Array.from(this.ipamInProgress.entries()).map(([id, info]) => `${id} (${info.resourceType.split("::").pop()}, started ${info.startTime.toISOString()})`);
|
|
422
|
+
logger.debug("CloudFormation", "IPAM resource started", {
|
|
423
|
+
logicalId: resourceEvent.logicalId,
|
|
424
|
+
resourceType: resourceEvent.resourceType,
|
|
425
|
+
concurrentIpamCount: this.ipamInProgress.size,
|
|
426
|
+
concurrentIpamResources: concurrent
|
|
427
|
+
});
|
|
428
|
+
if (this.eventLogger) {
|
|
429
|
+
this.eventLogger.writeEvent(resourceEvent, {
|
|
430
|
+
phase: "ipam_concurrency",
|
|
431
|
+
isMainStack: true,
|
|
432
|
+
parentStack: `concurrent=${this.ipamInProgress.size}: ${concurrent.join(", ")}`
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
else if (resourceEvent.status.includes("COMPLETE") ||
|
|
437
|
+
resourceEvent.status.includes("FAILED")) {
|
|
438
|
+
const wasInProgress = this.ipamInProgress.has(resourceEvent.logicalId);
|
|
439
|
+
const startInfo = this.ipamInProgress.get(resourceEvent.logicalId);
|
|
440
|
+
this.ipamInProgress.delete(resourceEvent.logicalId);
|
|
441
|
+
const durationMs = startInfo
|
|
442
|
+
? resourceEvent.timestamp.getTime() -
|
|
443
|
+
startInfo.startTime.getTime()
|
|
444
|
+
: undefined;
|
|
445
|
+
logger.debug("CloudFormation", `IPAM resource ${resourceEvent.status}`, {
|
|
446
|
+
logicalId: resourceEvent.logicalId,
|
|
447
|
+
resourceType: resourceEvent.resourceType,
|
|
448
|
+
status: resourceEvent.status,
|
|
449
|
+
statusReason: resourceEvent.statusReason,
|
|
450
|
+
durationMs,
|
|
451
|
+
wasTracked: wasInProgress,
|
|
452
|
+
remainingIpamCount: this.ipamInProgress.size,
|
|
453
|
+
remainingIpamResources: Array.from(this.ipamInProgress.keys())
|
|
454
|
+
});
|
|
455
|
+
if (resourceEvent.status.includes("FAILED") && this.eventLogger) {
|
|
456
|
+
// Snapshot all in-flight IPAM resources at failure time
|
|
457
|
+
const snapshot = Array.from(this.ipamInProgress.entries()).map(([id, info]) => ({
|
|
458
|
+
logicalId: id,
|
|
459
|
+
resourceType: info.resourceType,
|
|
460
|
+
startTime: info.startTime.toISOString(),
|
|
461
|
+
inFlightDurationMs: resourceEvent.timestamp.getTime() -
|
|
462
|
+
info.startTime.getTime()
|
|
463
|
+
}));
|
|
464
|
+
this.eventLogger.writeEvent(resourceEvent, {
|
|
465
|
+
phase: "ipam_failure_snapshot",
|
|
466
|
+
isMainStack: true,
|
|
467
|
+
parentStack: JSON.stringify({
|
|
468
|
+
failedResource: resourceEvent.logicalId,
|
|
469
|
+
reason: resourceEvent.statusReason,
|
|
470
|
+
durationMs,
|
|
471
|
+
concurrentIpamAtFailure: snapshot
|
|
472
|
+
})
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
if (this.eventLogger) {
|
|
478
|
+
this.eventLogger.writeEvent(resourceEvent);
|
|
479
|
+
}
|
|
480
|
+
logger.debug("CloudFormation", "pollEvents delivering event", {
|
|
481
|
+
stackName,
|
|
482
|
+
logicalId: resourceEvent.logicalId,
|
|
483
|
+
status: resourceEvent.status,
|
|
484
|
+
resourceType: resourceEvent.resourceType
|
|
485
|
+
});
|
|
486
|
+
onResourceUpdate(resourceEvent);
|
|
487
|
+
}
|
|
488
|
+
nextToken = reachedSeenEvents ? undefined : response.NextToken;
|
|
489
|
+
} while (nextToken);
|
|
490
|
+
if (stackComplete && this.activeNestedStacks.size > 0) {
|
|
491
|
+
return false;
|
|
492
|
+
}
|
|
493
|
+
return stackComplete;
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
const errorName = typeof error === "object" &&
|
|
497
|
+
error !== null &&
|
|
498
|
+
"name" in error &&
|
|
499
|
+
typeof error.name === "string"
|
|
500
|
+
? error.name
|
|
501
|
+
: undefined;
|
|
502
|
+
const errorMessage = getErrorMessage(error);
|
|
503
|
+
if (errorName === "ValidationError" &&
|
|
504
|
+
errorMessage.includes(STACK_NOT_FOUND_PATTERN)) {
|
|
505
|
+
return false;
|
|
506
|
+
}
|
|
507
|
+
if (errorName === "Throttling" ||
|
|
508
|
+
errorName === "TooManyRequestsException" ||
|
|
509
|
+
errorName === "ThrottlingException") {
|
|
510
|
+
return "throttled";
|
|
511
|
+
}
|
|
512
|
+
if (!errorMessage.includes(STACK_NOT_FOUND_PATTERN)) {
|
|
513
|
+
logger.error("CloudFormation", "Unexpected polling error", {
|
|
514
|
+
error: maskSensitiveOutput(errorMessage)
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
return false;
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async getStackStatus(stackName) {
|
|
521
|
+
try {
|
|
522
|
+
const command = new DescribeStacksCommand({
|
|
523
|
+
StackName: stackName
|
|
524
|
+
});
|
|
525
|
+
const client = this.aws.getClient(CloudFormationClient);
|
|
526
|
+
const response = await client.send(command);
|
|
527
|
+
if (response.Stacks && response.Stacks.length > 0) {
|
|
528
|
+
const stack = response.Stacks[0];
|
|
529
|
+
return {
|
|
530
|
+
status: stack.StackStatus || "UNKNOWN",
|
|
531
|
+
statusReason: stack.StackStatusReason
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
return null;
|
|
535
|
+
}
|
|
536
|
+
catch (error) {
|
|
537
|
+
logger.debug("CloudFormation", "getStackStatus failed", {
|
|
538
|
+
error: getErrorMessage(error)
|
|
539
|
+
});
|
|
540
|
+
return null;
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
async getCurrentResources(stackName) {
|
|
544
|
+
const resources = [];
|
|
545
|
+
try {
|
|
546
|
+
const command = new DescribeStackEventsCommand({
|
|
547
|
+
StackName: stackName
|
|
548
|
+
});
|
|
549
|
+
const client = this.aws.getClient(CloudFormationClient);
|
|
550
|
+
const response = await client.send(command);
|
|
551
|
+
if (!response.StackEvents) {
|
|
552
|
+
return resources;
|
|
553
|
+
}
|
|
554
|
+
const resourceMap = new Map();
|
|
555
|
+
for (const event of response.StackEvents) {
|
|
556
|
+
const logicalId = event.LogicalResourceId || "";
|
|
557
|
+
if (logicalId === stackName &&
|
|
558
|
+
event.ResourceType === "AWS::CloudFormation::Stack") {
|
|
559
|
+
continue;
|
|
560
|
+
}
|
|
561
|
+
const existing = resourceMap.get(logicalId);
|
|
562
|
+
if (!existing ||
|
|
563
|
+
(event.Timestamp && existing.timestamp < event.Timestamp)) {
|
|
564
|
+
resourceMap.set(logicalId, {
|
|
565
|
+
logicalId,
|
|
566
|
+
physicalId: event.PhysicalResourceId,
|
|
567
|
+
resourceType: event.ResourceType || "",
|
|
568
|
+
status: event.ResourceStatus || "",
|
|
569
|
+
statusReason: event.ResourceStatusReason,
|
|
570
|
+
timestamp: event.Timestamp || new Date()
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
return Array.from(resourceMap.values());
|
|
575
|
+
}
|
|
576
|
+
catch (error) {
|
|
577
|
+
const errorName = typeof error === "object" &&
|
|
578
|
+
error !== null &&
|
|
579
|
+
"name" in error &&
|
|
580
|
+
typeof error.name === "string"
|
|
581
|
+
? error.name
|
|
582
|
+
: undefined;
|
|
583
|
+
const errorMessage = getErrorMessage(error);
|
|
584
|
+
if (errorName === "ValidationError" &&
|
|
585
|
+
errorMessage.includes(STACK_NOT_FOUND_PATTERN)) {
|
|
586
|
+
return resources;
|
|
587
|
+
}
|
|
588
|
+
if (!errorMessage.includes(STACK_NOT_FOUND_PATTERN)) {
|
|
589
|
+
logger.error("CloudFormation", "Error getting current resources", {
|
|
590
|
+
error: maskSensitiveOutput(getErrorMessage(error))
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
return resources;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export declare class AWSError extends Error {
|
|
2
|
+
readonly code?: string | undefined;
|
|
3
|
+
constructor(message: string, code?: string | undefined);
|
|
4
|
+
}
|
|
5
|
+
export declare class NoRolesFoundError extends AWSError {
|
|
6
|
+
constructor(accountId?: string);
|
|
7
|
+
}
|
|
8
|
+
export declare class InvalidCredentialsError extends AWSError {
|
|
9
|
+
constructor(message: string);
|
|
10
|
+
}
|
|
11
|
+
export declare class SSOTokenExpiredError extends AWSError {
|
|
12
|
+
constructor();
|
|
13
|
+
}
|
|
14
|
+
export declare class MissingRegionError extends AWSError {
|
|
15
|
+
constructor();
|
|
16
|
+
}
|
|
17
|
+
export declare class ProfileNotFoundError extends AWSError {
|
|
18
|
+
constructor(profileName: string);
|
|
19
|
+
}
|
|
20
|
+
export declare function isAWSError(error: unknown): error is AWSError;
|
|
21
|
+
export declare function isNoRolesFoundError(error: unknown): error is NoRolesFoundError;
|
|
22
|
+
export declare function isSSOUnauthorizedError(error: unknown): boolean;
|
|
23
|
+
export declare class CommandError extends AWSError {
|
|
24
|
+
readonly tip?: string;
|
|
25
|
+
constructor(message: string, tip?: string);
|
|
26
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export class AWSError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
constructor(message, code) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.code = code;
|
|
6
|
+
this.name = this.constructor.name;
|
|
7
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export class NoRolesFoundError extends AWSError {
|
|
11
|
+
constructor(accountId) {
|
|
12
|
+
const message = accountId
|
|
13
|
+
? `No roles found for account ${accountId}`
|
|
14
|
+
: "No roles found for this account";
|
|
15
|
+
super(message, "NO_ROLES_FOUND");
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export class InvalidCredentialsError extends AWSError {
|
|
19
|
+
constructor(message) {
|
|
20
|
+
super(message, "INVALID_CREDENTIALS");
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
export class SSOTokenExpiredError extends AWSError {
|
|
24
|
+
constructor() {
|
|
25
|
+
super("SSO token has expired", "SSO_TOKEN_EXPIRED");
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
export class MissingRegionError extends AWSError {
|
|
29
|
+
constructor() {
|
|
30
|
+
super("Region is missing in AWS configuration and environment", "MISSING_REGION");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
export class ProfileNotFoundError extends AWSError {
|
|
34
|
+
constructor(profileName) {
|
|
35
|
+
super(`AWS profile '${profileName}' not found`, "PROFILE_NOT_FOUND");
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
export function isAWSError(error) {
|
|
39
|
+
return error instanceof AWSError;
|
|
40
|
+
}
|
|
41
|
+
export function isNoRolesFoundError(error) {
|
|
42
|
+
return error instanceof NoRolesFoundError;
|
|
43
|
+
}
|
|
44
|
+
export function isSSOUnauthorizedError(error) {
|
|
45
|
+
if (!error || typeof error !== "object")
|
|
46
|
+
return false;
|
|
47
|
+
return (("name" in error && error.name === "UnauthorizedException") ||
|
|
48
|
+
("Code" in error && error.Code === "UnauthorizedException") ||
|
|
49
|
+
("__type" in error &&
|
|
50
|
+
typeof error.__type === "string" &&
|
|
51
|
+
error.__type.includes("UnauthorizedException")));
|
|
52
|
+
}
|
|
53
|
+
export class CommandError extends AWSError {
|
|
54
|
+
tip;
|
|
55
|
+
constructor(message, tip) {
|
|
56
|
+
super(message, "COMMAND_ERROR");
|
|
57
|
+
this.tip = tip;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { type RegionInfo, DEFAULT_REGION, regions, AWS_REGIONS_METADATA, topRegions, commonRegions, parseRegionList, isValidRegion, isValidRegionFormat, getSuggestions, validateRegion, validateRegionList, filterDuplicateRegions, getRegionOptions, getRegionOptionsExcluding, getRegionName, createRegionFormatter } from "@fjall/generator";
|