grada-run 0.49.4 ā 0.49.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -1
- package/src/commands/add.js +1 -1
- package/src/commands/alerts.js +1 -1
- package/src/commands/apply.js +84 -5
- package/src/commands/db/backup.js +3 -3
- package/src/commands/db/connect.js +2 -2
- package/src/commands/db/enable-vector.js +1 -1
- package/src/commands/db/import.js +2 -2
- package/src/commands/db/migrate.js +7 -6
- package/src/commands/destroy.js +116 -10
- package/src/commands/diagnose.js +18 -5
- package/src/commands/domain.js +51 -4
- package/src/commands/exec.js +1 -1
- package/src/commands/gc.js +2 -2
- package/src/commands/init.js +7 -7
- package/src/commands/logs.js +20 -7
- package/src/commands/rollback.js +21 -6
- package/src/commands/secrets.js +19 -19
- package/src/commands/status.js +3 -3
- package/src/commands/wake.js +2 -2
- package/src/utils/addons.js +1 -1
- package/src/utils/backup.js +1 -1
- package/src/utils/ecs-runner.js +3 -3
- package/src/utils/generator.js +3 -3
- package/src/utils/prompts.js +1 -1
- package/src/utils/resolvers.js +8 -10
- package/src/utils/terraform-errors.js +47 -0
- package/src/utils/terraform.js +1 -1
- package/templates/README.md +2 -2
- package/templates/github/deploy-lambda.yml +2 -2
- package/templates/github/deploy-static.yml +1 -1
- package/templates/github/deploy.yml +2 -2
- package/templates/github/drift.yml +1 -1
- package/templates/github/plan.yml +2 -2
- package/templates/github/preview-lambda.yml +4 -4
- package/templates/github/preview.yml +4 -4
- package/templates/github/teardown.yml +3 -3
- package/templates/terraform/addons/bedrock.tf +1 -1
- package/templates/terraform/backend.tf +1 -1
- package/templates/terraform/cloudfront-lambda.tf +2 -2
- package/templates/terraform/cloudfront.tf +3 -3
- package/templates/terraform/database-aurora-postgresql.tf +1 -1
- package/templates/terraform/main.tf +3 -3
- package/templates/terraform/oidc.tf +29 -14
- package/templates/terraform/secrets.tf +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "grada-run",
|
|
3
|
-
"version": "0.49.
|
|
3
|
+
"version": "0.49.6",
|
|
4
4
|
"description": "Provision production-ready AWS infrastructure and CI/CD pipelines in seconds.",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=18.0.0"
|
|
@@ -111,6 +111,7 @@
|
|
|
111
111
|
"@aws-sdk/client-elastic-load-balancing-v2": "3.1119.0",
|
|
112
112
|
"@aws-sdk/client-iam": "3.1119.0",
|
|
113
113
|
"@aws-sdk/client-rds": "3.1119.0",
|
|
114
|
+
"@aws-sdk/client-route-53": "3.1119.0",
|
|
114
115
|
"@aws-sdk/client-s3": "3.1115.0",
|
|
115
116
|
"@aws-sdk/client-secrets-manager": "3.1112.0",
|
|
116
117
|
"@aws-sdk/client-sqs": "3.1119.0",
|
package/src/commands/add.js
CHANGED
|
@@ -706,7 +706,7 @@ export function renderWorkerAutoscalingBlock(hasWorker) {
|
|
|
706
706
|
function renderAddonTemplate(templateContent, templateVars = {}, conditionalBlocks = {}) {
|
|
707
707
|
let rendered = String(templateContent ?? '');
|
|
708
708
|
// Replacement FUNCTIONS (not strings): `$` patterns in values (cron
|
|
709
|
-
// commands, ARNs) land byte-identical
|
|
709
|
+
// commands, ARNs) land byte-identical.
|
|
710
710
|
for (const [key, value] of Object.entries(templateVars)) {
|
|
711
711
|
const literal = String(value ?? '');
|
|
712
712
|
rendered = rendered.replaceAll(`{{${key}}}`, () => literal);
|
package/src/commands/alerts.js
CHANGED
|
@@ -187,7 +187,7 @@ export function buildAlertsTf({ projectName, target = 'ecs', email, webhook, thr
|
|
|
187
187
|
const defaults = ALERT_TARGET_DEFAULTS[target] || ALERT_TARGET_DEFAULTS.ecs;
|
|
188
188
|
const limit = threshold ?? defaults.threshold;
|
|
189
189
|
// All targets use the workspace-aware local: flat ECS names let a
|
|
190
|
-
// preview apply clobber the prod alarm
|
|
190
|
+
// preview apply clobber the prod alarm. Existing trees
|
|
191
191
|
// migrate on the next `grada alerts` + apply (topic/alarm recreate).
|
|
192
192
|
const namePrefix = '${local.app_name}';
|
|
193
193
|
const aliasLine = target === 'static' ? ' provider = aws.us_east_1\n' : '';
|
package/src/commands/apply.js
CHANGED
|
@@ -9,7 +9,7 @@ import { failCommand, isProgrammaticCall, shouldAutoApprove, refuseIfNonInteract
|
|
|
9
9
|
import { parseFlags, normalizeOptions, normalizeArgv } from '../utils/args.js';
|
|
10
10
|
import { spawnSync } from 'child_process';
|
|
11
11
|
import { runTerraformCommand, getTerraformOutputs, readLastRunMarker, isInterruptedMarker, ensureTerraformInit, checkOutputsContract, DEFAULT_TERRAFORM_PHASE_TIMEOUT_MS, PROGRESS_MODES } from '../utils/terraform.js';
|
|
12
|
-
import { classifyTerraformError, countRecreateCollisions, getRunbook, ERROR_CODES } from '../utils/terraform-errors.js';
|
|
12
|
+
import { classifyTerraformError, countRecreateCollisions, getRunbook, ERROR_CODES, parseCollisionImports } from '../utils/terraform-errors.js';
|
|
13
13
|
import { readInitManifest, recordAppliedTarget } from '../utils/backup.js';
|
|
14
14
|
import { requireManagedTree, parseStateBucketBackend, upsertAutoTfvar } from '../utils/terraform-metadata.js';
|
|
15
15
|
import { readTerraformComputeTarget, resolveRegion, readRenderedContainerEnvNames } from '../utils/resolvers.js';
|
|
@@ -492,7 +492,7 @@ async function applyStackMain(input = {}) {
|
|
|
492
492
|
if (!sleepGuard.proceed) return sleepGuard.result;
|
|
493
493
|
}
|
|
494
494
|
|
|
495
|
-
// 1d. Preflight secret/env collisions
|
|
495
|
+
// 1d. Preflight secret/env collisions: stale,
|
|
496
496
|
// confirmed-through, or hand-edited keys files would die late on
|
|
497
497
|
// the raw ECS 400 ā fail here naming the key and the fix. ECS-only
|
|
498
498
|
// (Lambda maps no secrets into env), unconditional (a collision is
|
|
@@ -671,7 +671,7 @@ async function applyStackMain(input = {}) {
|
|
|
671
671
|
if (cfUrl) finalMessage += `\n š App URL: ${color.cyan(cfUrl)} ${color.dim('(global CDN ā share this link)')}`;
|
|
672
672
|
// The origin only answers CloudFront (listener rule 403s direct
|
|
673
673
|
// hits without the origin-secret header) ā label the URL as
|
|
674
|
-
// unreachable instead of a working bypass
|
|
674
|
+
// unreachable instead of a working bypass.
|
|
675
675
|
if (albUrl) finalMessage += `\n š¦ Direct URL: ${color.gray(albUrl)} ${color.dim('(CloudFront-only origin ā direct hits 403)')}`;
|
|
676
676
|
// Lambda stacks have no CDN: without a CloudFront URL the API
|
|
677
677
|
// URL is the primary link, not a debug bypass.
|
|
@@ -797,12 +797,54 @@ async function applyStackMain(input = {}) {
|
|
|
797
797
|
});
|
|
798
798
|
return failCommand({ exitCode: 1 });
|
|
799
799
|
}
|
|
800
|
+
// Guided import first: stderr names the collided resource
|
|
801
|
+
// and (for most families) its provider id ā import in-flow
|
|
802
|
+
// instead of hand-typing. Secrets are excluded (import needs
|
|
803
|
+
// the ARN, not derivable from stderr); declined, failed, or
|
|
804
|
+
// unparseable markers fall through to the manual runbook.
|
|
805
|
+
let collisionImportError = null;
|
|
806
|
+
if (options._collisionImported !== true) {
|
|
807
|
+
const pairs = parseCollisionImports(errString)
|
|
808
|
+
.filter((pair) => !pair.address.startsWith('aws_secretsmanager_secret.'));
|
|
809
|
+
if (pairs.length > 0) {
|
|
810
|
+
const refused = await refuseIfNonInteractive(options, {
|
|
811
|
+
message: '\nā apply needs approval to import collided resources into state.',
|
|
812
|
+
hint: ' Re-run with --auto-approve to import non-interactively.\n',
|
|
813
|
+
event: 'infrastructure_applied',
|
|
814
|
+
telemetry: { projectName: actualProjectName },
|
|
815
|
+
errorCode: 'COLLISION_IMPORT_NON_INTERACTIVE',
|
|
816
|
+
});
|
|
817
|
+
if (refused) return refused;
|
|
818
|
+
let proceed = shouldAutoApprove(options);
|
|
819
|
+
if (!proceed) {
|
|
820
|
+
const confirmed = await confirm({
|
|
821
|
+
message: `Import ${pairs.length} collided resource${pairs.length === 1 ? '' : 's'} into state and retry?`,
|
|
822
|
+
initialValue: true,
|
|
823
|
+
});
|
|
824
|
+
proceed = !isCancel(confirmed) && confirmed === true;
|
|
825
|
+
}
|
|
826
|
+
if (proceed) {
|
|
827
|
+
try {
|
|
828
|
+
for (const pair of pairs) {
|
|
829
|
+
await runTerraform(['import', pair.address, pair.id], tfDir, null, `Importing ${pair.address}`);
|
|
830
|
+
}
|
|
831
|
+
} catch (error) {
|
|
832
|
+
collisionImportError = error;
|
|
833
|
+
}
|
|
834
|
+
if (collisionImportError === null) {
|
|
835
|
+
trackEvent('collision_resources_imported', { projectName: actualProjectName, count: pairs.length });
|
|
836
|
+
log.message(color.cyan('Imported collided resources ā retrying apply...'));
|
|
837
|
+
return applyStack({ ...options, _skipPreview: true, _collisionImported: true });
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
}
|
|
841
|
+
}
|
|
800
842
|
trackEvent('bucket_recovery_probed', { projectName: actualProjectName, collisions });
|
|
801
843
|
await flushTelemetry();
|
|
802
844
|
const runbook = getRunbook('RECREATE_COLLISION', { headless: shouldAutoApprove(options) });
|
|
803
845
|
return failCommand({
|
|
804
846
|
message: `\nā ${runbook.message}`,
|
|
805
|
-
hint: ` ${runbook.hint}\n Import each address it names, e.g.:\n ${color.green('cd terraform && terraform import <address> <aws-id>')}\n`,
|
|
847
|
+
hint: ` ${runbook.hint}\n Import each address it names, e.g.:\n ${color.green('cd terraform && terraform import <address> <aws-id>')}\n${collisionImportError ? color.dim(` (Automatic import failed: ${collisionImportError.message})`) + '\n' : ''}`,
|
|
806
848
|
event: 'infrastructure_applied',
|
|
807
849
|
telemetry: { projectName: actualProjectName, error_code: 'RECREATE_COLLISION', ...costProps },
|
|
808
850
|
errorCode: 'RECREATE_COLLISION',
|
|
@@ -862,14 +904,51 @@ async function applyStackMain(input = {}) {
|
|
|
862
904
|
if (adoptUpsertError) {
|
|
863
905
|
log.message(color.dim(` (Automatic adoption failed: ${adoptUpsertError.message})`));
|
|
864
906
|
}
|
|
865
|
-
} else if (code === 'BACKEND_RECONFIGURE_REQUIRED' || code === 'CLOUDFRONT_IN_PROGRESS' || code === 'DELETION_PROTECTION_ACTIVE') {
|
|
907
|
+
} else if (code === 'BACKEND_RECONFIGURE_REQUIRED' || code === 'CLOUDFRONT_IN_PROGRESS' || code === 'DELETION_PROTECTION_ACTIVE' || code === 'SECRET_PENDING_DELETION') {
|
|
866
908
|
// Named but flow-neutral: runbook guidance with a coded
|
|
867
909
|
// failure. (RECREATE_COLLISION has its dedicated branch
|
|
868
910
|
// above; deletion protection's destroy flow lives in
|
|
869
911
|
// destroy.js.)
|
|
912
|
+
let reconfigureError = null;
|
|
913
|
+
if (code === 'BACKEND_RECONFIGURE_REQUIRED' && options._backendReconfigured !== true) {
|
|
914
|
+
// Offer the reconfigure in-flow instead of shelling the
|
|
915
|
+
// user out to raw terraform; declined or failed runs
|
|
916
|
+
// fall through to the runbook below.
|
|
917
|
+
const refused = await refuseIfNonInteractive(options, {
|
|
918
|
+
message: '\nā apply needs approval to reconfigure the Terraform backend.',
|
|
919
|
+
hint: ' Re-run with --auto-approve to reconfigure non-interactively.\n',
|
|
920
|
+
event: 'infrastructure_applied',
|
|
921
|
+
telemetry: { projectName: actualProjectName },
|
|
922
|
+
errorCode: 'BACKEND_RECONFIGURE_NON_INTERACTIVE',
|
|
923
|
+
});
|
|
924
|
+
if (refused) return refused;
|
|
925
|
+
let proceed = shouldAutoApprove(options);
|
|
926
|
+
if (!proceed) {
|
|
927
|
+
const confirmed = await confirm({
|
|
928
|
+
message: 'Reconfigure the Terraform backend (init -reconfigure) and retry?',
|
|
929
|
+
initialValue: true,
|
|
930
|
+
});
|
|
931
|
+
proceed = !isCancel(confirmed) && confirmed === true;
|
|
932
|
+
}
|
|
933
|
+
if (proceed) {
|
|
934
|
+
try {
|
|
935
|
+
await runTerraform(['init', '-reconfigure'], tfDir, null, 'Reconfiguring backend');
|
|
936
|
+
} catch (error) {
|
|
937
|
+
reconfigureError = error;
|
|
938
|
+
}
|
|
939
|
+
if (reconfigureError === null) {
|
|
940
|
+
trackEvent('backend_reconfigured', { projectName: actualProjectName });
|
|
941
|
+
log.message(color.cyan('Reconfigured the backend ā retrying apply...'));
|
|
942
|
+
return applyStack({ ...options, _skipPreview: true, _backendReconfigured: true });
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
}
|
|
870
946
|
const runbook = getRunbook(code, { headless: shouldAutoApprove(options), lockId });
|
|
871
947
|
log.error(color.red(`ā ${runbook.message}`));
|
|
872
948
|
log.message(` ${runbook.hint}\n`);
|
|
949
|
+
if (reconfigureError) {
|
|
950
|
+
log.message(color.dim(` (Automatic reconfigure failed: ${reconfigureError.message})`));
|
|
951
|
+
}
|
|
873
952
|
log.message(color.dim(errString.split('\n').slice(0, 8).join('\n')));
|
|
874
953
|
} else {
|
|
875
954
|
// Generic fallback for unknown errors
|
|
@@ -33,7 +33,7 @@ function snapshotSourceDb(snapshot, isCluster) {
|
|
|
33
33
|
return isCluster ? snapshot.DBClusterIdentifier : snapshot.DBInstanceIdentifier;
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
//
|
|
36
|
+
// The reused id names another database's snapshot.
|
|
37
37
|
// Shared by the wait and no-wait paths (they differ only in `waited`).
|
|
38
38
|
function failSnapshotMismatch({ spinnerHandle, snapshotId, foreignDb, targetId, projectName, waited, dbIdentifier, region }) {
|
|
39
39
|
spinnerHandle.stop(color.red('ā Snapshot belongs to another database.'));
|
|
@@ -202,7 +202,7 @@ async function runDbBackupMain(input = {}) {
|
|
|
202
202
|
|
|
203
203
|
if (noWait) {
|
|
204
204
|
if (reusedExisting) {
|
|
205
|
-
//
|
|
205
|
+
// Verify ownership even when not waiting ā one
|
|
206
206
|
// describe, fail closed on a foreign snapshot. A
|
|
207
207
|
// not-found here is transient (eventual consistency),
|
|
208
208
|
// so only proceed past it.
|
|
@@ -268,7 +268,7 @@ async function runDbBackupMain(input = {}) {
|
|
|
268
268
|
throw error;
|
|
269
269
|
}
|
|
270
270
|
const snapshot = snapshots[0] || null;
|
|
271
|
-
//
|
|
271
|
+
// An explicit --id can name another database's
|
|
272
272
|
// snapshot (AlreadyExists proves existence, not
|
|
273
273
|
// ownership) ā fail closed instead of adopting a
|
|
274
274
|
// foreign snapshot as our safety backup. Checked
|
|
@@ -272,7 +272,7 @@ async function runDbConnectMain(input = {}) {
|
|
|
272
272
|
});
|
|
273
273
|
|
|
274
274
|
// Node reports signal death as code null ā a killed tunnel is a
|
|
275
|
-
// failure naming the signal, never `?? 0` success
|
|
275
|
+
// failure naming the signal, never `?? 0` success.
|
|
276
276
|
const { code, signal } = await new Promise((resolve) => {
|
|
277
277
|
const child = spawnImpl('aws', ssmArgs, { stdio: 'inherit' });
|
|
278
278
|
child.on('error', (err) => {
|
|
@@ -288,7 +288,7 @@ async function runDbConnectMain(input = {}) {
|
|
|
288
288
|
// Telemetry carries only non-sensitive fields. Credentials never leave this process
|
|
289
289
|
// except to the terminal above and the local SSM session below.
|
|
290
290
|
// Success is recorded only for a clean tunnel exit: a failed tunnel
|
|
291
|
-
// reports the tunnel's own exit code instead of ok:true
|
|
291
|
+
// reports the tunnel's own exit code instead of ok:true.
|
|
292
292
|
if (code !== 0) {
|
|
293
293
|
const tunnelCode = Number.isInteger(code) && code > 0 ? Math.min(code, 255) : 1;
|
|
294
294
|
return failCommand({
|
|
@@ -322,7 +322,7 @@ async function runDbEnableVectorMain(input = {}) {
|
|
|
322
322
|
const exitCode = outcome.exitCode;
|
|
323
323
|
const stoppedReason = outcome.stoppedReason;
|
|
324
324
|
if (exitCode === 0) {
|
|
325
|
-
// Owner pairs its own start/stop
|
|
325
|
+
// Owner pairs its own start/stop: a callee must never stop its caller's spinner.
|
|
326
326
|
try { s.stop(color.green('pgvector extension enabled.')); } catch { /* already stopped */ }
|
|
327
327
|
console.log(color.green('\nā
pgvector extension enabled.'));
|
|
328
328
|
try {
|
|
@@ -159,7 +159,7 @@ function targetEnv(targetScheme, password) {
|
|
|
159
159
|
}
|
|
160
160
|
|
|
161
161
|
// Source-side dump env: the SOURCE is arbitrary user infra (often no
|
|
162
|
-
// TLS), so prefer TLS instead of requiring it
|
|
162
|
+
// TLS), so prefer TLS instead of requiring it. MySQL sources
|
|
163
163
|
// were never forced; the target side keeps `require`.
|
|
164
164
|
function sourceEnv(scheme, password) {
|
|
165
165
|
if (scheme === 'mysql') {
|
|
@@ -172,7 +172,7 @@ function waitForExit(child) {
|
|
|
172
172
|
return new Promise((resolve) => {
|
|
173
173
|
child.once('error', (error) => resolve({ code: 1, error }));
|
|
174
174
|
// Node reports signal death as code null ā resolve it as failure
|
|
175
|
-
// (with the signal), never `?? 0` success
|
|
175
|
+
// (with the signal), never `?? 0` success.
|
|
176
176
|
child.once('close', (code, signal) => resolve(signal ? { code: 1, signal } : { code: code ?? 0 }));
|
|
177
177
|
});
|
|
178
178
|
}
|
|
@@ -101,7 +101,7 @@ function renderGateBlock({ includeSetupNode, steps }) {
|
|
|
101
101
|
lines.push(
|
|
102
102
|
` - name: ${stepName}`,
|
|
103
103
|
// Gates run AWS calls: honor the credential-skip outcome like
|
|
104
|
-
// every other post-assume step
|
|
104
|
+
// every other post-assume step. Neutral on deploy
|
|
105
105
|
// (a failed assume fails the run before any gate runs).
|
|
106
106
|
` if: steps.aws-creds.outcome == 'success'`,
|
|
107
107
|
' run: |',
|
|
@@ -147,7 +147,7 @@ function injectGateBlock(content, { steps, anchor }) {
|
|
|
147
147
|
// Idempotently injects (or refreshes) the migration gate block into a
|
|
148
148
|
// deploy.yml document. Returns the updated content, or null when neither
|
|
149
149
|
// anchor step is present. Pure and unit-tested. The snapshot id embeds
|
|
150
|
-
// the database identifier
|
|
150
|
+
// the database identifier: bare pre-deploy-<sha> ids match no
|
|
151
151
|
// guardrail or gc name pattern, so they were never protected or retired.
|
|
152
152
|
export function injectMigrationGate(workflowContent, { cmd, cliVersion, dbIdentifier } = {}) {
|
|
153
153
|
if (typeof dbIdentifier !== 'string' || dbIdentifier.trim() === '') {
|
|
@@ -297,7 +297,7 @@ async function runDbMigrateMain(input = {}) {
|
|
|
297
297
|
|
|
298
298
|
// 2. --setup-ci: patch the workflow file, no AWS calls.
|
|
299
299
|
if (setupCi) {
|
|
300
|
-
//
|
|
300
|
+
// The gate runs migrations as an ephemeral ECS task ā
|
|
301
301
|
// refuse on lambda/static (no ECS to run on). Init auto-skips
|
|
302
302
|
// with a warning; this explicit command fails closed.
|
|
303
303
|
const setupTarget = readTerraformComputeTarget(cwd);
|
|
@@ -328,12 +328,13 @@ async function runDbMigrateMain(input = {}) {
|
|
|
328
328
|
resultExtra: { cluster, service, region },
|
|
329
329
|
});
|
|
330
330
|
}
|
|
331
|
-
//
|
|
331
|
+
// The gate always targets production (deploy.yml), so
|
|
332
332
|
// resolve the prod db id even with a preview workspace checked
|
|
333
333
|
// out locally. Aurora needs the cluster prefix: cluster
|
|
334
334
|
// snapshots named with the instance prefix would miss the
|
|
335
335
|
// cluster-snapshot guardrail pattern.
|
|
336
|
-
const
|
|
336
|
+
const setupDbKind = readTerraformDbKind(cwd);
|
|
337
|
+
const setupDbResolver = setupDbKind === 'cluster' ? resolveDbClusterIdentifier : resolveDbIdentifier;
|
|
337
338
|
const setupDbIdentifier = setupDbResolver({ projectName, workspace: 'default' }, cwd);
|
|
338
339
|
const updated = injectMigrationGate(workflowContent, { cmd: resolvedCmd, cliVersion: getCliVersion(), dbIdentifier: setupDbIdentifier });
|
|
339
340
|
if (updated === null) {
|
|
@@ -367,7 +368,7 @@ async function runDbMigrateMain(input = {}) {
|
|
|
367
368
|
}
|
|
368
369
|
}
|
|
369
370
|
}
|
|
370
|
-
await trackSuccess('db_migrate_run', { projectName, cmd_source: cmdSource, ci_setup: true });
|
|
371
|
+
await trackSuccess('db_migrate_run', { projectName, cmd_source: cmdSource, ci_setup: true, db_kind: setupDbKind });
|
|
371
372
|
outro(color.green('Setup complete.'));
|
|
372
373
|
return { ok: true, ciSetup: true, workflowFile, cmdSource };
|
|
373
374
|
}
|
package/src/commands/destroy.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import fsSync from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
import { intro, outro, confirm, spinner, cancel } from '@clack/prompts';
|
|
3
|
+
import { intro, outro, confirm, spinner, cancel, isCancel } from '@clack/prompts';
|
|
4
4
|
import color from 'picocolors';
|
|
5
5
|
import { RDSClient, DescribeDBInstancesCommand, DescribeDBClustersCommand, DescribeDBSnapshotsCommand, DescribeDBClusterSnapshotsCommand, StartDBInstanceCommand, StartDBClusterCommand, CreateDBSnapshotCommand, CreateDBClusterSnapshotCommand, ModifyDBInstanceCommand, ModifyDBClusterCommand } from '@aws-sdk/client-rds';
|
|
6
6
|
import { S3Client, ListObjectVersionsCommand } from '@aws-sdk/client-s3';
|
|
7
|
+
import { EC2Client, DescribeVpcsCommand, DescribeNetworkInterfacesCommand, DescribeSecurityGroupsCommand, DeleteSecurityGroupCommand } from '@aws-sdk/client-ec2';
|
|
7
8
|
import { teardownStateBucket, drainBucketObjects, resolveClient } from '../utils/aws.js';
|
|
8
9
|
import { checkDependency, pollUntil, sleep } from '../utils/system.js';
|
|
9
10
|
import { trackEvent, flushTelemetry, trackSuccess, setActiveCommandName, resetActiveCommandName } from '../core/telemetry.js';
|
|
@@ -14,6 +15,7 @@ import { findDbTarget, dbStatusOf, resolveDbIdentifier, resolveDbClusterIdentifi
|
|
|
14
15
|
import { requireManagedTree, parseStateBucketBackend } from '../utils/terraform-metadata.js';
|
|
15
16
|
import { guardSleepingEnv } from '../utils/sleep-state.js';
|
|
16
17
|
import { normalizeOptions, parseFlags, normalizeArgv } from '../utils/args.js';
|
|
18
|
+
import { resolveHeadless } from '../utils/resolvers.js';
|
|
17
19
|
import { truncateSpinnerText } from '../utils/spinner-text.js';
|
|
18
20
|
|
|
19
21
|
export const DEFAULT_DESTROY_DB_TIMEOUT_MS = 600000;
|
|
@@ -57,7 +59,37 @@ export function parseDestroyArgs(argv = []) {
|
|
|
57
59
|
// Returns `{ retried: true }` when the destroy recovered so the caller
|
|
58
60
|
// continues to bucket cleanup, else `{ retried: false, result }`.
|
|
59
61
|
// Exported for unit tests: every branch must honor this contract.
|
|
60
|
-
|
|
62
|
+
async function describeDependencyBlockers(ec2, projectName) {
|
|
63
|
+
const { Vpcs = [] } = await ec2.send(new DescribeVpcsCommand({
|
|
64
|
+
Filters: [
|
|
65
|
+
{ Name: 'tag:ManagedBy', Values: ['grada'] },
|
|
66
|
+
{ Name: 'tag:GradaProject', Values: [projectName] },
|
|
67
|
+
],
|
|
68
|
+
}));
|
|
69
|
+
if (Vpcs.length !== 1 || !Vpcs[0].VpcId) return null;
|
|
70
|
+
const vpcId = Vpcs[0].VpcId;
|
|
71
|
+
const [niResp, sgResp] = await Promise.all([
|
|
72
|
+
ec2.send(new DescribeNetworkInterfacesCommand({ Filters: [{ Name: 'vpc-id', Values: [vpcId] }] })),
|
|
73
|
+
ec2.send(new DescribeSecurityGroupsCommand({ Filters: [{ Name: 'vpc-id', Values: [vpcId] }] })),
|
|
74
|
+
]);
|
|
75
|
+
const owned = (tags = []) => tags.some((tag) => tag.Key === 'ManagedBy' && tag.Value === 'grada')
|
|
76
|
+
&& tags.some((tag) => tag.Key === 'GradaProject' && tag.Value === projectName);
|
|
77
|
+
return {
|
|
78
|
+
vpcId,
|
|
79
|
+
enis: (niResp.NetworkInterfaces || []).map((ni) => ({
|
|
80
|
+
id: ni.NetworkInterfaceId,
|
|
81
|
+
detail: ni.Attachment?.InstanceId || ni.Description || ni.InterfaceType || 'attached',
|
|
82
|
+
owned: owned(ni.TagSet),
|
|
83
|
+
})),
|
|
84
|
+
sgs: (sgResp.SecurityGroups || []).map((sg) => ({
|
|
85
|
+
id: sg.GroupId,
|
|
86
|
+
detail: sg.GroupName || '',
|
|
87
|
+
owned: owned(sg.Tags),
|
|
88
|
+
})),
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export async function handleDestroyTerraformError({ options, tfDirPath, attemptDestroy, error, region }) {
|
|
61
93
|
const actualProjectName = path.basename(process.cwd());
|
|
62
94
|
const errString = String(error.message || error);
|
|
63
95
|
const structuredCode = typeof error.code === 'string' && ERROR_CODES.includes(error.code) ? error.code : null;
|
|
@@ -123,7 +155,7 @@ export async function handleDestroyTerraformError({ options, tfDirPath, attemptD
|
|
|
123
155
|
return namedFail('STATE_LOCK_HELD');
|
|
124
156
|
}
|
|
125
157
|
|
|
126
|
-
//
|
|
158
|
+
// A drained bucket can still report versions when terraform
|
|
127
159
|
// runs (S3 empty-visibility race). Settle briefly, then retry the
|
|
128
160
|
// destroy exactly once ā a second BucketNotEmpty is terminal.
|
|
129
161
|
if (code === 'BUCKET_NOT_EMPTY') {
|
|
@@ -138,6 +170,68 @@ export async function handleDestroyTerraformError({ options, tfDirPath, attemptD
|
|
|
138
170
|
return { retried: true };
|
|
139
171
|
}
|
|
140
172
|
|
|
173
|
+
if (code === 'DESTROY_DEPENDENCY_VIOLATION') {
|
|
174
|
+
// Enumerate the blockers (read-only, always safe) so the failure
|
|
175
|
+
// names attachments instead of gesturing at the console.
|
|
176
|
+
// Grada-owned security groups can go in-flow; ENIs and foreign
|
|
177
|
+
// attachments are report-only ā detaching live interfaces is
|
|
178
|
+
// never automatic. Any discovery failure falls through to the
|
|
179
|
+
// plain runbook below.
|
|
180
|
+
let blockers = null;
|
|
181
|
+
let ec2 = null;
|
|
182
|
+
try {
|
|
183
|
+
ec2 = resolveClient(options.ec2Client, EC2Client, { region: region ?? process.env.AWS_REGION ?? 'us-east-2' });
|
|
184
|
+
blockers = await describeDependencyBlockers(ec2, actualProjectName);
|
|
185
|
+
} catch {
|
|
186
|
+
blockers = null;
|
|
187
|
+
}
|
|
188
|
+
if (blockers !== null) {
|
|
189
|
+
const ownedSgs = blockers.sgs.filter((sg) => sg.owned);
|
|
190
|
+
if (ownedSgs.length > 0) {
|
|
191
|
+
let proceed = shouldAutoApprove(options);
|
|
192
|
+
if (!proceed && !resolveHeadless(normalizeOptions(options))) {
|
|
193
|
+
const confirmed = await confirm({
|
|
194
|
+
message: `Delete ${ownedSgs.length} grada-owned security group${ownedSgs.length === 1 ? '' : 's'} blocking VPC deletion and retry?`,
|
|
195
|
+
initialValue: true,
|
|
196
|
+
});
|
|
197
|
+
proceed = !isCancel(confirmed) && confirmed === true;
|
|
198
|
+
}
|
|
199
|
+
if (proceed) {
|
|
200
|
+
try {
|
|
201
|
+
for (const sg of ownedSgs) {
|
|
202
|
+
await ec2.send(new DeleteSecurityGroupCommand({ GroupId: sg.id }));
|
|
203
|
+
}
|
|
204
|
+
} catch {
|
|
205
|
+
return namedFail('DESTROY_DEPENDENCY_VIOLATION');
|
|
206
|
+
}
|
|
207
|
+
trackEvent('dependency_blockers_removed', { projectName: actualProjectName, count: ownedSgs.length });
|
|
208
|
+
try {
|
|
209
|
+
await attemptDestroy();
|
|
210
|
+
} catch {
|
|
211
|
+
return namedFail('DESTROY_DEPENDENCY_VIOLATION');
|
|
212
|
+
}
|
|
213
|
+
return { retried: true };
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const lines = [
|
|
217
|
+
...blockers.enis.map((ni) => ` ENI ${ni.id} (${ni.detail})${ni.owned ? '' : ' ā foreign, remove by hand'}`),
|
|
218
|
+
...blockers.sgs.map((sg) => ` SG ${sg.id} (${sg.detail})${sg.owned ? '' : ' ā foreign, remove by hand'}`),
|
|
219
|
+
];
|
|
220
|
+
const runbook = getRunbook('DESTROY_DEPENDENCY_VIOLATION', { headless: shouldAutoApprove(options), lockId });
|
|
221
|
+
return {
|
|
222
|
+
retried: false,
|
|
223
|
+
result: failCommand({
|
|
224
|
+
message: `\nā VPC deletion blocked by ${blockers.enis.length + blockers.sgs.length} attachment(s) in ${blockers.vpcId}:\n${lines.join('\n')}\n ${runbook.hint}\n`,
|
|
225
|
+
useErrorStream: true,
|
|
226
|
+
event: 'infrastructure_destroyed',
|
|
227
|
+
telemetry: { projectName: actualProjectName, error_code: 'DESTROY_DEPENDENCY_VIOLATION' },
|
|
228
|
+
errorCode: 'DESTROY_DEPENDENCY_VIOLATION',
|
|
229
|
+
reason: 'destroy-dependency-violation',
|
|
230
|
+
}),
|
|
231
|
+
};
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
141
235
|
if (code === 'BACKEND_RECONFIGURE_REQUIRED' || code === 'DESTROY_DEPENDENCY_VIOLATION'
|
|
142
236
|
|| code === 'CLOUDFRONT_IN_PROGRESS' || code === 'DELETION_PROTECTION_ACTIVE') {
|
|
143
237
|
return namedFail(code);
|
|
@@ -280,6 +374,18 @@ async function destroyStackMain(input = {}) {
|
|
|
280
374
|
if (refused) return refused;
|
|
281
375
|
}
|
|
282
376
|
|
|
377
|
+
// Secrets enter the 30-day recovery window on destroy: re-applying
|
|
378
|
+
// inside the window fails until the secret is restored or
|
|
379
|
+
// force-deleted (see the apply runbook). Warn in both modes ā
|
|
380
|
+
// informational only, the confirm below still gates.
|
|
381
|
+
try {
|
|
382
|
+
if (fsSync.existsSync(path.join(tfDirPath, 'secrets.tf'))) {
|
|
383
|
+
console.warn(color.yellow('ā Vault secrets enter a 30-day recovery window on destroy ā re-applying within 30 days needs a restore or force-delete first.'));
|
|
384
|
+
}
|
|
385
|
+
} catch {
|
|
386
|
+
// Unreadable tree: the confirm below still gates.
|
|
387
|
+
}
|
|
388
|
+
|
|
283
389
|
const proceed = shouldAutoApprove(options) ? true : await confirm({
|
|
284
390
|
message: dbNeedsProtection
|
|
285
391
|
? color.red(`ā ļø WARNING: This will permanently destroy all AWS resources including database ${dbTarget.id} (a pre-destroy snapshot will be taken first). Are you absolutely sure?`)
|
|
@@ -308,7 +414,7 @@ async function destroyStackMain(input = {}) {
|
|
|
308
414
|
try {
|
|
309
415
|
await runInit();
|
|
310
416
|
} catch (error) {
|
|
311
|
-
const handled = await handleDestroyTerraformError({ options, tfDirPath, attemptDestroy: runInit, error });
|
|
417
|
+
const handled = await handleDestroyTerraformError({ options, tfDirPath, attemptDestroy: runInit, error, region });
|
|
312
418
|
if (!handled.retried) return handled.result;
|
|
313
419
|
}
|
|
314
420
|
|
|
@@ -447,11 +553,11 @@ async function destroyStackMain(input = {}) {
|
|
|
447
553
|
}
|
|
448
554
|
// Wait for the snapshot to reach `available` before touching the
|
|
449
555
|
// database: deleting while it is still `creating` races snapshot
|
|
450
|
-
// completion on large DBs
|
|
556
|
+
// completion on large DBs. A failed snapshot or a timeout
|
|
451
557
|
// refuses the destroy ā re-run with --allow-data-loss to skip.
|
|
452
558
|
// Re-message (never re-start): the phase spinner is already
|
|
453
559
|
// running, and a second start leaks clack's first interval so the
|
|
454
|
-
// session never exits
|
|
560
|
+
// session never exits.
|
|
455
561
|
s.message(truncateSpinnerText(`Waiting for snapshot ${snapshotId} to complete...`));
|
|
456
562
|
const snapshotReady = await pollUntil({
|
|
457
563
|
intervalMs: pollIntervalMs,
|
|
@@ -477,7 +583,7 @@ async function destroyStackMain(input = {}) {
|
|
|
477
583
|
if (status === 'failed' || status === 'deleted') {
|
|
478
584
|
return { done: true, value: { failed: true, reason: `snapshot ${status}` } };
|
|
479
585
|
}
|
|
480
|
-
//
|
|
586
|
+
// Surface PercentProgress while creating so long
|
|
481
587
|
// snapshots don't look wedged.
|
|
482
588
|
const detail = typeof percent === 'number' ? `${status || 'pending'} (${percent}% complete)` : (status || 'pending');
|
|
483
589
|
s.message(truncateSpinnerText(`Waiting for snapshot ${snapshotId} to complete... (${detail})`));
|
|
@@ -613,7 +719,7 @@ async function destroyStackMain(input = {}) {
|
|
|
613
719
|
resultExtra: { region },
|
|
614
720
|
});
|
|
615
721
|
}
|
|
616
|
-
//
|
|
722
|
+
// Versioned deletes can lag S3's empty-reads, so
|
|
617
723
|
// verify each drained bucket reads empty before terraform
|
|
618
724
|
// destroy runs ā otherwise destroy flakes on BucketNotEmpty.
|
|
619
725
|
// A bucket deleted out-of-band counts as empty.
|
|
@@ -672,7 +778,7 @@ async function destroyStackMain(input = {}) {
|
|
|
672
778
|
s.stop('AWS compute resources destroyed.');
|
|
673
779
|
} catch (error) {
|
|
674
780
|
s.stop(color.red('ā Terraform destroy failed.'));
|
|
675
|
-
const handled = await handleDestroyTerraformError({ options, tfDirPath, attemptDestroy, error });
|
|
781
|
+
const handled = await handleDestroyTerraformError({ options, tfDirPath, attemptDestroy, error, region });
|
|
676
782
|
if (!handled.retried) return handled.result;
|
|
677
783
|
}
|
|
678
784
|
|
|
@@ -690,7 +796,7 @@ async function destroyStackMain(input = {}) {
|
|
|
690
796
|
});
|
|
691
797
|
if (refused) return refused;
|
|
692
798
|
}
|
|
693
|
-
//
|
|
799
|
+
// "spin this back up" overstated recovery ā apply
|
|
694
800
|
// recreates compute, never the destroyed database. Name the
|
|
695
801
|
// snapshot (or its absence) so the choice is informed.
|
|
696
802
|
const dbRecoveryNote = snapshotReadyId
|
package/src/commands/diagnose.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawnSync } from 'child_process';
|
|
2
2
|
import { ECSClient, ListTasksCommand, DescribeTasksCommand } from '@aws-sdk/client-ecs';
|
|
3
|
-
import { CloudWatchLogsClient } from '@aws-sdk/client-cloudwatch-logs';
|
|
3
|
+
import { CloudWatchLogsClient, DescribeLogGroupsCommand } from '@aws-sdk/client-cloudwatch-logs';
|
|
4
4
|
import color from 'picocolors';
|
|
5
5
|
import { intro, outro, spinner } from '@clack/prompts';
|
|
6
6
|
import { trackEvent, flushTelemetry, trackSuccess, trackFailure, setActiveCommandName, resetActiveCommandName } from '../core/telemetry.js';
|
|
@@ -86,7 +86,7 @@ function isLogGroupNotFoundError(error) {
|
|
|
86
86
|
// Shared log tail for the ECS and Lambda paths: fetched lines, the honest
|
|
87
87
|
// fetch-failure warning (never "no events" when the fetch failed), the
|
|
88
88
|
// missing-group hint, or the genuine empty line ā in that order.
|
|
89
|
-
function printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix, region }) {
|
|
89
|
+
async function printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix, region, logsClient }) {
|
|
90
90
|
if (logs.length > 0) {
|
|
91
91
|
console.log(`\n ${color.bold(`Last ${logs.length} log lines (${color.cyan(logGroup)}):`)}`);
|
|
92
92
|
for (const line of logs) {
|
|
@@ -96,6 +96,19 @@ function printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchEr
|
|
|
96
96
|
console.log(color.yellow(`\nā Could not fetch recent logs from ${logGroup} (${fetchError}). Log health is unverified.`));
|
|
97
97
|
} else if (logGroupMissing) {
|
|
98
98
|
console.log(color.yellow(`\nā Log group not found (expected ${logGroup}).`));
|
|
99
|
+
try {
|
|
100
|
+
const resp = await logsClient.send(new DescribeLogGroupsCommand({ logGroupNamePrefix: listPrefix, limit: 10 }));
|
|
101
|
+
const names = (resp?.logGroups || []).map((group) => group.logGroupName).filter(Boolean).slice(0, 10);
|
|
102
|
+
if (names.length > 0) {
|
|
103
|
+
console.log(color.dim('Matching groups:'));
|
|
104
|
+
for (const name of names) console.log(color.dim(` ${name}`));
|
|
105
|
+
} else {
|
|
106
|
+
console.log(color.dim(`No log groups match "${listPrefix}".`));
|
|
107
|
+
}
|
|
108
|
+
return;
|
|
109
|
+
} catch {
|
|
110
|
+
// Listing failed: fall back to the CLI pointer below.
|
|
111
|
+
}
|
|
99
112
|
console.log(color.dim(`List matching groups with: aws logs describe-log-groups --log-group-name-prefix "${listPrefix}" --region ${region}`));
|
|
100
113
|
} else {
|
|
101
114
|
console.log(color.dim(`\n No recent log events found in ${logGroup}.`));
|
|
@@ -215,7 +228,7 @@ export async function runLambdaDiagnose(input = {}) {
|
|
|
215
228
|
console.log(` ${color.dim('State:')} ${state === 'Active' ? color.green(state) : color.yellow(state)} ${color.dim(`/ ${lastUpdateStatus}`)}`);
|
|
216
229
|
if (configuration.LastModified) console.log(` ${color.dim('Modified:')} ${configuration.LastModified}`);
|
|
217
230
|
|
|
218
|
-
printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix: '/aws/lambda/', region });
|
|
231
|
+
await printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix: '/aws/lambda/', region, logsClient });
|
|
219
232
|
|
|
220
233
|
if (healthy) {
|
|
221
234
|
outro(color.green('Diagnose complete. Nothing to fix!'));
|
|
@@ -262,7 +275,7 @@ async function runDiagnoseMain(input = {}) {
|
|
|
262
275
|
cwd = resolveCwd(options);
|
|
263
276
|
projectName = resolveProjectName(options, cwd);
|
|
264
277
|
region = resolveRegion(options, cwd);
|
|
265
|
-
// Workspace-namespaced resources
|
|
278
|
+
// Workspace-namespaced resources: --workspace (or the
|
|
266
279
|
// local .terraform/environment) targets a preview stack like
|
|
267
280
|
// rollback and db do. Explicit flags/env keep winning; telemetry
|
|
268
281
|
// keeps the raw project name.
|
|
@@ -489,7 +502,7 @@ async function runDiagnoseMain(input = {}) {
|
|
|
489
502
|
console.log(` ${color.dim('Container reason:')} ${color.yellow(containerReason)}`);
|
|
490
503
|
}
|
|
491
504
|
|
|
492
|
-
printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix: '/ecs/', region });
|
|
505
|
+
await printLogTail({ logGroup, logs, logGroupMissing, logFetchFailed, fetchError, listPrefix: '/ecs/', region, logsClient });
|
|
493
506
|
|
|
494
507
|
outro(color.green('Diagnose complete. Fix the error above, then redeploy. š'));
|
|
495
508
|
|