grada-run 0.49.11 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/alerts.js +275 -57
- package/src/commands/db/migrate.js +1 -1
- package/src/commands/status.js +84 -6
- package/src/data/action-pins.js +1 -1
- package/src/utils/golden-signals.js +4 -4
- package/src/utils/resolvers.js +16 -1
- package/templates/github/deploy-lambda.yml +1 -1
- package/templates/github/deploy-static.yml +1 -1
- package/templates/github/deploy.yml +1 -1
- package/templates/github/drift.yml +68 -0
- package/templates/github/preview.yml +69 -0
- package/templates/terraform/addons/bedrock.tf +3 -3
- package/templates/terraform/addons/cron-lambda.tf +21 -2
- package/templates/terraform/addons/cron.tf +21 -2
- package/templates/terraform/addons/s3.tf +6 -0
- package/templates/terraform/addons/ses.tf +36 -0
- package/templates/terraform/database-aurora-postgresql.tf +5 -0
- package/templates/terraform/database-mysql.tf +18 -0
- package/templates/terraform/database.tf +18 -0
- package/templates/terraform/main-lambda.tf +8 -2
- package/templates/terraform/main.tf +14 -4
- package/templates/terraform/oidc.tf +94 -1
package/package.json
CHANGED
package/src/commands/alerts.js
CHANGED
|
@@ -52,6 +52,30 @@ const TARGET_ANCHORS = {
|
|
|
52
52
|
static: { resource: 'resource "aws_cloudfront_distribution" "site"', errorCode: 'DISTRIBUTION_NOT_FOUND', reason: 'distribution-not-found' },
|
|
53
53
|
};
|
|
54
54
|
|
|
55
|
+
// Addon alarm anchors. Detection reads the addon FILE CONTENT (not just
|
|
56
|
+
// its presence) so a hand-removed resource can't produce a dangling
|
|
57
|
+
// alarm reference. Several anchors may share one id (either engine
|
|
58
|
+
// file lights up 'db:instance').
|
|
59
|
+
const ADDON_ALARM_ANCHORS = [
|
|
60
|
+
{ id: 'queue:sqs', file: 'sqs.tf', resource: 'resource "aws_sqs_queue" "dlq"' },
|
|
61
|
+
{ id: 'cron', file: 'cron.tf', resource: 'resource "aws_scheduler_schedule" "cron"' },
|
|
62
|
+
{ id: 'db:instance', file: 'database.tf', resource: 'resource "aws_db_instance" "postgres"' },
|
|
63
|
+
{ id: 'db:instance', file: 'database-mysql.tf', resource: 'resource "aws_db_instance" "postgres"' },
|
|
64
|
+
{ id: 'db:aurora', file: 'database-aurora-postgresql.tf', resource: 'resource "aws_rds_cluster" "postgres"' },
|
|
65
|
+
{ id: 'db:redis', file: 'redis.tf', resource: 'resource "aws_elasticache_replication_group" "redis"' },
|
|
66
|
+
{ id: 'email:ses', file: 'ses.tf', resource: 'resource "aws_ses_domain_identity" "ses"' },
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
export function detectAlertAddonFeatures(terraformDir) {
|
|
70
|
+
const features = {};
|
|
71
|
+
for (const anchor of ADDON_ALARM_ANCHORS) {
|
|
72
|
+
if (features[anchor.id]) continue;
|
|
73
|
+
const content = readFileSafe(path.join(terraformDir, anchor.file));
|
|
74
|
+
features[anchor.id] = Boolean(content && content.includes(anchor.resource));
|
|
75
|
+
}
|
|
76
|
+
return features;
|
|
77
|
+
}
|
|
78
|
+
|
|
55
79
|
export function parseAlertsArgs(argv = []) {
|
|
56
80
|
const args = normalizeArgv(argv);
|
|
57
81
|
if (args[0] === 'alerts') args.shift();
|
|
@@ -183,7 +207,228 @@ exports.handler = async (event) => {
|
|
|
183
207
|
// so every static alert resource is pinned to the aws.us_east_1 alias
|
|
184
208
|
// (runAlerts ensures main.tf declares it). IAM and archive_file are
|
|
185
209
|
// region-independent and stay unpinned.
|
|
186
|
-
|
|
210
|
+
// Primary notify-alarm spec per compute target. Addon alarms (DLQ
|
|
211
|
+
// depth, cron health, ...) append as further entries consumed the same
|
|
212
|
+
// way; each spec carries its own baked threshold while --threshold
|
|
213
|
+
// keeps driving the primary alarm only.
|
|
214
|
+
const PRIMARY_NOTIFY_ALARMS = {
|
|
215
|
+
lambda: {
|
|
216
|
+
tfName: 'notify_errors',
|
|
217
|
+
alarmName: 'errors-notify',
|
|
218
|
+
metricName: 'Errors',
|
|
219
|
+
namespace: 'AWS/Lambda',
|
|
220
|
+
statistic: 'Sum',
|
|
221
|
+
describe: ({ projectName, limit, evaluationPeriods }) =>
|
|
222
|
+
`${projectName} reports more than ${limit} Lambda errors in ${evaluationPeriods} minutes.`,
|
|
223
|
+
dimensions: { FunctionName: 'aws_lambda_function.app.function_name' },
|
|
224
|
+
},
|
|
225
|
+
static: {
|
|
226
|
+
tfName: 'notify_5xx_rate',
|
|
227
|
+
alarmName: '5xx-rate-notify',
|
|
228
|
+
metricName: '5xxErrorRate',
|
|
229
|
+
namespace: 'AWS/CloudFront',
|
|
230
|
+
statistic: 'Average',
|
|
231
|
+
describe: ({ projectName, limit, evaluationPeriods }) =>
|
|
232
|
+
`${projectName} serves more than a ${limit}% 5xx error rate over ${evaluationPeriods} minutes.`,
|
|
233
|
+
dimensions: { DistributionId: 'aws_cloudfront_distribution.site.id' },
|
|
234
|
+
},
|
|
235
|
+
ecs: {
|
|
236
|
+
tfName: 'notify_5xx',
|
|
237
|
+
alarmName: '5xx-notify',
|
|
238
|
+
metricName: 'HTTPCode_Target_5XX_Count',
|
|
239
|
+
namespace: 'AWS/ApplicationELB',
|
|
240
|
+
statistic: 'Sum',
|
|
241
|
+
describe: ({ limit, evaluationPeriods }) =>
|
|
242
|
+
`the ALB serves more than ${limit} 5XX errors in ${evaluationPeriods} minutes.`,
|
|
243
|
+
dimensions: { LoadBalancer: 'aws_lb.main.arn_suffix' },
|
|
244
|
+
},
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
// Renders one notify alarm block. Shape is shared by primary and addon
|
|
248
|
+
// alarms so every alarm reads (and diffs) the same way.
|
|
249
|
+
function renderNotifyAlarm(spec, { namePrefix, aliasLine, limit, evaluationPeriods, period, projectName }) {
|
|
250
|
+
const dimensions = spec.dimensions
|
|
251
|
+
? `\n dimensions = {\n${Object.entries(spec.dimensions).map(([key, value]) => ` ${key} = ${value}`).join('\n')}\n }\n`
|
|
252
|
+
: '';
|
|
253
|
+
return `resource "aws_cloudwatch_metric_alarm" "${spec.tfName}" {
|
|
254
|
+
${aliasLine} alarm_name = "${namePrefix}-${spec.alarmName}"
|
|
255
|
+
comparison_operator = "${spec.comparison || 'GreaterThanThreshold'}"
|
|
256
|
+
evaluation_periods = "${evaluationPeriods}"
|
|
257
|
+
metric_name = "${spec.metricName}"
|
|
258
|
+
namespace = "${spec.namespace}"
|
|
259
|
+
period = "${period}"
|
|
260
|
+
statistic = "${spec.statistic}"
|
|
261
|
+
threshold = "${limit}"
|
|
262
|
+
alarm_description = "Notifies ${namePrefix}-alerts when ${spec.describe({ projectName, limit, evaluationPeriods })}"
|
|
263
|
+
|
|
264
|
+
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
265
|
+
${dimensions}}` + '\n';
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// Addon notify alarms, rendered only when the tree carries the addon
|
|
269
|
+
// (see detectAlertAddonFeatures). Thresholds are baked per alarm;
|
|
270
|
+
// --threshold keeps driving the primary alarm only.
|
|
271
|
+
const ADDON_NOTIFY_ALARMS = [
|
|
272
|
+
{
|
|
273
|
+
id: 'queue:sqs',
|
|
274
|
+
tfName: 'notify_dlq_depth',
|
|
275
|
+
alarmName: 'dlq-depth-notify',
|
|
276
|
+
metricName: 'ApproximateNumberOfMessagesVisible',
|
|
277
|
+
namespace: 'AWS/SQS',
|
|
278
|
+
statistic: 'Maximum',
|
|
279
|
+
threshold: 1,
|
|
280
|
+
evaluationPeriods: 1,
|
|
281
|
+
period: 300,
|
|
282
|
+
describe: () => 'the dead-letter queue holds failed messages (redrive or inspect them).',
|
|
283
|
+
dimensions: { QueueName: 'aws_sqs_queue.dlq.name' },
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
id: 'cron',
|
|
287
|
+
tfName: 'notify_cron_target_errors',
|
|
288
|
+
alarmName: 'cron-target-errors-notify',
|
|
289
|
+
metricName: 'TargetErrorCount',
|
|
290
|
+
namespace: 'AWS/Scheduler',
|
|
291
|
+
statistic: 'Sum',
|
|
292
|
+
threshold: 1,
|
|
293
|
+
evaluationPeriods: 1,
|
|
294
|
+
period: 300,
|
|
295
|
+
describe: () => 'a scheduled run fails to invoke its target.',
|
|
296
|
+
dimensions: { ScheduleGroup: 'aws_scheduler_schedule_group.crons.name' },
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
id: 'cron',
|
|
300
|
+
tfName: 'notify_cron_dropped',
|
|
301
|
+
alarmName: 'cron-dropped-notify',
|
|
302
|
+
metricName: 'InvocationDroppedCount',
|
|
303
|
+
namespace: 'AWS/Scheduler',
|
|
304
|
+
statistic: 'Sum',
|
|
305
|
+
threshold: 1,
|
|
306
|
+
evaluationPeriods: 1,
|
|
307
|
+
period: 300,
|
|
308
|
+
describe: () => 'a scheduled run is dropped after retry exhaustion.',
|
|
309
|
+
dimensions: { ScheduleGroup: 'aws_scheduler_schedule_group.crons.name' },
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
id: 'db:instance',
|
|
313
|
+
tfName: 'notify_rds_storage_low',
|
|
314
|
+
alarmName: 'rds-storage-low-notify',
|
|
315
|
+
metricName: 'FreeStorageSpace',
|
|
316
|
+
namespace: 'AWS/RDS',
|
|
317
|
+
statistic: 'Minimum',
|
|
318
|
+
comparison: 'LessThanThreshold',
|
|
319
|
+
threshold: 2147483648,
|
|
320
|
+
evaluationPeriods: 1,
|
|
321
|
+
period: 300,
|
|
322
|
+
describe: () => 'the database drops below 2 GB of free storage.',
|
|
323
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
id: 'db:instance',
|
|
327
|
+
tfName: 'notify_rds_cpu_high',
|
|
328
|
+
alarmName: 'rds-cpu-high-notify',
|
|
329
|
+
metricName: 'CPUUtilization',
|
|
330
|
+
namespace: 'AWS/RDS',
|
|
331
|
+
statistic: 'Average',
|
|
332
|
+
threshold: 80,
|
|
333
|
+
evaluationPeriods: 3,
|
|
334
|
+
period: 300,
|
|
335
|
+
describe: () => 'database CPU stays above 80% for 15 minutes.',
|
|
336
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
id: 'db:instance',
|
|
340
|
+
tfName: 'notify_rds_connections_high',
|
|
341
|
+
alarmName: 'rds-connections-high-notify',
|
|
342
|
+
metricName: 'DatabaseConnections',
|
|
343
|
+
namespace: 'AWS/RDS',
|
|
344
|
+
statistic: 'Maximum',
|
|
345
|
+
threshold: 80,
|
|
346
|
+
evaluationPeriods: 2,
|
|
347
|
+
period: 300,
|
|
348
|
+
describe: () => 'database connections exceed 80 (micro exhausts near 112).',
|
|
349
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
id: 'db:aurora',
|
|
353
|
+
tfName: 'notify_rds_cpu_high',
|
|
354
|
+
alarmName: 'rds-cpu-high-notify',
|
|
355
|
+
metricName: 'CPUUtilization',
|
|
356
|
+
namespace: 'AWS/RDS',
|
|
357
|
+
statistic: 'Average',
|
|
358
|
+
threshold: 80,
|
|
359
|
+
evaluationPeriods: 3,
|
|
360
|
+
period: 300,
|
|
361
|
+
describe: () => 'database CPU stays above 80% for 15 minutes.',
|
|
362
|
+
dimensions: { DBClusterIdentifier: 'aws_rds_cluster.postgres.id' },
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
id: 'db:aurora',
|
|
366
|
+
tfName: 'notify_rds_connections_high',
|
|
367
|
+
alarmName: 'rds-connections-high-notify',
|
|
368
|
+
metricName: 'DatabaseConnections',
|
|
369
|
+
namespace: 'AWS/RDS',
|
|
370
|
+
statistic: 'Maximum',
|
|
371
|
+
threshold: 80,
|
|
372
|
+
evaluationPeriods: 2,
|
|
373
|
+
period: 300,
|
|
374
|
+
describe: () => 'database connections exceed 80.',
|
|
375
|
+
dimensions: { DBClusterIdentifier: 'aws_rds_cluster.postgres.id' },
|
|
376
|
+
},
|
|
377
|
+
{
|
|
378
|
+
id: 'db:redis',
|
|
379
|
+
tfName: 'notify_valkey_evictions',
|
|
380
|
+
alarmName: 'valkey-evictions-notify',
|
|
381
|
+
metricName: 'Evictions',
|
|
382
|
+
namespace: 'AWS/ElastiCache',
|
|
383
|
+
statistic: 'Sum',
|
|
384
|
+
threshold: 1,
|
|
385
|
+
evaluationPeriods: 1,
|
|
386
|
+
period: 300,
|
|
387
|
+
describe: () => 'valkey evicts keys (dedup/rate windows lose entries).',
|
|
388
|
+
dimensions: { ReplicationGroupId: 'aws_elasticache_replication_group.redis.id' },
|
|
389
|
+
},
|
|
390
|
+
{
|
|
391
|
+
id: 'db:redis',
|
|
392
|
+
tfName: 'notify_valkey_cpu_high',
|
|
393
|
+
alarmName: 'valkey-cpu-high-notify',
|
|
394
|
+
metricName: 'EngineCPUUtilization',
|
|
395
|
+
namespace: 'AWS/ElastiCache',
|
|
396
|
+
statistic: 'Average',
|
|
397
|
+
threshold: 80,
|
|
398
|
+
evaluationPeriods: 3,
|
|
399
|
+
period: 300,
|
|
400
|
+
describe: () => 'valkey engine CPU stays above 80% for 15 minutes.',
|
|
401
|
+
dimensions: { ReplicationGroupId: 'aws_elasticache_replication_group.redis.id' },
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
id: 'email:ses',
|
|
405
|
+
tfName: 'notify_ses_bounce_rate',
|
|
406
|
+
alarmName: 'ses-bounce-rate-notify',
|
|
407
|
+
metricName: 'Reputation.BounceRate',
|
|
408
|
+
namespace: 'AWS/SES',
|
|
409
|
+
statistic: 'Average',
|
|
410
|
+
threshold: 0.05,
|
|
411
|
+
evaluationPeriods: 1,
|
|
412
|
+
period: 3600,
|
|
413
|
+
describe: () => 'the account bounce rate exceeds 5%.',
|
|
414
|
+
dimensions: null,
|
|
415
|
+
},
|
|
416
|
+
{
|
|
417
|
+
id: 'email:ses',
|
|
418
|
+
tfName: 'notify_ses_complaint_rate',
|
|
419
|
+
alarmName: 'ses-complaint-rate-notify',
|
|
420
|
+
metricName: 'Reputation.ComplaintRate',
|
|
421
|
+
namespace: 'AWS/SES',
|
|
422
|
+
statistic: 'Average',
|
|
423
|
+
threshold: 0.001,
|
|
424
|
+
evaluationPeriods: 1,
|
|
425
|
+
period: 3600,
|
|
426
|
+
describe: () => 'the account complaint rate exceeds 0.1%.',
|
|
427
|
+
dimensions: null,
|
|
428
|
+
},
|
|
429
|
+
];
|
|
430
|
+
|
|
431
|
+
export function buildAlertsTf({ projectName, target = 'ecs', email, webhook, threshold, addons = {} }) {
|
|
187
432
|
const defaults = ALERT_TARGET_DEFAULTS[target] || ALERT_TARGET_DEFAULTS.ecs;
|
|
188
433
|
const limit = threshold ?? defaults.threshold;
|
|
189
434
|
// All targets use the workspace-aware local: flat ECS names let a
|
|
@@ -212,58 +457,29 @@ ${aliasLine} name = "${namePrefix}-alerts"
|
|
|
212
457
|
}
|
|
213
458
|
`;
|
|
214
459
|
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
period = "${defaults.period}"
|
|
239
|
-
statistic = "Average"
|
|
240
|
-
threshold = "${limit}"
|
|
241
|
-
alarm_description = "Notifies ${namePrefix}-alerts when ${projectName} serves more than a ${limit}% 5xx error rate over ${defaults.evaluationPeriods} minutes."
|
|
242
|
-
|
|
243
|
-
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
244
|
-
|
|
245
|
-
dimensions = {
|
|
246
|
-
DistributionId = aws_cloudfront_distribution.site.id
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
` : `resource "aws_cloudwatch_metric_alarm" "notify_5xx" {
|
|
250
|
-
alarm_name = "${namePrefix}-5xx-notify"
|
|
251
|
-
comparison_operator = "GreaterThanThreshold"
|
|
252
|
-
evaluation_periods = "${defaults.evaluationPeriods}"
|
|
253
|
-
metric_name = "HTTPCode_Target_5XX_Count"
|
|
254
|
-
namespace = "AWS/ApplicationELB"
|
|
255
|
-
period = "${defaults.period}"
|
|
256
|
-
statistic = "Sum"
|
|
257
|
-
threshold = "${limit}"
|
|
258
|
-
alarm_description = "Notifies ${namePrefix}-alerts when the ALB serves more than ${limit} 5XX errors in ${defaults.evaluationPeriods} minutes."
|
|
259
|
-
|
|
260
|
-
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
261
|
-
|
|
262
|
-
dimensions = {
|
|
263
|
-
LoadBalancer = aws_lb.main.arn_suffix
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
`;
|
|
460
|
+
const spec = PRIMARY_NOTIFY_ALARMS[target] || PRIMARY_NOTIFY_ALARMS.ecs;
|
|
461
|
+
const alarm = renderNotifyAlarm(spec, {
|
|
462
|
+
namePrefix,
|
|
463
|
+
aliasLine,
|
|
464
|
+
limit,
|
|
465
|
+
evaluationPeriods: defaults.evaluationPeriods,
|
|
466
|
+
period: defaults.period,
|
|
467
|
+
projectName,
|
|
468
|
+
});
|
|
469
|
+
// Addon alarms are regional metrics (no us-east-1 alias even on
|
|
470
|
+
// static trees) with baked thresholds.
|
|
471
|
+
const addonAlarms = ADDON_NOTIFY_ALARMS
|
|
472
|
+
.filter((entry) => addons && addons[entry.id])
|
|
473
|
+
.map((entry) => renderNotifyAlarm(entry, {
|
|
474
|
+
namePrefix,
|
|
475
|
+
aliasLine: '',
|
|
476
|
+
limit: entry.threshold,
|
|
477
|
+
evaluationPeriods: entry.evaluationPeriods,
|
|
478
|
+
period: entry.period,
|
|
479
|
+
projectName,
|
|
480
|
+
}))
|
|
481
|
+
.join('\n');
|
|
482
|
+
const addonSection = addonAlarms.length > 0 ? `\n${addonAlarms}` : '';
|
|
267
483
|
|
|
268
484
|
// NOTE: the email endpoint lands in git by design (the operator passed
|
|
269
485
|
// it explicitly); delivery still needs the recipient's confirmation
|
|
@@ -354,7 +570,7 @@ ${aliasLine} topic_arn = aws_sns_topic.alerts.arn
|
|
|
354
570
|
|
|
355
571
|
return `${header}
|
|
356
572
|
${topic}
|
|
357
|
-
${alarm}${emailSubscription}${forwarder}`;
|
|
573
|
+
${alarm}${addonSection}${emailSubscription}${forwarder}`;
|
|
358
574
|
}
|
|
359
575
|
|
|
360
576
|
// Adds the archive provider (used by the forwarder's data.archive_file)
|
|
@@ -458,6 +674,8 @@ async function runAlertsMain(input = {}) {
|
|
|
458
674
|
resultExtra: { region },
|
|
459
675
|
});
|
|
460
676
|
}
|
|
677
|
+
const addons = detectAlertAddonFeatures(path.join(cwd, 'terraform'));
|
|
678
|
+
const addonAlarmIds = ADDON_NOTIFY_ALARMS.filter((entry) => addons[entry.id]).map((entry) => entry.id);
|
|
461
679
|
const anchor = TARGET_ANCHORS[target] || TARGET_ANCHORS.ecs;
|
|
462
680
|
if (!mainTf.includes(anchor.resource)) {
|
|
463
681
|
const anchorName = target === 'lambda' ? 'aws_lambda_function.app'
|
|
@@ -573,7 +791,7 @@ async function runAlertsMain(input = {}) {
|
|
|
573
791
|
archiveAdded = ensureArchiveProvider(path.join(cwd, 'terraform', 'backend.tf'));
|
|
574
792
|
}
|
|
575
793
|
|
|
576
|
-
await atomicWriteFile(alertsPath, buildAlertsTf({ projectName, target, email, webhook, threshold }));
|
|
794
|
+
await atomicWriteFile(alertsPath, buildAlertsTf({ projectName, target, email, webhook, threshold, addons }));
|
|
577
795
|
if (webhook) {
|
|
578
796
|
await ensureForwarderZipIgnored(cwd);
|
|
579
797
|
}
|
|
@@ -583,7 +801,7 @@ async function runAlertsMain(input = {}) {
|
|
|
583
801
|
const { aliasAdded, archiveAdded } = scaffolded;
|
|
584
802
|
const destinations = email && webhook ? 'both' : webhook ? 'webhook' : email ? 'email' : 'manual';
|
|
585
803
|
const kind = webhook ? detectWebhookKind(webhook) : undefined;
|
|
586
|
-
console.log(color.green(`\n✅ Created ${ALERTS_TF_FILE} (SNS topic + ${target} notify alarm${webhook ? ' + chat forwarder' : ''}).`));
|
|
804
|
+
console.log(color.green(`\n✅ Created ${ALERTS_TF_FILE} (SNS topic + ${target} notify alarm${addonAlarmIds.length > 0 ? ` + ${addonAlarmIds.length} addon alarm${addonAlarmIds.length === 1 ? '' : 's'}` : ''}${webhook ? ' + chat forwarder' : ''}).`));
|
|
587
805
|
if (aliasAdded) console.log(color.dim(' Added the aws.us_east_1 provider alias to terraform/main.tf (CloudFront alarms only exist there).'));
|
|
588
806
|
if (archiveAdded) console.log(color.dim(' Pinned the archive provider in terraform/backend.tf (forwarder packaging).'));
|
|
589
807
|
console.log(color.dim(' Next steps:'));
|
|
@@ -605,7 +823,7 @@ async function runAlertsMain(input = {}) {
|
|
|
605
823
|
}
|
|
606
824
|
// Telemetry carries destination *kinds* only — never the email
|
|
607
825
|
// address or webhook URL (PII / bearer secret).
|
|
608
|
-
await trackSuccess('alerts_run', { projectName, action: 'scaffold', target, destinations, thresholdCustom: threshold !== undefined });
|
|
826
|
+
await trackSuccess('alerts_run', { projectName, action: 'scaffold', target, destinations, thresholdCustom: threshold !== undefined, addonAlarms: addonAlarmIds });
|
|
609
827
|
outro(color.green('Done.'));
|
|
610
828
|
return { ok: true, action: 'scaffold', projectName, region, target, destinations, file: ALERTS_TF_FILE };
|
|
611
829
|
}
|
package/src/commands/status.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawnSync } from 'child_process';
|
|
2
|
+
import path from 'path';
|
|
2
3
|
import { ECSClient, DescribeServicesCommand } from '@aws-sdk/client-ecs';
|
|
3
|
-
import { CloudWatchClient, DescribeAlarmsCommand } from '@aws-sdk/client-cloudwatch';
|
|
4
|
+
import { CloudWatchClient, DescribeAlarmsCommand, GetMetricDataCommand } from '@aws-sdk/client-cloudwatch';
|
|
4
5
|
import { CloudFrontClient, ListDistributionsCommand } from '@aws-sdk/client-cloudfront';
|
|
5
6
|
import color from 'picocolors';
|
|
6
7
|
import { intro, outro, spinner } from '@clack/prompts';
|
|
@@ -8,8 +9,9 @@ import { trackEvent, flushTelemetry, trackSuccess, trackFailure, setActiveComman
|
|
|
8
9
|
import { failCommand, failProjectNotInitialized, isProgrammaticCall, refuseUnknownFlags } from '../utils/command.js';
|
|
9
10
|
import { parseFlags, normalizeOptions, normalizeArgv } from '../utils/args.js';
|
|
10
11
|
import { handleAuthErrorBranch, resolveClient } from '../utils/aws.js';
|
|
11
|
-
import { resolveRegion, resolveProjectName, resolveCwd, readTerraformComputeTarget, readResourceSuffix, resolveNamespacedProject } from '../utils/resolvers.js';
|
|
12
|
+
import { resolveRegion, resolveProjectName, resolveCwd, readTerraformComputeTarget, readResourceSuffix, resolveNamespacedProject, safeSuffixedName } from '../utils/resolvers.js';
|
|
12
13
|
import { runDiagnose } from './diagnose.js';
|
|
14
|
+
import { detectAlertAddonFeatures } from './alerts.js';
|
|
13
15
|
import { fetchGoldenSignals } from '../utils/golden-signals.js';
|
|
14
16
|
import { requireManagedTree } from '../utils/terraform-metadata.js';
|
|
15
17
|
|
|
@@ -89,13 +91,15 @@ export function getLambdaHealth(configuration = {}) {
|
|
|
89
91
|
return { state, lastUpdateStatus, healthy };
|
|
90
92
|
}
|
|
91
93
|
|
|
92
|
-
function printLambdaDashboard({ functionName, configuration, health }) {
|
|
94
|
+
function printLambdaDashboard({ functionName, configuration, health, cron }) {
|
|
93
95
|
console.log('');
|
|
94
96
|
console.log(` ${color.bold('Function:')} ${color.cyan(functionName)} ${health.state === 'Active' ? color.green('(ACTIVE)') : color.yellow(`(${health.state})`)}`);
|
|
95
97
|
console.log(` ${color.bold('Last update:')} ${health.lastUpdateStatus === 'Successful' ? color.green(health.lastUpdateStatus) : color.yellow(health.lastUpdateStatus)}`);
|
|
96
98
|
if (configuration.LastModified) console.log(` ${color.dim('Modified:')} ${configuration.LastModified}`);
|
|
97
99
|
if (configuration.MemorySize) console.log(` ${color.dim('Memory:')} ${configuration.MemorySize} MB ${color.dim(`(timeout ${configuration.Timeout ?? '?'}s)`)}`);
|
|
98
100
|
if (configuration.Code?.ImageUri) console.log(` ${color.dim('Image:')} ${configuration.Code.ImageUri.split('/').pop()}`);
|
|
101
|
+
const cronLine = formatCronHealthLine(cron);
|
|
102
|
+
if (cronLine) console.log(` ${color.bold('Cron:')} ${cronLine}`);
|
|
99
103
|
console.log('');
|
|
100
104
|
}
|
|
101
105
|
|
|
@@ -212,6 +216,17 @@ export async function runLambdaStatus(input = {}) {
|
|
|
212
216
|
}
|
|
213
217
|
|
|
214
218
|
const health = getLambdaHealth(configuration);
|
|
219
|
+
// Cron run evidence, best-effort like the ECS path: detected from
|
|
220
|
+
// the tree, null when absent or unreachable.
|
|
221
|
+
let cron = null;
|
|
222
|
+
if (detectAlertAddonFeatures(path.join(cwd, 'terraform')).cron) {
|
|
223
|
+
try {
|
|
224
|
+
const cloudWatchClient = resolveClient(options.cloudWatchClient, CloudWatchClient, { region });
|
|
225
|
+
cron = await fetchCronHealth({ cloudWatchClient, scheduleGroup: safeSuffixedName(`${namespacedProject}${readResourceSuffix(cwd)}`, '-crons', 64) });
|
|
226
|
+
} catch {
|
|
227
|
+
cron = null;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
215
230
|
const payload = {
|
|
216
231
|
function: {
|
|
217
232
|
name: functionName,
|
|
@@ -224,6 +239,7 @@ export async function runLambdaStatus(input = {}) {
|
|
|
224
239
|
},
|
|
225
240
|
healthy: health.healthy,
|
|
226
241
|
signals: null,
|
|
242
|
+
cron,
|
|
227
243
|
region,
|
|
228
244
|
computeTarget: 'lambda',
|
|
229
245
|
};
|
|
@@ -235,7 +251,7 @@ export async function runLambdaStatus(input = {}) {
|
|
|
235
251
|
}
|
|
236
252
|
|
|
237
253
|
if (s) s.stop('Health check complete.\n');
|
|
238
|
-
printLambdaDashboard({ functionName, configuration, health });
|
|
254
|
+
printLambdaDashboard({ functionName, configuration, health, cron });
|
|
239
255
|
|
|
240
256
|
if (!health.healthy) {
|
|
241
257
|
console.log(color.yellow(DEGRADED_MESSAGE));
|
|
@@ -389,7 +405,55 @@ function printStaticDashboard({ distribution }) {
|
|
|
389
405
|
console.log('');
|
|
390
406
|
}
|
|
391
407
|
|
|
392
|
-
|
|
408
|
+
// Cron run evidence from scheduler metrics (the Scheduler API exposes
|
|
409
|
+
// no execution history): latest attempt plus 24h failure sums. Throws
|
|
410
|
+
// naturally — callers treat it as best-effort like golden signals.
|
|
411
|
+
export async function fetchCronHealth({ cloudWatchClient, scheduleGroup }) {
|
|
412
|
+
const endTime = new Date();
|
|
413
|
+
const startTime = new Date(endTime.getTime() - 24 * 60 * 60 * 1000);
|
|
414
|
+
const queries = [
|
|
415
|
+
['cronAttempts', 'InvocationAttemptCount'],
|
|
416
|
+
['cronErrors', 'TargetErrorCount'],
|
|
417
|
+
['cronDropped', 'InvocationDroppedCount'],
|
|
418
|
+
].map(([Id, MetricName]) => ({
|
|
419
|
+
Id,
|
|
420
|
+
MetricStat: {
|
|
421
|
+
Metric: {
|
|
422
|
+
Namespace: 'AWS/Scheduler',
|
|
423
|
+
MetricName,
|
|
424
|
+
Dimensions: [{ Name: 'ScheduleGroup', Value: scheduleGroup }],
|
|
425
|
+
},
|
|
426
|
+
Period: 3600,
|
|
427
|
+
Stat: 'Sum',
|
|
428
|
+
},
|
|
429
|
+
}));
|
|
430
|
+
const resp = await cloudWatchClient.send(new GetMetricDataCommand({
|
|
431
|
+
StartTime: startTime,
|
|
432
|
+
EndTime: endTime,
|
|
433
|
+
ScanBy: 'TimestampDescending',
|
|
434
|
+
MetricDataQueries: queries,
|
|
435
|
+
}));
|
|
436
|
+
const byId = new Map((resp.MetricDataResults || []).map((r) => [r.Id, r]));
|
|
437
|
+
const attempts = byId.get('cronAttempts');
|
|
438
|
+
let lastAttemptAt = null;
|
|
439
|
+
if (attempts) {
|
|
440
|
+
const stamps = (attempts.Timestamps || [])
|
|
441
|
+
.map((t, i) => [new Date(t).getTime(), attempts.Values[i]])
|
|
442
|
+
.filter(([, value]) => value > 0)
|
|
443
|
+
.map(([time]) => time);
|
|
444
|
+
if (stamps.length > 0) lastAttemptAt = new Date(Math.max(...stamps)).toISOString();
|
|
445
|
+
}
|
|
446
|
+
const sum = (id) => (byId.get(id)?.Values || []).reduce((total, value) => total + value, 0);
|
|
447
|
+
return { lastAttemptAt, targetErrors24h: sum('cronErrors'), dropped24h: sum('cronDropped') };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function formatCronHealthLine(cron) {
|
|
451
|
+
if (!cron) return null;
|
|
452
|
+
if (!cron.lastAttemptAt) return 'no runs in the last 24h';
|
|
453
|
+
return `last run ${cron.lastAttemptAt} · ${cron.targetErrors24h} target errors · ${cron.dropped24h} dropped (24h)`;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function printDashboard({ serviceName, cluster, health, signals, cron }) {
|
|
393
457
|
console.log('');
|
|
394
458
|
console.log(` ${color.bold('Service:')} ${color.cyan(serviceName)} ${health.status === 'ACTIVE' ? color.green('(ACTIVE)') : color.yellow(`(${health.status})`)}`);
|
|
395
459
|
console.log(` ${color.dim('Cluster:')} ${cluster}`);
|
|
@@ -412,6 +476,8 @@ function printDashboard({ serviceName, cluster, health, signals }) {
|
|
|
412
476
|
if (db?.kind === 'cluster') console.log(` ${color.dim('DB:')} ${db.capacityAcu ?? 'n/a'} ACU · ${formatSignalCount(db.connections)} conns`);
|
|
413
477
|
else if (db) console.log(` ${color.dim('DB:')} CPU ${formatSignalPercent(db.cpuPercent)} · ${formatSignalCount(db.connections)} conns`);
|
|
414
478
|
}
|
|
479
|
+
const cronLine = formatCronHealthLine(cron);
|
|
480
|
+
if (cronLine) console.log(` ${color.bold('Cron:')} ${cronLine}`);
|
|
415
481
|
console.log('');
|
|
416
482
|
}
|
|
417
483
|
|
|
@@ -529,6 +595,17 @@ async function runStatusMain(input = {}) {
|
|
|
529
595
|
signalsError = [`signals: ${error?.message || error}`];
|
|
530
596
|
}
|
|
531
597
|
|
|
598
|
+
// Cron run evidence, same best-effort deal: detected from the
|
|
599
|
+
// tree (not the target), null when absent or unreachable.
|
|
600
|
+
let cron = null;
|
|
601
|
+
if (detectAlertAddonFeatures(path.join(cwd, 'terraform')).cron) {
|
|
602
|
+
try {
|
|
603
|
+
cron = await fetchCronHealth({ cloudWatchClient, scheduleGroup: safeSuffixedName(`${namespacedProject}${resourceSuffix}`, '-crons', 64) });
|
|
604
|
+
} catch {
|
|
605
|
+
cron = null;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
532
609
|
const payload = {
|
|
533
610
|
service: {
|
|
534
611
|
name: serviceName,
|
|
@@ -541,6 +618,7 @@ async function runStatusMain(input = {}) {
|
|
|
541
618
|
alarms: health.alarms,
|
|
542
619
|
signals,
|
|
543
620
|
...(signalsError ? { signalsError } : {}),
|
|
621
|
+
cron,
|
|
544
622
|
healthy: health.healthy,
|
|
545
623
|
region,
|
|
546
624
|
};
|
|
@@ -552,7 +630,7 @@ async function runStatusMain(input = {}) {
|
|
|
552
630
|
}
|
|
553
631
|
|
|
554
632
|
if (s) s.stop('Health check complete.\n');
|
|
555
|
-
printDashboard({ serviceName, cluster, health, signals });
|
|
633
|
+
printDashboard({ serviceName, cluster, health, signals, cron });
|
|
556
634
|
|
|
557
635
|
if (!health.healthy) {
|
|
558
636
|
console.log(color.yellow(DEGRADED_MESSAGE));
|
package/src/data/action-pins.js
CHANGED
|
@@ -12,7 +12,7 @@ export const ACTION_PINS = {
|
|
|
12
12
|
'aquasecurity/trivy-action': { sha: 'ed142fd0673e97e23eac54620cfb913e5ce36c25', tag: 'v0.36.0' },
|
|
13
13
|
'aws-actions/amazon-ecr-login': { sha: '03f1aad4c6c7ffd436567f42f9384779290529bd', tag: 'v2' },
|
|
14
14
|
'aws-actions/configure-aws-credentials': { sha: '7474bc4690e29a8392af63c5b98e7449536d5c3a', tag: 'v4' },
|
|
15
|
-
'grada-run/grada-action': { sha: '
|
|
15
|
+
'grada-run/grada-action': { sha: '9df2e2c12314b6165e62f0e615edb01d6a81c40b', tag: 'v1' },
|
|
16
16
|
'hashicorp/setup-terraform': { sha: 'b9cd54a3c349d3f38e8881555d616ced269862dd', tag: 'v3' },
|
|
17
17
|
};
|
|
18
18
|
|
|
@@ -4,7 +4,7 @@ import { RDSClient } from '@aws-sdk/client-rds';
|
|
|
4
4
|
import { resolveClient } from './aws.js';
|
|
5
5
|
import { normalizeOptions } from './args.js';
|
|
6
6
|
import { findDbTarget, resolveDbIdentifier, resolveDbClusterIdentifier } from './rds.js';
|
|
7
|
-
import { readResourceSuffix } from './resolvers.js';
|
|
7
|
+
import { readResourceSuffix, safeSuffixedName } from './resolvers.js';
|
|
8
8
|
|
|
9
9
|
// Golden Signals: live CloudWatch telemetry for `grada status`.
|
|
10
10
|
//
|
|
@@ -21,10 +21,10 @@ import { readResourceSuffix } from './resolvers.js';
|
|
|
21
21
|
export const GOLDEN_SIGNALS_WINDOW_MINUTES = 15;
|
|
22
22
|
export const GOLDEN_SIGNALS_PERIOD_SECONDS = 300;
|
|
23
23
|
|
|
24
|
-
// Mirrors the generated ALB name (
|
|
25
|
-
// templates/terraform/main.tf) so lookup matches
|
|
24
|
+
// Mirrors the generated ALB name (the length-conditional in
|
|
25
|
+
// templates/terraform/main.tf) so lookup matches exactly, hashed or not.
|
|
26
26
|
export function resolveAlbName(projectName) {
|
|
27
|
-
return
|
|
27
|
+
return safeSuffixedName(projectName, '-alb', 32);
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
// `arn:aws:elasticloadbalancing:<region>:<acct>:loadbalancer/app/<name>/<id>`
|
package/src/utils/resolvers.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fsSync from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
import { randomBytes } from 'crypto';
|
|
3
|
+
import { randomBytes, createHash } from 'crypto';
|
|
4
4
|
import { normalizeOptions } from './args.js';
|
|
5
5
|
|
|
6
6
|
export const FALLBACK_REGION = 'us-east-2';
|
|
@@ -112,6 +112,21 @@ export function readTerraformRegion(cwd = process.cwd()) {
|
|
|
112
112
|
return match[1];
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
+
// Length-safe AWS resource name, mirroring the template conditional
|
|
116
|
+
// byte-for-byte: short names render identically; over-cap names take a
|
|
117
|
+
// deterministic md5 stem (never a truncating collision). `cap` is the
|
|
118
|
+
// service limit including suffix. Doubled hyphens at the cut point are
|
|
119
|
+
// left alone — valid in every name pattern here, and trimming would
|
|
120
|
+
// need identical semantics on both sides. Both sides must stay in
|
|
121
|
+
// sync — the parity tests below pin the shared cases.
|
|
122
|
+
export function safeSuffixedName(base, suffix, cap) {
|
|
123
|
+
const full = `${base}${suffix}`;
|
|
124
|
+
if (full.length <= cap) return full;
|
|
125
|
+
const stemLength = Math.max(cap - 1 - 5 - suffix.length, 1);
|
|
126
|
+
const stem = base.slice(0, stemLength);
|
|
127
|
+
return `${stem}-${createHash('md5').update(base, 'utf8').digest('hex').slice(0, 5)}${suffix}`;
|
|
128
|
+
}
|
|
129
|
+
|
|
115
130
|
export function resolveRegion(options = {}, cwd = process.cwd()) {
|
|
116
131
|
const opts = normalizeOptions(options);
|
|
117
132
|
if (typeof opts.region === 'string' && opts.region.trim()) {
|
|
@@ -141,7 +141,7 @@ jobs:
|
|
|
141
141
|
# Runs AFTER the push: the function image tags exist by the time the
|
|
142
142
|
# function is created or updated.
|
|
143
143
|
- name: Apply Infrastructure Changes
|
|
144
|
-
uses: grada-run/grada-action@
|
|
144
|
+
uses: grada-run/grada-action@9df2e2c12314b6165e62f0e615edb01d6a81c40b # v1
|
|
145
145
|
with:
|
|
146
146
|
aws-region: ${{ env.AWS_REGION }}
|
|
147
147
|
role-to-assume: arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/{{DEPLOY_ROLE_NAME}}
|
|
@@ -73,7 +73,7 @@ jobs:
|
|
|
73
73
|
|
|
74
74
|
# --- 3. INFRASTRUCTURE SYNC (The New Marketplace Action) ---
|
|
75
75
|
- name: Apply Infrastructure Changes
|
|
76
|
-
uses: grada-run/grada-action@
|
|
76
|
+
uses: grada-run/grada-action@9df2e2c12314b6165e62f0e615edb01d6a81c40b # v1
|
|
77
77
|
with:
|
|
78
78
|
aws-region: ${{ env.AWS_REGION }}
|
|
79
79
|
role-to-assume: arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/{{DEPLOY_ROLE_NAME}}
|
|
@@ -144,7 +144,7 @@ jobs:
|
|
|
144
144
|
# the tag exists. A gate failure below can therefore never strand
|
|
145
145
|
# the service on an unpullable revision.
|
|
146
146
|
- name: Apply Infrastructure Changes
|
|
147
|
-
uses: grada-run/grada-action@
|
|
147
|
+
uses: grada-run/grada-action@9df2e2c12314b6165e62f0e615edb01d6a81c40b # v1
|
|
148
148
|
with:
|
|
149
149
|
aws-region: ${{ env.AWS_REGION }}
|
|
150
150
|
role-to-assume: arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/{{DEPLOY_ROLE_NAME}}
|
|
@@ -195,3 +195,71 @@ jobs:
|
|
|
195
195
|
--arg summary "$SUMMARY" \
|
|
196
196
|
'{text: (":warning: Infrastructure drift detected in " + $repo + "\n" + $url + "\n```" + $summary + "```")}' > /tmp/drift-slack.json
|
|
197
197
|
curl -sS -X POST -H 'Content-Type: application/json' -d @/tmp/drift-slack.json "$SLACK_WEBHOOK_URL"
|
|
198
|
+
|
|
199
|
+
# Pre-deploy snapshots accumulate per push with no retention: collect
|
|
200
|
+
# them (plus orphaned images, log groups, and EIPs) on the same daily
|
|
201
|
+
# schedule. A separate job so its failures can't trip the drift
|
|
202
|
+
# check-failed logic (that keys on the plan job's outputs) — gc gets
|
|
203
|
+
# its own failure label instead.
|
|
204
|
+
gc:
|
|
205
|
+
name: Collect garbage snapshots and orphans
|
|
206
|
+
runs-on: ubuntu-latest
|
|
207
|
+
timeout-minutes: 15
|
|
208
|
+
|
|
209
|
+
steps:
|
|
210
|
+
- name: Checkout Code
|
|
211
|
+
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
212
|
+
|
|
213
|
+
- name: Configure AWS Credentials
|
|
214
|
+
id: aws-creds
|
|
215
|
+
uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4
|
|
216
|
+
continue-on-error: true
|
|
217
|
+
with:
|
|
218
|
+
role-to-assume: {{ROLE_ARN}}
|
|
219
|
+
aws-region: ${{ env.AWS_REGION }}
|
|
220
|
+
|
|
221
|
+
- name: Interpret credential failure
|
|
222
|
+
if: steps.aws-creds.outcome == 'failure'
|
|
223
|
+
run: |
|
|
224
|
+
echo "::notice::AWS credential assumption failed — skipping garbage collection this run."
|
|
225
|
+
|
|
226
|
+
- name: Collect garbage
|
|
227
|
+
id: gc
|
|
228
|
+
if: steps.aws-creds.outcome == 'success'
|
|
229
|
+
run: npx --yes grada-run gc --headless --yes --region ${{ env.AWS_REGION }}
|
|
230
|
+
|
|
231
|
+
# gc failures need a human (a stuck snapshot pattern rots silently),
|
|
232
|
+
# but under their own label — never the drift labels above. A green
|
|
233
|
+
# run closes the stale issue; cancelled/skipped runs stay silent.
|
|
234
|
+
- name: Report gc result
|
|
235
|
+
if: always() && steps.aws-creds.outcome == 'success'
|
|
236
|
+
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7
|
|
237
|
+
with:
|
|
238
|
+
script: |
|
|
239
|
+
const { owner, repo } = context.repo;
|
|
240
|
+
const label = 'grada-gc-failed';
|
|
241
|
+
const outcome = '${{ steps.gc.outcome }}';
|
|
242
|
+
const runUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`;
|
|
243
|
+
try {
|
|
244
|
+
await github.rest.issues.getLabel({ owner, repo, name: label });
|
|
245
|
+
} catch (error) {
|
|
246
|
+
if (error.status === 404) {
|
|
247
|
+
await github.rest.issues.createLabel({ owner, repo, name: label, color: 'd93f0b', description: 'Scheduled garbage collection failed' });
|
|
248
|
+
} else {
|
|
249
|
+
throw error;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
const { data: open } = await github.rest.issues.listForRepo({ owner, repo, state: 'open', labels: label });
|
|
253
|
+
const existing = open[0];
|
|
254
|
+
if (outcome === 'failure') {
|
|
255
|
+
const body = `## 🧹 Garbage Collection Failed\n\nThe scheduled gc run errored on \`${new Date().toISOString()}\` ([workflow run](${runUrl})) — snapshots and orphans are accumulating.\n\nFix the run (credentials, permissions, gc output above) and the issue closes on the next green run.`;
|
|
256
|
+
if (existing) {
|
|
257
|
+
await github.rest.issues.createComment({ owner, repo, issue_number: existing.number, body: `gc still failing on \`${new Date().toISOString()}\` ([run](${runUrl})).` });
|
|
258
|
+
await github.rest.issues.update({ owner, repo, issue_number: existing.number, body });
|
|
259
|
+
} else {
|
|
260
|
+
await github.rest.issues.create({ owner, repo, title: '🧹 Garbage Collection Failed', labels: [label], body });
|
|
261
|
+
}
|
|
262
|
+
} else if (outcome === 'success' && existing) {
|
|
263
|
+
await github.rest.issues.createComment({ owner, repo, issue_number: existing.number, body: `✅ Garbage collection healthy again on \`${new Date().toISOString()}\` ([run](${runUrl})); closing.` });
|
|
264
|
+
await github.rest.issues.update({ owner, repo, issue_number: existing.number, state: 'closed' });
|
|
265
|
+
}
|
|
@@ -21,6 +21,9 @@ env:
|
|
|
21
21
|
ECR_REPOSITORY: {{PROJECT_NAME}}{{RESOURCE_SUFFIX}}-repo
|
|
22
22
|
WORKSPACE: pr-${{ github.event.pull_request.number }}
|
|
23
23
|
CF_COMMENT: {{PROJECT_NAME}}-pr-${{ github.event.pull_request.number }}{{RESOURCE_SUFFIX}}-cdn
|
|
24
|
+
PREVIEW_CLUSTER: {{PROJECT_NAME}}-pr-${{ github.event.pull_request.number }}{{RESOURCE_SUFFIX}}-cluster
|
|
25
|
+
PREVIEW_SERVICE: {{PROJECT_NAME}}-pr-${{ github.event.pull_request.number }}{{RESOURCE_SUFFIX}}-service
|
|
26
|
+
PREVIEW_TASK_FAMILY: {{PROJECT_NAME}}-pr-${{ github.event.pull_request.number }}{{RESOURCE_SUFFIX}}-task
|
|
24
27
|
|
|
25
28
|
permissions:
|
|
26
29
|
id-token: write
|
|
@@ -99,6 +102,72 @@ jobs:
|
|
|
99
102
|
PREVIEW_URL=$(terraform output -raw cloudfront_url)
|
|
100
103
|
echo "PREVIEW_URL=$PREVIEW_URL" >> $GITHUB_ENV
|
|
101
104
|
|
|
105
|
+
# The image tag is static per PR (:pr-N), so a re-push changes no
|
|
106
|
+
# Terraform input — without these steps the service keeps running
|
|
107
|
+
# the first push's image. Mirror the deploy workflow: register a
|
|
108
|
+
# digest-pinned revision (immutable, so concurrent pushes can't
|
|
109
|
+
# swap the image mid-rollout), force the rollout, and verify it
|
|
110
|
+
# stuck.
|
|
111
|
+
- name: Register preview task definition revision
|
|
112
|
+
id: register-preview-task-def
|
|
113
|
+
if: steps.aws-creds.outcome == 'success'
|
|
114
|
+
env:
|
|
115
|
+
ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
|
|
116
|
+
run: |
|
|
117
|
+
DIGEST=$(aws ecr describe-images \
|
|
118
|
+
--repository-name ${{ env.ECR_REPOSITORY }} \
|
|
119
|
+
--image-ids imageTag=${{ env.WORKSPACE }} \
|
|
120
|
+
--query 'imageDetails[0].imageDigest' \
|
|
121
|
+
--output text \
|
|
122
|
+
--region ${{ env.AWS_REGION }})
|
|
123
|
+
IMAGE_URI=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_REPOSITORY }}@${DIGEST}
|
|
124
|
+
TASK_DEF=$(aws ecs describe-task-definition \
|
|
125
|
+
--task-definition ${{ env.PREVIEW_TASK_FAMILY }} \
|
|
126
|
+
--region ${{ env.AWS_REGION }} \
|
|
127
|
+
--query 'taskDefinition' \
|
|
128
|
+
--output json)
|
|
129
|
+
NEW_TASK_DEF=$(echo "$TASK_DEF" | jq \
|
|
130
|
+
--arg IMAGE "$IMAGE_URI" \
|
|
131
|
+
'.containerDefinitions[0].image = $IMAGE | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)')
|
|
132
|
+
NEW_TASK_ARN=$(aws ecs register-task-definition \
|
|
133
|
+
--region ${{ env.AWS_REGION }} \
|
|
134
|
+
--cli-input-json "$NEW_TASK_DEF" \
|
|
135
|
+
--query 'taskDefinition.taskDefinitionArn' \
|
|
136
|
+
--output text)
|
|
137
|
+
echo "task-arn=$NEW_TASK_ARN" >> $GITHUB_OUTPUT
|
|
138
|
+
|
|
139
|
+
- name: Force preview deployment
|
|
140
|
+
if: steps.aws-creds.outcome == 'success'
|
|
141
|
+
run: |
|
|
142
|
+
aws ecs update-service \
|
|
143
|
+
--cluster ${{ env.PREVIEW_CLUSTER }} \
|
|
144
|
+
--service ${{ env.PREVIEW_SERVICE }} \
|
|
145
|
+
--task-definition ${{ steps.register-preview-task-def.outputs.task-arn }} \
|
|
146
|
+
--force-new-deployment \
|
|
147
|
+
--region ${{ env.AWS_REGION }}
|
|
148
|
+
|
|
149
|
+
# update-service returns before the rollout finishes: wait for
|
|
150
|
+
# stability, then verify PRIMARY actually runs the registered
|
|
151
|
+
# revision — a circuit-breaker rollback re-stabilizes on the OLD
|
|
152
|
+
# revision, which the wait alone would report green.
|
|
153
|
+
- name: Wait for preview rollout
|
|
154
|
+
if: steps.aws-creds.outcome == 'success'
|
|
155
|
+
run: |
|
|
156
|
+
aws ecs wait services-stable \
|
|
157
|
+
--cluster ${{ env.PREVIEW_CLUSTER }} \
|
|
158
|
+
--services ${{ env.PREVIEW_SERVICE }} \
|
|
159
|
+
--region ${{ env.AWS_REGION }}
|
|
160
|
+
PRIMARY_TD=$(aws ecs describe-services \
|
|
161
|
+
--cluster ${{ env.PREVIEW_CLUSTER }} \
|
|
162
|
+
--services ${{ env.PREVIEW_SERVICE }} \
|
|
163
|
+
--query 'services[0].deployments[?status==`PRIMARY`].taskDefinition' \
|
|
164
|
+
--output text \
|
|
165
|
+
--region ${{ env.AWS_REGION }})
|
|
166
|
+
if [ "$PRIMARY_TD" != "${{ steps.register-preview-task-def.outputs.task-arn }}" ]; then
|
|
167
|
+
echo "Preview rollout of ${{ steps.register-preview-task-def.outputs.task-arn }} did not stick (PRIMARY runs $PRIMARY_TD) — the deployment failed or rolled back." >&2
|
|
168
|
+
exit 1
|
|
169
|
+
fi
|
|
170
|
+
|
|
102
171
|
# Every push redeploys the preview: clear its CDN so new HTML shows
|
|
103
172
|
# immediately instead of serving stale pages until TTL expiry.
|
|
104
173
|
- name: Invalidate CloudFront
|
|
@@ -7,10 +7,10 @@
|
|
|
7
7
|
# chosen model in the AWS Bedrock console first.
|
|
8
8
|
|
|
9
9
|
# --- Least-Privilege IAM Policy for the ECS Task Role ---
|
|
10
|
-
# Scoped to the chosen model in the deploy region: the
|
|
10
|
+
# Scoped to the chosen model in the deploy region and account: the
|
|
11
11
|
# foundation-model ARN names the model exactly, and the
|
|
12
12
|
# inference-profile line stays (profile-style model IDs invoke through
|
|
13
|
-
# profiles) but pinned to the region.
|
|
13
|
+
# profiles) but pinned to the region and the caller's account.
|
|
14
14
|
resource "aws_iam_role_policy" "ai_bedrock_access" {
|
|
15
15
|
name = "${local.app_name}-ai-bedrock-access"
|
|
16
16
|
role = aws_iam_role.task_role.id
|
|
@@ -25,7 +25,7 @@ resource "aws_iam_role_policy" "ai_bedrock_access" {
|
|
|
25
25
|
]
|
|
26
26
|
Resource = [
|
|
27
27
|
"arn:aws:bedrock:{{REGION}}::foundation-model/{{BEDROCK_MODEL_ID}}",
|
|
28
|
-
"arn:aws:bedrock:{{REGION}}
|
|
28
|
+
"arn:aws:bedrock:{{REGION}}:${data.aws_caller_identity.current.account_id}:inference-profile/*"
|
|
29
29
|
]
|
|
30
30
|
}
|
|
31
31
|
]
|
|
@@ -11,9 +11,19 @@
|
|
|
11
11
|
# in application code (the payload below carries the configured command), or
|
|
12
12
|
# point the schedule at a dedicated function.
|
|
13
13
|
|
|
14
|
+
locals {
|
|
15
|
+
# Length-safe names: short names render byte-identically; over-cap
|
|
16
|
+
# names hash-suffix instead of failing validation. Mirrors
|
|
17
|
+
# safeSuffixedName in resolvers.js.
|
|
18
|
+
scheduler_role_safe_name = length("${local.app_name}-cron-scheduler-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 20)}-${substr(md5(local.app_name), 0, 5)}-cron-scheduler-role" : "${local.app_name}-cron-scheduler-role"
|
|
19
|
+
schedule_full_name = "${local.app_name}-cron-{{CRON_NAME}}"
|
|
20
|
+
schedule_safe_name = length(local.schedule_full_name) > 64 ? "${substr(local.schedule_full_name, 0, 64 - 1 - 5)}-${substr(md5(local.schedule_full_name), 0, 5)}" : local.schedule_full_name
|
|
21
|
+
schedule_group_safe_name = length("${local.app_name}-crons") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 6)}-${substr(md5(local.app_name), 0, 5)}-crons" : "${local.app_name}-crons"
|
|
22
|
+
}
|
|
23
|
+
|
|
14
24
|
# --- IAM Role assumed by EventBridge Scheduler ---
|
|
15
25
|
resource "aws_iam_role" "scheduler_cron_role" {
|
|
16
|
-
name =
|
|
26
|
+
name = local.scheduler_role_safe_name
|
|
17
27
|
permissions_boundary = local.preview_boundary_arn
|
|
18
28
|
assume_role_policy = jsonencode({
|
|
19
29
|
Version = "2012-10-17"
|
|
@@ -48,9 +58,18 @@ resource "aws_iam_role_policy" "scheduler_cron_policy" {
|
|
|
48
58
|
})
|
|
49
59
|
}
|
|
50
60
|
|
|
61
|
+
# --- Schedule group (failure metrics scope) ---
|
|
62
|
+
# Scheduler metrics dimension by group only (no per-schedule dimension),
|
|
63
|
+
# so the project's schedules get their own group instead of the shared
|
|
64
|
+
# default one — the alerts.tf failure alarms stay project-scoped.
|
|
65
|
+
resource "aws_scheduler_schedule_group" "crons" {
|
|
66
|
+
name = local.schedule_group_safe_name
|
|
67
|
+
}
|
|
68
|
+
|
|
51
69
|
# --- Scheduled invocation (single schedule per project) ---
|
|
52
70
|
resource "aws_scheduler_schedule" "cron" {
|
|
53
|
-
name =
|
|
71
|
+
name = local.schedule_safe_name
|
|
72
|
+
group_name = aws_scheduler_schedule_group.crons.name
|
|
54
73
|
schedule_expression = "{{SCHEDULE_EXPRESSION}}"
|
|
55
74
|
schedule_expression_timezone = "{{SCHEDULE_TIMEZONE}}"
|
|
56
75
|
# Sleep guard: stays disabled while grada sleep's auto-vars are active
|
|
@@ -5,9 +5,19 @@
|
|
|
5
5
|
# Billing: $0/mo fixed baseline (first 14M EventBridge Scheduler
|
|
6
6
|
# invocations/mo free); billed only for Fargate seconds while the task runs.
|
|
7
7
|
|
|
8
|
+
locals {
|
|
9
|
+
# Length-safe names: short names render byte-identically; over-cap
|
|
10
|
+
# names hash-suffix instead of failing validation. Mirrors
|
|
11
|
+
# safeSuffixedName in resolvers.js.
|
|
12
|
+
scheduler_role_safe_name = length("${local.app_name}-cron-scheduler-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 20)}-${substr(md5(local.app_name), 0, 5)}-cron-scheduler-role" : "${local.app_name}-cron-scheduler-role"
|
|
13
|
+
schedule_full_name = "${local.app_name}-cron-{{CRON_NAME}}"
|
|
14
|
+
schedule_safe_name = length(local.schedule_full_name) > 64 ? "${substr(local.schedule_full_name, 0, 64 - 1 - 5)}-${substr(md5(local.schedule_full_name), 0, 5)}" : local.schedule_full_name
|
|
15
|
+
schedule_group_safe_name = length("${local.app_name}-crons") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 6)}-${substr(md5(local.app_name), 0, 5)}-crons" : "${local.app_name}-crons"
|
|
16
|
+
}
|
|
17
|
+
|
|
8
18
|
# --- IAM Role assumed by EventBridge Scheduler ---
|
|
9
19
|
resource "aws_iam_role" "scheduler_cron_role" {
|
|
10
|
-
name =
|
|
20
|
+
name = local.scheduler_role_safe_name
|
|
11
21
|
permissions_boundary = local.preview_boundary_arn
|
|
12
22
|
assume_role_policy = jsonencode({
|
|
13
23
|
Version = "2012-10-17"
|
|
@@ -55,9 +65,18 @@ resource "aws_iam_role_policy" "scheduler_cron_policy" {
|
|
|
55
65
|
})
|
|
56
66
|
}
|
|
57
67
|
|
|
68
|
+
# --- Schedule group (failure metrics scope) ---
|
|
69
|
+
# Scheduler metrics dimension by group only (no per-schedule dimension),
|
|
70
|
+
# so the project's schedules get their own group instead of the shared
|
|
71
|
+
# default one — the alerts.tf failure alarms stay project-scoped.
|
|
72
|
+
resource "aws_scheduler_schedule_group" "crons" {
|
|
73
|
+
name = local.schedule_group_safe_name
|
|
74
|
+
}
|
|
75
|
+
|
|
58
76
|
# --- Scheduled task (single schedule per project) ---
|
|
59
77
|
resource "aws_scheduler_schedule" "cron" {
|
|
60
|
-
name =
|
|
78
|
+
name = local.schedule_safe_name
|
|
79
|
+
group_name = aws_scheduler_schedule_group.crons.name
|
|
61
80
|
schedule_expression = "{{SCHEDULE_EXPRESSION}}"
|
|
62
81
|
schedule_expression_timezone = "{{SCHEDULE_TIMEZONE}}"
|
|
63
82
|
# Sleep guard: stays disabled while grada sleep's auto-vars are active
|
|
@@ -12,6 +12,12 @@ resource "aws_s3_bucket" "storage" {
|
|
|
12
12
|
# within S3's 63-character limit. Underscores are replaced because S3 names
|
|
13
13
|
# must be DNS-compatible (lowercase, hyphens only).
|
|
14
14
|
bucket = "${trimsuffix(substr(replace(lower(local.app_name), "_", "-"), 0, 42), "-")}-storage-${data.aws_caller_identity.current.account_id}"
|
|
15
|
+
# Preview teardowns destroy raw (no CLI drain step runs there), so
|
|
16
|
+
# preview buckets self-clean — versions and lone delete markers
|
|
17
|
+
# included. Prod (default workspace) stays protected. Caveat: manual
|
|
18
|
+
# `terraform workspace new <name>` workspaces match too (accepted —
|
|
19
|
+
# manual-name hazards are tracked separately).
|
|
20
|
+
force_destroy = terraform.workspace != "default"
|
|
15
21
|
}
|
|
16
22
|
|
|
17
23
|
resource "aws_s3_bucket_public_access_block" "storage" {
|
|
@@ -27,6 +27,38 @@ resource "aws_ses_domain_mail_from" "ses" {
|
|
|
27
27
|
behavior_on_mx_failure = "UseDefaultValue"
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
# --- Feedback channel ---
|
|
31
|
+
# Bounces and complaints publish to CloudWatch (dashboard + alarms)
|
|
32
|
+
# instead of vanishing. The v2 sender identity sets this set as its
|
|
33
|
+
# default, so app mail needs no per-send configuration; addresses
|
|
34
|
+
# under the verified domain above verify automatically.
|
|
35
|
+
resource "aws_ses_configuration_set" "main" {
|
|
36
|
+
count = terraform.workspace == "default" ? 1 : 0
|
|
37
|
+
name = "${local.app_name}-ses-events"
|
|
38
|
+
reputation_metrics_enabled = true
|
|
39
|
+
sending_enabled = true
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
resource "aws_ses_event_destination" "cloudwatch" {
|
|
43
|
+
count = terraform.workspace == "default" ? 1 : 0
|
|
44
|
+
name = "cloudwatch"
|
|
45
|
+
configuration_set_name = aws_ses_configuration_set.main[0].name
|
|
46
|
+
enabled = true
|
|
47
|
+
matching_types = ["bounce", "complaint", "send"]
|
|
48
|
+
|
|
49
|
+
cloudwatch_destination {
|
|
50
|
+
default_value = "default"
|
|
51
|
+
dimension_name = "ses:configuration-set"
|
|
52
|
+
value_source = "messageTag"
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
resource "aws_sesv2_email_identity" "sender" {
|
|
57
|
+
count = terraform.workspace == "default" ? 1 : 0
|
|
58
|
+
email_identity = "{{SES_FROM_EMAIL}}"
|
|
59
|
+
configuration_set_name = aws_ses_configuration_set.main[0].name
|
|
60
|
+
}
|
|
61
|
+
|
|
30
62
|
{{SES_ROUTE53_RECORDS_BLOCK}}
|
|
31
63
|
# --- Least-Privilege IAM Policy for the ECS Task Role ---
|
|
32
64
|
resource "aws_iam_role_policy" "ses_send" {
|
|
@@ -64,6 +96,10 @@ output "ses_dkim_tokens" {
|
|
|
64
96
|
value = terraform.workspace == "default" ? aws_ses_domain_dkim.ses[0].dkim_tokens : []
|
|
65
97
|
}
|
|
66
98
|
|
|
99
|
+
output "ses_configuration_set_name" {
|
|
100
|
+
value = terraform.workspace == "default" ? aws_ses_configuration_set.main[0].name : null
|
|
101
|
+
}
|
|
102
|
+
|
|
67
103
|
output "ses_spf_record" {
|
|
68
104
|
value = "v=spf1 include:amazonses.com ~all"
|
|
69
105
|
}
|
|
@@ -19,6 +19,11 @@ resource "aws_security_group" "rds" {
|
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
# 2b. No SSL parameter group yet: this template floats its engine
|
|
23
|
+
# version by design, and a cluster parameter group family must match
|
|
24
|
+
# it exactly — forcing a family would pin the engine by side effect.
|
|
25
|
+
# Deferred until the engine-version decision; the instance engines
|
|
26
|
+
# already enforce TLS via their own groups.
|
|
22
27
|
# 3. The Aurora PostgreSQL Serverless v2 Cluster (scale-to-zero)
|
|
23
28
|
resource "aws_rds_cluster" "postgres" {
|
|
24
29
|
cluster_identifier = "${local.app_name}-db-cluster"
|
|
@@ -19,6 +19,21 @@ resource "aws_security_group" "rds" {
|
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
# 2b. TLS enforcement: clients connect with TLS (see the
|
|
23
|
+
# database-connections guide), so require it server-side. Pending
|
|
24
|
+
# reboot: existing trees enforce at the next reboot, never as a
|
|
25
|
+
# surprise mid-apply restart (see the SSL upgrade runbook).
|
|
26
|
+
resource "aws_db_parameter_group" "ssl" {
|
|
27
|
+
name = "${local.app_name}-mysql80"
|
|
28
|
+
family = "mysql8.0"
|
|
29
|
+
|
|
30
|
+
parameter {
|
|
31
|
+
name = "require_secure_transport"
|
|
32
|
+
value = "ON"
|
|
33
|
+
apply_method = "pending-reboot"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
22
37
|
# 3. The MySQL Instance
|
|
23
38
|
resource "aws_db_instance" "postgres" {
|
|
24
39
|
identifier = "${local.app_name}-db"
|
|
@@ -36,6 +51,9 @@ resource "aws_db_instance" "postgres" {
|
|
|
36
51
|
maintenance_window = "sun:04:00-sun:05:00"
|
|
37
52
|
storage_encrypted = true
|
|
38
53
|
|
|
54
|
+
# TLS enforcement (see the parameter group above).
|
|
55
|
+
parameter_group_name = aws_db_parameter_group.ssl.name
|
|
56
|
+
|
|
39
57
|
# Strip dashes for the database name: RDS MySQL db_name must begin with a
|
|
40
58
|
# letter and contain only alphanumeric characters (e.g. my-project -> myproject)
|
|
41
59
|
db_name = replace(local.app_name, "-", "")
|
|
@@ -19,6 +19,21 @@ resource "aws_security_group" "rds" {
|
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
# 2b. TLS enforcement: clients connect with TLS (see the
|
|
23
|
+
# database-connections guide), so require it server-side. Pending
|
|
24
|
+
# reboot: existing trees enforce at the next reboot, never as a
|
|
25
|
+
# surprise mid-apply restart (see the SSL upgrade runbook).
|
|
26
|
+
resource "aws_db_parameter_group" "ssl" {
|
|
27
|
+
name = "${local.app_name}-postgres16"
|
|
28
|
+
family = "postgres16"
|
|
29
|
+
|
|
30
|
+
parameter {
|
|
31
|
+
name = "rds.force_ssl"
|
|
32
|
+
value = "1"
|
|
33
|
+
apply_method = "pending-reboot"
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
22
37
|
# 3. The PostgreSQL Instance
|
|
23
38
|
resource "aws_db_instance" "postgres" {
|
|
24
39
|
identifier = "${local.app_name}-db"
|
|
@@ -36,6 +51,9 @@ resource "aws_db_instance" "postgres" {
|
|
|
36
51
|
maintenance_window = "sun:04:00-sun:05:00"
|
|
37
52
|
storage_encrypted = true
|
|
38
53
|
|
|
54
|
+
# TLS enforcement (see the parameter group above).
|
|
55
|
+
parameter_group_name = aws_db_parameter_group.ssl.name
|
|
56
|
+
|
|
39
57
|
# Clean up dashes for the database name (e.g. my-project -> my_project)
|
|
40
58
|
db_name = replace(local.app_name, "-", "_")
|
|
41
59
|
username = "dbadmin"
|
|
@@ -40,6 +40,12 @@ locals {
|
|
|
40
40
|
|
|
41
41
|
# Full application name: project + workspace + collision suffix.
|
|
42
42
|
app_name = "{{PROJECT_NAME}}${local.env_suffix}${local.grada_resource_suffix}"
|
|
43
|
+
|
|
44
|
+
# Length-safe names: short names render byte-identically; over-cap
|
|
45
|
+
# names hash-suffix instead of failing IAM validation. Mirrors
|
|
46
|
+
# safeSuffixedName in resolvers.js.
|
|
47
|
+
task_role_safe_name = length("${local.app_name}-task-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 10)}-${substr(md5(local.app_name), 0, 5)}-task-role" : "${local.app_name}-task-role"
|
|
48
|
+
execution_role_safe_name = length("${local.app_name}-execution-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 16)}-${substr(md5(local.app_name), 0, 5)}-execution-role" : "${local.app_name}-execution-role"
|
|
43
49
|
}
|
|
44
50
|
|
|
45
51
|
# --- CloudWatch Logs ---
|
|
@@ -85,7 +91,7 @@ data "aws_iam_policy_document" "ecs_trust" {
|
|
|
85
91
|
}
|
|
86
92
|
|
|
87
93
|
resource "aws_iam_role" "execution_role" {
|
|
88
|
-
name =
|
|
94
|
+
name = local.execution_role_safe_name
|
|
89
95
|
assume_role_policy = data.aws_iam_policy_document.ecs_trust.json
|
|
90
96
|
permissions_boundary = local.preview_boundary_arn
|
|
91
97
|
}
|
|
@@ -110,7 +116,7 @@ data "aws_iam_policy_document" "lambda_task_trust" {
|
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
resource "aws_iam_role" "task_role" {
|
|
113
|
-
name =
|
|
119
|
+
name = local.task_role_safe_name
|
|
114
120
|
assume_role_policy = data.aws_iam_policy_document.lambda_task_trust.json
|
|
115
121
|
permissions_boundary = local.preview_boundary_arn
|
|
116
122
|
}
|
|
@@ -50,6 +50,16 @@ locals {
|
|
|
50
50
|
|
|
51
51
|
# Full application name: project + workspace + collision suffix.
|
|
52
52
|
app_name = "{{PROJECT_NAME}}${local.env_suffix}${local.grada_resource_suffix}"
|
|
53
|
+
|
|
54
|
+
# Length-safe names: short names render byte-identically; over-cap
|
|
55
|
+
# names hash-suffix (deterministic md5 stem) instead of truncating
|
|
56
|
+
# into collisions (ALB/TG) or failing validation (IAM). Mirrors
|
|
57
|
+
# safeSuffixedName in resolvers.js — stem is always
|
|
58
|
+
# cap - 1 - 5 - len(suffix). Existing trees never rename.
|
|
59
|
+
alb_safe_name = length("${local.app_name}-alb") > 32 ? "${substr(local.app_name, 0, 32 - 1 - 5 - 4)}-${substr(md5(local.app_name), 0, 5)}-alb" : "${local.app_name}-alb"
|
|
60
|
+
tg_safe_name = length("${local.app_name}-tg") > 32 ? "${substr(local.app_name, 0, 32 - 1 - 5 - 3)}-${substr(md5(local.app_name), 0, 5)}-tg" : "${local.app_name}-tg"
|
|
61
|
+
task_role_safe_name = length("${local.app_name}-task-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 10)}-${substr(md5(local.app_name), 0, 5)}-task-role" : "${local.app_name}-task-role"
|
|
62
|
+
execution_role_safe_name = length("${local.app_name}-execution-role") > 64 ? "${substr(local.app_name, 0, 64 - 1 - 5 - 16)}-${substr(md5(local.app_name), 0, 5)}-execution-role" : "${local.app_name}-execution-role"
|
|
53
63
|
}
|
|
54
64
|
|
|
55
65
|
# --- CloudWatch Logs ---
|
|
@@ -94,7 +104,7 @@ data "aws_iam_policy_document" "ecs_trust" {
|
|
|
94
104
|
}
|
|
95
105
|
|
|
96
106
|
resource "aws_iam_role" "execution_role" {
|
|
97
|
-
name =
|
|
107
|
+
name = local.execution_role_safe_name
|
|
98
108
|
assume_role_policy = data.aws_iam_policy_document.ecs_trust.json
|
|
99
109
|
permissions_boundary = local.preview_boundary_arn
|
|
100
110
|
}
|
|
@@ -107,7 +117,7 @@ resource "aws_iam_role_policy_attachment" "execution_role_policy" {
|
|
|
107
117
|
# --- IAM: Task Role ---
|
|
108
118
|
# Allows your application code running INSIDE the container to access AWS services
|
|
109
119
|
resource "aws_iam_role" "task_role" {
|
|
110
|
-
name =
|
|
120
|
+
name = local.task_role_safe_name
|
|
111
121
|
assume_role_policy = data.aws_iam_policy_document.ecs_trust.json
|
|
112
122
|
permissions_boundary = local.preview_boundary_arn
|
|
113
123
|
}
|
|
@@ -206,7 +216,7 @@ resource "aws_ecs_task_definition" "app" {
|
|
|
206
216
|
# --- Application Load Balancer ---
|
|
207
217
|
# trivy:ignore:AVD-AWS-0053 - This ALB is intended to be publicly facing behind CloudFront
|
|
208
218
|
resource "aws_lb" "main" {
|
|
209
|
-
name =
|
|
219
|
+
name = local.alb_safe_name
|
|
210
220
|
load_balancer_type = "application"
|
|
211
221
|
security_groups = [aws_security_group.alb.id]
|
|
212
222
|
subnets = aws_subnet.public[*].id
|
|
@@ -214,7 +224,7 @@ resource "aws_lb" "main" {
|
|
|
214
224
|
}
|
|
215
225
|
|
|
216
226
|
resource "aws_lb_target_group" "app" {
|
|
217
|
-
name =
|
|
227
|
+
name = local.tg_safe_name
|
|
218
228
|
port = {{PORT}}
|
|
219
229
|
protocol = "HTTP"
|
|
220
230
|
vpc_id = aws_vpc.main.id
|
|
@@ -49,7 +49,9 @@ locals {
|
|
|
49
49
|
# ${prod_prefix}-* pattern matches prod only. Anchoring also keeps
|
|
50
50
|
# the boundary under the 6144-char managed-policy quota that broke
|
|
51
51
|
# tier1 E2E at a 24-char prefix — prefer one anchored wildcard over
|
|
52
|
-
# several exact ARNs when adding entries.
|
|
52
|
+
# several exact ARNs when adding entries. Headroom is now single
|
|
53
|
+
# digits at 32-char prefixes: the next IAM addition must economize
|
|
54
|
+
# (shorter Sids, fewer verbs, or slimmer patterns) before extending.
|
|
53
55
|
guardrail_compute_abuse_resources = ["*"]
|
|
54
56
|
guardrail_snapshot_sharing_resources = ["*"]
|
|
55
57
|
guardrail_state_destruction_resources = [
|
|
@@ -110,6 +112,37 @@ locals {
|
|
|
110
112
|
guardrail_app_role_resources = [
|
|
111
113
|
"arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.prod_prefix}-*-role",
|
|
112
114
|
]
|
|
115
|
+
# Data-plane confinement: prod data ARNs shared by the
|
|
116
|
+
# DenyProdAppData twins. One statement is exact here, not
|
|
117
|
+
# lazy — every denied action only ever matches its own service's
|
|
118
|
+
# ARNs in a real call, so the cross-product cannot over-match.
|
|
119
|
+
guardrail_app_data_resources = [
|
|
120
|
+
"arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:${local.prod_prefix}-secrets-*",
|
|
121
|
+
"arn:aws:sqs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:${local.prod_prefix}-*",
|
|
122
|
+
]
|
|
123
|
+
# Lifecycle twin of the above, deliberately carve-free: no bounded
|
|
124
|
+
# role ever stops or reboots a database (sleep/wake and db commands
|
|
125
|
+
# run on user credentials), and one fewer condition keeps the pair
|
|
126
|
+
# under the 6144-char boundary quota.
|
|
127
|
+
guardrail_db_lifecycle_resources = [
|
|
128
|
+
"arn:aws:rds:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:db:${local.prod_prefix}-db*",
|
|
129
|
+
"arn:aws:rds:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:cluster:${local.prod_prefix}-db-cluster",
|
|
130
|
+
]
|
|
131
|
+
# Carve-out principal for the app-data twins: the exact prod task
|
|
132
|
+
# role ARN (workers send/receive prod queues). Singular on purpose:
|
|
133
|
+
# execution resolves taskdef secrets via GetSecretValue, which no
|
|
134
|
+
# statement here denies (plan refresh requires it — provider-proven),
|
|
135
|
+
# and nothing bounded writes secrets — so nothing else needs
|
|
136
|
+
# carving. Exact ARN, never a wildcard: a backdoor role could match
|
|
137
|
+
# a pattern by name since CreateRole sets its own trust inline.
|
|
138
|
+
# String-built, so static trees (no roles at all) stay valid. The
|
|
139
|
+
# scheduler is intentionally absent: RunTask plus PassRole only.
|
|
140
|
+
# Length-conditional like the role itself: over-cap trees hash the
|
|
141
|
+
# task name, and the carve-out must track the rendered name exactly
|
|
142
|
+
# (a stale plain ARN would silently uncarve workers).
|
|
143
|
+
guardrail_task_role_principals = [
|
|
144
|
+
length("${local.prod_prefix}-task-role") > 64 ? "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${substr(local.prod_prefix, 0, 64 - 1 - 5 - 10)}-${substr(md5(local.prod_prefix), 0, 5)}-task-role" : "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${local.prod_prefix}-task-role",
|
|
145
|
+
]
|
|
113
146
|
}
|
|
114
147
|
|
|
115
148
|
# Deploy role FIRST: readTerraformOidcTrust anchors on this block so re-init
|
|
@@ -430,6 +463,43 @@ resource "aws_iam_role_policy" "preview_guardrails" {
|
|
|
430
463
|
Action = ["iam:AttachRolePolicy", "iam:DetachRolePolicy", "iam:PutRolePolicy", "iam:DeleteRolePolicy", "iam:UpdateAssumeRolePolicy", "iam:DeleteRole"]
|
|
431
464
|
Resource = local.guardrail_app_role_resources
|
|
432
465
|
},
|
|
466
|
+
{
|
|
467
|
+
# Preview CI must never write prod secrets or touch prod queue
|
|
468
|
+
# traffic. GetSecretValue stays allowed: plan refreshes the
|
|
469
|
+
# version resource with preview credentials, and the provider
|
|
470
|
+
# reads it via GetSecretValue (provider source verified) — a
|
|
471
|
+
# deny would break every PR plan. Reads stay an accepted
|
|
472
|
+
# residual (see below), as does DeleteMessage, which needs a
|
|
473
|
+
# Receive handle to call: with Receive denied it is unreachable.
|
|
474
|
+
# The carve-out is mirrored here as a no-op (the preview role
|
|
475
|
+
# is never a task role) so the twins stay identical.
|
|
476
|
+
# Accepted residual: GetSecretValue + prod state reads (plan
|
|
477
|
+
# refresh requires both; state additionally exposes adopted
|
|
478
|
+
# live values), snapshot export/restore, Bedrock/SES spend
|
|
479
|
+
# verbs, DynamoDB data verbs, and RDS Start — follow-ups, not
|
|
480
|
+
# oversights.
|
|
481
|
+
Sid = "DenyProdAppData"
|
|
482
|
+
Effect = "Deny"
|
|
483
|
+
Action = [
|
|
484
|
+
"secretsmanager:Put*", "secretsmanager:Update*",
|
|
485
|
+
"sqs:Send*", "sqs:Receive*",
|
|
486
|
+
]
|
|
487
|
+
Resource = local.guardrail_app_data_resources
|
|
488
|
+
Condition = {
|
|
489
|
+
StringNotEquals = {
|
|
490
|
+
"aws:PrincipalArn" = local.guardrail_task_role_principals
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
},
|
|
494
|
+
{
|
|
495
|
+
# ...nor stop or reboot prod databases. Deliberately
|
|
496
|
+
# carve-free: no bounded role ever calls these (sleep/wake
|
|
497
|
+
# and db commands run on user credentials).
|
|
498
|
+
Sid = "DenyProdDbLifecycle"
|
|
499
|
+
Effect = "Deny"
|
|
500
|
+
Action = ["rds:Stop*", "rds:Reboot*"]
|
|
501
|
+
Resource = local.guardrail_db_lifecycle_resources
|
|
502
|
+
},
|
|
433
503
|
]
|
|
434
504
|
})
|
|
435
505
|
}
|
|
@@ -584,6 +654,29 @@ resource "aws_iam_policy" "preview_boundary" {
|
|
|
584
654
|
Action = ["iam:AttachRolePolicy", "iam:DetachRolePolicy", "iam:PutRolePolicy", "iam:DeleteRolePolicy", "iam:UpdateAssumeRolePolicy", "iam:DeleteRole"]
|
|
585
655
|
Resource = local.guardrail_app_role_resources
|
|
586
656
|
},
|
|
657
|
+
{
|
|
658
|
+
# Backdoor twin of DenyProdAppData: roles born under preview
|
|
659
|
+
# credentials get no prod vault writes or queue traffic either.
|
|
660
|
+
# The exact-ARN carve-out keeps workers sending/receiving.
|
|
661
|
+
Sid = "BoundaryDenyAppData"
|
|
662
|
+
Effect = "Deny"
|
|
663
|
+
Action = [
|
|
664
|
+
"secretsmanager:Put*", "secretsmanager:Update*",
|
|
665
|
+
"sqs:Send*", "sqs:Receive*",
|
|
666
|
+
]
|
|
667
|
+
Resource = local.guardrail_app_data_resources
|
|
668
|
+
Condition = {
|
|
669
|
+
StringNotEquals = {
|
|
670
|
+
"aws:PrincipalArn" = local.guardrail_task_role_principals
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
},
|
|
674
|
+
{
|
|
675
|
+
Sid = "BoundaryDenyDbLifecycle"
|
|
676
|
+
Effect = "Deny"
|
|
677
|
+
Action = ["rds:Stop*", "rds:Reboot*"]
|
|
678
|
+
Resource = local.guardrail_db_lifecycle_resources
|
|
679
|
+
},
|
|
587
680
|
{
|
|
588
681
|
# Carved PassRole may only pass to the ECS tasks
|
|
589
682
|
# service (mirrors the scheduler policy's own condition).
|