grada-run 0.49.10 → 0.50.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/commands/alerts.js +275 -57
- package/src/commands/db/migrate.js +1 -1
- package/src/commands/status.js +84 -6
- package/src/data/action-pins.js +1 -1
- package/src/utils/golden-signals.js +4 -4
- package/src/utils/resolvers.js +16 -1
- package/templates/github/deploy-lambda.yml +4 -1
- package/templates/github/deploy-static.yml +1 -1
- package/templates/github/deploy.yml +4 -1
- package/templates/github/drift.yml +71 -0
- package/templates/github/preview.yml +69 -0
- package/templates/terraform/addons/bedrock.tf +3 -3
- package/templates/terraform/addons/cron-lambda.tf +21 -2
- package/templates/terraform/addons/cron.tf +21 -2
- package/templates/terraform/addons/s3.tf +6 -0
- package/templates/terraform/addons/ses.tf +36 -0
- package/templates/terraform/database-aurora-postgresql.tf +5 -0
- package/templates/terraform/database-mysql.tf +18 -0
- package/templates/terraform/database.tf +18 -0
- package/templates/terraform/main-lambda.tf +8 -2
- package/templates/terraform/main.tf +14 -4
- package/templates/terraform/oidc.tf +94 -1
package/package.json
CHANGED
package/src/commands/alerts.js
CHANGED
|
@@ -52,6 +52,30 @@ const TARGET_ANCHORS = {
|
|
|
52
52
|
static: { resource: 'resource "aws_cloudfront_distribution" "site"', errorCode: 'DISTRIBUTION_NOT_FOUND', reason: 'distribution-not-found' },
|
|
53
53
|
};
|
|
54
54
|
|
|
55
|
+
// Addon alarm anchors. Detection reads the addon FILE CONTENT (not just
|
|
56
|
+
// its presence) so a hand-removed resource can't produce a dangling
|
|
57
|
+
// alarm reference. Several anchors may share one id (either engine
|
|
58
|
+
// file lights up 'db:instance').
|
|
59
|
+
const ADDON_ALARM_ANCHORS = [
|
|
60
|
+
{ id: 'queue:sqs', file: 'sqs.tf', resource: 'resource "aws_sqs_queue" "dlq"' },
|
|
61
|
+
{ id: 'cron', file: 'cron.tf', resource: 'resource "aws_scheduler_schedule" "cron"' },
|
|
62
|
+
{ id: 'db:instance', file: 'database.tf', resource: 'resource "aws_db_instance" "postgres"' },
|
|
63
|
+
{ id: 'db:instance', file: 'database-mysql.tf', resource: 'resource "aws_db_instance" "postgres"' },
|
|
64
|
+
{ id: 'db:aurora', file: 'database-aurora-postgresql.tf', resource: 'resource "aws_rds_cluster" "postgres"' },
|
|
65
|
+
{ id: 'db:redis', file: 'redis.tf', resource: 'resource "aws_elasticache_replication_group" "redis"' },
|
|
66
|
+
{ id: 'email:ses', file: 'ses.tf', resource: 'resource "aws_ses_domain_identity" "ses"' },
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
export function detectAlertAddonFeatures(terraformDir) {
|
|
70
|
+
const features = {};
|
|
71
|
+
for (const anchor of ADDON_ALARM_ANCHORS) {
|
|
72
|
+
if (features[anchor.id]) continue;
|
|
73
|
+
const content = readFileSafe(path.join(terraformDir, anchor.file));
|
|
74
|
+
features[anchor.id] = Boolean(content && content.includes(anchor.resource));
|
|
75
|
+
}
|
|
76
|
+
return features;
|
|
77
|
+
}
|
|
78
|
+
|
|
55
79
|
export function parseAlertsArgs(argv = []) {
|
|
56
80
|
const args = normalizeArgv(argv);
|
|
57
81
|
if (args[0] === 'alerts') args.shift();
|
|
@@ -183,7 +207,228 @@ exports.handler = async (event) => {
|
|
|
183
207
|
// so every static alert resource is pinned to the aws.us_east_1 alias
|
|
184
208
|
// (runAlerts ensures main.tf declares it). IAM and archive_file are
|
|
185
209
|
// region-independent and stay unpinned.
|
|
186
|
-
|
|
210
|
+
// Primary notify-alarm spec per compute target. Addon alarms (DLQ
|
|
211
|
+
// depth, cron health, ...) append as further entries consumed the same
|
|
212
|
+
// way; each spec carries its own baked threshold while --threshold
|
|
213
|
+
// keeps driving the primary alarm only.
|
|
214
|
+
const PRIMARY_NOTIFY_ALARMS = {
|
|
215
|
+
lambda: {
|
|
216
|
+
tfName: 'notify_errors',
|
|
217
|
+
alarmName: 'errors-notify',
|
|
218
|
+
metricName: 'Errors',
|
|
219
|
+
namespace: 'AWS/Lambda',
|
|
220
|
+
statistic: 'Sum',
|
|
221
|
+
describe: ({ projectName, limit, evaluationPeriods }) =>
|
|
222
|
+
`${projectName} reports more than ${limit} Lambda errors in ${evaluationPeriods} minutes.`,
|
|
223
|
+
dimensions: { FunctionName: 'aws_lambda_function.app.function_name' },
|
|
224
|
+
},
|
|
225
|
+
static: {
|
|
226
|
+
tfName: 'notify_5xx_rate',
|
|
227
|
+
alarmName: '5xx-rate-notify',
|
|
228
|
+
metricName: '5xxErrorRate',
|
|
229
|
+
namespace: 'AWS/CloudFront',
|
|
230
|
+
statistic: 'Average',
|
|
231
|
+
describe: ({ projectName, limit, evaluationPeriods }) =>
|
|
232
|
+
`${projectName} serves more than a ${limit}% 5xx error rate over ${evaluationPeriods} minutes.`,
|
|
233
|
+
dimensions: { DistributionId: 'aws_cloudfront_distribution.site.id' },
|
|
234
|
+
},
|
|
235
|
+
ecs: {
|
|
236
|
+
tfName: 'notify_5xx',
|
|
237
|
+
alarmName: '5xx-notify',
|
|
238
|
+
metricName: 'HTTPCode_Target_5XX_Count',
|
|
239
|
+
namespace: 'AWS/ApplicationELB',
|
|
240
|
+
statistic: 'Sum',
|
|
241
|
+
describe: ({ limit, evaluationPeriods }) =>
|
|
242
|
+
`the ALB serves more than ${limit} 5XX errors in ${evaluationPeriods} minutes.`,
|
|
243
|
+
dimensions: { LoadBalancer: 'aws_lb.main.arn_suffix' },
|
|
244
|
+
},
|
|
245
|
+
};
|
|
246
|
+
|
|
247
|
+
// Renders one notify alarm block. Shape is shared by primary and addon
|
|
248
|
+
// alarms so every alarm reads (and diffs) the same way.
|
|
249
|
+
function renderNotifyAlarm(spec, { namePrefix, aliasLine, limit, evaluationPeriods, period, projectName }) {
|
|
250
|
+
const dimensions = spec.dimensions
|
|
251
|
+
? `\n dimensions = {\n${Object.entries(spec.dimensions).map(([key, value]) => ` ${key} = ${value}`).join('\n')}\n }\n`
|
|
252
|
+
: '';
|
|
253
|
+
return `resource "aws_cloudwatch_metric_alarm" "${spec.tfName}" {
|
|
254
|
+
${aliasLine} alarm_name = "${namePrefix}-${spec.alarmName}"
|
|
255
|
+
comparison_operator = "${spec.comparison || 'GreaterThanThreshold'}"
|
|
256
|
+
evaluation_periods = "${evaluationPeriods}"
|
|
257
|
+
metric_name = "${spec.metricName}"
|
|
258
|
+
namespace = "${spec.namespace}"
|
|
259
|
+
period = "${period}"
|
|
260
|
+
statistic = "${spec.statistic}"
|
|
261
|
+
threshold = "${limit}"
|
|
262
|
+
alarm_description = "Notifies ${namePrefix}-alerts when ${spec.describe({ projectName, limit, evaluationPeriods })}"
|
|
263
|
+
|
|
264
|
+
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
265
|
+
${dimensions}}` + '\n';
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// Addon notify alarms, rendered only when the tree carries the addon
|
|
269
|
+
// (see detectAlertAddonFeatures). Thresholds are baked per alarm;
|
|
270
|
+
// --threshold keeps driving the primary alarm only.
|
|
271
|
+
const ADDON_NOTIFY_ALARMS = [
|
|
272
|
+
{
|
|
273
|
+
id: 'queue:sqs',
|
|
274
|
+
tfName: 'notify_dlq_depth',
|
|
275
|
+
alarmName: 'dlq-depth-notify',
|
|
276
|
+
metricName: 'ApproximateNumberOfMessagesVisible',
|
|
277
|
+
namespace: 'AWS/SQS',
|
|
278
|
+
statistic: 'Maximum',
|
|
279
|
+
threshold: 1,
|
|
280
|
+
evaluationPeriods: 1,
|
|
281
|
+
period: 300,
|
|
282
|
+
describe: () => 'the dead-letter queue holds failed messages (redrive or inspect them).',
|
|
283
|
+
dimensions: { QueueName: 'aws_sqs_queue.dlq.name' },
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
id: 'cron',
|
|
287
|
+
tfName: 'notify_cron_target_errors',
|
|
288
|
+
alarmName: 'cron-target-errors-notify',
|
|
289
|
+
metricName: 'TargetErrorCount',
|
|
290
|
+
namespace: 'AWS/Scheduler',
|
|
291
|
+
statistic: 'Sum',
|
|
292
|
+
threshold: 1,
|
|
293
|
+
evaluationPeriods: 1,
|
|
294
|
+
period: 300,
|
|
295
|
+
describe: () => 'a scheduled run fails to invoke its target.',
|
|
296
|
+
dimensions: { ScheduleGroup: 'aws_scheduler_schedule_group.crons.name' },
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
id: 'cron',
|
|
300
|
+
tfName: 'notify_cron_dropped',
|
|
301
|
+
alarmName: 'cron-dropped-notify',
|
|
302
|
+
metricName: 'InvocationDroppedCount',
|
|
303
|
+
namespace: 'AWS/Scheduler',
|
|
304
|
+
statistic: 'Sum',
|
|
305
|
+
threshold: 1,
|
|
306
|
+
evaluationPeriods: 1,
|
|
307
|
+
period: 300,
|
|
308
|
+
describe: () => 'a scheduled run is dropped after retry exhaustion.',
|
|
309
|
+
dimensions: { ScheduleGroup: 'aws_scheduler_schedule_group.crons.name' },
|
|
310
|
+
},
|
|
311
|
+
{
|
|
312
|
+
id: 'db:instance',
|
|
313
|
+
tfName: 'notify_rds_storage_low',
|
|
314
|
+
alarmName: 'rds-storage-low-notify',
|
|
315
|
+
metricName: 'FreeStorageSpace',
|
|
316
|
+
namespace: 'AWS/RDS',
|
|
317
|
+
statistic: 'Minimum',
|
|
318
|
+
comparison: 'LessThanThreshold',
|
|
319
|
+
threshold: 2147483648,
|
|
320
|
+
evaluationPeriods: 1,
|
|
321
|
+
period: 300,
|
|
322
|
+
describe: () => 'the database drops below 2 GB of free storage.',
|
|
323
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
id: 'db:instance',
|
|
327
|
+
tfName: 'notify_rds_cpu_high',
|
|
328
|
+
alarmName: 'rds-cpu-high-notify',
|
|
329
|
+
metricName: 'CPUUtilization',
|
|
330
|
+
namespace: 'AWS/RDS',
|
|
331
|
+
statistic: 'Average',
|
|
332
|
+
threshold: 80,
|
|
333
|
+
evaluationPeriods: 3,
|
|
334
|
+
period: 300,
|
|
335
|
+
describe: () => 'database CPU stays above 80% for 15 minutes.',
|
|
336
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
337
|
+
},
|
|
338
|
+
{
|
|
339
|
+
id: 'db:instance',
|
|
340
|
+
tfName: 'notify_rds_connections_high',
|
|
341
|
+
alarmName: 'rds-connections-high-notify',
|
|
342
|
+
metricName: 'DatabaseConnections',
|
|
343
|
+
namespace: 'AWS/RDS',
|
|
344
|
+
statistic: 'Maximum',
|
|
345
|
+
threshold: 80,
|
|
346
|
+
evaluationPeriods: 2,
|
|
347
|
+
period: 300,
|
|
348
|
+
describe: () => 'database connections exceed 80 (micro exhausts near 112).',
|
|
349
|
+
dimensions: { DBInstanceIdentifier: 'aws_db_instance.postgres.identifier' },
|
|
350
|
+
},
|
|
351
|
+
{
|
|
352
|
+
id: 'db:aurora',
|
|
353
|
+
tfName: 'notify_rds_cpu_high',
|
|
354
|
+
alarmName: 'rds-cpu-high-notify',
|
|
355
|
+
metricName: 'CPUUtilization',
|
|
356
|
+
namespace: 'AWS/RDS',
|
|
357
|
+
statistic: 'Average',
|
|
358
|
+
threshold: 80,
|
|
359
|
+
evaluationPeriods: 3,
|
|
360
|
+
period: 300,
|
|
361
|
+
describe: () => 'database CPU stays above 80% for 15 minutes.',
|
|
362
|
+
dimensions: { DBClusterIdentifier: 'aws_rds_cluster.postgres.id' },
|
|
363
|
+
},
|
|
364
|
+
{
|
|
365
|
+
id: 'db:aurora',
|
|
366
|
+
tfName: 'notify_rds_connections_high',
|
|
367
|
+
alarmName: 'rds-connections-high-notify',
|
|
368
|
+
metricName: 'DatabaseConnections',
|
|
369
|
+
namespace: 'AWS/RDS',
|
|
370
|
+
statistic: 'Maximum',
|
|
371
|
+
threshold: 80,
|
|
372
|
+
evaluationPeriods: 2,
|
|
373
|
+
period: 300,
|
|
374
|
+
describe: () => 'database connections exceed 80.',
|
|
375
|
+
dimensions: { DBClusterIdentifier: 'aws_rds_cluster.postgres.id' },
|
|
376
|
+
},
|
|
377
|
+
{
|
|
378
|
+
id: 'db:redis',
|
|
379
|
+
tfName: 'notify_valkey_evictions',
|
|
380
|
+
alarmName: 'valkey-evictions-notify',
|
|
381
|
+
metricName: 'Evictions',
|
|
382
|
+
namespace: 'AWS/ElastiCache',
|
|
383
|
+
statistic: 'Sum',
|
|
384
|
+
threshold: 1,
|
|
385
|
+
evaluationPeriods: 1,
|
|
386
|
+
period: 300,
|
|
387
|
+
describe: () => 'valkey evicts keys (dedup/rate windows lose entries).',
|
|
388
|
+
dimensions: { ReplicationGroupId: 'aws_elasticache_replication_group.redis.id' },
|
|
389
|
+
},
|
|
390
|
+
{
|
|
391
|
+
id: 'db:redis',
|
|
392
|
+
tfName: 'notify_valkey_cpu_high',
|
|
393
|
+
alarmName: 'valkey-cpu-high-notify',
|
|
394
|
+
metricName: 'EngineCPUUtilization',
|
|
395
|
+
namespace: 'AWS/ElastiCache',
|
|
396
|
+
statistic: 'Average',
|
|
397
|
+
threshold: 80,
|
|
398
|
+
evaluationPeriods: 3,
|
|
399
|
+
period: 300,
|
|
400
|
+
describe: () => 'valkey engine CPU stays above 80% for 15 minutes.',
|
|
401
|
+
dimensions: { ReplicationGroupId: 'aws_elasticache_replication_group.redis.id' },
|
|
402
|
+
},
|
|
403
|
+
{
|
|
404
|
+
id: 'email:ses',
|
|
405
|
+
tfName: 'notify_ses_bounce_rate',
|
|
406
|
+
alarmName: 'ses-bounce-rate-notify',
|
|
407
|
+
metricName: 'Reputation.BounceRate',
|
|
408
|
+
namespace: 'AWS/SES',
|
|
409
|
+
statistic: 'Average',
|
|
410
|
+
threshold: 0.05,
|
|
411
|
+
evaluationPeriods: 1,
|
|
412
|
+
period: 3600,
|
|
413
|
+
describe: () => 'the account bounce rate exceeds 5%.',
|
|
414
|
+
dimensions: null,
|
|
415
|
+
},
|
|
416
|
+
{
|
|
417
|
+
id: 'email:ses',
|
|
418
|
+
tfName: 'notify_ses_complaint_rate',
|
|
419
|
+
alarmName: 'ses-complaint-rate-notify',
|
|
420
|
+
metricName: 'Reputation.ComplaintRate',
|
|
421
|
+
namespace: 'AWS/SES',
|
|
422
|
+
statistic: 'Average',
|
|
423
|
+
threshold: 0.001,
|
|
424
|
+
evaluationPeriods: 1,
|
|
425
|
+
period: 3600,
|
|
426
|
+
describe: () => 'the account complaint rate exceeds 0.1%.',
|
|
427
|
+
dimensions: null,
|
|
428
|
+
},
|
|
429
|
+
];
|
|
430
|
+
|
|
431
|
+
export function buildAlertsTf({ projectName, target = 'ecs', email, webhook, threshold, addons = {} }) {
|
|
187
432
|
const defaults = ALERT_TARGET_DEFAULTS[target] || ALERT_TARGET_DEFAULTS.ecs;
|
|
188
433
|
const limit = threshold ?? defaults.threshold;
|
|
189
434
|
// All targets use the workspace-aware local: flat ECS names let a
|
|
@@ -212,58 +457,29 @@ ${aliasLine} name = "${namePrefix}-alerts"
|
|
|
212
457
|
}
|
|
213
458
|
`;
|
|
214
459
|
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
period = "${defaults.period}"
|
|
239
|
-
statistic = "Average"
|
|
240
|
-
threshold = "${limit}"
|
|
241
|
-
alarm_description = "Notifies ${namePrefix}-alerts when ${projectName} serves more than a ${limit}% 5xx error rate over ${defaults.evaluationPeriods} minutes."
|
|
242
|
-
|
|
243
|
-
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
244
|
-
|
|
245
|
-
dimensions = {
|
|
246
|
-
DistributionId = aws_cloudfront_distribution.site.id
|
|
247
|
-
}
|
|
248
|
-
}
|
|
249
|
-
` : `resource "aws_cloudwatch_metric_alarm" "notify_5xx" {
|
|
250
|
-
alarm_name = "${namePrefix}-5xx-notify"
|
|
251
|
-
comparison_operator = "GreaterThanThreshold"
|
|
252
|
-
evaluation_periods = "${defaults.evaluationPeriods}"
|
|
253
|
-
metric_name = "HTTPCode_Target_5XX_Count"
|
|
254
|
-
namespace = "AWS/ApplicationELB"
|
|
255
|
-
period = "${defaults.period}"
|
|
256
|
-
statistic = "Sum"
|
|
257
|
-
threshold = "${limit}"
|
|
258
|
-
alarm_description = "Notifies ${namePrefix}-alerts when the ALB serves more than ${limit} 5XX errors in ${defaults.evaluationPeriods} minutes."
|
|
259
|
-
|
|
260
|
-
alarm_actions = [aws_sns_topic.alerts.arn]
|
|
261
|
-
|
|
262
|
-
dimensions = {
|
|
263
|
-
LoadBalancer = aws_lb.main.arn_suffix
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
`;
|
|
460
|
+
const spec = PRIMARY_NOTIFY_ALARMS[target] || PRIMARY_NOTIFY_ALARMS.ecs;
|
|
461
|
+
const alarm = renderNotifyAlarm(spec, {
|
|
462
|
+
namePrefix,
|
|
463
|
+
aliasLine,
|
|
464
|
+
limit,
|
|
465
|
+
evaluationPeriods: defaults.evaluationPeriods,
|
|
466
|
+
period: defaults.period,
|
|
467
|
+
projectName,
|
|
468
|
+
});
|
|
469
|
+
// Addon alarms are regional metrics (no us-east-1 alias even on
|
|
470
|
+
// static trees) with baked thresholds.
|
|
471
|
+
const addonAlarms = ADDON_NOTIFY_ALARMS
|
|
472
|
+
.filter((entry) => addons && addons[entry.id])
|
|
473
|
+
.map((entry) => renderNotifyAlarm(entry, {
|
|
474
|
+
namePrefix,
|
|
475
|
+
aliasLine: '',
|
|
476
|
+
limit: entry.threshold,
|
|
477
|
+
evaluationPeriods: entry.evaluationPeriods,
|
|
478
|
+
period: entry.period,
|
|
479
|
+
projectName,
|
|
480
|
+
}))
|
|
481
|
+
.join('\n');
|
|
482
|
+
const addonSection = addonAlarms.length > 0 ? `\n${addonAlarms}` : '';
|
|
267
483
|
|
|
268
484
|
// NOTE: the email endpoint lands in git by design (the operator passed
|
|
269
485
|
// it explicitly); delivery still needs the recipient's confirmation
|
|
@@ -354,7 +570,7 @@ ${aliasLine} topic_arn = aws_sns_topic.alerts.arn
|
|
|
354
570
|
|
|
355
571
|
return `${header}
|
|
356
572
|
${topic}
|
|
357
|
-
${alarm}${emailSubscription}${forwarder}`;
|
|
573
|
+
${alarm}${addonSection}${emailSubscription}${forwarder}`;
|
|
358
574
|
}
|
|
359
575
|
|
|
360
576
|
// Adds the archive provider (used by the forwarder's data.archive_file)
|
|
@@ -458,6 +674,8 @@ async function runAlertsMain(input = {}) {
|
|
|
458
674
|
resultExtra: { region },
|
|
459
675
|
});
|
|
460
676
|
}
|
|
677
|
+
const addons = detectAlertAddonFeatures(path.join(cwd, 'terraform'));
|
|
678
|
+
const addonAlarmIds = ADDON_NOTIFY_ALARMS.filter((entry) => addons[entry.id]).map((entry) => entry.id);
|
|
461
679
|
const anchor = TARGET_ANCHORS[target] || TARGET_ANCHORS.ecs;
|
|
462
680
|
if (!mainTf.includes(anchor.resource)) {
|
|
463
681
|
const anchorName = target === 'lambda' ? 'aws_lambda_function.app'
|
|
@@ -573,7 +791,7 @@ async function runAlertsMain(input = {}) {
|
|
|
573
791
|
archiveAdded = ensureArchiveProvider(path.join(cwd, 'terraform', 'backend.tf'));
|
|
574
792
|
}
|
|
575
793
|
|
|
576
|
-
await atomicWriteFile(alertsPath, buildAlertsTf({ projectName, target, email, webhook, threshold }));
|
|
794
|
+
await atomicWriteFile(alertsPath, buildAlertsTf({ projectName, target, email, webhook, threshold, addons }));
|
|
577
795
|
if (webhook) {
|
|
578
796
|
await ensureForwarderZipIgnored(cwd);
|
|
579
797
|
}
|
|
@@ -583,7 +801,7 @@ async function runAlertsMain(input = {}) {
|
|
|
583
801
|
const { aliasAdded, archiveAdded } = scaffolded;
|
|
584
802
|
const destinations = email && webhook ? 'both' : webhook ? 'webhook' : email ? 'email' : 'manual';
|
|
585
803
|
const kind = webhook ? detectWebhookKind(webhook) : undefined;
|
|
586
|
-
console.log(color.green(`\n✅ Created ${ALERTS_TF_FILE} (SNS topic + ${target} notify alarm${webhook ? ' + chat forwarder' : ''}).`));
|
|
804
|
+
console.log(color.green(`\n✅ Created ${ALERTS_TF_FILE} (SNS topic + ${target} notify alarm${addonAlarmIds.length > 0 ? ` + ${addonAlarmIds.length} addon alarm${addonAlarmIds.length === 1 ? '' : 's'}` : ''}${webhook ? ' + chat forwarder' : ''}).`));
|
|
587
805
|
if (aliasAdded) console.log(color.dim(' Added the aws.us_east_1 provider alias to terraform/main.tf (CloudFront alarms only exist there).'));
|
|
588
806
|
if (archiveAdded) console.log(color.dim(' Pinned the archive provider in terraform/backend.tf (forwarder packaging).'));
|
|
589
807
|
console.log(color.dim(' Next steps:'));
|
|
@@ -605,7 +823,7 @@ async function runAlertsMain(input = {}) {
|
|
|
605
823
|
}
|
|
606
824
|
// Telemetry carries destination *kinds* only — never the email
|
|
607
825
|
// address or webhook URL (PII / bearer secret).
|
|
608
|
-
await trackSuccess('alerts_run', { projectName, action: 'scaffold', target, destinations, thresholdCustom: threshold !== undefined });
|
|
826
|
+
await trackSuccess('alerts_run', { projectName, action: 'scaffold', target, destinations, thresholdCustom: threshold !== undefined, addonAlarms: addonAlarmIds });
|
|
609
827
|
outro(color.green('Done.'));
|
|
610
828
|
return { ok: true, action: 'scaffold', projectName, region, target, destinations, file: ALERTS_TF_FILE };
|
|
611
829
|
}
|
package/src/commands/status.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { spawnSync } from 'child_process';
|
|
2
|
+
import path from 'path';
|
|
2
3
|
import { ECSClient, DescribeServicesCommand } from '@aws-sdk/client-ecs';
|
|
3
|
-
import { CloudWatchClient, DescribeAlarmsCommand } from '@aws-sdk/client-cloudwatch';
|
|
4
|
+
import { CloudWatchClient, DescribeAlarmsCommand, GetMetricDataCommand } from '@aws-sdk/client-cloudwatch';
|
|
4
5
|
import { CloudFrontClient, ListDistributionsCommand } from '@aws-sdk/client-cloudfront';
|
|
5
6
|
import color from 'picocolors';
|
|
6
7
|
import { intro, outro, spinner } from '@clack/prompts';
|
|
@@ -8,8 +9,9 @@ import { trackEvent, flushTelemetry, trackSuccess, trackFailure, setActiveComman
|
|
|
8
9
|
import { failCommand, failProjectNotInitialized, isProgrammaticCall, refuseUnknownFlags } from '../utils/command.js';
|
|
9
10
|
import { parseFlags, normalizeOptions, normalizeArgv } from '../utils/args.js';
|
|
10
11
|
import { handleAuthErrorBranch, resolveClient } from '../utils/aws.js';
|
|
11
|
-
import { resolveRegion, resolveProjectName, resolveCwd, readTerraformComputeTarget, readResourceSuffix, resolveNamespacedProject } from '../utils/resolvers.js';
|
|
12
|
+
import { resolveRegion, resolveProjectName, resolveCwd, readTerraformComputeTarget, readResourceSuffix, resolveNamespacedProject, safeSuffixedName } from '../utils/resolvers.js';
|
|
12
13
|
import { runDiagnose } from './diagnose.js';
|
|
14
|
+
import { detectAlertAddonFeatures } from './alerts.js';
|
|
13
15
|
import { fetchGoldenSignals } from '../utils/golden-signals.js';
|
|
14
16
|
import { requireManagedTree } from '../utils/terraform-metadata.js';
|
|
15
17
|
|
|
@@ -89,13 +91,15 @@ export function getLambdaHealth(configuration = {}) {
|
|
|
89
91
|
return { state, lastUpdateStatus, healthy };
|
|
90
92
|
}
|
|
91
93
|
|
|
92
|
-
function printLambdaDashboard({ functionName, configuration, health }) {
|
|
94
|
+
function printLambdaDashboard({ functionName, configuration, health, cron }) {
|
|
93
95
|
console.log('');
|
|
94
96
|
console.log(` ${color.bold('Function:')} ${color.cyan(functionName)} ${health.state === 'Active' ? color.green('(ACTIVE)') : color.yellow(`(${health.state})`)}`);
|
|
95
97
|
console.log(` ${color.bold('Last update:')} ${health.lastUpdateStatus === 'Successful' ? color.green(health.lastUpdateStatus) : color.yellow(health.lastUpdateStatus)}`);
|
|
96
98
|
if (configuration.LastModified) console.log(` ${color.dim('Modified:')} ${configuration.LastModified}`);
|
|
97
99
|
if (configuration.MemorySize) console.log(` ${color.dim('Memory:')} ${configuration.MemorySize} MB ${color.dim(`(timeout ${configuration.Timeout ?? '?'}s)`)}`);
|
|
98
100
|
if (configuration.Code?.ImageUri) console.log(` ${color.dim('Image:')} ${configuration.Code.ImageUri.split('/').pop()}`);
|
|
101
|
+
const cronLine = formatCronHealthLine(cron);
|
|
102
|
+
if (cronLine) console.log(` ${color.bold('Cron:')} ${cronLine}`);
|
|
99
103
|
console.log('');
|
|
100
104
|
}
|
|
101
105
|
|
|
@@ -212,6 +216,17 @@ export async function runLambdaStatus(input = {}) {
|
|
|
212
216
|
}
|
|
213
217
|
|
|
214
218
|
const health = getLambdaHealth(configuration);
|
|
219
|
+
// Cron run evidence, best-effort like the ECS path: detected from
|
|
220
|
+
// the tree, null when absent or unreachable.
|
|
221
|
+
let cron = null;
|
|
222
|
+
if (detectAlertAddonFeatures(path.join(cwd, 'terraform')).cron) {
|
|
223
|
+
try {
|
|
224
|
+
const cloudWatchClient = resolveClient(options.cloudWatchClient, CloudWatchClient, { region });
|
|
225
|
+
cron = await fetchCronHealth({ cloudWatchClient, scheduleGroup: safeSuffixedName(`${namespacedProject}${readResourceSuffix(cwd)}`, '-crons', 64) });
|
|
226
|
+
} catch {
|
|
227
|
+
cron = null;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
215
230
|
const payload = {
|
|
216
231
|
function: {
|
|
217
232
|
name: functionName,
|
|
@@ -224,6 +239,7 @@ export async function runLambdaStatus(input = {}) {
|
|
|
224
239
|
},
|
|
225
240
|
healthy: health.healthy,
|
|
226
241
|
signals: null,
|
|
242
|
+
cron,
|
|
227
243
|
region,
|
|
228
244
|
computeTarget: 'lambda',
|
|
229
245
|
};
|
|
@@ -235,7 +251,7 @@ export async function runLambdaStatus(input = {}) {
|
|
|
235
251
|
}
|
|
236
252
|
|
|
237
253
|
if (s) s.stop('Health check complete.\n');
|
|
238
|
-
printLambdaDashboard({ functionName, configuration, health });
|
|
254
|
+
printLambdaDashboard({ functionName, configuration, health, cron });
|
|
239
255
|
|
|
240
256
|
if (!health.healthy) {
|
|
241
257
|
console.log(color.yellow(DEGRADED_MESSAGE));
|
|
@@ -389,7 +405,55 @@ function printStaticDashboard({ distribution }) {
|
|
|
389
405
|
console.log('');
|
|
390
406
|
}
|
|
391
407
|
|
|
392
|
-
|
|
408
|
+
// Cron run evidence from scheduler metrics (the Scheduler API exposes
|
|
409
|
+
// no execution history): latest attempt plus 24h failure sums. Throws
|
|
410
|
+
// naturally — callers treat it as best-effort like golden signals.
|
|
411
|
+
export async function fetchCronHealth({ cloudWatchClient, scheduleGroup }) {
|
|
412
|
+
const endTime = new Date();
|
|
413
|
+
const startTime = new Date(endTime.getTime() - 24 * 60 * 60 * 1000);
|
|
414
|
+
const queries = [
|
|
415
|
+
['cronAttempts', 'InvocationAttemptCount'],
|
|
416
|
+
['cronErrors', 'TargetErrorCount'],
|
|
417
|
+
['cronDropped', 'InvocationDroppedCount'],
|
|
418
|
+
].map(([Id, MetricName]) => ({
|
|
419
|
+
Id,
|
|
420
|
+
MetricStat: {
|
|
421
|
+
Metric: {
|
|
422
|
+
Namespace: 'AWS/Scheduler',
|
|
423
|
+
MetricName,
|
|
424
|
+
Dimensions: [{ Name: 'ScheduleGroup', Value: scheduleGroup }],
|
|
425
|
+
},
|
|
426
|
+
Period: 3600,
|
|
427
|
+
Stat: 'Sum',
|
|
428
|
+
},
|
|
429
|
+
}));
|
|
430
|
+
const resp = await cloudWatchClient.send(new GetMetricDataCommand({
|
|
431
|
+
StartTime: startTime,
|
|
432
|
+
EndTime: endTime,
|
|
433
|
+
ScanBy: 'TimestampDescending',
|
|
434
|
+
MetricDataQueries: queries,
|
|
435
|
+
}));
|
|
436
|
+
const byId = new Map((resp.MetricDataResults || []).map((r) => [r.Id, r]));
|
|
437
|
+
const attempts = byId.get('cronAttempts');
|
|
438
|
+
let lastAttemptAt = null;
|
|
439
|
+
if (attempts) {
|
|
440
|
+
const stamps = (attempts.Timestamps || [])
|
|
441
|
+
.map((t, i) => [new Date(t).getTime(), attempts.Values[i]])
|
|
442
|
+
.filter(([, value]) => value > 0)
|
|
443
|
+
.map(([time]) => time);
|
|
444
|
+
if (stamps.length > 0) lastAttemptAt = new Date(Math.max(...stamps)).toISOString();
|
|
445
|
+
}
|
|
446
|
+
const sum = (id) => (byId.get(id)?.Values || []).reduce((total, value) => total + value, 0);
|
|
447
|
+
return { lastAttemptAt, targetErrors24h: sum('cronErrors'), dropped24h: sum('cronDropped') };
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
function formatCronHealthLine(cron) {
|
|
451
|
+
if (!cron) return null;
|
|
452
|
+
if (!cron.lastAttemptAt) return 'no runs in the last 24h';
|
|
453
|
+
return `last run ${cron.lastAttemptAt} · ${cron.targetErrors24h} target errors · ${cron.dropped24h} dropped (24h)`;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function printDashboard({ serviceName, cluster, health, signals, cron }) {
|
|
393
457
|
console.log('');
|
|
394
458
|
console.log(` ${color.bold('Service:')} ${color.cyan(serviceName)} ${health.status === 'ACTIVE' ? color.green('(ACTIVE)') : color.yellow(`(${health.status})`)}`);
|
|
395
459
|
console.log(` ${color.dim('Cluster:')} ${cluster}`);
|
|
@@ -412,6 +476,8 @@ function printDashboard({ serviceName, cluster, health, signals }) {
|
|
|
412
476
|
if (db?.kind === 'cluster') console.log(` ${color.dim('DB:')} ${db.capacityAcu ?? 'n/a'} ACU · ${formatSignalCount(db.connections)} conns`);
|
|
413
477
|
else if (db) console.log(` ${color.dim('DB:')} CPU ${formatSignalPercent(db.cpuPercent)} · ${formatSignalCount(db.connections)} conns`);
|
|
414
478
|
}
|
|
479
|
+
const cronLine = formatCronHealthLine(cron);
|
|
480
|
+
if (cronLine) console.log(` ${color.bold('Cron:')} ${cronLine}`);
|
|
415
481
|
console.log('');
|
|
416
482
|
}
|
|
417
483
|
|
|
@@ -529,6 +595,17 @@ async function runStatusMain(input = {}) {
|
|
|
529
595
|
signalsError = [`signals: ${error?.message || error}`];
|
|
530
596
|
}
|
|
531
597
|
|
|
598
|
+
// Cron run evidence, same best-effort deal: detected from the
|
|
599
|
+
// tree (not the target), null when absent or unreachable.
|
|
600
|
+
let cron = null;
|
|
601
|
+
if (detectAlertAddonFeatures(path.join(cwd, 'terraform')).cron) {
|
|
602
|
+
try {
|
|
603
|
+
cron = await fetchCronHealth({ cloudWatchClient, scheduleGroup: safeSuffixedName(`${namespacedProject}${resourceSuffix}`, '-crons', 64) });
|
|
604
|
+
} catch {
|
|
605
|
+
cron = null;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
532
609
|
const payload = {
|
|
533
610
|
service: {
|
|
534
611
|
name: serviceName,
|
|
@@ -541,6 +618,7 @@ async function runStatusMain(input = {}) {
|
|
|
541
618
|
alarms: health.alarms,
|
|
542
619
|
signals,
|
|
543
620
|
...(signalsError ? { signalsError } : {}),
|
|
621
|
+
cron,
|
|
544
622
|
healthy: health.healthy,
|
|
545
623
|
region,
|
|
546
624
|
};
|
|
@@ -552,7 +630,7 @@ async function runStatusMain(input = {}) {
|
|
|
552
630
|
}
|
|
553
631
|
|
|
554
632
|
if (s) s.stop('Health check complete.\n');
|
|
555
|
-
printDashboard({ serviceName, cluster, health, signals });
|
|
633
|
+
printDashboard({ serviceName, cluster, health, signals, cron });
|
|
556
634
|
|
|
557
635
|
if (!health.healthy) {
|
|
558
636
|
console.log(color.yellow(DEGRADED_MESSAGE));
|
package/src/data/action-pins.js
CHANGED
|
@@ -12,7 +12,7 @@ export const ACTION_PINS = {
|
|
|
12
12
|
'aquasecurity/trivy-action': { sha: 'ed142fd0673e97e23eac54620cfb913e5ce36c25', tag: 'v0.36.0' },
|
|
13
13
|
'aws-actions/amazon-ecr-login': { sha: '03f1aad4c6c7ffd436567f42f9384779290529bd', tag: 'v2' },
|
|
14
14
|
'aws-actions/configure-aws-credentials': { sha: '7474bc4690e29a8392af63c5b98e7449536d5c3a', tag: 'v4' },
|
|
15
|
-
'grada-run/grada-action': { sha: '
|
|
15
|
+
'grada-run/grada-action': { sha: '9df2e2c12314b6165e62f0e615edb01d6a81c40b', tag: 'v1' },
|
|
16
16
|
'hashicorp/setup-terraform': { sha: 'b9cd54a3c349d3f38e8881555d616ced269862dd', tag: 'v3' },
|
|
17
17
|
};
|
|
18
18
|
|
|
@@ -4,7 +4,7 @@ import { RDSClient } from '@aws-sdk/client-rds';
|
|
|
4
4
|
import { resolveClient } from './aws.js';
|
|
5
5
|
import { normalizeOptions } from './args.js';
|
|
6
6
|
import { findDbTarget, resolveDbIdentifier, resolveDbClusterIdentifier } from './rds.js';
|
|
7
|
-
import { readResourceSuffix } from './resolvers.js';
|
|
7
|
+
import { readResourceSuffix, safeSuffixedName } from './resolvers.js';
|
|
8
8
|
|
|
9
9
|
// Golden Signals: live CloudWatch telemetry for `grada status`.
|
|
10
10
|
//
|
|
@@ -21,10 +21,10 @@ import { readResourceSuffix } from './resolvers.js';
|
|
|
21
21
|
export const GOLDEN_SIGNALS_WINDOW_MINUTES = 15;
|
|
22
22
|
export const GOLDEN_SIGNALS_PERIOD_SECONDS = 300;
|
|
23
23
|
|
|
24
|
-
// Mirrors the generated ALB name (
|
|
25
|
-
// templates/terraform/main.tf) so lookup matches
|
|
24
|
+
// Mirrors the generated ALB name (the length-conditional in
|
|
25
|
+
// templates/terraform/main.tf) so lookup matches exactly, hashed or not.
|
|
26
26
|
export function resolveAlbName(projectName) {
|
|
27
|
-
return
|
|
27
|
+
return safeSuffixedName(projectName, '-alb', 32);
|
|
28
28
|
}
|
|
29
29
|
|
|
30
30
|
// `arn:aws:elasticloadbalancing:<region>:<acct>:loadbalancer/app/<name>/<id>`
|
package/src/utils/resolvers.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fsSync from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
import { randomBytes } from 'crypto';
|
|
3
|
+
import { randomBytes, createHash } from 'crypto';
|
|
4
4
|
import { normalizeOptions } from './args.js';
|
|
5
5
|
|
|
6
6
|
export const FALLBACK_REGION = 'us-east-2';
|
|
@@ -112,6 +112,21 @@ export function readTerraformRegion(cwd = process.cwd()) {
|
|
|
112
112
|
return match[1];
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
+
// Length-safe AWS resource name, mirroring the template conditional
|
|
116
|
+
// byte-for-byte: short names render identically; over-cap names take a
|
|
117
|
+
// deterministic md5 stem (never a truncating collision). `cap` is the
|
|
118
|
+
// service limit including suffix. Doubled hyphens at the cut point are
|
|
119
|
+
// left alone — valid in every name pattern here, and trimming would
|
|
120
|
+
// need identical semantics on both sides. Both sides must stay in
|
|
121
|
+
// sync — the parity tests below pin the shared cases.
|
|
122
|
+
export function safeSuffixedName(base, suffix, cap) {
|
|
123
|
+
const full = `${base}${suffix}`;
|
|
124
|
+
if (full.length <= cap) return full;
|
|
125
|
+
const stemLength = Math.max(cap - 1 - 5 - suffix.length, 1);
|
|
126
|
+
const stem = base.slice(0, stemLength);
|
|
127
|
+
return `${stem}-${createHash('md5').update(base, 'utf8').digest('hex').slice(0, 5)}${suffix}`;
|
|
128
|
+
}
|
|
129
|
+
|
|
115
130
|
export function resolveRegion(options = {}, cwd = process.cwd()) {
|
|
116
131
|
const opts = normalizeOptions(options);
|
|
117
132
|
if (typeof opts.region === 'string' && opts.region.trim()) {
|
|
@@ -76,6 +76,9 @@ jobs:
|
|
|
76
76
|
# point at an unpushed tag), but the repository itself is
|
|
77
77
|
# Terraform-managed — so it is bootstrapped with a targeted apply
|
|
78
78
|
# first. Idempotent on existing stacks.
|
|
79
|
+
- name: Setup Terraform
|
|
80
|
+
uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3
|
|
81
|
+
|
|
79
82
|
- name: Bootstrap ECR repository
|
|
80
83
|
working-directory: ./terraform
|
|
81
84
|
run: |
|
|
@@ -138,7 +141,7 @@ jobs:
|
|
|
138
141
|
# Runs AFTER the push: the function image tags exist by the time the
|
|
139
142
|
# function is created or updated.
|
|
140
143
|
- name: Apply Infrastructure Changes
|
|
141
|
-
uses: grada-run/grada-action@
|
|
144
|
+
uses: grada-run/grada-action@9df2e2c12314b6165e62f0e615edb01d6a81c40b # v1
|
|
142
145
|
with:
|
|
143
146
|
aws-region: ${{ env.AWS_REGION }}
|
|
144
147
|
role-to-assume: arn:aws:iam::{{AWS_ACCOUNT_ID}}:role/{{DEPLOY_ROLE_NAME}}
|