@prism-d1/cli 1.0.26 → 1.0.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/dist/assets/eval-harness/README.md +114 -0
  2. package/dist/assets/eval-harness/eval-config.json +10 -0
  3. package/dist/assets/eval-harness/rubrics/agent-quality.json +79 -0
  4. package/dist/assets/eval-harness/rubrics/api-response-quality.json +45 -0
  5. package/dist/assets/eval-harness/rubrics/code-quality.json +98 -0
  6. package/dist/assets/eval-harness/rubrics/security-compliance.json +145 -0
  7. package/dist/assets/eval-harness/rubrics/spec-compliance.json +67 -0
  8. package/dist/assets/eval-harness/run-eval.sh +122 -0
  9. package/dist/assets/github-workflows/README.md +110 -0
  10. package/dist/assets/github-workflows/prism-agent-eval.yml +313 -0
  11. package/dist/assets/github-workflows/prism-ai-metrics.yml +261 -0
  12. package/dist/assets/github-workflows/prism-dora-weekly.yml +334 -0
  13. package/dist/assets/github-workflows/prism-eval-gate.yml +310 -0
  14. package/dist/assets/infra/bin/app.ts +56 -0
  15. package/dist/assets/infra/cdk.json +12 -0
  16. package/dist/assets/infra/lib/api-stack.ts +347 -0
  17. package/dist/assets/infra/lib/constructs/bedrock-guardrail-construct.ts +201 -0
  18. package/dist/assets/infra/lib/constructs/guardrail-enforcer-construct.ts +59 -0
  19. package/dist/assets/infra/lib/constructs/prism-vpc-construct.ts +75 -0
  20. package/dist/assets/infra/lib/constructs/security-agent-construct.ts +266 -0
  21. package/dist/assets/infra/lib/dashboard-stack.ts +1392 -0
  22. package/dist/assets/infra/lib/lambda/api-handler.ts +477 -0
  23. package/dist/assets/infra/lib/lambda/defect-correlator.ts +142 -0
  24. package/dist/assets/infra/lib/lambda/exfiltration-detector.ts +100 -0
  25. package/dist/assets/infra/lib/lambda/layers/guardrail-enforcer/nodejs/guardrail-enforcer.js +53 -0
  26. package/dist/assets/infra/lib/lambda/metrics-processor.ts +748 -0
  27. package/dist/assets/infra/lib/lambda/security-agent-processor.ts +231 -0
  28. package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +120 -0
  29. package/dist/assets/infra/lib/lambda/security-response-automator.ts +130 -0
  30. package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +123 -0
  31. package/dist/assets/infra/lib/metrics-pipeline-stack.ts +701 -0
  32. package/dist/assets/infra/package.json +23 -0
  33. package/dist/assets/infra/tsconfig.json +24 -0
  34. package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
  35. package/dist/src/commands/bootstrapper/install-eval-harness.js +3 -4
  36. package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
  37. package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
  38. package/dist/src/commands/bootstrapper/install-git-hooks.js +2 -5
  39. package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
  40. package/dist/src/commands/securityagent/setup.d.ts.map +1 -1
  41. package/dist/src/commands/securityagent/setup.js +2 -3
  42. package/dist/src/commands/securityagent/setup.js.map +1 -1
  43. package/dist/src/commands/workshop/deploy-infra.d.ts.map +1 -1
  44. package/dist/src/commands/workshop/deploy-infra.js +2 -3
  45. package/dist/src/commands/workshop/deploy-infra.js.map +1 -1
  46. package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
  47. package/dist/src/commands/workshop/generate-demo-data.js +3 -8
  48. package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
  49. package/dist/src/commands/workshop/perform-pen-test.d.ts.map +1 -1
  50. package/dist/src/commands/workshop/perform-pen-test.js +5 -14
  51. package/dist/src/commands/workshop/perform-pen-test.js.map +1 -1
  52. package/dist/src/utils/root.d.ts +6 -0
  53. package/dist/src/utils/root.d.ts.map +1 -1
  54. package/dist/src/utils/root.js +29 -0
  55. package/dist/src/utils/root.js.map +1 -1
  56. package/package.json +2 -2
@@ -0,0 +1,748 @@
1
+ import {
2
+ DynamoDBClient,
3
+ PutItemCommand,
4
+ } from '@aws-sdk/client-dynamodb';
5
+ import {
6
+ CloudWatchClient,
7
+ PutMetricDataCommand,
8
+ MetricDatum,
9
+ StandardUnit,
10
+ } from '@aws-sdk/client-cloudwatch';
11
+
12
+ // ---- Types ----
13
+
14
+ interface AiContext {
15
+ tool: string;
16
+ model: string;
17
+ origin: string;
18
+ }
19
+
20
+ interface DoraMetrics {
21
+ deployment_frequency: number | null;
22
+ lead_time_seconds: number | null;
23
+ change_failure_rate: number | null;
24
+ mttr_seconds: number | null;
25
+ }
26
+
27
+ interface AiDoraMetrics {
28
+ ai_acceptance_rate: number | null;
29
+ ai_to_merge_ratio: number | null;
30
+ spec_to_code_hours: number | null;
31
+ post_merge_defect_rate: number | null;
32
+ eval_gate_pass_rate: number | null;
33
+ ai_test_coverage_delta: number | null;
34
+ total_input_tokens: number | null;
35
+ total_output_tokens: number | null;
36
+ total_cost_usd: number | null;
37
+ }
38
+
39
+ interface EvalDetail {
40
+ eval_id: string;
41
+ rubric: string;
42
+ result: 'PASS' | 'FAIL';
43
+ score: number;
44
+ input_file: string;
45
+ pr_number?: number;
46
+ criterion_scores?: Array<{ name: string; score: number; max_score: number; reasoning: string }>;
47
+ }
48
+
49
+ interface GuardrailTriggerDetail {
50
+ guardrail_id: string;
51
+ guardrail_name: string;
52
+ trigger_category: 'CONTENT_FILTER' | 'DENIED_TOPIC' | 'WORD_FILTER' | 'SENSITIVE_INFO' | 'CONTEXTUAL_GROUNDING';
53
+ trigger_type: string;
54
+ action_taken: 'BLOCK' | 'ANONYMIZE' | 'WARN';
55
+ agent_name: string;
56
+ invocation_id: string;
57
+ }
58
+
59
+ interface MCPToolCallDetail {
60
+ session_id: string;
61
+ client_id: string;
62
+ tool_name: string;
63
+ scopes_used: string[];
64
+ authorized: boolean;
65
+ risk_level: string;
66
+ duration_ms: number;
67
+ result_status: 'success' | 'error' | 'denied';
68
+ }
69
+
70
+
71
+ interface QualityDetail {
72
+ deployment_id: string;
73
+ ai_defect_rate: number;
74
+ human_defect_rate: number;
75
+ total_ai_commits: number;
76
+ total_human_commits: number;
77
+ }
78
+
79
+ interface SecurityDetail {
80
+ alert_type: string;
81
+ table_name: string;
82
+ principal_arn: string;
83
+ read_count: number;
84
+ window_start: string;
85
+ window_end: string;
86
+ }
87
+
88
+ interface SecurityAgentFinding {
89
+ finding_id: string;
90
+ phase: 'design_review' | 'code_review' | 'pen_test';
91
+ severity: 'CRITICAL' | 'HIGH' | 'MEDIUM' | 'LOW' | 'INFORMATIONAL';
92
+ cvss_score: number | null;
93
+ title: string;
94
+ category: string;
95
+ cwe_id: string | null;
96
+ exploit_validated: boolean;
97
+ compliance_mappings: string[];
98
+ ai_origin: string;
99
+ spec_ref: string | null;
100
+ found_at: string;
101
+ remediated_at: string | null;
102
+ }
103
+
104
+ interface SecurityRemediationDetail {
105
+ finding_id: string;
106
+ severity: string;
107
+ remediation_time_hours: number;
108
+ remediated_by_origin: string;
109
+ finding_phase: string;
110
+ }
111
+
112
+ interface MetricDetail {
113
+ team_id: string;
114
+ repo: string;
115
+ timestamp: string;
116
+ prism_level: number | string;
117
+ metric: { name: string; value: number; unit: string };
118
+ ai_context?: AiContext;
119
+ dora?: DoraMetrics;
120
+ ai_dora?: AiDoraMetrics;
121
+ agent?: {
122
+ agent_name: string;
123
+ steps_taken: number;
124
+ tools_invoked: number;
125
+ duration_ms: number;
126
+ tokens_used: number;
127
+ status: string;
128
+ guardrails_triggered: number;
129
+ };
130
+ eval?: EvalDetail;
131
+ guardrail?: GuardrailTriggerDetail;
132
+ mcp_tool_call?: MCPToolCallDetail;
133
+ quality?: QualityDetail;
134
+ security?: SecurityDetail;
135
+ security_agent_finding?: SecurityAgentFinding;
136
+ security_remediation?: SecurityRemediationDetail;
137
+ }
138
+
139
+ interface EventBridgeEvent {
140
+ source: string;
141
+ 'detail-type': string;
142
+ detail: MetricDetail;
143
+ }
144
+
145
+ // ---- Clients (reused across invocations) ----
146
+
147
+ const dynamoClient = new DynamoDBClient({});
148
+ const cloudwatchClient = new CloudWatchClient({});
149
+
150
+ const EVENTS_TABLE = process.env.EVENTS_TABLE!;
151
+ const METADATA_TABLE = process.env.METADATA_TABLE!;
152
+ const METRIC_NAMESPACE = process.env.METRIC_NAMESPACE ?? 'PRISM/D1/Velocity';
153
+
154
+ // ---- Handler ----
155
+
156
+ export async function handler(event: EventBridgeEvent): Promise<void> {
157
+ console.log('[metrics-processor] Received event:', JSON.stringify(event, null, 2));
158
+
159
+ const detailType = event['detail-type'];
160
+ const detail = event.detail;
161
+
162
+ console.log(`[metrics-processor] detail-type=${detailType} team_id=${detail?.team_id} repo=${detail?.repo} timestamp=${detail?.timestamp}`);
163
+ console.log(`[metrics-processor] dora=${JSON.stringify(detail?.dora)} ai_dora=${JSON.stringify(detail?.ai_dora)} metric=${JSON.stringify(detail?.metric)}`);
164
+
165
+ if (!detail.team_id) {
166
+ console.log('[metrics-processor] No team_id provided, defaulting to "no_team"');
167
+ detail.team_id = 'no_team';
168
+ }
169
+
170
+ if (!detail.repo || !detail.timestamp) {
171
+ console.error('[metrics-processor] VALIDATION FAILED: Missing required fields: repo or timestamp');
172
+ throw new Error('Event missing required fields');
173
+ }
174
+
175
+ const results = await Promise.allSettled([
176
+ writeEventToDynamo(detailType, detail),
177
+ writeMetadataToDynamo(detailType, detail),
178
+ publishCloudWatchMetrics(detailType, detail),
179
+ ]);
180
+
181
+ results.forEach((result, idx) => {
182
+ const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics'];
183
+ if (result.status === 'fulfilled') {
184
+ console.log(`[metrics-processor] ${labels[idx]} succeeded`);
185
+ } else {
186
+ console.error(`[metrics-processor] ${labels[idx]} FAILED:`, result.reason);
187
+ }
188
+ });
189
+
190
+ const failures = results.filter((r) => r.status === 'rejected');
191
+ if (failures.length > 0) {
192
+ throw new Error(`${failures.length} operation(s) failed — check logs above`);
193
+ }
194
+
195
+ console.log(`[metrics-processor] Successfully processed ${detailType} for ${detail.team_id}/${detail.repo}`);
196
+ }
197
+
198
+ // ---- DynamoDB events ----
199
+
200
+ async function writeEventToDynamo(
201
+ detailType: string,
202
+ detail: MetricDetail,
203
+ ): Promise<void> {
204
+ console.log(`[writeEventToDynamo] Writing event: pk=${detail.team_id}#${detail.repo} sk=${detail.timestamp} type=${detailType}`);
205
+ const ttl = Math.floor(Date.now() / 1000) + 365 * 24 * 60 * 60; // 365 days from now
206
+
207
+ const data: Record<string, unknown> = {
208
+ team_id: detail.team_id,
209
+ repo: detail.repo,
210
+ prism_level: detail.prism_level ?? '1',
211
+ };
212
+
213
+ if (detail.metric) {
214
+ data.metric = detail.metric;
215
+ }
216
+ if (detail.ai_context) {
217
+ data.ai_context = detail.ai_context;
218
+ }
219
+ if (detail.dora) {
220
+ data.dora = detail.dora;
221
+ }
222
+ if (detail.ai_dora) {
223
+ data.ai_dora = detail.ai_dora;
224
+ }
225
+
226
+ const item: Record<string, { S?: string; N?: string }> = {
227
+ pk: { S: `${detail.team_id}#${detail.repo}` },
228
+ sk: { S: detail.timestamp },
229
+ detail_type: { S: detailType },
230
+ data: { S: JSON.stringify(data) },
231
+ ttl: { N: ttl.toString() },
232
+ };
233
+
234
+ // Store spec_ref as a top-level attribute for GSI queries (spec-to-code calculation)
235
+ const specRef = (detail.ai_context as any)?.spec_ref
236
+ ?? (detail as any).spec_ref;
237
+ if (specRef && typeof specRef === 'string') {
238
+ item.spec_ref = { S: specRef };
239
+ }
240
+
241
+ // Store eval rubric as a top-level attribute for per-rubric queries
242
+ if (detail.eval?.rubric) {
243
+ item.eval_rubric = { S: detail.eval.rubric };
244
+ }
245
+
246
+ // Store finding_id for Security Agent finding queries
247
+ if (detail.security_agent_finding?.finding_id) {
248
+ item.finding_id = { S: detail.security_agent_finding.finding_id };
249
+ }
250
+ if (detail.security_remediation?.finding_id) {
251
+ item.finding_id = { S: detail.security_remediation.finding_id };
252
+ }
253
+
254
+ await dynamoClient.send(
255
+ new PutItemCommand({
256
+ TableName: EVENTS_TABLE,
257
+ Item: item,
258
+ }),
259
+ );
260
+ }
261
+
262
+ // ---- DynamoDB metadata ----
263
+
264
+ async function writeMetadataToDynamo(
265
+ detailType: string,
266
+ detail: MetricDetail,
267
+ ): Promise<void> {
268
+ console.log(`[writeMetadataToDynamo] Writing metadata: team_id=${detail.team_id} repo=${detail.repo} type=${detailType}`);
269
+ const item: Record<string, { S?: string; N?: string }> = {
270
+ team_id: { S: detail.team_id },
271
+ repo: { S: detail.repo },
272
+ last_event_type: { S: detailType },
273
+ last_updated: { S: detail.timestamp },
274
+ prism_level: { N: String(detail.prism_level ?? 1) },
275
+ };
276
+
277
+ if (detail.ai_context?.tool) {
278
+ item.ai_tool = { S: detail.ai_context.tool };
279
+ }
280
+ if (detail.ai_context?.origin) {
281
+ item.ai_origin = { S: detail.ai_context.origin };
282
+ }
283
+
284
+ // For assessment events, store the full PRISM level and primary metric
285
+ if (detailType === 'prism.d1.assessment' && detail.metric) {
286
+ item.assessment_metric = { S: detail.metric.name };
287
+ item.assessment_value = { N: detail.metric.value.toString() };
288
+ }
289
+
290
+ // Store latest DORA snapshot — only numeric fields as N attributes
291
+ if (detail.dora) {
292
+ for (const [key, val] of Object.entries(detail.dora)) {
293
+ if (val == null) continue;
294
+ if (typeof val === 'number') {
295
+ item[`dora_${key}`] = { N: val.toString() };
296
+ } else if (typeof val === 'string' && !isNaN(Number(val))) {
297
+ item[`dora_${key}`] = { N: val };
298
+ }
299
+ // Skip non-numeric values (e.g. deploy_sha) — they don't belong in N attributes
300
+ }
301
+ }
302
+
303
+ // Store latest AI-DORA snapshot — only numeric fields
304
+ if (detail.ai_dora) {
305
+ for (const [key, val] of Object.entries(detail.ai_dora)) {
306
+ if (val == null) continue;
307
+ if (typeof val === 'object') continue; // Skip nested objects like tool_breakdown
308
+ if (typeof val === 'number') {
309
+ item[`ai_dora_${key}`] = { N: val.toString() };
310
+ } else if (typeof val === 'string' && !isNaN(Number(val))) {
311
+ item[`ai_dora_${key}`] = { N: val };
312
+ }
313
+ }
314
+ }
315
+
316
+ await dynamoClient.send(
317
+ new PutItemCommand({
318
+ TableName: METADATA_TABLE,
319
+ Item: item,
320
+ }),
321
+ );
322
+ }
323
+
324
+ // ---- CloudWatch custom metrics ----
325
+
326
+ async function publishCloudWatchMetrics(
327
+ detailType: string,
328
+ detail: MetricDetail,
329
+ ): Promise<void> {
330
+ console.log(`[publishCloudWatchMetrics] Starting for ${detailType}, namespace=${METRIC_NAMESPACE}`);
331
+ console.log(`[publishCloudWatchMetrics] dora fields: deployment_frequency=${detail.dora?.deployment_frequency} lead_time_seconds=${detail.dora?.lead_time_seconds} change_failure_rate=${detail.dora?.change_failure_rate} mttr_seconds=${detail.dora?.mttr_seconds}`);
332
+ console.log(`[publishCloudWatchMetrics] ai_dora fields: ai_acceptance_rate=${detail.ai_dora?.ai_acceptance_rate} ai_to_merge_ratio=${detail.ai_dora?.ai_to_merge_ratio} eval_gate_pass_rate=${detail.ai_dora?.eval_gate_pass_rate}`);
333
+
334
+ const sharedDimensions = [
335
+ { Name: 'TeamId', Value: detail.team_id },
336
+ { Name: 'Repository', Value: detail.repo },
337
+ ];
338
+
339
+ // Add AIOrigin dimension when available — enables dashboard filtering
340
+ // by ai-generated vs ai-assisted vs human
341
+ const aiOrigin = detail.ai_context?.origin;
342
+ const dimensionsWithOrigin = aiOrigin
343
+ ? [...sharedDimensions, { Name: 'AIOrigin', Value: aiOrigin }]
344
+ : sharedDimensions;
345
+
346
+ // Clamp timestamp: CloudWatch rejects timestamps >2h in the future
347
+ const eventTime = new Date(detail.timestamp);
348
+ const metricTimestamp = eventTime.getTime() > Date.now() ? new Date() : eventTime;
349
+
350
+ const metricData: MetricDatum[] = [];
351
+
352
+ // Primary metric — published with both dimension sets for flexibility:
353
+ // 1. With AIOrigin: allows filtering by origin type
354
+ // 2. Without AIOrigin: allows aggregate queries across all origins
355
+ if (detail.metric?.value != null) {
356
+ metricData.push({
357
+ MetricName: detail.metric.name,
358
+ Value: detail.metric.value,
359
+ Unit: mapUnit(detail.metric.unit),
360
+ Dimensions: sharedDimensions,
361
+ Timestamp: metricTimestamp,
362
+ });
363
+ if (aiOrigin) {
364
+ metricData.push({
365
+ MetricName: detail.metric.name,
366
+ Value: detail.metric.value,
367
+ Unit: mapUnit(detail.metric.unit),
368
+ Dimensions: dimensionsWithOrigin,
369
+ Timestamp: metricTimestamp,
370
+ });
371
+ }
372
+ }
373
+
374
+ // DORA metrics — published with AIOrigin dimension when available
375
+ if (detail.dora) {
376
+ const doraDims = aiOrigin ? dimensionsWithOrigin : sharedDimensions;
377
+ if (detail.dora.deployment_frequency != null) {
378
+ metricData.push({
379
+ MetricName: 'DeploymentFrequency',
380
+ Value: detail.dora.deployment_frequency,
381
+ Unit: StandardUnit.Count,
382
+ Dimensions: sharedDimensions,
383
+ Timestamp: metricTimestamp,
384
+ });
385
+ if (aiOrigin) {
386
+ metricData.push({
387
+ MetricName: 'DeploymentFrequency',
388
+ Value: detail.dora.deployment_frequency,
389
+ Unit: StandardUnit.Count,
390
+ Dimensions: doraDims,
391
+ Timestamp: metricTimestamp,
392
+ });
393
+ }
394
+ }
395
+ if (detail.dora.lead_time_seconds != null) {
396
+ metricData.push({
397
+ MetricName: 'LeadTimeForChanges',
398
+ Value: detail.dora.lead_time_seconds,
399
+ Unit: StandardUnit.Seconds,
400
+ Dimensions: sharedDimensions,
401
+ Timestamp: metricTimestamp,
402
+ });
403
+ if (aiOrigin) {
404
+ metricData.push({
405
+ MetricName: 'LeadTimeForChanges',
406
+ Value: detail.dora.lead_time_seconds,
407
+ Unit: StandardUnit.Seconds,
408
+ Dimensions: doraDims,
409
+ Timestamp: metricTimestamp,
410
+ });
411
+ }
412
+ }
413
+ if (detail.dora.change_failure_rate != null) {
414
+ const cfrValue = detail.dora.change_failure_rate <= 1 ? detail.dora.change_failure_rate * 100 : detail.dora.change_failure_rate;
415
+ metricData.push({
416
+ MetricName: 'ChangeFailureRate',
417
+ Value: cfrValue,
418
+ Unit: StandardUnit.Percent,
419
+ Dimensions: sharedDimensions,
420
+ Timestamp: metricTimestamp,
421
+ });
422
+ }
423
+ if (detail.dora.mttr_seconds != null) {
424
+ metricData.push({
425
+ MetricName: 'MTTR',
426
+ Value: detail.dora.mttr_seconds,
427
+ Unit: StandardUnit.Seconds,
428
+ Dimensions: sharedDimensions,
429
+ Timestamp: metricTimestamp,
430
+ });
431
+ }
432
+ }
433
+
434
+ // AI-DORA metrics — scale 0–1 ratios to 0–100 for CloudWatch Percent unit
435
+ if (detail.ai_dora) {
436
+ const aiDoraMap: Array<[string, number | null, StandardUnit, boolean]> = [
437
+ ['AIAcceptanceRate', detail.ai_dora.ai_acceptance_rate, StandardUnit.Percent, true],
438
+ ['AIToMergeRatio', detail.ai_dora.ai_to_merge_ratio, StandardUnit.Percent, true],
439
+ ['SpecToCodeHours', detail.ai_dora.spec_to_code_hours, StandardUnit.Count, false],
440
+ ['PostMergeDefectRate', detail.ai_dora.post_merge_defect_rate, StandardUnit.Percent, true],
441
+ ['EvalGatePassRate', detail.ai_dora.eval_gate_pass_rate, StandardUnit.Percent, true],
442
+ ['AITestCoverageDelta', detail.ai_dora.ai_test_coverage_delta, StandardUnit.Percent, true],
443
+ ['AIInputTokens', detail.ai_dora.total_input_tokens, StandardUnit.Count, false],
444
+ ['AIOutputTokens', detail.ai_dora.total_output_tokens, StandardUnit.Count, false],
445
+ ['AICostUSD', detail.ai_dora.total_cost_usd, StandardUnit.None, false],
446
+ ];
447
+
448
+ for (const [name, value, unit, scaleToPercent] of aiDoraMap) {
449
+ if (value != null) {
450
+ const publishValue = scaleToPercent && value <= 1 ? value * 100 : value;
451
+ metricData.push({
452
+ MetricName: name,
453
+ Value: publishValue,
454
+ Unit: unit,
455
+ Dimensions: sharedDimensions,
456
+ Timestamp: metricTimestamp,
457
+ });
458
+ }
459
+ }
460
+ }
461
+
462
+ // Agent metrics
463
+ if (detail.agent) {
464
+ const agent = detail.agent;
465
+ const agentDimensions = [
466
+ ...sharedDimensions,
467
+ { Name: 'AgentName', Value: agent.agent_name ?? 'unknown' },
468
+ ];
469
+
470
+ const agentMetrics: Array<[string, number | null, StandardUnit]> = [
471
+ ['AgentInvocationCount', 1, StandardUnit.Count],
472
+ ['AgentStepCount', agent.steps_taken ?? null, StandardUnit.Count],
473
+ ['AgentDurationMs', agent.duration_ms ?? null, StandardUnit.Milliseconds],
474
+ ['AgentTokensUsed', agent.tokens_used ?? null, StandardUnit.Count],
475
+ ['AgentToolInvocationCount', agent.tools_invoked ?? null, StandardUnit.Count],
476
+ ['AgentGuardrailTriggerCount', agent.guardrails_triggered ?? null, StandardUnit.Count],
477
+ ['AgentSuccessRate', agent.status === 'success' ? 100 : 0, StandardUnit.Percent],
478
+ ];
479
+
480
+ for (const [name, value, unit] of agentMetrics) {
481
+ if (value != null) {
482
+ // Publish with AgentName dimension (for per-agent drill-down)
483
+ metricData.push({
484
+ MetricName: name,
485
+ Value: value,
486
+ Unit: unit,
487
+ Dimensions: agentDimensions,
488
+ Timestamp: metricTimestamp,
489
+ });
490
+ // Also publish without AgentName (for aggregate dashboard queries)
491
+ metricData.push({
492
+ MetricName: name,
493
+ Value: value,
494
+ Unit: unit,
495
+ Dimensions: sharedDimensions,
496
+ Timestamp: metricTimestamp,
497
+ });
498
+ }
499
+ }
500
+ }
501
+
502
+ // Eval metrics — per-rubric pass rate
503
+ if (detail.eval) {
504
+ const rubricDimensions = [
505
+ ...sharedDimensions,
506
+ { Name: 'RubricName', Value: detail.eval.rubric ?? 'unknown' },
507
+ ];
508
+ metricData.push({
509
+ MetricName: 'EvalGatePassRateByRubric',
510
+ Value: detail.eval.result === 'PASS' ? 100 : 0,
511
+ Unit: StandardUnit.Percent,
512
+ Dimensions: rubricDimensions,
513
+ Timestamp: metricTimestamp,
514
+ });
515
+ metricData.push({
516
+ MetricName: 'EvalScore',
517
+ Value: detail.eval.score ?? 0,
518
+ Unit: StandardUnit.None,
519
+ Dimensions: rubricDimensions,
520
+ Timestamp: metricTimestamp,
521
+ });
522
+ }
523
+
524
+ // Guardrail metrics — per-category trigger tracking
525
+ if (detail.guardrail) {
526
+ const guardrailDimensions = [
527
+ ...sharedDimensions,
528
+ { Name: 'TriggerCategory', Value: detail.guardrail.trigger_category },
529
+ { Name: 'AgentName', Value: detail.guardrail.agent_name ?? 'unknown' },
530
+ ];
531
+ metricData.push({
532
+ MetricName: 'GuardrailTriggerCount',
533
+ Value: 1,
534
+ Unit: StandardUnit.Count,
535
+ Dimensions: guardrailDimensions,
536
+ Timestamp: metricTimestamp,
537
+ });
538
+ if (detail.guardrail.action_taken === 'BLOCK') {
539
+ metricData.push({
540
+ MetricName: 'GuardrailBlockCount',
541
+ Value: 1,
542
+ Unit: StandardUnit.Count,
543
+ Dimensions: sharedDimensions,
544
+ Timestamp: metricTimestamp,
545
+ });
546
+ }
547
+ if (detail.guardrail.action_taken === 'ANONYMIZE') {
548
+ metricData.push({
549
+ MetricName: 'GuardrailAnonymizeCount',
550
+ Value: 1,
551
+ Unit: StandardUnit.Count,
552
+ Dimensions: sharedDimensions,
553
+ Timestamp: metricTimestamp,
554
+ });
555
+ }
556
+ }
557
+
558
+ // MCP tool call metrics
559
+ if (detail.mcp_tool_call) {
560
+ const mcpDimensions = [
561
+ ...sharedDimensions,
562
+ { Name: 'ToolName', Value: detail.mcp_tool_call.tool_name },
563
+ ];
564
+ metricData.push({
565
+ MetricName: 'MCPToolCallCount',
566
+ Value: 1,
567
+ Unit: StandardUnit.Count,
568
+ Dimensions: mcpDimensions,
569
+ Timestamp: metricTimestamp,
570
+ });
571
+ if (!detail.mcp_tool_call.authorized) {
572
+ metricData.push({
573
+ MetricName: 'MCPAuthDeniedCount',
574
+ Value: 1,
575
+ Unit: StandardUnit.Count,
576
+ Dimensions: mcpDimensions,
577
+ Timestamp: metricTimestamp,
578
+ });
579
+ }
580
+ if (detail.mcp_tool_call.duration_ms != null) {
581
+ metricData.push({
582
+ MetricName: 'MCPToolCallDurationMs',
583
+ Value: detail.mcp_tool_call.duration_ms,
584
+ Unit: StandardUnit.Milliseconds,
585
+ Dimensions: mcpDimensions,
586
+ Timestamp: metricTimestamp,
587
+ });
588
+ }
589
+ }
590
+
591
+
592
+
593
+ // Quality / defect rate metrics
594
+ if (detail.quality) {
595
+ metricData.push(
596
+ {
597
+ MetricName: 'PostMergeDefectRateAI',
598
+ Value: detail.quality.ai_defect_rate,
599
+ Unit: StandardUnit.Percent,
600
+ Dimensions: sharedDimensions,
601
+ Timestamp: metricTimestamp,
602
+ },
603
+ {
604
+ MetricName: 'PostMergeDefectRateHuman',
605
+ Value: detail.quality.human_defect_rate,
606
+ Unit: StandardUnit.Percent,
607
+ Dimensions: sharedDimensions,
608
+ Timestamp: metricTimestamp,
609
+ },
610
+ );
611
+ }
612
+
613
+ // Security / exfiltration metrics
614
+ if (detail.security) {
615
+ metricData.push({
616
+ MetricName: 'ExfiltrationAlertCount',
617
+ Value: 1,
618
+ Unit: StandardUnit.Count,
619
+ Dimensions: sharedDimensions,
620
+ Timestamp: metricTimestamp,
621
+ });
622
+ }
623
+
624
+ // AWS Security Agent finding metrics
625
+ if (detail.security_agent_finding) {
626
+ const finding = detail.security_agent_finding;
627
+ const phaseDimensions = [
628
+ ...sharedDimensions,
629
+ { Name: 'Phase', Value: finding.phase },
630
+ { Name: 'Severity', Value: finding.severity },
631
+ ];
632
+ metricData.push({
633
+ MetricName: 'SecurityFindingCount',
634
+ Value: 1,
635
+ Unit: StandardUnit.Count,
636
+ Dimensions: phaseDimensions,
637
+ Timestamp: metricTimestamp,
638
+ });
639
+ if (finding.severity === 'CRITICAL' || finding.severity === 'HIGH') {
640
+ metricData.push({
641
+ MetricName: 'SecurityCriticalFindingCount',
642
+ Value: 1,
643
+ Unit: StandardUnit.Count,
644
+ Dimensions: sharedDimensions,
645
+ Timestamp: metricTimestamp,
646
+ });
647
+ }
648
+ if (finding.ai_origin) {
649
+ metricData.push({
650
+ MetricName: 'SecurityFindingByOrigin',
651
+ Value: 1,
652
+ Unit: StandardUnit.Count,
653
+ Dimensions: [
654
+ ...sharedDimensions,
655
+ { Name: 'AIOrigin', Value: finding.ai_origin },
656
+ ],
657
+ Timestamp: metricTimestamp,
658
+ });
659
+ }
660
+ if (finding.cvss_score != null) {
661
+ metricData.push({
662
+ MetricName: 'SecurityFindingCVSS',
663
+ Value: finding.cvss_score,
664
+ Unit: StandardUnit.None,
665
+ Dimensions: phaseDimensions,
666
+ Timestamp: metricTimestamp,
667
+ });
668
+ }
669
+ metricData.push({
670
+ MetricName: 'SecurityScanCount',
671
+ Value: 1,
672
+ Unit: StandardUnit.Count,
673
+ Dimensions: [
674
+ ...sharedDimensions,
675
+ { Name: 'Phase', Value: finding.phase },
676
+ ],
677
+ Timestamp: metricTimestamp,
678
+ });
679
+ }
680
+
681
+ // Security remediation metrics
682
+ if (detail.security_remediation) {
683
+ const remediation = detail.security_remediation;
684
+ metricData.push({
685
+ MetricName: 'SecurityRemediationTimeHours',
686
+ Value: remediation.remediation_time_hours,
687
+ Unit: StandardUnit.Count,
688
+ Dimensions: [
689
+ ...sharedDimensions,
690
+ { Name: 'Severity', Value: remediation.severity },
691
+ { Name: 'AIOrigin', Value: remediation.remediated_by_origin },
692
+ ],
693
+ Timestamp: metricTimestamp,
694
+ });
695
+ }
696
+
697
+ // Also publish all metrics WITHOUT dimensions for aggregate dashboard views.
698
+ // CloudWatch treats dimensioned and dimensionless metrics as separate time series.
699
+ // The dashboard-stack.ts widgets query without dimensions, so we need both.
700
+ const dimensionlessMetrics: MetricDatum[] = metricData
701
+ .filter((m) => m.Dimensions && m.Dimensions.length > 0)
702
+ .map((m) => ({
703
+ ...m,
704
+ Dimensions: [],
705
+ }));
706
+ metricData.push(...dimensionlessMetrics);
707
+
708
+ if (metricData.length === 0) {
709
+ console.log('[publishCloudWatchMetrics] No metrics to publish — metricData is empty');
710
+ return;
711
+ }
712
+
713
+ console.log(`[publishCloudWatchMetrics] Publishing ${metricData.length} metric data points`);
714
+ metricData.forEach((m, i) => {
715
+ console.log(`[publishCloudWatchMetrics] [${i}] ${m.MetricName}=${m.Value} unit=${m.Unit} dims=${JSON.stringify(m.Dimensions)}`);
716
+ });
717
+
718
+ // CloudWatch accepts max 1000 metric data points per call; batch in chunks of 25
719
+ const batchSize = 25;
720
+ for (let i = 0; i < metricData.length; i += batchSize) {
721
+ const batch = metricData.slice(i, i + batchSize);
722
+ console.log(`[publishCloudWatchMetrics] Sending batch ${Math.floor(i / batchSize) + 1} with ${batch.length} metrics`);
723
+ try {
724
+ await cloudwatchClient.send(
725
+ new PutMetricDataCommand({
726
+ Namespace: METRIC_NAMESPACE,
727
+ MetricData: batch,
728
+ }),
729
+ );
730
+ console.log(`[publishCloudWatchMetrics] Batch ${Math.floor(i / batchSize) + 1} sent successfully`);
731
+ } catch (err) {
732
+ console.error(`[publishCloudWatchMetrics] Batch ${Math.floor(i / batchSize) + 1} FAILED:`, err);
733
+ throw err;
734
+ }
735
+ }
736
+ }
737
+
738
+ function mapUnit(unit: string): StandardUnit {
739
+ const unitMap: Record<string, StandardUnit> = {
740
+ count: StandardUnit.Count,
741
+ percent: StandardUnit.Percent,
742
+ seconds: StandardUnit.Seconds,
743
+ milliseconds: StandardUnit.Milliseconds,
744
+ bytes: StandardUnit.Bytes,
745
+ none: StandardUnit.None,
746
+ };
747
+ return unitMap[unit?.toLowerCase()] ?? StandardUnit.None;
748
+ }