@prism-d1/cli 1.0.27 → 1.0.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/eval-harness/README.md +114 -0
- package/dist/assets/eval-harness/eval-config.json +10 -0
- package/dist/assets/eval-harness/rubrics/agent-quality.json +79 -0
- package/dist/assets/eval-harness/rubrics/api-response-quality.json +45 -0
- package/dist/assets/eval-harness/rubrics/code-quality.json +98 -0
- package/dist/assets/eval-harness/rubrics/security-compliance.json +145 -0
- package/dist/assets/eval-harness/rubrics/spec-compliance.json +67 -0
- package/dist/assets/eval-harness/run-eval.sh +122 -0
- package/dist/assets/github-workflows/README.md +110 -0
- package/dist/assets/github-workflows/prism-agent-eval.yml +313 -0
- package/dist/assets/github-workflows/prism-ai-metrics.yml +261 -0
- package/dist/assets/github-workflows/prism-dora-weekly.yml +334 -0
- package/dist/assets/github-workflows/prism-eval-gate.yml +310 -0
- package/dist/assets/infra/bin/app.ts +56 -0
- package/dist/assets/infra/cdk.json +12 -0
- package/dist/assets/infra/lib/api-stack.ts +347 -0
- package/dist/assets/infra/lib/constructs/bedrock-guardrail-construct.ts +201 -0
- package/dist/assets/infra/lib/constructs/guardrail-enforcer-construct.ts +59 -0
- package/dist/assets/infra/lib/constructs/prism-vpc-construct.ts +75 -0
- package/dist/assets/infra/lib/constructs/security-agent-construct.ts +266 -0
- package/dist/assets/infra/lib/dashboard-stack.ts +1392 -0
- package/dist/assets/infra/lib/lambda/api-handler.ts +477 -0
- package/dist/assets/infra/lib/lambda/defect-correlator.ts +142 -0
- package/dist/assets/infra/lib/lambda/exfiltration-detector.ts +100 -0
- package/dist/assets/infra/lib/lambda/layers/guardrail-enforcer/nodejs/guardrail-enforcer.js +53 -0
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +748 -0
- package/dist/assets/infra/lib/lambda/security-agent-processor.ts +231 -0
- package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +120 -0
- package/dist/assets/infra/lib/lambda/security-response-automator.ts +130 -0
- package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +123 -0
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +701 -0
- package/dist/assets/infra/package.json +23 -0
- package/dist/assets/infra/tsconfig.json +24 -0
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +3 -4
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.js +2 -5
- package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
- package/dist/src/commands/securityagent/setup.d.ts.map +1 -1
- package/dist/src/commands/securityagent/setup.js +2 -3
- package/dist/src/commands/securityagent/setup.js.map +1 -1
- package/dist/src/commands/workshop/deploy-infra.d.ts.map +1 -1
- package/dist/src/commands/workshop/deploy-infra.js +2 -3
- package/dist/src/commands/workshop/deploy-infra.js.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.js +3 -8
- package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
- package/dist/src/commands/workshop/perform-pen-test.d.ts.map +1 -1
- package/dist/src/commands/workshop/perform-pen-test.js +5 -14
- package/dist/src/commands/workshop/perform-pen-test.js.map +1 -1
- package/dist/src/utils/root.d.ts +6 -0
- package/dist/src/utils/root.d.ts.map +1 -1
- package/dist/src/utils/root.js +29 -0
- package/dist/src/utils/root.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,748 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DynamoDBClient,
|
|
3
|
+
PutItemCommand,
|
|
4
|
+
} from '@aws-sdk/client-dynamodb';
|
|
5
|
+
import {
|
|
6
|
+
CloudWatchClient,
|
|
7
|
+
PutMetricDataCommand,
|
|
8
|
+
MetricDatum,
|
|
9
|
+
StandardUnit,
|
|
10
|
+
} from '@aws-sdk/client-cloudwatch';
|
|
11
|
+
|
|
12
|
+
// ---- Types ----
|
|
13
|
+
|
|
14
|
+
interface AiContext {
|
|
15
|
+
tool: string;
|
|
16
|
+
model: string;
|
|
17
|
+
origin: string;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface DoraMetrics {
|
|
21
|
+
deployment_frequency: number | null;
|
|
22
|
+
lead_time_seconds: number | null;
|
|
23
|
+
change_failure_rate: number | null;
|
|
24
|
+
mttr_seconds: number | null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface AiDoraMetrics {
|
|
28
|
+
ai_acceptance_rate: number | null;
|
|
29
|
+
ai_to_merge_ratio: number | null;
|
|
30
|
+
spec_to_code_hours: number | null;
|
|
31
|
+
post_merge_defect_rate: number | null;
|
|
32
|
+
eval_gate_pass_rate: number | null;
|
|
33
|
+
ai_test_coverage_delta: number | null;
|
|
34
|
+
total_input_tokens: number | null;
|
|
35
|
+
total_output_tokens: number | null;
|
|
36
|
+
total_cost_usd: number | null;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
interface EvalDetail {
|
|
40
|
+
eval_id: string;
|
|
41
|
+
rubric: string;
|
|
42
|
+
result: 'PASS' | 'FAIL';
|
|
43
|
+
score: number;
|
|
44
|
+
input_file: string;
|
|
45
|
+
pr_number?: number;
|
|
46
|
+
criterion_scores?: Array<{ name: string; score: number; max_score: number; reasoning: string }>;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface GuardrailTriggerDetail {
|
|
50
|
+
guardrail_id: string;
|
|
51
|
+
guardrail_name: string;
|
|
52
|
+
trigger_category: 'CONTENT_FILTER' | 'DENIED_TOPIC' | 'WORD_FILTER' | 'SENSITIVE_INFO' | 'CONTEXTUAL_GROUNDING';
|
|
53
|
+
trigger_type: string;
|
|
54
|
+
action_taken: 'BLOCK' | 'ANONYMIZE' | 'WARN';
|
|
55
|
+
agent_name: string;
|
|
56
|
+
invocation_id: string;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface MCPToolCallDetail {
|
|
60
|
+
session_id: string;
|
|
61
|
+
client_id: string;
|
|
62
|
+
tool_name: string;
|
|
63
|
+
scopes_used: string[];
|
|
64
|
+
authorized: boolean;
|
|
65
|
+
risk_level: string;
|
|
66
|
+
duration_ms: number;
|
|
67
|
+
result_status: 'success' | 'error' | 'denied';
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
interface QualityDetail {
|
|
72
|
+
deployment_id: string;
|
|
73
|
+
ai_defect_rate: number;
|
|
74
|
+
human_defect_rate: number;
|
|
75
|
+
total_ai_commits: number;
|
|
76
|
+
total_human_commits: number;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
interface SecurityDetail {
|
|
80
|
+
alert_type: string;
|
|
81
|
+
table_name: string;
|
|
82
|
+
principal_arn: string;
|
|
83
|
+
read_count: number;
|
|
84
|
+
window_start: string;
|
|
85
|
+
window_end: string;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
interface SecurityAgentFinding {
|
|
89
|
+
finding_id: string;
|
|
90
|
+
phase: 'design_review' | 'code_review' | 'pen_test';
|
|
91
|
+
severity: 'CRITICAL' | 'HIGH' | 'MEDIUM' | 'LOW' | 'INFORMATIONAL';
|
|
92
|
+
cvss_score: number | null;
|
|
93
|
+
title: string;
|
|
94
|
+
category: string;
|
|
95
|
+
cwe_id: string | null;
|
|
96
|
+
exploit_validated: boolean;
|
|
97
|
+
compliance_mappings: string[];
|
|
98
|
+
ai_origin: string;
|
|
99
|
+
spec_ref: string | null;
|
|
100
|
+
found_at: string;
|
|
101
|
+
remediated_at: string | null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
interface SecurityRemediationDetail {
|
|
105
|
+
finding_id: string;
|
|
106
|
+
severity: string;
|
|
107
|
+
remediation_time_hours: number;
|
|
108
|
+
remediated_by_origin: string;
|
|
109
|
+
finding_phase: string;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
interface MetricDetail {
|
|
113
|
+
team_id: string;
|
|
114
|
+
repo: string;
|
|
115
|
+
timestamp: string;
|
|
116
|
+
prism_level: number | string;
|
|
117
|
+
metric: { name: string; value: number; unit: string };
|
|
118
|
+
ai_context?: AiContext;
|
|
119
|
+
dora?: DoraMetrics;
|
|
120
|
+
ai_dora?: AiDoraMetrics;
|
|
121
|
+
agent?: {
|
|
122
|
+
agent_name: string;
|
|
123
|
+
steps_taken: number;
|
|
124
|
+
tools_invoked: number;
|
|
125
|
+
duration_ms: number;
|
|
126
|
+
tokens_used: number;
|
|
127
|
+
status: string;
|
|
128
|
+
guardrails_triggered: number;
|
|
129
|
+
};
|
|
130
|
+
eval?: EvalDetail;
|
|
131
|
+
guardrail?: GuardrailTriggerDetail;
|
|
132
|
+
mcp_tool_call?: MCPToolCallDetail;
|
|
133
|
+
quality?: QualityDetail;
|
|
134
|
+
security?: SecurityDetail;
|
|
135
|
+
security_agent_finding?: SecurityAgentFinding;
|
|
136
|
+
security_remediation?: SecurityRemediationDetail;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
interface EventBridgeEvent {
|
|
140
|
+
source: string;
|
|
141
|
+
'detail-type': string;
|
|
142
|
+
detail: MetricDetail;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// ---- Clients (reused across invocations) ----
|
|
146
|
+
|
|
147
|
+
const dynamoClient = new DynamoDBClient({});
|
|
148
|
+
const cloudwatchClient = new CloudWatchClient({});
|
|
149
|
+
|
|
150
|
+
const EVENTS_TABLE = process.env.EVENTS_TABLE!;
|
|
151
|
+
const METADATA_TABLE = process.env.METADATA_TABLE!;
|
|
152
|
+
const METRIC_NAMESPACE = process.env.METRIC_NAMESPACE ?? 'PRISM/D1/Velocity';
|
|
153
|
+
|
|
154
|
+
// ---- Handler ----
|
|
155
|
+
|
|
156
|
+
export async function handler(event: EventBridgeEvent): Promise<void> {
|
|
157
|
+
console.log('[metrics-processor] Received event:', JSON.stringify(event, null, 2));
|
|
158
|
+
|
|
159
|
+
const detailType = event['detail-type'];
|
|
160
|
+
const detail = event.detail;
|
|
161
|
+
|
|
162
|
+
console.log(`[metrics-processor] detail-type=${detailType} team_id=${detail?.team_id} repo=${detail?.repo} timestamp=${detail?.timestamp}`);
|
|
163
|
+
console.log(`[metrics-processor] dora=${JSON.stringify(detail?.dora)} ai_dora=${JSON.stringify(detail?.ai_dora)} metric=${JSON.stringify(detail?.metric)}`);
|
|
164
|
+
|
|
165
|
+
if (!detail.team_id) {
|
|
166
|
+
console.log('[metrics-processor] No team_id provided, defaulting to "no_team"');
|
|
167
|
+
detail.team_id = 'no_team';
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (!detail.repo || !detail.timestamp) {
|
|
171
|
+
console.error('[metrics-processor] VALIDATION FAILED: Missing required fields: repo or timestamp');
|
|
172
|
+
throw new Error('Event missing required fields');
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
const results = await Promise.allSettled([
|
|
176
|
+
writeEventToDynamo(detailType, detail),
|
|
177
|
+
writeMetadataToDynamo(detailType, detail),
|
|
178
|
+
publishCloudWatchMetrics(detailType, detail),
|
|
179
|
+
]);
|
|
180
|
+
|
|
181
|
+
results.forEach((result, idx) => {
|
|
182
|
+
const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics'];
|
|
183
|
+
if (result.status === 'fulfilled') {
|
|
184
|
+
console.log(`[metrics-processor] ${labels[idx]} succeeded`);
|
|
185
|
+
} else {
|
|
186
|
+
console.error(`[metrics-processor] ${labels[idx]} FAILED:`, result.reason);
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
const failures = results.filter((r) => r.status === 'rejected');
|
|
191
|
+
if (failures.length > 0) {
|
|
192
|
+
throw new Error(`${failures.length} operation(s) failed — check logs above`);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
console.log(`[metrics-processor] Successfully processed ${detailType} for ${detail.team_id}/${detail.repo}`);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// ---- DynamoDB events ----
|
|
199
|
+
|
|
200
|
+
async function writeEventToDynamo(
|
|
201
|
+
detailType: string,
|
|
202
|
+
detail: MetricDetail,
|
|
203
|
+
): Promise<void> {
|
|
204
|
+
console.log(`[writeEventToDynamo] Writing event: pk=${detail.team_id}#${detail.repo} sk=${detail.timestamp} type=${detailType}`);
|
|
205
|
+
const ttl = Math.floor(Date.now() / 1000) + 365 * 24 * 60 * 60; // 365 days from now
|
|
206
|
+
|
|
207
|
+
const data: Record<string, unknown> = {
|
|
208
|
+
team_id: detail.team_id,
|
|
209
|
+
repo: detail.repo,
|
|
210
|
+
prism_level: detail.prism_level ?? '1',
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
if (detail.metric) {
|
|
214
|
+
data.metric = detail.metric;
|
|
215
|
+
}
|
|
216
|
+
if (detail.ai_context) {
|
|
217
|
+
data.ai_context = detail.ai_context;
|
|
218
|
+
}
|
|
219
|
+
if (detail.dora) {
|
|
220
|
+
data.dora = detail.dora;
|
|
221
|
+
}
|
|
222
|
+
if (detail.ai_dora) {
|
|
223
|
+
data.ai_dora = detail.ai_dora;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const item: Record<string, { S?: string; N?: string }> = {
|
|
227
|
+
pk: { S: `${detail.team_id}#${detail.repo}` },
|
|
228
|
+
sk: { S: detail.timestamp },
|
|
229
|
+
detail_type: { S: detailType },
|
|
230
|
+
data: { S: JSON.stringify(data) },
|
|
231
|
+
ttl: { N: ttl.toString() },
|
|
232
|
+
};
|
|
233
|
+
|
|
234
|
+
// Store spec_ref as a top-level attribute for GSI queries (spec-to-code calculation)
|
|
235
|
+
const specRef = (detail.ai_context as any)?.spec_ref
|
|
236
|
+
?? (detail as any).spec_ref;
|
|
237
|
+
if (specRef && typeof specRef === 'string') {
|
|
238
|
+
item.spec_ref = { S: specRef };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
// Store eval rubric as a top-level attribute for per-rubric queries
|
|
242
|
+
if (detail.eval?.rubric) {
|
|
243
|
+
item.eval_rubric = { S: detail.eval.rubric };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Store finding_id for Security Agent finding queries
|
|
247
|
+
if (detail.security_agent_finding?.finding_id) {
|
|
248
|
+
item.finding_id = { S: detail.security_agent_finding.finding_id };
|
|
249
|
+
}
|
|
250
|
+
if (detail.security_remediation?.finding_id) {
|
|
251
|
+
item.finding_id = { S: detail.security_remediation.finding_id };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
await dynamoClient.send(
|
|
255
|
+
new PutItemCommand({
|
|
256
|
+
TableName: EVENTS_TABLE,
|
|
257
|
+
Item: item,
|
|
258
|
+
}),
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// ---- DynamoDB metadata ----
|
|
263
|
+
|
|
264
|
+
async function writeMetadataToDynamo(
|
|
265
|
+
detailType: string,
|
|
266
|
+
detail: MetricDetail,
|
|
267
|
+
): Promise<void> {
|
|
268
|
+
console.log(`[writeMetadataToDynamo] Writing metadata: team_id=${detail.team_id} repo=${detail.repo} type=${detailType}`);
|
|
269
|
+
const item: Record<string, { S?: string; N?: string }> = {
|
|
270
|
+
team_id: { S: detail.team_id },
|
|
271
|
+
repo: { S: detail.repo },
|
|
272
|
+
last_event_type: { S: detailType },
|
|
273
|
+
last_updated: { S: detail.timestamp },
|
|
274
|
+
prism_level: { N: String(detail.prism_level ?? 1) },
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
if (detail.ai_context?.tool) {
|
|
278
|
+
item.ai_tool = { S: detail.ai_context.tool };
|
|
279
|
+
}
|
|
280
|
+
if (detail.ai_context?.origin) {
|
|
281
|
+
item.ai_origin = { S: detail.ai_context.origin };
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// For assessment events, store the full PRISM level and primary metric
|
|
285
|
+
if (detailType === 'prism.d1.assessment' && detail.metric) {
|
|
286
|
+
item.assessment_metric = { S: detail.metric.name };
|
|
287
|
+
item.assessment_value = { N: detail.metric.value.toString() };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// Store latest DORA snapshot — only numeric fields as N attributes
|
|
291
|
+
if (detail.dora) {
|
|
292
|
+
for (const [key, val] of Object.entries(detail.dora)) {
|
|
293
|
+
if (val == null) continue;
|
|
294
|
+
if (typeof val === 'number') {
|
|
295
|
+
item[`dora_${key}`] = { N: val.toString() };
|
|
296
|
+
} else if (typeof val === 'string' && !isNaN(Number(val))) {
|
|
297
|
+
item[`dora_${key}`] = { N: val };
|
|
298
|
+
}
|
|
299
|
+
// Skip non-numeric values (e.g. deploy_sha) — they don't belong in N attributes
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// Store latest AI-DORA snapshot — only numeric fields
|
|
304
|
+
if (detail.ai_dora) {
|
|
305
|
+
for (const [key, val] of Object.entries(detail.ai_dora)) {
|
|
306
|
+
if (val == null) continue;
|
|
307
|
+
if (typeof val === 'object') continue; // Skip nested objects like tool_breakdown
|
|
308
|
+
if (typeof val === 'number') {
|
|
309
|
+
item[`ai_dora_${key}`] = { N: val.toString() };
|
|
310
|
+
} else if (typeof val === 'string' && !isNaN(Number(val))) {
|
|
311
|
+
item[`ai_dora_${key}`] = { N: val };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
await dynamoClient.send(
|
|
317
|
+
new PutItemCommand({
|
|
318
|
+
TableName: METADATA_TABLE,
|
|
319
|
+
Item: item,
|
|
320
|
+
}),
|
|
321
|
+
);
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// ---- CloudWatch custom metrics ----
|
|
325
|
+
|
|
326
|
+
async function publishCloudWatchMetrics(
|
|
327
|
+
detailType: string,
|
|
328
|
+
detail: MetricDetail,
|
|
329
|
+
): Promise<void> {
|
|
330
|
+
console.log(`[publishCloudWatchMetrics] Starting for ${detailType}, namespace=${METRIC_NAMESPACE}`);
|
|
331
|
+
console.log(`[publishCloudWatchMetrics] dora fields: deployment_frequency=${detail.dora?.deployment_frequency} lead_time_seconds=${detail.dora?.lead_time_seconds} change_failure_rate=${detail.dora?.change_failure_rate} mttr_seconds=${detail.dora?.mttr_seconds}`);
|
|
332
|
+
console.log(`[publishCloudWatchMetrics] ai_dora fields: ai_acceptance_rate=${detail.ai_dora?.ai_acceptance_rate} ai_to_merge_ratio=${detail.ai_dora?.ai_to_merge_ratio} eval_gate_pass_rate=${detail.ai_dora?.eval_gate_pass_rate}`);
|
|
333
|
+
|
|
334
|
+
const sharedDimensions = [
|
|
335
|
+
{ Name: 'TeamId', Value: detail.team_id },
|
|
336
|
+
{ Name: 'Repository', Value: detail.repo },
|
|
337
|
+
];
|
|
338
|
+
|
|
339
|
+
// Add AIOrigin dimension when available — enables dashboard filtering
|
|
340
|
+
// by ai-generated vs ai-assisted vs human
|
|
341
|
+
const aiOrigin = detail.ai_context?.origin;
|
|
342
|
+
const dimensionsWithOrigin = aiOrigin
|
|
343
|
+
? [...sharedDimensions, { Name: 'AIOrigin', Value: aiOrigin }]
|
|
344
|
+
: sharedDimensions;
|
|
345
|
+
|
|
346
|
+
// Clamp timestamp: CloudWatch rejects timestamps >2h in the future
|
|
347
|
+
const eventTime = new Date(detail.timestamp);
|
|
348
|
+
const metricTimestamp = eventTime.getTime() > Date.now() ? new Date() : eventTime;
|
|
349
|
+
|
|
350
|
+
const metricData: MetricDatum[] = [];
|
|
351
|
+
|
|
352
|
+
// Primary metric — published with both dimension sets for flexibility:
|
|
353
|
+
// 1. With AIOrigin: allows filtering by origin type
|
|
354
|
+
// 2. Without AIOrigin: allows aggregate queries across all origins
|
|
355
|
+
if (detail.metric?.value != null) {
|
|
356
|
+
metricData.push({
|
|
357
|
+
MetricName: detail.metric.name,
|
|
358
|
+
Value: detail.metric.value,
|
|
359
|
+
Unit: mapUnit(detail.metric.unit),
|
|
360
|
+
Dimensions: sharedDimensions,
|
|
361
|
+
Timestamp: metricTimestamp,
|
|
362
|
+
});
|
|
363
|
+
if (aiOrigin) {
|
|
364
|
+
metricData.push({
|
|
365
|
+
MetricName: detail.metric.name,
|
|
366
|
+
Value: detail.metric.value,
|
|
367
|
+
Unit: mapUnit(detail.metric.unit),
|
|
368
|
+
Dimensions: dimensionsWithOrigin,
|
|
369
|
+
Timestamp: metricTimestamp,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
// DORA metrics — published with AIOrigin dimension when available
|
|
375
|
+
if (detail.dora) {
|
|
376
|
+
const doraDims = aiOrigin ? dimensionsWithOrigin : sharedDimensions;
|
|
377
|
+
if (detail.dora.deployment_frequency != null) {
|
|
378
|
+
metricData.push({
|
|
379
|
+
MetricName: 'DeploymentFrequency',
|
|
380
|
+
Value: detail.dora.deployment_frequency,
|
|
381
|
+
Unit: StandardUnit.Count,
|
|
382
|
+
Dimensions: sharedDimensions,
|
|
383
|
+
Timestamp: metricTimestamp,
|
|
384
|
+
});
|
|
385
|
+
if (aiOrigin) {
|
|
386
|
+
metricData.push({
|
|
387
|
+
MetricName: 'DeploymentFrequency',
|
|
388
|
+
Value: detail.dora.deployment_frequency,
|
|
389
|
+
Unit: StandardUnit.Count,
|
|
390
|
+
Dimensions: doraDims,
|
|
391
|
+
Timestamp: metricTimestamp,
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
if (detail.dora.lead_time_seconds != null) {
|
|
396
|
+
metricData.push({
|
|
397
|
+
MetricName: 'LeadTimeForChanges',
|
|
398
|
+
Value: detail.dora.lead_time_seconds,
|
|
399
|
+
Unit: StandardUnit.Seconds,
|
|
400
|
+
Dimensions: sharedDimensions,
|
|
401
|
+
Timestamp: metricTimestamp,
|
|
402
|
+
});
|
|
403
|
+
if (aiOrigin) {
|
|
404
|
+
metricData.push({
|
|
405
|
+
MetricName: 'LeadTimeForChanges',
|
|
406
|
+
Value: detail.dora.lead_time_seconds,
|
|
407
|
+
Unit: StandardUnit.Seconds,
|
|
408
|
+
Dimensions: doraDims,
|
|
409
|
+
Timestamp: metricTimestamp,
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
if (detail.dora.change_failure_rate != null) {
|
|
414
|
+
const cfrValue = detail.dora.change_failure_rate <= 1 ? detail.dora.change_failure_rate * 100 : detail.dora.change_failure_rate;
|
|
415
|
+
metricData.push({
|
|
416
|
+
MetricName: 'ChangeFailureRate',
|
|
417
|
+
Value: cfrValue,
|
|
418
|
+
Unit: StandardUnit.Percent,
|
|
419
|
+
Dimensions: sharedDimensions,
|
|
420
|
+
Timestamp: metricTimestamp,
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
if (detail.dora.mttr_seconds != null) {
|
|
424
|
+
metricData.push({
|
|
425
|
+
MetricName: 'MTTR',
|
|
426
|
+
Value: detail.dora.mttr_seconds,
|
|
427
|
+
Unit: StandardUnit.Seconds,
|
|
428
|
+
Dimensions: sharedDimensions,
|
|
429
|
+
Timestamp: metricTimestamp,
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// AI-DORA metrics — scale 0–1 ratios to 0–100 for CloudWatch Percent unit
|
|
435
|
+
if (detail.ai_dora) {
|
|
436
|
+
const aiDoraMap: Array<[string, number | null, StandardUnit, boolean]> = [
|
|
437
|
+
['AIAcceptanceRate', detail.ai_dora.ai_acceptance_rate, StandardUnit.Percent, true],
|
|
438
|
+
['AIToMergeRatio', detail.ai_dora.ai_to_merge_ratio, StandardUnit.Percent, true],
|
|
439
|
+
['SpecToCodeHours', detail.ai_dora.spec_to_code_hours, StandardUnit.Count, false],
|
|
440
|
+
['PostMergeDefectRate', detail.ai_dora.post_merge_defect_rate, StandardUnit.Percent, true],
|
|
441
|
+
['EvalGatePassRate', detail.ai_dora.eval_gate_pass_rate, StandardUnit.Percent, true],
|
|
442
|
+
['AITestCoverageDelta', detail.ai_dora.ai_test_coverage_delta, StandardUnit.Percent, true],
|
|
443
|
+
['AIInputTokens', detail.ai_dora.total_input_tokens, StandardUnit.Count, false],
|
|
444
|
+
['AIOutputTokens', detail.ai_dora.total_output_tokens, StandardUnit.Count, false],
|
|
445
|
+
['AICostUSD', detail.ai_dora.total_cost_usd, StandardUnit.None, false],
|
|
446
|
+
];
|
|
447
|
+
|
|
448
|
+
for (const [name, value, unit, scaleToPercent] of aiDoraMap) {
|
|
449
|
+
if (value != null) {
|
|
450
|
+
const publishValue = scaleToPercent && value <= 1 ? value * 100 : value;
|
|
451
|
+
metricData.push({
|
|
452
|
+
MetricName: name,
|
|
453
|
+
Value: publishValue,
|
|
454
|
+
Unit: unit,
|
|
455
|
+
Dimensions: sharedDimensions,
|
|
456
|
+
Timestamp: metricTimestamp,
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
// Agent metrics
|
|
463
|
+
if (detail.agent) {
|
|
464
|
+
const agent = detail.agent;
|
|
465
|
+
const agentDimensions = [
|
|
466
|
+
...sharedDimensions,
|
|
467
|
+
{ Name: 'AgentName', Value: agent.agent_name ?? 'unknown' },
|
|
468
|
+
];
|
|
469
|
+
|
|
470
|
+
const agentMetrics: Array<[string, number | null, StandardUnit]> = [
|
|
471
|
+
['AgentInvocationCount', 1, StandardUnit.Count],
|
|
472
|
+
['AgentStepCount', agent.steps_taken ?? null, StandardUnit.Count],
|
|
473
|
+
['AgentDurationMs', agent.duration_ms ?? null, StandardUnit.Milliseconds],
|
|
474
|
+
['AgentTokensUsed', agent.tokens_used ?? null, StandardUnit.Count],
|
|
475
|
+
['AgentToolInvocationCount', agent.tools_invoked ?? null, StandardUnit.Count],
|
|
476
|
+
['AgentGuardrailTriggerCount', agent.guardrails_triggered ?? null, StandardUnit.Count],
|
|
477
|
+
['AgentSuccessRate', agent.status === 'success' ? 100 : 0, StandardUnit.Percent],
|
|
478
|
+
];
|
|
479
|
+
|
|
480
|
+
for (const [name, value, unit] of agentMetrics) {
|
|
481
|
+
if (value != null) {
|
|
482
|
+
// Publish with AgentName dimension (for per-agent drill-down)
|
|
483
|
+
metricData.push({
|
|
484
|
+
MetricName: name,
|
|
485
|
+
Value: value,
|
|
486
|
+
Unit: unit,
|
|
487
|
+
Dimensions: agentDimensions,
|
|
488
|
+
Timestamp: metricTimestamp,
|
|
489
|
+
});
|
|
490
|
+
// Also publish without AgentName (for aggregate dashboard queries)
|
|
491
|
+
metricData.push({
|
|
492
|
+
MetricName: name,
|
|
493
|
+
Value: value,
|
|
494
|
+
Unit: unit,
|
|
495
|
+
Dimensions: sharedDimensions,
|
|
496
|
+
Timestamp: metricTimestamp,
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// Eval metrics — per-rubric pass rate
|
|
503
|
+
if (detail.eval) {
|
|
504
|
+
const rubricDimensions = [
|
|
505
|
+
...sharedDimensions,
|
|
506
|
+
{ Name: 'RubricName', Value: detail.eval.rubric ?? 'unknown' },
|
|
507
|
+
];
|
|
508
|
+
metricData.push({
|
|
509
|
+
MetricName: 'EvalGatePassRateByRubric',
|
|
510
|
+
Value: detail.eval.result === 'PASS' ? 100 : 0,
|
|
511
|
+
Unit: StandardUnit.Percent,
|
|
512
|
+
Dimensions: rubricDimensions,
|
|
513
|
+
Timestamp: metricTimestamp,
|
|
514
|
+
});
|
|
515
|
+
metricData.push({
|
|
516
|
+
MetricName: 'EvalScore',
|
|
517
|
+
Value: detail.eval.score ?? 0,
|
|
518
|
+
Unit: StandardUnit.None,
|
|
519
|
+
Dimensions: rubricDimensions,
|
|
520
|
+
Timestamp: metricTimestamp,
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// Guardrail metrics — per-category trigger tracking
|
|
525
|
+
if (detail.guardrail) {
|
|
526
|
+
const guardrailDimensions = [
|
|
527
|
+
...sharedDimensions,
|
|
528
|
+
{ Name: 'TriggerCategory', Value: detail.guardrail.trigger_category },
|
|
529
|
+
{ Name: 'AgentName', Value: detail.guardrail.agent_name ?? 'unknown' },
|
|
530
|
+
];
|
|
531
|
+
metricData.push({
|
|
532
|
+
MetricName: 'GuardrailTriggerCount',
|
|
533
|
+
Value: 1,
|
|
534
|
+
Unit: StandardUnit.Count,
|
|
535
|
+
Dimensions: guardrailDimensions,
|
|
536
|
+
Timestamp: metricTimestamp,
|
|
537
|
+
});
|
|
538
|
+
if (detail.guardrail.action_taken === 'BLOCK') {
|
|
539
|
+
metricData.push({
|
|
540
|
+
MetricName: 'GuardrailBlockCount',
|
|
541
|
+
Value: 1,
|
|
542
|
+
Unit: StandardUnit.Count,
|
|
543
|
+
Dimensions: sharedDimensions,
|
|
544
|
+
Timestamp: metricTimestamp,
|
|
545
|
+
});
|
|
546
|
+
}
|
|
547
|
+
if (detail.guardrail.action_taken === 'ANONYMIZE') {
|
|
548
|
+
metricData.push({
|
|
549
|
+
MetricName: 'GuardrailAnonymizeCount',
|
|
550
|
+
Value: 1,
|
|
551
|
+
Unit: StandardUnit.Count,
|
|
552
|
+
Dimensions: sharedDimensions,
|
|
553
|
+
Timestamp: metricTimestamp,
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// MCP tool call metrics
|
|
559
|
+
if (detail.mcp_tool_call) {
|
|
560
|
+
const mcpDimensions = [
|
|
561
|
+
...sharedDimensions,
|
|
562
|
+
{ Name: 'ToolName', Value: detail.mcp_tool_call.tool_name },
|
|
563
|
+
];
|
|
564
|
+
metricData.push({
|
|
565
|
+
MetricName: 'MCPToolCallCount',
|
|
566
|
+
Value: 1,
|
|
567
|
+
Unit: StandardUnit.Count,
|
|
568
|
+
Dimensions: mcpDimensions,
|
|
569
|
+
Timestamp: metricTimestamp,
|
|
570
|
+
});
|
|
571
|
+
if (!detail.mcp_tool_call.authorized) {
|
|
572
|
+
metricData.push({
|
|
573
|
+
MetricName: 'MCPAuthDeniedCount',
|
|
574
|
+
Value: 1,
|
|
575
|
+
Unit: StandardUnit.Count,
|
|
576
|
+
Dimensions: mcpDimensions,
|
|
577
|
+
Timestamp: metricTimestamp,
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
if (detail.mcp_tool_call.duration_ms != null) {
|
|
581
|
+
metricData.push({
|
|
582
|
+
MetricName: 'MCPToolCallDurationMs',
|
|
583
|
+
Value: detail.mcp_tool_call.duration_ms,
|
|
584
|
+
Unit: StandardUnit.Milliseconds,
|
|
585
|
+
Dimensions: mcpDimensions,
|
|
586
|
+
Timestamp: metricTimestamp,
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
// Quality / defect rate metrics
|
|
594
|
+
if (detail.quality) {
|
|
595
|
+
metricData.push(
|
|
596
|
+
{
|
|
597
|
+
MetricName: 'PostMergeDefectRateAI',
|
|
598
|
+
Value: detail.quality.ai_defect_rate,
|
|
599
|
+
Unit: StandardUnit.Percent,
|
|
600
|
+
Dimensions: sharedDimensions,
|
|
601
|
+
Timestamp: metricTimestamp,
|
|
602
|
+
},
|
|
603
|
+
{
|
|
604
|
+
MetricName: 'PostMergeDefectRateHuman',
|
|
605
|
+
Value: detail.quality.human_defect_rate,
|
|
606
|
+
Unit: StandardUnit.Percent,
|
|
607
|
+
Dimensions: sharedDimensions,
|
|
608
|
+
Timestamp: metricTimestamp,
|
|
609
|
+
},
|
|
610
|
+
);
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
// Security / exfiltration metrics
|
|
614
|
+
if (detail.security) {
|
|
615
|
+
metricData.push({
|
|
616
|
+
MetricName: 'ExfiltrationAlertCount',
|
|
617
|
+
Value: 1,
|
|
618
|
+
Unit: StandardUnit.Count,
|
|
619
|
+
Dimensions: sharedDimensions,
|
|
620
|
+
Timestamp: metricTimestamp,
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
// AWS Security Agent finding metrics
|
|
625
|
+
if (detail.security_agent_finding) {
|
|
626
|
+
const finding = detail.security_agent_finding;
|
|
627
|
+
const phaseDimensions = [
|
|
628
|
+
...sharedDimensions,
|
|
629
|
+
{ Name: 'Phase', Value: finding.phase },
|
|
630
|
+
{ Name: 'Severity', Value: finding.severity },
|
|
631
|
+
];
|
|
632
|
+
metricData.push({
|
|
633
|
+
MetricName: 'SecurityFindingCount',
|
|
634
|
+
Value: 1,
|
|
635
|
+
Unit: StandardUnit.Count,
|
|
636
|
+
Dimensions: phaseDimensions,
|
|
637
|
+
Timestamp: metricTimestamp,
|
|
638
|
+
});
|
|
639
|
+
if (finding.severity === 'CRITICAL' || finding.severity === 'HIGH') {
|
|
640
|
+
metricData.push({
|
|
641
|
+
MetricName: 'SecurityCriticalFindingCount',
|
|
642
|
+
Value: 1,
|
|
643
|
+
Unit: StandardUnit.Count,
|
|
644
|
+
Dimensions: sharedDimensions,
|
|
645
|
+
Timestamp: metricTimestamp,
|
|
646
|
+
});
|
|
647
|
+
}
|
|
648
|
+
if (finding.ai_origin) {
|
|
649
|
+
metricData.push({
|
|
650
|
+
MetricName: 'SecurityFindingByOrigin',
|
|
651
|
+
Value: 1,
|
|
652
|
+
Unit: StandardUnit.Count,
|
|
653
|
+
Dimensions: [
|
|
654
|
+
...sharedDimensions,
|
|
655
|
+
{ Name: 'AIOrigin', Value: finding.ai_origin },
|
|
656
|
+
],
|
|
657
|
+
Timestamp: metricTimestamp,
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
if (finding.cvss_score != null) {
|
|
661
|
+
metricData.push({
|
|
662
|
+
MetricName: 'SecurityFindingCVSS',
|
|
663
|
+
Value: finding.cvss_score,
|
|
664
|
+
Unit: StandardUnit.None,
|
|
665
|
+
Dimensions: phaseDimensions,
|
|
666
|
+
Timestamp: metricTimestamp,
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
metricData.push({
|
|
670
|
+
MetricName: 'SecurityScanCount',
|
|
671
|
+
Value: 1,
|
|
672
|
+
Unit: StandardUnit.Count,
|
|
673
|
+
Dimensions: [
|
|
674
|
+
...sharedDimensions,
|
|
675
|
+
{ Name: 'Phase', Value: finding.phase },
|
|
676
|
+
],
|
|
677
|
+
Timestamp: metricTimestamp,
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// Security remediation metrics
|
|
682
|
+
if (detail.security_remediation) {
|
|
683
|
+
const remediation = detail.security_remediation;
|
|
684
|
+
metricData.push({
|
|
685
|
+
MetricName: 'SecurityRemediationTimeHours',
|
|
686
|
+
Value: remediation.remediation_time_hours,
|
|
687
|
+
Unit: StandardUnit.Count,
|
|
688
|
+
Dimensions: [
|
|
689
|
+
...sharedDimensions,
|
|
690
|
+
{ Name: 'Severity', Value: remediation.severity },
|
|
691
|
+
{ Name: 'AIOrigin', Value: remediation.remediated_by_origin },
|
|
692
|
+
],
|
|
693
|
+
Timestamp: metricTimestamp,
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
// Also publish all metrics WITHOUT dimensions for aggregate dashboard views.
|
|
698
|
+
// CloudWatch treats dimensioned and dimensionless metrics as separate time series.
|
|
699
|
+
// The dashboard-stack.ts widgets query without dimensions, so we need both.
|
|
700
|
+
const dimensionlessMetrics: MetricDatum[] = metricData
|
|
701
|
+
.filter((m) => m.Dimensions && m.Dimensions.length > 0)
|
|
702
|
+
.map((m) => ({
|
|
703
|
+
...m,
|
|
704
|
+
Dimensions: [],
|
|
705
|
+
}));
|
|
706
|
+
metricData.push(...dimensionlessMetrics);
|
|
707
|
+
|
|
708
|
+
if (metricData.length === 0) {
|
|
709
|
+
console.log('[publishCloudWatchMetrics] No metrics to publish — metricData is empty');
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
console.log(`[publishCloudWatchMetrics] Publishing ${metricData.length} metric data points`);
|
|
714
|
+
metricData.forEach((m, i) => {
|
|
715
|
+
console.log(`[publishCloudWatchMetrics] [${i}] ${m.MetricName}=${m.Value} unit=${m.Unit} dims=${JSON.stringify(m.Dimensions)}`);
|
|
716
|
+
});
|
|
717
|
+
|
|
718
|
+
// CloudWatch accepts max 1000 metric data points per call; batch in chunks of 25
|
|
719
|
+
const batchSize = 25;
|
|
720
|
+
for (let i = 0; i < metricData.length; i += batchSize) {
|
|
721
|
+
const batch = metricData.slice(i, i + batchSize);
|
|
722
|
+
console.log(`[publishCloudWatchMetrics] Sending batch ${Math.floor(i / batchSize) + 1} with ${batch.length} metrics`);
|
|
723
|
+
try {
|
|
724
|
+
await cloudwatchClient.send(
|
|
725
|
+
new PutMetricDataCommand({
|
|
726
|
+
Namespace: METRIC_NAMESPACE,
|
|
727
|
+
MetricData: batch,
|
|
728
|
+
}),
|
|
729
|
+
);
|
|
730
|
+
console.log(`[publishCloudWatchMetrics] Batch ${Math.floor(i / batchSize) + 1} sent successfully`);
|
|
731
|
+
} catch (err) {
|
|
732
|
+
console.error(`[publishCloudWatchMetrics] Batch ${Math.floor(i / batchSize) + 1} FAILED:`, err);
|
|
733
|
+
throw err;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
function mapUnit(unit: string): StandardUnit {
|
|
739
|
+
const unitMap: Record<string, StandardUnit> = {
|
|
740
|
+
count: StandardUnit.Count,
|
|
741
|
+
percent: StandardUnit.Percent,
|
|
742
|
+
seconds: StandardUnit.Seconds,
|
|
743
|
+
milliseconds: StandardUnit.Milliseconds,
|
|
744
|
+
bytes: StandardUnit.Bytes,
|
|
745
|
+
none: StandardUnit.None,
|
|
746
|
+
};
|
|
747
|
+
return unitMap[unit?.toLowerCase()] ?? StandardUnit.None;
|
|
748
|
+
}
|