@prism-d1/cli 1.0.83 → 1.0.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/coding-agent/README.md +824 -0
- package/dist/assets/coding-agent/agent.py +381 -0
- package/dist/assets/coding-agent/config.py +338 -0
- package/dist/assets/coding-agent/deploy/Dockerfile +86 -0
- package/dist/assets/coding-agent/deploy/create_harness.py +144 -0
- package/dist/assets/coding-agent/deploy/deploy-harness.sh +201 -0
- package/dist/assets/coding-agent/deploy/prism-coding-agent.yml +399 -0
- package/dist/assets/coding-agent/eval/run_eval.py +430 -0
- package/dist/assets/coding-agent/eval/run_harness_eval.py +262 -0
- package/dist/assets/coding-agent/eval/stubs/001-tags-element-validation.json +11 -0
- package/dist/assets/coding-agent/eval/stubs/002-status-filter.json +11 -0
- package/dist/assets/coding-agent/eval/stubs/003-refuse-test-deletion.json +11 -0
- package/dist/assets/coding-agent/fixtures/001-tags-element-validation.json +12 -0
- package/dist/assets/coding-agent/fixtures/002-status-filter.json +12 -0
- package/dist/assets/coding-agent/fixtures/003-refuse-test-deletion.json +10 -0
- package/dist/assets/coding-agent/pyproject.toml +28 -0
- package/dist/assets/coding-agent/requirements.txt +8 -0
- package/dist/assets/coding-agent/system_prompt.py +290 -0
- package/dist/assets/coding-agent/tools/__init__.py +11 -0
- package/dist/assets/coding-agent/tools/create_pr.py +141 -0
- package/dist/assets/coding-agent/tools/git_ops.py +197 -0
- package/dist/assets/github-workflows/prism-agent-eval.yml +8 -3
- package/dist/assets/github-workflows/prism-ai-metrics.yml +19 -6
- package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +14 -9
- package/dist/assets/github-workflows/prism-eval-gate.yml +11 -6
- package/dist/assets/gitlab-workflows/prism-agent-eval.yml +5 -3
- package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +5 -3
- package/dist/assets/gitlab-workflows/prism-eval-gate.yml +5 -3
- package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +134 -2
- package/dist/assets/infra/lib/dashboard-stack.ts +79 -95
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +99 -1
- package/dist/assets/infra/lib/lambda/otel-receiver.ts +318 -29
- package/dist/assets/infra/lib/lambda/productivity-widget.ts +85 -4
- package/dist/assets/infra/lib/lambda/velocity-widget.ts +265 -0
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +5 -0
- package/dist/src/commands/bootstrapper/install-coding-agent.d.ts +61 -0
- package/dist/src/commands/bootstrapper/install-coding-agent.d.ts.map +1 -0
- package/dist/src/commands/bootstrapper/install-coding-agent.js +599 -0
- package/dist/src/commands/bootstrapper/install-coding-agent.js.map +1 -0
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +16 -6
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.js +13 -6
- package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +9 -6
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.js +23 -0
- package/dist/src/commands/bootstrapper/setup-github-oidc.js.map +1 -1
- package/package.json +2 -2
|
@@ -27,6 +27,8 @@ import * as logs from 'aws-cdk-lib/aws-logs';
|
|
|
27
27
|
import * as apigwv2 from 'aws-cdk-lib/aws-apigatewayv2';
|
|
28
28
|
import { HttpLambdaIntegration } from 'aws-cdk-lib/aws-apigatewayv2-integrations';
|
|
29
29
|
import { HttpJwtAuthorizer } from 'aws-cdk-lib/aws-apigatewayv2-authorizers';
|
|
30
|
+
import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
|
|
31
|
+
import * as ssm from 'aws-cdk-lib/aws-ssm';
|
|
30
32
|
import * as path from 'path';
|
|
31
33
|
import { Construct } from 'constructs';
|
|
32
34
|
import { NagSuppressions } from 'cdk-nag';
|
|
@@ -54,6 +56,10 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
54
56
|
public readonly archiveBucket: s3.Bucket;
|
|
55
57
|
public readonly userPool?: cognito.UserPool;
|
|
56
58
|
public readonly userPoolClient?: cognito.UserPoolClient;
|
|
59
|
+
/** Machine-to-machine client used by the coding agent's telemetry emitter. */
|
|
60
|
+
public readonly agentClient?: cognito.UserPoolClient;
|
|
61
|
+
/** Its credentials, read in CI with the ephemeral OIDC role. */
|
|
62
|
+
public readonly agentSecret?: secretsmanager.Secret;
|
|
57
63
|
/** ARN of the CloudWatch custom-widget Lambda (Developer Productivity table). */
|
|
58
64
|
public productivityWidgetArn?: string;
|
|
59
65
|
/** The otel-receiver Lambda — other constructs may grant themselves invoke
|
|
@@ -74,6 +80,10 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
74
80
|
let issuer: string;
|
|
75
81
|
let clientId: string;
|
|
76
82
|
let identityClaim: string;
|
|
83
|
+
let domainPrefix = '';
|
|
84
|
+
let agentClientId = '';
|
|
85
|
+
let machineIdentities = '';
|
|
86
|
+
let tokenEndpoint = '';
|
|
77
87
|
|
|
78
88
|
if (byoIdp) {
|
|
79
89
|
issuer = props.externalIssuer!.replace(/\/$/, '');
|
|
@@ -102,8 +112,9 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
102
112
|
});
|
|
103
113
|
|
|
104
114
|
// Hosted UI domain — prefix must be globally unique per region.
|
|
115
|
+
domainPrefix = `prism-d1-otel-${cdk.Aws.ACCOUNT_ID}`;
|
|
105
116
|
this.userPool.addDomain('Domain', {
|
|
106
|
-
cognitoDomain: { domainPrefix
|
|
117
|
+
cognitoDomain: { domainPrefix },
|
|
107
118
|
});
|
|
108
119
|
|
|
109
120
|
// Public client (no secret) — codeburn does Authorization Code + PKCE
|
|
@@ -131,6 +142,85 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
131
142
|
// Cognito ACCESS tokens carry `username` (no email claim) — with
|
|
132
143
|
// email-alias sign-in and admin-created users, username == email.
|
|
133
144
|
identityClaim = props.identityClaim ?? 'username';
|
|
145
|
+
|
|
146
|
+
// -----------------------------------------------------------------
|
|
147
|
+
// Machine-to-machine path, for the coding agent's own telemetry
|
|
148
|
+
// -----------------------------------------------------------------
|
|
149
|
+
// The agent runs in CI, so there is no human to complete an
|
|
150
|
+
// authorization-code flow. SAX-02 Outcome 3 names Cognito's
|
|
151
|
+
// client-credentials flow as the machine-to-machine pattern, with a
|
|
152
|
+
// resource server carrying fine-grained scopes; the alternative — a
|
|
153
|
+
// Cognito user password in a repository secret — is what Outcome 1 lists
|
|
154
|
+
// as "hardcoding credentials in application code or environment
|
|
155
|
+
// variables".
|
|
156
|
+
//
|
|
157
|
+
// Client credentials require a resource server: Cognito rejects the grant
|
|
158
|
+
// for a client whose only scopes are the standard openid/email/profile.
|
|
159
|
+
const resourceServer = this.userPool.addResourceServer('PrismResourceServer', {
|
|
160
|
+
identifier: 'prism',
|
|
161
|
+
scopes: [{ scopeName: 'emit', scopeDescription: 'Emit usage and attribution spans' }],
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
this.agentClient = this.userPool.addClient('CodingAgentClient', {
|
|
165
|
+
userPoolClientName: 'prism-coding-agent',
|
|
166
|
+
generateSecret: true,
|
|
167
|
+
// No authorization-code or SRP flow: this identity is never a person, and
|
|
168
|
+
// leaving a human flow enabled on it would be a second way in.
|
|
169
|
+
authFlows: {},
|
|
170
|
+
oAuth: {
|
|
171
|
+
flows: { clientCredentials: true, authorizationCodeGrant: false },
|
|
172
|
+
scopes: [cognito.OAuthScope.resourceServer(resourceServer, {
|
|
173
|
+
scopeName: 'emit',
|
|
174
|
+
scopeDescription: 'Emit usage and attribution spans',
|
|
175
|
+
})],
|
|
176
|
+
},
|
|
177
|
+
accessTokenValidity: cdk.Duration.hours(1),
|
|
178
|
+
enableTokenRevocation: true,
|
|
179
|
+
preventUserExistenceErrors: true,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
// The credentials the CI job reads with its ephemeral OIDC role. Named
|
|
183
|
+
// under prism-d1- because setup-github-oidc grants GetSecretValue scoped
|
|
184
|
+
// to that prefix rather than to '*' — the role is assumed by a workflow
|
|
185
|
+
// that runs agent-authored code.
|
|
186
|
+
this.agentSecret = new secretsmanager.Secret(this, 'CodingAgentOidcSecret', {
|
|
187
|
+
secretName: 'prism-d1-agent-oidc',
|
|
188
|
+
description: 'Cognito client credentials for the PRISM coding agent telemetry emitter',
|
|
189
|
+
secretObjectValue: {
|
|
190
|
+
client_id: cdk.SecretValue.unsafePlainText(this.agentClient.userPoolClientId),
|
|
191
|
+
// unsafePlainText names the risk accurately and it does not apply here:
|
|
192
|
+
// the value is a CloudFormation reference resolved at deploy time, not a
|
|
193
|
+
// literal in source. The client id is not sensitive; the secret is, and
|
|
194
|
+
// it never leaves CloudFormation except into this secret.
|
|
195
|
+
client_secret: this.agentClient.userPoolClientSecret,
|
|
196
|
+
},
|
|
197
|
+
removalPolicy: cdk.RemovalPolicy.RETAIN,
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
// Cognito app-client secrets cannot be rotated in place — you would recreate
|
|
201
|
+
// the client, which invalidates every outstanding token and every copy of the
|
|
202
|
+
// old secret. The suppression documents why rotation is not configured rather
|
|
203
|
+
// than silently ignoring the rule.
|
|
204
|
+
NagSuppressions.addResourceSuppressions(this.agentSecret, [
|
|
205
|
+
{
|
|
206
|
+
id: 'AwsSolutions-SMG4',
|
|
207
|
+
reason:
|
|
208
|
+
'Cognito app-client secrets are immutable: rotating requires replacing the ' +
|
|
209
|
+
'client, which invalidates all outstanding tokens. Rotation would be a ' +
|
|
210
|
+
'client-replacement Lambda on a 90-day schedule, acceptable for a workshop ' +
|
|
211
|
+
'sample but not implemented here. The secret is scoped to one user pool and ' +
|
|
212
|
+
'grants only the prism/emit scope.',
|
|
213
|
+
},
|
|
214
|
+
]);
|
|
215
|
+
|
|
216
|
+
agentClientId = this.agentClient.userPoolClientId;
|
|
217
|
+
// A client-credentials token has no username, no email, and a `sub` equal to
|
|
218
|
+
// the app client id — so identity has to be mapped rather than read. Mapped
|
|
219
|
+
// here, at deploy time, precisely so it is NOT caller-supplied: a payload
|
|
220
|
+
// that could name its own author would let any holder of any token post as
|
|
221
|
+
// anyone.
|
|
222
|
+
machineIdentities = JSON.stringify({ [agentClientId]: 'prism-coding-agent' });
|
|
223
|
+
tokenEndpoint = `https://${domainPrefix}.auth.${cdk.Aws.REGION}.amazoncognito.com/oauth2/token`;
|
|
134
224
|
}
|
|
135
225
|
|
|
136
226
|
// -------------------------------------------------------
|
|
@@ -191,6 +281,9 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
191
281
|
IDENTITY_CLAIM: identityClaim,
|
|
192
282
|
OIDC_ISSUER: issuer,
|
|
193
283
|
OIDC_CLIENT_ID: clientId,
|
|
284
|
+
// clientId -> identity, for tokens that carry no username. Deploy-time
|
|
285
|
+
// config so the author of a span is never caller-supplied.
|
|
286
|
+
MACHINE_IDENTITIES: machineIdentities,
|
|
194
287
|
MAX_BATCH_SIZE: '1000',
|
|
195
288
|
SPAN_TTL_DAYS: '90',
|
|
196
289
|
},
|
|
@@ -271,8 +364,12 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
271
364
|
|
|
272
365
|
// HTTP API JWT authorizers validate the `aud` claim, or `client_id` for
|
|
273
366
|
// Cognito access tokens — both paths work with jwtAudience = [clientId].
|
|
367
|
+
// Both client ids: API Gateway validates `aud`, or `client_id` for Cognito
|
|
368
|
+
// access tokens, and a client-credentials token carries the M2M client id.
|
|
369
|
+
// Listing only the codeburn client would reject every agent span with a 401
|
|
370
|
+
// that looks like a bad token rather than a missing audience entry.
|
|
274
371
|
const authorizer = new HttpJwtAuthorizer('OtelJwtAuthorizer', issuer, {
|
|
275
|
-
jwtAudience: [clientId],
|
|
372
|
+
jwtAudience: agentClientId ? [clientId, agentClientId] : [clientId],
|
|
276
373
|
});
|
|
277
374
|
|
|
278
375
|
// Discovery doc — unauthenticated by design (it only exposes issuer +
|
|
@@ -365,5 +462,40 @@ export class OtelCollectorConstruct extends Construct {
|
|
|
365
462
|
});
|
|
366
463
|
poolOutput.overrideLogicalId('OtelUserPoolId');
|
|
367
464
|
}
|
|
465
|
+
if (this.agentSecret) {
|
|
466
|
+
// SSM parameters: the workflow reads these at run time instead of requiring
|
|
467
|
+
// GitHub org variables. One source of truth, written by CDK at deploy time,
|
|
468
|
+
// cannot drift from what's actually deployed.
|
|
469
|
+
const ssmPrefix = '/prism/d1';
|
|
470
|
+
new ssm.StringParameter(this, 'ParamCollectorUrl', {
|
|
471
|
+
parameterName: `${ssmPrefix}/collector-url`,
|
|
472
|
+
stringValue: this.httpApi.apiEndpoint,
|
|
473
|
+
description: 'OTEL collector URL — read by the coding agent workflow',
|
|
474
|
+
tier: ssm.ParameterTier.STANDARD,
|
|
475
|
+
});
|
|
476
|
+
new ssm.StringParameter(this, 'ParamTokenEndpoint', {
|
|
477
|
+
parameterName: `${ssmPrefix}/token-endpoint`,
|
|
478
|
+
stringValue: tokenEndpoint,
|
|
479
|
+
description: 'Cognito token endpoint — read by the coding agent workflow',
|
|
480
|
+
tier: ssm.ParameterTier.STANDARD,
|
|
481
|
+
});
|
|
482
|
+
new ssm.StringParameter(this, 'ParamAgentSecretId', {
|
|
483
|
+
parameterName: `${ssmPrefix}/agent-secret-id`,
|
|
484
|
+
stringValue: this.agentSecret.secretName,
|
|
485
|
+
description: 'Secrets Manager id for the M2M client credentials',
|
|
486
|
+
tier: ssm.ParameterTier.STANDARD,
|
|
487
|
+
});
|
|
488
|
+
|
|
489
|
+
// Outputs kept for discoverability — printed by `cdk deploy`, but no longer
|
|
490
|
+
// need to be manually copied into GitHub.
|
|
491
|
+
new cdk.CfnOutput(this, 'PrismAgentTokenEndpoint', {
|
|
492
|
+
value: tokenEndpoint,
|
|
493
|
+
description: 'Also available as SSM /prism/d1/token-endpoint',
|
|
494
|
+
}).overrideLogicalId('PrismAgentTokenEndpoint');
|
|
495
|
+
new cdk.CfnOutput(this, 'PrismAgentSecretId', {
|
|
496
|
+
value: this.agentSecret.secretName,
|
|
497
|
+
description: 'Also available as SSM /prism/d1/agent-secret-id',
|
|
498
|
+
}).overrideLogicalId('PrismAgentSecretId');
|
|
499
|
+
}
|
|
368
500
|
}
|
|
369
501
|
}
|
|
@@ -92,47 +92,19 @@ export class DashboardStack extends cdk.Stack {
|
|
|
92
92
|
});
|
|
93
93
|
|
|
94
94
|
teamDashboard.addWidgets(
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
stacked: true,
|
|
103
|
-
width: 8,
|
|
104
|
-
height: 6,
|
|
105
|
-
leftYAxis: { min: 0, label: 'Commits' },
|
|
106
|
-
}),
|
|
107
|
-
new cloudwatch.GraphWidget({
|
|
108
|
-
title: 'Merge Ratio: AI vs Human',
|
|
109
|
-
left: [
|
|
110
|
-
// Both lines from attribution spans. Divergence is the signal:
|
|
111
|
-
// AI code merging at a materially lower rate than human code means
|
|
112
|
-
// review friction or quality problems, not just "AI is used a lot".
|
|
113
|
-
new cloudwatch.MathExpression({
|
|
114
|
-
expression: 'FILL(IF(FILL(aiCommits, 0) > 0, 100 * FILL(mergedAi, 0) / FILL(aiCommits, 0)), REPEAT)',
|
|
115
|
-
usingMetrics: {
|
|
116
|
-
mergedAi: dailySum('MergedAICommits'),
|
|
117
|
-
aiCommits: dailySum('AICommits'),
|
|
118
|
-
},
|
|
119
|
-
label: 'AI merge rate (%)',
|
|
120
|
-
period: cdk.Duration.days(1),
|
|
121
|
-
}),
|
|
122
|
-
new cloudwatch.MathExpression({
|
|
123
|
-
expression: 'FILL(IF(FILL(humanCommits, 0) > 0, 100 * FILL(mergedHuman, 0) / FILL(humanCommits, 0)), REPEAT)',
|
|
124
|
-
usingMetrics: {
|
|
125
|
-
mergedHuman: dailySum('MergedHumanCommits'),
|
|
126
|
-
humanCommits: dailySum('HumanCommits'),
|
|
127
|
-
},
|
|
128
|
-
label: 'Human merge rate (%)',
|
|
129
|
-
period: cdk.Duration.days(1),
|
|
130
|
-
}),
|
|
131
|
-
],
|
|
132
|
-
width: 8,
|
|
95
|
+
// Commits / Day and Merge Ratio replaced by DDB-backed custom widget to
|
|
96
|
+
// avoid CloudWatch increment-only double-count when CI-seeded human items
|
|
97
|
+
// are later upgraded to AI by codeburn attribution.
|
|
98
|
+
...(props?.velocityWidgetArn ? [new cloudwatch.CustomWidget({
|
|
99
|
+
functionArn: props.velocityWidgetArn,
|
|
100
|
+
title: 'Commits & Merge Rate (AI vs Human)',
|
|
101
|
+
width: 16,
|
|
133
102
|
height: 6,
|
|
134
|
-
|
|
135
|
-
|
|
103
|
+
params: { view: 'commits-trend' },
|
|
104
|
+
updateOnRefresh: true,
|
|
105
|
+
updateOnResize: false,
|
|
106
|
+
updateOnTimeRangeChange: true,
|
|
107
|
+
})] : []),
|
|
136
108
|
new cloudwatch.GraphWidget({
|
|
137
109
|
title: 'AI Defect Trend (reverted / merged)',
|
|
138
110
|
left: [
|
|
@@ -232,18 +204,17 @@ export class DashboardStack extends cdk.Stack {
|
|
|
232
204
|
});
|
|
233
205
|
|
|
234
206
|
execDashboard.addWidgets(
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
weeklySum('HumanCommits', 'Human commits'),
|
|
240
|
-
],
|
|
241
|
-
view: cloudwatch.GraphWidgetView.BAR,
|
|
242
|
-
stacked: true,
|
|
207
|
+
// AI vs Human Commits (Weekly) replaced by DDB-backed custom widget.
|
|
208
|
+
...(props?.velocityWidgetArn ? [new cloudwatch.CustomWidget({
|
|
209
|
+
functionArn: props.velocityWidgetArn,
|
|
210
|
+
title: 'AI vs Human Commits & Merge Rate',
|
|
243
211
|
width: 8,
|
|
244
212
|
height: 6,
|
|
245
|
-
|
|
246
|
-
|
|
213
|
+
params: { view: 'commits-trend' },
|
|
214
|
+
updateOnRefresh: true,
|
|
215
|
+
updateOnResize: false,
|
|
216
|
+
updateOnTimeRangeChange: true,
|
|
217
|
+
})] : []),
|
|
247
218
|
new cloudwatch.GraphWidget({
|
|
248
219
|
title: 'AI Spend Trend (Weekly)',
|
|
249
220
|
left: [weeklySum('AICostUSD', 'Spend ($)')],
|
|
@@ -273,27 +244,11 @@ export class DashboardStack extends cdk.Stack {
|
|
|
273
244
|
);
|
|
274
245
|
|
|
275
246
|
// Row 4: quality — "is AI code as reliable as human code?"
|
|
247
|
+
// Merge Rate: AI vs Human is already included in the commits-trend custom
|
|
248
|
+
// widget above (DDB-backed, avoids CloudWatch double-count). Only the AI
|
|
249
|
+
// Defect Trend (RevertedAICommits / MergedAICommits) remains as CloudWatch
|
|
250
|
+
// native because it uses AI-only metrics with no upgrade double-count issue.
|
|
276
251
|
execDashboard.addWidgets(
|
|
277
|
-
new cloudwatch.GraphWidget({
|
|
278
|
-
title: 'Merge Rate: AI vs Human (Weekly)',
|
|
279
|
-
left: [
|
|
280
|
-
new cloudwatch.MathExpression({
|
|
281
|
-
expression: 'FILL(IF(FILL(xAi, 0) > 0, 100 * FILL(xMergedAi, 0) / FILL(xAi, 0)), REPEAT)',
|
|
282
|
-
usingMetrics: { xMergedAi: weeklySum('MergedAICommits'), xAi: weeklySum('AICommits') },
|
|
283
|
-
label: 'AI merge rate (%)',
|
|
284
|
-
period: cdk.Duration.days(7),
|
|
285
|
-
}),
|
|
286
|
-
new cloudwatch.MathExpression({
|
|
287
|
-
expression: 'FILL(IF(FILL(xHuman, 0) > 0, 100 * FILL(xMergedHuman, 0) / FILL(xHuman, 0)), REPEAT)',
|
|
288
|
-
usingMetrics: { xMergedHuman: weeklySum('MergedHumanCommits'), xHuman: weeklySum('HumanCommits') },
|
|
289
|
-
label: 'Human merge rate (%)',
|
|
290
|
-
period: cdk.Duration.days(7),
|
|
291
|
-
}),
|
|
292
|
-
],
|
|
293
|
-
width: 12,
|
|
294
|
-
height: 6,
|
|
295
|
-
leftYAxis: { min: 0, max: 100, label: 'Percent' },
|
|
296
|
-
}),
|
|
297
252
|
new cloudwatch.GraphWidget({
|
|
298
253
|
title: 'AI Defect Trend (Weekly)',
|
|
299
254
|
left: [
|
|
@@ -739,11 +694,26 @@ export class DashboardStack extends cdk.Stack {
|
|
|
739
694
|
});
|
|
740
695
|
|
|
741
696
|
devDashboard.addWidgets(
|
|
742
|
-
|
|
743
|
-
|
|
697
|
+
// AI Share and AI Merge Rate are DDB-backed (accurate, no CloudWatch delay)
|
|
698
|
+
new cloudwatch.CustomWidget({
|
|
699
|
+
functionArn: props.velocityWidgetArn ?? '',
|
|
700
|
+
title: 'AI Share of Commits',
|
|
701
|
+
width: 6,
|
|
702
|
+
height: 4,
|
|
703
|
+
params: { view: 'ai-share-kpi' },
|
|
704
|
+
updateOnRefresh: true,
|
|
705
|
+
updateOnResize: false,
|
|
706
|
+
updateOnTimeRangeChange: true,
|
|
744
707
|
}),
|
|
745
|
-
|
|
746
|
-
|
|
708
|
+
new cloudwatch.CustomWidget({
|
|
709
|
+
functionArn: props.velocityWidgetArn ?? '',
|
|
710
|
+
title: 'AI Merge Rate',
|
|
711
|
+
width: 6,
|
|
712
|
+
height: 4,
|
|
713
|
+
params: { view: 'merge-rate-kpi' },
|
|
714
|
+
updateOnRefresh: true,
|
|
715
|
+
updateOnResize: false,
|
|
716
|
+
updateOnTimeRangeChange: true,
|
|
747
717
|
}),
|
|
748
718
|
ratioKpi('Cost per Shipped Commit ($)', 'IF(FILL(kMerged2, 0) > 0, FILL(kCost, 0) / FILL(kMerged2, 0))', {
|
|
749
719
|
kCost: dailyMetric('AICostUSD', 'kCost'), kMerged2: dailyMetric('MergedAICommits', 'kMerged2'),
|
|
@@ -757,19 +727,18 @@ export class DashboardStack extends cdk.Stack {
|
|
|
757
727
|
}),
|
|
758
728
|
);
|
|
759
729
|
|
|
760
|
-
// --- Row 2: Org trends
|
|
730
|
+
// --- Row 2: Org trends ---
|
|
761
731
|
devDashboard.addWidgets(
|
|
762
|
-
|
|
732
|
+
// Commits bar chart — just AI vs Human stacked bars, no KPIs or merge rate
|
|
733
|
+
new cloudwatch.CustomWidget({
|
|
734
|
+
functionArn: props.velocityWidgetArn ?? '',
|
|
763
735
|
title: 'Commits / Day (AI vs Human)',
|
|
764
|
-
left: [
|
|
765
|
-
dailyMetric('AICommits', 'AI commits'),
|
|
766
|
-
dailyMetric('HumanCommits', 'Human commits'),
|
|
767
|
-
],
|
|
768
|
-
view: cloudwatch.GraphWidgetView.BAR,
|
|
769
|
-
stacked: true,
|
|
770
736
|
width: 8,
|
|
771
737
|
height: 6,
|
|
772
|
-
|
|
738
|
+
params: { view: 'commits-bar' },
|
|
739
|
+
updateOnRefresh: true,
|
|
740
|
+
updateOnResize: false,
|
|
741
|
+
updateOnTimeRangeChange: true,
|
|
773
742
|
}),
|
|
774
743
|
new cloudwatch.GraphWidget({
|
|
775
744
|
title: 'AI Spend / Day',
|
|
@@ -778,23 +747,38 @@ export class DashboardStack extends cdk.Stack {
|
|
|
778
747
|
height: 6,
|
|
779
748
|
leftYAxis: { min: 0, label: 'USD' },
|
|
780
749
|
}),
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
expression: 'FILL(IF(FILL(tAi, 0) > 0, 100 * FILL(tMerged, 0) / FILL(tAi, 0)), REPEAT)',
|
|
786
|
-
usingMetrics: { tMerged: dailyMetric('MergedAICommits', 'tMerged'), tAi: dailyMetric('AICommits', 'tAi') },
|
|
787
|
-
label: 'Merge ratio (%)',
|
|
788
|
-
period: cdk.Duration.days(1),
|
|
789
|
-
}),
|
|
790
|
-
],
|
|
750
|
+
// Merge Rate Trend — DDB-backed line chart replacing CloudWatch AI Merge Ratio Trend
|
|
751
|
+
new cloudwatch.CustomWidget({
|
|
752
|
+
functionArn: props.velocityWidgetArn ?? '',
|
|
753
|
+
title: 'Merge Rate Trend (AI vs Human)',
|
|
791
754
|
width: 8,
|
|
792
755
|
height: 6,
|
|
793
|
-
|
|
756
|
+
params: { view: 'merge-rate-trend' },
|
|
757
|
+
updateOnRefresh: true,
|
|
758
|
+
updateOnResize: false,
|
|
759
|
+
updateOnTimeRangeChange: true,
|
|
760
|
+
}),
|
|
761
|
+
);
|
|
762
|
+
|
|
763
|
+
// --- Row 3: The coding agent's own work ---
|
|
764
|
+
// Its own row rather than a column in the team table: autonomous work is not
|
|
765
|
+
// human-assisted AI use, and the receiver keeps it out of `totals` for the
|
|
766
|
+
// same reason. Placed above the human table because an agent that ships more
|
|
767
|
+
// than a developer should not be discovered by scrolling.
|
|
768
|
+
devDashboard.addWidgets(
|
|
769
|
+
new cloudwatch.CustomWidget({
|
|
770
|
+
functionArn: props.productivityWidgetArn,
|
|
771
|
+
title: 'PRISM Coding Agent — issues, commits and spend',
|
|
772
|
+
width: 24,
|
|
773
|
+
height: 7,
|
|
774
|
+
params: { view: 'coding-agent' },
|
|
775
|
+
updateOnRefresh: true,
|
|
776
|
+
updateOnResize: false,
|
|
777
|
+
updateOnTimeRangeChange: true,
|
|
794
778
|
}),
|
|
795
779
|
);
|
|
796
780
|
|
|
797
|
-
// --- Row
|
|
781
|
+
// --- Row 4: Team comparison table (custom widget, view=table) ---
|
|
798
782
|
devDashboard.addWidgets(
|
|
799
783
|
new cloudwatch.CustomWidget({
|
|
800
784
|
functionArn: props.productivityWidgetArn,
|
|
@@ -196,6 +196,11 @@ const cloudwatchClient = new CloudWatchClient({});
|
|
|
196
196
|
const EVENTS_TABLE = process.env.EVENTS_TABLE!;
|
|
197
197
|
const METADATA_TABLE = process.env.METADATA_TABLE!;
|
|
198
198
|
const METRIC_NAMESPACE = process.env.METRIC_NAMESPACE ?? 'PRISM/D1/Velocity';
|
|
199
|
+
// Optional: when set, PR events seed the attribution store with default-human
|
|
200
|
+
// COMMIT# items so that commits not yet pushed by codeburn are visible as human
|
|
201
|
+
// rather than absent. The otel-receiver's condition expression allows a later
|
|
202
|
+
// codeburn push to upgrade human → ai-generated, so ordering does not matter.
|
|
203
|
+
const AI_USAGE_TABLE = process.env.AI_USAGE_TABLE;
|
|
199
204
|
|
|
200
205
|
|
|
201
206
|
// ---- Handler ----
|
|
@@ -220,9 +225,10 @@ export async function handler(event: EventBridgeEvent): Promise<void> {
|
|
|
220
225
|
writeEventToDynamo(detailType, detail),
|
|
221
226
|
writeMetadataToDynamo(detailType, detail),
|
|
222
227
|
publishCloudWatchMetrics(detailType, detail),
|
|
228
|
+
seedCommitAttribution(detailType, detail),
|
|
223
229
|
]);
|
|
224
230
|
|
|
225
|
-
const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics'];
|
|
231
|
+
const labels = ['writeEventToDynamo', 'writeMetadataToDynamo', 'publishCloudWatchMetrics', 'seedCommitAttribution'];
|
|
226
232
|
results.forEach((result, idx) => {
|
|
227
233
|
if (result.status === 'rejected') {
|
|
228
234
|
console.error(`[metrics-processor] ${labels[idx]} FAILED:`, result.reason);
|
|
@@ -844,3 +850,95 @@ function mapUnit(unit: string): StandardUnit {
|
|
|
844
850
|
};
|
|
845
851
|
return unitMap[unit?.toLowerCase()] ?? StandardUnit.None;
|
|
846
852
|
}
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
// ---- Commit attribution seeding ----
|
|
856
|
+
|
|
857
|
+
/**
|
|
858
|
+
* When a prism.d1.pr event arrives with commit_shas, seed a COMMIT# item for
|
|
859
|
+
* each SHA in the attribution store with ai_origin=human and in_main=true.
|
|
860
|
+
*
|
|
861
|
+
* This ensures human commits are explicitly represented rather than invisible.
|
|
862
|
+
* The otel-receiver's condition expression allows a later codeburn push to
|
|
863
|
+
* upgrade human → ai-generated:
|
|
864
|
+
*
|
|
865
|
+
* OR (ai_origin = :human AND :ao = :ai)
|
|
866
|
+
*
|
|
867
|
+
* So ordering between PR merge and codeburn sync does not matter:
|
|
868
|
+
* - If codeburn pushed first: item already has ai_origin=ai-generated, this
|
|
869
|
+
* write is rejected by the condition (never downgrade ai→human). ✓
|
|
870
|
+
* - If PR merges first: item is written as human, codeburn upgrades later. ✓
|
|
871
|
+
* - Simultaneous: DDB conditional write is atomic, one wins. ✓
|
|
872
|
+
*
|
|
873
|
+
* The repo format from the CI workflow is bare (owner/name), but codeburn
|
|
874
|
+
* stores with a host prefix (github.com/owner/name). We write under the bare
|
|
875
|
+
* form because the /v1/repos and /v1/productivity endpoints scan ALL COMMIT#
|
|
876
|
+
* items regardless of repo format, and repoKeyCandidates() in the security
|
|
877
|
+
* processor already handles both forms for point lookups.
|
|
878
|
+
*/
|
|
879
|
+
async function seedCommitAttribution(detailType: string, detail: any): Promise<void> {
|
|
880
|
+
// Only process prism.d1.pr events that carry commit SHAs
|
|
881
|
+
if (detailType !== 'prism.d1.pr') return;
|
|
882
|
+
if (!AI_USAGE_TABLE) return;
|
|
883
|
+
|
|
884
|
+
const pr = detail.pr as PrDetail | undefined;
|
|
885
|
+
const shas = pr?.commit_shas;
|
|
886
|
+
if (!shas || shas.length === 0) return;
|
|
887
|
+
|
|
888
|
+
const authors: string[] = (pr as any)?.commit_authors ?? [];
|
|
889
|
+
const repo = detail.repo as string;
|
|
890
|
+
// CI workflows now emit fully-qualified repo names (github.com/owner/repo
|
|
891
|
+
// or gitlab.com/group/project) matching codeburn's convention. No
|
|
892
|
+
// normalization needed — the ConditionExpression hits existing items directly.
|
|
893
|
+
const timestamp = detail.timestamp as string ?? new Date().toISOString();
|
|
894
|
+
const ttl = Math.floor(Date.now() / 1000) + 365 * 86400; // 1 year
|
|
895
|
+
|
|
896
|
+
let seeded = 0;
|
|
897
|
+
for (let i = 0; i < shas.length; i++) {
|
|
898
|
+
const sha = shas[i];
|
|
899
|
+
const author = authors[i] ?? ''; // parallel array; may be shorter or absent
|
|
900
|
+
try {
|
|
901
|
+
await dynamoClient.send(new PutItemCommand({
|
|
902
|
+
TableName: AI_USAGE_TABLE,
|
|
903
|
+
Item: {
|
|
904
|
+
pk: { S: `REPO#${repo}` },
|
|
905
|
+
sk: { S: `COMMIT#${sha}` },
|
|
906
|
+
record_type: { S: 'OTEL_ATTR_COMMIT' },
|
|
907
|
+
ai_origin: { S: 'human' },
|
|
908
|
+
ai_tool: { S: 'none' },
|
|
909
|
+
ai_model: { S: '' },
|
|
910
|
+
origin_source: { S: 'ci-seeded' },
|
|
911
|
+
in_main: { BOOL: true },
|
|
912
|
+
was_reverted: { BOOL: false },
|
|
913
|
+
timestamp: { S: timestamp },
|
|
914
|
+
updated_at: { S: new Date().toISOString() },
|
|
915
|
+
ttl: { N: String(ttl) },
|
|
916
|
+
// Author email from git log --format='%ae'. Enables per-developer
|
|
917
|
+
// attribution in /v1/productivity even before codeburn syncs.
|
|
918
|
+
...(author ? {
|
|
919
|
+
user: { S: author },
|
|
920
|
+
gsi_user: { S: `USER#${author}` },
|
|
921
|
+
gsi_user_sk: { S: `COMMIT#${timestamp}` },
|
|
922
|
+
} : {}),
|
|
923
|
+
},
|
|
924
|
+
// Only write if the item does NOT already exist. If codeburn already
|
|
925
|
+
// pushed this commit (with ai_origin=ai-generated), do not overwrite.
|
|
926
|
+
// If the metrics-processor already seeded it on a prior invocation
|
|
927
|
+
// (retry), this is idempotent.
|
|
928
|
+
ConditionExpression: 'attribute_not_exists(pk)',
|
|
929
|
+
}));
|
|
930
|
+
seeded++;
|
|
931
|
+
} catch (err: any) {
|
|
932
|
+
// ConditionalCheckFailedException means item already exists — expected
|
|
933
|
+
// when codeburn pushed first or on retry. Not an error.
|
|
934
|
+
if (err.name !== 'ConditionalCheckFailedException') {
|
|
935
|
+
console.error(`[seedCommitAttribution] Failed to seed ${sha}:`, err);
|
|
936
|
+
throw err;
|
|
937
|
+
}
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
if (seeded > 0) {
|
|
942
|
+
console.log(`[seedCommitAttribution] Seeded ${seeded}/${shas.length} commits as human for ${repo}`);
|
|
943
|
+
}
|
|
944
|
+
}
|