@prism-d1/cli 1.0.77 → 1.0.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/assets/github-workflows/prism-ai-metrics.yml +83 -122
  2. package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +66 -0
  3. package/dist/assets/github-workflows/prism-eval-gate.yml +1 -1
  4. package/dist/assets/gitlab-workflows/.gitlab-ci.yml +0 -3
  5. package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +53 -56
  6. package/dist/assets/infra/bin/app.ts +2 -0
  7. package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +62 -0
  8. package/dist/assets/infra/lib/dashboard-stack.ts +541 -1144
  9. package/dist/assets/infra/lib/lambda/attribution-metrics-publisher.ts +248 -0
  10. package/dist/assets/infra/lib/lambda/event-schema.ts +144 -0
  11. package/dist/assets/infra/lib/lambda/metrics-processor.ts +124 -36
  12. package/dist/assets/infra/lib/lambda/otel-receiver.ts +927 -16
  13. package/dist/assets/infra/lib/lambda/productivity-widget.ts +203 -0
  14. package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +17 -2
  15. package/dist/assets/infra/lib/lambda/velocity-widget.ts +1483 -0
  16. package/dist/assets/infra/lib/metrics-pipeline-stack.ts +191 -119
  17. package/dist/assets/infra/package.json +2 -1
  18. package/dist/assets/infra/tsconfig.json +9 -2
  19. package/dist/src/commands/bootstrapper/install-eval-harness.d.ts +8 -2
  20. package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
  21. package/dist/src/commands/bootstrapper/install-eval-harness.js +32 -32
  22. package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
  23. package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
  24. package/dist/src/commands/bootstrapper/install-github-workflows.js +7 -2
  25. package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
  26. package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
  27. package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +7 -1
  28. package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
  29. package/dist/src/commands/bootstrapper/setup-otel-sync.d.ts.map +1 -1
  30. package/dist/src/commands/bootstrapper/setup-otel-sync.js +3 -2
  31. package/dist/src/commands/bootstrapper/setup-otel-sync.js.map +1 -1
  32. package/dist/src/utils/region.d.ts +50 -0
  33. package/dist/src/utils/region.d.ts.map +1 -0
  34. package/dist/src/utils/region.js +62 -0
  35. package/dist/src/utils/region.js.map +1 -0
  36. package/package.json +2 -2
  37. package/dist/assets/eval-harness/README.md +0 -144
  38. package/dist/assets/github-workflows/README.md +0 -110
  39. package/dist/assets/github-workflows/prism-dora-weekly.yml +0 -334
  40. package/dist/assets/gitlab-workflows/prism-dora-weekly.yml +0 -181
  41. package/dist/assets/infra/lib/lambda/defect-correlator.ts +0 -142
  42. package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +0 -123
  43. package/dist/assets/metric-hooks/README.md +0 -107
@@ -1,334 +0,0 @@
1
- # prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
2
- #
3
- # Copy this file to .github/workflows/prism-dora-weekly.yml in your repo.
4
- #
5
- # Methodology:
6
- # - Deployment Frequency: merged PRs to main (proxy for deploys)
7
- # - Lead Time: avg hours from PR open to merge
8
- # - Change Failure Rate: PRs with "revert|hotfix|rollback" in title / total merged
9
- # - MTTR: avg lead time of revert/hotfix PRs
10
- # - AI metrics: derived from AI-Origin commit trailers
11
- #
12
- # Required setup:
13
- # - OIDC provider configured for GitHub Actions in your AWS account
14
- # - IAM role with events:PutEvents and cloudwatch:PutMetricData permissions
15
- # - Repository secret PRISM_METRICS_ROLE_ARN set to the OIDC role ARN
16
- # - .prism/config.json committed to the repo (created by: bash prism-cli bootstrapper install-git-hooks)
17
-
18
- name: PRISM DORA Weekly Assessment
19
-
20
- on:
21
- schedule:
22
- - cron: '0 9 * * 1' # Every Monday at 09:00 UTC
23
- workflow_dispatch: # Allow manual trigger
24
-
25
- permissions:
26
- id-token: write
27
- contents: read
28
- pull-requests: read
29
-
30
- jobs:
31
- dora-assessment:
32
- runs-on: ubuntu-latest
33
- steps:
34
- - name: Checkout
35
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36
- with:
37
- fetch-depth: 0
38
-
39
- - name: Configure AWS credentials (OIDC)
40
- uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
41
- with:
42
- role-to-assume: ${{ secrets.PRISM_METRICS_ROLE_ARN }}
43
- aws-region: us-west-2
44
-
45
- - name: Read team ID from .prism/config.json
46
- id: prism-config
47
- run: |
48
- if [[ -f .prism/config.json ]]; then
49
- TEAM_ID=$(jq -r '.team_id' .prism/config.json)
50
- else
51
- echo "::error::.prism/config.json not found. Run bootstrapper/metric-hooks/install.sh first."
52
- exit 1
53
- fi
54
- echo "team_id=${TEAM_ID}" >> "$GITHUB_OUTPUT"
55
-
56
- - name: Calculate DORA metrics (past 7 days)
57
- id: dora
58
- env:
59
- GH_TOKEN: ${{ github.token }}
60
- run: |
61
- set -o pipefail
62
- SINCE=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
63
- NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
64
- REPO="${{ github.repository }}"
65
-
66
- echo "Assessment period: ${SINCE} to ${NOW}"
67
-
68
- # ---------------------------------------------------------------
69
- # 1. Deployment Frequency — merged PRs to main (proxy for deploys)
70
- # ---------------------------------------------------------------
71
- MERGED_PRS=$(gh pr list --repo "${REPO}" --state merged --base main \
72
- --search "merged:>=${SINCE}" --json number,title,mergedAt --limit 200 2>/dev/null || echo "[]")
73
- DEPLOY_COUNT=$(echo "${MERGED_PRS}" | jq 'length')
74
- DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
75
-
76
- # ---------------------------------------------------------------
77
- # 2. Lead Time for Changes — avg time from PR open to merge
78
- # ---------------------------------------------------------------
79
- LEAD_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
80
- --search "merged:>=${SINCE}" --json createdAt,mergedAt --limit 200 2>/dev/null || echo "[]")
81
-
82
- AVG_LEAD_TIME_HOURS=$(echo "${LEAD_TIMES}" | jq '
83
- if length == 0 then 0
84
- else
85
- [.[] |
86
- (((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
87
- (.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
88
- ] | add / length | . * 100 | round / 100
89
- end
90
- ')
91
-
92
- # ---------------------------------------------------------------
93
- # 3. Change Failure Rate — PRs with "revert" or "hotfix" in title
94
- # ---------------------------------------------------------------
95
- FAILURE_PRS=$(echo "${MERGED_PRS}" | jq '[.[] | select(.title? // "" | test("revert|hotfix|rollback"; "i"))] | length' 2>/dev/null || echo "0")
96
- if [[ "${DEPLOY_COUNT}" -gt 0 ]]; then
97
- CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_PRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./')
98
- else
99
- CHANGE_FAILURE_RATE="0"
100
- fi
101
-
102
- # ---------------------------------------------------------------
103
- # 4. Mean Time to Recovery — avg time for hotfix/revert PRs
104
- # ---------------------------------------------------------------
105
- RECOVERY_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
106
- --search "merged:>=${SINCE}" --json title,createdAt,mergedAt --limit 200 2>/dev/null \
107
- | jq '[.[] | select(.title | test("revert|hotfix|rollback"; "i"))]')
108
-
109
- MTTR_HOURS=$(echo "${RECOVERY_TIMES}" | jq '
110
- if length == 0 then 0
111
- else
112
- [.[] |
113
- (((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
114
- (.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
115
- ] | add / length | . * 100 | round / 100
116
- end
117
- ')
118
-
119
- echo "deploy_count=${DEPLOY_COUNT}" >> "$GITHUB_OUTPUT"
120
- echo "deploy_frequency=${DEPLOY_FREQUENCY}" >> "$GITHUB_OUTPUT"
121
- echo "avg_lead_time_hours=${AVG_LEAD_TIME_HOURS}" >> "$GITHUB_OUTPUT"
122
- echo "change_failure_rate=${CHANGE_FAILURE_RATE}" >> "$GITHUB_OUTPUT"
123
- echo "mttr_hours=${MTTR_HOURS}" >> "$GITHUB_OUTPUT"
124
-
125
- - name: Calculate AI-DORA metrics (past 7 days)
126
- id: ai-dora
127
- run: |
128
- set -o pipefail
129
- SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
130
-
131
- # All commits in the past 7 days on main
132
- TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline main 2>/dev/null | wc -l | tr -d ' ')
133
-
134
- # AI-origin breakdown — count per-commit to avoid overcounting
135
- AI_GENERATED=0
136
- AI_ASSISTED=0
137
- HUMAN_ONLY=0
138
- SPEC_REFS=0
139
- CLAUDE_CODE_COUNT=0
140
- KIRO_COUNT=0
141
- Q_DEV_COUNT=0
142
-
143
- for SHA in $(git log --since="${SINCE_DATE}" --format='%H' main 2>/dev/null); do
144
- MSG=$(git log -1 --format='%B' "${SHA}")
145
- if echo "${MSG}" | grep -q '^AI-Origin: ai-generated'; then
146
- AI_GENERATED=$((AI_GENERATED + 1))
147
- elif echo "${MSG}" | grep -q '^AI-Origin: ai-assisted'; then
148
- AI_ASSISTED=$((AI_ASSISTED + 1))
149
- elif echo "${MSG}" | grep -q '^AI-Origin: human'; then
150
- HUMAN_ONLY=$((HUMAN_ONLY + 1))
151
- fi
152
- echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
153
- TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
154
- case "${TOOL}" in
155
- *claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
156
- *kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
157
- *q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
158
- esac
159
- done
160
-
161
- UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
162
- [[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
163
-
164
- AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
165
- if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
166
- AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
167
- else
168
- AI_ADOPTION_RATE="0"
169
- fi
170
-
171
- AI_TO_MERGE_RATIO="${AI_ADOPTION_RATE}"
172
-
173
- if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
174
- SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
175
- else
176
- SPEC_COVERAGE="0"
177
- fi
178
-
179
- echo "total_commits=${TOTAL_COMMITS}" >> "$GITHUB_OUTPUT"
180
- echo "ai_generated=${AI_GENERATED}" >> "$GITHUB_OUTPUT"
181
- echo "ai_assisted=${AI_ASSISTED}" >> "$GITHUB_OUTPUT"
182
- echo "human_only=${HUMAN_ONLY}" >> "$GITHUB_OUTPUT"
183
- echo "untagged=${UNTAGGED}" >> "$GITHUB_OUTPUT"
184
- echo "ai_adoption_rate=${AI_ADOPTION_RATE}" >> "$GITHUB_OUTPUT"
185
- echo "ai_to_merge_ratio=${AI_TO_MERGE_RATIO}" >> "$GITHUB_OUTPUT"
186
- echo "spec_coverage=${SPEC_COVERAGE}" >> "$GITHUB_OUTPUT"
187
- echo "claude_code_count=${CLAUDE_CODE_COUNT}" >> "$GITHUB_OUTPUT"
188
- echo "kiro_count=${KIRO_COUNT}" >> "$GITHUB_OUTPUT"
189
- echo "q_dev_count=${Q_DEV_COUNT}" >> "$GITHUB_OUTPUT"
190
-
191
- - name: Emit prism.d1.assessment event
192
- run: |
193
- TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
194
-
195
- ASSESSMENT_EVENT=$(jq -n \
196
- --arg team_id "${{ steps.prism-config.outputs.team_id }}" \
197
- --arg repo "${{ github.repository }}" \
198
- --arg timestamp "${TIMESTAMP}" \
199
- --argjson deploy_count "${{ steps.dora.outputs.deploy_count }}" \
200
- --argjson deploy_frequency "${{ steps.dora.outputs.deploy_frequency }}" \
201
- --argjson avg_lead_time_hours "${{ steps.dora.outputs.avg_lead_time_hours }}" \
202
- --argjson change_failure_rate "${{ steps.dora.outputs.change_failure_rate }}" \
203
- --argjson mttr_hours "${{ steps.dora.outputs.mttr_hours }}" \
204
- --argjson total_commits "${{ steps.ai-dora.outputs.total_commits }}" \
205
- --argjson ai_generated "${{ steps.ai-dora.outputs.ai_generated }}" \
206
- --argjson ai_assisted "${{ steps.ai-dora.outputs.ai_assisted }}" \
207
- --argjson human_only "${{ steps.ai-dora.outputs.human_only }}" \
208
- --argjson untagged "${{ steps.ai-dora.outputs.untagged }}" \
209
- --argjson ai_adoption_rate "${{ steps.ai-dora.outputs.ai_adoption_rate }}" \
210
- --argjson ai_to_merge_ratio "${{ steps.ai-dora.outputs.ai_to_merge_ratio }}" \
211
- --argjson spec_coverage "${{ steps.ai-dora.outputs.spec_coverage }}" \
212
- --argjson claude_code_count "${{ steps.ai-dora.outputs.claude_code_count }}" \
213
- --argjson kiro_count "${{ steps.ai-dora.outputs.kiro_count }}" \
214
- --argjson q_dev_count "${{ steps.ai-dora.outputs.q_dev_count }}" \
215
- '{
216
- "team_id": $team_id,
217
- "repo": $repo,
218
- "timestamp": $timestamp,
219
- "prism_level": 2,
220
- "metric": {
221
- "name": "weekly_assessment",
222
- "value": 1,
223
- "unit": "assessment"
224
- },
225
- "dora": {
226
- "deployment_frequency": $deploy_frequency,
227
- "deployments": $deploy_count,
228
- "lead_time_hours": $avg_lead_time_hours,
229
- "change_failure_rate": $change_failure_rate,
230
- "mttr_hours": $mttr_hours
231
- },
232
- "ai_dora": {
233
- "ai_adoption_rate": $ai_adoption_rate,
234
- "ai_to_merge_ratio": $ai_to_merge_ratio,
235
- "total_commits": $total_commits,
236
- "ai_generated_commits": $ai_generated,
237
- "ai_assisted_commits": $ai_assisted,
238
- "human_commits": $human_only,
239
- "untagged_commits": $untagged,
240
- "spec_coverage": $spec_coverage,
241
- "tool_breakdown": {
242
- "claude_code": $claude_code_count,
243
- "kiro": $kiro_count,
244
- "q_developer": $q_dev_count
245
- }
246
- }
247
- }')
248
-
249
- aws events put-events \
250
- --region us-west-2 \
251
- --entries "[{
252
- \"Source\": \"prism.d1.velocity\",
253
- \"DetailType\": \"prism.d1.assessment\",
254
- \"EventBusName\": \"prism-d1-metrics\",
255
- \"Detail\": $(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')
256
- }]"
257
-
258
- echo "Weekly DORA assessment emitted successfully."
259
-
260
- - name: Emit tool breakdown and adoption metrics to CloudWatch
261
- run: |
262
- TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
263
- TEAM_ID="${{ steps.prism-config.outputs.team_id }}"
264
- REPO="${{ github.repository }}"
265
- NAMESPACE="PRISM/D1/Velocity"
266
-
267
- # AI Adoption Rate
268
- aws cloudwatch put-metric-data \
269
- --region us-west-2 \
270
- --namespace "${NAMESPACE}" \
271
- --metric-data "[{
272
- \"MetricName\": \"AIAdoptionRate\",
273
- \"Value\": ${{ steps.ai-dora.outputs.ai_adoption_rate }},
274
- \"Unit\": \"Percent\",
275
- \"Timestamp\": \"${TIMESTAMP}\",
276
- \"Dimensions\": [
277
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
278
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
279
- ]
280
- }]"
281
-
282
- # Spec Coverage
283
- aws cloudwatch put-metric-data \
284
- --region us-west-2 \
285
- --namespace "${NAMESPACE}" \
286
- --metric-data "[{
287
- \"MetricName\": \"SpecCoverage\",
288
- \"Value\": ${{ steps.ai-dora.outputs.spec_coverage }},
289
- \"Unit\": \"Percent\",
290
- \"Timestamp\": \"${TIMESTAMP}\",
291
- \"Dimensions\": [
292
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
293
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
294
- ]
295
- }]"
296
-
297
- # Tool breakdown — one metric per tool
298
- for TOOL_DATA in \
299
- "ClaudeCodeCommits:${{ steps.ai-dora.outputs.claude_code_count }}" \
300
- "KiroCommits:${{ steps.ai-dora.outputs.kiro_count }}" \
301
- "QDeveloperCommits:${{ steps.ai-dora.outputs.q_dev_count }}" \
302
- "UntaggedCommits:${{ steps.ai-dora.outputs.untagged }}"; do
303
-
304
- METRIC_NAME="${TOOL_DATA%%:*}"
305
- METRIC_VALUE="${TOOL_DATA##*:}"
306
-
307
- aws cloudwatch put-metric-data \
308
- --region us-west-2 \
309
- --namespace "${NAMESPACE}" \
310
- --metric-data "[{
311
- \"MetricName\": \"${METRIC_NAME}\",
312
- \"Value\": ${METRIC_VALUE},
313
- \"Unit\": \"Count\",
314
- \"Timestamp\": \"${TIMESTAMP}\",
315
- \"Dimensions\": [
316
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
317
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
318
- ]
319
- }]"
320
- done
321
-
322
- echo "Tool breakdown and adoption metrics emitted to CloudWatch."
323
- echo ""
324
- echo "=== DORA Metrics ==="
325
- echo "Deployment Frequency: ${{ steps.dora.outputs.deploy_frequency }}/day"
326
- echo "Lead Time: ${{ steps.dora.outputs.avg_lead_time_hours }} hours"
327
- echo "Change Failure Rate: ${{ steps.dora.outputs.change_failure_rate }}"
328
- echo "MTTR: ${{ steps.dora.outputs.mttr_hours }} hours"
329
- echo ""
330
- echo "=== AI-DORA Metrics ==="
331
- echo "AI Adoption Rate: ${{ steps.ai-dora.outputs.ai_adoption_rate }}"
332
- echo "AI-to-Merge Ratio: ${{ steps.ai-dora.outputs.ai_to_merge_ratio }}"
333
- echo "Spec Coverage: ${{ steps.ai-dora.outputs.spec_coverage }}"
334
- echo "Total Commits: ${{ steps.ai-dora.outputs.total_commits }}"
@@ -1,181 +0,0 @@
1
- # prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
2
- #
3
- # Include this file in your .gitlab-ci.yml:
4
- # include:
5
- # - local: '.prism/gitlab-workflows/prism-dora-weekly.yml'
6
- #
7
- # Required setup:
8
- # - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role with events:PutEvents + cloudwatch:PutMetricData)
9
- # - CI/CD variable AWS_REGION (default: us-west-2)
10
- # - .prism/config.json committed to the repo
11
- # - Pipeline schedule configured: weekly on Monday at 09:00 UTC
12
-
13
- prism-dora-weekly:
14
- stage: test
15
- image:
16
- name: amazon/aws-cli:latest
17
- entrypoint: [""]
18
- id_tokens:
19
- GITLAB_OIDC_TOKEN:
20
- aud: https://gitlab.com
21
- rules:
22
- - if: '$CI_PIPELINE_SOURCE == "schedule"'
23
- - if: '$CI_PIPELINE_SOURCE == "web"'
24
- when: manual
25
- variables:
26
- AWS_DEFAULT_REGION: us-west-2
27
- before_script:
28
- - yum install -y -q jq bc git
29
- - git fetch --unshallow || true
30
- - |
31
- # Assume AWS role via GitLab OIDC
32
- if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
33
- echo "Assuming role via OIDC: ${PRISM_METRICS_ROLE_ARN}"
34
- if CREDS=$(aws sts assume-role-with-web-identity \
35
- --role-arn "${PRISM_METRICS_ROLE_ARN}" \
36
- --role-session-name "gitlab-ci-${CI_JOB_ID}" \
37
- --web-identity-token "${GITLAB_OIDC_TOKEN}" \
38
- --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
39
- --output text 2>&1); then
40
- export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
41
- export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
42
- export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
43
- echo "✓ OIDC credentials acquired."
44
- else
45
- echo "⚠️ OIDC assume-role failed: ${CREDS}"
46
- fi
47
- elif [[ -n "${PRISM_METRICS_ROLE_ARN}" ]]; then
48
- echo "⚠️ PRISM_METRICS_ROLE_ARN set but GITLAB_OIDC_TOKEN is empty."
49
- fi
50
- script:
51
- - |
52
- # Skip if no AWS credentials available
53
- if ! aws sts get-caller-identity &>/dev/null; then
54
- echo "⚠️ No AWS credentials. Set PRISM_METRICS_ROLE_ARN or configure AWS access."
55
- exit 0
56
- fi
57
-
58
- TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
59
- SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
60
- REPO="${CI_PROJECT_PATH}"
61
- NAMESPACE="PRISM/D1/Velocity"
62
-
63
- # ---------------------------------------------------------------
64
- # 1. Deployment Frequency — merged MRs to default branch (past 7 days)
65
- # ---------------------------------------------------------------
66
- SINCE_ISO=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
67
- MERGED_MRS=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
68
- "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests?state=merged&updated_after=${SINCE_ISO}&per_page=100" 2>/dev/null || echo "[]")
69
- DEPLOY_COUNT=$(echo "${MERGED_MRS}" | jq 'length')
70
- DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
71
-
72
- # ---------------------------------------------------------------
73
- # 2. Lead Time — avg time from MR open to merge
74
- # ---------------------------------------------------------------
75
- AVG_LEAD_TIME_HOURS=$(echo "${MERGED_MRS}" | jq '
76
- if length == 0 then 0
77
- else [.[] | select(.merged_at != null and .created_at != null) |
78
- ((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
79
- (.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
80
- ] | if length > 0 then add / length | . * 100 | round / 100 else 0 end
81
- end')
82
-
83
- # ---------------------------------------------------------------
84
- # 3. Change Failure Rate — MRs with "revert|hotfix|rollback" in title
85
- # ---------------------------------------------------------------
86
- FAILURE_MRS=$(echo "${MERGED_MRS}" | jq '[.[] | select(.title // "" | test("revert|hotfix|rollback"; "i"))] | length')
87
- [[ "${DEPLOY_COUNT}" -gt 0 ]] && CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_MRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./') || CHANGE_FAILURE_RATE="0"
88
-
89
- # ---------------------------------------------------------------
90
- # 4. MTTR — avg time for hotfix/revert MRs
91
- # ---------------------------------------------------------------
92
- MTTR_HOURS=$(echo "${MERGED_MRS}" | jq '
93
- [.[] | select(.title // "" | test("revert|hotfix|rollback"; "i")) |
94
- select(.merged_at != null and .created_at != null) |
95
- ((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
96
- (.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
97
- ] | if length > 0 then add / length | . * 100 | round / 100 else 0 end')
98
-
99
- # ---------------------------------------------------------------
100
- # 5. AI-DORA metrics from commit trailers
101
- # ---------------------------------------------------------------
102
- TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline "${CI_DEFAULT_BRANCH}" 2>/dev/null | wc -l | tr -d ' ')
103
- AI_GENERATED=0; AI_ASSISTED=0; HUMAN_ONLY=0; SPEC_REFS=0
104
- CLAUDE_CODE_COUNT=0; KIRO_COUNT=0; Q_DEV_COUNT=0
105
-
106
- for SHA in $(git log --since="${SINCE_DATE}" --format='%H' "${CI_DEFAULT_BRANCH}" 2>/dev/null); do
107
- MSG=$(git log -1 --format='%B' "${SHA}")
108
- echo "${MSG}" | grep -q '^AI-Origin: ai-generated' && AI_GENERATED=$((AI_GENERATED + 1))
109
- echo "${MSG}" | grep -q '^AI-Origin: ai-assisted' && AI_ASSISTED=$((AI_ASSISTED + 1))
110
- echo "${MSG}" | grep -q '^AI-Origin: human' && HUMAN_ONLY=$((HUMAN_ONLY + 1))
111
- echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
112
- TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
113
- case "${TOOL}" in
114
- *claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
115
- *kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
116
- *q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
117
- esac
118
- done
119
-
120
- UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
121
- [[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
122
- AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
123
- [[ "${TOTAL_COMMITS}" -gt 0 ]] && AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || AI_ADOPTION_RATE="0"
124
- [[ "${TOTAL_COMMITS}" -gt 0 ]] && SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || SPEC_COVERAGE="0"
125
-
126
- # ---------------------------------------------------------------
127
- # Emit assessment event
128
- # ---------------------------------------------------------------
129
- TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
130
-
131
- ASSESSMENT_EVENT=$(jq -n \
132
- --arg team_id "${TEAM_ID}" --arg repo "${REPO}" --arg timestamp "${TIMESTAMP}" \
133
- --argjson deploy_count "${DEPLOY_COUNT}" --argjson deploy_frequency "${DEPLOY_FREQUENCY}" \
134
- --argjson avg_lead_time_hours "${AVG_LEAD_TIME_HOURS}" \
135
- --argjson change_failure_rate "${CHANGE_FAILURE_RATE}" --argjson mttr_hours "${MTTR_HOURS}" \
136
- --argjson total_commits "${TOTAL_COMMITS}" --argjson ai_generated "${AI_GENERATED}" \
137
- --argjson ai_assisted "${AI_ASSISTED}" --argjson human_only "${HUMAN_ONLY}" \
138
- --argjson untagged "${UNTAGGED}" --argjson ai_adoption_rate "${AI_ADOPTION_RATE}" \
139
- --argjson spec_coverage "${SPEC_COVERAGE}" \
140
- --argjson claude_code "${CLAUDE_CODE_COUNT}" --argjson kiro "${KIRO_COUNT}" --argjson q_dev "${Q_DEV_COUNT}" \
141
- '{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:2,
142
- metric:{name:"weekly_assessment",value:1,unit:"assessment"},
143
- dora:{deployment_frequency:$deploy_frequency, deployments:$deploy_count,
144
- lead_time_hours:$avg_lead_time_hours, change_failure_rate:$change_failure_rate,
145
- mttr_hours:$mttr_hours},
146
- ai_dora:{ai_adoption_rate:$ai_adoption_rate, ai_to_merge_ratio:$ai_adoption_rate,
147
- total_commits:$total_commits, ai_generated_commits:$ai_generated,
148
- ai_assisted_commits:$ai_assisted, human_commits:$human_only,
149
- untagged_commits:$untagged, spec_coverage:$spec_coverage,
150
- tool_breakdown:{claude_code:$claude_code, kiro:$kiro, q_developer:$q_dev}}}')
151
-
152
- aws events put-events --region "${AWS_DEFAULT_REGION}" \
153
- --entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.assessment\",
154
- \"EventBusName\":\"prism-d1-metrics\",
155
- \"Detail\":$(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')}]"
156
-
157
- # Emit CloudWatch metrics
158
- for METRIC_DATA in \
159
- "AIAdoptionRate:${AI_ADOPTION_RATE}:Percent" \
160
- "SpecCoverage:${SPEC_COVERAGE}:Percent" \
161
- "ClaudeCodeCommits:${CLAUDE_CODE_COUNT}:Count" \
162
- "KiroCommits:${KIRO_COUNT}:Count" \
163
- "QDeveloperCommits:${Q_DEV_COUNT}:Count" \
164
- "UntaggedCommits:${UNTAGGED}:Count"; do
165
- METRIC_NAME="${METRIC_DATA%%:*}"; REST="${METRIC_DATA#*:}"
166
- METRIC_VALUE="${REST%%:*}"; METRIC_UNIT="${REST#*:}"
167
- aws cloudwatch put-metric-data --region "${AWS_DEFAULT_REGION}" --namespace "${NAMESPACE}" \
168
- --metric-data "[{\"MetricName\":\"${METRIC_NAME}\",\"Value\":${METRIC_VALUE},
169
- \"Unit\":\"${METRIC_UNIT}\",\"Timestamp\":\"${TIMESTAMP}\",
170
- \"Dimensions\":[{\"Name\":\"TeamId\",\"Value\":\"${TEAM_ID}\"},{\"Name\":\"Repository\",\"Value\":\"${REPO}\"}]}]"
171
- done
172
-
173
- echo "=== DORA Metrics ==="
174
- echo "Deployment Frequency: ${DEPLOY_FREQUENCY}/day"
175
- echo "Lead Time: ${AVG_LEAD_TIME_HOURS} hours"
176
- echo "Change Failure Rate: ${CHANGE_FAILURE_RATE}"
177
- echo "MTTR: ${MTTR_HOURS} hours"
178
- echo "=== AI-DORA Metrics ==="
179
- echo "AI Adoption Rate: ${AI_ADOPTION_RATE}"
180
- echo "Spec Coverage: ${SPEC_COVERAGE}"
181
- echo "Total Commits: ${TOTAL_COMMITS}"
@@ -1,142 +0,0 @@
1
- import {
2
- DynamoDBClient,
3
- QueryCommand,
4
- } from '@aws-sdk/client-dynamodb';
5
- import {
6
- EventBridgeClient,
7
- PutEventsCommand,
8
- } from '@aws-sdk/client-eventbridge';
9
-
10
- const dynamoClient = new DynamoDBClient({});
11
- const eventBridgeClient = new EventBridgeClient({});
12
-
13
- const EVENTS_TABLE = process.env.EVENTS_TABLE!;
14
- const EVENT_BUS_NAME = process.env.EVENT_BUS_NAME!;
15
- const LOOKBACK_HOURS = parseInt(process.env.LOOKBACK_HOURS ?? '24', 10);
16
-
17
- interface DeployEvent {
18
- source: string;
19
- 'detail-type': string;
20
- detail: {
21
- team_id: string;
22
- repo: string;
23
- timestamp: string;
24
- dora?: {
25
- change_failure_rate?: number;
26
- };
27
- };
28
- }
29
-
30
- /**
31
- * Triggered by failed deployment events.
32
- * Queries recent commits by AI origin, calculates AI vs human defect rates,
33
- * and emits a prism.d1.quality event.
34
- */
35
- export async function handler(event: DeployEvent): Promise<void> {
36
- const detail = event.detail;
37
-
38
- // Only process deployment failures
39
- const cfr = detail.dora?.change_failure_rate ?? 0;
40
- if (cfr === 0) {
41
- console.log('Deployment succeeded, no defect correlation needed');
42
- return;
43
- }
44
-
45
- console.log(`Processing failed deployment for ${detail.team_id}/${detail.repo}`);
46
-
47
- const deployTime = new Date(detail.timestamp);
48
- const lookbackStart = new Date(deployTime.getTime() - LOOKBACK_HOURS * 60 * 60 * 1000);
49
-
50
- // Query recent commit events for this repo
51
- try {
52
- const result = await dynamoClient.send(
53
- new QueryCommand({
54
- TableName: EVENTS_TABLE,
55
- KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
56
- ExpressionAttributeValues: {
57
- ':pk': { S: `${detail.team_id}#${detail.repo}` },
58
- ':start': { S: lookbackStart.toISOString() },
59
- ':end': { S: deployTime.toISOString() },
60
- },
61
- }),
62
- );
63
-
64
- if (!result.Items || result.Items.length === 0) {
65
- console.log('No recent commits found in lookback window');
66
- return;
67
- }
68
-
69
- // Count commits by AI origin
70
- let aiCommitCount = 0;
71
- let humanCommitCount = 0;
72
-
73
- for (const item of result.Items) {
74
- const detailType = item.detail_type?.S ?? '';
75
- if (detailType !== 'prism.d1.commit') continue;
76
-
77
- const data = JSON.parse(item.data?.S ?? '{}');
78
- const origin = data.ai_context?.origin ?? 'human';
79
-
80
- if (origin === 'ai-generated' || origin === 'ai-assisted') {
81
- aiCommitCount++;
82
- } else {
83
- humanCommitCount++;
84
- }
85
- }
86
-
87
- const totalCommits = aiCommitCount + humanCommitCount;
88
- if (totalCommits === 0) {
89
- console.log('No commit events in lookback window');
90
- return;
91
- }
92
-
93
- // Calculate defect rates using proportional distribution.
94
- // Since we can't attribute a specific failure to a specific commit,
95
- // we distribute the change failure rate proportionally by code origin.
96
- // This gives a directional signal: if 80% of commits are AI and the
97
- // deploy failed, AI code carries 80% of the attributed failure weight.
98
- const aiDefectRate = aiCommitCount > 0 ? (cfr * aiCommitCount) / totalCommits : 0;
99
- const humanDefectRate = humanCommitCount > 0 ? (cfr * humanCommitCount) / totalCommits : 0;
100
-
101
- const qualityEvent = {
102
- team_id: detail.team_id,
103
- repo: detail.repo,
104
- timestamp: detail.timestamp,
105
- prism_level: 2,
106
- metric: {
107
- name: 'post_merge_defect_rate',
108
- value: cfr,
109
- unit: 'percent',
110
- },
111
- ai_dora: {
112
- post_merge_defect_rate: cfr,
113
- },
114
- quality: {
115
- deployment_id: `deploy-${Date.now()}`,
116
- ai_defect_rate: Math.round(aiDefectRate * 10000) / 10000,
117
- human_defect_rate: Math.round(humanDefectRate * 10000) / 10000,
118
- total_ai_commits: aiCommitCount,
119
- total_human_commits: humanCommitCount,
120
- },
121
- };
122
-
123
- await eventBridgeClient.send(
124
- new PutEventsCommand({
125
- Entries: [
126
- {
127
- Source: 'prism.d1.velocity',
128
- DetailType: 'prism.d1.quality',
129
- EventBusName: EVENT_BUS_NAME,
130
- Detail: JSON.stringify(qualityEvent),
131
- },
132
- ],
133
- }),
134
- );
135
-
136
- console.log(
137
- `Emitted quality event: AI defect rate ${aiDefectRate.toFixed(4)}, Human defect rate ${humanDefectRate.toFixed(4)}, ${aiCommitCount} AI commits, ${humanCommitCount} human commits`,
138
- );
139
- } catch (err) {
140
- console.error('Failed to correlate defects:', err);
141
- }
142
- }