@prism-d1/cli 1.0.79 → 1.0.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/github-workflows/prism-ai-metrics.yml +83 -122
- package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +66 -0
- package/dist/assets/github-workflows/prism-eval-gate.yml +1 -1
- package/dist/assets/gitlab-workflows/.gitlab-ci.yml +0 -3
- package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +53 -56
- package/dist/assets/infra/bin/app.ts +1 -0
- package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +4 -0
- package/dist/assets/infra/lib/dashboard-stack.ts +378 -1235
- package/dist/assets/infra/lib/lambda/event-schema.ts +144 -0
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +124 -36
- package/dist/assets/infra/lib/lambda/otel-receiver.ts +249 -35
- package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +17 -2
- package/dist/assets/infra/lib/lambda/velocity-widget.ts +1483 -0
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +111 -119
- package/dist/assets/infra/package.json +2 -1
- package/dist/assets/infra/tsconfig.json +9 -2
- package/dist/src/commands/assessment/web.d.ts.map +1 -1
- package/dist/src/commands/assessment/web.js +25 -2
- package/dist/src/commands/assessment/web.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts +8 -2
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +46 -36
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.js +10 -14
- package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.js +7 -2
- package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +7 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-github-oidc.js +50 -27
- package/dist/src/commands/bootstrapper/setup-github-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js +69 -25
- package/dist/src/commands/bootstrapper/setup-gitlab-oidc.js.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/setup-otel-sync.js +145 -53
- package/dist/src/commands/bootstrapper/setup-otel-sync.js.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.d.ts +41 -2
- package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.js +391 -167
- package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
- package/dist/src/commands/workshop/verify-setup.d.ts.map +1 -1
- package/dist/src/commands/workshop/verify-setup.js +75 -14
- package/dist/src/commands/workshop/verify-setup.js.map +1 -1
- package/dist/src/utils/exec.d.ts +56 -0
- package/dist/src/utils/exec.d.ts.map +1 -0
- package/dist/src/utils/exec.js +72 -0
- package/dist/src/utils/exec.js.map +1 -0
- package/dist/src/utils/region.d.ts +50 -0
- package/dist/src/utils/region.d.ts.map +1 -0
- package/dist/src/utils/region.js +62 -0
- package/dist/src/utils/region.js.map +1 -0
- package/dist/src/utils/tempfile.d.ts +19 -0
- package/dist/src/utils/tempfile.d.ts.map +1 -0
- package/dist/src/utils/tempfile.js +32 -0
- package/dist/src/utils/tempfile.js.map +1 -0
- package/dist/src/utils/validate.d.ts +54 -0
- package/dist/src/utils/validate.d.ts.map +1 -0
- package/dist/src/utils/validate.js +143 -0
- package/dist/src/utils/validate.js.map +1 -0
- package/package.json +2 -2
- package/dist/assets/eval-harness/README.md +0 -144
- package/dist/assets/github-workflows/README.md +0 -110
- package/dist/assets/github-workflows/prism-dora-weekly.yml +0 -334
- package/dist/assets/gitlab-workflows/prism-dora-weekly.yml +0 -181
- package/dist/assets/infra/lib/lambda/defect-correlator.ts +0 -142
- package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +0 -123
- package/dist/assets/metric-hooks/README.md +0 -107
|
@@ -1,334 +0,0 @@
|
|
|
1
|
-
# prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
|
|
2
|
-
#
|
|
3
|
-
# Copy this file to .github/workflows/prism-dora-weekly.yml in your repo.
|
|
4
|
-
#
|
|
5
|
-
# Methodology:
|
|
6
|
-
# - Deployment Frequency: merged PRs to main (proxy for deploys)
|
|
7
|
-
# - Lead Time: avg hours from PR open to merge
|
|
8
|
-
# - Change Failure Rate: PRs with "revert|hotfix|rollback" in title / total merged
|
|
9
|
-
# - MTTR: avg lead time of revert/hotfix PRs
|
|
10
|
-
# - AI metrics: derived from AI-Origin commit trailers
|
|
11
|
-
#
|
|
12
|
-
# Required setup:
|
|
13
|
-
# - OIDC provider configured for GitHub Actions in your AWS account
|
|
14
|
-
# - IAM role with events:PutEvents and cloudwatch:PutMetricData permissions
|
|
15
|
-
# - Repository secret PRISM_METRICS_ROLE_ARN set to the OIDC role ARN
|
|
16
|
-
# - .prism/config.json committed to the repo (created by: bash prism-cli bootstrapper install-git-hooks)
|
|
17
|
-
|
|
18
|
-
name: PRISM DORA Weekly Assessment
|
|
19
|
-
|
|
20
|
-
on:
|
|
21
|
-
schedule:
|
|
22
|
-
- cron: '0 9 * * 1' # Every Monday at 09:00 UTC
|
|
23
|
-
workflow_dispatch: # Allow manual trigger
|
|
24
|
-
|
|
25
|
-
permissions:
|
|
26
|
-
id-token: write
|
|
27
|
-
contents: read
|
|
28
|
-
pull-requests: read
|
|
29
|
-
|
|
30
|
-
jobs:
|
|
31
|
-
dora-assessment:
|
|
32
|
-
runs-on: ubuntu-latest
|
|
33
|
-
steps:
|
|
34
|
-
- name: Checkout
|
|
35
|
-
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
36
|
-
with:
|
|
37
|
-
fetch-depth: 0
|
|
38
|
-
|
|
39
|
-
- name: Configure AWS credentials (OIDC)
|
|
40
|
-
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
|
|
41
|
-
with:
|
|
42
|
-
role-to-assume: ${{ secrets.PRISM_METRICS_ROLE_ARN }}
|
|
43
|
-
aws-region: us-west-2
|
|
44
|
-
|
|
45
|
-
- name: Read team ID from .prism/config.json
|
|
46
|
-
id: prism-config
|
|
47
|
-
run: |
|
|
48
|
-
if [[ -f .prism/config.json ]]; then
|
|
49
|
-
TEAM_ID=$(jq -r '.team_id' .prism/config.json)
|
|
50
|
-
else
|
|
51
|
-
echo "::error::.prism/config.json not found. Run bootstrapper/metric-hooks/install.sh first."
|
|
52
|
-
exit 1
|
|
53
|
-
fi
|
|
54
|
-
echo "team_id=${TEAM_ID}" >> "$GITHUB_OUTPUT"
|
|
55
|
-
|
|
56
|
-
- name: Calculate DORA metrics (past 7 days)
|
|
57
|
-
id: dora
|
|
58
|
-
env:
|
|
59
|
-
GH_TOKEN: ${{ github.token }}
|
|
60
|
-
run: |
|
|
61
|
-
set -o pipefail
|
|
62
|
-
SINCE=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
|
|
63
|
-
NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
64
|
-
REPO="${{ github.repository }}"
|
|
65
|
-
|
|
66
|
-
echo "Assessment period: ${SINCE} to ${NOW}"
|
|
67
|
-
|
|
68
|
-
# ---------------------------------------------------------------
|
|
69
|
-
# 1. Deployment Frequency — merged PRs to main (proxy for deploys)
|
|
70
|
-
# ---------------------------------------------------------------
|
|
71
|
-
MERGED_PRS=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
72
|
-
--search "merged:>=${SINCE}" --json number,title,mergedAt --limit 200 2>/dev/null || echo "[]")
|
|
73
|
-
DEPLOY_COUNT=$(echo "${MERGED_PRS}" | jq 'length')
|
|
74
|
-
DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
|
|
75
|
-
|
|
76
|
-
# ---------------------------------------------------------------
|
|
77
|
-
# 2. Lead Time for Changes — avg time from PR open to merge
|
|
78
|
-
# ---------------------------------------------------------------
|
|
79
|
-
LEAD_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
80
|
-
--search "merged:>=${SINCE}" --json createdAt,mergedAt --limit 200 2>/dev/null || echo "[]")
|
|
81
|
-
|
|
82
|
-
AVG_LEAD_TIME_HOURS=$(echo "${LEAD_TIMES}" | jq '
|
|
83
|
-
if length == 0 then 0
|
|
84
|
-
else
|
|
85
|
-
[.[] |
|
|
86
|
-
(((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
87
|
-
(.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
|
|
88
|
-
] | add / length | . * 100 | round / 100
|
|
89
|
-
end
|
|
90
|
-
')
|
|
91
|
-
|
|
92
|
-
# ---------------------------------------------------------------
|
|
93
|
-
# 3. Change Failure Rate — PRs with "revert" or "hotfix" in title
|
|
94
|
-
# ---------------------------------------------------------------
|
|
95
|
-
FAILURE_PRS=$(echo "${MERGED_PRS}" | jq '[.[] | select(.title? // "" | test("revert|hotfix|rollback"; "i"))] | length' 2>/dev/null || echo "0")
|
|
96
|
-
if [[ "${DEPLOY_COUNT}" -gt 0 ]]; then
|
|
97
|
-
CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_PRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./')
|
|
98
|
-
else
|
|
99
|
-
CHANGE_FAILURE_RATE="0"
|
|
100
|
-
fi
|
|
101
|
-
|
|
102
|
-
# ---------------------------------------------------------------
|
|
103
|
-
# 4. Mean Time to Recovery — avg time for hotfix/revert PRs
|
|
104
|
-
# ---------------------------------------------------------------
|
|
105
|
-
RECOVERY_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
106
|
-
--search "merged:>=${SINCE}" --json title,createdAt,mergedAt --limit 200 2>/dev/null \
|
|
107
|
-
| jq '[.[] | select(.title | test("revert|hotfix|rollback"; "i"))]')
|
|
108
|
-
|
|
109
|
-
MTTR_HOURS=$(echo "${RECOVERY_TIMES}" | jq '
|
|
110
|
-
if length == 0 then 0
|
|
111
|
-
else
|
|
112
|
-
[.[] |
|
|
113
|
-
(((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
114
|
-
(.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
|
|
115
|
-
] | add / length | . * 100 | round / 100
|
|
116
|
-
end
|
|
117
|
-
')
|
|
118
|
-
|
|
119
|
-
echo "deploy_count=${DEPLOY_COUNT}" >> "$GITHUB_OUTPUT"
|
|
120
|
-
echo "deploy_frequency=${DEPLOY_FREQUENCY}" >> "$GITHUB_OUTPUT"
|
|
121
|
-
echo "avg_lead_time_hours=${AVG_LEAD_TIME_HOURS}" >> "$GITHUB_OUTPUT"
|
|
122
|
-
echo "change_failure_rate=${CHANGE_FAILURE_RATE}" >> "$GITHUB_OUTPUT"
|
|
123
|
-
echo "mttr_hours=${MTTR_HOURS}" >> "$GITHUB_OUTPUT"
|
|
124
|
-
|
|
125
|
-
- name: Calculate AI-DORA metrics (past 7 days)
|
|
126
|
-
id: ai-dora
|
|
127
|
-
run: |
|
|
128
|
-
set -o pipefail
|
|
129
|
-
SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
|
|
130
|
-
|
|
131
|
-
# All commits in the past 7 days on main
|
|
132
|
-
TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline main 2>/dev/null | wc -l | tr -d ' ')
|
|
133
|
-
|
|
134
|
-
# AI-origin breakdown — count per-commit to avoid overcounting
|
|
135
|
-
AI_GENERATED=0
|
|
136
|
-
AI_ASSISTED=0
|
|
137
|
-
HUMAN_ONLY=0
|
|
138
|
-
SPEC_REFS=0
|
|
139
|
-
CLAUDE_CODE_COUNT=0
|
|
140
|
-
KIRO_COUNT=0
|
|
141
|
-
Q_DEV_COUNT=0
|
|
142
|
-
|
|
143
|
-
for SHA in $(git log --since="${SINCE_DATE}" --format='%H' main 2>/dev/null); do
|
|
144
|
-
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
145
|
-
if echo "${MSG}" | grep -q '^AI-Origin: ai-generated'; then
|
|
146
|
-
AI_GENERATED=$((AI_GENERATED + 1))
|
|
147
|
-
elif echo "${MSG}" | grep -q '^AI-Origin: ai-assisted'; then
|
|
148
|
-
AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
149
|
-
elif echo "${MSG}" | grep -q '^AI-Origin: human'; then
|
|
150
|
-
HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
151
|
-
fi
|
|
152
|
-
echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
|
|
153
|
-
TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
|
|
154
|
-
case "${TOOL}" in
|
|
155
|
-
*claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
|
|
156
|
-
*kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
|
|
157
|
-
*q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
|
|
158
|
-
esac
|
|
159
|
-
done
|
|
160
|
-
|
|
161
|
-
UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
|
|
162
|
-
[[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
|
|
163
|
-
|
|
164
|
-
AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
|
|
165
|
-
if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
|
|
166
|
-
AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
|
|
167
|
-
else
|
|
168
|
-
AI_ADOPTION_RATE="0"
|
|
169
|
-
fi
|
|
170
|
-
|
|
171
|
-
AI_TO_MERGE_RATIO="${AI_ADOPTION_RATE}"
|
|
172
|
-
|
|
173
|
-
if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
|
|
174
|
-
SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
|
|
175
|
-
else
|
|
176
|
-
SPEC_COVERAGE="0"
|
|
177
|
-
fi
|
|
178
|
-
|
|
179
|
-
echo "total_commits=${TOTAL_COMMITS}" >> "$GITHUB_OUTPUT"
|
|
180
|
-
echo "ai_generated=${AI_GENERATED}" >> "$GITHUB_OUTPUT"
|
|
181
|
-
echo "ai_assisted=${AI_ASSISTED}" >> "$GITHUB_OUTPUT"
|
|
182
|
-
echo "human_only=${HUMAN_ONLY}" >> "$GITHUB_OUTPUT"
|
|
183
|
-
echo "untagged=${UNTAGGED}" >> "$GITHUB_OUTPUT"
|
|
184
|
-
echo "ai_adoption_rate=${AI_ADOPTION_RATE}" >> "$GITHUB_OUTPUT"
|
|
185
|
-
echo "ai_to_merge_ratio=${AI_TO_MERGE_RATIO}" >> "$GITHUB_OUTPUT"
|
|
186
|
-
echo "spec_coverage=${SPEC_COVERAGE}" >> "$GITHUB_OUTPUT"
|
|
187
|
-
echo "claude_code_count=${CLAUDE_CODE_COUNT}" >> "$GITHUB_OUTPUT"
|
|
188
|
-
echo "kiro_count=${KIRO_COUNT}" >> "$GITHUB_OUTPUT"
|
|
189
|
-
echo "q_dev_count=${Q_DEV_COUNT}" >> "$GITHUB_OUTPUT"
|
|
190
|
-
|
|
191
|
-
- name: Emit prism.d1.assessment event
|
|
192
|
-
run: |
|
|
193
|
-
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
194
|
-
|
|
195
|
-
ASSESSMENT_EVENT=$(jq -n \
|
|
196
|
-
--arg team_id "${{ steps.prism-config.outputs.team_id }}" \
|
|
197
|
-
--arg repo "${{ github.repository }}" \
|
|
198
|
-
--arg timestamp "${TIMESTAMP}" \
|
|
199
|
-
--argjson deploy_count "${{ steps.dora.outputs.deploy_count }}" \
|
|
200
|
-
--argjson deploy_frequency "${{ steps.dora.outputs.deploy_frequency }}" \
|
|
201
|
-
--argjson avg_lead_time_hours "${{ steps.dora.outputs.avg_lead_time_hours }}" \
|
|
202
|
-
--argjson change_failure_rate "${{ steps.dora.outputs.change_failure_rate }}" \
|
|
203
|
-
--argjson mttr_hours "${{ steps.dora.outputs.mttr_hours }}" \
|
|
204
|
-
--argjson total_commits "${{ steps.ai-dora.outputs.total_commits }}" \
|
|
205
|
-
--argjson ai_generated "${{ steps.ai-dora.outputs.ai_generated }}" \
|
|
206
|
-
--argjson ai_assisted "${{ steps.ai-dora.outputs.ai_assisted }}" \
|
|
207
|
-
--argjson human_only "${{ steps.ai-dora.outputs.human_only }}" \
|
|
208
|
-
--argjson untagged "${{ steps.ai-dora.outputs.untagged }}" \
|
|
209
|
-
--argjson ai_adoption_rate "${{ steps.ai-dora.outputs.ai_adoption_rate }}" \
|
|
210
|
-
--argjson ai_to_merge_ratio "${{ steps.ai-dora.outputs.ai_to_merge_ratio }}" \
|
|
211
|
-
--argjson spec_coverage "${{ steps.ai-dora.outputs.spec_coverage }}" \
|
|
212
|
-
--argjson claude_code_count "${{ steps.ai-dora.outputs.claude_code_count }}" \
|
|
213
|
-
--argjson kiro_count "${{ steps.ai-dora.outputs.kiro_count }}" \
|
|
214
|
-
--argjson q_dev_count "${{ steps.ai-dora.outputs.q_dev_count }}" \
|
|
215
|
-
'{
|
|
216
|
-
"team_id": $team_id,
|
|
217
|
-
"repo": $repo,
|
|
218
|
-
"timestamp": $timestamp,
|
|
219
|
-
"prism_level": 2,
|
|
220
|
-
"metric": {
|
|
221
|
-
"name": "weekly_assessment",
|
|
222
|
-
"value": 1,
|
|
223
|
-
"unit": "assessment"
|
|
224
|
-
},
|
|
225
|
-
"dora": {
|
|
226
|
-
"deployment_frequency": $deploy_frequency,
|
|
227
|
-
"deployments": $deploy_count,
|
|
228
|
-
"lead_time_hours": $avg_lead_time_hours,
|
|
229
|
-
"change_failure_rate": $change_failure_rate,
|
|
230
|
-
"mttr_hours": $mttr_hours
|
|
231
|
-
},
|
|
232
|
-
"ai_dora": {
|
|
233
|
-
"ai_adoption_rate": $ai_adoption_rate,
|
|
234
|
-
"ai_to_merge_ratio": $ai_to_merge_ratio,
|
|
235
|
-
"total_commits": $total_commits,
|
|
236
|
-
"ai_generated_commits": $ai_generated,
|
|
237
|
-
"ai_assisted_commits": $ai_assisted,
|
|
238
|
-
"human_commits": $human_only,
|
|
239
|
-
"untagged_commits": $untagged,
|
|
240
|
-
"spec_coverage": $spec_coverage,
|
|
241
|
-
"tool_breakdown": {
|
|
242
|
-
"claude_code": $claude_code_count,
|
|
243
|
-
"kiro": $kiro_count,
|
|
244
|
-
"q_developer": $q_dev_count
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}')
|
|
248
|
-
|
|
249
|
-
aws events put-events \
|
|
250
|
-
--region us-west-2 \
|
|
251
|
-
--entries "[{
|
|
252
|
-
\"Source\": \"prism.d1.velocity\",
|
|
253
|
-
\"DetailType\": \"prism.d1.assessment\",
|
|
254
|
-
\"EventBusName\": \"prism-d1-metrics\",
|
|
255
|
-
\"Detail\": $(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
256
|
-
}]"
|
|
257
|
-
|
|
258
|
-
echo "Weekly DORA assessment emitted successfully."
|
|
259
|
-
|
|
260
|
-
- name: Emit tool breakdown and adoption metrics to CloudWatch
|
|
261
|
-
run: |
|
|
262
|
-
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
263
|
-
TEAM_ID="${{ steps.prism-config.outputs.team_id }}"
|
|
264
|
-
REPO="${{ github.repository }}"
|
|
265
|
-
NAMESPACE="PRISM/D1/Velocity"
|
|
266
|
-
|
|
267
|
-
# AI Adoption Rate
|
|
268
|
-
aws cloudwatch put-metric-data \
|
|
269
|
-
--region us-west-2 \
|
|
270
|
-
--namespace "${NAMESPACE}" \
|
|
271
|
-
--metric-data "[{
|
|
272
|
-
\"MetricName\": \"AIAdoptionRate\",
|
|
273
|
-
\"Value\": ${{ steps.ai-dora.outputs.ai_adoption_rate }},
|
|
274
|
-
\"Unit\": \"Percent\",
|
|
275
|
-
\"Timestamp\": \"${TIMESTAMP}\",
|
|
276
|
-
\"Dimensions\": [
|
|
277
|
-
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
278
|
-
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
279
|
-
]
|
|
280
|
-
}]"
|
|
281
|
-
|
|
282
|
-
# Spec Coverage
|
|
283
|
-
aws cloudwatch put-metric-data \
|
|
284
|
-
--region us-west-2 \
|
|
285
|
-
--namespace "${NAMESPACE}" \
|
|
286
|
-
--metric-data "[{
|
|
287
|
-
\"MetricName\": \"SpecCoverage\",
|
|
288
|
-
\"Value\": ${{ steps.ai-dora.outputs.spec_coverage }},
|
|
289
|
-
\"Unit\": \"Percent\",
|
|
290
|
-
\"Timestamp\": \"${TIMESTAMP}\",
|
|
291
|
-
\"Dimensions\": [
|
|
292
|
-
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
293
|
-
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
294
|
-
]
|
|
295
|
-
}]"
|
|
296
|
-
|
|
297
|
-
# Tool breakdown — one metric per tool
|
|
298
|
-
for TOOL_DATA in \
|
|
299
|
-
"ClaudeCodeCommits:${{ steps.ai-dora.outputs.claude_code_count }}" \
|
|
300
|
-
"KiroCommits:${{ steps.ai-dora.outputs.kiro_count }}" \
|
|
301
|
-
"QDeveloperCommits:${{ steps.ai-dora.outputs.q_dev_count }}" \
|
|
302
|
-
"UntaggedCommits:${{ steps.ai-dora.outputs.untagged }}"; do
|
|
303
|
-
|
|
304
|
-
METRIC_NAME="${TOOL_DATA%%:*}"
|
|
305
|
-
METRIC_VALUE="${TOOL_DATA##*:}"
|
|
306
|
-
|
|
307
|
-
aws cloudwatch put-metric-data \
|
|
308
|
-
--region us-west-2 \
|
|
309
|
-
--namespace "${NAMESPACE}" \
|
|
310
|
-
--metric-data "[{
|
|
311
|
-
\"MetricName\": \"${METRIC_NAME}\",
|
|
312
|
-
\"Value\": ${METRIC_VALUE},
|
|
313
|
-
\"Unit\": \"Count\",
|
|
314
|
-
\"Timestamp\": \"${TIMESTAMP}\",
|
|
315
|
-
\"Dimensions\": [
|
|
316
|
-
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
317
|
-
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
318
|
-
]
|
|
319
|
-
}]"
|
|
320
|
-
done
|
|
321
|
-
|
|
322
|
-
echo "Tool breakdown and adoption metrics emitted to CloudWatch."
|
|
323
|
-
echo ""
|
|
324
|
-
echo "=== DORA Metrics ==="
|
|
325
|
-
echo "Deployment Frequency: ${{ steps.dora.outputs.deploy_frequency }}/day"
|
|
326
|
-
echo "Lead Time: ${{ steps.dora.outputs.avg_lead_time_hours }} hours"
|
|
327
|
-
echo "Change Failure Rate: ${{ steps.dora.outputs.change_failure_rate }}"
|
|
328
|
-
echo "MTTR: ${{ steps.dora.outputs.mttr_hours }} hours"
|
|
329
|
-
echo ""
|
|
330
|
-
echo "=== AI-DORA Metrics ==="
|
|
331
|
-
echo "AI Adoption Rate: ${{ steps.ai-dora.outputs.ai_adoption_rate }}"
|
|
332
|
-
echo "AI-to-Merge Ratio: ${{ steps.ai-dora.outputs.ai_to_merge_ratio }}"
|
|
333
|
-
echo "Spec Coverage: ${{ steps.ai-dora.outputs.spec_coverage }}"
|
|
334
|
-
echo "Total Commits: ${{ steps.ai-dora.outputs.total_commits }}"
|
|
@@ -1,181 +0,0 @@
|
|
|
1
|
-
# prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
|
|
2
|
-
#
|
|
3
|
-
# Include this file in your .gitlab-ci.yml:
|
|
4
|
-
# include:
|
|
5
|
-
# - local: '.prism/gitlab-workflows/prism-dora-weekly.yml'
|
|
6
|
-
#
|
|
7
|
-
# Required setup:
|
|
8
|
-
# - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role with events:PutEvents + cloudwatch:PutMetricData)
|
|
9
|
-
# - CI/CD variable AWS_REGION (default: us-west-2)
|
|
10
|
-
# - .prism/config.json committed to the repo
|
|
11
|
-
# - Pipeline schedule configured: weekly on Monday at 09:00 UTC
|
|
12
|
-
|
|
13
|
-
prism-dora-weekly:
|
|
14
|
-
stage: test
|
|
15
|
-
image:
|
|
16
|
-
name: amazon/aws-cli:latest
|
|
17
|
-
entrypoint: [""]
|
|
18
|
-
id_tokens:
|
|
19
|
-
GITLAB_OIDC_TOKEN:
|
|
20
|
-
aud: https://gitlab.com
|
|
21
|
-
rules:
|
|
22
|
-
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
23
|
-
- if: '$CI_PIPELINE_SOURCE == "web"'
|
|
24
|
-
when: manual
|
|
25
|
-
variables:
|
|
26
|
-
AWS_DEFAULT_REGION: us-west-2
|
|
27
|
-
before_script:
|
|
28
|
-
- yum install -y -q jq bc git
|
|
29
|
-
- git fetch --unshallow || true
|
|
30
|
-
- |
|
|
31
|
-
# Assume AWS role via GitLab OIDC
|
|
32
|
-
if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
|
|
33
|
-
echo "Assuming role via OIDC: ${PRISM_METRICS_ROLE_ARN}"
|
|
34
|
-
if CREDS=$(aws sts assume-role-with-web-identity \
|
|
35
|
-
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
36
|
-
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
37
|
-
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
38
|
-
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
39
|
-
--output text 2>&1); then
|
|
40
|
-
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
|
|
41
|
-
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
|
|
42
|
-
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
|
|
43
|
-
echo "✓ OIDC credentials acquired."
|
|
44
|
-
else
|
|
45
|
-
echo "⚠️ OIDC assume-role failed: ${CREDS}"
|
|
46
|
-
fi
|
|
47
|
-
elif [[ -n "${PRISM_METRICS_ROLE_ARN}" ]]; then
|
|
48
|
-
echo "⚠️ PRISM_METRICS_ROLE_ARN set but GITLAB_OIDC_TOKEN is empty."
|
|
49
|
-
fi
|
|
50
|
-
script:
|
|
51
|
-
- |
|
|
52
|
-
# Skip if no AWS credentials available
|
|
53
|
-
if ! aws sts get-caller-identity &>/dev/null; then
|
|
54
|
-
echo "⚠️ No AWS credentials. Set PRISM_METRICS_ROLE_ARN or configure AWS access."
|
|
55
|
-
exit 0
|
|
56
|
-
fi
|
|
57
|
-
|
|
58
|
-
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
59
|
-
SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
|
|
60
|
-
REPO="${CI_PROJECT_PATH}"
|
|
61
|
-
NAMESPACE="PRISM/D1/Velocity"
|
|
62
|
-
|
|
63
|
-
# ---------------------------------------------------------------
|
|
64
|
-
# 1. Deployment Frequency — merged MRs to default branch (past 7 days)
|
|
65
|
-
# ---------------------------------------------------------------
|
|
66
|
-
SINCE_ISO=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
|
|
67
|
-
MERGED_MRS=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
|
|
68
|
-
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests?state=merged&updated_after=${SINCE_ISO}&per_page=100" 2>/dev/null || echo "[]")
|
|
69
|
-
DEPLOY_COUNT=$(echo "${MERGED_MRS}" | jq 'length')
|
|
70
|
-
DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
|
|
71
|
-
|
|
72
|
-
# ---------------------------------------------------------------
|
|
73
|
-
# 2. Lead Time — avg time from MR open to merge
|
|
74
|
-
# ---------------------------------------------------------------
|
|
75
|
-
AVG_LEAD_TIME_HOURS=$(echo "${MERGED_MRS}" | jq '
|
|
76
|
-
if length == 0 then 0
|
|
77
|
-
else [.[] | select(.merged_at != null and .created_at != null) |
|
|
78
|
-
((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
79
|
-
(.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
|
|
80
|
-
] | if length > 0 then add / length | . * 100 | round / 100 else 0 end
|
|
81
|
-
end')
|
|
82
|
-
|
|
83
|
-
# ---------------------------------------------------------------
|
|
84
|
-
# 3. Change Failure Rate — MRs with "revert|hotfix|rollback" in title
|
|
85
|
-
# ---------------------------------------------------------------
|
|
86
|
-
FAILURE_MRS=$(echo "${MERGED_MRS}" | jq '[.[] | select(.title // "" | test("revert|hotfix|rollback"; "i"))] | length')
|
|
87
|
-
[[ "${DEPLOY_COUNT}" -gt 0 ]] && CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_MRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./') || CHANGE_FAILURE_RATE="0"
|
|
88
|
-
|
|
89
|
-
# ---------------------------------------------------------------
|
|
90
|
-
# 4. MTTR — avg time for hotfix/revert MRs
|
|
91
|
-
# ---------------------------------------------------------------
|
|
92
|
-
MTTR_HOURS=$(echo "${MERGED_MRS}" | jq '
|
|
93
|
-
[.[] | select(.title // "" | test("revert|hotfix|rollback"; "i")) |
|
|
94
|
-
select(.merged_at != null and .created_at != null) |
|
|
95
|
-
((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
96
|
-
(.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
|
|
97
|
-
] | if length > 0 then add / length | . * 100 | round / 100 else 0 end')
|
|
98
|
-
|
|
99
|
-
# ---------------------------------------------------------------
|
|
100
|
-
# 5. AI-DORA metrics from commit trailers
|
|
101
|
-
# ---------------------------------------------------------------
|
|
102
|
-
TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline "${CI_DEFAULT_BRANCH}" 2>/dev/null | wc -l | tr -d ' ')
|
|
103
|
-
AI_GENERATED=0; AI_ASSISTED=0; HUMAN_ONLY=0; SPEC_REFS=0
|
|
104
|
-
CLAUDE_CODE_COUNT=0; KIRO_COUNT=0; Q_DEV_COUNT=0
|
|
105
|
-
|
|
106
|
-
for SHA in $(git log --since="${SINCE_DATE}" --format='%H' "${CI_DEFAULT_BRANCH}" 2>/dev/null); do
|
|
107
|
-
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
108
|
-
echo "${MSG}" | grep -q '^AI-Origin: ai-generated' && AI_GENERATED=$((AI_GENERATED + 1))
|
|
109
|
-
echo "${MSG}" | grep -q '^AI-Origin: ai-assisted' && AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
110
|
-
echo "${MSG}" | grep -q '^AI-Origin: human' && HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
111
|
-
echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
|
|
112
|
-
TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
|
|
113
|
-
case "${TOOL}" in
|
|
114
|
-
*claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
|
|
115
|
-
*kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
|
|
116
|
-
*q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
|
|
117
|
-
esac
|
|
118
|
-
done
|
|
119
|
-
|
|
120
|
-
UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
|
|
121
|
-
[[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
|
|
122
|
-
AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
|
|
123
|
-
[[ "${TOTAL_COMMITS}" -gt 0 ]] && AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || AI_ADOPTION_RATE="0"
|
|
124
|
-
[[ "${TOTAL_COMMITS}" -gt 0 ]] && SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || SPEC_COVERAGE="0"
|
|
125
|
-
|
|
126
|
-
# ---------------------------------------------------------------
|
|
127
|
-
# Emit assessment event
|
|
128
|
-
# ---------------------------------------------------------------
|
|
129
|
-
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
130
|
-
|
|
131
|
-
ASSESSMENT_EVENT=$(jq -n \
|
|
132
|
-
--arg team_id "${TEAM_ID}" --arg repo "${REPO}" --arg timestamp "${TIMESTAMP}" \
|
|
133
|
-
--argjson deploy_count "${DEPLOY_COUNT}" --argjson deploy_frequency "${DEPLOY_FREQUENCY}" \
|
|
134
|
-
--argjson avg_lead_time_hours "${AVG_LEAD_TIME_HOURS}" \
|
|
135
|
-
--argjson change_failure_rate "${CHANGE_FAILURE_RATE}" --argjson mttr_hours "${MTTR_HOURS}" \
|
|
136
|
-
--argjson total_commits "${TOTAL_COMMITS}" --argjson ai_generated "${AI_GENERATED}" \
|
|
137
|
-
--argjson ai_assisted "${AI_ASSISTED}" --argjson human_only "${HUMAN_ONLY}" \
|
|
138
|
-
--argjson untagged "${UNTAGGED}" --argjson ai_adoption_rate "${AI_ADOPTION_RATE}" \
|
|
139
|
-
--argjson spec_coverage "${SPEC_COVERAGE}" \
|
|
140
|
-
--argjson claude_code "${CLAUDE_CODE_COUNT}" --argjson kiro "${KIRO_COUNT}" --argjson q_dev "${Q_DEV_COUNT}" \
|
|
141
|
-
'{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:2,
|
|
142
|
-
metric:{name:"weekly_assessment",value:1,unit:"assessment"},
|
|
143
|
-
dora:{deployment_frequency:$deploy_frequency, deployments:$deploy_count,
|
|
144
|
-
lead_time_hours:$avg_lead_time_hours, change_failure_rate:$change_failure_rate,
|
|
145
|
-
mttr_hours:$mttr_hours},
|
|
146
|
-
ai_dora:{ai_adoption_rate:$ai_adoption_rate, ai_to_merge_ratio:$ai_adoption_rate,
|
|
147
|
-
total_commits:$total_commits, ai_generated_commits:$ai_generated,
|
|
148
|
-
ai_assisted_commits:$ai_assisted, human_commits:$human_only,
|
|
149
|
-
untagged_commits:$untagged, spec_coverage:$spec_coverage,
|
|
150
|
-
tool_breakdown:{claude_code:$claude_code, kiro:$kiro, q_developer:$q_dev}}}')
|
|
151
|
-
|
|
152
|
-
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
153
|
-
--entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.assessment\",
|
|
154
|
-
\"EventBusName\":\"prism-d1-metrics\",
|
|
155
|
-
\"Detail\":$(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')}]"
|
|
156
|
-
|
|
157
|
-
# Emit CloudWatch metrics
|
|
158
|
-
for METRIC_DATA in \
|
|
159
|
-
"AIAdoptionRate:${AI_ADOPTION_RATE}:Percent" \
|
|
160
|
-
"SpecCoverage:${SPEC_COVERAGE}:Percent" \
|
|
161
|
-
"ClaudeCodeCommits:${CLAUDE_CODE_COUNT}:Count" \
|
|
162
|
-
"KiroCommits:${KIRO_COUNT}:Count" \
|
|
163
|
-
"QDeveloperCommits:${Q_DEV_COUNT}:Count" \
|
|
164
|
-
"UntaggedCommits:${UNTAGGED}:Count"; do
|
|
165
|
-
METRIC_NAME="${METRIC_DATA%%:*}"; REST="${METRIC_DATA#*:}"
|
|
166
|
-
METRIC_VALUE="${REST%%:*}"; METRIC_UNIT="${REST#*:}"
|
|
167
|
-
aws cloudwatch put-metric-data --region "${AWS_DEFAULT_REGION}" --namespace "${NAMESPACE}" \
|
|
168
|
-
--metric-data "[{\"MetricName\":\"${METRIC_NAME}\",\"Value\":${METRIC_VALUE},
|
|
169
|
-
\"Unit\":\"${METRIC_UNIT}\",\"Timestamp\":\"${TIMESTAMP}\",
|
|
170
|
-
\"Dimensions\":[{\"Name\":\"TeamId\",\"Value\":\"${TEAM_ID}\"},{\"Name\":\"Repository\",\"Value\":\"${REPO}\"}]}]"
|
|
171
|
-
done
|
|
172
|
-
|
|
173
|
-
echo "=== DORA Metrics ==="
|
|
174
|
-
echo "Deployment Frequency: ${DEPLOY_FREQUENCY}/day"
|
|
175
|
-
echo "Lead Time: ${AVG_LEAD_TIME_HOURS} hours"
|
|
176
|
-
echo "Change Failure Rate: ${CHANGE_FAILURE_RATE}"
|
|
177
|
-
echo "MTTR: ${MTTR_HOURS} hours"
|
|
178
|
-
echo "=== AI-DORA Metrics ==="
|
|
179
|
-
echo "AI Adoption Rate: ${AI_ADOPTION_RATE}"
|
|
180
|
-
echo "Spec Coverage: ${SPEC_COVERAGE}"
|
|
181
|
-
echo "Total Commits: ${TOTAL_COMMITS}"
|
|
@@ -1,142 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
DynamoDBClient,
|
|
3
|
-
QueryCommand,
|
|
4
|
-
} from '@aws-sdk/client-dynamodb';
|
|
5
|
-
import {
|
|
6
|
-
EventBridgeClient,
|
|
7
|
-
PutEventsCommand,
|
|
8
|
-
} from '@aws-sdk/client-eventbridge';
|
|
9
|
-
|
|
10
|
-
const dynamoClient = new DynamoDBClient({});
|
|
11
|
-
const eventBridgeClient = new EventBridgeClient({});
|
|
12
|
-
|
|
13
|
-
const EVENTS_TABLE = process.env.EVENTS_TABLE!;
|
|
14
|
-
const EVENT_BUS_NAME = process.env.EVENT_BUS_NAME!;
|
|
15
|
-
const LOOKBACK_HOURS = parseInt(process.env.LOOKBACK_HOURS ?? '24', 10);
|
|
16
|
-
|
|
17
|
-
interface DeployEvent {
|
|
18
|
-
source: string;
|
|
19
|
-
'detail-type': string;
|
|
20
|
-
detail: {
|
|
21
|
-
team_id: string;
|
|
22
|
-
repo: string;
|
|
23
|
-
timestamp: string;
|
|
24
|
-
dora?: {
|
|
25
|
-
change_failure_rate?: number;
|
|
26
|
-
};
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Triggered by failed deployment events.
|
|
32
|
-
* Queries recent commits by AI origin, calculates AI vs human defect rates,
|
|
33
|
-
* and emits a prism.d1.quality event.
|
|
34
|
-
*/
|
|
35
|
-
export async function handler(event: DeployEvent): Promise<void> {
|
|
36
|
-
const detail = event.detail;
|
|
37
|
-
|
|
38
|
-
// Only process deployment failures
|
|
39
|
-
const cfr = detail.dora?.change_failure_rate ?? 0;
|
|
40
|
-
if (cfr === 0) {
|
|
41
|
-
console.log('Deployment succeeded, no defect correlation needed');
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
console.log(`Processing failed deployment for ${detail.team_id}/${detail.repo}`);
|
|
46
|
-
|
|
47
|
-
const deployTime = new Date(detail.timestamp);
|
|
48
|
-
const lookbackStart = new Date(deployTime.getTime() - LOOKBACK_HOURS * 60 * 60 * 1000);
|
|
49
|
-
|
|
50
|
-
// Query recent commit events for this repo
|
|
51
|
-
try {
|
|
52
|
-
const result = await dynamoClient.send(
|
|
53
|
-
new QueryCommand({
|
|
54
|
-
TableName: EVENTS_TABLE,
|
|
55
|
-
KeyConditionExpression: 'pk = :pk AND sk BETWEEN :start AND :end',
|
|
56
|
-
ExpressionAttributeValues: {
|
|
57
|
-
':pk': { S: `${detail.team_id}#${detail.repo}` },
|
|
58
|
-
':start': { S: lookbackStart.toISOString() },
|
|
59
|
-
':end': { S: deployTime.toISOString() },
|
|
60
|
-
},
|
|
61
|
-
}),
|
|
62
|
-
);
|
|
63
|
-
|
|
64
|
-
if (!result.Items || result.Items.length === 0) {
|
|
65
|
-
console.log('No recent commits found in lookback window');
|
|
66
|
-
return;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
// Count commits by AI origin
|
|
70
|
-
let aiCommitCount = 0;
|
|
71
|
-
let humanCommitCount = 0;
|
|
72
|
-
|
|
73
|
-
for (const item of result.Items) {
|
|
74
|
-
const detailType = item.detail_type?.S ?? '';
|
|
75
|
-
if (detailType !== 'prism.d1.commit') continue;
|
|
76
|
-
|
|
77
|
-
const data = JSON.parse(item.data?.S ?? '{}');
|
|
78
|
-
const origin = data.ai_context?.origin ?? 'human';
|
|
79
|
-
|
|
80
|
-
if (origin === 'ai-generated' || origin === 'ai-assisted') {
|
|
81
|
-
aiCommitCount++;
|
|
82
|
-
} else {
|
|
83
|
-
humanCommitCount++;
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const totalCommits = aiCommitCount + humanCommitCount;
|
|
88
|
-
if (totalCommits === 0) {
|
|
89
|
-
console.log('No commit events in lookback window');
|
|
90
|
-
return;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
// Calculate defect rates using proportional distribution.
|
|
94
|
-
// Since we can't attribute a specific failure to a specific commit,
|
|
95
|
-
// we distribute the change failure rate proportionally by code origin.
|
|
96
|
-
// This gives a directional signal: if 80% of commits are AI and the
|
|
97
|
-
// deploy failed, AI code carries 80% of the attributed failure weight.
|
|
98
|
-
const aiDefectRate = aiCommitCount > 0 ? (cfr * aiCommitCount) / totalCommits : 0;
|
|
99
|
-
const humanDefectRate = humanCommitCount > 0 ? (cfr * humanCommitCount) / totalCommits : 0;
|
|
100
|
-
|
|
101
|
-
const qualityEvent = {
|
|
102
|
-
team_id: detail.team_id,
|
|
103
|
-
repo: detail.repo,
|
|
104
|
-
timestamp: detail.timestamp,
|
|
105
|
-
prism_level: 2,
|
|
106
|
-
metric: {
|
|
107
|
-
name: 'post_merge_defect_rate',
|
|
108
|
-
value: cfr,
|
|
109
|
-
unit: 'percent',
|
|
110
|
-
},
|
|
111
|
-
ai_dora: {
|
|
112
|
-
post_merge_defect_rate: cfr,
|
|
113
|
-
},
|
|
114
|
-
quality: {
|
|
115
|
-
deployment_id: `deploy-${Date.now()}`,
|
|
116
|
-
ai_defect_rate: Math.round(aiDefectRate * 10000) / 10000,
|
|
117
|
-
human_defect_rate: Math.round(humanDefectRate * 10000) / 10000,
|
|
118
|
-
total_ai_commits: aiCommitCount,
|
|
119
|
-
total_human_commits: humanCommitCount,
|
|
120
|
-
},
|
|
121
|
-
};
|
|
122
|
-
|
|
123
|
-
await eventBridgeClient.send(
|
|
124
|
-
new PutEventsCommand({
|
|
125
|
-
Entries: [
|
|
126
|
-
{
|
|
127
|
-
Source: 'prism.d1.velocity',
|
|
128
|
-
DetailType: 'prism.d1.quality',
|
|
129
|
-
EventBusName: EVENT_BUS_NAME,
|
|
130
|
-
Detail: JSON.stringify(qualityEvent),
|
|
131
|
-
},
|
|
132
|
-
],
|
|
133
|
-
}),
|
|
134
|
-
);
|
|
135
|
-
|
|
136
|
-
console.log(
|
|
137
|
-
`Emitted quality event: AI defect rate ${aiDefectRate.toFixed(4)}, Human defect rate ${humanDefectRate.toFixed(4)}, ${aiCommitCount} AI commits, ${humanCommitCount} human commits`,
|
|
138
|
-
);
|
|
139
|
-
} catch (err) {
|
|
140
|
-
console.error('Failed to correlate defects:', err);
|
|
141
|
-
}
|
|
142
|
-
}
|