@prism-d1/cli 1.0.27 → 1.0.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/eval-harness/README.md +114 -0
- package/dist/assets/eval-harness/eval-config.json +10 -0
- package/dist/assets/eval-harness/rubrics/agent-quality.json +79 -0
- package/dist/assets/eval-harness/rubrics/api-response-quality.json +45 -0
- package/dist/assets/eval-harness/rubrics/code-quality.json +98 -0
- package/dist/assets/eval-harness/rubrics/security-compliance.json +145 -0
- package/dist/assets/eval-harness/rubrics/spec-compliance.json +67 -0
- package/dist/assets/eval-harness/run-eval.sh +122 -0
- package/dist/assets/github-workflows/README.md +110 -0
- package/dist/assets/github-workflows/prism-agent-eval.yml +313 -0
- package/dist/assets/github-workflows/prism-ai-metrics.yml +261 -0
- package/dist/assets/github-workflows/prism-dora-weekly.yml +334 -0
- package/dist/assets/github-workflows/prism-eval-gate.yml +310 -0
- package/dist/assets/infra/bin/app.ts +56 -0
- package/dist/assets/infra/cdk.json +12 -0
- package/dist/assets/infra/lib/api-stack.ts +347 -0
- package/dist/assets/infra/lib/constructs/bedrock-guardrail-construct.ts +201 -0
- package/dist/assets/infra/lib/constructs/guardrail-enforcer-construct.ts +59 -0
- package/dist/assets/infra/lib/constructs/prism-vpc-construct.ts +75 -0
- package/dist/assets/infra/lib/constructs/security-agent-construct.ts +266 -0
- package/dist/assets/infra/lib/dashboard-stack.ts +1392 -0
- package/dist/assets/infra/lib/lambda/api-handler.ts +477 -0
- package/dist/assets/infra/lib/lambda/defect-correlator.ts +142 -0
- package/dist/assets/infra/lib/lambda/exfiltration-detector.ts +100 -0
- package/dist/assets/infra/lib/lambda/layers/guardrail-enforcer/nodejs/guardrail-enforcer.js +53 -0
- package/dist/assets/infra/lib/lambda/metrics-processor.ts +748 -0
- package/dist/assets/infra/lib/lambda/security-agent-processor.ts +231 -0
- package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +120 -0
- package/dist/assets/infra/lib/lambda/security-response-automator.ts +130 -0
- package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +123 -0
- package/dist/assets/infra/lib/metrics-pipeline-stack.ts +701 -0
- package/dist/assets/infra/package.json +23 -0
- package/dist/assets/infra/tsconfig.json +24 -0
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +3 -4
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-git-hooks.js +2 -5
- package/dist/src/commands/bootstrapper/install-git-hooks.js.map +1 -1
- package/dist/src/commands/securityagent/setup.d.ts.map +1 -1
- package/dist/src/commands/securityagent/setup.js +2 -3
- package/dist/src/commands/securityagent/setup.js.map +1 -1
- package/dist/src/commands/workshop/deploy-infra.d.ts.map +1 -1
- package/dist/src/commands/workshop/deploy-infra.js +2 -3
- package/dist/src/commands/workshop/deploy-infra.js.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.d.ts.map +1 -1
- package/dist/src/commands/workshop/generate-demo-data.js +3 -8
- package/dist/src/commands/workshop/generate-demo-data.js.map +1 -1
- package/dist/src/commands/workshop/perform-pen-test.d.ts.map +1 -1
- package/dist/src/commands/workshop/perform-pen-test.js +5 -14
- package/dist/src/commands/workshop/perform-pen-test.js.map +1 -1
- package/dist/src/utils/root.d.ts +6 -0
- package/dist/src/utils/root.d.ts.map +1 -1
- package/dist/src/utils/root.js +29 -0
- package/dist/src/utils/root.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
# prism-ai-metrics.yml — Emit AI-assisted development metrics on PR merge.
|
|
2
|
+
#
|
|
3
|
+
# Copy this file to .github/workflows/prism-ai-metrics.yml in your repo.
|
|
4
|
+
#
|
|
5
|
+
# Required setup:
|
|
6
|
+
# - OIDC provider configured for GitHub Actions in your AWS account
|
|
7
|
+
# - IAM role with events:PutEvents on the prism-d1-metrics bus
|
|
8
|
+
# - Repository secret PRISM_METRICS_ROLE_ARN set to the OIDC role ARN
|
|
9
|
+
# - .prism/config.json committed to the repo (created by: bash prism-cli bootstrapper install-git-hooks)
|
|
10
|
+
|
|
11
|
+
name: PRISM AI Metrics
|
|
12
|
+
|
|
13
|
+
on:
|
|
14
|
+
pull_request:
|
|
15
|
+
types: [closed]
|
|
16
|
+
branches: [main, master]
|
|
17
|
+
|
|
18
|
+
permissions:
|
|
19
|
+
id-token: write
|
|
20
|
+
contents: read
|
|
21
|
+
pull-requests: read
|
|
22
|
+
|
|
23
|
+
jobs:
|
|
24
|
+
emit-metrics:
|
|
25
|
+
if: github.event.pull_request.merged == true
|
|
26
|
+
runs-on: ubuntu-latest
|
|
27
|
+
steps:
|
|
28
|
+
- name: Checkout
|
|
29
|
+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
30
|
+
with:
|
|
31
|
+
fetch-depth: 0
|
|
32
|
+
|
|
33
|
+
- name: Configure AWS credentials (OIDC)
|
|
34
|
+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
|
|
35
|
+
with:
|
|
36
|
+
role-to-assume: ${{ secrets.PRISM_METRICS_ROLE_ARN }}
|
|
37
|
+
aws-region: us-west-2
|
|
38
|
+
|
|
39
|
+
- name: Read team ID from .prism/config.json
|
|
40
|
+
id: prism-config
|
|
41
|
+
run: |
|
|
42
|
+
if [[ -f .prism/config.json ]]; then
|
|
43
|
+
TEAM_ID=$(jq -r '.team_id // ""' .prism/config.json 2>/dev/null || echo "")
|
|
44
|
+
else
|
|
45
|
+
TEAM_ID=""
|
|
46
|
+
fi
|
|
47
|
+
if [[ -z "${TEAM_ID}" ]]; then
|
|
48
|
+
TEAM_ID="no_team"
|
|
49
|
+
fi
|
|
50
|
+
echo "team_id=${TEAM_ID}" >> "$GITHUB_OUTPUT"
|
|
51
|
+
|
|
52
|
+
- name: Calculate AI-to-merge ratio
|
|
53
|
+
id: ai-ratio
|
|
54
|
+
run: |
|
|
55
|
+
BASE_SHA="${{ github.event.pull_request.base.sha }}"
|
|
56
|
+
HEAD_SHA="${{ github.event.pull_request.merge_commit_sha }}"
|
|
57
|
+
|
|
58
|
+
TOTAL_COMMITS=$(git rev-list --count "${BASE_SHA}..${HEAD_SHA}")
|
|
59
|
+
|
|
60
|
+
# Count AI origin per-commit (not per-line) to avoid overcounting
|
|
61
|
+
AI_GENERATED=0
|
|
62
|
+
AI_ASSISTED=0
|
|
63
|
+
HUMAN_ONLY=0
|
|
64
|
+
TOTAL_INPUT_TOKENS=0
|
|
65
|
+
TOTAL_OUTPUT_TOKENS=0
|
|
66
|
+
TOTAL_COST="0"
|
|
67
|
+
for SHA in $(git rev-list "${BASE_SHA}..${HEAD_SHA}"); do
|
|
68
|
+
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
69
|
+
if echo "${MSG}" | grep -q '^AI-Origin: ai-generated'; then
|
|
70
|
+
AI_GENERATED=$((AI_GENERATED + 1))
|
|
71
|
+
elif echo "${MSG}" | grep -q '^AI-Origin: ai-assisted'; then
|
|
72
|
+
AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
73
|
+
elif echo "${MSG}" | grep -q '^AI-Origin: human'; then
|
|
74
|
+
HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
75
|
+
fi
|
|
76
|
+
|
|
77
|
+
# Accumulate token usage trailers (clamped to prevent injection)
|
|
78
|
+
INPUT_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Input-Tokens: )\d+' || echo "0")
|
|
79
|
+
OUTPUT_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Output-Tokens: )\d+' || echo "0")
|
|
80
|
+
COST_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Cost: )[0-9.]+' || echo "0")
|
|
81
|
+
[[ ${INPUT_T} -gt 1000000 ]] && INPUT_T=0
|
|
82
|
+
[[ ${OUTPUT_T} -gt 1000000 ]] && OUTPUT_T=0
|
|
83
|
+
(( $(echo "${COST_T} > 100" | bc -l) )) && COST_T=0
|
|
84
|
+
TOTAL_INPUT_TOKENS=$((TOTAL_INPUT_TOKENS + INPUT_T))
|
|
85
|
+
TOTAL_OUTPUT_TOKENS=$((TOTAL_OUTPUT_TOKENS + OUTPUT_T))
|
|
86
|
+
TOTAL_COST=$(echo "${TOTAL_COST} + ${COST_T}" | bc -l)
|
|
87
|
+
done
|
|
88
|
+
|
|
89
|
+
AI_COMMITS=$((AI_GENERATED + AI_ASSISTED))
|
|
90
|
+
if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
|
|
91
|
+
AI_RATIO=$(echo "scale=4; ${AI_COMMITS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
|
|
92
|
+
else
|
|
93
|
+
AI_RATIO="0"
|
|
94
|
+
fi
|
|
95
|
+
|
|
96
|
+
# Detect tools used
|
|
97
|
+
TOOLS_USED=$(git log "${BASE_SHA}..${HEAD_SHA}" --format='%B' | grep '^AI-Origin:' | sort -u | head -5 | tr '\n' ',' | sed 's/,$//')
|
|
98
|
+
|
|
99
|
+
echo "total_commits=${TOTAL_COMMITS}" >> "$GITHUB_OUTPUT"
|
|
100
|
+
echo "ai_commits=${AI_COMMITS}" >> "$GITHUB_OUTPUT"
|
|
101
|
+
echo "ai_ratio=${AI_RATIO}" >> "$GITHUB_OUTPUT"
|
|
102
|
+
echo "ai_generated=${AI_GENERATED}" >> "$GITHUB_OUTPUT"
|
|
103
|
+
echo "ai_assisted=${AI_ASSISTED}" >> "$GITHUB_OUTPUT"
|
|
104
|
+
echo "human_only=${HUMAN_ONLY}" >> "$GITHUB_OUTPUT"
|
|
105
|
+
echo "total_input_tokens=${TOTAL_INPUT_TOKENS}" >> "$GITHUB_OUTPUT"
|
|
106
|
+
echo "total_output_tokens=${TOTAL_OUTPUT_TOKENS}" >> "$GITHUB_OUTPUT"
|
|
107
|
+
echo "total_cost=${TOTAL_COST}" >> "$GITHUB_OUTPUT"
|
|
108
|
+
|
|
109
|
+
- name: Calculate lead time
|
|
110
|
+
id: lead-time
|
|
111
|
+
run: |
|
|
112
|
+
CREATED_AT="${{ github.event.pull_request.created_at }}"
|
|
113
|
+
MERGED_AT="${{ github.event.pull_request.merged_at }}"
|
|
114
|
+
|
|
115
|
+
CREATED_TS=$(date -d "${CREATED_AT}" +%s 2>/dev/null || date -jf "%Y-%m-%dT%H:%M:%SZ" "${CREATED_AT}" +%s)
|
|
116
|
+
MERGED_TS=$(date -d "${MERGED_AT}" +%s 2>/dev/null || date -jf "%Y-%m-%dT%H:%M:%SZ" "${MERGED_AT}" +%s)
|
|
117
|
+
|
|
118
|
+
LEAD_TIME_SECONDS=$((MERGED_TS - CREATED_TS))
|
|
119
|
+
LEAD_TIME_HOURS=$(echo "scale=2; ${LEAD_TIME_SECONDS} / 3600" | bc -l | sed 's/^\./0./')
|
|
120
|
+
|
|
121
|
+
echo "lead_time_seconds=${LEAD_TIME_SECONDS}" >> "$GITHUB_OUTPUT"
|
|
122
|
+
echo "lead_time_hours=${LEAD_TIME_HOURS}" >> "$GITHUB_OUTPUT"
|
|
123
|
+
|
|
124
|
+
- name: Calculate AI acceptance rate
|
|
125
|
+
id: acceptance
|
|
126
|
+
env:
|
|
127
|
+
GH_TOKEN: ${{ github.token }}
|
|
128
|
+
run: |
|
|
129
|
+
PR_NUMBER="${{ github.event.pull_request.number }}"
|
|
130
|
+
REPO="${{ github.repository }}"
|
|
131
|
+
|
|
132
|
+
# Get review decisions for this PR
|
|
133
|
+
REVIEWS=$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/reviews" \
|
|
134
|
+
--jq '[.[] | select(.state == "APPROVED" or .state == "CHANGES_REQUESTED")]' 2>/dev/null || echo "[]")
|
|
135
|
+
|
|
136
|
+
APPROVED=$(echo "${REVIEWS}" | jq '[.[] | select(.state == "APPROVED")] | length')
|
|
137
|
+
CHANGES_REQUESTED=$(echo "${REVIEWS}" | jq '[.[] | select(.state == "CHANGES_REQUESTED")] | length')
|
|
138
|
+
TOTAL_REVIEWS=$((APPROVED + CHANGES_REQUESTED))
|
|
139
|
+
|
|
140
|
+
if [[ "${TOTAL_REVIEWS}" -gt 0 ]]; then
|
|
141
|
+
ACCEPTANCE_RATE=$(echo "scale=4; ${APPROVED} / ${TOTAL_REVIEWS}" | bc -l | sed 's/^\./0./')
|
|
142
|
+
else
|
|
143
|
+
ACCEPTANCE_RATE="1" # No reviews = accepted by default
|
|
144
|
+
fi
|
|
145
|
+
|
|
146
|
+
# Only meaningful for AI PRs
|
|
147
|
+
AI_RATIO="${{ steps.ai-ratio.outputs.ai_ratio }}"
|
|
148
|
+
if [[ $(echo "${AI_RATIO} > 0" | bc -l) -eq 1 ]]; then
|
|
149
|
+
AI_ACCEPTANCE_RATE="${ACCEPTANCE_RATE}"
|
|
150
|
+
else
|
|
151
|
+
AI_ACCEPTANCE_RATE=""
|
|
152
|
+
fi
|
|
153
|
+
|
|
154
|
+
echo "acceptance_rate=${ACCEPTANCE_RATE}" >> "$GITHUB_OUTPUT"
|
|
155
|
+
echo "ai_acceptance_rate=${AI_ACCEPTANCE_RATE}" >> "$GITHUB_OUTPUT"
|
|
156
|
+
echo "approved=${APPROVED}" >> "$GITHUB_OUTPUT"
|
|
157
|
+
echo "changes_requested=${CHANGES_REQUESTED}" >> "$GITHUB_OUTPUT"
|
|
158
|
+
|
|
159
|
+
- name: Emit prism.d1.pr event
|
|
160
|
+
run: |
|
|
161
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
162
|
+
TEAM_ID="${{ steps.prism-config.outputs.team_id }}"
|
|
163
|
+
REPO="${{ github.repository }}"
|
|
164
|
+
|
|
165
|
+
PR_EVENT=$(jq -n \
|
|
166
|
+
--arg team_id "${TEAM_ID}" \
|
|
167
|
+
--arg repo "${REPO}" \
|
|
168
|
+
--arg timestamp "${TIMESTAMP}" \
|
|
169
|
+
--argjson ai_ratio "${{ steps.ai-ratio.outputs.ai_ratio }}" \
|
|
170
|
+
--argjson total_commits "${{ steps.ai-ratio.outputs.total_commits }}" \
|
|
171
|
+
--argjson ai_commits "${{ steps.ai-ratio.outputs.ai_commits }}" \
|
|
172
|
+
--argjson ai_generated "${{ steps.ai-ratio.outputs.ai_generated }}" \
|
|
173
|
+
--argjson ai_assisted "${{ steps.ai-ratio.outputs.ai_assisted }}" \
|
|
174
|
+
--argjson lead_time_seconds "${{ steps.lead-time.outputs.lead_time_seconds }}" \
|
|
175
|
+
--arg pr_number "${{ github.event.pull_request.number }}" \
|
|
176
|
+
--arg pr_author "${{ github.event.pull_request.user.login }}" \
|
|
177
|
+
--argjson approved "${{ steps.acceptance.outputs.approved }}" \
|
|
178
|
+
--argjson changes_requested "${{ steps.acceptance.outputs.changes_requested }}" \
|
|
179
|
+
--arg acceptance_rate "${{ steps.acceptance.outputs.acceptance_rate }}" \
|
|
180
|
+
--arg ai_acceptance_rate "${{ steps.acceptance.outputs.ai_acceptance_rate }}" \
|
|
181
|
+
--argjson total_input_tokens "${{ steps.ai-ratio.outputs.total_input_tokens }}" \
|
|
182
|
+
--argjson total_output_tokens "${{ steps.ai-ratio.outputs.total_output_tokens }}" \
|
|
183
|
+
--arg total_cost "${{ steps.ai-ratio.outputs.total_cost }}" \
|
|
184
|
+
'{
|
|
185
|
+
"team_id": $team_id,
|
|
186
|
+
"repo": $repo,
|
|
187
|
+
"timestamp": $timestamp,
|
|
188
|
+
"prism_level": 2,
|
|
189
|
+
"metric": {
|
|
190
|
+
"name": "ai_to_merge_ratio",
|
|
191
|
+
"value": $ai_ratio,
|
|
192
|
+
"unit": "ratio"
|
|
193
|
+
},
|
|
194
|
+
"ai_context": {
|
|
195
|
+
"tool": "github-actions",
|
|
196
|
+
"model": "n/a",
|
|
197
|
+
"origin": (if $ai_ratio > 0.5 then "ai-assisted" elif $ai_ratio > 0 then "ai-assisted" else "human" end)
|
|
198
|
+
},
|
|
199
|
+
"dora": {
|
|
200
|
+
"deployment_frequency": 1,
|
|
201
|
+
"lead_time_seconds": $lead_time_seconds
|
|
202
|
+
},
|
|
203
|
+
"ai_dora": {
|
|
204
|
+
"ai_to_merge_ratio": $ai_ratio,
|
|
205
|
+
"ai_acceptance_rate": (if $ai_acceptance_rate != "" then ($ai_acceptance_rate | tonumber) else null end),
|
|
206
|
+
"total_commits": $total_commits,
|
|
207
|
+
"ai_commits": $ai_commits,
|
|
208
|
+
"ai_generated_commits": $ai_generated,
|
|
209
|
+
"ai_assisted_commits": $ai_assisted,
|
|
210
|
+
"total_input_tokens": $total_input_tokens,
|
|
211
|
+
"total_output_tokens": $total_output_tokens,
|
|
212
|
+
"total_cost_usd": (if $total_cost != "0" and $total_cost != "" then ($total_cost | tonumber) else 0 end)
|
|
213
|
+
},
|
|
214
|
+
"pr": {
|
|
215
|
+
"number": ($pr_number | tonumber),
|
|
216
|
+
"author": $pr_author,
|
|
217
|
+
"reviews_approved": $approved,
|
|
218
|
+
"reviews_changes_requested": $changes_requested
|
|
219
|
+
}
|
|
220
|
+
}')
|
|
221
|
+
|
|
222
|
+
aws events put-events \
|
|
223
|
+
--region us-west-2 \
|
|
224
|
+
--entries "[{
|
|
225
|
+
\"Source\": \"prism.d1.velocity\",
|
|
226
|
+
\"DetailType\": \"prism.d1.pr\",
|
|
227
|
+
\"EventBusName\": \"prism-d1-metrics\",
|
|
228
|
+
\"Detail\": $(echo "${PR_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
229
|
+
}]"
|
|
230
|
+
|
|
231
|
+
- name: Emit prism.d1.deploy event
|
|
232
|
+
run: |
|
|
233
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
234
|
+
|
|
235
|
+
DEPLOY_EVENT=$(jq -n \
|
|
236
|
+
--arg team_id "${{ steps.prism-config.outputs.team_id }}" \
|
|
237
|
+
--arg repo "${{ github.repository }}" \
|
|
238
|
+
--arg timestamp "${TIMESTAMP}" \
|
|
239
|
+
'{
|
|
240
|
+
"team_id": $team_id,
|
|
241
|
+
"repo": $repo,
|
|
242
|
+
"timestamp": $timestamp,
|
|
243
|
+
"prism_level": 2,
|
|
244
|
+
"metric": {
|
|
245
|
+
"name": "deployment",
|
|
246
|
+
"value": 1,
|
|
247
|
+
"unit": "count"
|
|
248
|
+
},
|
|
249
|
+
"dora": {
|
|
250
|
+
"deployment_frequency": 1
|
|
251
|
+
}
|
|
252
|
+
}')
|
|
253
|
+
|
|
254
|
+
aws events put-events \
|
|
255
|
+
--region us-west-2 \
|
|
256
|
+
--entries "[{
|
|
257
|
+
\"Source\": \"prism.d1.velocity\",
|
|
258
|
+
\"DetailType\": \"prism.d1.deploy\",
|
|
259
|
+
\"EventBusName\": \"prism-d1-metrics\",
|
|
260
|
+
\"Detail\": $(echo "${DEPLOY_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
261
|
+
}]"
|
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
# prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
|
|
2
|
+
#
|
|
3
|
+
# Copy this file to .github/workflows/prism-dora-weekly.yml in your repo.
|
|
4
|
+
#
|
|
5
|
+
# Methodology:
|
|
6
|
+
# - Deployment Frequency: merged PRs to main (proxy for deploys)
|
|
7
|
+
# - Lead Time: avg hours from PR open to merge
|
|
8
|
+
# - Change Failure Rate: PRs with "revert|hotfix|rollback" in title / total merged
|
|
9
|
+
# - MTTR: avg lead time of revert/hotfix PRs
|
|
10
|
+
# - AI metrics: derived from AI-Origin commit trailers
|
|
11
|
+
#
|
|
12
|
+
# Required setup:
|
|
13
|
+
# - OIDC provider configured for GitHub Actions in your AWS account
|
|
14
|
+
# - IAM role with events:PutEvents and cloudwatch:PutMetricData permissions
|
|
15
|
+
# - Repository secret PRISM_METRICS_ROLE_ARN set to the OIDC role ARN
|
|
16
|
+
# - .prism/config.json committed to the repo (created by: bash prism-cli bootstrapper install-git-hooks)
|
|
17
|
+
|
|
18
|
+
name: PRISM DORA Weekly Assessment
|
|
19
|
+
|
|
20
|
+
on:
|
|
21
|
+
schedule:
|
|
22
|
+
- cron: '0 9 * * 1' # Every Monday at 09:00 UTC
|
|
23
|
+
workflow_dispatch: # Allow manual trigger
|
|
24
|
+
|
|
25
|
+
permissions:
|
|
26
|
+
id-token: write
|
|
27
|
+
contents: read
|
|
28
|
+
pull-requests: read
|
|
29
|
+
|
|
30
|
+
jobs:
|
|
31
|
+
dora-assessment:
|
|
32
|
+
runs-on: ubuntu-latest
|
|
33
|
+
steps:
|
|
34
|
+
- name: Checkout
|
|
35
|
+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
36
|
+
with:
|
|
37
|
+
fetch-depth: 0
|
|
38
|
+
|
|
39
|
+
- name: Configure AWS credentials (OIDC)
|
|
40
|
+
uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
|
|
41
|
+
with:
|
|
42
|
+
role-to-assume: ${{ secrets.PRISM_METRICS_ROLE_ARN }}
|
|
43
|
+
aws-region: us-west-2
|
|
44
|
+
|
|
45
|
+
- name: Read team ID from .prism/config.json
|
|
46
|
+
id: prism-config
|
|
47
|
+
run: |
|
|
48
|
+
if [[ -f .prism/config.json ]]; then
|
|
49
|
+
TEAM_ID=$(jq -r '.team_id' .prism/config.json)
|
|
50
|
+
else
|
|
51
|
+
echo "::error::.prism/config.json not found. Run bootstrapper/metric-hooks/install.sh first."
|
|
52
|
+
exit 1
|
|
53
|
+
fi
|
|
54
|
+
echo "team_id=${TEAM_ID}" >> "$GITHUB_OUTPUT"
|
|
55
|
+
|
|
56
|
+
- name: Calculate DORA metrics (past 7 days)
|
|
57
|
+
id: dora
|
|
58
|
+
env:
|
|
59
|
+
GH_TOKEN: ${{ github.token }}
|
|
60
|
+
run: |
|
|
61
|
+
set -o pipefail
|
|
62
|
+
SINCE=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
|
|
63
|
+
NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
64
|
+
REPO="${{ github.repository }}"
|
|
65
|
+
|
|
66
|
+
echo "Assessment period: ${SINCE} to ${NOW}"
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------
|
|
69
|
+
# 1. Deployment Frequency — merged PRs to main (proxy for deploys)
|
|
70
|
+
# ---------------------------------------------------------------
|
|
71
|
+
MERGED_PRS=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
72
|
+
--search "merged:>=${SINCE}" --json number,title,mergedAt --limit 200 2>/dev/null || echo "[]")
|
|
73
|
+
DEPLOY_COUNT=$(echo "${MERGED_PRS}" | jq 'length')
|
|
74
|
+
DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
|
|
75
|
+
|
|
76
|
+
# ---------------------------------------------------------------
|
|
77
|
+
# 2. Lead Time for Changes — avg time from PR open to merge
|
|
78
|
+
# ---------------------------------------------------------------
|
|
79
|
+
LEAD_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
80
|
+
--search "merged:>=${SINCE}" --json createdAt,mergedAt --limit 200 2>/dev/null || echo "[]")
|
|
81
|
+
|
|
82
|
+
AVG_LEAD_TIME_HOURS=$(echo "${LEAD_TIMES}" | jq '
|
|
83
|
+
if length == 0 then 0
|
|
84
|
+
else
|
|
85
|
+
[.[] |
|
|
86
|
+
(((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
87
|
+
(.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
|
|
88
|
+
] | add / length | . * 100 | round / 100
|
|
89
|
+
end
|
|
90
|
+
')
|
|
91
|
+
|
|
92
|
+
# ---------------------------------------------------------------
|
|
93
|
+
# 3. Change Failure Rate — PRs with "revert" or "hotfix" in title
|
|
94
|
+
# ---------------------------------------------------------------
|
|
95
|
+
FAILURE_PRS=$(echo "${MERGED_PRS}" | jq '[.[] | select(.title? // "" | test("revert|hotfix|rollback"; "i"))] | length' 2>/dev/null || echo "0")
|
|
96
|
+
if [[ "${DEPLOY_COUNT}" -gt 0 ]]; then
|
|
97
|
+
CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_PRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./')
|
|
98
|
+
else
|
|
99
|
+
CHANGE_FAILURE_RATE="0"
|
|
100
|
+
fi
|
|
101
|
+
|
|
102
|
+
# ---------------------------------------------------------------
|
|
103
|
+
# 4. Mean Time to Recovery — avg time for hotfix/revert PRs
|
|
104
|
+
# ---------------------------------------------------------------
|
|
105
|
+
RECOVERY_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
|
|
106
|
+
--search "merged:>=${SINCE}" --json title,createdAt,mergedAt --limit 200 2>/dev/null \
|
|
107
|
+
| jq '[.[] | select(.title | test("revert|hotfix|rollback"; "i"))]')
|
|
108
|
+
|
|
109
|
+
MTTR_HOURS=$(echo "${RECOVERY_TIMES}" | jq '
|
|
110
|
+
if length == 0 then 0
|
|
111
|
+
else
|
|
112
|
+
[.[] |
|
|
113
|
+
(((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
114
|
+
(.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
|
|
115
|
+
] | add / length | . * 100 | round / 100
|
|
116
|
+
end
|
|
117
|
+
')
|
|
118
|
+
|
|
119
|
+
echo "deploy_count=${DEPLOY_COUNT}" >> "$GITHUB_OUTPUT"
|
|
120
|
+
echo "deploy_frequency=${DEPLOY_FREQUENCY}" >> "$GITHUB_OUTPUT"
|
|
121
|
+
echo "avg_lead_time_hours=${AVG_LEAD_TIME_HOURS}" >> "$GITHUB_OUTPUT"
|
|
122
|
+
echo "change_failure_rate=${CHANGE_FAILURE_RATE}" >> "$GITHUB_OUTPUT"
|
|
123
|
+
echo "mttr_hours=${MTTR_HOURS}" >> "$GITHUB_OUTPUT"
|
|
124
|
+
|
|
125
|
+
- name: Calculate AI-DORA metrics (past 7 days)
|
|
126
|
+
id: ai-dora
|
|
127
|
+
run: |
|
|
128
|
+
set -o pipefail
|
|
129
|
+
SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
|
|
130
|
+
|
|
131
|
+
# All commits in the past 7 days on main
|
|
132
|
+
TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline main 2>/dev/null | wc -l | tr -d ' ')
|
|
133
|
+
|
|
134
|
+
# AI-origin breakdown — count per-commit to avoid overcounting
|
|
135
|
+
AI_GENERATED=0
|
|
136
|
+
AI_ASSISTED=0
|
|
137
|
+
HUMAN_ONLY=0
|
|
138
|
+
SPEC_REFS=0
|
|
139
|
+
CLAUDE_CODE_COUNT=0
|
|
140
|
+
KIRO_COUNT=0
|
|
141
|
+
Q_DEV_COUNT=0
|
|
142
|
+
|
|
143
|
+
for SHA in $(git log --since="${SINCE_DATE}" --format='%H' main 2>/dev/null); do
|
|
144
|
+
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
145
|
+
if echo "${MSG}" | grep -q '^AI-Origin: ai-generated'; then
|
|
146
|
+
AI_GENERATED=$((AI_GENERATED + 1))
|
|
147
|
+
elif echo "${MSG}" | grep -q '^AI-Origin: ai-assisted'; then
|
|
148
|
+
AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
149
|
+
elif echo "${MSG}" | grep -q '^AI-Origin: human'; then
|
|
150
|
+
HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
151
|
+
fi
|
|
152
|
+
echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
|
|
153
|
+
TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
|
|
154
|
+
case "${TOOL}" in
|
|
155
|
+
*claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
|
|
156
|
+
*kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
|
|
157
|
+
*q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
|
|
158
|
+
esac
|
|
159
|
+
done
|
|
160
|
+
|
|
161
|
+
UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
|
|
162
|
+
[[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
|
|
163
|
+
|
|
164
|
+
AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
|
|
165
|
+
if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
|
|
166
|
+
AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
|
|
167
|
+
else
|
|
168
|
+
AI_ADOPTION_RATE="0"
|
|
169
|
+
fi
|
|
170
|
+
|
|
171
|
+
AI_TO_MERGE_RATIO="${AI_ADOPTION_RATE}"
|
|
172
|
+
|
|
173
|
+
if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
|
|
174
|
+
SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
|
|
175
|
+
else
|
|
176
|
+
SPEC_COVERAGE="0"
|
|
177
|
+
fi
|
|
178
|
+
|
|
179
|
+
echo "total_commits=${TOTAL_COMMITS}" >> "$GITHUB_OUTPUT"
|
|
180
|
+
echo "ai_generated=${AI_GENERATED}" >> "$GITHUB_OUTPUT"
|
|
181
|
+
echo "ai_assisted=${AI_ASSISTED}" >> "$GITHUB_OUTPUT"
|
|
182
|
+
echo "human_only=${HUMAN_ONLY}" >> "$GITHUB_OUTPUT"
|
|
183
|
+
echo "untagged=${UNTAGGED}" >> "$GITHUB_OUTPUT"
|
|
184
|
+
echo "ai_adoption_rate=${AI_ADOPTION_RATE}" >> "$GITHUB_OUTPUT"
|
|
185
|
+
echo "ai_to_merge_ratio=${AI_TO_MERGE_RATIO}" >> "$GITHUB_OUTPUT"
|
|
186
|
+
echo "spec_coverage=${SPEC_COVERAGE}" >> "$GITHUB_OUTPUT"
|
|
187
|
+
echo "claude_code_count=${CLAUDE_CODE_COUNT}" >> "$GITHUB_OUTPUT"
|
|
188
|
+
echo "kiro_count=${KIRO_COUNT}" >> "$GITHUB_OUTPUT"
|
|
189
|
+
echo "q_dev_count=${Q_DEV_COUNT}" >> "$GITHUB_OUTPUT"
|
|
190
|
+
|
|
191
|
+
- name: Emit prism.d1.assessment event
|
|
192
|
+
run: |
|
|
193
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
194
|
+
|
|
195
|
+
ASSESSMENT_EVENT=$(jq -n \
|
|
196
|
+
--arg team_id "${{ steps.prism-config.outputs.team_id }}" \
|
|
197
|
+
--arg repo "${{ github.repository }}" \
|
|
198
|
+
--arg timestamp "${TIMESTAMP}" \
|
|
199
|
+
--argjson deploy_count "${{ steps.dora.outputs.deploy_count }}" \
|
|
200
|
+
--argjson deploy_frequency "${{ steps.dora.outputs.deploy_frequency }}" \
|
|
201
|
+
--argjson avg_lead_time_hours "${{ steps.dora.outputs.avg_lead_time_hours }}" \
|
|
202
|
+
--argjson change_failure_rate "${{ steps.dora.outputs.change_failure_rate }}" \
|
|
203
|
+
--argjson mttr_hours "${{ steps.dora.outputs.mttr_hours }}" \
|
|
204
|
+
--argjson total_commits "${{ steps.ai-dora.outputs.total_commits }}" \
|
|
205
|
+
--argjson ai_generated "${{ steps.ai-dora.outputs.ai_generated }}" \
|
|
206
|
+
--argjson ai_assisted "${{ steps.ai-dora.outputs.ai_assisted }}" \
|
|
207
|
+
--argjson human_only "${{ steps.ai-dora.outputs.human_only }}" \
|
|
208
|
+
--argjson untagged "${{ steps.ai-dora.outputs.untagged }}" \
|
|
209
|
+
--argjson ai_adoption_rate "${{ steps.ai-dora.outputs.ai_adoption_rate }}" \
|
|
210
|
+
--argjson ai_to_merge_ratio "${{ steps.ai-dora.outputs.ai_to_merge_ratio }}" \
|
|
211
|
+
--argjson spec_coverage "${{ steps.ai-dora.outputs.spec_coverage }}" \
|
|
212
|
+
--argjson claude_code_count "${{ steps.ai-dora.outputs.claude_code_count }}" \
|
|
213
|
+
--argjson kiro_count "${{ steps.ai-dora.outputs.kiro_count }}" \
|
|
214
|
+
--argjson q_dev_count "${{ steps.ai-dora.outputs.q_dev_count }}" \
|
|
215
|
+
'{
|
|
216
|
+
"team_id": $team_id,
|
|
217
|
+
"repo": $repo,
|
|
218
|
+
"timestamp": $timestamp,
|
|
219
|
+
"prism_level": 2,
|
|
220
|
+
"metric": {
|
|
221
|
+
"name": "weekly_assessment",
|
|
222
|
+
"value": 1,
|
|
223
|
+
"unit": "assessment"
|
|
224
|
+
},
|
|
225
|
+
"dora": {
|
|
226
|
+
"deployment_frequency": $deploy_frequency,
|
|
227
|
+
"deployments": $deploy_count,
|
|
228
|
+
"lead_time_hours": $avg_lead_time_hours,
|
|
229
|
+
"change_failure_rate": $change_failure_rate,
|
|
230
|
+
"mttr_hours": $mttr_hours
|
|
231
|
+
},
|
|
232
|
+
"ai_dora": {
|
|
233
|
+
"ai_adoption_rate": $ai_adoption_rate,
|
|
234
|
+
"ai_to_merge_ratio": $ai_to_merge_ratio,
|
|
235
|
+
"total_commits": $total_commits,
|
|
236
|
+
"ai_generated_commits": $ai_generated,
|
|
237
|
+
"ai_assisted_commits": $ai_assisted,
|
|
238
|
+
"human_commits": $human_only,
|
|
239
|
+
"untagged_commits": $untagged,
|
|
240
|
+
"spec_coverage": $spec_coverage,
|
|
241
|
+
"tool_breakdown": {
|
|
242
|
+
"claude_code": $claude_code_count,
|
|
243
|
+
"kiro": $kiro_count,
|
|
244
|
+
"q_developer": $q_dev_count
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
}')
|
|
248
|
+
|
|
249
|
+
aws events put-events \
|
|
250
|
+
--region us-west-2 \
|
|
251
|
+
--entries "[{
|
|
252
|
+
\"Source\": \"prism.d1.velocity\",
|
|
253
|
+
\"DetailType\": \"prism.d1.assessment\",
|
|
254
|
+
\"EventBusName\": \"prism-d1-metrics\",
|
|
255
|
+
\"Detail\": $(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
256
|
+
}]"
|
|
257
|
+
|
|
258
|
+
echo "Weekly DORA assessment emitted successfully."
|
|
259
|
+
|
|
260
|
+
- name: Emit tool breakdown and adoption metrics to CloudWatch
|
|
261
|
+
run: |
|
|
262
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
263
|
+
TEAM_ID="${{ steps.prism-config.outputs.team_id }}"
|
|
264
|
+
REPO="${{ github.repository }}"
|
|
265
|
+
NAMESPACE="PRISM/D1/Velocity"
|
|
266
|
+
|
|
267
|
+
# AI Adoption Rate
|
|
268
|
+
aws cloudwatch put-metric-data \
|
|
269
|
+
--region us-west-2 \
|
|
270
|
+
--namespace "${NAMESPACE}" \
|
|
271
|
+
--metric-data "[{
|
|
272
|
+
\"MetricName\": \"AIAdoptionRate\",
|
|
273
|
+
\"Value\": ${{ steps.ai-dora.outputs.ai_adoption_rate }},
|
|
274
|
+
\"Unit\": \"Percent\",
|
|
275
|
+
\"Timestamp\": \"${TIMESTAMP}\",
|
|
276
|
+
\"Dimensions\": [
|
|
277
|
+
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
278
|
+
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
279
|
+
]
|
|
280
|
+
}]"
|
|
281
|
+
|
|
282
|
+
# Spec Coverage
|
|
283
|
+
aws cloudwatch put-metric-data \
|
|
284
|
+
--region us-west-2 \
|
|
285
|
+
--namespace "${NAMESPACE}" \
|
|
286
|
+
--metric-data "[{
|
|
287
|
+
\"MetricName\": \"SpecCoverage\",
|
|
288
|
+
\"Value\": ${{ steps.ai-dora.outputs.spec_coverage }},
|
|
289
|
+
\"Unit\": \"Percent\",
|
|
290
|
+
\"Timestamp\": \"${TIMESTAMP}\",
|
|
291
|
+
\"Dimensions\": [
|
|
292
|
+
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
293
|
+
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
294
|
+
]
|
|
295
|
+
}]"
|
|
296
|
+
|
|
297
|
+
# Tool breakdown — one metric per tool
|
|
298
|
+
for TOOL_DATA in \
|
|
299
|
+
"ClaudeCodeCommits:${{ steps.ai-dora.outputs.claude_code_count }}" \
|
|
300
|
+
"KiroCommits:${{ steps.ai-dora.outputs.kiro_count }}" \
|
|
301
|
+
"QDeveloperCommits:${{ steps.ai-dora.outputs.q_dev_count }}" \
|
|
302
|
+
"UntaggedCommits:${{ steps.ai-dora.outputs.untagged }}"; do
|
|
303
|
+
|
|
304
|
+
METRIC_NAME="${TOOL_DATA%%:*}"
|
|
305
|
+
METRIC_VALUE="${TOOL_DATA##*:}"
|
|
306
|
+
|
|
307
|
+
aws cloudwatch put-metric-data \
|
|
308
|
+
--region us-west-2 \
|
|
309
|
+
--namespace "${NAMESPACE}" \
|
|
310
|
+
--metric-data "[{
|
|
311
|
+
\"MetricName\": \"${METRIC_NAME}\",
|
|
312
|
+
\"Value\": ${METRIC_VALUE},
|
|
313
|
+
\"Unit\": \"Count\",
|
|
314
|
+
\"Timestamp\": \"${TIMESTAMP}\",
|
|
315
|
+
\"Dimensions\": [
|
|
316
|
+
{\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
|
|
317
|
+
{\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
|
|
318
|
+
]
|
|
319
|
+
}]"
|
|
320
|
+
done
|
|
321
|
+
|
|
322
|
+
echo "Tool breakdown and adoption metrics emitted to CloudWatch."
|
|
323
|
+
echo ""
|
|
324
|
+
echo "=== DORA Metrics ==="
|
|
325
|
+
echo "Deployment Frequency: ${{ steps.dora.outputs.deploy_frequency }}/day"
|
|
326
|
+
echo "Lead Time: ${{ steps.dora.outputs.avg_lead_time_hours }} hours"
|
|
327
|
+
echo "Change Failure Rate: ${{ steps.dora.outputs.change_failure_rate }}"
|
|
328
|
+
echo "MTTR: ${{ steps.dora.outputs.mttr_hours }} hours"
|
|
329
|
+
echo ""
|
|
330
|
+
echo "=== AI-DORA Metrics ==="
|
|
331
|
+
echo "AI Adoption Rate: ${{ steps.ai-dora.outputs.ai_adoption_rate }}"
|
|
332
|
+
echo "AI-to-Merge Ratio: ${{ steps.ai-dora.outputs.ai_to_merge_ratio }}"
|
|
333
|
+
echo "Spec Coverage: ${{ steps.ai-dora.outputs.spec_coverage }}"
|
|
334
|
+
echo "Total Commits: ${{ steps.ai-dora.outputs.total_commits }}"
|