@prism-d1/cli 1.0.79 → 1.0.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/assets/github-workflows/prism-ai-metrics.yml +83 -122
  2. package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +66 -0
  3. package/dist/assets/github-workflows/prism-eval-gate.yml +1 -1
  4. package/dist/assets/gitlab-workflows/.gitlab-ci.yml +0 -3
  5. package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +53 -56
  6. package/dist/assets/infra/bin/app.ts +1 -0
  7. package/dist/assets/infra/lib/constructs/otel-collector-construct.ts +4 -0
  8. package/dist/assets/infra/lib/dashboard-stack.ts +378 -1235
  9. package/dist/assets/infra/lib/lambda/event-schema.ts +144 -0
  10. package/dist/assets/infra/lib/lambda/metrics-processor.ts +124 -36
  11. package/dist/assets/infra/lib/lambda/otel-receiver.ts +249 -35
  12. package/dist/assets/infra/lib/lambda/security-remediation-tracker.ts +17 -2
  13. package/dist/assets/infra/lib/lambda/velocity-widget.ts +1483 -0
  14. package/dist/assets/infra/lib/metrics-pipeline-stack.ts +110 -119
  15. package/dist/assets/infra/package.json +2 -1
  16. package/dist/assets/infra/tsconfig.json +9 -2
  17. package/dist/src/commands/bootstrapper/install-eval-harness.d.ts +8 -2
  18. package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
  19. package/dist/src/commands/bootstrapper/install-eval-harness.js +32 -32
  20. package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
  21. package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
  22. package/dist/src/commands/bootstrapper/install-github-workflows.js +7 -2
  23. package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
  24. package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
  25. package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +7 -1
  26. package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
  27. package/dist/src/utils/region.d.ts +50 -0
  28. package/dist/src/utils/region.d.ts.map +1 -0
  29. package/dist/src/utils/region.js +62 -0
  30. package/dist/src/utils/region.js.map +1 -0
  31. package/package.json +2 -2
  32. package/dist/assets/eval-harness/README.md +0 -144
  33. package/dist/assets/github-workflows/README.md +0 -110
  34. package/dist/assets/github-workflows/prism-dora-weekly.yml +0 -334
  35. package/dist/assets/gitlab-workflows/prism-dora-weekly.yml +0 -181
  36. package/dist/assets/infra/lib/lambda/defect-correlator.ts +0 -142
  37. package/dist/assets/infra/lib/lambda/spec-to-code-calculator.ts +0 -123
  38. package/dist/assets/metric-hooks/README.md +0 -107
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Region templating for bootstrapper assets.
3
+ *
4
+ * The shipped workflow assets are authored against DEFAULT_REGION. Every
5
+ * installer that copies one into a user's repo must run the content through
6
+ * applyRegion() so the whole file targets a single region.
7
+ *
8
+ * Why a bare-token replace rather than per-syntax rules:
9
+ *
10
+ * The workflows spell the region three different ways -- `aws-region:` (the
11
+ * configure-aws-credentials input), `--region` (an AWS CLI flag), and
12
+ * `AWS_REGION:` / `AWS_DEFAULT_REGION:` (step-level env vars). The installers
13
+ * previously matched a subset each: install-github-workflows handled
14
+ * `aws-region:` and `--region` but not the env form, install-gitlab-workflows
15
+ * handled only the env form, and install-eval-harness did a raw copyFileSync
16
+ * with no substitution at all.
17
+ *
18
+ * A partial rewrite is worse than none. It vends OIDC credentials for the
19
+ * requested region while leaving the AWS Continuum scan and event-emission
20
+ * calls pointed at DEFAULT_REGION, so those calls run against a region where
21
+ * the agent space, scan bucket, and SSM parameters were never created. The
22
+ * eval gate then fails closed and the merge blocks -- a symptom several steps
23
+ * removed from a region string the installer claimed to template.
24
+ *
25
+ * Matching the bare token means a fourth spelling cannot silently escape.
26
+ * This is safe here because no asset embeds the region inside an ARN, Bedrock
27
+ * model id, or endpoint hostname -- verified by assertNoDefaultRegion() in the
28
+ * test suite. If that ever changes, this function needs an exclusion list
29
+ * rather than a looser regex.
30
+ *
31
+ * Keeping every call in one region also satisfies the regional isolation
32
+ * requirement that credentials and the resources they touch stay within a
33
+ * single region boundary (SAX-03 Outcome 3), whose named pitfall is exactly
34
+ * a credential in one region reaching a resource in another.
35
+ */
36
+ /** The region the bootstrapper assets are authored against. */
37
+ export const DEFAULT_REGION = 'us-west-2';
38
+ /**
39
+ * Rewrites every occurrence of the authored region to `region`.
40
+ *
41
+ * Word-boundary anchored so a longer region name that merely contains the
42
+ * token cannot be partially rewritten.
43
+ */
44
+ export function applyRegion(content, region) {
45
+ if (!region || region === DEFAULT_REGION)
46
+ return content;
47
+ return content.replace(new RegExp(`\\b${DEFAULT_REGION}\\b`, 'g'), region);
48
+ }
49
+ /**
50
+ * Returns the `file:line` of every surviving reference to the authored region.
51
+ * Used by installers to warn, and by tests to assert a clean rewrite.
52
+ */
53
+ export function findDefaultRegionRefs(content, label = '') {
54
+ const hits = [];
55
+ content.split('\n').forEach((line, i) => {
56
+ if (new RegExp(`\\b${DEFAULT_REGION}\\b`).test(line)) {
57
+ hits.push(`${label}${label ? ':' : ''}${i + 1}: ${line.trim()}`);
58
+ }
59
+ });
60
+ return hits;
61
+ }
62
+ //# sourceMappingURL=region.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"region.js","sourceRoot":"","sources":["../../../src/utils/region.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,+DAA+D;AAC/D,MAAM,CAAC,MAAM,cAAc,GAAG,WAAW,CAAC;AAE1C;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe,EAAE,MAAc;IACzD,IAAI,CAAC,MAAM,IAAI,MAAM,KAAK,cAAc;QAAE,OAAO,OAAO,CAAC;IACzD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,MAAM,cAAc,KAAK,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;AAC7E,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,qBAAqB,CAAC,OAAe,EAAE,KAAK,GAAG,EAAE;IAC/D,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtC,IAAI,IAAI,MAAM,CAAC,MAAM,cAAc,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACrD,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACnE,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC;AACd,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@prism-d1/cli",
3
- "version": "1.0.79",
3
+ "version": "1.0.80",
4
4
  "description": "PRISM D1 Velocity CLI — AI development lifecycle assessment tool",
5
5
  "author": "andklee",
6
6
  "engines": {
@@ -22,7 +22,7 @@
22
22
  "access": "public"
23
23
  },
24
24
  "scripts": {
25
- "build": "npx tsc && mkdir -p dist/assets/infra && cp -r ../bootstrapper/metric-hooks ../bootstrapper/eval-harness ../bootstrapper/github-workflows ../bootstrapper/gitlab-workflows dist/assets/ && cp -r ../infra/bin ../infra/lib ../infra/package.json ../infra/tsconfig.json ../infra/cdk.json dist/assets/infra/",
25
+ "build": "npx tsc && rm -rf dist/assets && mkdir -p dist/assets/infra && cp -r ../bootstrapper/metric-hooks ../bootstrapper/eval-harness ../bootstrapper/github-workflows ../bootstrapper/gitlab-workflows dist/assets/ && cp -r ../infra/bin ../infra/lib ../infra/package.json ../infra/tsconfig.json ../infra/cdk.json dist/assets/infra/",
26
26
  "prepublishOnly": "npm run build",
27
27
  "start": "tsx bin/prism-cli.ts",
28
28
  "dev": "tsx bin/prism-cli.ts"
@@ -1,144 +0,0 @@
1
- # Eval Harness
2
-
3
- Evaluate code quality in CI using either **kiro-cli headless mode** (recommended) or Amazon Bedrock rubrics.
4
-
5
- ## Modes
6
-
7
- | Mode | Install Command | How It Works |
8
- |---|---|---|
9
- | **Kiro** (recommended) | `prism-cli bootstrapper install-eval-harness --mode kiro` | kiro-cli reads files, follows `.kiro/steering/code-review.md`, outputs structured JSON findings |
10
- | **Bedrock** (legacy) | `prism-cli bootstrapper install-eval-harness --with-rubrics` | Per-file rubric scoring via Bedrock API, weighted average client-side |
11
-
12
- ## Kiro Mode
13
-
14
- ```bash
15
- prism-cli bootstrapper install-eval-harness --mode kiro
16
- ```
17
-
18
- This installs:
19
- - `.kiro/steering/code-review.md` — review rules (plain English)
20
- - `.github/workflows/prism-eval-gate.yml` — kiro-cli headless CI workflow
21
-
22
- **Requirements:**
23
- - `KIRO_API_KEY` repository secret (generate at https://app.kiro.dev → Settings → API Keys)
24
- - (Optional) `PRISM_METRICS_ROLE_ARN` for EventBridge metrics + AWS Continuum security scanning
25
-
26
- **How it works:**
27
- 1. PR opened → workflow triggers
28
- 2. kiro-cli reads changed files + the diff via `--trust-all-tools`
29
- 3. Outputs structured JSON: findings with file/line, severity, score
30
- 4. Gate fails if any high-severity finding or score < 0.82
31
- 5. PR comment posted with findings table
32
-
33
- ## Bedrock Mode (Legacy)
34
-
35
- ## Bedrock Mode (Legacy) — Install
36
-
37
- ```bash
38
- # Workshop mode — empty rubrics, create your own
39
- bash prism-cli bootstrapper install-eval-harness
40
-
41
- # Production mode — includes all 5 rubrics
42
- bash prism-cli bootstrapper install-eval-harness --with-rubrics
43
-
44
- # Non-interactive
45
- bash prism-cli bootstrapper install-eval-harness --model us.anthropic.claude-haiku-4-5-20251001-v1:0 --threshold 0.82 --with-rubrics
46
- ```
47
-
48
- This installs into your repo:
49
- - `.prism/.prism/eval-harness/run-eval.sh` — evaluation script
50
- - `.prism/.prism/eval-harness/eval-config.json` — model, threshold, region
51
- - `.prism/.prism/eval-harness/rubrics/` — rubric JSON files
52
- - `.github/workflows/prism-eval-gate.yml` — CI workflow
53
-
54
- ## Usage
55
-
56
- ```bash
57
- # Evaluate a single file
58
- ./.prism/.prism/eval-harness/run-eval.sh .prism/.prism/eval-harness/rubrics/code-quality.json src/handler.ts
59
-
60
- # With a spec file (for spec-compliance rubric)
61
- ./.prism/.prism/eval-harness/run-eval.sh .prism/.prism/eval-harness/rubrics/spec-compliance.json src/api.ts --spec specs/api.md
62
- ```
63
-
64
- ### Output
65
-
66
- ```
67
- correctness: 0.9 — Handles all inputs correctly including edge cases
68
- readability: 0.85 — Clear naming, minor style inconsistency in helper
69
- ...
70
-
71
- Score: 0.8720
72
- Result: PASS
73
- Hallucinations: 0
74
- ```
75
-
76
- Exit codes: `0` = pass, `1` = fail, `2` = error.
77
-
78
- ## Configuration
79
-
80
- `eval-config.json`:
81
-
82
- | Field | Description | Default |
83
- |---|---|---|
84
- | `pass_threshold` | Minimum score to pass (0-1) | `0.82` |
85
- | `eval_model_id` | Bedrock model for evaluation | `us.anthropic.claude-haiku-4-5-20251001-v1:0` |
86
- | `aws_region` | AWS region | `us-west-2` |
87
- | `event_bus` | EventBridge bus name | `prism-d1-metrics` |
88
- | `emit_to_eventbridge` | Emit events (workflow handles this) | `true` |
89
-
90
- ## Rubrics
91
-
92
- Five production rubrics are available:
93
-
94
- | Rubric | Auto-selected when file path matches |
95
- |---|---|
96
- | `code-quality.json` | Default fallback |
97
- | `api-response-quality.json` | `api`, `handler`, `route`, `controller` |
98
- | `agent-quality.json` | `agent`, `assistant`, `orchestrat`, `workflow`, `chain` |
99
- | `security-compliance.json` | `auth`, `security`, `guard`, `policy`, `iam`, `crypto` |
100
- | `spec-compliance.json` | Used when commit has `Spec-Ref:` trailer |
101
-
102
- ### Creating a Custom Rubric
103
-
104
- ```json
105
- {
106
- "rubric_name": "my-rubric",
107
- "criteria": [
108
- {
109
- "name": "criterion_name",
110
- "weight": 0.30,
111
- "description": "What this measures",
112
- "scoring": "How to score 0.0-1.0"
113
- }
114
- ]
115
- }
116
- ```
117
-
118
- Weights must sum to 1.0. The script calculates the weighted average client-side (does not trust the LLM to do math).
119
-
120
- ## CI Workflow
121
-
122
- The `prism-eval-gate.yml` workflow:
123
-
124
- 1. Detects commits with `AI-Origin:` trailers
125
- 2. Identifies changed source files from those commits
126
- 3. Auto-selects a rubric per file based on path
127
- 4. Runs `run-eval.sh` per file
128
- 5. Posts a PR comment with per-file scores
129
- 6. Waits for AWS Security Agent review (if installed)
130
- 7. Emits `prism.d1.eval` event to EventBridge
131
- 8. Fails the check if any file scores below threshold or Security Agent finds issues
132
-
133
- ### Requirements
134
-
135
- - OIDC provider configured for GitHub Actions
136
- - IAM role with `bedrock:InvokeModel` + `events:PutEvents`
137
- - Repository secret `PRISM_METRICS_ROLE_ARN`
138
- - `.prism/config.json` with `team_id`
139
-
140
- ## Uninstall
141
-
142
- ```bash
143
- bash prism-cli bootstrapper install-eval-harness --uninstall
144
- ```
@@ -1,110 +0,0 @@
1
- # GitHub Actions Workflows
2
-
3
- Reusable GitHub Actions workflows for PRISM D1 Velocity metric collection.
4
-
5
- ## Workflows
6
-
7
- | Workflow | Trigger | Purpose |
8
- |---|---|---|
9
- | `prism-ai-metrics.yml` | PR merge to main | Calculates AI-to-merge ratio, token usage, lead time. Emits `prism.d1.pr` + `prism.d1.deploy` events |
10
- | `prism-eval-gate.yml` | PR open/update | Evaluates AI-generated code per-file with auto-selected rubrics, waits for Security Agent, blocks merge on failure |
11
- | `prism-agent-eval.yml` | PR modifying agent code | Runs agent in mock mode, evaluates output with agent-quality rubric |
12
- | `prism-dora-weekly.yml` | Weekly (Monday 09:00 UTC) | Calculates DORA + AI-DORA metrics, emits to EventBridge + CloudWatch |
13
-
14
- ## Setup
15
-
16
- ### 1. Configure AWS OIDC
17
-
18
- ```bash
19
- bash prism-cli bootstrapper setup-github-oidc
20
- ```
21
-
22
- This interactively creates:
23
- - OIDC identity provider for `token.actions.githubusercontent.com`
24
- - IAM role `GitHubActions-<repo>` with trust policy scoped to your repo
25
- - Inline policy with `events:PutEvents` and `bedrock:InvokeModel`
26
-
27
- For the weekly workflow, manually add `cloudwatch:PutMetricData` to the role policy.
28
-
29
- ### 2. Set Repository Secret
30
-
31
- In GitHub: Settings → Secrets and variables → Actions → New repository secret:
32
-
33
- | Secret | Value |
34
- |---|---|
35
- | `PRISM_METRICS_ROLE_ARN` | ARN printed by `setup-github-oidc` |
36
-
37
- ### 3. Install Git Hooks + Config
38
-
39
- ```bash
40
- bash prism-cli bootstrapper install-git-hooks
41
- ```
42
-
43
- Creates `.prism/config.json` with your team ID (read by all workflows).
44
-
45
- ### 4. Install Eval Harness
46
-
47
- ```bash
48
- # Workshop mode — bring your own rubric
49
- bash prism-cli bootstrapper install-eval-harness
50
-
51
- # Production mode — includes all 5 rubrics
52
- bash prism-cli bootstrapper install-eval-harness --with-rubrics
53
- ```
54
-
55
- This copies `.prism/eval-harness/`, `eval-config.json`, rubrics, and the `prism-eval-gate.yml` workflow.
56
-
57
- ### 5. Copy Remaining Workflows
58
-
59
- ```bash
60
- mkdir -p .github/workflows
61
- cp bootstrapper/github-workflows/prism-ai-metrics.yml .github/workflows/
62
- cp bootstrapper/github-workflows/prism-dora-weekly.yml .github/workflows/
63
- # Optional — only if you have agents with --mock support:
64
- cp bootstrapper/github-workflows/prism-agent-eval.yml .github/workflows/
65
- ```
66
-
67
- ## IAM Permissions
68
-
69
- The OIDC role needs:
70
-
71
- | Permission | Used by |
72
- |---|---|
73
- | `events:PutEvents` | All workflows |
74
- | `bedrock:InvokeModel` | eval-gate, agent-eval |
75
- | `cloudwatch:PutMetricData` | dora-weekly |
76
-
77
- ## Customization
78
-
79
- | Setting | How |
80
- |---|---|
81
- | Branch | Edit `branches` in each workflow |
82
- | AWS region | Edit `aws-region` field + EventBridge commands |
83
- | Eval threshold | Edit `.prism/.prism/eval-harness/eval-config.json` → `pass_threshold` |
84
- | Eval model | Edit `.prism/.prism/eval-harness/eval-config.json` → `eval_model_id` |
85
- | Weekly schedule | Edit cron in `prism-dora-weekly.yml` (default: `0 9 * * 1`) |
86
-
87
- ## Events Emitted
88
-
89
- | Detail Type | Source Workflow | Destination |
90
- |---|---|---|
91
- | `prism.d1.pr` | ai-metrics | EventBridge |
92
- | `prism.d1.deploy` | ai-metrics | EventBridge |
93
- | `prism.d1.eval` | eval-gate | EventBridge |
94
- | `prism.d1.agent.eval` | agent-eval | EventBridge |
95
- | `prism.d1.assessment` | dora-weekly | EventBridge |
96
- | `prism.d1.security.code_review` | eval-gate (Security Agent) | EventBridge |
97
- | `AIAdoptionRate`, `SpecCoverage`, tool counts | dora-weekly | CloudWatch |
98
-
99
- All EventBridge events use source `prism.d1.velocity` and bus `prism-d1-metrics`.
100
-
101
- ## Troubleshooting
102
-
103
- | Issue | Solution |
104
- |---|---|
105
- | OIDC auth fails | Verify trust policy `sub` matches `repo:org/repo:*` |
106
- | EventBridge put fails | Check `events:PutEvents` on bus ARN |
107
- | Eval gate always skips | Ensure commits have `AI-Origin:` trailers (install git hooks) |
108
- | Weekly not running | Workflow must exist on default branch; test with `workflow_dispatch` |
109
- | Agent eval skips | No `agent/main.py` found — add `--mock` support to your agent |
110
- | Security Agent timeout | Agent takes 2+ min to start; workflow waits up to 12 min total |
@@ -1,334 +0,0 @@
1
- # prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
2
- #
3
- # Copy this file to .github/workflows/prism-dora-weekly.yml in your repo.
4
- #
5
- # Methodology:
6
- # - Deployment Frequency: merged PRs to main (proxy for deploys)
7
- # - Lead Time: avg hours from PR open to merge
8
- # - Change Failure Rate: PRs with "revert|hotfix|rollback" in title / total merged
9
- # - MTTR: avg lead time of revert/hotfix PRs
10
- # - AI metrics: derived from AI-Origin commit trailers
11
- #
12
- # Required setup:
13
- # - OIDC provider configured for GitHub Actions in your AWS account
14
- # - IAM role with events:PutEvents and cloudwatch:PutMetricData permissions
15
- # - Repository secret PRISM_METRICS_ROLE_ARN set to the OIDC role ARN
16
- # - .prism/config.json committed to the repo (created by: bash prism-cli bootstrapper install-git-hooks)
17
-
18
- name: PRISM DORA Weekly Assessment
19
-
20
- on:
21
- schedule:
22
- - cron: '0 9 * * 1' # Every Monday at 09:00 UTC
23
- workflow_dispatch: # Allow manual trigger
24
-
25
- permissions:
26
- id-token: write
27
- contents: read
28
- pull-requests: read
29
-
30
- jobs:
31
- dora-assessment:
32
- runs-on: ubuntu-latest
33
- steps:
34
- - name: Checkout
35
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
36
- with:
37
- fetch-depth: 0
38
-
39
- - name: Configure AWS credentials (OIDC)
40
- uses: aws-actions/configure-aws-credentials@b47578312673ae6fa5b5096b330d9fbac3d116df # v4.2.1
41
- with:
42
- role-to-assume: ${{ secrets.PRISM_METRICS_ROLE_ARN }}
43
- aws-region: us-west-2
44
-
45
- - name: Read team ID from .prism/config.json
46
- id: prism-config
47
- run: |
48
- if [[ -f .prism/config.json ]]; then
49
- TEAM_ID=$(jq -r '.team_id' .prism/config.json)
50
- else
51
- echo "::error::.prism/config.json not found. Run bootstrapper/metric-hooks/install.sh first."
52
- exit 1
53
- fi
54
- echo "team_id=${TEAM_ID}" >> "$GITHUB_OUTPUT"
55
-
56
- - name: Calculate DORA metrics (past 7 days)
57
- id: dora
58
- env:
59
- GH_TOKEN: ${{ github.token }}
60
- run: |
61
- set -o pipefail
62
- SINCE=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
63
- NOW=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
64
- REPO="${{ github.repository }}"
65
-
66
- echo "Assessment period: ${SINCE} to ${NOW}"
67
-
68
- # ---------------------------------------------------------------
69
- # 1. Deployment Frequency — merged PRs to main (proxy for deploys)
70
- # ---------------------------------------------------------------
71
- MERGED_PRS=$(gh pr list --repo "${REPO}" --state merged --base main \
72
- --search "merged:>=${SINCE}" --json number,title,mergedAt --limit 200 2>/dev/null || echo "[]")
73
- DEPLOY_COUNT=$(echo "${MERGED_PRS}" | jq 'length')
74
- DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
75
-
76
- # ---------------------------------------------------------------
77
- # 2. Lead Time for Changes — avg time from PR open to merge
78
- # ---------------------------------------------------------------
79
- LEAD_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
80
- --search "merged:>=${SINCE}" --json createdAt,mergedAt --limit 200 2>/dev/null || echo "[]")
81
-
82
- AVG_LEAD_TIME_HOURS=$(echo "${LEAD_TIMES}" | jq '
83
- if length == 0 then 0
84
- else
85
- [.[] |
86
- (((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
87
- (.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
88
- ] | add / length | . * 100 | round / 100
89
- end
90
- ')
91
-
92
- # ---------------------------------------------------------------
93
- # 3. Change Failure Rate — PRs with "revert" or "hotfix" in title
94
- # ---------------------------------------------------------------
95
- FAILURE_PRS=$(echo "${MERGED_PRS}" | jq '[.[] | select(.title? // "" | test("revert|hotfix|rollback"; "i"))] | length' 2>/dev/null || echo "0")
96
- if [[ "${DEPLOY_COUNT}" -gt 0 ]]; then
97
- CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_PRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./')
98
- else
99
- CHANGE_FAILURE_RATE="0"
100
- fi
101
-
102
- # ---------------------------------------------------------------
103
- # 4. Mean Time to Recovery — avg time for hotfix/revert PRs
104
- # ---------------------------------------------------------------
105
- RECOVERY_TIMES=$(gh pr list --repo "${REPO}" --state merged --base main \
106
- --search "merged:>=${SINCE}" --json title,createdAt,mergedAt --limit 200 2>/dev/null \
107
- | jq '[.[] | select(.title | test("revert|hotfix|rollback"; "i"))]')
108
-
109
- MTTR_HOURS=$(echo "${RECOVERY_TIMES}" | jq '
110
- if length == 0 then 0
111
- else
112
- [.[] |
113
- (((.mergedAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
114
- (.createdAt | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600)
115
- ] | add / length | . * 100 | round / 100
116
- end
117
- ')
118
-
119
- echo "deploy_count=${DEPLOY_COUNT}" >> "$GITHUB_OUTPUT"
120
- echo "deploy_frequency=${DEPLOY_FREQUENCY}" >> "$GITHUB_OUTPUT"
121
- echo "avg_lead_time_hours=${AVG_LEAD_TIME_HOURS}" >> "$GITHUB_OUTPUT"
122
- echo "change_failure_rate=${CHANGE_FAILURE_RATE}" >> "$GITHUB_OUTPUT"
123
- echo "mttr_hours=${MTTR_HOURS}" >> "$GITHUB_OUTPUT"
124
-
125
- - name: Calculate AI-DORA metrics (past 7 days)
126
- id: ai-dora
127
- run: |
128
- set -o pipefail
129
- SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
130
-
131
- # All commits in the past 7 days on main
132
- TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline main 2>/dev/null | wc -l | tr -d ' ')
133
-
134
- # AI-origin breakdown — count per-commit to avoid overcounting
135
- AI_GENERATED=0
136
- AI_ASSISTED=0
137
- HUMAN_ONLY=0
138
- SPEC_REFS=0
139
- CLAUDE_CODE_COUNT=0
140
- KIRO_COUNT=0
141
- Q_DEV_COUNT=0
142
-
143
- for SHA in $(git log --since="${SINCE_DATE}" --format='%H' main 2>/dev/null); do
144
- MSG=$(git log -1 --format='%B' "${SHA}")
145
- if echo "${MSG}" | grep -q '^AI-Origin: ai-generated'; then
146
- AI_GENERATED=$((AI_GENERATED + 1))
147
- elif echo "${MSG}" | grep -q '^AI-Origin: ai-assisted'; then
148
- AI_ASSISTED=$((AI_ASSISTED + 1))
149
- elif echo "${MSG}" | grep -q '^AI-Origin: human'; then
150
- HUMAN_ONLY=$((HUMAN_ONLY + 1))
151
- fi
152
- echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
153
- TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
154
- case "${TOOL}" in
155
- *claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
156
- *kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
157
- *q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
158
- esac
159
- done
160
-
161
- UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
162
- [[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
163
-
164
- AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
165
- if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
166
- AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
167
- else
168
- AI_ADOPTION_RATE="0"
169
- fi
170
-
171
- AI_TO_MERGE_RATIO="${AI_ADOPTION_RATE}"
172
-
173
- if [[ "${TOTAL_COMMITS}" -gt 0 ]]; then
174
- SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./')
175
- else
176
- SPEC_COVERAGE="0"
177
- fi
178
-
179
- echo "total_commits=${TOTAL_COMMITS}" >> "$GITHUB_OUTPUT"
180
- echo "ai_generated=${AI_GENERATED}" >> "$GITHUB_OUTPUT"
181
- echo "ai_assisted=${AI_ASSISTED}" >> "$GITHUB_OUTPUT"
182
- echo "human_only=${HUMAN_ONLY}" >> "$GITHUB_OUTPUT"
183
- echo "untagged=${UNTAGGED}" >> "$GITHUB_OUTPUT"
184
- echo "ai_adoption_rate=${AI_ADOPTION_RATE}" >> "$GITHUB_OUTPUT"
185
- echo "ai_to_merge_ratio=${AI_TO_MERGE_RATIO}" >> "$GITHUB_OUTPUT"
186
- echo "spec_coverage=${SPEC_COVERAGE}" >> "$GITHUB_OUTPUT"
187
- echo "claude_code_count=${CLAUDE_CODE_COUNT}" >> "$GITHUB_OUTPUT"
188
- echo "kiro_count=${KIRO_COUNT}" >> "$GITHUB_OUTPUT"
189
- echo "q_dev_count=${Q_DEV_COUNT}" >> "$GITHUB_OUTPUT"
190
-
191
- - name: Emit prism.d1.assessment event
192
- run: |
193
- TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
194
-
195
- ASSESSMENT_EVENT=$(jq -n \
196
- --arg team_id "${{ steps.prism-config.outputs.team_id }}" \
197
- --arg repo "${{ github.repository }}" \
198
- --arg timestamp "${TIMESTAMP}" \
199
- --argjson deploy_count "${{ steps.dora.outputs.deploy_count }}" \
200
- --argjson deploy_frequency "${{ steps.dora.outputs.deploy_frequency }}" \
201
- --argjson avg_lead_time_hours "${{ steps.dora.outputs.avg_lead_time_hours }}" \
202
- --argjson change_failure_rate "${{ steps.dora.outputs.change_failure_rate }}" \
203
- --argjson mttr_hours "${{ steps.dora.outputs.mttr_hours }}" \
204
- --argjson total_commits "${{ steps.ai-dora.outputs.total_commits }}" \
205
- --argjson ai_generated "${{ steps.ai-dora.outputs.ai_generated }}" \
206
- --argjson ai_assisted "${{ steps.ai-dora.outputs.ai_assisted }}" \
207
- --argjson human_only "${{ steps.ai-dora.outputs.human_only }}" \
208
- --argjson untagged "${{ steps.ai-dora.outputs.untagged }}" \
209
- --argjson ai_adoption_rate "${{ steps.ai-dora.outputs.ai_adoption_rate }}" \
210
- --argjson ai_to_merge_ratio "${{ steps.ai-dora.outputs.ai_to_merge_ratio }}" \
211
- --argjson spec_coverage "${{ steps.ai-dora.outputs.spec_coverage }}" \
212
- --argjson claude_code_count "${{ steps.ai-dora.outputs.claude_code_count }}" \
213
- --argjson kiro_count "${{ steps.ai-dora.outputs.kiro_count }}" \
214
- --argjson q_dev_count "${{ steps.ai-dora.outputs.q_dev_count }}" \
215
- '{
216
- "team_id": $team_id,
217
- "repo": $repo,
218
- "timestamp": $timestamp,
219
- "prism_level": 2,
220
- "metric": {
221
- "name": "weekly_assessment",
222
- "value": 1,
223
- "unit": "assessment"
224
- },
225
- "dora": {
226
- "deployment_frequency": $deploy_frequency,
227
- "deployments": $deploy_count,
228
- "lead_time_hours": $avg_lead_time_hours,
229
- "change_failure_rate": $change_failure_rate,
230
- "mttr_hours": $mttr_hours
231
- },
232
- "ai_dora": {
233
- "ai_adoption_rate": $ai_adoption_rate,
234
- "ai_to_merge_ratio": $ai_to_merge_ratio,
235
- "total_commits": $total_commits,
236
- "ai_generated_commits": $ai_generated,
237
- "ai_assisted_commits": $ai_assisted,
238
- "human_commits": $human_only,
239
- "untagged_commits": $untagged,
240
- "spec_coverage": $spec_coverage,
241
- "tool_breakdown": {
242
- "claude_code": $claude_code_count,
243
- "kiro": $kiro_count,
244
- "q_developer": $q_dev_count
245
- }
246
- }
247
- }')
248
-
249
- aws events put-events \
250
- --region us-west-2 \
251
- --entries "[{
252
- \"Source\": \"prism.d1.velocity\",
253
- \"DetailType\": \"prism.d1.assessment\",
254
- \"EventBusName\": \"prism-d1-metrics\",
255
- \"Detail\": $(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')
256
- }]"
257
-
258
- echo "Weekly DORA assessment emitted successfully."
259
-
260
- - name: Emit tool breakdown and adoption metrics to CloudWatch
261
- run: |
262
- TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
263
- TEAM_ID="${{ steps.prism-config.outputs.team_id }}"
264
- REPO="${{ github.repository }}"
265
- NAMESPACE="PRISM/D1/Velocity"
266
-
267
- # AI Adoption Rate
268
- aws cloudwatch put-metric-data \
269
- --region us-west-2 \
270
- --namespace "${NAMESPACE}" \
271
- --metric-data "[{
272
- \"MetricName\": \"AIAdoptionRate\",
273
- \"Value\": ${{ steps.ai-dora.outputs.ai_adoption_rate }},
274
- \"Unit\": \"Percent\",
275
- \"Timestamp\": \"${TIMESTAMP}\",
276
- \"Dimensions\": [
277
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
278
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
279
- ]
280
- }]"
281
-
282
- # Spec Coverage
283
- aws cloudwatch put-metric-data \
284
- --region us-west-2 \
285
- --namespace "${NAMESPACE}" \
286
- --metric-data "[{
287
- \"MetricName\": \"SpecCoverage\",
288
- \"Value\": ${{ steps.ai-dora.outputs.spec_coverage }},
289
- \"Unit\": \"Percent\",
290
- \"Timestamp\": \"${TIMESTAMP}\",
291
- \"Dimensions\": [
292
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
293
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
294
- ]
295
- }]"
296
-
297
- # Tool breakdown — one metric per tool
298
- for TOOL_DATA in \
299
- "ClaudeCodeCommits:${{ steps.ai-dora.outputs.claude_code_count }}" \
300
- "KiroCommits:${{ steps.ai-dora.outputs.kiro_count }}" \
301
- "QDeveloperCommits:${{ steps.ai-dora.outputs.q_dev_count }}" \
302
- "UntaggedCommits:${{ steps.ai-dora.outputs.untagged }}"; do
303
-
304
- METRIC_NAME="${TOOL_DATA%%:*}"
305
- METRIC_VALUE="${TOOL_DATA##*:}"
306
-
307
- aws cloudwatch put-metric-data \
308
- --region us-west-2 \
309
- --namespace "${NAMESPACE}" \
310
- --metric-data "[{
311
- \"MetricName\": \"${METRIC_NAME}\",
312
- \"Value\": ${METRIC_VALUE},
313
- \"Unit\": \"Count\",
314
- \"Timestamp\": \"${TIMESTAMP}\",
315
- \"Dimensions\": [
316
- {\"Name\": \"TeamId\", \"Value\": \"${TEAM_ID}\"},
317
- {\"Name\": \"Repository\", \"Value\": \"${REPO}\"}
318
- ]
319
- }]"
320
- done
321
-
322
- echo "Tool breakdown and adoption metrics emitted to CloudWatch."
323
- echo ""
324
- echo "=== DORA Metrics ==="
325
- echo "Deployment Frequency: ${{ steps.dora.outputs.deploy_frequency }}/day"
326
- echo "Lead Time: ${{ steps.dora.outputs.avg_lead_time_hours }} hours"
327
- echo "Change Failure Rate: ${{ steps.dora.outputs.change_failure_rate }}"
328
- echo "MTTR: ${{ steps.dora.outputs.mttr_hours }} hours"
329
- echo ""
330
- echo "=== AI-DORA Metrics ==="
331
- echo "AI Adoption Rate: ${{ steps.ai-dora.outputs.ai_adoption_rate }}"
332
- echo "AI-to-Merge Ratio: ${{ steps.ai-dora.outputs.ai_to_merge_ratio }}"
333
- echo "Spec Coverage: ${{ steps.ai-dora.outputs.spec_coverage }}"
334
- echo "Total Commits: ${{ steps.ai-dora.outputs.total_commits }}"