@prism-d1/cli 1.0.43 → 1.0.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/gitlab-workflows/.gitlab-ci.yml +17 -0
- package/dist/assets/gitlab-workflows/prism-agent-eval.yml +132 -0
- package/dist/assets/gitlab-workflows/prism-ai-metrics.yml +139 -0
- package/dist/assets/gitlab-workflows/prism-dora-weekly.yml +166 -0
- package/dist/assets/gitlab-workflows/prism-eval-gate.yml +139 -0
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts +14 -0
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -0
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +53 -0
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -0
- package/package.json +2 -2
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# PRISM D1 Velocity — GitLab CI/CD Pipeline
|
|
2
|
+
#
|
|
3
|
+
# Copy this file to .gitlab-ci.yml in your repo root.
|
|
4
|
+
# Individual workflow files should be placed in .prism/gitlab-workflows/
|
|
5
|
+
#
|
|
6
|
+
# Required setup:
|
|
7
|
+
# 1. Run: prism-cli bootstrapper setup-gitlab-oidc
|
|
8
|
+
# 2. Add CI/CD variable PRISM_METRICS_ROLE_ARN with the output role ARN
|
|
9
|
+
# 3. Copy gitlab-workflows/*.yml to .prism/gitlab-workflows/ in your repo
|
|
10
|
+
# 4. (Optional) Set GITLAB_OIDC_AUDIENCE if not using gitlab.aws.dev
|
|
11
|
+
# 5. (Optional) Create a pipeline schedule for prism-dora-weekly (Monday 09:00 UTC)
|
|
12
|
+
|
|
13
|
+
include:
|
|
14
|
+
- local: '.prism/gitlab-workflows/prism-ai-metrics.yml'
|
|
15
|
+
- local: '.prism/gitlab-workflows/prism-dora-weekly.yml'
|
|
16
|
+
- local: '.prism/gitlab-workflows/prism-agent-eval.yml'
|
|
17
|
+
- local: '.prism/gitlab-workflows/prism-eval-gate.yml'
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# prism-agent-eval.yml — Evaluate agent outputs on MRs that modify agent code.
|
|
2
|
+
#
|
|
3
|
+
# Include this file in your .gitlab-ci.yml:
|
|
4
|
+
# include:
|
|
5
|
+
# - local: '.prism/gitlab-workflows/prism-agent-eval.yml'
|
|
6
|
+
#
|
|
7
|
+
# Required setup:
|
|
8
|
+
# - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role with bedrock:InvokeModel + events:PutEvents)
|
|
9
|
+
# - CI/CD variable AWS_REGION (default: us-west-2)
|
|
10
|
+
# - .prism/config.json committed to the repo
|
|
11
|
+
# - .prism/eval-harness/ installed (run: prism-cli bootstrapper install-eval-harness)
|
|
12
|
+
# - (Optional) Agent entry point supports --mock flag for dry-run execution
|
|
13
|
+
|
|
14
|
+
prism-agent-eval:
|
|
15
|
+
stage: test
|
|
16
|
+
image: python:3.12-slim
|
|
17
|
+
id_tokens:
|
|
18
|
+
GITLAB_OIDC_TOKEN:
|
|
19
|
+
aud: https://gitlab.aws.dev
|
|
20
|
+
rules:
|
|
21
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
22
|
+
changes:
|
|
23
|
+
- agent/**
|
|
24
|
+
- agents/**
|
|
25
|
+
- mcp-servers/**
|
|
26
|
+
- "**/*agent*.py"
|
|
27
|
+
- "**/*agent*.ts"
|
|
28
|
+
variables:
|
|
29
|
+
AWS_DEFAULT_REGION: ${AWS_REGION:-us-west-2}
|
|
30
|
+
before_script:
|
|
31
|
+
- apt-get update -qq && apt-get install -y -qq jq bc git curl unzip
|
|
32
|
+
- curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
|
|
33
|
+
- unzip -q /tmp/awscliv2.zip -d /tmp && /tmp/aws/install
|
|
34
|
+
- pip install --quiet strands-agents strands-agents-tools requests
|
|
35
|
+
- git fetch --unshallow || true
|
|
36
|
+
- |
|
|
37
|
+
# Assume AWS role via GitLab OIDC
|
|
38
|
+
if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
|
|
39
|
+
CREDS=$(aws sts assume-role-with-web-identity \
|
|
40
|
+
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
41
|
+
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
42
|
+
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
43
|
+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
44
|
+
--output text)
|
|
45
|
+
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
|
|
46
|
+
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
|
|
47
|
+
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
|
|
48
|
+
fi
|
|
49
|
+
script:
|
|
50
|
+
- |
|
|
51
|
+
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
52
|
+
BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA}"
|
|
53
|
+
HEAD_SHA="${CI_COMMIT_SHA}"
|
|
54
|
+
|
|
55
|
+
# Identify agent changes
|
|
56
|
+
AGENT_FILES=$(git diff --name-only "${BASE_SHA}..${HEAD_SHA}" \
|
|
57
|
+
| grep -E '(^agent/|^agents/|^mcp-servers/|agent.*\.(py|ts)$)' || true)
|
|
58
|
+
|
|
59
|
+
if [[ -z "${AGENT_FILES}" ]]; then
|
|
60
|
+
echo "No agent-related files changed. Skipping."
|
|
61
|
+
exit 0
|
|
62
|
+
fi
|
|
63
|
+
|
|
64
|
+
FILE_COUNT=$(echo "${AGENT_FILES}" | wc -l | tr -d ' ')
|
|
65
|
+
echo "Found ${FILE_COUNT} changed agent files."
|
|
66
|
+
|
|
67
|
+
# Run agent in mock mode
|
|
68
|
+
AGENT_ENTRY=""
|
|
69
|
+
for CANDIDATE in agent/main.py agents/main.py agent.py; do
|
|
70
|
+
[[ -f "${CANDIDATE}" ]] && AGENT_ENTRY="${CANDIDATE}" && break
|
|
71
|
+
done
|
|
72
|
+
|
|
73
|
+
MOCK_RAN="false"
|
|
74
|
+
if [[ -n "${AGENT_ENTRY}" ]]; then
|
|
75
|
+
echo "Running agent in mock mode: ${AGENT_ENTRY}"
|
|
76
|
+
MOCK_RAN="true"
|
|
77
|
+
set +e
|
|
78
|
+
python "${AGENT_ENTRY}" --mock > /tmp/agent-output.json 2>/tmp/agent-stderr.log
|
|
79
|
+
AGENT_EXIT=$?
|
|
80
|
+
set -e
|
|
81
|
+
else
|
|
82
|
+
echo "{}" > /tmp/agent-output.json
|
|
83
|
+
AGENT_EXIT=0
|
|
84
|
+
fi
|
|
85
|
+
|
|
86
|
+
# Evaluate
|
|
87
|
+
chmod +x .prism/eval-harness/run-eval.sh
|
|
88
|
+
RUBRIC=".prism/eval-harness/rubrics/agent-quality.json"
|
|
89
|
+
|
|
90
|
+
if [[ -s /tmp/agent-output.json ]]; then
|
|
91
|
+
EVAL_TARGET="/tmp/agent-output.json"
|
|
92
|
+
else
|
|
93
|
+
EVAL_TARGET=$(echo "${AGENT_FILES}" | head -1)
|
|
94
|
+
fi
|
|
95
|
+
|
|
96
|
+
set +e
|
|
97
|
+
OUTPUT=$(./.prism/eval-harness/run-eval.sh "${RUBRIC}" "${EVAL_TARGET}" 2>&1)
|
|
98
|
+
set -e
|
|
99
|
+
|
|
100
|
+
SCORE=$(echo "${OUTPUT}" | grep '^Score:' | awk '{print $2}' || echo "0")
|
|
101
|
+
RESULT=$(echo "${OUTPUT}" | grep '^Result:' | awk '{print $2}' || echo "UNKNOWN")
|
|
102
|
+
|
|
103
|
+
echo "Agent eval: score=${SCORE} result=${RESULT}"
|
|
104
|
+
|
|
105
|
+
# Post MR note
|
|
106
|
+
if [[ -n "${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" ]]; then
|
|
107
|
+
EMOJI=$([[ "${RESULT}" == "PASS" ]] && echo "✅" || echo "❌")
|
|
108
|
+
NOTE="## ${EMOJI} PRISM Agent Eval — ${RESULT}\n\n**Score**: ${SCORE} | **Agent Files**: ${FILE_COUNT} | **Mock**: ${MOCK_RAN} (exit ${AGENT_EXIT})"
|
|
109
|
+
curl -s --request POST --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
|
|
110
|
+
--form "body=${NOTE}" \
|
|
111
|
+
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" > /dev/null
|
|
112
|
+
fi
|
|
113
|
+
|
|
114
|
+
# Emit event
|
|
115
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
116
|
+
EVAL_EVENT=$(jq -n \
|
|
117
|
+
--arg team_id "${TEAM_ID}" --arg repo "${CI_PROJECT_PATH}" --arg timestamp "${TIMESTAMP}" \
|
|
118
|
+
--argjson score "${SCORE:-0}" --arg result "${RESULT}" --argjson file_count "${FILE_COUNT}" \
|
|
119
|
+
--arg mr_iid "${CI_MERGE_REQUEST_IID}" --arg mock_ran "${MOCK_RAN}" \
|
|
120
|
+
'{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:3,
|
|
121
|
+
metric:{name:"agent_eval_score",value:$score,unit:"score"},
|
|
122
|
+
ai_context:{tool:"bedrock-eval",model:"anthropic.claude-sonnet-4-20250514",origin:"ai-generated"},
|
|
123
|
+
ai_dora:{eval_gate_pass_rate:(if $result == "PASS" then 1.0 else 0.0 end)},
|
|
124
|
+
agent_eval:{file_count:$file_count, result:$result, mock_ran:($mock_ran == "true"),
|
|
125
|
+
mr_iid:($mr_iid | tonumber), rubric:"agent-quality"}}')
|
|
126
|
+
|
|
127
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
128
|
+
--entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.agent.eval\",
|
|
129
|
+
\"EventBusName\":\"prism-d1-metrics\",
|
|
130
|
+
\"Detail\":$(echo "${EVAL_EVENT}" | jq -c '.' | jq -Rs '.')}]"
|
|
131
|
+
|
|
132
|
+
[[ "${RESULT}" == "FAIL" ]] && exit 1 || exit 0
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# prism-ai-metrics.yml — Emit AI-assisted development metrics on MR merge.
|
|
2
|
+
#
|
|
3
|
+
# Include this file in your .gitlab-ci.yml:
|
|
4
|
+
# include:
|
|
5
|
+
# - local: '.prism/gitlab-workflows/prism-ai-metrics.yml'
|
|
6
|
+
#
|
|
7
|
+
# Required setup:
|
|
8
|
+
# - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role ARN with events:PutEvents)
|
|
9
|
+
# - CI/CD variable AWS_REGION (default: us-west-2)
|
|
10
|
+
# - .prism/config.json committed to the repo
|
|
11
|
+
# - AWS credentials configured via OIDC or CI/CD variables
|
|
12
|
+
|
|
13
|
+
prism-ai-metrics:
|
|
14
|
+
stage: .post
|
|
15
|
+
image: amazon/aws-cli:latest
|
|
16
|
+
id_tokens:
|
|
17
|
+
GITLAB_OIDC_TOKEN:
|
|
18
|
+
aud: https://gitlab.aws.dev
|
|
19
|
+
rules:
|
|
20
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_STATE == "merged"'
|
|
21
|
+
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"'
|
|
22
|
+
variables:
|
|
23
|
+
AWS_DEFAULT_REGION: ${AWS_REGION:-us-west-2}
|
|
24
|
+
before_script:
|
|
25
|
+
- yum install -y -q jq bc git
|
|
26
|
+
- git fetch --unshallow || true
|
|
27
|
+
- |
|
|
28
|
+
# Assume AWS role via GitLab OIDC
|
|
29
|
+
if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
|
|
30
|
+
CREDS=$(aws sts assume-role-with-web-identity \
|
|
31
|
+
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
32
|
+
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
33
|
+
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
34
|
+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
35
|
+
--output text)
|
|
36
|
+
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
|
|
37
|
+
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
|
|
38
|
+
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
|
|
39
|
+
fi
|
|
40
|
+
script:
|
|
41
|
+
- |
|
|
42
|
+
# Read team ID
|
|
43
|
+
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
44
|
+
|
|
45
|
+
# Determine commit range
|
|
46
|
+
if [[ -n "${CI_MERGE_REQUEST_DIFF_BASE_SHA}" ]]; then
|
|
47
|
+
BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA}"
|
|
48
|
+
HEAD_SHA="${CI_COMMIT_SHA}"
|
|
49
|
+
else
|
|
50
|
+
BASE_SHA=$(git rev-parse HEAD~1 2>/dev/null || git rev-parse HEAD)
|
|
51
|
+
HEAD_SHA="${CI_COMMIT_SHA}"
|
|
52
|
+
fi
|
|
53
|
+
|
|
54
|
+
TOTAL_COMMITS=$(git rev-list --count "${BASE_SHA}..${HEAD_SHA}" 2>/dev/null || echo "1")
|
|
55
|
+
|
|
56
|
+
# Count AI origin per-commit
|
|
57
|
+
AI_GENERATED=0; AI_ASSISTED=0; HUMAN_ONLY=0
|
|
58
|
+
TOTAL_INPUT_TOKENS=0; TOTAL_OUTPUT_TOKENS=0; TOTAL_COST="0"
|
|
59
|
+
|
|
60
|
+
for SHA in $(git rev-list "${BASE_SHA}..${HEAD_SHA}" 2>/dev/null || echo "${HEAD_SHA}"); do
|
|
61
|
+
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
62
|
+
echo "${MSG}" | grep -q '^AI-Origin: ai-generated' && AI_GENERATED=$((AI_GENERATED + 1))
|
|
63
|
+
echo "${MSG}" | grep -q '^AI-Origin: ai-assisted' && AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
64
|
+
echo "${MSG}" | grep -q '^AI-Origin: human' && HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
65
|
+
|
|
66
|
+
INPUT_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Input-Tokens: )\d+' || echo "0")
|
|
67
|
+
OUTPUT_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Output-Tokens: )\d+' || echo "0")
|
|
68
|
+
COST_T=$(echo "${MSG}" | grep -oP '(?<=^AI-Cost: )[0-9.]+' || echo "0")
|
|
69
|
+
[[ ${INPUT_T} -gt 1000000 ]] && INPUT_T=0
|
|
70
|
+
[[ ${OUTPUT_T} -gt 1000000 ]] && OUTPUT_T=0
|
|
71
|
+
TOTAL_INPUT_TOKENS=$((TOTAL_INPUT_TOKENS + INPUT_T))
|
|
72
|
+
TOTAL_OUTPUT_TOKENS=$((TOTAL_OUTPUT_TOKENS + OUTPUT_T))
|
|
73
|
+
TOTAL_COST=$(echo "${TOTAL_COST} + ${COST_T}" | bc -l)
|
|
74
|
+
done
|
|
75
|
+
|
|
76
|
+
AI_COMMITS=$((AI_GENERATED + AI_ASSISTED))
|
|
77
|
+
[[ "${TOTAL_COMMITS}" -gt 0 ]] && AI_RATIO=$(echo "scale=4; ${AI_COMMITS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || AI_RATIO="0"
|
|
78
|
+
|
|
79
|
+
# Calculate lead time (MR created to merged)
|
|
80
|
+
LEAD_TIME_SECONDS=0
|
|
81
|
+
if [[ -n "${CI_MERGE_REQUEST_CREATED_AT}" ]]; then
|
|
82
|
+
CREATED_TS=$(date -d "${CI_MERGE_REQUEST_CREATED_AT}" +%s 2>/dev/null || echo "0")
|
|
83
|
+
MERGED_TS=$(date +%s)
|
|
84
|
+
LEAD_TIME_SECONDS=$((MERGED_TS - CREATED_TS))
|
|
85
|
+
fi
|
|
86
|
+
|
|
87
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
88
|
+
|
|
89
|
+
PR_EVENT=$(jq -n \
|
|
90
|
+
--arg team_id "${TEAM_ID}" \
|
|
91
|
+
--arg repo "${CI_PROJECT_PATH}" \
|
|
92
|
+
--arg timestamp "${TIMESTAMP}" \
|
|
93
|
+
--argjson ai_ratio "${AI_RATIO}" \
|
|
94
|
+
--argjson total_commits "${TOTAL_COMMITS}" \
|
|
95
|
+
--argjson ai_commits "${AI_COMMITS}" \
|
|
96
|
+
--argjson ai_generated "${AI_GENERATED}" \
|
|
97
|
+
--argjson ai_assisted "${AI_ASSISTED}" \
|
|
98
|
+
--argjson lead_time_seconds "${LEAD_TIME_SECONDS}" \
|
|
99
|
+
--arg mr_iid "${CI_MERGE_REQUEST_IID:-0}" \
|
|
100
|
+
--arg mr_author "${GITLAB_USER_LOGIN}" \
|
|
101
|
+
--argjson total_input_tokens "${TOTAL_INPUT_TOKENS}" \
|
|
102
|
+
--argjson total_output_tokens "${TOTAL_OUTPUT_TOKENS}" \
|
|
103
|
+
--arg total_cost "${TOTAL_COST}" \
|
|
104
|
+
'{
|
|
105
|
+
team_id: $team_id, repo: $repo, timestamp: $timestamp, prism_level: 2,
|
|
106
|
+
metric: {name: "ai_to_merge_ratio", value: $ai_ratio, unit: "ratio"},
|
|
107
|
+
ai_context: {tool: "gitlab-ci", model: "n/a",
|
|
108
|
+
origin: (if $ai_ratio > 0 then "ai-assisted" else "human" end)},
|
|
109
|
+
dora: {deployment_frequency: 1, lead_time_seconds: $lead_time_seconds},
|
|
110
|
+
ai_dora: {ai_to_merge_ratio: $ai_ratio, total_commits: $total_commits,
|
|
111
|
+
ai_commits: $ai_commits, ai_generated_commits: $ai_generated,
|
|
112
|
+
ai_assisted_commits: $ai_assisted,
|
|
113
|
+
total_input_tokens: $total_input_tokens,
|
|
114
|
+
total_output_tokens: $total_output_tokens,
|
|
115
|
+
total_cost_usd: (if $total_cost != "0" and $total_cost != "" then ($total_cost | tonumber) else 0 end)},
|
|
116
|
+
mr: {iid: ($mr_iid | tonumber), author: $mr_author}
|
|
117
|
+
}')
|
|
118
|
+
|
|
119
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
120
|
+
--entries "[{
|
|
121
|
+
\"Source\": \"prism.d1.velocity\",
|
|
122
|
+
\"DetailType\": \"prism.d1.pr\",
|
|
123
|
+
\"EventBusName\": \"prism-d1-metrics\",
|
|
124
|
+
\"Detail\": $(echo "${PR_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
125
|
+
}]"
|
|
126
|
+
|
|
127
|
+
# Also emit deploy event
|
|
128
|
+
DEPLOY_EVENT=$(jq -n --arg team_id "${TEAM_ID}" --arg repo "${CI_PROJECT_PATH}" --arg timestamp "${TIMESTAMP}" \
|
|
129
|
+
'{team_id: $team_id, repo: $repo, timestamp: $timestamp, prism_level: 2,
|
|
130
|
+
metric: {name: "deployment", value: 1, unit: "count"},
|
|
131
|
+
dora: {deployment_frequency: 1}}')
|
|
132
|
+
|
|
133
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
134
|
+
--entries "[{
|
|
135
|
+
\"Source\": \"prism.d1.velocity\",
|
|
136
|
+
\"DetailType\": \"prism.d1.deploy\",
|
|
137
|
+
\"EventBusName\": \"prism-d1-metrics\",
|
|
138
|
+
\"Detail\": $(echo "${DEPLOY_EVENT}" | jq -c '.' | jq -Rs '.')
|
|
139
|
+
}]"
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# prism-dora-weekly.yml — Weekly DORA + AI-DORA metrics assessment.
|
|
2
|
+
#
|
|
3
|
+
# Include this file in your .gitlab-ci.yml:
|
|
4
|
+
# include:
|
|
5
|
+
# - local: '.prism/gitlab-workflows/prism-dora-weekly.yml'
|
|
6
|
+
#
|
|
7
|
+
# Required setup:
|
|
8
|
+
# - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role with events:PutEvents + cloudwatch:PutMetricData)
|
|
9
|
+
# - CI/CD variable AWS_REGION (default: us-west-2)
|
|
10
|
+
# - .prism/config.json committed to the repo
|
|
11
|
+
# - Pipeline schedule configured: weekly on Monday at 09:00 UTC
|
|
12
|
+
|
|
13
|
+
prism-dora-weekly:
|
|
14
|
+
stage: .post
|
|
15
|
+
image: amazon/aws-cli:latest
|
|
16
|
+
id_tokens:
|
|
17
|
+
GITLAB_OIDC_TOKEN:
|
|
18
|
+
aud: https://gitlab.aws.dev
|
|
19
|
+
rules:
|
|
20
|
+
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
21
|
+
- if: '$CI_PIPELINE_SOURCE == "web"'
|
|
22
|
+
when: manual
|
|
23
|
+
variables:
|
|
24
|
+
AWS_DEFAULT_REGION: ${AWS_REGION:-us-west-2}
|
|
25
|
+
before_script:
|
|
26
|
+
- yum install -y -q jq bc git curl
|
|
27
|
+
- git fetch --unshallow || true
|
|
28
|
+
- |
|
|
29
|
+
# Assume AWS role via GitLab OIDC
|
|
30
|
+
if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
|
|
31
|
+
CREDS=$(aws sts assume-role-with-web-identity \
|
|
32
|
+
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
33
|
+
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
34
|
+
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
35
|
+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
36
|
+
--output text)
|
|
37
|
+
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
|
|
38
|
+
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
|
|
39
|
+
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
|
|
40
|
+
fi
|
|
41
|
+
script:
|
|
42
|
+
- |
|
|
43
|
+
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
44
|
+
SINCE_DATE=$(date -u -d '7 days ago' +"%Y-%m-%d")
|
|
45
|
+
REPO="${CI_PROJECT_PATH}"
|
|
46
|
+
NAMESPACE="PRISM/D1/Velocity"
|
|
47
|
+
|
|
48
|
+
# ---------------------------------------------------------------
|
|
49
|
+
# 1. Deployment Frequency — merged MRs to default branch (past 7 days)
|
|
50
|
+
# ---------------------------------------------------------------
|
|
51
|
+
SINCE_ISO=$(date -u -d '7 days ago' +"%Y-%m-%dT%H:%M:%SZ")
|
|
52
|
+
MERGED_MRS=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
|
|
53
|
+
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests?state=merged&updated_after=${SINCE_ISO}&per_page=100" 2>/dev/null || echo "[]")
|
|
54
|
+
DEPLOY_COUNT=$(echo "${MERGED_MRS}" | jq 'length')
|
|
55
|
+
DEPLOY_FREQUENCY=$(echo "scale=2; ${DEPLOY_COUNT} / 7" | bc -l | sed 's/^\./0./')
|
|
56
|
+
|
|
57
|
+
# ---------------------------------------------------------------
|
|
58
|
+
# 2. Lead Time — avg time from MR open to merge
|
|
59
|
+
# ---------------------------------------------------------------
|
|
60
|
+
AVG_LEAD_TIME_HOURS=$(echo "${MERGED_MRS}" | jq '
|
|
61
|
+
if length == 0 then 0
|
|
62
|
+
else [.[] | select(.merged_at != null and .created_at != null) |
|
|
63
|
+
((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
64
|
+
(.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
|
|
65
|
+
] | if length > 0 then add / length | . * 100 | round / 100 else 0 end
|
|
66
|
+
end')
|
|
67
|
+
|
|
68
|
+
# ---------------------------------------------------------------
|
|
69
|
+
# 3. Change Failure Rate — MRs with "revert|hotfix|rollback" in title
|
|
70
|
+
# ---------------------------------------------------------------
|
|
71
|
+
FAILURE_MRS=$(echo "${MERGED_MRS}" | jq '[.[] | select(.title // "" | test("revert|hotfix|rollback"; "i"))] | length')
|
|
72
|
+
[[ "${DEPLOY_COUNT}" -gt 0 ]] && CHANGE_FAILURE_RATE=$(echo "scale=4; ${FAILURE_MRS} / ${DEPLOY_COUNT}" | bc -l | sed 's/^\./0./') || CHANGE_FAILURE_RATE="0"
|
|
73
|
+
|
|
74
|
+
# ---------------------------------------------------------------
|
|
75
|
+
# 4. MTTR — avg time for hotfix/revert MRs
|
|
76
|
+
# ---------------------------------------------------------------
|
|
77
|
+
MTTR_HOURS=$(echo "${MERGED_MRS}" | jq '
|
|
78
|
+
[.[] | select(.title // "" | test("revert|hotfix|rollback"; "i")) |
|
|
79
|
+
select(.merged_at != null and .created_at != null) |
|
|
80
|
+
((.merged_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime) -
|
|
81
|
+
(.created_at | sub("\\.[0-9]+Z$"; "Z") | strptime("%Y-%m-%dT%H:%M:%SZ") | mktime)) / 3600
|
|
82
|
+
] | if length > 0 then add / length | . * 100 | round / 100 else 0 end')
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------
|
|
85
|
+
# 5. AI-DORA metrics from commit trailers
|
|
86
|
+
# ---------------------------------------------------------------
|
|
87
|
+
TOTAL_COMMITS=$(git log --since="${SINCE_DATE}" --oneline "${CI_DEFAULT_BRANCH}" 2>/dev/null | wc -l | tr -d ' ')
|
|
88
|
+
AI_GENERATED=0; AI_ASSISTED=0; HUMAN_ONLY=0; SPEC_REFS=0
|
|
89
|
+
CLAUDE_CODE_COUNT=0; KIRO_COUNT=0; Q_DEV_COUNT=0
|
|
90
|
+
|
|
91
|
+
for SHA in $(git log --since="${SINCE_DATE}" --format='%H' "${CI_DEFAULT_BRANCH}" 2>/dev/null); do
|
|
92
|
+
MSG=$(git log -1 --format='%B' "${SHA}")
|
|
93
|
+
echo "${MSG}" | grep -q '^AI-Origin: ai-generated' && AI_GENERATED=$((AI_GENERATED + 1))
|
|
94
|
+
echo "${MSG}" | grep -q '^AI-Origin: ai-assisted' && AI_ASSISTED=$((AI_ASSISTED + 1))
|
|
95
|
+
echo "${MSG}" | grep -q '^AI-Origin: human' && HUMAN_ONLY=$((HUMAN_ONLY + 1))
|
|
96
|
+
echo "${MSG}" | grep -q '^Spec-Ref:' && SPEC_REFS=$((SPEC_REFS + 1))
|
|
97
|
+
TOOL=$(echo "${MSG}" | grep -oP '(?<=^AI-Tool: ).+' || true)
|
|
98
|
+
case "${TOOL}" in
|
|
99
|
+
*claude-code*) CLAUDE_CODE_COUNT=$((CLAUDE_CODE_COUNT + 1)) ;;
|
|
100
|
+
*kiro*) KIRO_COUNT=$((KIRO_COUNT + 1)) ;;
|
|
101
|
+
*q-developer*) Q_DEV_COUNT=$((Q_DEV_COUNT + 1)) ;;
|
|
102
|
+
esac
|
|
103
|
+
done
|
|
104
|
+
|
|
105
|
+
UNTAGGED=$((TOTAL_COMMITS - AI_GENERATED - AI_ASSISTED - HUMAN_ONLY))
|
|
106
|
+
[[ "${UNTAGGED}" -lt 0 ]] && UNTAGGED=0
|
|
107
|
+
AI_TOTAL=$((AI_GENERATED + AI_ASSISTED))
|
|
108
|
+
[[ "${TOTAL_COMMITS}" -gt 0 ]] && AI_ADOPTION_RATE=$(echo "scale=4; ${AI_TOTAL} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || AI_ADOPTION_RATE="0"
|
|
109
|
+
[[ "${TOTAL_COMMITS}" -gt 0 ]] && SPEC_COVERAGE=$(echo "scale=4; ${SPEC_REFS} / ${TOTAL_COMMITS}" | bc -l | sed 's/^\./0./') || SPEC_COVERAGE="0"
|
|
110
|
+
|
|
111
|
+
# ---------------------------------------------------------------
|
|
112
|
+
# Emit assessment event
|
|
113
|
+
# ---------------------------------------------------------------
|
|
114
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
115
|
+
|
|
116
|
+
ASSESSMENT_EVENT=$(jq -n \
|
|
117
|
+
--arg team_id "${TEAM_ID}" --arg repo "${REPO}" --arg timestamp "${TIMESTAMP}" \
|
|
118
|
+
--argjson deploy_count "${DEPLOY_COUNT}" --argjson deploy_frequency "${DEPLOY_FREQUENCY}" \
|
|
119
|
+
--argjson avg_lead_time_hours "${AVG_LEAD_TIME_HOURS}" \
|
|
120
|
+
--argjson change_failure_rate "${CHANGE_FAILURE_RATE}" --argjson mttr_hours "${MTTR_HOURS}" \
|
|
121
|
+
--argjson total_commits "${TOTAL_COMMITS}" --argjson ai_generated "${AI_GENERATED}" \
|
|
122
|
+
--argjson ai_assisted "${AI_ASSISTED}" --argjson human_only "${HUMAN_ONLY}" \
|
|
123
|
+
--argjson untagged "${UNTAGGED}" --argjson ai_adoption_rate "${AI_ADOPTION_RATE}" \
|
|
124
|
+
--argjson spec_coverage "${SPEC_COVERAGE}" \
|
|
125
|
+
--argjson claude_code "${CLAUDE_CODE_COUNT}" --argjson kiro "${KIRO_COUNT}" --argjson q_dev "${Q_DEV_COUNT}" \
|
|
126
|
+
'{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:2,
|
|
127
|
+
metric:{name:"weekly_assessment",value:1,unit:"assessment"},
|
|
128
|
+
dora:{deployment_frequency:$deploy_frequency, deployments:$deploy_count,
|
|
129
|
+
lead_time_hours:$avg_lead_time_hours, change_failure_rate:$change_failure_rate,
|
|
130
|
+
mttr_hours:$mttr_hours},
|
|
131
|
+
ai_dora:{ai_adoption_rate:$ai_adoption_rate, ai_to_merge_ratio:$ai_adoption_rate,
|
|
132
|
+
total_commits:$total_commits, ai_generated_commits:$ai_generated,
|
|
133
|
+
ai_assisted_commits:$ai_assisted, human_commits:$human_only,
|
|
134
|
+
untagged_commits:$untagged, spec_coverage:$spec_coverage,
|
|
135
|
+
tool_breakdown:{claude_code:$claude_code, kiro:$kiro, q_developer:$q_dev}}}')
|
|
136
|
+
|
|
137
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
138
|
+
--entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.assessment\",
|
|
139
|
+
\"EventBusName\":\"prism-d1-metrics\",
|
|
140
|
+
\"Detail\":$(echo "${ASSESSMENT_EVENT}" | jq -c '.' | jq -Rs '.')}]"
|
|
141
|
+
|
|
142
|
+
# Emit CloudWatch metrics
|
|
143
|
+
for METRIC_DATA in \
|
|
144
|
+
"AIAdoptionRate:${AI_ADOPTION_RATE}:Percent" \
|
|
145
|
+
"SpecCoverage:${SPEC_COVERAGE}:Percent" \
|
|
146
|
+
"ClaudeCodeCommits:${CLAUDE_CODE_COUNT}:Count" \
|
|
147
|
+
"KiroCommits:${KIRO_COUNT}:Count" \
|
|
148
|
+
"QDeveloperCommits:${Q_DEV_COUNT}:Count" \
|
|
149
|
+
"UntaggedCommits:${UNTAGGED}:Count"; do
|
|
150
|
+
METRIC_NAME="${METRIC_DATA%%:*}"; REST="${METRIC_DATA#*:}"
|
|
151
|
+
METRIC_VALUE="${REST%%:*}"; METRIC_UNIT="${REST#*:}"
|
|
152
|
+
aws cloudwatch put-metric-data --region "${AWS_DEFAULT_REGION}" --namespace "${NAMESPACE}" \
|
|
153
|
+
--metric-data "[{\"MetricName\":\"${METRIC_NAME}\",\"Value\":${METRIC_VALUE},
|
|
154
|
+
\"Unit\":\"${METRIC_UNIT}\",\"Timestamp\":\"${TIMESTAMP}\",
|
|
155
|
+
\"Dimensions\":[{\"Name\":\"TeamId\",\"Value\":\"${TEAM_ID}\"},{\"Name\":\"Repository\",\"Value\":\"${REPO}\"}]}]"
|
|
156
|
+
done
|
|
157
|
+
|
|
158
|
+
echo "=== DORA Metrics ==="
|
|
159
|
+
echo "Deployment Frequency: ${DEPLOY_FREQUENCY}/day"
|
|
160
|
+
echo "Lead Time: ${AVG_LEAD_TIME_HOURS} hours"
|
|
161
|
+
echo "Change Failure Rate: ${CHANGE_FAILURE_RATE}"
|
|
162
|
+
echo "MTTR: ${MTTR_HOURS} hours"
|
|
163
|
+
echo "=== AI-DORA Metrics ==="
|
|
164
|
+
echo "AI Adoption Rate: ${AI_ADOPTION_RATE}"
|
|
165
|
+
echo "Spec Coverage: ${SPEC_COVERAGE}"
|
|
166
|
+
echo "Total Commits: ${TOTAL_COMMITS}"
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# prism-eval-gate.yml — Evaluate AI-generated code quality on MRs.
|
|
2
|
+
#
|
|
3
|
+
# Include this file in your .gitlab-ci.yml:
|
|
4
|
+
# include:
|
|
5
|
+
# - local: '.prism/gitlab-workflows/prism-eval-gate.yml'
|
|
6
|
+
#
|
|
7
|
+
# Required setup:
|
|
8
|
+
# - CI/CD variable PRISM_METRICS_ROLE_ARN (IAM role with bedrock:InvokeModel + events:PutEvents)
|
|
9
|
+
# - CI/CD variable AWS_REGION (default: us-west-2)
|
|
10
|
+
# - .prism/eval-harness/ directory in your repo (run: prism-cli bootstrapper install-eval-harness)
|
|
11
|
+
# - At least one rubric JSON in .prism/eval-harness/rubrics/
|
|
12
|
+
|
|
13
|
+
prism-eval-gate:
|
|
14
|
+
stage: test
|
|
15
|
+
image: amazon/aws-cli:latest
|
|
16
|
+
id_tokens:
|
|
17
|
+
GITLAB_OIDC_TOKEN:
|
|
18
|
+
aud: https://gitlab.aws.dev
|
|
19
|
+
rules:
|
|
20
|
+
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
|
|
21
|
+
variables:
|
|
22
|
+
AWS_DEFAULT_REGION: ${AWS_REGION:-us-west-2}
|
|
23
|
+
before_script:
|
|
24
|
+
- yum install -y -q jq bc git curl
|
|
25
|
+
- git fetch --unshallow || true
|
|
26
|
+
- |
|
|
27
|
+
# Assume AWS role via GitLab OIDC
|
|
28
|
+
if [[ -n "${PRISM_METRICS_ROLE_ARN}" && -n "${GITLAB_OIDC_TOKEN}" ]]; then
|
|
29
|
+
CREDS=$(aws sts assume-role-with-web-identity \
|
|
30
|
+
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
31
|
+
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
32
|
+
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
33
|
+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
34
|
+
--output text)
|
|
35
|
+
export AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}')
|
|
36
|
+
export AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}')
|
|
37
|
+
export AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}')
|
|
38
|
+
fi
|
|
39
|
+
script:
|
|
40
|
+
- |
|
|
41
|
+
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
42
|
+
BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA}"
|
|
43
|
+
HEAD_SHA="${CI_COMMIT_SHA}"
|
|
44
|
+
|
|
45
|
+
# Identify AI-generated commits
|
|
46
|
+
AI_SHAS=""
|
|
47
|
+
for SHA in $(git rev-list "${BASE_SHA}..${HEAD_SHA}" 2>/dev/null); do
|
|
48
|
+
if git log -1 --format='%B' "${SHA}" | grep -q 'AI-Origin: ai-generated\|AI-Origin: ai-assisted'; then
|
|
49
|
+
AI_SHAS="${AI_SHAS} ${SHA}"
|
|
50
|
+
fi
|
|
51
|
+
done
|
|
52
|
+
AI_SHAS=$(echo "${AI_SHAS}" | xargs)
|
|
53
|
+
|
|
54
|
+
if [[ -z "${AI_SHAS}" ]]; then
|
|
55
|
+
echo "No AI-generated code detected. Skipping eval gate."
|
|
56
|
+
# Post skip note
|
|
57
|
+
curl -s --request POST --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
|
|
58
|
+
--form "body=✅ **PRISM Eval Gate** — No AI-generated code detected. Skipped." \
|
|
59
|
+
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" > /dev/null 2>&1 || true
|
|
60
|
+
exit 0
|
|
61
|
+
fi
|
|
62
|
+
|
|
63
|
+
# Collect changed source files from AI commits
|
|
64
|
+
CHANGED_FILES=""
|
|
65
|
+
for SHA in ${AI_SHAS}; do
|
|
66
|
+
FILES=$(git diff-tree --no-commit-id --name-only -r "${SHA}" \
|
|
67
|
+
| grep -E '\.(ts|js|py|go|java|rs|tsx|jsx)$' | grep -v '\.test\.\|\.spec\.' || true)
|
|
68
|
+
CHANGED_FILES="${CHANGED_FILES}\n${FILES}"
|
|
69
|
+
done
|
|
70
|
+
echo -e "${CHANGED_FILES}" | sort -u | grep -v '^$' > /tmp/ai-changed-files.txt || true
|
|
71
|
+
FILE_COUNT=$(wc -l < /tmp/ai-changed-files.txt | tr -d ' ')
|
|
72
|
+
|
|
73
|
+
# Run evaluations
|
|
74
|
+
chmod +x .prism/eval-harness/run-eval.sh
|
|
75
|
+
PASS_COUNT=0; FAIL_COUNT=0; TOTAL_SCORE=0; TOTAL_HALLUCINATIONS=0; RESULTS=""
|
|
76
|
+
|
|
77
|
+
while IFS= read -r FILE; do
|
|
78
|
+
[[ -z "${FILE}" || ! -f "${FILE}" ]] && continue
|
|
79
|
+
|
|
80
|
+
# Auto-select rubric
|
|
81
|
+
if echo "${FILE}" | grep -qiE '(agent|assistant|orchestrat|workflow|chain)'; then
|
|
82
|
+
RUBRIC=".prism/eval-harness/rubrics/agent-quality.json"
|
|
83
|
+
elif echo "${FILE}" | grep -qiE '(auth|security|guard|policy|iam|permission|crypto)'; then
|
|
84
|
+
RUBRIC=".prism/eval-harness/rubrics/security-compliance.json"
|
|
85
|
+
elif echo "${FILE}" | grep -qiE '(api|handler|route|controller)'; then
|
|
86
|
+
RUBRIC=".prism/eval-harness/rubrics/api-response-quality.json"
|
|
87
|
+
else
|
|
88
|
+
RUBRIC=".prism/eval-harness/rubrics/code-quality.json"
|
|
89
|
+
fi
|
|
90
|
+
[[ -f "${RUBRIC}" ]] || RUBRIC=$(ls .prism/eval-harness/rubrics/*.json 2>/dev/null | head -1)
|
|
91
|
+
[[ -f "${RUBRIC}" ]] || { echo "No rubrics found"; exit 1; }
|
|
92
|
+
|
|
93
|
+
set +e
|
|
94
|
+
OUTPUT=$(./.prism/eval-harness/run-eval.sh "${RUBRIC}" "${FILE}" 2>&1)
|
|
95
|
+
set -e
|
|
96
|
+
|
|
97
|
+
SCORE=$(echo "${OUTPUT}" | grep '^Score:' | awk '{print $2}' || echo "0")
|
|
98
|
+
RESULT=$(echo "${OUTPUT}" | grep '^Result:' | awk '{print $2}' || echo "UNKNOWN")
|
|
99
|
+
HALLUC=$(echo "${OUTPUT}" | grep '^Hallucinations:' | awk '{print $2}' || echo "0")
|
|
100
|
+
|
|
101
|
+
TOTAL_SCORE=$(echo "${TOTAL_SCORE} + ${SCORE}" | bc -l)
|
|
102
|
+
TOTAL_HALLUCINATIONS=$((TOTAL_HALLUCINATIONS + HALLUC))
|
|
103
|
+
|
|
104
|
+
[[ "${RESULT}" == "SKIP" ]] && continue
|
|
105
|
+
[[ "${RESULT}" == "PASS" ]] && PASS_COUNT=$((PASS_COUNT + 1)) || FAIL_COUNT=$((FAIL_COUNT + 1))
|
|
106
|
+
RESULTS="${RESULTS}\n| \`${FILE}\` | $(basename "${RUBRIC}" .json) | ${SCORE} | ${RESULT} |"
|
|
107
|
+
done < /tmp/ai-changed-files.txt
|
|
108
|
+
|
|
109
|
+
TOTAL_FILES=$((PASS_COUNT + FAIL_COUNT))
|
|
110
|
+
[[ "${TOTAL_FILES}" -gt 0 ]] && AVG_SCORE=$(echo "scale=4; ${TOTAL_SCORE} / ${TOTAL_FILES}" | bc -l) || AVG_SCORE="0"
|
|
111
|
+
[[ "${FAIL_COUNT}" -gt 0 ]] && OVERALL="FAIL" || OVERALL="PASS"
|
|
112
|
+
|
|
113
|
+
# Post MR note
|
|
114
|
+
EMOJI=$([[ "${OVERALL}" == "PASS" ]] && echo "✅" || echo "❌")
|
|
115
|
+
NOTE="## ${EMOJI} PRISM Eval Gate — ${OVERALL}\n\n**Score**: ${AVG_SCORE} | **Files**: ${TOTAL_FILES} | **Pass**: ${PASS_COUNT} | **Fail**: ${FAIL_COUNT} | **Hallucinations**: ${TOTAL_HALLUCINATIONS}\n\n| File | Rubric | Score | Result |\n|---|---|---|---|\n$(echo -e "${RESULTS}")\n\n> Threshold: 0.82 | Model: claude-sonnet-4-20250514"
|
|
116
|
+
curl -s --request POST --header "PRIVATE-TOKEN: ${GITLAB_TOKEN:-${CI_JOB_TOKEN}}" \
|
|
117
|
+
--form "body=${NOTE}" \
|
|
118
|
+
"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes" > /dev/null 2>&1 || true
|
|
119
|
+
|
|
120
|
+
# Emit event
|
|
121
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
122
|
+
EVAL_EVENT=$(jq -n \
|
|
123
|
+
--arg team_id "${TEAM_ID}" --arg repo "${CI_PROJECT_PATH}" --arg timestamp "${TIMESTAMP}" \
|
|
124
|
+
--argjson score "${AVG_SCORE}" --arg result "${OVERALL}" \
|
|
125
|
+
--argjson total_files "${TOTAL_FILES}" --argjson pass_count "${PASS_COUNT}" \
|
|
126
|
+
--argjson fail_count "${FAIL_COUNT}" --argjson hallucinations "${TOTAL_HALLUCINATIONS}" \
|
|
127
|
+
--arg mr_iid "${CI_MERGE_REQUEST_IID}" \
|
|
128
|
+
'{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:2,
|
|
129
|
+
metric:{name:"eval_score",value:$score,unit:"score"},
|
|
130
|
+
ai_context:{tool:"bedrock-eval",model:"anthropic.claude-sonnet-4-20250514",origin:"ai-generated"},
|
|
131
|
+
eval:{total_files:$total_files, pass_count:$pass_count, fail_count:$fail_count,
|
|
132
|
+
result:$result, mr_iid:($mr_iid | tonumber), hallucinations_caught:$hallucinations}}')
|
|
133
|
+
|
|
134
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
135
|
+
--entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.eval\",
|
|
136
|
+
\"EventBusName\":\"prism-d1-metrics\",
|
|
137
|
+
\"Detail\":$(echo "${EVAL_EVENT}" | jq -c '.' | jq -Rs '.')}]"
|
|
138
|
+
|
|
139
|
+
[[ "${OVERALL}" == "FAIL" ]] && exit 1 || exit 0
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
declare const _default: {
|
|
2
|
+
description: string;
|
|
3
|
+
options: {
|
|
4
|
+
flags: string;
|
|
5
|
+
description: string;
|
|
6
|
+
default: string;
|
|
7
|
+
}[];
|
|
8
|
+
action(opts: {
|
|
9
|
+
gitlabUrl?: string;
|
|
10
|
+
outputDir: string;
|
|
11
|
+
}): Promise<void>;
|
|
12
|
+
};
|
|
13
|
+
export default _default;
|
|
14
|
+
//# sourceMappingURL=install-gitlab-workflows.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"install-gitlab-workflows.d.ts","sourceRoot":"","sources":["../../../../src/commands/bootstrapper/install-gitlab-workflows.ts"],"names":[],"mappings":";;;;;;;iBAsBqB;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;;AAN9D,wBA2CE"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { createInterface } from 'node:readline';
|
|
4
|
+
import { getAssetPath } from '../../utils/root.js';
|
|
5
|
+
function prompt(question, defaultValue) {
|
|
6
|
+
const rl = createInterface({ input: process.stdin, output: process.stdout });
|
|
7
|
+
const suffix = defaultValue ? ` [${defaultValue}]` : '';
|
|
8
|
+
return new Promise((resolve) => {
|
|
9
|
+
rl.question(`${question}${suffix}: `, (answer) => {
|
|
10
|
+
rl.close();
|
|
11
|
+
resolve(answer.trim() || defaultValue || '');
|
|
12
|
+
});
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
export default {
|
|
16
|
+
description: 'Install GitLab CI workflow templates into the current repo',
|
|
17
|
+
options: [
|
|
18
|
+
{ flags: '--gitlab-url <url>', description: 'GitLab instance URL for OIDC audience', default: 'https://gitlab.aws.dev' },
|
|
19
|
+
{ flags: '--output-dir <dir>', description: 'Output directory', default: '.prism/gitlab-workflows' },
|
|
20
|
+
],
|
|
21
|
+
async action(opts) {
|
|
22
|
+
const gitlabUrl = opts.gitlabUrl || await prompt('GitLab instance URL', 'https://gitlab.aws.dev');
|
|
23
|
+
const outputDir = opts.outputDir;
|
|
24
|
+
console.log(`\n📦 Installing GitLab CI workflows`);
|
|
25
|
+
console.log(` Audience: ${gitlabUrl}`);
|
|
26
|
+
console.log(` Output: ${outputDir}/\n`);
|
|
27
|
+
mkdirSync(outputDir, { recursive: true });
|
|
28
|
+
const assetDir = getAssetPath(import.meta.url, 'gitlab-workflows/.gitlab-ci.yml').replace('/.gitlab-ci.yml', '');
|
|
29
|
+
const files = readdirSync(assetDir).filter(f => f.endsWith('.yml'));
|
|
30
|
+
for (const file of files) {
|
|
31
|
+
let content = readFileSync(join(assetDir, file), 'utf-8');
|
|
32
|
+
// Template the audience
|
|
33
|
+
content = content.replace(/aud: https:\/\/gitlab\.aws\.dev/g, `aud: ${gitlabUrl}`);
|
|
34
|
+
writeFileSync(join(outputDir, file), content);
|
|
35
|
+
console.log(` ✓ ${file}`);
|
|
36
|
+
}
|
|
37
|
+
// Update .gitlab-ci.yml include paths
|
|
38
|
+
const ciFile = join(outputDir, '.gitlab-ci.yml');
|
|
39
|
+
if (existsSync(ciFile)) {
|
|
40
|
+
let ci = readFileSync(ciFile, 'utf-8');
|
|
41
|
+
ci = ci.replace(/\.prism\/gitlab-workflows/g, outputDir);
|
|
42
|
+
writeFileSync(ciFile, ci);
|
|
43
|
+
}
|
|
44
|
+
console.log(`\n✅ Installed ${files.length} workflow files to ${outputDir}/`);
|
|
45
|
+
console.log(`\nNext steps:`);
|
|
46
|
+
console.log(` 1. Copy ${outputDir}/.gitlab-ci.yml to your repo root`);
|
|
47
|
+
console.log(` 2. Run: prism-cli bootstrapper setup-gitlab-oidc`);
|
|
48
|
+
console.log(` 3. Add CI/CD variable PRISM_METRICS_ROLE_ARN in GitLab`);
|
|
49
|
+
console.log(` 4. (Optional) Create pipeline schedule for weekly DORA assessment`);
|
|
50
|
+
console.log('');
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
//# sourceMappingURL=install-gitlab-workflows.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"install-gitlab-workflows.js","sourceRoot":"","sources":["../../../../src/commands/bootstrapper/install-gitlab-workflows.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAC1F,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEnD,SAAS,MAAM,CAAC,QAAgB,EAAE,YAAqB;IACrD,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC,KAAK,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IACxD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,EAAE,CAAC,QAAQ,CAAC,GAAG,QAAQ,GAAG,MAAM,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE;YAC/C,EAAE,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,YAAY,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,eAAe;IACb,WAAW,EAAE,4DAA4D;IACzE,OAAO,EAAE;QACP,EAAE,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,uCAAuC,EAAE,OAAO,EAAE,wBAAwB,EAAE;QACxH,EAAE,KAAK,EAAE,oBAAoB,EAAE,WAAW,EAAE,kBAAkB,EAAE,OAAO,EAAE,yBAAyB,EAAE;KACrG;IACD,KAAK,CAAC,MAAM,CAAC,IAA+C;QAC1D,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,MAAM,CAAC,qBAAqB,EAAE,wBAAwB,CAAC,CAAC;QAClG,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,gBAAgB,SAAS,KAAK,CAAC,CAAC;QAE5C,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE1C,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,iCAAiC,CAAC,CAAC,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACjH,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;QAEpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAC1D,wBAAwB;YACxB,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,kCAAkC,EAAE,QAAQ,SAAS,EAAE,CAAC,CAAC;YACnF,aAAa,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAC7B,CAAC;QAED,sCAAsC;QACtC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACjD,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvB,IAAI,EAAE,GAAG,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YACvC,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,4BAA4B,EAAE,SAAS,CAAC,CAAC;YACzD,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC5B,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,CAAC,MAAM,sBAAsB,SAAS,GAAG,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,SAAS,mCAAmC,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;QAClE,OAAO,CAAC,GAAG,CAAC,0DAA0D,CAAC,CAAC;QACxE,OAAO,CAAC,GAAG,CAAC,qEAAqE,CAAC,CAAC;QACnF,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;CACF,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prism-d1/cli",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.45",
|
|
4
4
|
"description": "PRISM D1 Velocity CLI — AI development lifecycle assessment tool",
|
|
5
5
|
"author": "andklee",
|
|
6
6
|
"engines": {
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
"access": "public"
|
|
23
23
|
},
|
|
24
24
|
"scripts": {
|
|
25
|
-
"build": "npx tsc && mkdir -p dist/assets/infra && cp -r ../bootstrapper/metric-hooks ../bootstrapper/eval-harness ../bootstrapper/github-workflows dist/assets/ && cp -r ../infra/bin ../infra/lib ../infra/package.json ../infra/tsconfig.json ../infra/cdk.json dist/assets/infra/",
|
|
25
|
+
"build": "npx tsc && mkdir -p dist/assets/infra && cp -r ../bootstrapper/metric-hooks ../bootstrapper/eval-harness ../bootstrapper/github-workflows ../bootstrapper/gitlab-workflows dist/assets/ && cp -r ../infra/bin ../infra/lib ../infra/package.json ../infra/tsconfig.json ../infra/cdk.json dist/assets/infra/",
|
|
26
26
|
"prepublishOnly": "npm run build",
|
|
27
27
|
"start": "tsx bin/prism-cli.ts",
|
|
28
28
|
"dev": "tsx bin/prism-cli.ts"
|