@prism-d1/cli 1.0.93 → 1.0.94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/github-workflows/prism-eval-gate-kiro.yml +311 -1
- package/dist/assets/gitlab-workflows/prism-eval-gate.yml +333 -116
- package/dist/assets/infra/lib/lambda/velocity-widget.ts +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts +5 -5
- package/dist/src/commands/bootstrapper/install-eval-harness.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-eval-harness.js +157 -75
- package/dist/src/commands/bootstrapper/install-eval-harness.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.d.ts +6 -2
- package/dist/src/commands/bootstrapper/install-github-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-github-workflows.js +36 -48
- package/dist/src/commands/bootstrapper/install-github-workflows.js.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.d.ts.map +1 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js +10 -1
- package/dist/src/commands/bootstrapper/install-gitlab-workflows.js.map +1 -1
- package/package.json +1 -1
- package/dist/assets/github-workflows/prism-eval-gate.yml +0 -490
|
@@ -1,14 +1,27 @@
|
|
|
1
|
-
# prism-eval-gate.yml —
|
|
1
|
+
# prism-eval-gate.yml — AI code quality gate using kiro-cli headless mode,
|
|
2
|
+
# plus gitleaks secret scanning.
|
|
2
3
|
#
|
|
3
4
|
# Include this file in your .gitlab-ci.yml:
|
|
4
5
|
# include:
|
|
5
6
|
# - local: '.prism/gitlab-workflows/prism-eval-gate.yml'
|
|
6
7
|
#
|
|
7
8
|
# Required setup:
|
|
8
|
-
# - CI/CD variable
|
|
9
|
+
# - CI/CD variable KIRO_API_KEY (masked). Generate at https://app.kiro.dev
|
|
9
10
|
# - PRISM_AWS_REGION variable in the variables: block below controls the region
|
|
10
|
-
# -
|
|
11
|
-
#
|
|
11
|
+
# - (Optional) CI/CD variable PRISM_METRICS_ROLE_ARN for EventBridge metrics
|
|
12
|
+
#
|
|
13
|
+
# The Bedrock rubric gate that used to live in this file is retired. It scored
|
|
14
|
+
# changed files against .prism/eval-harness/rubrics/*.json via run-eval.sh; the
|
|
15
|
+
# review is now agentic (kiro-cli against .kiro/steering/code-review.md). The
|
|
16
|
+
# job name and include path are unchanged so existing .gitlab-ci.yml files and
|
|
17
|
+
# any merge-request approval rule matching this job keep working.
|
|
18
|
+
#
|
|
19
|
+
# .prism/eval-harness/ is still installed and still used — by prism-agent-eval,
|
|
20
|
+
# which remains Bedrock-scored. It is no longer read by this gate.
|
|
21
|
+
#
|
|
22
|
+
# Secret scanning needs no CI/CD variable and no AWS role, so unlike the review
|
|
23
|
+
# it also functions on fork merge requests, where GitLab withholds protected
|
|
24
|
+
# variables and KIRO_API_KEY is therefore absent.
|
|
12
25
|
|
|
13
26
|
prism-eval-gate:
|
|
14
27
|
stage: test
|
|
@@ -24,138 +37,342 @@ prism-eval-gate:
|
|
|
24
37
|
PRISM_AWS_REGION: us-west-2
|
|
25
38
|
AWS_DEFAULT_REGION: $PRISM_AWS_REGION
|
|
26
39
|
PRISM_REPO: ${CI_SERVER_HOST}/${CI_PROJECT_PATH}
|
|
40
|
+
# Full history: the scan and the review both diff BASE..HEAD, and a shallow
|
|
41
|
+
# clone makes that range unresolvable.
|
|
42
|
+
GIT_DEPTH: 0
|
|
43
|
+
# Pinned deliberately, with the digest verified out of band. Floating to
|
|
44
|
+
# latest would let an upstream release change what this gate blocks without
|
|
45
|
+
# a repo change; reading the checksum from the release's own checksums.txt
|
|
46
|
+
# would confirm nothing if the release itself were replaced.
|
|
47
|
+
GITLEAKS_VERSION: "8.28.0"
|
|
48
|
+
GITLEAKS_SHA256: "a65b5253807a68ac0cafa4414031fd740aeb55f54fb7e55f386acb52e6a840eb"
|
|
27
49
|
before_script:
|
|
28
|
-
|
|
29
|
-
|
|
50
|
+
# unzip is not optional: the kiro-cli installer hard-errors without it.
|
|
51
|
+
# tar/gzip are for the gitleaks tarball, which the aws-cli image also lacks.
|
|
52
|
+
- yum install -y -q jq git tar gzip unzip which >/dev/null 2>&1 || yum install -y jq git tar gzip unzip
|
|
53
|
+
- git fetch --unshallow 2>/dev/null || true
|
|
54
|
+
script:
|
|
30
55
|
- |
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
56
|
+
set -u
|
|
57
|
+
GATE_FAILED=0
|
|
58
|
+
|
|
59
|
+
BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
|
|
60
|
+
HEAD_SHA="${CI_COMMIT_SHA:-}"
|
|
61
|
+
TEAM_ID=$(jq -r '.team_id // "no_team"' .prism/config.json 2>/dev/null || echo "no_team")
|
|
62
|
+
|
|
63
|
+
# ===================================================================
|
|
64
|
+
# 1. Secret scanning (gitleaks) — runs FIRST and NEVER skips.
|
|
65
|
+
# ===================================================================
|
|
66
|
+
# Ordered first because it is the cheapest gate and a committed
|
|
67
|
+
# credential should not wait behind a model call to surface.
|
|
68
|
+
#
|
|
69
|
+
# Every failure path below fails the gate closed. This is deliberately
|
|
70
|
+
# unlike the review in step 2, which skips when KIRO_API_KEY is absent
|
|
71
|
+
# because GitLab structurally withholds protected variables from fork
|
|
72
|
+
# pipelines. gitleaks needs no variable and no role, so "unavailable"
|
|
73
|
+
# here is an install failure, not a configuration choice — and a secret
|
|
74
|
+
# scanner that reports green without having scanned is worse than none,
|
|
75
|
+
# because it launders an exposure into a passing pipeline.
|
|
76
|
+
GITLEAKS_LEAKS=""
|
|
77
|
+
GITLEAKS_ERROR=""
|
|
78
|
+
|
|
79
|
+
echo "── Secret scan (gitleaks ${GITLEAKS_VERSION}) ──"
|
|
80
|
+
TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
|
|
81
|
+
URL="https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${TARBALL}"
|
|
82
|
+
|
|
83
|
+
if ! curl -fsSL --retry 3 --retry-delay 2 -o "/tmp/${TARBALL}" "${URL}"; then
|
|
84
|
+
echo "❌ Could not download gitleaks ${GITLEAKS_VERSION} — this MR has NOT been secret-scanned."
|
|
85
|
+
GITLEAKS_ERROR="true"
|
|
86
|
+
elif ! (cd /tmp && echo "${GITLEAKS_SHA256} ${TARBALL}" | sha256sum -c -); then
|
|
87
|
+
echo "❌ gitleaks checksum mismatch — refusing to install an unverified binary."
|
|
88
|
+
echo " Expected ${GITLEAKS_SHA256}"
|
|
89
|
+
echo " If you bumped GITLEAKS_VERSION, update GITLEAKS_SHA256 to match."
|
|
90
|
+
GITLEAKS_ERROR="true"
|
|
91
|
+
else
|
|
92
|
+
tar -xzf "/tmp/${TARBALL}" -C /tmp gitleaks
|
|
93
|
+
chmod +x /tmp/gitleaks
|
|
94
|
+
/tmp/gitleaks version
|
|
95
|
+
|
|
96
|
+
if [ -z "${BASE_SHA}" ] || [ -z "${HEAD_SHA}" ]; then
|
|
97
|
+
echo "❌ Missing base/head SHA — cannot determine what to scan, failing closed."
|
|
98
|
+
GITLEAKS_ERROR="true"
|
|
44
99
|
else
|
|
45
|
-
|
|
100
|
+
# Scope is the commits this MR introduces, NOT the whole history.
|
|
101
|
+
# A full-history scan re-reports every secret predating the gate, so
|
|
102
|
+
# the gate could never go green and would be switched off within a
|
|
103
|
+
# week. Pre-existing leaks need a one-time `gitleaks git` sweep plus a
|
|
104
|
+
# rotation — work an unrelated MR cannot do.
|
|
105
|
+
#
|
|
106
|
+
# --redact is load-bearing: without it gitleaks prints the matched
|
|
107
|
+
# secret into the job log, which is world-readable on a public
|
|
108
|
+
# project. That turns the scanner into a second disclosure channel
|
|
109
|
+
# for the thing it just found.
|
|
110
|
+
#
|
|
111
|
+
# Smoke-testing this gate: verify your fixture with gitleaks itself
|
|
112
|
+
# before trusting a green result. A hand-written fake credential can
|
|
113
|
+
# look completely plausible and still match no rule, in which case
|
|
114
|
+
# "no leaks found" is correct and the gate only appears broken. Never
|
|
115
|
+
# infer that from the rule internals -- they are version specific and
|
|
116
|
+
# change without notice. Ask the tool:
|
|
117
|
+
#
|
|
118
|
+
# gitleaks dir /path/to/fixture --redact --no-banner -v
|
|
119
|
+
#
|
|
120
|
+
# A non-zero finding count there is the only evidence the fixture is
|
|
121
|
+
# usable. Two known traps it will surface: gitleaks' defaults allowlist
|
|
122
|
+
# AWS's own documentation keys (AKIAIOSFODNN7EXAMPLE), and tooling that
|
|
123
|
+
# redacts credential-shaped text in transit can rewrite a key NAME out
|
|
124
|
+
# of a fixture, disarming the rules that need a nearby keyword.
|
|
125
|
+
#
|
|
126
|
+
# A PEM block is the most robust fixture -- no charset constraints, no
|
|
127
|
+
# keyword needed -- and fires `private-key`.
|
|
128
|
+
GL_ARGS="git ${CI_PROJECT_DIR} --log-opts=${BASE_SHA}..${HEAD_SHA}"
|
|
129
|
+
GL_ARGS="${GL_ARGS} --report-format=json --report-path=/tmp/gitleaks-report.json"
|
|
130
|
+
GL_ARGS="${GL_ARGS} --redact --exit-code=1 --no-banner"
|
|
131
|
+
if [ -f .prism/gitleaks-baseline.json ]; then
|
|
132
|
+
GL_ARGS="${GL_ARGS} --baseline-path=.prism/gitleaks-baseline.json"
|
|
133
|
+
echo "Using baseline: .prism/gitleaks-baseline.json"
|
|
134
|
+
fi
|
|
135
|
+
[ -f .gitleaks.toml ] && echo "Using repo config: .gitleaks.toml"
|
|
136
|
+
|
|
137
|
+
set +e
|
|
138
|
+
# shellcheck disable=SC2086
|
|
139
|
+
/tmp/gitleaks ${GL_ARGS}
|
|
140
|
+
RC=$?
|
|
141
|
+
set -e
|
|
142
|
+
|
|
143
|
+
case "${RC}" in
|
|
144
|
+
0)
|
|
145
|
+
echo "✅ No secrets detected in ${BASE_SHA}..${HEAD_SHA}"
|
|
146
|
+
GITLEAKS_LEAKS=0
|
|
147
|
+
;;
|
|
148
|
+
1)
|
|
149
|
+
# Exit 1 means leaks were found. If the count cannot be read the
|
|
150
|
+
# scan still found something, so this must not degrade to 0.
|
|
151
|
+
GITLEAKS_LEAKS=$(jq 'length' /tmp/gitleaks-report.json 2>/dev/null || echo "")
|
|
152
|
+
if [ -z "${GITLEAKS_LEAKS}" ]; then
|
|
153
|
+
echo "❌ gitleaks reported findings but the report could not be parsed — failing closed."
|
|
154
|
+
GITLEAKS_ERROR="true"
|
|
155
|
+
else
|
|
156
|
+
echo "❌ gitleaks found ${GITLEAKS_LEAKS} secret(s) in this MR's commits:"
|
|
157
|
+
# Only safe fields. The secret is never echoed, even though
|
|
158
|
+
# --redact already replaced it.
|
|
159
|
+
jq -r '.[] | " [\(.RuleID)] \(.File):\(.StartLine) in commit \(.Commit[0:8])"' \
|
|
160
|
+
/tmp/gitleaks-report.json || true
|
|
161
|
+
fi
|
|
162
|
+
;;
|
|
163
|
+
*)
|
|
164
|
+
echo "❌ gitleaks exited ${RC} — the scan produced no verdict, so this MR is not verified."
|
|
165
|
+
GITLEAKS_ERROR="true"
|
|
166
|
+
;;
|
|
167
|
+
esac
|
|
46
168
|
fi
|
|
47
|
-
elif [[ -n "${PRISM_METRICS_ROLE_ARN}" ]]; then
|
|
48
|
-
echo "⚠️ PRISM_METRICS_ROLE_ARN set but GITLAB_OIDC_TOKEN is empty."
|
|
49
|
-
fi
|
|
50
|
-
script:
|
|
51
|
-
- |
|
|
52
|
-
# Skip if no AWS credentials available
|
|
53
|
-
if ! aws sts get-caller-identity &>/dev/null; then
|
|
54
|
-
echo "⚠️ No AWS credentials. Set PRISM_METRICS_ROLE_ARN or configure AWS access."
|
|
55
|
-
exit 0
|
|
56
169
|
fi
|
|
57
170
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
HEAD_SHA="${CI_COMMIT_SHA}"
|
|
61
|
-
|
|
62
|
-
# Skip if eval harness not installed
|
|
63
|
-
if [[ ! -f ".prism/eval-harness/run-eval.sh" ]]; then
|
|
64
|
-
echo "Eval harness not installed. Run: prism-cli bootstrapper install-eval-harness"
|
|
65
|
-
echo "Skipping eval gate."
|
|
66
|
-
exit 0
|
|
171
|
+
if [ "${GITLEAKS_ERROR}" = "true" ] || { [ -n "${GITLEAKS_LEAKS}" ] && [ "${GITLEAKS_LEAKS}" != "0" ]; }; then
|
|
172
|
+
GATE_FAILED=1
|
|
67
173
|
fi
|
|
68
174
|
|
|
69
|
-
#
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
175
|
+
# ===================================================================
|
|
176
|
+
# 2. Agentic code review (kiro-cli headless)
|
|
177
|
+
# ===================================================================
|
|
178
|
+
REVIEW_RESULT="SKIPPED"
|
|
179
|
+
REVIEW_SCORE=""
|
|
180
|
+
|
|
181
|
+
echo ""
|
|
182
|
+
echo "── Code quality review (kiro-cli) ──"
|
|
183
|
+
if [ -z "${KIRO_API_KEY:-}" ]; then
|
|
184
|
+
# Fork MRs receive no protected variables. Reported, never silently passed.
|
|
185
|
+
echo "⚠️ KIRO_API_KEY unavailable (fork MR or unset variable) — review skipped."
|
|
186
|
+
elif [ -z "${BASE_SHA}" ] || [ -z "${HEAD_SHA}" ]; then
|
|
187
|
+
echo "⚠️ Missing base/head SHA — review skipped."
|
|
188
|
+
else
|
|
189
|
+
FILES=$(git diff --name-only "${BASE_SHA}..${HEAD_SHA}" \
|
|
190
|
+
| grep -E '\.(ts|js|py|go|java|rs|tsx|jsx)$' \
|
|
191
|
+
| grep -vE '\.test\.|\.spec\.|__test__|_test\.' \
|
|
192
|
+
| head -30 | tr '\n' ' ' || true)
|
|
193
|
+
|
|
194
|
+
if [ -z "${FILES}" ]; then
|
|
195
|
+
echo "No source files changed — review skipped."
|
|
196
|
+
else
|
|
197
|
+
# Installs to $HOME/.local/bin, which is not on PATH in this image.
|
|
198
|
+
export KIRO_API_KEY="$(printf '%s' "${KIRO_API_KEY}" | tr -d '[:space:]')"
|
|
199
|
+
curl -fsSL https://cli.kiro.dev/install | bash || true
|
|
200
|
+
export PATH="${HOME}/.local/bin:${PATH}"
|
|
201
|
+
|
|
202
|
+
if ! command -v kiro-cli >/dev/null 2>&1; then
|
|
203
|
+
# FAIL CLOSED: a review that cannot run is not a passing review.
|
|
204
|
+
echo "❌ kiro-cli is not on PATH after install — review could not run."
|
|
205
|
+
REVIEW_RESULT="ERROR"
|
|
206
|
+
else
|
|
207
|
+
kiro-cli --version || true
|
|
208
|
+
# Short prompt on purpose: kiro-cli has a bug putting long stdin
|
|
209
|
+
# into HTTP headers, so let it read the patch off disk itself.
|
|
210
|
+
git diff "${BASE_SHA}..${HEAD_SHA}" -- ${FILES} > /tmp/mr-diff.patch
|
|
211
|
+
|
|
212
|
+
set +e
|
|
213
|
+
REVIEW_OUTPUT=$(kiro-cli chat --no-interactive --trust-all-tools --model auto \
|
|
214
|
+
"Review /tmp/mr-diff.patch for code quality. Read it, then output ONLY JSON: {\"findings\":[{\"file\":\"path\",\"line\":1,\"severity\":\"high|medium|low\",\"category\":\"str\",\"message\":\"str\"}],\"score\":0.0,\"summary\":\"str\"}")
|
|
215
|
+
set -e
|
|
216
|
+
|
|
217
|
+
CLEANED=$(printf '%s' "${REVIEW_OUTPUT}" | sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g' | tr -d '\r')
|
|
218
|
+
REVIEW_JSON=$(printf '%s' "${CLEANED}" | tr '\n' ' ' | grep -o '{.*}' || true)
|
|
219
|
+
|
|
220
|
+
if [ -z "${REVIEW_JSON}" ] || ! printf '%s' "${REVIEW_JSON}" | jq -e '.score' >/dev/null 2>&1; then
|
|
221
|
+
echo "❌ Could not parse kiro-cli review output — failing closed."
|
|
222
|
+
printf '%s' "${REVIEW_OUTPUT:-<empty>}" | head -c 2000
|
|
223
|
+
echo ""
|
|
224
|
+
REVIEW_RESULT="ERROR"
|
|
225
|
+
else
|
|
226
|
+
printf '%s' "${REVIEW_JSON}" > /tmp/kiro-review.json
|
|
227
|
+
REVIEW_SCORE=$(jq -r '.score' /tmp/kiro-review.json)
|
|
228
|
+
REVIEW_HIGH=$(jq '[.findings[] | select(.severity=="high")] | length' /tmp/kiro-review.json)
|
|
229
|
+
REVIEW_TOTAL=$(jq '.findings | length' /tmp/kiro-review.json)
|
|
230
|
+
REVIEW_SUMMARY=$(jq -r '.summary // "No summary"' /tmp/kiro-review.json | head -1 | cut -c1-200)
|
|
231
|
+
|
|
232
|
+
PASS=$(awk -v s="${REVIEW_SCORE}" 'BEGIN { print (s >= 0.82) ? "true" : "false" }')
|
|
233
|
+
if [ "${REVIEW_HIGH}" -gt 0 ] || [ "${PASS}" != "true" ]; then
|
|
234
|
+
REVIEW_RESULT="FAIL"
|
|
235
|
+
else
|
|
236
|
+
REVIEW_RESULT="PASS"
|
|
237
|
+
fi
|
|
238
|
+
|
|
239
|
+
echo "Score: ${REVIEW_SCORE} | Findings: ${REVIEW_TOTAL} (${REVIEW_HIGH} high) | ${REVIEW_RESULT}"
|
|
240
|
+
echo "Summary: ${REVIEW_SUMMARY}"
|
|
241
|
+
jq -r '.findings[] | " [\(.severity)] \(.file):\(.line) — \(.message)"' /tmp/kiro-review.json || true
|
|
242
|
+
fi
|
|
243
|
+
fi
|
|
74
244
|
fi
|
|
75
|
-
|
|
76
|
-
AI_SHAS=$(echo "${AI_SHAS}" | xargs)
|
|
245
|
+
fi
|
|
77
246
|
|
|
78
|
-
if [
|
|
79
|
-
|
|
80
|
-
exit 0
|
|
247
|
+
if [ "${REVIEW_RESULT}" = "FAIL" ] || [ "${REVIEW_RESULT}" = "ERROR" ]; then
|
|
248
|
+
GATE_FAILED=1
|
|
81
249
|
fi
|
|
82
250
|
|
|
83
|
-
#
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
# Auto-select rubric
|
|
101
|
-
if echo "${FILE}" | grep -qiE '(agent|assistant|orchestrat|workflow|chain)'; then
|
|
102
|
-
RUBRIC=".prism/eval-harness/rubrics/agent-quality.json"
|
|
103
|
-
elif echo "${FILE}" | grep -qiE '(auth|security|guard|policy|iam|permission|crypto)'; then
|
|
104
|
-
RUBRIC=".prism/eval-harness/rubrics/security-compliance.json"
|
|
105
|
-
elif echo "${FILE}" | grep -qiE '(api|handler|route|controller)'; then
|
|
106
|
-
RUBRIC=".prism/eval-harness/rubrics/api-response-quality.json"
|
|
251
|
+
# ===================================================================
|
|
252
|
+
# 3. Telemetry (best effort — never changes the gate verdict)
|
|
253
|
+
# ===================================================================
|
|
254
|
+
echo ""
|
|
255
|
+
echo "── Metrics ──"
|
|
256
|
+
if [ -n "${PRISM_METRICS_ROLE_ARN:-}" ] && [ -n "${GITLAB_OIDC_TOKEN:-}" ]; then
|
|
257
|
+
if CREDS=$(aws sts assume-role-with-web-identity \
|
|
258
|
+
--role-arn "${PRISM_METRICS_ROLE_ARN}" \
|
|
259
|
+
--role-session-name "gitlab-ci-${CI_JOB_ID}" \
|
|
260
|
+
--web-identity-token "${GITLAB_OIDC_TOKEN}" \
|
|
261
|
+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
|
|
262
|
+
--output text 2>&1); then
|
|
263
|
+
AWS_ACCESS_KEY_ID=$(echo "${CREDS}" | awk '{print $1}'); export AWS_ACCESS_KEY_ID
|
|
264
|
+
AWS_SECRET_ACCESS_KEY=$(echo "${CREDS}" | awk '{print $2}'); export AWS_SECRET_ACCESS_KEY
|
|
265
|
+
AWS_SESSION_TOKEN=$(echo "${CREDS}" | awk '{print $3}'); export AWS_SESSION_TOKEN
|
|
266
|
+
echo "✓ OIDC credentials acquired."
|
|
107
267
|
else
|
|
108
|
-
|
|
268
|
+
echo "⚠️ OIDC assume-role failed — skipping metrics."
|
|
109
269
|
fi
|
|
110
|
-
|
|
111
|
-
|
|
270
|
+
else
|
|
271
|
+
echo "⚠️ PRISM_METRICS_ROLE_ARN or GITLAB_OIDC_TOKEN unset — skipping metrics."
|
|
272
|
+
fi
|
|
112
273
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
set -e
|
|
274
|
+
if aws sts get-caller-identity >/dev/null 2>&1; then
|
|
275
|
+
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
116
276
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
277
|
+
# Eval event — only when there is a real score (not SKIPPED/ERROR).
|
|
278
|
+
if [ -n "${REVIEW_SCORE}" ]; then
|
|
279
|
+
DETAIL=$(jq -n -c \
|
|
280
|
+
--arg team_id "${TEAM_ID}" --arg repo "${PRISM_REPO}" --arg timestamp "${TIMESTAMP}" \
|
|
281
|
+
--arg score "${REVIEW_SCORE}" --arg result "${REVIEW_RESULT}" \
|
|
282
|
+
--arg mr_iid "${CI_MERGE_REQUEST_IID:-0}" \
|
|
283
|
+
'{team_id:$team_id,repo:$repo,timestamp:$timestamp,prism_level:2,
|
|
284
|
+
metric:{name:"eval_score",value:($score|tonumber),unit:"score"},
|
|
285
|
+
ai_context:{tool:"kiro-cli",model:"claude-haiku-4.5",origin:"ai-generated"},
|
|
286
|
+
eval:{score:($score|tonumber),result:$result,rubric:"kiro-headless",
|
|
287
|
+
mr_iid:($mr_iid|tonumber),method:"kiro-headless"}}')
|
|
288
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" --entries \
|
|
289
|
+
"[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.eval\",\"EventBusName\":\"prism-d1-metrics\",\"Detail\":$(printf '%s' "${DETAIL}" | jq -Rs '.')}]" || true
|
|
290
|
+
echo "Emitted prism.d1.eval"
|
|
291
|
+
fi
|
|
120
292
|
|
|
121
|
-
|
|
122
|
-
|
|
293
|
+
# gitleaks findings — feeds the CISO dashboard. Without this the gate
|
|
294
|
+
# blocks the MR but the exposure never reaches the metrics pipeline, so
|
|
295
|
+
# "secrets caught per 100 commits" is unanswerable.
|
|
296
|
+
if [ -n "${GITLEAKS_LEAKS}" ] && [ "${GITLEAKS_LEAKS}" != "0" ] && [ -s /tmp/gitleaks-report.json ]; then
|
|
297
|
+
# Deferred origin join: emit commit SHAs as facts and let dashboards
|
|
298
|
+
# resolve AI-vs-human at render time. Attribution spans arrive on a
|
|
299
|
+
# ~12h codeburn schedule, so a verdict computed now would usually be
|
|
300
|
+
# wrong and would be frozen into an immutable event.
|
|
301
|
+
COMMIT_SHAS=$(git rev-list "${BASE_SHA}..${HEAD_SHA}" 2>/dev/null | head -50 | jq -R . | jq -sc . || echo '[]')
|
|
123
302
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
303
|
+
# Severity drives real automation: prism.d1.security.code_review also
|
|
304
|
+
# routes to the security-response-automator, which on CRITICAL/HIGH
|
|
305
|
+
# writes a penalty record and fires the SecurityCriticalFinding alarm.
|
|
306
|
+
# So the mapping is fail-safe rather than uniform — provider-specific
|
|
307
|
+
# rules page as HIGH, while gitleaks' catch-all entropy rule
|
|
308
|
+
# (generic-api-key, which fires on ordinary `secret = ...` lines)
|
|
309
|
+
# reports MEDIUM: still recorded, still blocking the MR, not paging.
|
|
310
|
+
# An unrecognised NEW rule id defaults to HIGH.
|
|
311
|
+
#
|
|
312
|
+
# CWE-798 = Use of Hard-coded Credentials. Fingerprint is gitleaks'
|
|
313
|
+
# own stable id (commit:file:rule:line), so a re-run re-emits the same
|
|
314
|
+
# finding_id instead of inflating counts. Only safe fields are
|
|
315
|
+
# projected — never .Secret/.Match (already REDACTED), and never
|
|
316
|
+
# .Message, which is the commit message and could hold the credential.
|
|
317
|
+
ENTRIES=$(jq -c \
|
|
318
|
+
--arg team_id "${TEAM_ID}" --arg repo "${PRISM_REPO}" \
|
|
319
|
+
--argjson commit_shas "${COMMIT_SHAS}" \
|
|
320
|
+
--argjson mr_iid "${CI_MERGE_REQUEST_IID:-0}" \
|
|
321
|
+
--argjson low_confidence '["generic-api-key"]' \
|
|
322
|
+
'[ .[] | . as $f | {
|
|
323
|
+
Source: "prism.d1.velocity",
|
|
324
|
+
DetailType: "prism.d1.security.code_review",
|
|
325
|
+
EventBusName: "prism-d1-metrics",
|
|
326
|
+
Detail: ({
|
|
327
|
+
team_id: $team_id, repo: $repo, timestamp: (now|todate), prism_level: 3,
|
|
328
|
+
metric: { name: "security_finding", value: 1, unit: "count" },
|
|
329
|
+
ai_context: { tool: "gitleaks", model: "gitleaks-8.28.0" },
|
|
330
|
+
security_agent_finding: {
|
|
331
|
+
finding_id: ("gl-" + ($f.Fingerprint // "unknown")),
|
|
332
|
+
phase: "code_review",
|
|
333
|
+
severity: (if ($low_confidence | index($f.RuleID)) then "MEDIUM" else "HIGH" end),
|
|
334
|
+
category: ($f.RuleID // "unknown"),
|
|
335
|
+
title: ($f.Description // $f.RuleID // "Committed secret detected"),
|
|
336
|
+
cwe_id: "CWE-798", cvss_score: null, exploit_validated: false,
|
|
337
|
+
commit_shas: $commit_shas, pr_number: $mr_iid,
|
|
338
|
+
found_at: (now|todate)
|
|
339
|
+
}
|
|
340
|
+
} | tostring)
|
|
341
|
+
} ]' /tmp/gitleaks-report.json 2>/dev/null || echo '[]')
|
|
128
342
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
343
|
+
TOTAL=$(printf '%s' "${ENTRIES}" | jq 'length')
|
|
344
|
+
if [ "${TOTAL}" -gt 0 ]; then
|
|
345
|
+
# EventBridge accepts max 10 entries per call
|
|
346
|
+
i=0
|
|
347
|
+
while [ "${i}" -lt "${TOTAL}" ]; do
|
|
348
|
+
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
349
|
+
--entries "$(printf '%s' "${ENTRIES}" | jq -c ".[${i}:$((i+10))]")" || true
|
|
350
|
+
i=$((i+10))
|
|
351
|
+
done
|
|
352
|
+
echo "Emitted ${TOTAL} gitleaks finding events"
|
|
353
|
+
fi
|
|
354
|
+
fi
|
|
355
|
+
fi
|
|
132
356
|
|
|
133
|
-
#
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
echo "## ${EMOJI} PRISM Eval Gate — ${OVERALL}"
|
|
137
|
-
echo "Score: ${AVG_SCORE} | Files: ${TOTAL_FILES} | Pass: ${PASS_COUNT} | Fail: ${FAIL_COUNT} | Hallucinations: ${TOTAL_HALLUCINATIONS}"
|
|
138
|
-
echo -e "${RESULTS}"
|
|
357
|
+
# ===================================================================
|
|
358
|
+
# 4. Verdict
|
|
359
|
+
# ===================================================================
|
|
139
360
|
echo ""
|
|
361
|
+
if [ "${GATE_FAILED}" -ne 0 ]; then
|
|
362
|
+
echo "════════════════════════════════════════"
|
|
363
|
+
echo " ❌ PRISM Eval Gate FAILED"
|
|
364
|
+
echo "════════════════════════════════════════"
|
|
365
|
+
if [ "${GITLEAKS_ERROR}" = "true" ]; then
|
|
366
|
+
echo " Secret scan failed closed: this MR has NOT been scanned for credentials."
|
|
367
|
+
elif [ -n "${GITLEAKS_LEAKS}" ] && [ "${GITLEAKS_LEAKS}" != "0" ]; then
|
|
368
|
+
echo " ${GITLEAKS_LEAKS} secret(s) found. Rotate the credential first —"
|
|
369
|
+
echo " deleting it from the branch does not un-expose what was pushed."
|
|
370
|
+
echo " False positive? Add an allowlist entry to .gitleaks.toml."
|
|
371
|
+
fi
|
|
372
|
+
[ "${REVIEW_RESULT}" = "ERROR" ] && echo " Review failed closed: output could not be parsed."
|
|
373
|
+
[ "${REVIEW_RESULT}" = "FAIL" ] && echo " Review score ${REVIEW_SCORE} below threshold, or high-severity findings."
|
|
374
|
+
exit 1
|
|
375
|
+
fi
|
|
140
376
|
|
|
141
|
-
|
|
142
|
-
TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
|
143
|
-
EVAL_EVENT=$(jq -n \
|
|
144
|
-
--arg team_id "${TEAM_ID}" --arg repo "${PRISM_REPO}" --arg timestamp "${TIMESTAMP}" \
|
|
145
|
-
--argjson score "${AVG_SCORE}" --arg result "${OVERALL}" \
|
|
146
|
-
--argjson total_files "${TOTAL_FILES}" --argjson pass_count "${PASS_COUNT}" \
|
|
147
|
-
--argjson fail_count "${FAIL_COUNT}" --argjson hallucinations "${TOTAL_HALLUCINATIONS}" \
|
|
148
|
-
--arg mr_iid "${CI_MERGE_REQUEST_IID}" \
|
|
149
|
-
'{team_id:$team_id, repo:$repo, timestamp:$timestamp, prism_level:2,
|
|
150
|
-
metric:{name:"eval_score",value:$score,unit:"score"},
|
|
151
|
-
ai_context:{tool:"bedrock-eval",model:"anthropic.claude-sonnet-4-20250514",origin:"ai-generated"},
|
|
152
|
-
eval:{total_files:$total_files, pass_count:$pass_count, fail_count:$fail_count,
|
|
153
|
-
result:$result, mr_iid:($mr_iid | tonumber), hallucinations_caught:$hallucinations}}')
|
|
154
|
-
|
|
155
|
-
aws events put-events --region "${AWS_DEFAULT_REGION}" \
|
|
156
|
-
--entries "[{\"Source\":\"prism.d1.velocity\",\"DetailType\":\"prism.d1.eval\",
|
|
157
|
-
\"EventBusName\":\"prism-d1-metrics\",
|
|
158
|
-
\"Detail\":$(echo "${EVAL_EVENT}" | jq -c '.' | jq -Rs '.')}]"
|
|
159
|
-
|
|
160
|
-
[[ "${OVERALL:-PASS}" == "FAIL" ]] && exit 1
|
|
377
|
+
echo "✅ PRISM Eval Gate passed (secret scan: clean, review: ${REVIEW_RESULT})"
|
|
161
378
|
exit 0
|
|
@@ -743,7 +743,7 @@ async function renderExecSecurity(fromIso: string, toIso: string, p: Palette): P
|
|
|
743
743
|
async function renderEval(fromIso: string, toIso: string, p: Palette): Promise<string> {
|
|
744
744
|
const evals = await queryEvents('prism.d1.eval', fromIso, toIso);
|
|
745
745
|
if (evals.length === 0) {
|
|
746
|
-
return emptyState('prism.d1.eval', 'Populated by prism-eval-gate-kiro.yml
|
|
746
|
+
return emptyState('prism.d1.eval', 'Populated by prism-eval-gate-kiro.yml on pull requests, and prism-agent-eval.yml on agent changes.', p);
|
|
747
747
|
}
|
|
748
748
|
|
|
749
749
|
const byRubric = new Map<string, { total: number; pass: number; scoreSum: number }>();
|
|
@@ -3,18 +3,18 @@ declare const _default: {
|
|
|
3
3
|
options: ({
|
|
4
4
|
flags: string;
|
|
5
5
|
description: string;
|
|
6
|
-
default
|
|
6
|
+
default: string;
|
|
7
7
|
} | {
|
|
8
8
|
flags: string;
|
|
9
9
|
description: string;
|
|
10
|
-
default
|
|
10
|
+
default?: undefined;
|
|
11
11
|
})[];
|
|
12
12
|
action(opts: {
|
|
13
13
|
mode?: string;
|
|
14
|
-
withRubrics?: boolean;
|
|
15
|
-
model?: string;
|
|
16
|
-
threshold?: string;
|
|
17
14
|
region?: string;
|
|
15
|
+
agentEvalModel?: string;
|
|
16
|
+
agentEvalThreshold?: string;
|
|
17
|
+
skipAgentEvalHarness?: boolean;
|
|
18
18
|
uninstall?: boolean;
|
|
19
19
|
}): Promise<void>;
|
|
20
20
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"install-eval-harness.d.ts","sourceRoot":"","sources":["../../../../src/commands/bootstrapper/install-eval-harness.ts"],"names":[],"mappings":";;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"install-eval-harness.d.ts","sourceRoot":"","sources":["../../../../src/commands/bootstrapper/install-eval-harness.ts"],"names":[],"mappings":";;;;;;;;;;;iBA4KqB;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,oBAAoB,CAAC,EAAE,OAAO,CAAC;QAC/B,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB;;AAjBH,wBAqEE"}
|