@onlooker-community/ecosystem 0.34.0 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/writing-tests/SKILL.md +27 -0
- package/.claude-plugin/plugin.json +1 -1
- package/.github/workflows/release.yml +14 -14
- package/.release-please-manifest.json +7 -7
- package/AGENTS.md +116 -0
- package/CHANGELOG.md +175 -0
- package/CLAUDE.md +5 -0
- package/docs/lesson-promotion-pipeline.md +6 -6
- package/docs/superpowers/plans/2026-08-10-lesson-confirmation.md +1273 -0
- package/docs/superpowers/plans/2026-08-11-lesson-judging.md +1232 -0
- package/docs/superpowers/plans/2026-08-11-lesson-unconfirm.md +458 -0
- package/docs/superpowers/plans/2026-08-12-author-key.md +567 -0
- package/docs/superpowers/plans/2026-08-13-approved-pool.md +747 -0
- package/docs/superpowers/plans/2026-08-13-librarian-cluster.md +392 -0
- package/docs/superpowers/plans/2026-08-14-criterion-scores-consumers.md +1374 -0
- package/docs/superpowers/plans/2026-08-14-criterion-scores-schema.md +277 -0
- package/docs/superpowers/plans/2026-08-16-cartographer-undocumented-entity.md +741 -0
- package/docs/superpowers/specs/2026-08-10-lesson-confirmation-design.md +213 -0
- package/docs/superpowers/specs/2026-08-11-lesson-judging-design.md +267 -0
- package/docs/superpowers/specs/2026-08-11-lesson-unconfirm-design.md +152 -0
- package/docs/superpowers/specs/2026-08-12-author-key-design.md +317 -0
- package/docs/superpowers/specs/2026-08-13-approved-pool-design.md +232 -0
- package/docs/superpowers/specs/2026-08-14-criterion-scores-design.md +254 -0
- package/docs/superpowers/specs/2026-08-16-cartographer-undocumented-entity-design.md +249 -0
- package/package.json +2 -2
- package/plugins/assayer/.claude-plugin/plugin.json +1 -1
- package/plugins/assayer/CHANGELOG.md +7 -0
- package/plugins/assayer/scripts/lib/assayer-config.sh +6 -0
- package/plugins/cartographer/.claude-plugin/plugin.json +1 -1
- package/plugins/cartographer/CHANGELOG.md +54 -0
- package/plugins/cartographer/README.md +56 -1
- package/plugins/cartographer/config.json +7 -1
- package/plugins/cartographer/scripts/hooks/cartographer-post-write.sh +4 -4
- package/plugins/cartographer/scripts/hooks/cartographer-session-start.sh +7 -4
- package/plugins/cartographer/scripts/lib/cartographer-collect.sh +8 -1
- package/plugins/cartographer/scripts/lib/cartographer-config.sh +26 -0
- package/plugins/cartographer/scripts/lib/cartographer-events.sh +90 -0
- package/plugins/cartographer/scripts/lib/cartographer-filter.sh +105 -0
- package/plugins/cartographer/scripts/lib/cartographer-omission.sh +130 -0
- package/plugins/cartographer/scripts/lib/cartographer-resolve.sh +132 -0
- package/plugins/cartographer/scripts/run-audit.sh +136 -53
- package/plugins/cartographer/skills/cartographer/SKILL.md +33 -9
- package/plugins/curator/.claude-plugin/plugin.json +1 -1
- package/plugins/curator/CHANGELOG.md +7 -0
- package/plugins/curator/scripts/lib/curator-emit.sh +2 -1
- package/plugins/historian/.claude-plugin/plugin.json +1 -1
- package/plugins/historian/CHANGELOG.md +7 -0
- package/plugins/historian/scripts/lib/historian-emit.sh +2 -1
- package/plugins/librarian/.claude-plugin/plugin.json +1 -1
- package/plugins/librarian/CHANGELOG.md +71 -0
- package/plugins/librarian/config.json +30 -1
- package/plugins/librarian/docs/adr/002-agent-definitions-are-shared-assets.md +82 -0
- package/plugins/librarian/scripts/hooks/librarian-session-end.sh +28 -0
- package/plugins/librarian/scripts/hooks/librarian-session-start.sh +42 -12
- package/plugins/librarian/scripts/lib/librarian-author-key.sh +279 -0
- package/plugins/librarian/scripts/lib/librarian-cli.sh +346 -2
- package/plugins/librarian/scripts/lib/librarian-emit.sh +2 -1
- package/plugins/librarian/scripts/lib/librarian-lesson-judge.sh +349 -0
- package/plugins/librarian/scripts/lib/librarian-lesson-promote.sh +225 -0
- package/plugins/librarian/scripts/lib/librarian-lesson-review.sh +312 -0
- package/plugins/librarian/scripts/lib/librarian-lesson-rubric.sh +56 -0
- package/plugins/librarian/scripts/lib/librarian-lesson-storage.sh +82 -13
- package/plugins/librarian/scripts/lib/librarian-lesson-transform.sh +6 -1
- package/plugins/librarian/scripts/lib/librarian-lesson-validate.sh +131 -62
- package/plugins/librarian/skills/librarian/SKILL.md +98 -1
- package/plugins/tribunal/.claude-plugin/plugin.json +1 -1
- package/plugins/tribunal/CHANGELOG.md +63 -0
- package/plugins/tribunal/agents/tribunal-judge-adversarial.md +34 -0
- package/plugins/tribunal/agents/tribunal-judge-security.md +33 -0
- package/plugins/tribunal/agents/tribunal-judge-standard.md +12 -0
- package/plugins/tribunal/config.json +1 -0
- package/plugins/tribunal/scripts/lib/tribunal-aggregate.sh +154 -9
- package/plugins/tribunal/scripts/lib/tribunal-gate.sh +195 -16
- package/plugins/tribunal/skills/tribunal/SKILL.md +5 -3
- package/scripts/lib/prompt-rules.sh +6 -1
- package/test/bats/archivist-inject.bats +1 -1
- package/test/bats/assayer-extract.bats +2 -2
- package/test/bats/bursar-session-start.bats +3 -3
- package/test/bats/cartographer-config.bats +43 -0
- package/test/bats/cartographer-events.bats +267 -0
- package/test/bats/cartographer-filter.bats +169 -0
- package/test/bats/cartographer-lock.bats +3 -3
- package/test/bats/cartographer-omission.bats +202 -0
- package/test/bats/cartographer-resolve.bats +287 -0
- package/test/bats/cartographer-run-audit.bats +325 -0
- package/test/bats/compass-sanitizer.bats +11 -11
- package/test/bats/compass-transcript.bats +2 -2
- package/test/bats/config.bats +15 -15
- package/test/bats/curator-session-start.bats +10 -3
- package/test/bats/emit-payload-default.bats +52 -0
- package/test/bats/governor-ledger.bats +1 -1
- package/test/bats/historian-prompt-submit.bats +1 -1
- package/test/bats/inspector-post-write-hook.bats +4 -4
- package/test/bats/librarian-author-key.bats +477 -0
- package/test/bats/librarian-cli.bats +16 -16
- package/test/bats/librarian-lesson-judge.bats +884 -0
- package/test/bats/librarian-lesson-promote.bats +552 -0
- package/test/bats/librarian-lesson-review.bats +900 -0
- package/test/bats/librarian-lesson-transform.bats +17 -3
- package/test/bats/librarian-session-end.bats +161 -0
- package/test/bats/librarian-session-start.bats +36 -2
- package/test/bats/lineage-config.bats +1 -1
- package/test/bats/lineage-redact.bats +5 -5
- package/test/bats/session-tracker.bats +4 -4
- package/test/bats/tribunal-aggregate.bats +354 -1
- package/test/bats/tribunal-gate.bats +631 -0
- package/test/bats/tribunal-judge-agents.bats +119 -0
- package/test/bats/tribunal-jury.bats +1 -1
- package/test/bats/turn-tracker.bats +1 -1
- package/test/bats/warden-sanitizer.bats +3 -3
- package/test/bats/worktree-tracker.bats +2 -2
- package/test/node/lesson-validate-agreement.test.mjs +35 -6
|
@@ -0,0 +1,884 @@
|
|
|
1
|
+
#!/usr/bin/env bats
|
|
2
|
+
|
|
3
|
+
# `run --separate-stderr` (used below) requires bats >= 1.5.0.
|
|
4
|
+
bats_require_minimum_version 1.5.0
|
|
5
|
+
|
|
6
|
+
setup() {
|
|
7
|
+
source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
|
|
8
|
+
setup_test_env
|
|
9
|
+
|
|
10
|
+
PLUGIN_ROOT="${REPO_ROOT}/plugins/librarian"
|
|
11
|
+
export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT"
|
|
12
|
+
export ONLOOKER_ECOSYSTEM_ROOT="$REPO_ROOT"
|
|
13
|
+
|
|
14
|
+
PROJECT_REPO="${BATS_TEST_TMPDIR}/repo"
|
|
15
|
+
mkdir -p "$PROJECT_REPO"
|
|
16
|
+
git -C "$PROJECT_REPO" init -q
|
|
17
|
+
git -C "$PROJECT_REPO" config user.email t@example.com
|
|
18
|
+
git -C "$PROJECT_REPO" config user.name "Test"
|
|
19
|
+
git -C "$PROJECT_REPO" remote add origin git@github.com:org/fixture.git
|
|
20
|
+
|
|
21
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-config.sh"
|
|
22
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-project-key.sh"
|
|
23
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-lesson-rubric.sh"
|
|
24
|
+
librarian_config_load "$PROJECT_REPO"
|
|
25
|
+
|
|
26
|
+
PROJECT_KEY=$(librarian_project_key "$PROJECT_REPO")
|
|
27
|
+
|
|
28
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-storage.sh"
|
|
29
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-lesson-storage.sh"
|
|
30
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-lesson-judge.sh"
|
|
31
|
+
# librarian_cli_lessons_list (exercised by the CLI tests below) calls
|
|
32
|
+
# librarian_lesson_list_by_status, which lives here — not pulled in by
|
|
33
|
+
# any of the sources above.
|
|
34
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-lesson-review.sh"
|
|
35
|
+
source "${PLUGIN_ROOT}/scripts/lib/librarian-cli.sh"
|
|
36
|
+
|
|
37
|
+
# A claude stub that fails loudly. Any path asserted to spend no tokens
|
|
38
|
+
# must not invoke it. Same technique that proved stage 5's unavailable
|
|
39
|
+
# path and stage 6's no-model guarantee.
|
|
40
|
+
STUB_BIN="${BATS_TEST_TMPDIR}/bin"
|
|
41
|
+
mkdir -p "$STUB_BIN"
|
|
42
|
+
cat > "${STUB_BIN}/claude" <<'STUB'
|
|
43
|
+
#!/usr/bin/env bash
|
|
44
|
+
echo "claude was invoked but this path must spend no tokens" >&2
|
|
45
|
+
exit 99
|
|
46
|
+
STUB
|
|
47
|
+
chmod +x "${STUB_BIN}/claude"
|
|
48
|
+
export PATH="${STUB_BIN}:${PATH}"
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
@test "org visibility selects the lesson-promotion rubric" {
|
|
52
|
+
run librarian_lesson_rubric_id_for_visibility "org"
|
|
53
|
+
[ "$status" -eq 0 ]
|
|
54
|
+
[ "$output" = "lesson-promotion" ]
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
@test "public visibility selects the public rubric" {
|
|
58
|
+
run librarian_lesson_rubric_id_for_visibility "public"
|
|
59
|
+
[ "$status" -eq 0 ]
|
|
60
|
+
[ "$output" = "lesson-promotion-public" ]
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
@test "private visibility selects no rubric" {
|
|
64
|
+
run librarian_lesson_rubric_id_for_visibility "private"
|
|
65
|
+
[ "$status" -eq 0 ]
|
|
66
|
+
[ "$output" = "" ]
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
@test "an unknown visibility is refused" {
|
|
70
|
+
run librarian_lesson_rubric_id_for_visibility "everyone"
|
|
71
|
+
[ "$status" -ne 0 ]
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
@test "both the org and public rubrics gate on majority" {
|
|
75
|
+
# unanimous was the public tier's stand-in protection and never worked at
|
|
76
|
+
# this panel size — see ecosystem-j74. Both rubrics are majority now; what
|
|
77
|
+
# still differs between them is the criteria, pinned separately below.
|
|
78
|
+
local org public
|
|
79
|
+
org=$(librarian_lesson_rubric_get "lesson-promotion")
|
|
80
|
+
public=$(librarian_lesson_rubric_get "lesson-promotion-public")
|
|
81
|
+
[ "$(printf '%s' "$org" | jq -r '.gate_policy')" = "majority" ]
|
|
82
|
+
[ "$(printf '%s' "$public" | jq -r '.gate_policy')" = "majority" ]
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
@test "both rubrics carry a 0.75 score threshold and two judge types" {
|
|
86
|
+
local r
|
|
87
|
+
for r in lesson-promotion lesson-promotion-public; do
|
|
88
|
+
local got
|
|
89
|
+
got=$(librarian_lesson_rubric_get "$r")
|
|
90
|
+
[ "$(printf '%s' "$got" | jq -r '.score_threshold')" = "0.75" ]
|
|
91
|
+
[ "$(printf '%s' "$got" | jq -c '.judge_types')" = '["standard","adversarial"]' ]
|
|
92
|
+
done
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
@test "neither rubric carries a max_iterations knob" {
|
|
96
|
+
# There is no Actor in this pipeline, so a retry setting would be a knob
|
|
97
|
+
# that cannot do anything. See the spec's "There is no Actor" section.
|
|
98
|
+
local r
|
|
99
|
+
for r in lesson-promotion lesson-promotion-public; do
|
|
100
|
+
local got
|
|
101
|
+
got=$(librarian_lesson_rubric_get "$r")
|
|
102
|
+
[ "$(printf '%s' "$got" | jq 'has("max_iterations")')" = "false" ]
|
|
103
|
+
done
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
@test "each rubric's criterion weights sum to exactly 1.00" {
|
|
107
|
+
# Tribunal validates each weight in [0,1] but never their total. An
|
|
108
|
+
# unnormalized set would silently mis-score the moment ecosystem-pht
|
|
109
|
+
# implements real weighted_mean.
|
|
110
|
+
local r
|
|
111
|
+
for r in lesson-promotion lesson-promotion-public; do
|
|
112
|
+
local sum
|
|
113
|
+
sum=$(librarian_lesson_rubric_get "$r" | jq '[.criteria[].weight] | add | . * 100 | round')
|
|
114
|
+
[ "$sum" -eq 100 ]
|
|
115
|
+
done
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
@test "only the public rubric carries the disclosure criterion" {
|
|
119
|
+
local org public
|
|
120
|
+
org=$(librarian_lesson_rubric_get "lesson-promotion" | jq -c '[.criteria[].name]')
|
|
121
|
+
public=$(librarian_lesson_rubric_get "lesson-promotion-public" | jq -c '[.criteria[].name]')
|
|
122
|
+
[ "$org" = '["grounding","scope_accuracy","generality"]' ]
|
|
123
|
+
[ "$public" = '["grounding","scope_accuracy","generality","disclosure"]' ]
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
@test "disclosure carries the highest floor of any criterion" {
|
|
127
|
+
local r floor others_max
|
|
128
|
+
r=$(librarian_lesson_rubric_get "lesson-promotion-public")
|
|
129
|
+
floor=$(printf '%s' "$r" | jq '.criteria[] | select(.name == "disclosure") | .min_pass')
|
|
130
|
+
others_max=$(printf '%s' "$r" | jq '[.criteria[] | select(.name != "disclosure") | .min_pass] | max')
|
|
131
|
+
[ "$floor" = "0.9" ] || return 1
|
|
132
|
+
[ "$(jq -n --argjson a "$floor" --argjson b "$others_max" '$a > $b')" = "true" ] || return 1
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
@test "an unknown rubric id is refused and echoes nothing" {
|
|
136
|
+
run librarian_lesson_rubric_get "no-such-rubric"
|
|
137
|
+
[ "$status" -ne 0 ]
|
|
138
|
+
[ "$output" = "" ]
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
_seed_confirmed() {
|
|
142
|
+
local id="$1" visibility="$2"
|
|
143
|
+
local dir="$(librarian_lessons_dir "$PROJECT_KEY")/proposals"
|
|
144
|
+
mkdir -p "$dir"
|
|
145
|
+
jq -n --arg id "$id" --arg v "$visibility" \
|
|
146
|
+
--arg t "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
147
|
+
'{ id: $id, artifact_id: "art-\($id)", status: "confirmed",
|
|
148
|
+
visibility: $v, confirmed_at: $t,
|
|
149
|
+
candidate: { claim: "Prefer jq -c for compact output",
|
|
150
|
+
rationale: "Readable diffs",
|
|
151
|
+
evidence: { resolution: "Applied and verified" },
|
|
152
|
+
applies_to: { stack: ["bash"],
|
|
153
|
+
scope: { kind: "versioned", versions: ">=3.2" } } } }' \
|
|
154
|
+
> "${dir}/${id}.json"
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
_status_of() {
|
|
158
|
+
jq -r '.status' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/${1}.json"
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
# Two judges, both passing. Aggregate 0.85 at org / 0.88 at public — both clear
|
|
162
|
+
# the 0.75 threshold, and every criterion sits above its floor.
|
|
163
|
+
#
|
|
164
|
+
# criterion_scores are the superset of both rubrics' criteria, so the same
|
|
165
|
+
# fixture serves org (grounding / scope_accuracy / generality) and public (those
|
|
166
|
+
# plus disclosure). They are not decoration: librarian_lesson_judge refuses a
|
|
167
|
+
# panel that leaves a floored criterion unscored, so a fixture without them is
|
|
168
|
+
# UNJUDGED (2) rather than a verdict. disclosure is 0.95 on both judges because
|
|
169
|
+
# its floor is 0.9 and the floor is now the panel MINIMUM.
|
|
170
|
+
_verdicts_pass() {
|
|
171
|
+
printf '%s' '[{"judge_type":"standard","score":0.9,"passed":true,"confidence":0.9,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.9,"disclosure":0.95},"feedback_summary":"Well grounded."},{"judge_type":"adversarial","score":0.8,"passed":true,"confidence":0.8,"criterion_scores":{"grounding":0.8,"scope_accuracy":0.8,"generality":0.8,"disclosure":0.95},"feedback_summary":"Holds up."}]'
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
# Split panel: the aggregate still clears the threshold and no floor is
|
|
175
|
+
# violated, so the jury policy is the only thing that blocks it.
|
|
176
|
+
_verdicts_split() {
|
|
177
|
+
printf '%s' '[{"judge_type":"standard","score":0.95,"passed":true,"confidence":0.9,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.95,"disclosure":0.95},"feedback_summary":"Strong."},{"judge_type":"adversarial","score":0.75,"passed":false,"confidence":0.8,"criterion_scores":{"grounding":0.75,"scope_accuracy":0.75,"generality":0.75,"disclosure":0.95},"feedback_summary":"Scope claim is not supported."}]'
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
@test "aggregate averages the judges' scores" {
|
|
181
|
+
run librarian_lesson_aggregate "$(_verdicts_pass)"
|
|
182
|
+
[ "$status" -eq 0 ]
|
|
183
|
+
# 0.9 + 0.8 = 1.7 / 2 = 0.85
|
|
184
|
+
[ "$(printf '%s' "$output" | awk '{printf "%.2f", $1}')" = "0.85" ]
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
@test "aggregate refuses an empty panel" {
|
|
188
|
+
run librarian_lesson_aggregate '[]'
|
|
189
|
+
[ "$status" -ne 0 ]
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
@test "a unanimous gate blocks when one judge blocks" {
|
|
193
|
+
run librarian_lesson_gate "unanimous" "$(_verdicts_split)" "0.85" "0.75"
|
|
194
|
+
[ "$status" -eq 0 ]
|
|
195
|
+
[ "$(printf '%s' "$output" | jq -r '.passed')" = "false" ]
|
|
196
|
+
[ "$(printf '%s' "$output" | jq -r '.reason')" = "jury_not_unanimous" ]
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
@test "majority and unanimous diverge on a two-of-three panel" {
|
|
200
|
+
# Two judges with one dissenter does NOT clear majority either — it needs
|
|
201
|
+
# strictly more than half. A three-judge panel is the smallest one where
|
|
202
|
+
# the two policies actually disagree, which is what this pins.
|
|
203
|
+
local three='[{"judge_type":"standard","score":0.95,"passed":true},{"judge_type":"adversarial","score":0.9,"passed":true},{"judge_type":"standard","score":0.75,"passed":false}]'
|
|
204
|
+
run librarian_lesson_gate "majority" "$three" "0.867" "0.75"
|
|
205
|
+
[ "$status" -eq 0 ]
|
|
206
|
+
[ "$(printf '%s' "$output" | jq -r '.passed')" = "true" ]
|
|
207
|
+
|
|
208
|
+
run librarian_lesson_gate "unanimous" "$three" "0.867" "0.75"
|
|
209
|
+
[ "$(printf '%s' "$output" | jq -r '.passed')" = "false" ]
|
|
210
|
+
[ "$(printf '%s' "$output" | jq -r '.reason')" = "jury_not_unanimous" ]
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
@test "a below-threshold aggregate blocks even when every judge passed" {
|
|
214
|
+
local low='[{"judge_type":"standard","score":0.6,"passed":true},{"judge_type":"adversarial","score":0.6,"passed":true}]'
|
|
215
|
+
run librarian_lesson_gate "majority" "$low" "0.6" "0.75"
|
|
216
|
+
[ "$(printf '%s' "$output" | jq -r '.passed')" = "false" ]
|
|
217
|
+
[ "$(printf '%s' "$output" | jq -r '.reason')" = "below_threshold" ]
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
@test "a private candidate is approved with no model call and no judges" {
|
|
221
|
+
_seed_confirmed "priv01" "private"
|
|
222
|
+
run librarian_lesson_judge "$PROJECT_KEY" "priv01" '[]'
|
|
223
|
+
[ "$status" -eq 0 ]
|
|
224
|
+
[ "$(_status_of priv01)" = "approved" ]
|
|
225
|
+
|
|
226
|
+
local v
|
|
227
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/priv01.json")
|
|
228
|
+
[ "$(printf '%s' "$v" | jq -r '.reason')" = "private_no_jury" ]
|
|
229
|
+
[ "$(printf '%s' "$v" | jq -c '.judges')" = "[]" ]
|
|
230
|
+
[ "$(printf '%s' "$v" | jq -r '.rubric_id')" = "null" ]
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
@test "an org candidate the jury passes is approved under the org rubric" {
|
|
234
|
+
_seed_confirmed "org01" "org"
|
|
235
|
+
run librarian_lesson_judge "$PROJECT_KEY" "org01" "$(_verdicts_pass)"
|
|
236
|
+
[ "$status" -eq 0 ]
|
|
237
|
+
[ "$(_status_of org01)" = "approved" ]
|
|
238
|
+
|
|
239
|
+
local v
|
|
240
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/org01.json")
|
|
241
|
+
[ "$(printf '%s' "$v" | jq -r '.rubric_id')" = "lesson-promotion" ]
|
|
242
|
+
[ "$(printf '%s' "$v" | jq -r '.gate_policy')" = "majority" ]
|
|
243
|
+
[ "$(printf '%s' "$v" | jq '.judges | length')" -eq 2 ]
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
@test "a public candidate one judge blocks is rejected though the aggregate clears" {
|
|
247
|
+
# The mean here is 0.85, well above the 0.75 threshold; only the jury
|
|
248
|
+
# policy stops it. A split 1-of-2 panel fails majority the same way it
|
|
249
|
+
# failed unanimous at this panel size — see ecosystem-j74.
|
|
250
|
+
_seed_confirmed "pub01" "public"
|
|
251
|
+
run librarian_lesson_judge "$PROJECT_KEY" "pub01" "$(_verdicts_split)"
|
|
252
|
+
[ "$status" -eq 0 ]
|
|
253
|
+
[ "$(_status_of pub01)" = "rejected" ]
|
|
254
|
+
|
|
255
|
+
local v
|
|
256
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/pub01.json")
|
|
257
|
+
[ "$(printf '%s' "$v" | jq -r '.rubric_id')" = "lesson-promotion-public" ]
|
|
258
|
+
[ "$(printf '%s' "$v" | jq -r '.gate_policy')" = "majority" ]
|
|
259
|
+
[ "$(printf '%s' "$v" | jq -r '.reason')" = "jury_not_majority" ]
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
@test "the same split panel is rejected at org, but for the majority reason" {
|
|
263
|
+
# The verdicts are identical to the public case above. 1 of 2 passing
|
|
264
|
+
# clears neither policy, so both tiers reject — what this pins is that the
|
|
265
|
+
# RUBRIC actually differs by visibility, via the recorded reason and id.
|
|
266
|
+
# Without this, the public test alone would pass even if both visibilities
|
|
267
|
+
# resolved to the same rubric.
|
|
268
|
+
_seed_confirmed "org02" "org"
|
|
269
|
+
run librarian_lesson_judge "$PROJECT_KEY" "org02" "$(_verdicts_split)"
|
|
270
|
+
[ "$status" -eq 0 ]
|
|
271
|
+
[ "$(_status_of org02)" = "rejected" ]
|
|
272
|
+
|
|
273
|
+
local v
|
|
274
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/org02.json")
|
|
275
|
+
[ "$(printf '%s' "$v" | jq -r '.rubric_id')" = "lesson-promotion" ]
|
|
276
|
+
[ "$(printf '%s' "$v" | jq -r '.gate_policy')" = "majority" ]
|
|
277
|
+
[ "$(printf '%s' "$v" | jq -r '.reason')" = "jury_not_majority" ]
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
@test "a three-element panel (one extra judge) is unjudged, not judged with an extra vote" {
|
|
281
|
+
# Both builtin rubrics declare exactly two judge_types (standard,
|
|
282
|
+
# adversarial). A three-element panel can no longer produce a real
|
|
283
|
+
# outcome divergence between majority and unanimous through
|
|
284
|
+
# librarian_lesson_judge: the composition check requires the panel's
|
|
285
|
+
# judge_type multiset to equal the rubric's, so an extra judge is
|
|
286
|
+
# rejected before the gate ever runs. (For a real two-judge panel,
|
|
287
|
+
# majority and unanimous are the same decision anyway: majority needs
|
|
288
|
+
# strictly more than half, which for count=2 means both pass — exactly
|
|
289
|
+
# what unanimous requires.) The gate-level divergence between the two
|
|
290
|
+
# policies on a hypothetical three-judge panel is still pinned directly
|
|
291
|
+
# by "majority and unanimous diverge on a two-of-three panel" above.
|
|
292
|
+
local three='[{"judge_type":"standard","score":0.95,"passed":true},{"judge_type":"adversarial","score":0.9,"passed":true},{"judge_type":"standard","score":0.8,"passed":false}]'
|
|
293
|
+
|
|
294
|
+
_seed_confirmed "tier01" "org"
|
|
295
|
+
local before_org
|
|
296
|
+
before_org=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/tier01.json")
|
|
297
|
+
run librarian_lesson_judge "$PROJECT_KEY" "tier01" "$three"
|
|
298
|
+
[ "$status" -eq 2 ]
|
|
299
|
+
[ "$(_status_of tier01)" = "confirmed" ]
|
|
300
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/tier01.json")" = "$before_org" ]
|
|
301
|
+
|
|
302
|
+
_seed_confirmed "tier02" "public"
|
|
303
|
+
local before_public
|
|
304
|
+
before_public=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/tier02.json")
|
|
305
|
+
run librarian_lesson_judge "$PROJECT_KEY" "tier02" "$three"
|
|
306
|
+
[ "$status" -eq 2 ]
|
|
307
|
+
[ "$(_status_of tier02)" = "confirmed" ]
|
|
308
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/tier02.json")" = "$before_public" ]
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
@test "a one-judge panel on a public candidate is unjudged, not approved by a lone judge" {
|
|
312
|
+
# The sharpest case: a single approving judge is trivially unanimous, so
|
|
313
|
+
# without the composition check this would promote a public lesson on
|
|
314
|
+
# one vote — the exact failure the unanimous policy exists to prevent.
|
|
315
|
+
_seed_confirmed "solo01" "public"
|
|
316
|
+
local before
|
|
317
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/solo01.json")
|
|
318
|
+
|
|
319
|
+
run librarian_lesson_judge "$PROJECT_KEY" "solo01" \
|
|
320
|
+
'[{"judge_type":"standard","score":0.9,"passed":true}]'
|
|
321
|
+
[ "$status" -eq 2 ]
|
|
322
|
+
[ "$(_status_of solo01)" = "confirmed" ]
|
|
323
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/solo01.json")" = "$before" ]
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
@test "a one-judge panel on an org candidate is unjudged" {
|
|
327
|
+
_seed_confirmed "solo02" "org"
|
|
328
|
+
local before
|
|
329
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/solo02.json")
|
|
330
|
+
|
|
331
|
+
run librarian_lesson_judge "$PROJECT_KEY" "solo02" \
|
|
332
|
+
'[{"judge_type":"standard","score":0.9,"passed":true}]'
|
|
333
|
+
[ "$status" -eq 2 ]
|
|
334
|
+
[ "$(_status_of solo02)" = "confirmed" ]
|
|
335
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/solo02.json")" = "$before" ]
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
@test "two judges of the same type is unjudged, not a stand-in for the missing type" {
|
|
339
|
+
# Two "standard" verdicts and no "adversarial" one is individually
|
|
340
|
+
# well-typed and even sized right, so only a composition check catches
|
|
341
|
+
# it.
|
|
342
|
+
_seed_confirmed "dup01" "org"
|
|
343
|
+
local before
|
|
344
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/dup01.json")
|
|
345
|
+
|
|
346
|
+
run librarian_lesson_judge "$PROJECT_KEY" "dup01" \
|
|
347
|
+
'[{"judge_type":"standard","score":0.9,"passed":true},{"judge_type":"standard","score":0.8,"passed":true}]'
|
|
348
|
+
[ "$status" -eq 2 ]
|
|
349
|
+
[ "$(_status_of dup01)" = "confirmed" ]
|
|
350
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/dup01.json")" = "$before" ]
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
@test "a rejected proposal keeps its file, correctly marked rejected with a recorded verdict" {
|
|
354
|
+
# librarian_lesson_seen scans proposals/ by artifact_id; deleting the file
|
|
355
|
+
# would let the same artifact re-propose and re-pay tokens next scan.
|
|
356
|
+
# _seed_confirmed already creates this file before the call, so asserting
|
|
357
|
+
# only -f here would pass even if the function body did nothing at all —
|
|
358
|
+
# the assertion must pin what a *rejected* proposal looks like, not
|
|
359
|
+
# merely that the file was not deleted.
|
|
360
|
+
_seed_confirmed "rej01" "public"
|
|
361
|
+
run librarian_lesson_judge "$PROJECT_KEY" "rej01" "$(_verdicts_split)"
|
|
362
|
+
[ "$status" -eq 0 ]
|
|
363
|
+
[ -f "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/rej01.json" ]
|
|
364
|
+
[ "$(_status_of rej01)" = "rejected" ]
|
|
365
|
+
|
|
366
|
+
local v
|
|
367
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/rej01.json")
|
|
368
|
+
[ "$(printf '%s' "$v" | jq -r '.passed')" = "false" ]
|
|
369
|
+
[ "$(printf '%s' "$v" | jq '.judges | length')" -eq 2 ]
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
@test "judge writes the verdict atomically, never truncating in place" {
|
|
373
|
+
# Same discriminator as the three atomic-write tests in
|
|
374
|
+
# librarian-lesson-review.bats: `printf > path` truncates before writing,
|
|
375
|
+
# so an interrupted write leaves a zero-byte proposal that every verb
|
|
376
|
+
# refuses and list_pending hides — unrecoverable even by unconfirm. A
|
|
377
|
+
# read-only-dir test would NOT catch this: it blocks the open entirely,
|
|
378
|
+
# so the truncating code also leaves the original intact. What
|
|
379
|
+
# distinguishes atomic from not is that the write lands somewhere else
|
|
380
|
+
# first, so spy on the rename.
|
|
381
|
+
_seed_confirmed "atomic01" "org"
|
|
382
|
+
|
|
383
|
+
local marker="${BATS_TEST_TMPDIR}/mv-called"
|
|
384
|
+
rm -f "$marker"
|
|
385
|
+
mv() { printf '%s -> %s\n' "$1" "$2" >> "$marker"; command mv "$@"; }
|
|
386
|
+
|
|
387
|
+
run librarian_lesson_judge "$PROJECT_KEY" "atomic01" "$(_verdicts_pass)"
|
|
388
|
+
[ "$status" -eq 0 ]
|
|
389
|
+
|
|
390
|
+
[ -f "$marker" ]
|
|
391
|
+
grep -q "proposals/atomic01.json" "$marker" || return 1
|
|
392
|
+
[ "$(_status_of atomic01)" = "approved" ]
|
|
393
|
+
unset -f mv
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
@test "a malformed verdict leaves the candidate confirmed and writes nothing" {
|
|
397
|
+
_seed_confirmed "bad01" "org"
|
|
398
|
+
local before
|
|
399
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad01.json")
|
|
400
|
+
|
|
401
|
+
run librarian_lesson_judge "$PROJECT_KEY" "bad01" '[{"judge_type":"standard","score":"not-a-number","passed":true}]'
|
|
402
|
+
[ "$status" -eq 2 ]
|
|
403
|
+
[ "$(_status_of bad01)" = "confirmed" ]
|
|
404
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad01.json")" = "$before" ]
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
@test "unparseable verdict JSON is unjudged, not rejected" {
|
|
408
|
+
_seed_confirmed "bad02" "org"
|
|
409
|
+
run librarian_lesson_judge "$PROJECT_KEY" "bad02" 'this is not json'
|
|
410
|
+
[ "$status" -eq 2 ]
|
|
411
|
+
[ "$(_status_of bad02)" = "confirmed" ]
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
@test "an empty panel on a non-private candidate is unjudged" {
|
|
415
|
+
_seed_confirmed "bad03" "public"
|
|
416
|
+
run librarian_lesson_judge "$PROJECT_KEY" "bad03" '[]'
|
|
417
|
+
[ "$status" -eq 2 ]
|
|
418
|
+
[ "$(_status_of bad03)" = "confirmed" ]
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
# The three malformed-panel tests above are all double-covered downstream:
|
|
422
|
+
# a non-numeric score fails jq arithmetic inside librarian_lesson_aggregate,
|
|
423
|
+
# unparseable JSON fails its `jq 'length'`, and an empty array is refused by
|
|
424
|
+
# librarian_lesson_aggregate's own zero-length check. Every one of those
|
|
425
|
+
# still returns 2 even with the `usable` guard deleted, so none of them
|
|
426
|
+
# actually pins the guard itself. A wrong-typed or absent `passed` field is
|
|
427
|
+
# the one shape that reaches neither check: the scores parse fine, so the
|
|
428
|
+
# aggregate succeeds, and jq's `select(.passed == true)` on a non-boolean or
|
|
429
|
+
# missing `passed` just quietly evaluates false instead of erroring. Without
|
|
430
|
+
# the guard, that panel gets judged as though every judge blocked — written
|
|
431
|
+
# `rejected` rather than left `confirmed` for a retry, which is exactly the
|
|
432
|
+
# "malformed panel permanently buries a good lesson" failure this stage
|
|
433
|
+
# exists to prevent.
|
|
434
|
+
@test "a wrong-typed passed field is unjudged, not silently scored as a block" {
|
|
435
|
+
_seed_confirmed "bad04" "org"
|
|
436
|
+
local before
|
|
437
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad04.json")
|
|
438
|
+
|
|
439
|
+
run librarian_lesson_judge "$PROJECT_KEY" "bad04" \
|
|
440
|
+
'[{"judge_type":"standard","score":0.9,"passed":"true"},{"judge_type":"adversarial","score":0.85,"passed":"true"}]'
|
|
441
|
+
[ "$status" -eq 2 ]
|
|
442
|
+
[ "$(_status_of bad04)" = "confirmed" ]
|
|
443
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad04.json")" = "$before" ]
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
@test "a missing passed field is unjudged, not silently scored as a block" {
|
|
447
|
+
_seed_confirmed "bad05" "org"
|
|
448
|
+
local before
|
|
449
|
+
before=$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad05.json")
|
|
450
|
+
|
|
451
|
+
run librarian_lesson_judge "$PROJECT_KEY" "bad05" \
|
|
452
|
+
'[{"judge_type":"standard","score":0.9},{"judge_type":"adversarial","score":0.85}]'
|
|
453
|
+
[ "$status" -eq 2 ]
|
|
454
|
+
[ "$(_status_of bad05)" = "confirmed" ]
|
|
455
|
+
[ "$(cat "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/bad05.json")" = "$before" ]
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
@test "judging proceeds only from confirmed" {
|
|
459
|
+
_seed_confirmed "st01" "org"
|
|
460
|
+
local path="$(librarian_lessons_dir "$PROJECT_KEY")/proposals/st01.json"
|
|
461
|
+
local tmp="${BATS_TEST_TMPDIR}/st01.json"
|
|
462
|
+
jq '.status = "pending"' "$path" > "$tmp" && mv "$tmp" "$path"
|
|
463
|
+
|
|
464
|
+
run librarian_lesson_judge "$PROJECT_KEY" "st01" "$(_verdicts_pass)"
|
|
465
|
+
[ "$status" -eq 1 ]
|
|
466
|
+
[[ "$output" == *"pending"* ]] || return 1
|
|
467
|
+
[ "$(_status_of st01)" = "pending" ]
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
@test "re-judging an already-approved candidate is refused, naming the status" {
|
|
471
|
+
_seed_confirmed "st02" "org"
|
|
472
|
+
run librarian_lesson_judge "$PROJECT_KEY" "st02" "$(_verdicts_pass)"
|
|
473
|
+
[ "$status" -eq 0 ]
|
|
474
|
+
|
|
475
|
+
run librarian_lesson_judge "$PROJECT_KEY" "st02" "$(_verdicts_pass)"
|
|
476
|
+
[ "$status" -eq 1 ]
|
|
477
|
+
[[ "$output" == *"approved"* ]] || return 1
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
@test "a missing lesson is refused" {
|
|
481
|
+
run librarian_lesson_judge "$PROJECT_KEY" "nope01" "$(_verdicts_pass)"
|
|
482
|
+
[ "$status" -eq 1 ]
|
|
483
|
+
[[ "$output" == *"not found"* ]] || return 1
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
@test "lessons judge records a verdict through the CLI" {
|
|
487
|
+
_seed_confirmed "cli01" "org"
|
|
488
|
+
run librarian_cli lessons judge "cli01" "$(_verdicts_pass)" "$PROJECT_REPO"
|
|
489
|
+
[ "$status" -eq 0 ]
|
|
490
|
+
[ "$(_status_of cli01)" = "approved" ]
|
|
491
|
+
[[ "$output" == *"approved"* ]] || return 1
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
@test "lessons judge reports an unjudged candidate distinctly from a rejection" {
|
|
495
|
+
_seed_confirmed "cli02" "org"
|
|
496
|
+
run librarian_cli lessons judge "cli02" '[{"judge_type":"standard","score":"nope","passed":true}]' "$PROJECT_REPO"
|
|
497
|
+
[ "$status" -eq 2 ]
|
|
498
|
+
[[ "$output" == *"could not be judged"* ]] || return 1
|
|
499
|
+
[ "$(_status_of cli02)" = "confirmed" ]
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
@test "lessons judge requires a lesson id" {
|
|
503
|
+
run librarian_cli lessons judge
|
|
504
|
+
[ "$status" -ne 0 ]
|
|
505
|
+
[[ "$output" == *"usage:"* && "$output" == *"judge"* ]] || return 1
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
@test "lessons judge requires verdicts" {
|
|
509
|
+
run librarian_cli lessons judge "cli03"
|
|
510
|
+
[ "$status" -ne 0 ]
|
|
511
|
+
[[ "$output" == *"usage:"* && "$output" == *"judge"* ]] || return 1
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
@test "lessons judge rejects an unknown flag" {
|
|
515
|
+
_seed_confirmed "cli04" "org"
|
|
516
|
+
run librarian_cli lessons judge "cli04" "$(_verdicts_pass)" --force
|
|
517
|
+
[ "$status" -ne 0 ]
|
|
518
|
+
[[ "$output" == *"unknown option"* && "$output" == *"--force"* ]] || return 1
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
@test "a rubric missing from config is refused with a reason, not silently" {
|
|
522
|
+
# Reachable by config drift: the visibility map still names a rubric that
|
|
523
|
+
# librarian.lesson_judging.rubrics no longer defines. State stays safe —
|
|
524
|
+
# nothing is written — but a user sees only an exit code.
|
|
525
|
+
_seed_confirmed "cfg01" "org"
|
|
526
|
+
_LIBRARIAN_CONFIG=$(printf '%s' "$_LIBRARIAN_CONFIG" | jq 'del(.librarian.lesson_judging.rubrics)')
|
|
527
|
+
|
|
528
|
+
run --separate-stderr librarian_lesson_judge "$PROJECT_KEY" "cfg01" "$(_verdicts_pass)"
|
|
529
|
+
[ "$status" -eq 1 ]
|
|
530
|
+
[ "$output" = "" ]
|
|
531
|
+
[[ "$stderr" == *"rubric"* ]] || return 1
|
|
532
|
+
[ "$(_status_of cfg01)" = "confirmed" ]
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
@test "lessons list --confirmed --json emits rows carrying visibility" {
|
|
536
|
+
_seed_confirmed "js01" "public"
|
|
537
|
+
_seed_confirmed "js02" "org"
|
|
538
|
+
run librarian_cli lessons list --confirmed --json "$PROJECT_REPO"
|
|
539
|
+
[ "$status" -eq 0 ]
|
|
540
|
+
[ "$(printf '%s' "$output" | jq 'length')" -eq 2 ]
|
|
541
|
+
[ "$(printf '%s' "$output" | jq -r '[.[] | select(.visibility == "public")] | length')" -eq 1 ]
|
|
542
|
+
[ "$(printf '%s' "$output" | jq -r '.[0] | has("id")')" = "true" ]
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
@test "lessons list --json on an empty set emits an empty array, not prose" {
|
|
546
|
+
# The skill parses this; a human-readable empty-state message would break it.
|
|
547
|
+
run librarian_cli lessons list --confirmed --json "$PROJECT_REPO"
|
|
548
|
+
[ "$status" -eq 0 ]
|
|
549
|
+
[ "$output" = "[]" ]
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
@test "bare lessons list is unchanged by the --json addition" {
|
|
553
|
+
run librarian_cli lessons list "$PROJECT_REPO"
|
|
554
|
+
[ "$status" -eq 0 ]
|
|
555
|
+
[ "$output" = "No pending lessons." ]
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
PUBLIC_RUBRIC='{"id":"lesson-promotion-public","criteria":[
|
|
559
|
+
{"name":"grounding","weight":0.32,"min_pass":0.7},
|
|
560
|
+
{"name":"scope_accuracy","weight":0.24,"min_pass":0.7},
|
|
561
|
+
{"name":"generality","weight":0.14,"min_pass":0.6},
|
|
562
|
+
{"name":"disclosure","weight":0.30,"min_pass":0.9}],
|
|
563
|
+
"score_threshold":0.75,"gate_policy":"majority"}'
|
|
564
|
+
|
|
565
|
+
@test "lesson aggregate weights criteria when scores are present" {
|
|
566
|
+
local w m verdicts
|
|
567
|
+
verdicts='[
|
|
568
|
+
{"judge_type":"standard","score":0.5,"passed":true,"criterion_scores":{"grounding":1.0,"scope_accuracy":1.0,"generality":1.0,"disclosure":0.0}},
|
|
569
|
+
{"judge_type":"adversarial","score":0.5,"passed":true,"criterion_scores":{"grounding":1.0,"scope_accuracy":1.0,"generality":1.0,"disclosure":0.0}}
|
|
570
|
+
]'
|
|
571
|
+
w=$(librarian_lesson_aggregate "$verdicts" "$PUBLIC_RUBRIC")
|
|
572
|
+
m=$(librarian_lesson_aggregate "$verdicts")
|
|
573
|
+
# weighted: 0.32+0.24+0.14 = 0.70 over a weight sum of 1.0. Plain mean: 0.5.
|
|
574
|
+
awk -v a="$w" -v b="$m" 'BEGIN { exit !(a != b) }' || return 1
|
|
575
|
+
awk -v a="$w" 'BEGIN { exit !(a > 0.699 && a < 0.701) }'
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
@test "lesson aggregate degrades to the plain mean without criterion scores" {
|
|
579
|
+
local out
|
|
580
|
+
out=$(librarian_lesson_aggregate \
|
|
581
|
+
'[{"judge_type":"standard","score":0.8,"passed":true},
|
|
582
|
+
{"judge_type":"adversarial","score":0.6,"passed":true}]' "$PUBLIC_RUBRIC")
|
|
583
|
+
awk -v a="$out" 'BEGIN { exit !(a > 0.699 && a < 0.701) }'
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
@test "lesson aggregate still returns 1 on an empty panel" {
|
|
587
|
+
run librarian_lesson_aggregate '[]' "$PUBLIC_RUBRIC"
|
|
588
|
+
[ "$status" -eq 1 ]
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
@test "lesson aggregate normalizes weights that do not sum to 1.0" {
|
|
592
|
+
# librarian_lesson_rubric_get validates NOTHING, so a mis-summed rubric
|
|
593
|
+
# reaches this function where tribunal's validator would have refused it.
|
|
594
|
+
local out
|
|
595
|
+
out=$(librarian_lesson_aggregate \
|
|
596
|
+
'[{"judge_type":"standard","score":0.1,"passed":true,"criterion_scores":{"grounding":1.0,"disclosure":0.0}}]' \
|
|
597
|
+
'{"criteria":[{"name":"grounding","weight":1.8,"min_pass":0.7},{"name":"disclosure","weight":0.2,"min_pass":0.9}]}')
|
|
598
|
+
awk -v a="$out" 'BEGIN { exit !(a > 0.899 && a < 0.901) }'
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
@test "a low disclosure score blocks a public lesson under majority" {
|
|
602
|
+
# The reason this whole thread exists: disclosure's 0.9 floor blocks even
|
|
603
|
+
# though both judges passed and the aggregate clears 0.75.
|
|
604
|
+
local out
|
|
605
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
606
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9,"disclosure":0.4}},
|
|
607
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9,"disclosure":0.4}}
|
|
608
|
+
]' "0.78" "0.75" "$PUBLIC_RUBRIC")
|
|
609
|
+
printf '%s' "$out" | jq -e '.passed == false' >/dev/null || return 1
|
|
610
|
+
printf '%s' "$out" | jq -e '.reason == "criterion_floor"' >/dev/null || return 1
|
|
611
|
+
printf '%s' "$out" | jq -e '.failed_criterion == "disclosure"' >/dev/null
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
@test "a generality score above its floor does not block a public lesson" {
|
|
615
|
+
# Pins generality's 0.6 floor: 0.65 clears it. This is NOT a difference from
|
|
616
|
+
# the old `unanimous` policy — both judges pass, so unanimous accepted it
|
|
617
|
+
# too, and at a two-judge panel no fixture can tell the policies apart.
|
|
618
|
+
# See ecosystem-j74.
|
|
619
|
+
local out
|
|
620
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
621
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.65,"disclosure":0.95}},
|
|
622
|
+
{"judge_type":"adversarial","score":0.85,"passed":true,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.65,"disclosure":0.95}}
|
|
623
|
+
]' "0.88" "0.75" "$PUBLIC_RUBRIC")
|
|
624
|
+
printf '%s' "$out" | jq -e '.passed == true' >/dev/null
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
@test "lesson gate: a verdict with no criterion_scores key at all never blocks" {
|
|
628
|
+
# Every verdict emitted before judges shipped criterion_scores. Note this
|
|
629
|
+
# case is caught by the OUTER type guard and never reaches has() — it does
|
|
630
|
+
# NOT pin the per-criterion absence guard. The next test does that.
|
|
631
|
+
local out
|
|
632
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
633
|
+
{"judge_type":"standard","score":0.9,"passed":true},
|
|
634
|
+
{"judge_type":"adversarial","score":0.85,"passed":true}
|
|
635
|
+
]' "0.88" "0.75" "$PUBLIC_RUBRIC")
|
|
636
|
+
printf '%s' "$out" | jq -e '.passed == true and .reason == "gate_passed"' >/dev/null
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
@test "lesson gate: scores present but one floored criterion omitted does not block" {
|
|
640
|
+
# THE test that pins the has() guard. These verdicts DO carry
|
|
641
|
+
# criterion_scores, so they survive the outer type guard and reach the
|
|
642
|
+
# per-criterion lookup — but `disclosure`, whose floor is 0.9, is absent.
|
|
643
|
+
# Substituting `// 0` for has() makes disclosure read as 0.0 and blocks.
|
|
644
|
+
#
|
|
645
|
+
# Written as its own test because the case above cannot fail when has() is
|
|
646
|
+
# deleted: its fixture is filtered one layer earlier. Two different absences
|
|
647
|
+
# sharing one test is how an outer guard silently stands in for an inner one
|
|
648
|
+
# — this project has hit that shape eight times.
|
|
649
|
+
local out
|
|
650
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
651
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9}},
|
|
652
|
+
{"judge_type":"adversarial","score":0.85,"passed":true,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.85}}
|
|
653
|
+
]' "0.88" "0.75" "$PUBLIC_RUBRIC")
|
|
654
|
+
printf '%s' "$out" | jq -e '.passed == true' >/dev/null || return 1
|
|
655
|
+
printf '%s' "$out" | jq -e '.reason == "gate_passed"' >/dev/null
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
@test "lesson gate: a criterion scored zero does block" {
|
|
659
|
+
local out
|
|
660
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
661
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"disclosure":0.0}},
|
|
662
|
+
{"judge_type":"adversarial","score":0.85,"passed":true,"criterion_scores":{"disclosure":0.0}}
|
|
663
|
+
]' "0.88" "0.75" "$PUBLIC_RUBRIC")
|
|
664
|
+
printf '%s' "$out" | jq -e '.reason == "criterion_floor"' >/dev/null || return 1
|
|
665
|
+
printf '%s' "$out" | jq -e '.failed_criterion == "disclosure"' >/dev/null
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
@test "lesson gate: below_threshold still wins over criterion_floor" {
|
|
669
|
+
local out
|
|
670
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
671
|
+
{"judge_type":"standard","score":0.2,"passed":true,"criterion_scores":{"disclosure":0.1}},
|
|
672
|
+
{"judge_type":"adversarial","score":0.2,"passed":true,"criterion_scores":{"disclosure":0.1}}
|
|
673
|
+
]' "0.20" "0.75" "$PUBLIC_RUBRIC")
|
|
674
|
+
printf '%s' "$out" | jq -e '.reason == "below_threshold"' >/dev/null
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
@test "lesson gate: jury policy still wins over criterion_floor" {
|
|
678
|
+
local out
|
|
679
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
680
|
+
{"judge_type":"standard","score":0.9,"passed":false,"criterion_scores":{"disclosure":0.1}},
|
|
681
|
+
{"judge_type":"adversarial","score":0.9,"passed":false,"criterion_scores":{"disclosure":0.1}}
|
|
682
|
+
]' "0.90" "0.75" "$PUBLIC_RUBRIC")
|
|
683
|
+
printf '%s' "$out" | jq -e '.reason == "jury_not_majority"' >/dev/null
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
@test "lesson gate: works with no rubric at all" {
|
|
687
|
+
local out
|
|
688
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
689
|
+
{"judge_type":"standard","score":0.9,"passed":true},
|
|
690
|
+
{"judge_type":"adversarial","score":0.85,"passed":true}
|
|
691
|
+
]' "0.88" "0.75")
|
|
692
|
+
printf '%s' "$out" | jq -e '.passed == true' >/dev/null
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
@test "lesson gate: a hyphenated criterion name gates correctly" {
|
|
696
|
+
local out
|
|
697
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
698
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"scope-accuracy":0.1}},
|
|
699
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"scope-accuracy":0.1}}
|
|
700
|
+
]' "0.90" "0.75" '{"criteria":[{"name":"scope-accuracy","weight":1.0,"min_pass":0.7}]}')
|
|
701
|
+
printf '%s' "$out" | jq -e '.failed_criterion == "scope-accuracy"' >/dev/null
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
@test "an end-to-end public judge is rejected by a criterion floor, not just the unit gate" {
|
|
705
|
+
# Pins Step 5's threading, not just librarian_lesson_gate in isolation: both
|
|
706
|
+
# judges pass unanimously and the mean (0.875) clears the 0.75 threshold, so
|
|
707
|
+
# without the rubric reaching the gate through librarian_lesson_judge this
|
|
708
|
+
# would approve. disclosure's 0.9 floor is what actually blocks it.
|
|
709
|
+
_seed_confirmed "floor01" "public"
|
|
710
|
+
local verdicts='[
|
|
711
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9,"disclosure":0.4}},
|
|
712
|
+
{"judge_type":"adversarial","score":0.85,"passed":true,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.85,"disclosure":0.4}}
|
|
713
|
+
]'
|
|
714
|
+
run librarian_lesson_judge "$PROJECT_KEY" "floor01" "$verdicts"
|
|
715
|
+
[ "$status" -eq 0 ]
|
|
716
|
+
[ "$(_status_of floor01)" = "rejected" ]
|
|
717
|
+
|
|
718
|
+
local v
|
|
719
|
+
v=$(jq -c '.verdict' "$(librarian_lessons_dir "$PROJECT_KEY")/proposals/floor01.json")
|
|
720
|
+
[ "$(printf '%s' "$v" | jq -r '.reason')" = "criterion_floor" ]
|
|
721
|
+
}
|
|
722
|
+
|
|
723
|
+
@test "the public rubric no longer relies on unanimous" {
|
|
724
|
+
# unanimous was a stand-in for the disclosure floor and never worked: at the
|
|
725
|
+
# configured panel of 2, unanimous and majority are the same function for
|
|
726
|
+
# every possible pass count. ecosystem-j74.
|
|
727
|
+
local policy
|
|
728
|
+
policy=$(jq -r '.librarian.lesson_judging.rubrics[]
|
|
729
|
+
| select(.id == "lesson-promotion-public") | .gate_policy' \
|
|
730
|
+
"${REPO_ROOT}/plugins/librarian/config.json")
|
|
731
|
+
[ "$policy" = "majority" ]
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
@test "the public rubric keeps disclosure's floor at 0.9" {
|
|
735
|
+
# The floor is now the ONLY thing making public stricter than org. If this
|
|
736
|
+
# drops, the public tier silently loses its protection entirely.
|
|
737
|
+
local floor
|
|
738
|
+
floor=$(jq -r '.librarian.lesson_judging.rubrics[]
|
|
739
|
+
| select(.id == "lesson-promotion-public")
|
|
740
|
+
| .criteria[] | select(.name == "disclosure") | .min_pass' \
|
|
741
|
+
"${REPO_ROOT}/plugins/librarian/config.json")
|
|
742
|
+
[ "$floor" = "0.9" ]
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
@test "org and public rubrics differ by more than their gate policy" {
|
|
746
|
+
# Both are `majority` now. If the criteria ever converge too, the two tiers
|
|
747
|
+
# become indistinguishable and the public tier is inert again — the exact
|
|
748
|
+
# shape of j74.
|
|
749
|
+
local org_crit pub_crit
|
|
750
|
+
org_crit=$(jq -c '[.librarian.lesson_judging.rubrics[]
|
|
751
|
+
| select(.id == "lesson-promotion") | .criteria[].name] | sort' \
|
|
752
|
+
"${REPO_ROOT}/plugins/librarian/config.json")
|
|
753
|
+
pub_crit=$(jq -c '[.librarian.lesson_judging.rubrics[]
|
|
754
|
+
| select(.id == "lesson-promotion-public") | .criteria[].name] | sort' \
|
|
755
|
+
"${REPO_ROOT}/plugins/librarian/config.json")
|
|
756
|
+
[ "$org_crit" != "$pub_crit" ]
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
@test "the librarian walk names every public rubric criterion to its judges" {
|
|
760
|
+
# A bare grep for "criterion_scores" passes even when the walk hands judges
|
|
761
|
+
# the wrong key names — which silently disables the floor, since no key
|
|
762
|
+
# matches the rubric. Assert the actual names, read from config.
|
|
763
|
+
local skill name
|
|
764
|
+
skill="${REPO_ROOT}/plugins/librarian/skills/librarian/SKILL.md"
|
|
765
|
+
grep -q 'criterion_scores' "$skill" || return 1
|
|
766
|
+
|
|
767
|
+
while IFS= read -r name; do
|
|
768
|
+
grep -q "$name" "$skill" || return 1
|
|
769
|
+
done < <(jq -r '.librarian.lesson_judging.rubrics[]
|
|
770
|
+
| select(.id == "lesson-promotion-public") | .criteria[].name' \
|
|
771
|
+
"${REPO_ROOT}/plugins/librarian/config.json")
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
@test "a floor rejection records which criterion failed" {
|
|
775
|
+
# "reason": "criterion_floor" without the criterion name is no more
|
|
776
|
+
# actionable than "blocked" — the whole argument for a distinct reason
|
|
777
|
+
# was naming the thing that failed.
|
|
778
|
+
_seed_confirmed "floorname01" "public"
|
|
779
|
+
local verdicts='[
|
|
780
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9,"disclosure":0.4}},
|
|
781
|
+
{"judge_type":"adversarial","score":0.85,"passed":true,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.85,"disclosure":0.4}}
|
|
782
|
+
]'
|
|
783
|
+
run librarian_lesson_judge "$PROJECT_KEY" "floorname01" "$verdicts"
|
|
784
|
+
[ "$status" -eq 0 ] || return 1
|
|
785
|
+
[ "$(_status_of floorname01)" = "rejected" ] || return 1
|
|
786
|
+
|
|
787
|
+
local path
|
|
788
|
+
path="$(librarian_lessons_dir "$PROJECT_KEY")/proposals/floorname01.json"
|
|
789
|
+
jq -e '.verdict.reason == "criterion_floor"' "$path" >/dev/null || return 1
|
|
790
|
+
jq -e '.verdict.failed_criterion == "disclosure"' "$path" >/dev/null
|
|
791
|
+
}
|
|
792
|
+
|
|
793
|
+
@test "an approved verdict carries no failed_criterion key" {
|
|
794
|
+
# Absent rather than null: a key that is always present but usually empty
|
|
795
|
+
# invites `// ""` at the read site, which is how this pipeline has
|
|
796
|
+
# repeatedly lost the absent-vs-empty distinction.
|
|
797
|
+
_seed_confirmed "floorname02" "public"
|
|
798
|
+
local verdicts='[
|
|
799
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.9,"disclosure":0.95}},
|
|
800
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"grounding":0.9,"scope_accuracy":0.9,"generality":0.9,"disclosure":0.95}}
|
|
801
|
+
]'
|
|
802
|
+
run librarian_lesson_judge "$PROJECT_KEY" "floorname02" "$verdicts"
|
|
803
|
+
[ "$status" -eq 0 ] || return 1
|
|
804
|
+
|
|
805
|
+
local path
|
|
806
|
+
path="$(librarian_lessons_dir "$PROJECT_KEY")/proposals/floorname02.json"
|
|
807
|
+
jq -e '.verdict | has("failed_criterion") | not' "$path" >/dev/null
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
@test "a floored criterion no judge scored is UNJUDGED, not approved" {
|
|
811
|
+
# C2: librarian dispatches tribunal's judge agents, whose shipped example
|
|
812
|
+
# keys are tribunal's rubric. A judge following that example emits keys
|
|
813
|
+
# matching nothing here, disclosure never runs, and the lesson publishes.
|
|
814
|
+
# Refusing is right: the candidate stays confirmed and is retried.
|
|
815
|
+
_seed_confirmed "cov01" "public"
|
|
816
|
+
local verdicts='[
|
|
817
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"correctness":0.9,"completeness":0.9,"safety":0.2,"clarity":0.9}},
|
|
818
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"correctness":0.9,"completeness":0.9,"safety":0.2,"clarity":0.9}}
|
|
819
|
+
]'
|
|
820
|
+
run --separate-stderr librarian_lesson_judge "$PROJECT_KEY" "cov01" "$verdicts"
|
|
821
|
+
[ "$status" -eq 2 ] || return 1
|
|
822
|
+
[ "$(_status_of cov01)" = "confirmed" ] || return 1
|
|
823
|
+
local re='disclosure'
|
|
824
|
+
[[ "$stderr" =~ $re ]]
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
@test "a panel that scored every floored criterion but too little weight is UNJUDGED" {
|
|
828
|
+
# Isolates the coverage fraction from the unscored-floors check above.
|
|
829
|
+
#
|
|
830
|
+
# Both SHIPPED rubrics floor every one of their criteria, so with them a
|
|
831
|
+
# floor-complete panel always covers 1.0 of the weight and this branch is
|
|
832
|
+
# unreachable: a fixture built on the org rubric would be caught by the
|
|
833
|
+
# unscored check and would still pass with the coverage guard deleted —
|
|
834
|
+
# the outer-guard-stands-in-for-the-inner shape this repo keeps hitting.
|
|
835
|
+
# ADR-004 lets a user override `rubrics`, so an unfloored criterion is a
|
|
836
|
+
# real configuration, and it is the only one that reaches this guard.
|
|
837
|
+
_LIBRARIAN_CONFIG=$(printf '%s' "$_LIBRARIAN_CONFIG" | jq '
|
|
838
|
+
.librarian.lesson_judging.rubrics = [
|
|
839
|
+
{ id: "lesson-promotion",
|
|
840
|
+
criteria: [ { name: "grounding", weight: 0.2, min_pass: 0.7 },
|
|
841
|
+
{ name: "depth", weight: 0.8 } ],
|
|
842
|
+
score_threshold: 0.75,
|
|
843
|
+
judge_types: ["standard", "adversarial"],
|
|
844
|
+
gate_policy: "majority" } ]')
|
|
845
|
+
|
|
846
|
+
_seed_confirmed "cov02" "org"
|
|
847
|
+
# grounding is the only floored criterion and both judges scored it, so the
|
|
848
|
+
# unscored check passes. Coverage is 0.2 of 1.0 — well under the 0.6 floor.
|
|
849
|
+
local verdicts='[
|
|
850
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.9}},
|
|
851
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"grounding":0.9}}
|
|
852
|
+
]'
|
|
853
|
+
run --separate-stderr librarian_lesson_judge "$PROJECT_KEY" "cov02" "$verdicts"
|
|
854
|
+
[ "$status" -eq 2 ] || return 1
|
|
855
|
+
[ "$(_status_of cov02)" = "confirmed" ] || return 1
|
|
856
|
+
local re='rubric weight'
|
|
857
|
+
[[ "$stderr" =~ $re ]]
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
@test "a below_threshold block still names the criterion that failed its floor" {
|
|
861
|
+
# The worst disclosure failures drag the aggregate under threshold, so
|
|
862
|
+
# before this they landed as below_threshold with no failed_criterion —
|
|
863
|
+
# the diagnostic absent exactly where it matters most.
|
|
864
|
+
local out
|
|
865
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
866
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.95,"disclosure":0.0}},
|
|
867
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.95,"disclosure":0.0}}
|
|
868
|
+
]' "0.665" "0.75" "$PUBLIC_RUBRIC")
|
|
869
|
+
printf '%s' "$out" | jq -e '.reason == "below_threshold"' >/dev/null || return 1
|
|
870
|
+
printf '%s' "$out" | jq -e '.failed_criterion == "disclosure"' >/dev/null
|
|
871
|
+
}
|
|
872
|
+
|
|
873
|
+
@test "a lesson floor uses the lowest judge score, not the mean" {
|
|
874
|
+
local out
|
|
875
|
+
out=$(librarian_lesson_gate "majority" '[
|
|
876
|
+
{"judge_type":"standard","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.95,"disclosure":1.0}},
|
|
877
|
+
{"judge_type":"adversarial","score":0.9,"passed":true,"criterion_scores":{"grounding":0.95,"scope_accuracy":0.95,"generality":0.95,"disclosure":0.85}}
|
|
878
|
+
]' "0.93" "0.75" "$PUBLIC_RUBRIC")
|
|
879
|
+
# The discriminating pair: the mean of 1.0/0.85 is 0.925 and CLEARS the 0.9
|
|
880
|
+
# floor, so a mean-based floor passes this panel. The min, 0.85, does not.
|
|
881
|
+
# 0.99/0.8 would not discriminate — that mean is 0.895, already under 0.9.
|
|
882
|
+
printf '%s' "$out" | jq -e '.reason == "criterion_floor"' >/dev/null || return 1
|
|
883
|
+
printf '%s' "$out" | jq -e '.failed_criterion == "disclosure"' >/dev/null
|
|
884
|
+
}
|