@chrono-meta/fh-gate 3.1.4 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/registry/agent_cards.json +1 -1
- package/.claude/rules/fh_4axis_gate.md +26 -1
- package/.claude-plugin/marketplace.json +3 -3
- package/AGENTS.md +8 -2
- package/CATALOG.md +4 -4
- package/CHEATSHEET.md +1 -1
- package/CLAUDE.md +54 -10
- package/README.ja.md +3 -3
- package/README.ko.md +3 -3
- package/README.md +3 -3
- package/README.zh.md +3 -3
- package/docs/OUTPUT_EVIDENCE.md +2 -2
- package/docs/STANDARDS_ALIGNMENT.md +1 -1
- package/docs/codex-compat.md +1 -1
- package/knowledge/shared/harness-core/agents_md_runtime_details.md +2 -2
- package/knowledge/shared/harness-core/claude_md_gate_details.md +26 -0
- package/knowledge/shared/harness-core/fh_three_layer_canon.md +1 -1
- package/knowledge/shared/harness-core/field_verdict_crossfamily_gate.md +76 -3
- package/knowledge/shared/harness-core/governance_engineering_definition.md +89 -0
- package/knowledge/shared/harness-core/iso_ai_standards_crosswalk.md +1 -1
- package/knowledge/shared/harness-core/skill_quality_rubric.md +1 -1
- package/knowledge/shared/learnings/subagent_invocations_log.yaml +99 -0
- package/knowledge/shared/rules/auto_project_mapping.md +1 -1
- package/knowledge/shared/rules/modes_and_value.md +2 -2
- package/package.json +12 -1
- package/plugins/fh-commons/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-commons/README.md +38 -0
- package/plugins/fh-commons/skills/preprep/SKILL.md +23 -0
- package/plugins/fh-commons/skills/preprep/fixtures/font_revert_probe.py +92 -0
- package/plugins/fh-commons/skills/preprep/lane_font.py +462 -0
- package/plugins/fh-commons/skills/preprep/preprep.py +16 -1
- package/plugins/fh-commons/skills/preprep/surfaces.example.yaml +17 -0
- package/plugins/fh-commons/skills/preprep/test_lane_font.py +452 -0
- package/plugins/fh-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-meta/CHANGELOG.md +164 -0
- package/plugins/fh-meta/skills/agent-composer/SKILL.md +2 -2
- package/plugins/fh-meta/skills/auto-decorrelation/SKILL.md +40 -0
- package/plugins/fh-meta/skills/frontier-digest/SKILL.md +1 -1
- package/plugins/fh-meta/skills/frontier-digest/SKILL_detail.md +43 -5
- package/plugins/fh-meta/skills/{hub-cc-pr-reviewer → harness-pr-reviewer}/SKILL.md +75 -3
- package/plugins/fh-meta/skills/{hub-cc-pr-reviewer → harness-pr-reviewer}/SKILL_detail.md +2 -2
- package/plugins/fh-meta/skills/harvest-loop/SKILL_detail.md +2 -2
- package/plugins/fh-meta/skills/install-doctor/SKILL.md +1 -1
- package/plugins/fh-meta/skills/install-wizard/SKILL.md +1 -1
- package/plugins/fh-meta/skills/meta-prompt-builder/SKILL.md +1 -1
- package/plugins/fh-meta/skills/pipeline-conductor/SKILL.md +1 -1
- package/plugins/fh-meta/skills/plugin-recommender/SKILL.md +1 -1
- package/plugins/fh-meta/skills/sim-conductor/SKILL.md +1 -1
- package/plugins/fh-qp/.claude-plugin/plugin.json +1 -1
- package/scripts/doc_claim_triad_scan.py +303 -0
- package/scripts/finding_fleet.sh +558 -0
- package/scripts/finding_pipeline.sh +572 -0
- package/scripts/finding_verifier.sh +173 -0
- package/scripts/finding_verify.py +468 -0
- package/scripts/frontier_digest_autopilot.sh +3 -3
- package/scripts/gate_pathspec_check.sh +1 -0
- package/scripts/gate_shape_scan.sh +120 -0
- package/scripts/selfcheck.sh +50 -0
- package/scripts/test_doc_claim_triad_lanes.sh +124 -0
- package/scripts/test_finding_pipeline_lanes.sh +2011 -0
- package/scripts/test_gate_shape_scan_lanes.sh +47 -0
- package/scripts/test_heavy_classifier_lanes.sh +13 -3
- package/scripts/test_marker_crossfamily_lanes.sh +75 -3
- package/scripts/test_marker_standpoint_lanes.sh +31 -6
- package/scripts/test_preprep_font_lanes.sh +87 -0
- package/templates/.git-hooks/pre-commit +158 -11
- package/templates/PRE-PUBLISH-CHECKLIST.md +29 -0
- package/templates/local_fh_context.md +1 -1
- package/templates/regression_guard.sh +1 -1
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# finding_verifier.sh — the wrapper finding_verify.py's --verifier contract asks for, and which did
|
|
3
|
+
# not exist. Reads findings JSONL on stdin, writes verdict JSONL on stdout:
|
|
4
|
+
# {"id","verdict":"confirmed|false-positive|needs-debate","why"}
|
|
5
|
+
# With --audit it answers the drop-audit protocol instead:
|
|
6
|
+
# {"id","verdict":"correct-drop|wrong-drop|uncertain","why"}
|
|
7
|
+
#
|
|
8
|
+
# WHY THIS EXISTS: finding_verify.py runs whatever shell command you hand it. Until this file, the
|
|
9
|
+
# only commands that satisfied its protocol were the hardcoded stubs inside the lane suite, so the
|
|
10
|
+
# pipeline could only ever report status=UNVERIFIED (rc=3) against real findings. A pipeline whose
|
|
11
|
+
# only live path is its own fixture is not wired (CLAUDE.md §built-but-not-wired).
|
|
12
|
+
#
|
|
13
|
+
# 🟥 THE VERIFIER MUST SEE THE CODE, NOT ONLY THE CLAIM. A verdict on "is this finding real" that is
|
|
14
|
+
# reached from the claim text alone measures plausibility, not truth — the model agrees with a
|
|
15
|
+
# well-written wrong claim. So --target is REQUIRED and its content is put in front of the model.
|
|
16
|
+
#
|
|
17
|
+
# EXIT 0 answered · 2 usage · 3 the CLI failed or produced no parsable verdict (caller degrades;
|
|
18
|
+
# finding_verify.py turns an empty answer into `unverified`, never into a silent pass)
|
|
19
|
+
set -uo pipefail
|
|
20
|
+
|
|
21
|
+
CODEX="${FH_CODEX_BIN:-$(command -v codex 2>/dev/null || echo "$HOME/.npm-global/bin/codex")}"
|
|
22
|
+
AGY="${FH_AGY_BIN:-$(command -v agy 2>/dev/null || echo "$HOME/.local/bin/agy")}"
|
|
23
|
+
|
|
24
|
+
FAMILY=""; TARGET=""; MODE="verify"; KEEP=""
|
|
25
|
+
# 호출자(파이프라인)가 env 로 보존 경로를 줄 수 있다 — 검증 패스의 CLI stderr(토큰 회계)가 여기서 산다.
|
|
26
|
+
# --keep 플래그가 뒤에서 덮으므로 env 는 «폴백» 이지 강제가 아니다. (2026-09-10: 0바이트 ≠ 0 토큰)
|
|
27
|
+
KEEP="${FH_VERIFIER_KEEP:-}"
|
|
28
|
+
usage() { echo "usage: finding_verifier.sh --family codex|gemini --target <file> [--audit] [--keep <dir>]" >&2; exit 2; }
|
|
29
|
+
# `shift 2` with only one word left FAILS and consumes nothing, so the loop spins forever. Under
|
|
30
|
+
# `set -uo pipefail` (no errexit) nothing stops it. codex reproduced a hang on a trailing --family.
|
|
31
|
+
need() { [ $# -ge 2 ] || { echo "finding_verifier: $1 needs a value" >&2; exit 2; }; }
|
|
32
|
+
while [ $# -gt 0 ]; do
|
|
33
|
+
case "$1" in
|
|
34
|
+
--family) need "$@"; FAMILY="$2"; shift 2 ;;
|
|
35
|
+
--target) need "$@"; TARGET="$2"; shift 2 ;;
|
|
36
|
+
--audit) MODE="audit"; shift ;;
|
|
37
|
+
--keep) need "$@"; KEEP="$2"; shift 2 ;;
|
|
38
|
+
*) usage ;;
|
|
39
|
+
esac
|
|
40
|
+
done
|
|
41
|
+
[ -n "$FAMILY" ] || usage
|
|
42
|
+
# `-f` is a TYPE test, not a readability test: an unreadable regular file passes it, the later `cat`
|
|
43
|
+
# fails silently, and the model then judges claims WITHOUT the source — which is the one thing the
|
|
44
|
+
# header above says must never happen. Test readability, and check the read itself below.
|
|
45
|
+
[ -n "$TARGET" ] && [ -f "$TARGET" ] && [ -r "$TARGET" ] \
|
|
46
|
+
|| { echo "finding_verifier: --target must name a readable file" >&2; exit 2; }
|
|
47
|
+
# 🟥 심링크 거부 — 이 파일의 내용은 «외부 모델로 전송»된다. 체크아웃 안의 링크가 바깥
|
|
48
|
+
# 자격증명을 가리키면, 리뷰하려던 코드 대신 그 비밀이 프롬프트가 된다(residency 위반).
|
|
49
|
+
# cross-family security review 2026-09-09.
|
|
50
|
+
if [ -L "$TARGET" ]; then
|
|
51
|
+
echo "finding_verifier: --target is a symlink — refusing (content is SENT to an external model)" >&2
|
|
52
|
+
exit 2
|
|
53
|
+
fi
|
|
54
|
+
# 프롬프트 파일은 소스 전문을 담는다 — 0644 로 남기지 않는다.
|
|
55
|
+
umask 077
|
|
56
|
+
|
|
57
|
+
WORK="$(mktemp -d 2>/dev/null)" || { echo "finding_verifier: mktemp failed" >&2; exit 3; }
|
|
58
|
+
# 🟥 R2 #7: 검증과 감사가 같은 keep 을 쓰면 감사가 검증 err.txt(토큰 회계)를 덮는다 — 드롭이 있는 런에서
|
|
59
|
+
# «바로 그 런» 의 회계가 또 버려진다. MODE 별 하위 디렉터리로 가른다. R2 #1: 심링크 잎은 복사 전에 거부.
|
|
60
|
+
# 🟥 R3 #3: the root is checked BEFORE the mode dir is appended — appended first, a symlink root became
|
|
61
|
+
# an ancestor and passed both the dir check and the leaf check (path probe: INPUT_IS_SYMLINK → GUARD_ALLOWS).
|
|
62
|
+
# R4 #2: `link/` makes -L false (the trailing slash resolves through the link) — strip separators first, keep "/" intact.
|
|
63
|
+
# R6 #5: `link/.` and `link//.` also resolve through the link — strip trailing "/." and "/" until stable.
|
|
64
|
+
while [ "${#KEEP}" -gt 1 ]; do
|
|
65
|
+
case "$KEEP" in
|
|
66
|
+
*/.) KEEP="${KEEP%/.}" ;;
|
|
67
|
+
*/) KEEP="${KEEP%/}" ;;
|
|
68
|
+
*) break ;;
|
|
69
|
+
esac
|
|
70
|
+
done
|
|
71
|
+
if [ -n "$KEEP" ] && [ -L "$KEEP" ]; then echo "finding_verifier: --keep root is a symlink: $KEEP — refusing" >&2; exit 2; fi
|
|
72
|
+
if [ -n "$KEEP" ] && [ -z "${FH_VERIFIER_KEEP_FLAT:-}" ]; then KEEP="$KEEP/keep_$MODE"; fi
|
|
73
|
+
cleanup() {
|
|
74
|
+
if [ -n "$KEEP" ]; then
|
|
75
|
+
[ -L "$KEEP" ] && { echo "finding_verifier: refusing to keep through a symlink: $KEEP" >&2; rm -rf "$WORK"; return; }
|
|
76
|
+
mkdir -p "$KEEP" 2>/dev/null
|
|
77
|
+
for _f in "$WORK"/*; do
|
|
78
|
+
_dst="$KEEP/$(basename "$_f")"
|
|
79
|
+
[ -L "$_dst" ] && { echo "finding_verifier: refusing to keep through a symlink: $_dst" >&2; continue; }
|
|
80
|
+
cp "$_f" "$_dst" 2>/dev/null
|
|
81
|
+
done
|
|
82
|
+
fi
|
|
83
|
+
rm -rf "$WORK"
|
|
84
|
+
}
|
|
85
|
+
trap cleanup EXIT
|
|
86
|
+
|
|
87
|
+
cat > "$WORK/findings.jsonl"
|
|
88
|
+
if [ ! -s "$WORK/findings.jsonl" ]; then exit 0; fi # nothing asked, nothing to answer
|
|
89
|
+
|
|
90
|
+
if [ "$MODE" = "verify" ]; then
|
|
91
|
+
HEAD='You are an independent verifier from a different model family than the reviewer who wrote the
|
|
92
|
+
claims below. For EACH claim decide whether it is real, judged against the file itself.
|
|
93
|
+
|
|
94
|
+
Output ONLY JSON Lines, one object per claim id, nothing else — no prose, no code fences:
|
|
95
|
+
{"id":"<the id verbatim>","verdict":"confirmed|false-positive|needs-debate","why":"<one line, cite the line number you checked>"}
|
|
96
|
+
|
|
97
|
+
confirmed = you read the cited location and the described failure can actually occur there.
|
|
98
|
+
false-positive = the location does not say what the claim says, or the failure cannot occur.
|
|
99
|
+
needs-debate = it depends on a caller or config you cannot see from this file alone.
|
|
100
|
+
|
|
101
|
+
Answer for EVERY id. Do not invent ids. Do not add findings of your own.'
|
|
102
|
+
else
|
|
103
|
+
HEAD='You are auditing DELETIONS made by a different reviewer. Each record below is a claim that was
|
|
104
|
+
dropped as a false positive. For EACH one decide whether dropping it was right, judged against the file.
|
|
105
|
+
|
|
106
|
+
Output ONLY JSON Lines, one object per id, nothing else — no prose, no code fences:
|
|
107
|
+
{"id":"<the id verbatim>","verdict":"correct-drop|wrong-drop|uncertain","why":"<one line, cite the line number you checked>"}
|
|
108
|
+
|
|
109
|
+
correct-drop = the claim really was wrong; deleting it was right.
|
|
110
|
+
wrong-drop = the claim was true and was deleted in error (it will be reinstated).
|
|
111
|
+
uncertain = you cannot tell from this file alone.
|
|
112
|
+
|
|
113
|
+
Answer for EVERY id. A drop you cannot justify is not "correct" by default.'
|
|
114
|
+
fi
|
|
115
|
+
|
|
116
|
+
{ printf '%s\n\n===== CLAIMS =====\n' "$HEAD"; cat "$WORK/findings.jsonl"
|
|
117
|
+
printf '\n===== FILE: %s =====\n' "$(basename "$TARGET")"; cat "$TARGET"; } > "$WORK/prompt.txt" \
|
|
118
|
+
|| { echo "finding_verifier: could not build the prompt (source unreadable?)" >&2; exit 3; }
|
|
119
|
+
# A prompt that does not actually contain the file is a verdict reached from the claim text alone.
|
|
120
|
+
if ! grep -q "===== FILE: $(basename "$TARGET") =====" "$WORK/prompt.txt"; then
|
|
121
|
+
echo "finding_verifier: source did not reach the prompt — refusing to ask" >&2; exit 3
|
|
122
|
+
fi
|
|
123
|
+
|
|
124
|
+
case "$FAMILY" in
|
|
125
|
+
codex) "$CODEX" exec --sandbox read-only --skip-git-repo-check -m gpt-6-astra \
|
|
126
|
+
-c model_reasoning_effort="high" < "$WORK/prompt.txt" > "$WORK/raw.txt" 2>"$WORK/err.txt" ;;
|
|
127
|
+
gemini) "$AGY" --mode plan --sandbox --model gemini-3.8-flash-high --output-format text --print-timeout 20m \
|
|
128
|
+
-p "$(cat "$WORK/prompt.txt")" < /dev/null > "$WORK/raw.txt" 2>"$WORK/err.txt" ;;
|
|
129
|
+
*) echo "finding_verifier: unknown family '$FAMILY' (codex|gemini)" >&2; exit 2 ;;
|
|
130
|
+
esac
|
|
131
|
+
RC=$?
|
|
132
|
+
|
|
133
|
+
# The CLI's own exit code is not the WHOLE verdict — codex has exited non-zero while still printing
|
|
134
|
+
# usable output, and zero while printing none — but the first draft used it for nothing at all, which
|
|
135
|
+
# made the "cli failed" guard decorative (cross-family finding, 2026-09-09). It is now one of two
|
|
136
|
+
# inputs: a non-zero CLI that nevertheless answered EVERY claim asked is reported and allowed; a
|
|
137
|
+
# non-zero CLI that answered only some is a degraded run (exit 3), because the missing answers and the
|
|
138
|
+
# crash have the same cause and "fewer verdicts" would otherwise read as "fewer problems".
|
|
139
|
+
ASKED=$(grep -c '^{' "$WORK/findings.jsonl" 2>/dev/null); ASKED=${ASKED:-0}
|
|
140
|
+
MODE="$MODE" /usr/bin/python3 - "$WORK/raw.txt" <<'PY'
|
|
141
|
+
import json, os, sys
|
|
142
|
+
allowed = (("confirmed", "false-positive", "needs-debate") if os.environ["MODE"] == "verify"
|
|
143
|
+
else ("correct-drop", "wrong-drop", "uncertain"))
|
|
144
|
+
n = 0
|
|
145
|
+
for line in open(sys.argv[1], encoding="utf-8", errors="replace"):
|
|
146
|
+
line = line.strip().lstrip("")
|
|
147
|
+
if not line.startswith("{"):
|
|
148
|
+
continue # tolerate banners and fences, same as finding_fleet.sh
|
|
149
|
+
try:
|
|
150
|
+
d = json.loads(line)
|
|
151
|
+
except json.JSONDecodeError:
|
|
152
|
+
continue
|
|
153
|
+
if not d.get("id") or d.get("verdict") not in allowed:
|
|
154
|
+
continue # an out-of-enum verdict is dropped, never coerced
|
|
155
|
+
n += 1
|
|
156
|
+
print(json.dumps({"id": d["id"], "verdict": d["verdict"], "why": d.get("why", "")},
|
|
157
|
+
ensure_ascii=False))
|
|
158
|
+
open(os.path.join(os.path.dirname(sys.argv[1]), "n_answered"), "w").write(str(n))
|
|
159
|
+
sys.exit(0 if n else 3)
|
|
160
|
+
PY
|
|
161
|
+
PRC=$?
|
|
162
|
+
if [ "$PRC" -ne 0 ]; then
|
|
163
|
+
echo "finding_verifier: family=$FAMILY mode=$MODE cli_rc=$RC — no parsable verdict" >&2
|
|
164
|
+
head -c 400 "$WORK/err.txt" >&2 2>/dev/null
|
|
165
|
+
exit 3
|
|
166
|
+
fi
|
|
167
|
+
ANSWERED=$(cat "$WORK/n_answered" 2>/dev/null); ANSWERED=${ANSWERED:-0}
|
|
168
|
+
if [ "$RC" -ne 0 ] && [ "$ANSWERED" -lt "$ASKED" ]; then
|
|
169
|
+
echo "finding_verifier: family=$FAMILY mode=$MODE cli_rc=$RC answered=$ANSWERED/$ASKED — partial answer from a failed CLI, degrading" >&2
|
|
170
|
+
exit 3
|
|
171
|
+
fi
|
|
172
|
+
[ "$RC" -ne 0 ] && echo "finding_verifier: family=$FAMILY cli_rc=$RC but answered $ANSWERED/$ASKED — allowed, recorded" >&2
|
|
173
|
+
exit 0
|
|
@@ -0,0 +1,468 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""finding_verify.py — typed review findings in, cross-family verdicts on, false positives dropped BY CODE.
|
|
3
|
+
|
|
4
|
+
WHY THIS EXISTS. FH's review output is prose end to end: a governor reads, judges, and writes. Nothing
|
|
5
|
+
in that path can be counted, filtered, or handed to a second family, because a prose finding has no
|
|
6
|
+
fields. Measured 2026-09-08 on eight GHSA cases x3: FH's review made 52 claims of which 5 were wrong
|
|
7
|
+
about the code (9.6%); a sibling harness (octo) made 73 claims -- 40% MORE -- with 2 wrong (2.7%). Its
|
|
8
|
+
advantage is not better reading. Its pipeline emits findings as typed JSON, has a different-family
|
|
9
|
+
agent stamp each one `confirmed|false-positive|needs-debate`, and then DELETES the false positives with
|
|
10
|
+
a filter. Judgment stays with a model; the drop is mechanical. This script is that stage for FH.
|
|
11
|
+
|
|
12
|
+
WHAT IS MECHANIZED AND WHAT IS NOT (CLAUDE.md 'Mechanization Boundary'). The channel is mechanized:
|
|
13
|
+
every finding carries a verdict, the verdict comes from a command that is not the author, the drop is
|
|
14
|
+
performed by code, and what was dropped is written down. The judgment -- is this claim true of the
|
|
15
|
+
source -- is made by whatever model the verifier command runs, never frozen here. This file contains
|
|
16
|
+
no rule about what makes a finding wrong.
|
|
17
|
+
|
|
18
|
+
DEGRADE DIRECTION. A review surface is reversible, so an unreachable verifier does not block. It must
|
|
19
|
+
not be silent either: with no verifier every finding is stamped `unverified`, NOTHING is dropped, the
|
|
20
|
+
status is UNVERIFIED and the exit code says so. An unverified run must never read as a clean one.
|
|
21
|
+
|
|
22
|
+
THE DROP SIDE IS MEASURED TOO, OR THE RUN SAYS IT WAS NOT. A stage that deletes claims improves any
|
|
23
|
+
precision number for free: delete enough and nothing wrong survives. So the error rate of the SURVIVORS
|
|
24
|
+
is not a result on its own — it is only meaningful beside the error rate of the DELETIONS. Measured on
|
|
25
|
+
this pipeline's first real use, 2026-09-08: the verifier dropped a claim that the project's own earlier
|
|
26
|
+
record grades a real A-tier defect. One drop, one wrong. That is why `--audit-verifier` exists and why
|
|
27
|
+
the summary line carries `drop_audit=UNAUDITED` in bold terms when drops happened and nobody checked
|
|
28
|
+
them. The auditor must not be the family that made the drop; when it is the family that PRODUCED the
|
|
29
|
+
finding, that is an appeal by an interested party and is recorded as `audit_role=appeal`, not hidden.
|
|
30
|
+
|
|
31
|
+
INPUT JSONL, one finding per line:
|
|
32
|
+
{"id","file","line","severity","category","title","detail","confidence","producer_family"}
|
|
33
|
+
`id` and `title` are required; the rest are optional and pass through untouched.
|
|
34
|
+
When `producer_family` is present and equals the verifier's family, that finding is stamped
|
|
35
|
+
`unverified` rather than judged -- see the note above VERDICTS.
|
|
36
|
+
EXIT 0 verified, survivors · 1 verified, nothing survived · 2 usage/schema ·
|
|
37
|
+
3 UNVERIFIED (something was never judged) · 4 drops never audited ·
|
|
38
|
+
5 SEEDED control degraded (a known-true finding was deleted), inconclusive
|
|
39
|
+
(the verifier abstained on one) or absent
|
|
40
|
+
(the control never entered the run — which is not the same as passing).
|
|
41
|
+
AUDITOR Optional, and required for the drop-side number to exist. Same protocol as the verifier, but
|
|
42
|
+
it receives only the DROPPED findings and answers {"id","verdict":"correct-drop|wrong-drop|
|
|
43
|
+
uncertain","why"}. A `wrong-drop` finding is moved back into confirmed.jsonl with
|
|
44
|
+
`reinstated: true` -- the audit is not advisory, it reverses the deletion.
|
|
45
|
+
VERIFIER A command that reads the findings JSONL on stdin and writes JSONL verdicts on stdout:
|
|
46
|
+
{"id","verdict":"confirmed|false-positive|needs-debate","why"}
|
|
47
|
+
Set it with --verifier or FH_VERIFY_CMD. Run it as a DIFFERENT model family than the author;
|
|
48
|
+
this script cannot check that, and says so rather than pretending to.
|
|
49
|
+
OUTPUT <out>/confirmed.jsonl survivors (confirmed + needs-debate, the latter flagged)
|
|
50
|
+
<out>/dropped.jsonl false positives, with the verifier's reason -- never silent
|
|
51
|
+
stdout one summary line, machine-readable
|
|
52
|
+
EXIT 0 verified and (no drops, or drops audited) with >=1 survivor · 1 same but nothing survives
|
|
53
|
+
3 UNVERIFIED (degraded) · 4 drops happened and were never audited · 2 usage or schema error
|
|
54
|
+
"""
|
|
55
|
+
import argparse, json, os, subprocess, sys
|
|
56
|
+
|
|
57
|
+
REQUIRED = ("id", "title")
|
|
58
|
+
VERDICTS = ("confirmed", "false-positive", "needs-debate")
|
|
59
|
+
AUDIT_VERDICTS = ("correct-drop", "wrong-drop", "uncertain")
|
|
60
|
+
|
|
61
|
+
# A finding is never verified by the family that produced it. That is the one property of the record
|
|
62
|
+
# this file enforces on its own: same-family review shares the author's blind spot, so a verdict from
|
|
63
|
+
# the producer is not a second opinion. It is a channel rule, not a judgment -- the script does not
|
|
64
|
+
# decide whether the claim is true, only that the party answering must not be the party asking.
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _schema_error(msg):
|
|
68
|
+
# R4 #3: a string SystemExit exits 1 — the documented code for "verified, nothing survived".
|
|
69
|
+
# Schema rejection is exit 2, and a consumer treating 0/1 as "completed" must not see 1 here.
|
|
70
|
+
print(msg, file=sys.stderr)
|
|
71
|
+
sys.exit(2)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def read_findings(path):
|
|
75
|
+
out, seen = [], set()
|
|
76
|
+
src = sys.stdin if path == "-" else open(path, encoding="utf-8")
|
|
77
|
+
for n, line in enumerate(src, 1):
|
|
78
|
+
line = line.strip()
|
|
79
|
+
if not line:
|
|
80
|
+
continue
|
|
81
|
+
try:
|
|
82
|
+
d = json.loads(line)
|
|
83
|
+
except json.JSONDecodeError as e:
|
|
84
|
+
_schema_error(f"finding_verify: line {n} is not JSON: {e}")
|
|
85
|
+
if not isinstance(d, dict):
|
|
86
|
+
# R5 #3: `[]` / `null` parse fine and then crash on .get() with exit 1 (= "nothing survived")
|
|
87
|
+
_schema_error(f"finding_verify: line {n} is not a JSON object")
|
|
88
|
+
for k in REQUIRED:
|
|
89
|
+
if not d.get(k):
|
|
90
|
+
_schema_error(f"finding_verify: line {n} missing required field '{k}'")
|
|
91
|
+
# 🟥 R4 #7: audit/verdict metadata is OURS to write — a producer row arriving with
|
|
92
|
+
# `reinstated: true` was counted as recovered coverage by the driver (probe: 1/2 → 2/2).
|
|
93
|
+
for k in ("verdict", "verify_note", "reinstated", "drop_verdict", "audit_note", "audit_why"):
|
|
94
|
+
d.pop(k, None)
|
|
95
|
+
# 🟥 R3 #1: verdicts are keyed by str(id) (run_verifier), so 101 and "101" are ONE key
|
|
96
|
+
# there — uniqueness here must use the same form, or both rows take whichever verdict
|
|
97
|
+
# came last (reproduced: false-positive + confirmed → confirmed=2, exit 0).
|
|
98
|
+
if str(d["id"]) in seen:
|
|
99
|
+
_schema_error(f"finding_verify: duplicate id {d['id']!r} on line {n} (ids compare as strings)")
|
|
100
|
+
seen.add(str(d["id"]))
|
|
101
|
+
out.append(d)
|
|
102
|
+
return out
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def run_verifier(cmd, findings, allowed=VERDICTS):
|
|
106
|
+
"""Returns (verdicts_by_id, error_or_None). Any failure degrades; it never raises.
|
|
107
|
+
|
|
108
|
+
`allowed` is the verdict vocabulary. The audit pass speaks a different one, and a verdict outside
|
|
109
|
+
the expected set is dropped rather than coerced -- a stage that silently reinterprets an unknown
|
|
110
|
+
label is how an unanswered question becomes an answer."""
|
|
111
|
+
payload = "\n".join(json.dumps(f, ensure_ascii=False) for f in findings) + "\n"
|
|
112
|
+
# 🟥 A LIST MEANS argv; A STRING MEANS A SHELL. The caller decides, and the shipped caller
|
|
113
|
+
# (finding_pipeline.sh) now hands a list, so no shell parses our paths.
|
|
114
|
+
#
|
|
115
|
+
# Why this branch exists (cross-family security review, 2026-09-09, reproduced on dash):
|
|
116
|
+
# the string form is executed with `shell=True`, i.e. by `/bin/sh`. Quoting the interpolated
|
|
117
|
+
# path with bash's `printf %q` is NOT enough, because %q emits bash-only `$'...'` for a path
|
|
118
|
+
# containing a newline, and `/bin/sh` on most Linux distributions is **dash**, which does not
|
|
119
|
+
# understand that syntax -- the quoting comes apart and a crafted filename executes a second
|
|
120
|
+
# command. macOS cannot observe this at all: its /bin/sh is bash-derived, so a local run is
|
|
121
|
+
# green while the shipped npm package is not. The fix is not better escaping; it is not
|
|
122
|
+
# handing a shell the string in the first place.
|
|
123
|
+
shell = isinstance(cmd, str)
|
|
124
|
+
try:
|
|
125
|
+
p = subprocess.run(cmd, shell=shell, input=payload, capture_output=True,
|
|
126
|
+
text=True, timeout=int(os.environ.get("FH_VERIFY_TIMEOUT", "1200")))
|
|
127
|
+
except Exception as e: # noqa: BLE001 - degrade on anything
|
|
128
|
+
return {}, f"verifier did not run: {e}"
|
|
129
|
+
if p.returncode != 0:
|
|
130
|
+
return {}, f"verifier exit {p.returncode}: {(p.stderr or '').strip()[:200]}"
|
|
131
|
+
got = {}
|
|
132
|
+
for line in p.stdout.splitlines():
|
|
133
|
+
line = line.strip()
|
|
134
|
+
if not line or not line.startswith("{"):
|
|
135
|
+
continue # tolerate chatter around the JSONL
|
|
136
|
+
try:
|
|
137
|
+
d = json.loads(line)
|
|
138
|
+
except json.JSONDecodeError:
|
|
139
|
+
continue
|
|
140
|
+
if d.get("id") and d.get("verdict") in allowed:
|
|
141
|
+
# 🟥 ids are keyed AS STRINGS. A JSONL row may carry an integer id and a verifier may
|
|
142
|
+
# answer with the string form (or vice versa); `verdicts.get(101)` then misses
|
|
143
|
+
# `{"101": ...}` and the finding comes back `unverified` forever — a finding with an
|
|
144
|
+
# integer id could never be verified at all. Found while writing the seeded-control
|
|
145
|
+
# lane for the same type mismatch (cross-family round 4, finding 7; this second half was
|
|
146
|
+
# not in the report — the lane surfaced it).
|
|
147
|
+
got[str(d["id"])] = d
|
|
148
|
+
if not got:
|
|
149
|
+
return {}, "verifier returned no parseable verdict"
|
|
150
|
+
return got, None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def main():
|
|
154
|
+
ap = argparse.ArgumentParser(add_help=True)
|
|
155
|
+
ap.add_argument("findings", help="JSONL file, or - for stdin")
|
|
156
|
+
ap.add_argument("--out", required=True, help="directory for confirmed.jsonl / dropped.jsonl")
|
|
157
|
+
ap.add_argument("--verifier-argv", default=None,
|
|
158
|
+
help="JSON array form of --verifier. Executed as argv (no shell), which is the "
|
|
159
|
+
"only form immune to path-shaped injection. Wins over --verifier.")
|
|
160
|
+
ap.add_argument("--audit-verifier-argv", default=None,
|
|
161
|
+
help="JSON array form of --audit-verifier. Same reason.")
|
|
162
|
+
ap.add_argument("--verifier", default=os.environ.get("FH_VERIFY_CMD", ""),
|
|
163
|
+
help="shell command; findings JSONL on stdin, verdict JSONL on stdout")
|
|
164
|
+
ap.add_argument("--family", default=os.environ.get("FH_VERIFY_FAMILY", "unstated"),
|
|
165
|
+
help="model family of the verifier, recorded verbatim and never checked")
|
|
166
|
+
ap.add_argument("--audit-verifier", default=os.environ.get("FH_AUDIT_CMD", ""),
|
|
167
|
+
help="command that re-checks the DROPPED findings; without it the run is UNAUDITED")
|
|
168
|
+
ap.add_argument("--seeded", default="",
|
|
169
|
+
help="comma-separated finding ids that are KNOWN-TRUE. They are ordinary rows in "
|
|
170
|
+
"the input; the verifier is never told which they are. A filter that buys "
|
|
171
|
+
"precision by deleting reports itself by deleting these.")
|
|
172
|
+
ap.add_argument("--seeded-file", default="",
|
|
173
|
+
help="file with one known-true finding id per line (same meaning as --seeded)")
|
|
174
|
+
ap.add_argument("--audit-family", default=os.environ.get("FH_AUDIT_FAMILY", "unstated"),
|
|
175
|
+
help="model family of the auditor; must differ from the verifier's")
|
|
176
|
+
a = ap.parse_args()
|
|
177
|
+
|
|
178
|
+
# argv 형태가 있으면 그것이 실행 형태다 — 문자열은 셸을 타므로 후순위다.
|
|
179
|
+
def _as_argv(raw, label):
|
|
180
|
+
if not raw:
|
|
181
|
+
return None
|
|
182
|
+
try:
|
|
183
|
+
v = json.loads(raw)
|
|
184
|
+
except json.JSONDecodeError as e:
|
|
185
|
+
raise SystemExit("%s must be a JSON array: %s" % (label, e))
|
|
186
|
+
if not (isinstance(v, list) and v and all(isinstance(x, str) for x in v)):
|
|
187
|
+
raise SystemExit("%s must be a non-empty JSON array of strings" % label)
|
|
188
|
+
return v
|
|
189
|
+
|
|
190
|
+
a.verifier = _as_argv(a.verifier_argv, "--verifier-argv") or a.verifier
|
|
191
|
+
a.audit_verifier = _as_argv(a.audit_verifier_argv, "--audit-verifier-argv") or a.audit_verifier
|
|
192
|
+
|
|
193
|
+
# 🟥 The seeded control is parsed BEFORE anything runs. Parsing it at the end meant a bad
|
|
194
|
+
# control file surfaced only after the verifier had run and the output files and the VERIFIED
|
|
195
|
+
# summary were already written — and a UnicodeDecodeError there escaped `except OSError` and
|
|
196
|
+
# exited 1, which is this CLI's documented code for "verified, nothing survived". A consumer
|
|
197
|
+
# accepting 0 and 1 would have read a configuration failure as a completed run.
|
|
198
|
+
# (cross-family review 2026-09-09, findings 3 and 4, both reproduced.)
|
|
199
|
+
seeded = [x.strip() for x in a.seeded.split(",") if x.strip()]
|
|
200
|
+
if a.seeded_file:
|
|
201
|
+
try:
|
|
202
|
+
with open(a.seeded_file, encoding="utf-8") as fh:
|
|
203
|
+
from_file = [ln.strip() for ln in fh if ln.strip() and not ln.startswith("#")]
|
|
204
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
205
|
+
print("finding_verify: --seeded-file unusable: %s" % e, file=sys.stderr)
|
|
206
|
+
return 2
|
|
207
|
+
# 🟥 "option supplied but it yielded nothing" is NOT "option omitted". A repository-controlled
|
|
208
|
+
# control file that goes empty would otherwise silently turn calibration off and still exit 0.
|
|
209
|
+
if not from_file:
|
|
210
|
+
print("finding_verify: --seeded-file %r yielded no ids — a control file that declares "
|
|
211
|
+
"nothing is a disabled control, not an absent one" % a.seeded_file, file=sys.stderr)
|
|
212
|
+
return 2
|
|
213
|
+
seeded += from_file
|
|
214
|
+
seeded = sorted(set(seeded))
|
|
215
|
+
|
|
216
|
+
# 문자열이든 리스트든 «비어 있나»를 같은 방법으로 묻는다 — 리스트에 .strip() 은 없다.
|
|
217
|
+
def _configured(cmd):
|
|
218
|
+
return bool(cmd) if isinstance(cmd, list) else bool(str(cmd or "").strip())
|
|
219
|
+
|
|
220
|
+
findings = read_findings(a.findings)
|
|
221
|
+
os.makedirs(a.out, exist_ok=True)
|
|
222
|
+
|
|
223
|
+
if not _configured(a.verifier):
|
|
224
|
+
verdicts, err = {}, "no verifier configured (--verifier / FH_VERIFY_CMD)"
|
|
225
|
+
else:
|
|
226
|
+
verdicts, err = run_verifier(a.verifier, findings)
|
|
227
|
+
|
|
228
|
+
confirmed, dropped, debate, unverified = [], [], 0, 0
|
|
229
|
+
for f in findings:
|
|
230
|
+
v = verdicts.get(str(f["id"]))
|
|
231
|
+
if v is None:
|
|
232
|
+
# Degraded, or the verifier skipped this one. Keep it, mark it, never drop it silently.
|
|
233
|
+
f = dict(f, verdict="unverified",
|
|
234
|
+
verify_note=err or "verifier returned no verdict for this finding")
|
|
235
|
+
unverified += 1
|
|
236
|
+
confirmed.append(f)
|
|
237
|
+
continue
|
|
238
|
+
prod = f.get("producer_family")
|
|
239
|
+
# 🟥 ABSENT is not CLEAN. This guard is the one property this file claims to enforce, and
|
|
240
|
+
# until 2026-09-09 it hung on an OPTIONAL field: omit `producer_family` and the check was
|
|
241
|
+
# skipped entirely, so a family verified its own findings and the run reported
|
|
242
|
+
# `status=VERIFIED rc=0`. Reproduced with a known pair — same table, same verifier, the
|
|
243
|
+
# field the only difference: with it `unverified=1 rc=3`, without it `confirmed=1 rc=0`.
|
|
244
|
+
# A missing producer cannot PROVE the verifier is not the author, so the fail-closed
|
|
245
|
+
# answer is the same one an actual self-verification gets: `unverified`, never a silent
|
|
246
|
+
# pass. (The wired path never reached this — `finding_pipeline.sh` refuses to route a
|
|
247
|
+
# table with no usable `producer_family` (exit 3) and `finding_fleet.sh` always stamps it
|
|
248
|
+
# — but this script ships its own CLI, and a hand-built table is a supported entry point.)
|
|
249
|
+
if not prod:
|
|
250
|
+
f = dict(f, verdict="unverified", verify_note="finding declares no producer_family; "
|
|
251
|
+
f"cannot establish that the verifier ({a.family}) is not its author")
|
|
252
|
+
unverified += 1
|
|
253
|
+
confirmed.append(f)
|
|
254
|
+
continue
|
|
255
|
+
if prod == a.family:
|
|
256
|
+
f = dict(f, verdict="unverified", verify_note="verifier is the producing family "
|
|
257
|
+
f"({a.family}); a finding is not verified by its own author")
|
|
258
|
+
unverified += 1
|
|
259
|
+
confirmed.append(f)
|
|
260
|
+
continue
|
|
261
|
+
f = dict(f, verdict=v["verdict"], verify_why=v.get("why", ""), verify_family=a.family)
|
|
262
|
+
if v["verdict"] == "false-positive":
|
|
263
|
+
dropped.append(f)
|
|
264
|
+
else:
|
|
265
|
+
if v["verdict"] == "needs-debate":
|
|
266
|
+
debate += 1
|
|
267
|
+
confirmed.append(f)
|
|
268
|
+
|
|
269
|
+
# ── drop audit ────────────────────────────────────────────────────────────────────────────────
|
|
270
|
+
# Nothing here judges whether a drop was right; it routes the question to a party that did not make
|
|
271
|
+
# the drop, and moves a reversed drop back. The refusal to report a bare precision number when this
|
|
272
|
+
# did not run is the mechanized part.
|
|
273
|
+
audited = wrong_drops = reinstated = 0
|
|
274
|
+
# 감사가 `dropped` 를 재할당하기 전에 «필터가 무엇을 지웠나» 를 얼려 둔다. SEEDED 는 이것을 읽는다.
|
|
275
|
+
pre_audit_dropped_ids = [f.get("id") for f in dropped]
|
|
276
|
+
audit_status = "UNAUDITED"
|
|
277
|
+
audit_note = ""
|
|
278
|
+
if dropped and _configured(a.audit_verifier):
|
|
279
|
+
if a.audit_family == a.family:
|
|
280
|
+
audit_note = ("auditor is the family that made the drop (%s) -- refused; a deletion is not "
|
|
281
|
+
"checked by the party that made it" % a.family)
|
|
282
|
+
else:
|
|
283
|
+
av, aerr = run_verifier(a.audit_verifier, dropped, AUDIT_VERDICTS)
|
|
284
|
+
if aerr:
|
|
285
|
+
audit_note = "auditor did not answer: " + aerr
|
|
286
|
+
else:
|
|
287
|
+
kept = []
|
|
288
|
+
for d in dropped:
|
|
289
|
+
r = av.get(str(d["id"])) # R3 #7: same normalization as the verify lookup
|
|
290
|
+
if r is None:
|
|
291
|
+
kept.append(dict(d, drop_verdict="unaudited"))
|
|
292
|
+
continue
|
|
293
|
+
audited += 1
|
|
294
|
+
role = "appeal" if d.get("producer_family") == a.audit_family else "independent"
|
|
295
|
+
d = dict(d, drop_verdict=r["verdict"], drop_why=r.get("why", ""),
|
|
296
|
+
audit_family=a.audit_family, audit_role=role)
|
|
297
|
+
if r["verdict"] == "wrong-drop":
|
|
298
|
+
wrong_drops += 1
|
|
299
|
+
reinstated += 1
|
|
300
|
+
# 🟥 A reinstated row used to keep its ORIGINAL top-level
|
|
301
|
+
# `verdict: "false-positive"` while moving into confirmed.jsonl. Every
|
|
302
|
+
# consumer that counts decisions by top-level verdict then lost it from both
|
|
303
|
+
# sides — the driver's coverage read 0% on a run that was fully judged and
|
|
304
|
+
# audited. The row's verdict must state the decision that now stands; the
|
|
305
|
+
# superseded one is kept under its own key rather than deleted.
|
|
306
|
+
# (cross-family round 4, gemini family, A severity — three codex rounds
|
|
307
|
+
# missed it because they were the same family that wrote the counting fix.)
|
|
308
|
+
confirmed.append(dict(d, reinstated=True,
|
|
309
|
+
pre_audit_verdict=d.get("verdict"),
|
|
310
|
+
verdict="confirmed"))
|
|
311
|
+
else:
|
|
312
|
+
kept.append(d)
|
|
313
|
+
dropped = kept
|
|
314
|
+
# 🟥 AUDITED must mean EVERY drop was answered. Setting it unconditionally let an
|
|
315
|
+
# auditor that answered one unrelated id produce `audited=0 drop_audit=AUDITED` and
|
|
316
|
+
# exit 0 — the deletions went unchecked while the record said they were checked.
|
|
317
|
+
# (cross-family review 2026-09-09, reproduced; an EMPTY audit already returned 4, so
|
|
318
|
+
# the hole was specifically the PARTIAL answer.) A partial audit is not an audit.
|
|
319
|
+
unanswered = sum(1 for d in dropped if d.get("drop_verdict") == "unaudited")
|
|
320
|
+
if unanswered:
|
|
321
|
+
audit_status = "PARTIAL"
|
|
322
|
+
audit_note = ("auditor answered %d of %d drops; %d unanswered — a partial audit "
|
|
323
|
+
"is not an audit" % (audited, audited + unanswered, unanswered))
|
|
324
|
+
else:
|
|
325
|
+
audit_status = "AUDITED"
|
|
326
|
+
elif not dropped:
|
|
327
|
+
audit_status = "NO-DROPS"
|
|
328
|
+
|
|
329
|
+
for name, rows in (("confirmed.jsonl", confirmed), ("dropped.jsonl", dropped)):
|
|
330
|
+
with open(os.path.join(a.out, name), "w", encoding="utf-8") as fh:
|
|
331
|
+
for r in rows:
|
|
332
|
+
fh.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
333
|
+
|
|
334
|
+
# 🟥 COVERAGE IS NOT OPTIONAL. An error rate computed over judged findings while the unjudged
|
|
335
|
+
# ones sit outside the denominator is a rate at an unstated operating point, and two arms with
|
|
336
|
+
# different abstention rates are then not comparable at all. This is a named, documented flaw in
|
|
337
|
+
# the selective-classification literature (evaluation "assumes fixed working points",
|
|
338
|
+
# arXiv:2407.01032), and our own five-arm table is an instance of it: UNVERIFIABLE was ~half the
|
|
339
|
+
# claims and was silently dropped from the denominator. So the line carries coverage
|
|
340
|
+
# unconditionally, exactly like DROPS does — same discipline, second application.
|
|
341
|
+
# 🟥 `needs-debate` IS NOT A DECISION. Counting it as covered let every finding come back
|
|
342
|
+
# `needs-debate` and still print coverage=100% — the exact thing coverage exists to prevent
|
|
343
|
+
# (cross-family review 2026-09-09, finding 2, reproduced). Decision coverage = findings that got
|
|
344
|
+
# a RESOLVED truth judgment; debate and unverified are both abstentions, of different kinds.
|
|
345
|
+
# Percentage is FLOORED, never rounded: 200/201 must not print 100%.
|
|
346
|
+
judged = len(findings) - unverified - debate
|
|
347
|
+
pct = (judged * 100) // len(findings) if findings else 0
|
|
348
|
+
# 🟥 `VERIFIED` must not be stamped on a run in which nothing was decided. An all-debate run
|
|
349
|
+
# left `unverified == 0`, so the summary said VERIFIED while coverage said 0% — the exit code was
|
|
350
|
+
# already fixed to 3 but the human-readable half still lied.
|
|
351
|
+
# (cross-family round 5, gemini family, A severity — a half-fix that stopped at the exit code.)
|
|
352
|
+
status = "UNVERIFIED" if (unverified or (findings and judged == 0)) else "VERIFIED"
|
|
353
|
+
print("FINDINGS in={} confirmed={} dropped={} debate={} unverified={} coverage={}/{} ({}%) "
|
|
354
|
+
"family={} status={}{}".format(
|
|
355
|
+
# `confirmed=` excludes BOTH abstention kinds. Debate rows live in the confirmed list for
|
|
356
|
+
# output purposes, but counting them as confirmations double-reports them beside `debate=`.
|
|
357
|
+
len(findings), len(confirmed) - unverified - debate, len(dropped), debate, unverified,
|
|
358
|
+
judged, len(findings), pct,
|
|
359
|
+
a.family, status, "" if not err else " reason=" + err.replace("\n", " ")))
|
|
360
|
+
# 🟥 The drop line is unconditional. A survivor-side number without it is a precision claim made by
|
|
361
|
+
# deleting, and this pipeline does not let a reader compute one without seeing whether the
|
|
362
|
+
# deletions were checked.
|
|
363
|
+
print("DROPS dropped={} audited={} wrong_drops={} reinstated={} auditor={} drop_audit={}{}".format(
|
|
364
|
+
len(dropped), audited, wrong_drops, reinstated, a.audit_family, audit_status,
|
|
365
|
+
"" if not audit_note else " reason=" + audit_note.replace("\n", " ")))
|
|
366
|
+
# 🟥 SEEDED — the known-pair discipline applied to the FILTER, not to a scanner.
|
|
367
|
+
# This repo has required known-pair calibration of instruments for a long time and had never
|
|
368
|
+
# once applied it to the deletion stage, which is also an instrument. Known-true findings are
|
|
369
|
+
# mixed into the input as ordinary rows; their ids live only in this process and never reach the
|
|
370
|
+
# verifier's prompt. A stage that buys precision by deleting therefore reports itself.
|
|
371
|
+
if not seeded:
|
|
372
|
+
seed_status, s_present, s_kept, s_dropped, s_abstained = "NOT_PROVIDED", 0, 0, 0, 0
|
|
373
|
+
else:
|
|
374
|
+
# 🟥 ids are compared AS STRINGS on both sides. A JSONL row may legitimately carry an
|
|
375
|
+
# integer id, and `"101" in {101}` is False in Python — the control then reported ABSENT
|
|
376
|
+
# (exit 5) on a run where the seed was right there. (cross-family round 4, finding 7.)
|
|
377
|
+
# 씨앗은 «라우팅 id» 로도 «원래 멤버 id» 로도 선언할 수 있다. fleet 이 id 를 재번호하므로
|
|
378
|
+
# 호출자 어휘로 선언하려면 후자가 필요하다.
|
|
379
|
+
ids_in = {str(f.get("id")) for f in findings}
|
|
380
|
+
ids_in |= {str(f["member_id"]) for f in findings if f.get("member_id") is not None}
|
|
381
|
+
present = [i for i in seeded if str(i) in ids_in]
|
|
382
|
+
def _row_matches(f, sid):
|
|
383
|
+
# R8 #3: a missing alias must not become the string "None" — a legitimate seed named "None"
|
|
384
|
+
# matched every alias-less row and reported AMBIGUOUS.
|
|
385
|
+
mid = f.get("member_id")
|
|
386
|
+
return str(f.get("id")) == str(sid) or (mid is not None and str(mid) == str(sid))
|
|
387
|
+
# 🟥 A SEED MUST RESOLVE TO EXACTLY ONE ROW. `member_id` is the member's own id and is only
|
|
388
|
+
# locally unique — two fleet members can both emit `1`. Binding the seed to every matching
|
|
389
|
+
# row then makes an unrelated member's drop read as "the control was deleted", and a
|
|
390
|
+
# perfectly legitimate run fails closed with exit 5. A false alarm on a fail-closed surface
|
|
391
|
+
# is not a safe default: it trains the override. So an ambiguous declaration is reported AS
|
|
392
|
+
# ambiguous, by name, instead of being silently resolved the pessimistic way.
|
|
393
|
+
# (cross-family round 6, gemini family, A severity.)
|
|
394
|
+
ambiguous = [i for i in present if sum(1 for f in findings if _row_matches(f, i)) > 1]
|
|
395
|
+
# 🟥 THE PRE-AUDIT DELETION SET, not the post-audit one. `dropped` is reassigned when the
|
|
396
|
+
# auditor reinstates a wrong drop, so reading it here meant: verifier deletes the known-true
|
|
397
|
+
# seed → auditor puts it back → SEEDED prints CLEAN, exit 0. The filter demonstrably deleted
|
|
398
|
+
# a control and the control said it passed. Reinstatement repairs the OUTPUT; it does not
|
|
399
|
+
# establish that the FILTER passed, and the filter is what this control measures.
|
|
400
|
+
# (cross-family review 2026-09-09, finding 1 — A severity, reproduced.)
|
|
401
|
+
dropped_ids = {str(i) for i in pre_audit_dropped_ids}
|
|
402
|
+
# 🟥 SURVIVING IS NOT PASSING. The first version asked only "was the seed deleted?", so a
|
|
403
|
+
# verifier that ABSTAINED on a known-true finding (`needs-debate`, or unverified) reported
|
|
404
|
+
# kept=1 status=CLEAN exit 0 — precision bought by not deciding instead of by deleting,
|
|
405
|
+
# which is the same purchase through a different door. A seed passes only when it received a
|
|
406
|
+
# positive decision. (cross-family round 4, gemini family, A severity.)
|
|
407
|
+
abstained_verdicts = {"needs-debate", "unverified"}
|
|
408
|
+
verdict_of = {}
|
|
409
|
+
for f in confirmed:
|
|
410
|
+
verdict_of[str(f.get("id"))] = f.get("verdict")
|
|
411
|
+
s_present = len(present)
|
|
412
|
+
dropped_rows = [f for f in findings if str(f.get("id")) in dropped_ids]
|
|
413
|
+
s_dropped = len([i for i in present if any(_row_matches(f, i) for f in dropped_rows)])
|
|
414
|
+
s_abstained = len([i for i in present
|
|
415
|
+
if not any(_row_matches(f, i) for f in dropped_rows)
|
|
416
|
+
and any(_row_matches(f, i) and f.get("verdict") in abstained_verdicts
|
|
417
|
+
for f in confirmed)])
|
|
418
|
+
s_kept = s_present - s_dropped - s_abstained
|
|
419
|
+
if ambiguous:
|
|
420
|
+
seed_status = "AMBIGUOUS"
|
|
421
|
+
print("finding_verify: seed(s) %s match more than one finding — member ids are only "
|
|
422
|
+
"locally unique; declare the routing id instead" % ",".join(ambiguous),
|
|
423
|
+
file=sys.stderr)
|
|
424
|
+
elif len(present) < len(seeded):
|
|
425
|
+
# A control that never entered the run is not a passing control. It looks exactly like a
|
|
426
|
+
# clean one from the outside, which is the whole reason this branch exists.
|
|
427
|
+
# 🟥 R3 #2: `not present` only caught TOTAL absence — z1 present + `missing` absent
|
|
428
|
+
# printed CLEAN exit 0. Any declared seed missing is ABSENT.
|
|
429
|
+
seed_status = "ABSENT"
|
|
430
|
+
elif s_dropped:
|
|
431
|
+
seed_status = "DEGRADED"
|
|
432
|
+
elif s_abstained:
|
|
433
|
+
seed_status = "INCONCLUSIVE"
|
|
434
|
+
else:
|
|
435
|
+
seed_status = "CLEAN"
|
|
436
|
+
print("SEEDED declared={} present={} kept={} dropped={} abstained={} status={}".format(
|
|
437
|
+
len(seeded), s_present, s_kept, s_dropped,
|
|
438
|
+
s_abstained if seeded else 0, seed_status))
|
|
439
|
+
|
|
440
|
+
# 🟥 THE SEED VERDICT IS CHECKED FIRST. It used to sit after the two exit-3 branches, so a
|
|
441
|
+
# single unrelated `unverified` finding anywhere in the batch masked a DEGRADED control: the
|
|
442
|
+
# split returned 3, and in the driver rank_of(3) < rank_of(5), so "the filter deleted a
|
|
443
|
+
# known-true finding" was suppressed into a generic unverified exit. The calibration verdict is
|
|
444
|
+
# the more specific and the more serious fact, and it is reported as such.
|
|
445
|
+
# (cross-family round 5, gemini family, A severity.)
|
|
446
|
+
# 🟥 A VERIFIER THAT DID NOT RUN IS AN EXECUTION FAILURE, NOT A CONTROL AMBIGUITY. When the
|
|
447
|
+
# verifier command crashes, every finding degrades to `unverified`, the seed among them becomes
|
|
448
|
+
# `INCONCLUSIVE`, and the run used to exit 5 — reporting a calibration problem for what is
|
|
449
|
+
# actually "the tool did not execute". The execution fact wins. (cross-family round 6.)
|
|
450
|
+
if err and unverified:
|
|
451
|
+
return 3
|
|
452
|
+
if seed_status in ("ABSENT", "DEGRADED", "INCONCLUSIVE", "AMBIGUOUS"):
|
|
453
|
+
return 5
|
|
454
|
+
if unverified:
|
|
455
|
+
return 3
|
|
456
|
+
# 🟥 ZERO DECISIONS IS NOT A PASS. Every finding coming back `needs-debate` left `unverified=0`,
|
|
457
|
+
# so status stamped VERIFIED and the run exited 0 while coverage said 0% — a caller reading exit
|
|
458
|
+
# codes saw a completed run in which nothing was actually judged.
|
|
459
|
+
# (cross-family round 4, gemini family, A severity.)
|
|
460
|
+
if findings and judged == 0:
|
|
461
|
+
return 3 # a known-true finding was deleted, or the control never ran
|
|
462
|
+
if audit_status in ("UNAUDITED", "PARTIAL"):
|
|
463
|
+
return 4 # drops happened and nobody checked them: not a completed run
|
|
464
|
+
return 0 if confirmed else 1
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
if __name__ == "__main__":
|
|
468
|
+
sys.exit(main())
|