@chrono-meta/fh-gate 3.2.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/registry/agent_cards.json +1 -1
- package/.claude/rules/fh_4axis_gate.md +25 -0
- package/.claude-plugin/marketplace.json +3 -3
- package/AGENTS.md +2 -2
- package/CATALOG.md +4 -4
- package/CHEATSHEET.md +1 -1
- package/CLAUDE.md +4 -4
- package/docs/OUTPUT_EVIDENCE.md +1 -1
- package/docs/STANDARDS_ALIGNMENT.md +1 -1
- package/docs/codex-compat.md +1 -1
- package/knowledge/shared/harness-core/agents_md_runtime_details.md +2 -2
- package/knowledge/shared/harness-core/field_verdict_crossfamily_gate.md +30 -3
- package/knowledge/shared/harness-core/iso_ai_standards_crosswalk.md +1 -1
- package/knowledge/shared/harness-core/skill_quality_rubric.md +1 -1
- package/knowledge/shared/learnings/subagent_invocations_log.yaml +31 -0
- package/knowledge/shared/rules/modes_and_value.md +2 -2
- package/package.json +2 -1
- package/plugins/fh-commons/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-commons/README.md +38 -0
- package/plugins/fh-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/fh-meta/CHANGELOG.md +95 -0
- package/plugins/fh-meta/skills/agent-composer/SKILL.md +2 -2
- package/plugins/fh-meta/skills/auto-decorrelation/SKILL.md +40 -0
- package/plugins/fh-meta/skills/frontier-digest/SKILL.md +1 -1
- package/plugins/fh-meta/skills/frontier-digest/SKILL_detail.md +43 -5
- package/plugins/fh-meta/skills/{hub-cc-pr-reviewer → harness-pr-reviewer}/SKILL.md +75 -3
- package/plugins/fh-meta/skills/{hub-cc-pr-reviewer → harness-pr-reviewer}/SKILL_detail.md +2 -2
- package/plugins/fh-meta/skills/harvest-loop/SKILL_detail.md +2 -2
- package/plugins/fh-meta/skills/install-doctor/SKILL.md +1 -1
- package/plugins/fh-meta/skills/install-wizard/SKILL.md +1 -1
- package/plugins/fh-meta/skills/meta-prompt-builder/SKILL.md +1 -1
- package/plugins/fh-meta/skills/pipeline-conductor/SKILL.md +1 -1
- package/plugins/fh-meta/skills/plugin-recommender/SKILL.md +1 -1
- package/plugins/fh-meta/skills/sim-conductor/SKILL.md +1 -1
- package/plugins/fh-qp/.claude-plugin/plugin.json +1 -1
- package/scripts/finding_fleet.sh +391 -13
- package/scripts/finding_pipeline.sh +370 -11
- package/scripts/finding_verifier.sh +31 -2
- package/scripts/finding_verify.py +198 -13
- package/scripts/frontier_digest_autopilot.sh +3 -3
- package/scripts/gate_shape_scan.sh +16 -2
- package/scripts/test_finding_pipeline_lanes.sh +1556 -4
- package/scripts/test_gate_shape_scan_lanes.sh +11 -0
- package/scripts/test_marker_crossfamily_lanes.sh +75 -3
- package/scripts/test_marker_standpoint_lanes.sh +31 -6
- package/templates/.git-hooks/pre-commit +152 -6
- package/templates/local_fh_context.md +1 -1
- package/templates/regression_guard.sh +1 -1
|
@@ -33,6 +33,11 @@ INPUT JSONL, one finding per line:
|
|
|
33
33
|
`id` and `title` are required; the rest are optional and pass through untouched.
|
|
34
34
|
When `producer_family` is present and equals the verifier's family, that finding is stamped
|
|
35
35
|
`unverified` rather than judged -- see the note above VERDICTS.
|
|
36
|
+
EXIT 0 verified, survivors · 1 verified, nothing survived · 2 usage/schema ·
|
|
37
|
+
3 UNVERIFIED (something was never judged) · 4 drops never audited ·
|
|
38
|
+
5 SEEDED control degraded (a known-true finding was deleted), inconclusive
|
|
39
|
+
(the verifier abstained on one) or absent
|
|
40
|
+
(the control never entered the run — which is not the same as passing).
|
|
36
41
|
AUDITOR Optional, and required for the drop-side number to exist. Same protocol as the verifier, but
|
|
37
42
|
it receives only the DROPPED findings and answers {"id","verdict":"correct-drop|wrong-drop|
|
|
38
43
|
uncertain","why"}. A `wrong-drop` finding is moved back into confirmed.jsonl with
|
|
@@ -59,6 +64,13 @@ AUDIT_VERDICTS = ("correct-drop", "wrong-drop", "uncertain")
|
|
|
59
64
|
# decide whether the claim is true, only that the party answering must not be the party asking.
|
|
60
65
|
|
|
61
66
|
|
|
67
|
+
def _schema_error(msg):
|
|
68
|
+
# R4 #3: a string SystemExit exits 1 — the documented code for "verified, nothing survived".
|
|
69
|
+
# Schema rejection is exit 2, and a consumer treating 0/1 as "completed" must not see 1 here.
|
|
70
|
+
print(msg, file=sys.stderr)
|
|
71
|
+
sys.exit(2)
|
|
72
|
+
|
|
73
|
+
|
|
62
74
|
def read_findings(path):
|
|
63
75
|
out, seen = [], set()
|
|
64
76
|
src = sys.stdin if path == "-" else open(path, encoding="utf-8")
|
|
@@ -69,13 +81,23 @@ def read_findings(path):
|
|
|
69
81
|
try:
|
|
70
82
|
d = json.loads(line)
|
|
71
83
|
except json.JSONDecodeError as e:
|
|
72
|
-
|
|
84
|
+
_schema_error(f"finding_verify: line {n} is not JSON: {e}")
|
|
85
|
+
if not isinstance(d, dict):
|
|
86
|
+
# R5 #3: `[]` / `null` parse fine and then crash on .get() with exit 1 (= "nothing survived")
|
|
87
|
+
_schema_error(f"finding_verify: line {n} is not a JSON object")
|
|
73
88
|
for k in REQUIRED:
|
|
74
89
|
if not d.get(k):
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
90
|
+
_schema_error(f"finding_verify: line {n} missing required field '{k}'")
|
|
91
|
+
# 🟥 R4 #7: audit/verdict metadata is OURS to write — a producer row arriving with
|
|
92
|
+
# `reinstated: true` was counted as recovered coverage by the driver (probe: 1/2 → 2/2).
|
|
93
|
+
for k in ("verdict", "verify_note", "reinstated", "drop_verdict", "audit_note", "audit_why"):
|
|
94
|
+
d.pop(k, None)
|
|
95
|
+
# 🟥 R3 #1: verdicts are keyed by str(id) (run_verifier), so 101 and "101" are ONE key
|
|
96
|
+
# there — uniqueness here must use the same form, or both rows take whichever verdict
|
|
97
|
+
# came last (reproduced: false-positive + confirmed → confirmed=2, exit 0).
|
|
98
|
+
if str(d["id"]) in seen:
|
|
99
|
+
_schema_error(f"finding_verify: duplicate id {d['id']!r} on line {n} (ids compare as strings)")
|
|
100
|
+
seen.add(str(d["id"]))
|
|
79
101
|
out.append(d)
|
|
80
102
|
return out
|
|
81
103
|
|
|
@@ -101,7 +123,7 @@ def run_verifier(cmd, findings, allowed=VERDICTS):
|
|
|
101
123
|
shell = isinstance(cmd, str)
|
|
102
124
|
try:
|
|
103
125
|
p = subprocess.run(cmd, shell=shell, input=payload, capture_output=True,
|
|
104
|
-
text=True, timeout=int(os.environ.get("FH_VERIFY_TIMEOUT", "
|
|
126
|
+
text=True, timeout=int(os.environ.get("FH_VERIFY_TIMEOUT", "1200")))
|
|
105
127
|
except Exception as e: # noqa: BLE001 - degrade on anything
|
|
106
128
|
return {}, f"verifier did not run: {e}"
|
|
107
129
|
if p.returncode != 0:
|
|
@@ -116,7 +138,13 @@ def run_verifier(cmd, findings, allowed=VERDICTS):
|
|
|
116
138
|
except json.JSONDecodeError:
|
|
117
139
|
continue
|
|
118
140
|
if d.get("id") and d.get("verdict") in allowed:
|
|
119
|
-
|
|
141
|
+
# 🟥 ids are keyed AS STRINGS. A JSONL row may carry an integer id and a verifier may
|
|
142
|
+
# answer with the string form (or vice versa); `verdicts.get(101)` then misses
|
|
143
|
+
# `{"101": ...}` and the finding comes back `unverified` forever — a finding with an
|
|
144
|
+
# integer id could never be verified at all. Found while writing the seeded-control
|
|
145
|
+
# lane for the same type mismatch (cross-family round 4, finding 7; this second half was
|
|
146
|
+
# not in the report — the lane surfaced it).
|
|
147
|
+
got[str(d["id"])] = d
|
|
120
148
|
if not got:
|
|
121
149
|
return {}, "verifier returned no parseable verdict"
|
|
122
150
|
return got, None
|
|
@@ -137,6 +165,12 @@ def main():
|
|
|
137
165
|
help="model family of the verifier, recorded verbatim and never checked")
|
|
138
166
|
ap.add_argument("--audit-verifier", default=os.environ.get("FH_AUDIT_CMD", ""),
|
|
139
167
|
help="command that re-checks the DROPPED findings; without it the run is UNAUDITED")
|
|
168
|
+
ap.add_argument("--seeded", default="",
|
|
169
|
+
help="comma-separated finding ids that are KNOWN-TRUE. They are ordinary rows in "
|
|
170
|
+
"the input; the verifier is never told which they are. A filter that buys "
|
|
171
|
+
"precision by deleting reports itself by deleting these.")
|
|
172
|
+
ap.add_argument("--seeded-file", default="",
|
|
173
|
+
help="file with one known-true finding id per line (same meaning as --seeded)")
|
|
140
174
|
ap.add_argument("--audit-family", default=os.environ.get("FH_AUDIT_FAMILY", "unstated"),
|
|
141
175
|
help="model family of the auditor; must differ from the verifier's")
|
|
142
176
|
a = ap.parse_args()
|
|
@@ -156,6 +190,29 @@ def main():
|
|
|
156
190
|
a.verifier = _as_argv(a.verifier_argv, "--verifier-argv") or a.verifier
|
|
157
191
|
a.audit_verifier = _as_argv(a.audit_verifier_argv, "--audit-verifier-argv") or a.audit_verifier
|
|
158
192
|
|
|
193
|
+
# 🟥 The seeded control is parsed BEFORE anything runs. Parsing it at the end meant a bad
|
|
194
|
+
# control file surfaced only after the verifier had run and the output files and the VERIFIED
|
|
195
|
+
# summary were already written — and a UnicodeDecodeError there escaped `except OSError` and
|
|
196
|
+
# exited 1, which is this CLI's documented code for "verified, nothing survived". A consumer
|
|
197
|
+
# accepting 0 and 1 would have read a configuration failure as a completed run.
|
|
198
|
+
# (cross-family review 2026-09-09, findings 3 and 4, both reproduced.)
|
|
199
|
+
seeded = [x.strip() for x in a.seeded.split(",") if x.strip()]
|
|
200
|
+
if a.seeded_file:
|
|
201
|
+
try:
|
|
202
|
+
with open(a.seeded_file, encoding="utf-8") as fh:
|
|
203
|
+
from_file = [ln.strip() for ln in fh if ln.strip() and not ln.startswith("#")]
|
|
204
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
205
|
+
print("finding_verify: --seeded-file unusable: %s" % e, file=sys.stderr)
|
|
206
|
+
return 2
|
|
207
|
+
# 🟥 "option supplied but it yielded nothing" is NOT "option omitted". A repository-controlled
|
|
208
|
+
# control file that goes empty would otherwise silently turn calibration off and still exit 0.
|
|
209
|
+
if not from_file:
|
|
210
|
+
print("finding_verify: --seeded-file %r yielded no ids — a control file that declares "
|
|
211
|
+
"nothing is a disabled control, not an absent one" % a.seeded_file, file=sys.stderr)
|
|
212
|
+
return 2
|
|
213
|
+
seeded += from_file
|
|
214
|
+
seeded = sorted(set(seeded))
|
|
215
|
+
|
|
159
216
|
# 문자열이든 리스트든 «비어 있나»를 같은 방법으로 묻는다 — 리스트에 .strip() 은 없다.
|
|
160
217
|
def _configured(cmd):
|
|
161
218
|
return bool(cmd) if isinstance(cmd, list) else bool(str(cmd or "").strip())
|
|
@@ -170,7 +227,7 @@ def main():
|
|
|
170
227
|
|
|
171
228
|
confirmed, dropped, debate, unverified = [], [], 0, 0
|
|
172
229
|
for f in findings:
|
|
173
|
-
v = verdicts.get(f["id"])
|
|
230
|
+
v = verdicts.get(str(f["id"]))
|
|
174
231
|
if v is None:
|
|
175
232
|
# Degraded, or the verifier skipped this one. Keep it, mark it, never drop it silently.
|
|
176
233
|
f = dict(f, verdict="unverified",
|
|
@@ -214,6 +271,8 @@ def main():
|
|
|
214
271
|
# the drop, and moves a reversed drop back. The refusal to report a bare precision number when this
|
|
215
272
|
# did not run is the mechanized part.
|
|
216
273
|
audited = wrong_drops = reinstated = 0
|
|
274
|
+
# 감사가 `dropped` 를 재할당하기 전에 «필터가 무엇을 지웠나» 를 얼려 둔다. SEEDED 는 이것을 읽는다.
|
|
275
|
+
pre_audit_dropped_ids = [f.get("id") for f in dropped]
|
|
217
276
|
audit_status = "UNAUDITED"
|
|
218
277
|
audit_note = ""
|
|
219
278
|
if dropped and _configured(a.audit_verifier):
|
|
@@ -227,7 +286,7 @@ def main():
|
|
|
227
286
|
else:
|
|
228
287
|
kept = []
|
|
229
288
|
for d in dropped:
|
|
230
|
-
r = av.get(d["id"])
|
|
289
|
+
r = av.get(str(d["id"])) # R3 #7: same normalization as the verify lookup
|
|
231
290
|
if r is None:
|
|
232
291
|
kept.append(dict(d, drop_verdict="unaudited"))
|
|
233
292
|
continue
|
|
@@ -238,7 +297,17 @@ def main():
|
|
|
238
297
|
if r["verdict"] == "wrong-drop":
|
|
239
298
|
wrong_drops += 1
|
|
240
299
|
reinstated += 1
|
|
241
|
-
|
|
300
|
+
# 🟥 A reinstated row used to keep its ORIGINAL top-level
|
|
301
|
+
# `verdict: "false-positive"` while moving into confirmed.jsonl. Every
|
|
302
|
+
# consumer that counts decisions by top-level verdict then lost it from both
|
|
303
|
+
# sides — the driver's coverage read 0% on a run that was fully judged and
|
|
304
|
+
# audited. The row's verdict must state the decision that now stands; the
|
|
305
|
+
# superseded one is kept under its own key rather than deleted.
|
|
306
|
+
# (cross-family round 4, gemini family, A severity — three codex rounds
|
|
307
|
+
# missed it because they were the same family that wrote the counting fix.)
|
|
308
|
+
confirmed.append(dict(d, reinstated=True,
|
|
309
|
+
pre_audit_verdict=d.get("verdict"),
|
|
310
|
+
verdict="confirmed"))
|
|
242
311
|
else:
|
|
243
312
|
kept.append(d)
|
|
244
313
|
dropped = kept
|
|
@@ -262,9 +331,31 @@ def main():
|
|
|
262
331
|
for r in rows:
|
|
263
332
|
fh.write(json.dumps(r, ensure_ascii=False) + "\n")
|
|
264
333
|
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
334
|
+
# 🟥 COVERAGE IS NOT OPTIONAL. An error rate computed over judged findings while the unjudged
|
|
335
|
+
# ones sit outside the denominator is a rate at an unstated operating point, and two arms with
|
|
336
|
+
# different abstention rates are then not comparable at all. This is a named, documented flaw in
|
|
337
|
+
# the selective-classification literature (evaluation "assumes fixed working points",
|
|
338
|
+
# arXiv:2407.01032), and our own five-arm table is an instance of it: UNVERIFIABLE was ~half the
|
|
339
|
+
# claims and was silently dropped from the denominator. So the line carries coverage
|
|
340
|
+
# unconditionally, exactly like DROPS does — same discipline, second application.
|
|
341
|
+
# 🟥 `needs-debate` IS NOT A DECISION. Counting it as covered let every finding come back
|
|
342
|
+
# `needs-debate` and still print coverage=100% — the exact thing coverage exists to prevent
|
|
343
|
+
# (cross-family review 2026-09-09, finding 2, reproduced). Decision coverage = findings that got
|
|
344
|
+
# a RESOLVED truth judgment; debate and unverified are both abstentions, of different kinds.
|
|
345
|
+
# Percentage is FLOORED, never rounded: 200/201 must not print 100%.
|
|
346
|
+
judged = len(findings) - unverified - debate
|
|
347
|
+
pct = (judged * 100) // len(findings) if findings else 0
|
|
348
|
+
# 🟥 `VERIFIED` must not be stamped on a run in which nothing was decided. An all-debate run
|
|
349
|
+
# left `unverified == 0`, so the summary said VERIFIED while coverage said 0% — the exit code was
|
|
350
|
+
# already fixed to 3 but the human-readable half still lied.
|
|
351
|
+
# (cross-family round 5, gemini family, A severity — a half-fix that stopped at the exit code.)
|
|
352
|
+
status = "UNVERIFIED" if (unverified or (findings and judged == 0)) else "VERIFIED"
|
|
353
|
+
print("FINDINGS in={} confirmed={} dropped={} debate={} unverified={} coverage={}/{} ({}%) "
|
|
354
|
+
"family={} status={}{}".format(
|
|
355
|
+
# `confirmed=` excludes BOTH abstention kinds. Debate rows live in the confirmed list for
|
|
356
|
+
# output purposes, but counting them as confirmations double-reports them beside `debate=`.
|
|
357
|
+
len(findings), len(confirmed) - unverified - debate, len(dropped), debate, unverified,
|
|
358
|
+
judged, len(findings), pct,
|
|
268
359
|
a.family, status, "" if not err else " reason=" + err.replace("\n", " ")))
|
|
269
360
|
# 🟥 The drop line is unconditional. A survivor-side number without it is a precision claim made by
|
|
270
361
|
# deleting, and this pipeline does not let a reader compute one without seeing whether the
|
|
@@ -272,8 +363,102 @@ def main():
|
|
|
272
363
|
print("DROPS dropped={} audited={} wrong_drops={} reinstated={} auditor={} drop_audit={}{}".format(
|
|
273
364
|
len(dropped), audited, wrong_drops, reinstated, a.audit_family, audit_status,
|
|
274
365
|
"" if not audit_note else " reason=" + audit_note.replace("\n", " ")))
|
|
366
|
+
# 🟥 SEEDED — the known-pair discipline applied to the FILTER, not to a scanner.
|
|
367
|
+
# This repo has required known-pair calibration of instruments for a long time and had never
|
|
368
|
+
# once applied it to the deletion stage, which is also an instrument. Known-true findings are
|
|
369
|
+
# mixed into the input as ordinary rows; their ids live only in this process and never reach the
|
|
370
|
+
# verifier's prompt. A stage that buys precision by deleting therefore reports itself.
|
|
371
|
+
if not seeded:
|
|
372
|
+
seed_status, s_present, s_kept, s_dropped, s_abstained = "NOT_PROVIDED", 0, 0, 0, 0
|
|
373
|
+
else:
|
|
374
|
+
# 🟥 ids are compared AS STRINGS on both sides. A JSONL row may legitimately carry an
|
|
375
|
+
# integer id, and `"101" in {101}` is False in Python — the control then reported ABSENT
|
|
376
|
+
# (exit 5) on a run where the seed was right there. (cross-family round 4, finding 7.)
|
|
377
|
+
# 씨앗은 «라우팅 id» 로도 «원래 멤버 id» 로도 선언할 수 있다. fleet 이 id 를 재번호하므로
|
|
378
|
+
# 호출자 어휘로 선언하려면 후자가 필요하다.
|
|
379
|
+
ids_in = {str(f.get("id")) for f in findings}
|
|
380
|
+
ids_in |= {str(f["member_id"]) for f in findings if f.get("member_id") is not None}
|
|
381
|
+
present = [i for i in seeded if str(i) in ids_in]
|
|
382
|
+
def _row_matches(f, sid):
|
|
383
|
+
# R8 #3: a missing alias must not become the string "None" — a legitimate seed named "None"
|
|
384
|
+
# matched every alias-less row and reported AMBIGUOUS.
|
|
385
|
+
mid = f.get("member_id")
|
|
386
|
+
return str(f.get("id")) == str(sid) or (mid is not None and str(mid) == str(sid))
|
|
387
|
+
# 🟥 A SEED MUST RESOLVE TO EXACTLY ONE ROW. `member_id` is the member's own id and is only
|
|
388
|
+
# locally unique — two fleet members can both emit `1`. Binding the seed to every matching
|
|
389
|
+
# row then makes an unrelated member's drop read as "the control was deleted", and a
|
|
390
|
+
# perfectly legitimate run fails closed with exit 5. A false alarm on a fail-closed surface
|
|
391
|
+
# is not a safe default: it trains the override. So an ambiguous declaration is reported AS
|
|
392
|
+
# ambiguous, by name, instead of being silently resolved the pessimistic way.
|
|
393
|
+
# (cross-family round 6, gemini family, A severity.)
|
|
394
|
+
ambiguous = [i for i in present if sum(1 for f in findings if _row_matches(f, i)) > 1]
|
|
395
|
+
# 🟥 THE PRE-AUDIT DELETION SET, not the post-audit one. `dropped` is reassigned when the
|
|
396
|
+
# auditor reinstates a wrong drop, so reading it here meant: verifier deletes the known-true
|
|
397
|
+
# seed → auditor puts it back → SEEDED prints CLEAN, exit 0. The filter demonstrably deleted
|
|
398
|
+
# a control and the control said it passed. Reinstatement repairs the OUTPUT; it does not
|
|
399
|
+
# establish that the FILTER passed, and the filter is what this control measures.
|
|
400
|
+
# (cross-family review 2026-09-09, finding 1 — A severity, reproduced.)
|
|
401
|
+
dropped_ids = {str(i) for i in pre_audit_dropped_ids}
|
|
402
|
+
# 🟥 SURVIVING IS NOT PASSING. The first version asked only "was the seed deleted?", so a
|
|
403
|
+
# verifier that ABSTAINED on a known-true finding (`needs-debate`, or unverified) reported
|
|
404
|
+
# kept=1 status=CLEAN exit 0 — precision bought by not deciding instead of by deleting,
|
|
405
|
+
# which is the same purchase through a different door. A seed passes only when it received a
|
|
406
|
+
# positive decision. (cross-family round 4, gemini family, A severity.)
|
|
407
|
+
abstained_verdicts = {"needs-debate", "unverified"}
|
|
408
|
+
verdict_of = {}
|
|
409
|
+
for f in confirmed:
|
|
410
|
+
verdict_of[str(f.get("id"))] = f.get("verdict")
|
|
411
|
+
s_present = len(present)
|
|
412
|
+
dropped_rows = [f for f in findings if str(f.get("id")) in dropped_ids]
|
|
413
|
+
s_dropped = len([i for i in present if any(_row_matches(f, i) for f in dropped_rows)])
|
|
414
|
+
s_abstained = len([i for i in present
|
|
415
|
+
if not any(_row_matches(f, i) for f in dropped_rows)
|
|
416
|
+
and any(_row_matches(f, i) and f.get("verdict") in abstained_verdicts
|
|
417
|
+
for f in confirmed)])
|
|
418
|
+
s_kept = s_present - s_dropped - s_abstained
|
|
419
|
+
if ambiguous:
|
|
420
|
+
seed_status = "AMBIGUOUS"
|
|
421
|
+
print("finding_verify: seed(s) %s match more than one finding — member ids are only "
|
|
422
|
+
"locally unique; declare the routing id instead" % ",".join(ambiguous),
|
|
423
|
+
file=sys.stderr)
|
|
424
|
+
elif len(present) < len(seeded):
|
|
425
|
+
# A control that never entered the run is not a passing control. It looks exactly like a
|
|
426
|
+
# clean one from the outside, which is the whole reason this branch exists.
|
|
427
|
+
# 🟥 R3 #2: `not present` only caught TOTAL absence — z1 present + `missing` absent
|
|
428
|
+
# printed CLEAN exit 0. Any declared seed missing is ABSENT.
|
|
429
|
+
seed_status = "ABSENT"
|
|
430
|
+
elif s_dropped:
|
|
431
|
+
seed_status = "DEGRADED"
|
|
432
|
+
elif s_abstained:
|
|
433
|
+
seed_status = "INCONCLUSIVE"
|
|
434
|
+
else:
|
|
435
|
+
seed_status = "CLEAN"
|
|
436
|
+
print("SEEDED declared={} present={} kept={} dropped={} abstained={} status={}".format(
|
|
437
|
+
len(seeded), s_present, s_kept, s_dropped,
|
|
438
|
+
s_abstained if seeded else 0, seed_status))
|
|
439
|
+
|
|
440
|
+
# 🟥 THE SEED VERDICT IS CHECKED FIRST. It used to sit after the two exit-3 branches, so a
|
|
441
|
+
# single unrelated `unverified` finding anywhere in the batch masked a DEGRADED control: the
|
|
442
|
+
# split returned 3, and in the driver rank_of(3) < rank_of(5), so "the filter deleted a
|
|
443
|
+
# known-true finding" was suppressed into a generic unverified exit. The calibration verdict is
|
|
444
|
+
# the more specific and the more serious fact, and it is reported as such.
|
|
445
|
+
# (cross-family round 5, gemini family, A severity.)
|
|
446
|
+
# 🟥 A VERIFIER THAT DID NOT RUN IS AN EXECUTION FAILURE, NOT A CONTROL AMBIGUITY. When the
|
|
447
|
+
# verifier command crashes, every finding degrades to `unverified`, the seed among them becomes
|
|
448
|
+
# `INCONCLUSIVE`, and the run used to exit 5 — reporting a calibration problem for what is
|
|
449
|
+
# actually "the tool did not execute". The execution fact wins. (cross-family round 6.)
|
|
450
|
+
if err and unverified:
|
|
451
|
+
return 3
|
|
452
|
+
if seed_status in ("ABSENT", "DEGRADED", "INCONCLUSIVE", "AMBIGUOUS"):
|
|
453
|
+
return 5
|
|
275
454
|
if unverified:
|
|
276
455
|
return 3
|
|
456
|
+
# 🟥 ZERO DECISIONS IS NOT A PASS. Every finding coming back `needs-debate` left `unverified=0`,
|
|
457
|
+
# so status stamped VERIFIED and the run exited 0 while coverage said 0% — a caller reading exit
|
|
458
|
+
# codes saw a completed run in which nothing was actually judged.
|
|
459
|
+
# (cross-family round 4, gemini family, A severity.)
|
|
460
|
+
if findings and judged == 0:
|
|
461
|
+
return 3 # a known-true finding was deleted, or the control never ran
|
|
277
462
|
if audit_status in ("UNAUDITED", "PARTIAL"):
|
|
278
463
|
return 4 # drops happened and nobody checked them: not a completed run
|
|
279
464
|
return 0 if confirmed else 1
|
|
@@ -22,10 +22,10 @@
|
|
|
22
22
|
# BACK-END SHIPPING DOCTRINE (operator, 2026-08-15): "개발의 앞단-영혼심기, 중간단-탈상관 가속화,
|
|
23
23
|
# 뒷단 출하전-4단검증 및 하네스오너 리뷰" — the back end of shipping is 4-axis verification AND a
|
|
24
24
|
# standpoint-axis review, not 4-axis alone. CORRECTED same session (an earlier draft of this
|
|
25
|
-
# header named fh-meta:
|
|
25
|
+
# header named fh-meta:harness-pr-reviewer here — wrong skill, caught by the operator: "내가 말한건
|
|
26
26
|
# 하네스오너(너) 아니라 그 하네스에 에이전트가 들어가서 그 입장에서 리뷰한다는거야"). "하네스오너
|
|
27
27
|
# 리뷰" is NOT the human operator (that gate is merge, unconditionally human, unchanged) NOR
|
|
28
|
-
#
|
|
28
|
+
# harness-pr-reviewer (which checks FH's diff against FH's OWN conventions — same-repo
|
|
29
29
|
# self-consistency, a different question). It is the STANDPOINT AXIS
|
|
30
30
|
# (`knowledge/shared/harness-core/field_verdict_crossfamily_gate.md §7`, the mechanism behind the
|
|
31
31
|
# if(kakao)26 keynote's p15 "(c) 탈상관의 확장" slide — "계열을 늘려도 못 잡는 결함이 있습니다,
|
|
@@ -197,7 +197,7 @@ BACK-END CHECKPOINTS (operator instruction, 2026-08-15 - required steps on this
|
|
|
197
197
|
2. STANDPOINT AXIS (knowledge/shared/harness-core/field_verdict_crossfamily_gate.md §7 - read it before applying this step, this summary is not the full spec): does the diff alter ANOTHER harness's actual behavior, gate outcome, or interaction contract (not merely touch a path that happens to be synced elsewhere - the trigger is behavioral, not file-class)? Most ordinary FH self-improvement from a digest signal will correctly land on standpoint: not-applicable - that is a correct, expected answer, not a shortfall to fix. This is a genuinely different axis from "harness-owner reviewing FH's own conventions" - it means an agent actually running the diff's effect FROM the standpoint of the OTHER harness's own repo (family diversity alone does not catch this: the mechanism behind the if(kakao)26 keynote's p15 slide "(c) 탈상관의 확장" - "계열을 늘려도 못 잡는 결함이 있습니다, 입장을 바꾸면 보입니다" - §7 is built on three field incidents where full cross-family review missed a defect that only one execution-from-the-target's-own-repo caught).
|
|
198
198
|
- If NOT applicable (the common case): record standpoint: not-applicable in the marker/signal, state briefly what was checked (per the spec's own discipline - asserting non-applicability without naming what was checked is indistinguishable from not having looked at all), and move on.
|
|
199
199
|
- If applicable: check whether a local clone of the target harness exists on this machine (e.g. under the parent of ${FH_DIR} - sibling directories like pmh-dev, qasp-dev, or similar). If one exists, run the change against THAT repo's own content/rules from its own standpoint (tier2) - this must happen BEFORE any push or gh pr create for THIS diff, same non-negotiable pre-push timing as the irreversibility check above and for the identical reason (PR #370, 2026-08-14: a post-PR standpoint review still caught 2 residency leaks that had already sat in public view before the fix - public exposure is effectively irreversible, so this cannot run after the diff is visible). If it finds anything, fix it in the local diff and re-run until clean - only then push and open the PR (or, if step 1 already routed to hold, fold the finding into that signal file instead). If no local clone of the target harness is reachable, record standpoint: DEGRADED_NO_TARGET_ACCESS (could not, not did not) and proceed - do not block indefinitely on a target you structurally cannot reach, but do not silently claim not-applicable either when it actually is applicable and merely unreachable.
|
|
200
|
-
Do NOT conflate this with fh-meta:challenger (family/adversarial-correctness axis, already required by the 4-axis gate above) or with fh-meta:
|
|
200
|
+
Do NOT conflate this with fh-meta:challenger (family/adversarial-correctness axis, already required by the 4-axis gate above) or with fh-meta:harness-pr-reviewer (checks FH's own diff against FH's OWN baseline conventions - a same-repo self-consistency check, not a standpoint-axis review at all). All three are different lenses; running one is not a substitute for another.
|
|
201
201
|
|
|
202
202
|
3. If the diff is neither irreversible/load-bearing (step 1: NO) nor standpoint-applicable (step 2: NO or DEGRADED) - the common case, ordinary small reversible self-improvement: skip straight to branch/commit/push/PR below.
|
|
203
203
|
|
|
@@ -46,7 +46,15 @@ scan_one() { # $1=file → prints hits, returns 0 hit / 1 none / 3 unscannable
|
|
|
46
46
|
if /usr/bin/file -b --mime-encoding "$f" 2>/dev/null | /usr/bin/grep -q '^binary$'; then
|
|
47
47
|
echo "UNSCANNABLE $f (binary)"; return 3; fi
|
|
48
48
|
# one pass: drop comment-led lines, then classify (VERDICT wins over EXPOSURE over IRREV per line)
|
|
49
|
-
|
|
49
|
+
# 🟥 pmh-dev #76 (2026-09-11): 여러 줄 docstring 의 «안쪽» 줄은 줄머리가 따옴표가 아니라 COMMENT_RE 를 통과했다 —
|
|
50
|
+
# 실물: 파이썬 테스트의 docstring 안 «fail-safe … 봉인» 문장이 GATE-SHAPED 로 지목됨. 줄 단위 grep 앞에
|
|
51
|
+
# """ / ''' 짝 상태를 awk 로 추적해 열린 동안의 줄을 전부 버린다(여는 줄·닫는 줄 포함). 한 줄 docstring 은
|
|
52
|
+
# 짝수라 상태가 안 바뀌고 종전 COMMENT_RE 가 그대로 거른다. 잔여(이름으로): JS 템플릿 리터럴(`) · 일반 문자열 안의 """.
|
|
53
|
+
body=$(/usr/bin/awk -v q="'''" -v t='"""' '
|
|
54
|
+
{ line=$0; n=gsub(t, "&", line); n+=gsub(q, "&", line)
|
|
55
|
+
if (indoc) { if (n % 2 == 1) indoc=0; next }
|
|
56
|
+
if (n % 2 == 1) { indoc=1; next }
|
|
57
|
+
print NR ":" $0 }' "$f" 2>/dev/null | /usr/bin/grep -Ev "^[0-9]+:${COMMENT_RE#^}" || true)
|
|
50
58
|
v=$(printf '%s\n' "$body" | /usr/bin/grep -Ei "$VERDICT_RE" || true)
|
|
51
59
|
v2=$(printf '%s\n' "$body" | /usr/bin/grep -E "$ENUM_RE" || true)
|
|
52
60
|
e=$(printf '%s\n' "$body" | /usr/bin/grep -E "$EXPOSURE_RE" || true)
|
|
@@ -64,7 +72,7 @@ scan_one() { # $1=file → prints hits, returns 0 hit / 1 none / 3 unscannable
|
|
|
64
72
|
}
|
|
65
73
|
|
|
66
74
|
selftest() { # known pair — positive must hit, negative must not, comment-only must not
|
|
67
|
-
local d rc_pos rc_pos2 rc_neg rc_cmt rc_bin rc_irr rc_bnd rc_star fails=0
|
|
75
|
+
local d rc_pos rc_pos2 rc_neg rc_cmt rc_bin rc_irr rc_bnd rc_star rc_doc rc_doc2 fails=0
|
|
68
76
|
d="$(mktemp -d 2>/dev/null)" || d=""
|
|
69
77
|
[ -n "$d" ] && [ -w "$d" ] || { echo "SELFTEST: ENV-BLOCKED (mktemp -d failed) — result unmeasured, not a pass"; return 3; }
|
|
70
78
|
printf 'export class S {\n start() {\n this.app.listen(this.config.port, () => {});\n }\n}\n' > "$d/pos_exposure.ts"
|
|
@@ -75,6 +83,8 @@ selftest() { # known pair — positive must hit, negative must not, comment-only
|
|
|
75
83
|
printf 'set -e\ngit push origin main \\\n --force\n' > "$d/pos_irrev.sh"
|
|
76
84
|
printf 'authored_by = "x"\nallowance = 3\nauthor = "y"\n' > "$d/neg_boundary.py"
|
|
77
85
|
printf 'int f(int *allow) {\n *allow = 1;\n return 0;\n}\n' > "$d/pos_star.c"
|
|
86
|
+
printf 'def t():\n """docstring\n PASS allow true — fail-safe 계통은 봉인\n """\n return 1\n' > "$d/neg_docstring.py" # pmh-dev #76
|
|
87
|
+
printf 'def t():\n """docstring\n PASS allow true\n """\n return Verdict.ALLOW\n' > "$d/pos_after_docstring.py"
|
|
78
88
|
scan_one "$d/pos_exposure.ts" >/dev/null; rc_pos=$?
|
|
79
89
|
scan_one "$d/pos_verdict.py" >/dev/null; rc_pos2=$?
|
|
80
90
|
scan_one "$d/neg_util.py" >/dev/null; rc_neg=$?
|
|
@@ -83,6 +93,8 @@ selftest() { # known pair — positive must hit, negative must not, comment-only
|
|
|
83
93
|
scan_one "$d/pos_irrev.sh" >/dev/null; rc_irr=$?
|
|
84
94
|
scan_one "$d/neg_boundary.py" >/dev/null; rc_bnd=$?
|
|
85
95
|
scan_one "$d/pos_star.c" >/dev/null; rc_star=$?
|
|
96
|
+
scan_one "$d/neg_docstring.py" >/dev/null; rc_doc=$?
|
|
97
|
+
scan_one "$d/pos_after_docstring.py" >/dev/null; rc_doc2=$?
|
|
86
98
|
[ "$rc_pos" -eq 0 ] && echo " ✅ known-positive exposure (listen()) → GATE-SHAPED" || { echo " ❌ known-positive exposure rc=$rc_pos"; fails=1; }
|
|
87
99
|
[ "$rc_pos2" -eq 0 ] && echo " ✅ known-positive verdict (ALLOW/DENY) → GATE-SHAPED" || { echo " ❌ known-positive verdict rc=$rc_pos2"; fails=1; }
|
|
88
100
|
[ "$rc_neg" -eq 1 ] && echo " ✅ known-negative util (docstring 'Pass') → NOT" || { echo " ❌ known-negative util rc=$rc_neg"; fails=1; }
|
|
@@ -91,6 +103,8 @@ selftest() { # known pair — positive must hit, negative must not, comment-only
|
|
|
91
103
|
[ "$rc_irr" -eq 0 ] && echo " ✅ known-positive irreversible (continued --force line) → GATE-SHAPED" || { echo " ❌ known-positive irrev rc=$rc_irr"; fails=1; }
|
|
92
104
|
[ "$rc_bnd" -eq 1 ] && echo " ✅ boundary: author/authored/allowance → NOT" || { echo " ❌ boundary rc=$rc_bnd"; fails=1; }
|
|
93
105
|
[ "$rc_star" -eq 0 ] && echo " ✅ code line starting with *allow (not a comment) → GATE-SHAPED" || { echo " ❌ star-code rc=$rc_star"; fails=1; }
|
|
106
|
+
[ "$rc_doc" -eq 1 ] && echo " ✅ multi-line docstring interior ('PASS allow') → NOT (pmh-dev #76)" || { echo " ❌ docstring interior rc=$rc_doc"; fails=1; }
|
|
107
|
+
[ "$rc_doc2" -eq 0 ] && echo " ✅ control: real verdict line AFTER the docstring → GATE-SHAPED" || { echo " ❌ after-docstring rc=$rc_doc2"; fails=1; }
|
|
94
108
|
/bin/rm -rf "$d"
|
|
95
109
|
[ "$fails" -eq 0 ] && { echo "SELFTEST: PASS"; return 0; } || { echo "SELFTEST: FAIL"; return 3; }
|
|
96
110
|
}
|