@dzhechkov/p-replicator 1.5.16 → 1.5.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +51 -23
- package/README.md +5 -3
- package/package.json +1 -1
- package/sbom.json +106 -36
- package/templates/.claude/skills/goap-research-ed25519/SKILL.md +340 -47
- package/templates/.claude/skills/goap-research-ed25519/scripts/check_report_evidence.py +359 -3
- package/templates/.claude/skills/goap-research-ed25519/scripts/ed25519_verifier.py +386 -13
- package/templates/.claude/skills/goap-research-ed25519/scripts/fixture_legacy_v2_fact.json +23 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/fixtures_field_cases.json +133 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/goap_planner.py +314 -44
- package/templates/.claude/skills/goap-research-ed25519/scripts/learning_bridge.py +891 -280
- package/templates/.claude/skills/goap-research-ed25519/scripts/population_match.py +591 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/risk_statement.py +289 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_ed25519_verifier.py +57 -2
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_evidence_provenance.py +970 -306
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_goap_planner.py +420 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_population_match.py +544 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_risk_absolute.py +239 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_signature_v3.py +554 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_suite_completeness.py +90 -0
- package/tests/snapshot/baseline.json +24 -9
|
@@ -0,0 +1,544 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
ADR-001 Confirmation tests — population applicability (T-9…T-11, T-15, T-18).
|
|
4
|
+
|
|
5
|
+
cd .../goap-research-ed25519/scripts && python3 -m unittest discover -s . -p 'test_*.py' -v
|
|
6
|
+
|
|
7
|
+
Each test names the property it proves. The four field cases are the acceptance bar:
|
|
8
|
+
they are the four times a correctly-signed, correctly-sourced fact flipped a real
|
|
9
|
+
conclusion, and they are replayed from a COMMITTED fixture, not restated inline.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import itertools
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import sys
|
|
16
|
+
|
|
17
|
+
# Import NOTHING local before this line (a stray __pycache__ in this vendored skill
|
|
18
|
+
# reads as canonical drift and turns an unrelated repo test red).
|
|
19
|
+
sys.dont_write_bytecode = True
|
|
20
|
+
|
|
21
|
+
import unittest
|
|
22
|
+
|
|
23
|
+
import population_match as pm
|
|
24
|
+
import check_report_evidence as gate
|
|
25
|
+
|
|
26
|
+
FIXTURES = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixtures_field_cases.json")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def _load_cases():
|
|
30
|
+
with open(FIXTURES, "r", encoding="utf-8") as handle:
|
|
31
|
+
return json.load(handle)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class FieldReversalTests(unittest.TestCase):
|
|
35
|
+
"""T-9 — the four real reversals, each NAMING its discrepancy (AC-1…AC-4)."""
|
|
36
|
+
|
|
37
|
+
def setUp(self):
|
|
38
|
+
self.data = _load_cases()
|
|
39
|
+
|
|
40
|
+
def _match_for(self, case):
|
|
41
|
+
profile = dict(self.data["patient_profile"])
|
|
42
|
+
if "patient_profile_override" in case:
|
|
43
|
+
profile = dict(case["patient_profile_override"])
|
|
44
|
+
return pm.match_from_fact(case["study_population"], profile)
|
|
45
|
+
|
|
46
|
+
def test_four_field_reversals_name_their_discrepancy(self):
|
|
47
|
+
"""A verdict alone is worthless: «не просто вердикт, а перечисление того, чем
|
|
48
|
+
именно пациент отличается». Every non-full verdict must carry a discrepancy
|
|
49
|
+
naming the field, the study requirement, the patient value and the source's
|
|
50
|
+
own words."""
|
|
51
|
+
real = [c for c in self.data["cases"] if c["kind"] == "real-field-reversal"]
|
|
52
|
+
self.assertEqual(len(real), 4, "the four field reversals are the acceptance bar")
|
|
53
|
+
for case in real:
|
|
54
|
+
with self.subTest(case=case["id"]):
|
|
55
|
+
match = self._match_for(case)
|
|
56
|
+
self.assertEqual(match.verdict, case["expected_verdict"])
|
|
57
|
+
self.assertEqual(len(match.discrepancies), len(case["expected_discrepancies"]))
|
|
58
|
+
for actual, expected in zip(match.discrepancies, case["expected_discrepancies"]):
|
|
59
|
+
self.assertEqual(actual.field, expected["field"])
|
|
60
|
+
self.assertEqual(actual.kind, expected["kind"])
|
|
61
|
+
self.assertEqual(actual.direction, expected["direction"])
|
|
62
|
+
self.assertTrue(actual.study_requirement.strip(), "must state the study's criterion")
|
|
63
|
+
self.assertTrue(actual.patient_value.strip(), "must state the patient's value")
|
|
64
|
+
self.assertTrue(actual.verbatim.strip(), "must quote the source's own words")
|
|
65
|
+
rendered = pm.render_population_match(match)
|
|
66
|
+
self.assertIn(case["expected_verdict"], rendered)
|
|
67
|
+
for expected in case["expected_discrepancies"]:
|
|
68
|
+
self.assertIn(expected["field"], rendered)
|
|
69
|
+
|
|
70
|
+
def test_eligibility_and_baseline_are_different_failures(self):
|
|
71
|
+
"""Cases 1/4 are `partial`, cases 2/3 are `none`, and the difference is the
|
|
72
|
+
criterion KIND, not the analyte. An inclusion-criteria-only model would
|
|
73
|
+
reproduce two of the four reversals and call it done."""
|
|
74
|
+
verdicts = {c["id"]: self._match_for(c).verdict
|
|
75
|
+
for c in self.data["cases"] if c["kind"] == "real-field-reversal"}
|
|
76
|
+
self.assertEqual(verdicts["testosterone-weight-loss"], "partial")
|
|
77
|
+
self.assertEqual(verdicts["omega3-ldl"], "partial")
|
|
78
|
+
self.assertEqual(verdicts["ed-rct-bmi30"], "none")
|
|
79
|
+
self.assertEqual(verdicts["traverse-cv-safety"], "none")
|
|
80
|
+
|
|
81
|
+
def test_synthetic_cases(self):
|
|
82
|
+
"""AC-5 (missing patient value) and AC-6 (full)."""
|
|
83
|
+
for case in [c for c in self.data["cases"] if c["kind"] == "synthetic"]:
|
|
84
|
+
with self.subTest(case=case["id"]):
|
|
85
|
+
self.assertEqual(self._match_for(case).verdict, case["expected_verdict"])
|
|
86
|
+
|
|
87
|
+
def test_same_matcher_no_per_case_branch(self):
|
|
88
|
+
"""FR-5 — all six cases flow through the SAME evaluator with different DATA."""
|
|
89
|
+
for case in self.data["cases"]:
|
|
90
|
+
with self.subTest(case=case["id"]):
|
|
91
|
+
population = pm.parse_study_population(case["study_population"])
|
|
92
|
+
profile = pm.PatientProfile(case.get("patient_profile_override", self.data["patient_profile"]))
|
|
93
|
+
discrepancies = pm.evaluate(population, profile)
|
|
94
|
+
self.assertEqual(pm.derive_verdict(discrepancies), case["expected_verdict"])
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class VerdictOrderingTests(unittest.TestCase):
|
|
98
|
+
"""T-10 / AM-6 — `unknown` is a distinct FOURTH verdict, never folded into partial."""
|
|
99
|
+
|
|
100
|
+
def _population(self, **criteria):
|
|
101
|
+
return pm.parse_study_population({"description": "d", "criteria": criteria})
|
|
102
|
+
|
|
103
|
+
def test_unknown_criterion_is_unknown_not_partial(self):
|
|
104
|
+
"""The paper does not state the axis. That is not a mild match."""
|
|
105
|
+
population = self._population(
|
|
106
|
+
bmi_min={"kind": "eligibility", "verbatim": "obese adults",
|
|
107
|
+
"unstated_reason": "the paper reports no BMI inclusion range"})
|
|
108
|
+
match = pm.match(population, pm.PatientProfile({"bmi": 25}))
|
|
109
|
+
self.assertEqual(match.verdict, "unknown")
|
|
110
|
+
self.assertNotEqual(match.verdict, "partial")
|
|
111
|
+
self.assertEqual(match.discrepancies[0].kind, "criterion-unstated")
|
|
112
|
+
|
|
113
|
+
def test_missing_patient_value_is_unknown(self):
|
|
114
|
+
"""The profile does not carry the axis — never a default, never a guess."""
|
|
115
|
+
population = self._population(
|
|
116
|
+
bmi_min={"op": ">=", "value": 30, "kind": "eligibility", "verbatim": "BMI >= 30"})
|
|
117
|
+
match = pm.match(population, pm.PatientProfile({"sex": "male"}))
|
|
118
|
+
self.assertEqual(match.verdict, "unknown")
|
|
119
|
+
self.assertEqual(match.discrepancies[0].kind, "patient-value-missing")
|
|
120
|
+
self.assertEqual(match.discrepancies[0].direction, "absent")
|
|
121
|
+
|
|
122
|
+
def test_wholly_unstated_population_is_unknown_with_its_reason(self):
|
|
123
|
+
population = pm.StudyPopulation.unstated("the abstract never describes who was enrolled")
|
|
124
|
+
match = pm.match(population, pm.PatientProfile({"bmi": 25}))
|
|
125
|
+
self.assertEqual(match.verdict, "unknown")
|
|
126
|
+
self.assertIn("never describes", match.discrepancies[0].study_requirement)
|
|
127
|
+
|
|
128
|
+
def test_eligibility_exclusion_outranks_unknown(self):
|
|
129
|
+
"""An established exclusion is knowledge; an unestablished criterion is not."""
|
|
130
|
+
population = self._population(
|
|
131
|
+
bmi_min={"op": ">=", "value": 30, "kind": "eligibility", "verbatim": "BMI >= 30"},
|
|
132
|
+
hba1c_max={"op": "<=", "value": 7, "kind": "eligibility", "verbatim": "HbA1c <= 7%"})
|
|
133
|
+
match = pm.match(population, pm.PatientProfile({"bmi": 25}))
|
|
134
|
+
self.assertEqual(match.verdict, "none", "none outranks unknown")
|
|
135
|
+
|
|
136
|
+
def test_full_iff_no_discrepancies_over_every_kind_subset(self):
|
|
137
|
+
"""D-10, exhaustively: over all 2^4 = 16 subsets of DISCREPANCY_KINDS,
|
|
138
|
+
`derive_verdict(d) == 'full'` IFF `d == []`. Imports the SAME `derive_verdict`
|
|
139
|
+
the runtime calls — one definition of the verdict predicate, shared."""
|
|
140
|
+
def sample(kind):
|
|
141
|
+
return pm.Discrepancy(field="bmi", kind=kind, patient_value="25",
|
|
142
|
+
study_requirement="bmi >= 30", verbatim="obese adults",
|
|
143
|
+
direction="absent" if kind.endswith(("unstated", "missing")) else "below")
|
|
144
|
+
|
|
145
|
+
seen = 0
|
|
146
|
+
for size in range(len(pm.DISCREPANCY_KINDS) + 1):
|
|
147
|
+
for combo in itertools.combinations(pm.DISCREPANCY_KINDS, size):
|
|
148
|
+
seen += 1
|
|
149
|
+
discrepancies = [sample(kind) for kind in combo]
|
|
150
|
+
verdict = pm.derive_verdict(discrepancies)
|
|
151
|
+
self.assertEqual(verdict == "full", len(discrepancies) == 0,
|
|
152
|
+
f"biconditional broken for {combo}")
|
|
153
|
+
self.assertIn(verdict, pm.POPULATION_MATCH_VERDICTS)
|
|
154
|
+
self.assertEqual(seen, 16, "2^4 states")
|
|
155
|
+
|
|
156
|
+
def test_verdict_cannot_be_asserted_out_of_thin_air(self):
|
|
157
|
+
"""D-9 — `derive_verdict` is the ONLY producer. A PopulationMatch whose verdict
|
|
158
|
+
was not derived from its own discrepancies refuses to exist."""
|
|
159
|
+
discrepancy = pm.Discrepancy(field="bmi", kind="eligibility-excluded", patient_value="25",
|
|
160
|
+
study_requirement="bmi >= 30", verbatim="BMI >= 30", direction="below")
|
|
161
|
+
with self.assertRaises(pm.PopulationError):
|
|
162
|
+
pm.PopulationMatch(verdict="full", discrepancies=(discrepancy,))
|
|
163
|
+
with self.assertRaises(pm.PopulationError):
|
|
164
|
+
pm.PopulationMatch(verdict="partial", discrepancies=())
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class VocabularyTests(unittest.TestCase):
|
|
168
|
+
"""T-11 / D-16 — the closed vocabulary fails LOUDLY, never silently."""
|
|
169
|
+
|
|
170
|
+
def test_unlisted_criterion_field_raises(self):
|
|
171
|
+
with self.assertRaises(pm.PopulationError) as ctx:
|
|
172
|
+
pm.parse_study_population({"description": "d", "criteria": {
|
|
173
|
+
"dialysis_status": {"op": "==", "value": "none", "kind": "eligibility",
|
|
174
|
+
"verbatim": "not on dialysis"}}})
|
|
175
|
+
self.assertIn("CRITERION_FIELDS", str(ctx.exception))
|
|
176
|
+
|
|
177
|
+
def test_unit_suffixes_resolve_to_the_same_axis(self):
|
|
178
|
+
self.assertEqual(pm.normalize_field("triglycerides_mg_dl_min"), "triglycerides")
|
|
179
|
+
self.assertEqual(pm.normalize_field("bmi_min"), "bmi")
|
|
180
|
+
self.assertEqual(pm.normalize_field("BMI"), "bmi")
|
|
181
|
+
|
|
182
|
+
def test_unrecognised_patient_key_can_never_satisfy_a_criterion(self):
|
|
183
|
+
"""The asymmetry is deliberate: a criterion naming an unlisted axis RAISES; a
|
|
184
|
+
patient value naming one is KEPT under its raw key, so the axis reads
|
|
185
|
+
`patient-value-missing` → `unknown`, never `full`."""
|
|
186
|
+
profile = pm.PatientProfile({"homa_ir": 3.1, "bmi": 33})
|
|
187
|
+
self.assertIn("homa_ir", profile.values)
|
|
188
|
+
population = pm.parse_study_population({"description": "d", "criteria": {
|
|
189
|
+
"bmi_min": {"op": ">=", "value": 30, "kind": "eligibility", "verbatim": "BMI >= 30"}}})
|
|
190
|
+
self.assertEqual(pm.match(population, profile).verdict, "full")
|
|
191
|
+
|
|
192
|
+
def test_blank_shapes_raise(self):
|
|
193
|
+
"""T-2's value-object half: `{description: 'adults', criteria: {}}` is not a shape."""
|
|
194
|
+
with self.assertRaises(pm.PopulationError):
|
|
195
|
+
pm.StudyPopulation(description=" ", criteria=())
|
|
196
|
+
with self.assertRaises(pm.PopulationError):
|
|
197
|
+
pm.StudyPopulation(description="adults", criteria=())
|
|
198
|
+
with self.assertRaises(pm.PopulationError):
|
|
199
|
+
pm.StudyPopulation.unstated(" ")
|
|
200
|
+
|
|
201
|
+
def test_criterion_kind_is_never_defaulted(self):
|
|
202
|
+
"""A bare scalar cannot state the kind, and the kind decides the verdict."""
|
|
203
|
+
with self.assertRaises(pm.PopulationError) as ctx:
|
|
204
|
+
pm.parse_study_population({"description": "d", "criteria": {"bmi_min": 30}})
|
|
205
|
+
self.assertIn("kind", str(ctx.exception))
|
|
206
|
+
|
|
207
|
+
def test_verbatim_is_mandatory(self):
|
|
208
|
+
with self.assertRaises(pm.PopulationError):
|
|
209
|
+
pm.parse_study_population({"description": "d", "criteria": {
|
|
210
|
+
"bmi_min": {"op": ">=", "value": 30, "kind": "eligibility"}}})
|
|
211
|
+
|
|
212
|
+
def test_incomparable_patient_value_refuses_loudly(self):
|
|
213
|
+
population = pm.parse_study_population({"description": "d", "criteria": {
|
|
214
|
+
"bmi_min": {"op": ">=", "value": 30, "kind": "eligibility", "verbatim": "BMI >= 30"}}})
|
|
215
|
+
with self.assertRaises(pm.PopulationError):
|
|
216
|
+
pm.match(population, pm.PatientProfile({"bmi": "не измерялся"}))
|
|
217
|
+
|
|
218
|
+
def test_renderer_iterates_the_allowlist_not_the_dataclass(self):
|
|
219
|
+
"""D-5's sibling for output: DISCREPANCY_KEYS is an allowlist over emitted
|
|
220
|
+
structure, so the renderer's key set is exactly the tuple."""
|
|
221
|
+
self.assertEqual(set(pm.DISCREPANCY_KEYS),
|
|
222
|
+
{"field", "kind", "patient_value", "study_requirement",
|
|
223
|
+
"verbatim", "locator", "direction"})
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
class GateIntegrationTests(unittest.TestCase):
|
|
227
|
+
"""T-15 / T-18 — POPULATION_MATCH is load-bearing at the report gate, not decorative."""
|
|
228
|
+
|
|
229
|
+
def setUp(self):
|
|
230
|
+
self.data = _load_cases()
|
|
231
|
+
self.case = next(c for c in self.data["cases"] if c["id"] == "ed-rct-bmi30")
|
|
232
|
+
self.fact = {
|
|
233
|
+
"claim": "Erectile dysfunction is reversible in this population",
|
|
234
|
+
"source_url": "https://pubmed.ncbi.nlm.nih.gov/1",
|
|
235
|
+
"evidence_class": "LISTING_ONLY",
|
|
236
|
+
"study_population": self.case["study_population"],
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
def test_unmarked_population_mismatch_fires_gate(self):
|
|
240
|
+
bare = ("Conclusion: Erectile dysfunction is reversible in this population, so weight "
|
|
241
|
+
"loss should be recommended.")
|
|
242
|
+
findings, _ = gate.evaluate_population(bare, [self.fact], self.data["patient_profile"])
|
|
243
|
+
self.assertTrue(findings, "an unmarked mismatch must fire")
|
|
244
|
+
self.assertEqual(findings[0].kind, "UNMARKED_POPULATION_MISMATCH")
|
|
245
|
+
self.assertIn("bmi", findings[0].detail)
|
|
246
|
+
|
|
247
|
+
marked = ("Conclusion: Erectile dysfunction is reversible in this population "
|
|
248
|
+
"(POPULATION_MATCH none — the study population required bmi >= 30 and this "
|
|
249
|
+
"patient's bmi is 25), so the finding does not transfer.")
|
|
250
|
+
findings, _ = gate.evaluate_population(marked, [self.fact], self.data["patient_profile"])
|
|
251
|
+
self.assertFalse(findings, "a marker naming the diverging axis clears it")
|
|
252
|
+
|
|
253
|
+
def test_every_occurrence_needs_its_own_population_marker(self):
|
|
254
|
+
claim = self.fact["claim"]
|
|
255
|
+
text = (f"Early on: {claim} (POPULATION_MATCH none — bmi 25 vs bmi >= 30)."
|
|
256
|
+
+ " filler." * 150
|
|
257
|
+
+ f" Later we repeat that {claim} with no warning at all.")
|
|
258
|
+
findings, _ = gate.evaluate_population(text, [self.fact], self.data["patient_profile"])
|
|
259
|
+
self.assertTrue(findings, "a marker on page 1 does not warn the reader on page 2")
|
|
260
|
+
|
|
261
|
+
def test_generic_caveat_without_the_axis_does_not_clear_it(self):
|
|
262
|
+
text = ("Conclusion: Erectile dysfunction is reversible in this population. "
|
|
263
|
+
"Note on study population: results may not generalise.")
|
|
264
|
+
findings, _ = gate.evaluate_population(text, [self.fact], self.data["patient_profile"])
|
|
265
|
+
self.assertTrue(findings, "boilerplate that names no axis tells the reader nothing")
|
|
266
|
+
|
|
267
|
+
def test_unknown_verdict_has_its_own_finding_kind(self):
|
|
268
|
+
fact = dict(self.fact, study_population={
|
|
269
|
+
"description": "population not stated", "criteria": {},
|
|
270
|
+
"unstated_reason": "the abstract never describes who was enrolled"})
|
|
271
|
+
text = "Conclusion: Erectile dysfunction is reversible in this population."
|
|
272
|
+
findings, _ = gate.evaluate_population(text, [fact], self.data["patient_profile"])
|
|
273
|
+
self.assertEqual(findings[0].kind, "POPULATION_UNKNOWN_UNMARKED")
|
|
274
|
+
|
|
275
|
+
def test_missing_study_population_needs_no_patient_to_be_wrong(self):
|
|
276
|
+
fact = dict(self.fact, study_population={})
|
|
277
|
+
text = "Conclusion: Erectile dysfunction is reversible in this population."
|
|
278
|
+
findings, _ = gate.evaluate_population(text, [fact], None)
|
|
279
|
+
self.assertEqual(findings[0].kind, "MISSING_STUDY_POPULATION")
|
|
280
|
+
|
|
281
|
+
def test_legacy_fact_counted_not_folded_into_clean(self):
|
|
282
|
+
"""T-18 / D-7 — a used v1/v2 fact gets its OWN named line AND its own finding.
|
|
283
|
+
|
|
284
|
+
AMENDED by QE G4. The first version asserted `assertFalse(findings)` and so
|
|
285
|
+
encoded the defect it was written to prevent: the count went up, no finding was
|
|
286
|
+
raised, and `main()` returned 0 on a report resting ENTIRELY on facts the gate
|
|
287
|
+
cannot judge — while this module's own docstring claimed they are «never folded
|
|
288
|
+
into clean (D-7)». A count nobody has to read is the same shape as
|
|
289
|
+
"inconclusive reads as pass".
|
|
290
|
+
"""
|
|
291
|
+
legacy = {"claim": "Erectile dysfunction is reversible in this population",
|
|
292
|
+
"source_url": "u", "evidence_class": "LISTING_ONLY"}
|
|
293
|
+
text = "Conclusion: Erectile dysfunction is reversible in this population."
|
|
294
|
+
findings, counts = gate.evaluate_population(text, [legacy], self.data["patient_profile"])
|
|
295
|
+
self.assertEqual([f.kind for f in findings], ["LEGACY_POPULATION_UNJUDGEABLE"],
|
|
296
|
+
"unjudgeable is not clean")
|
|
297
|
+
self.assertEqual(counts["legacy-population-unknown"], 1, "…and it is COUNTED, by name")
|
|
298
|
+
rendered = gate.render_population_and_risk(counts, True)
|
|
299
|
+
self.assertIn("legacy-population-unknown", rendered)
|
|
300
|
+
|
|
301
|
+
def test_absent_profile_is_printed_never_silently_passed(self):
|
|
302
|
+
"""D-17 — unevaluable is never clean, and the gate SAYS which check did not run."""
|
|
303
|
+
text = "Conclusion: Erectile dysfunction is reversible in this population."
|
|
304
|
+
findings, counts = gate.evaluate_population(text, [self.fact], None)
|
|
305
|
+
self.assertFalse(findings)
|
|
306
|
+
self.assertIn(gate.POPULATION_UNCHECKED_LINE, gate.render_population_and_risk(counts, False))
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
class UnattestedPopulationTests(unittest.TestCase):
|
|
310
|
+
"""QE G1 — the slice's central guarantee, for the facts it was meant to protect.
|
|
311
|
+
|
|
312
|
+
A pre-v3 signed message does not cover `study_population`. So a legitimately-signed
|
|
313
|
+
v1/v2 fact can be handed a fabricated population in a text editor, with
|
|
314
|
+
`schema_version` pinned to its OWN schema so the signature still verifies, and the
|
|
315
|
+
gate used to answer `POPULATION_MATCH full` with zero findings.
|
|
316
|
+
|
|
317
|
+
MEASURED before the fix, against this same committed fixture: `verified=True
|
|
318
|
+
conf=0.5 schema=2`, `signed_fields` correctly omitting `study_population`, counts
|
|
319
|
+
`{'population-checked': 1, 'population-full': 1}`, findings `[]`.
|
|
320
|
+
"""
|
|
321
|
+
|
|
322
|
+
FIXTURE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
323
|
+
"fixture_legacy_v2_fact.json")
|
|
324
|
+
|
|
325
|
+
def _injected_fact(self, **overrides):
|
|
326
|
+
with open(self.FIXTURE, "r", encoding="utf-8") as handle:
|
|
327
|
+
fact = dict(json.load(handle)["fact"])
|
|
328
|
+
fact["study_population"] = {
|
|
329
|
+
"description": "adults aged 18-99, any BMI",
|
|
330
|
+
"criteria": {"bmi_min": {"value": 0, "kind": "baseline",
|
|
331
|
+
"verbatim": "any BMI", "locator": "[fabricated]"}},
|
|
332
|
+
}
|
|
333
|
+
fact["schema_version"] = 2 # pinned to its OWN schema, so the signature holds
|
|
334
|
+
fact.update(overrides)
|
|
335
|
+
return fact
|
|
336
|
+
|
|
337
|
+
def _report(self, fact):
|
|
338
|
+
return "The finding: %s. See the table.\n" % fact["claim"]
|
|
339
|
+
|
|
340
|
+
def test_the_injected_population_still_verifies_which_is_the_premise(self):
|
|
341
|
+
"""Not a proof of the fix — the SETUP the fix has to survive. If this ever goes
|
|
342
|
+
red the attack changed shape and the test below is measuring something else."""
|
|
343
|
+
import ed25519_verifier as ev
|
|
344
|
+
if ev.CRYPTO_BACKEND is None:
|
|
345
|
+
self.skipTest("No Ed25519 backend installed")
|
|
346
|
+
fact = self._injected_fact()
|
|
347
|
+
result = ev.Ed25519Verifier().verify_fact(ev.SignedFact.from_dict(fact))
|
|
348
|
+
self.assertTrue(result.verified, "the signature covers the v2 fields and still holds")
|
|
349
|
+
self.assertEqual(result.schema_version, 2)
|
|
350
|
+
self.assertNotIn("study_population", result.signed_fields,
|
|
351
|
+
"the signed message does not cover the injected field — that IS the hole")
|
|
352
|
+
|
|
353
|
+
def test_unattested_population_is_refused_never_matched(self):
|
|
354
|
+
fact = self._injected_fact()
|
|
355
|
+
findings, counts = gate.evaluate_population(self._report(fact), [fact], {"bmi": 25})
|
|
356
|
+
self.assertEqual([f.kind for f in findings], ["UNATTESTED_STUDY_POPULATION"])
|
|
357
|
+
self.assertEqual(counts["population-unattested"], 1)
|
|
358
|
+
self.assertEqual(counts["population-full"], 0,
|
|
359
|
+
"an unsigned population must never produce a clean full match")
|
|
360
|
+
self.assertEqual(counts["population-checked"], 0,
|
|
361
|
+
"it is not CHECKED against the patient at all — that is the point")
|
|
362
|
+
self.assertIn("population-unattested", gate.render_population_and_risk(counts, True))
|
|
363
|
+
|
|
364
|
+
def test_a_genuine_v3_population_is_still_matched(self):
|
|
365
|
+
"""DISCRIMINATION in the other direction: the refusal must be about ATTESTATION,
|
|
366
|
+
not about populations in general. Same fixture, same fabricated population, but
|
|
367
|
+
presented as v3 — now it is evaluated (and, being satisfied, matches full)."""
|
|
368
|
+
fact = self._injected_fact(schema_version=3)
|
|
369
|
+
findings, counts = gate.evaluate_population(self._report(fact), [fact], {"bmi": 25})
|
|
370
|
+
self.assertEqual(counts["population-unattested"], 0)
|
|
371
|
+
self.assertEqual(counts["population-checked"], 1)
|
|
372
|
+
self.assertEqual(counts["population-full"], 1)
|
|
373
|
+
self.assertEqual(findings, [])
|
|
374
|
+
|
|
375
|
+
def test_a_malformed_schema_version_reads_as_unattested_not_as_v3(self):
|
|
376
|
+
"""QE G6's composition with G1: a version the dispatch cannot parse is UNKNOWN,
|
|
377
|
+
and unknown is never 'attested'. It also must not raise out of the gate."""
|
|
378
|
+
fact = self._injected_fact(schema_version="not-a-number")
|
|
379
|
+
findings, counts = gate.evaluate_population(self._report(fact), [fact], {"bmi": 25})
|
|
380
|
+
self.assertEqual([f.kind for f in findings], ["UNATTESTED_STUDY_POPULATION"])
|
|
381
|
+
self.assertEqual(counts["population-unattested"], 1)
|
|
382
|
+
|
|
383
|
+
def test_the_gate_exits_one_on_an_unattested_population(self):
|
|
384
|
+
import tempfile
|
|
385
|
+
fact = self._injected_fact()
|
|
386
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
387
|
+
report = os.path.join(tmp, "r.md")
|
|
388
|
+
facts = os.path.join(tmp, "f.json")
|
|
389
|
+
profile = os.path.join(tmp, "p.json")
|
|
390
|
+
with open(report, "w", encoding="utf-8") as fh:
|
|
391
|
+
fh.write(self._report(fact))
|
|
392
|
+
with open(facts, "w", encoding="utf-8") as fh:
|
|
393
|
+
json.dump([fact], fh)
|
|
394
|
+
with open(profile, "w", encoding="utf-8") as fh:
|
|
395
|
+
json.dump({"bmi": 25}, fh)
|
|
396
|
+
self.assertEqual(
|
|
397
|
+
gate.main(["--report", report, "--facts", facts, "--profile", profile]), 1)
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
class LegacyFactExitCodeTests(unittest.TestCase):
|
|
401
|
+
"""QE G4 — a report resting entirely on legacy facts must not exit 0.
|
|
402
|
+
|
|
403
|
+
MEASURED before the fix: one used fact with no `study_population` produced
|
|
404
|
+
`counts['legacy-population-unknown'] = 1`, ZERO findings, and `main()` returned 0.
|
|
405
|
+
"""
|
|
406
|
+
|
|
407
|
+
def _write(self, tmp, fact, report_text):
|
|
408
|
+
report = os.path.join(tmp, "r.md")
|
|
409
|
+
facts = os.path.join(tmp, "f.json")
|
|
410
|
+
profile = os.path.join(tmp, "p.json")
|
|
411
|
+
with open(report, "w", encoding="utf-8") as fh:
|
|
412
|
+
fh.write(report_text)
|
|
413
|
+
with open(facts, "w", encoding="utf-8") as fh:
|
|
414
|
+
json.dump([fact], fh)
|
|
415
|
+
with open(profile, "w", encoding="utf-8") as fh:
|
|
416
|
+
json.dump({"bmi": 25}, fh)
|
|
417
|
+
return report, facts, profile
|
|
418
|
+
|
|
419
|
+
def test_legacy_only_report_exits_one_with_and_without_a_profile(self):
|
|
420
|
+
import tempfile
|
|
421
|
+
fixture = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
422
|
+
"fixture_legacy_v2_fact.json")
|
|
423
|
+
with open(fixture, "r", encoding="utf-8") as handle:
|
|
424
|
+
fact = json.load(handle)["fact"]
|
|
425
|
+
text = ("The finding: %s (LISTING_ONLY, карточка не открывалась).\n" % fact["claim"])
|
|
426
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
427
|
+
report, facts, profile = self._write(tmp, fact, text)
|
|
428
|
+
self.assertEqual(
|
|
429
|
+
gate.main(["--report", report, "--facts", facts, "--profile", profile]), 1,
|
|
430
|
+
"a report resting on facts the gate cannot judge is unjudged, not clean")
|
|
431
|
+
self.assertEqual(
|
|
432
|
+
gate.main(["--report", report, "--facts", facts]), 1,
|
|
433
|
+
"the verdict does not depend on the patient — the fact is unjudgeable for anyone")
|
|
434
|
+
|
|
435
|
+
def test_an_unused_legacy_fact_is_still_not_this_gate_s_business(self):
|
|
436
|
+
"""The bound on G4's fix: only USED facts are judged. A ledger may hold legacy
|
|
437
|
+
records the report never leans on, and those are not violations."""
|
|
438
|
+
legacy = {"claim": "some entirely unrelated claim about ferritin kinetics",
|
|
439
|
+
"source_url": "u", "evidence_class": "LISTING_ONLY"}
|
|
440
|
+
findings, counts = gate.evaluate_population(
|
|
441
|
+
"This report discusses lipoprotein(a) and nothing else at all.", [legacy], {"bmi": 25})
|
|
442
|
+
self.assertEqual(findings, [])
|
|
443
|
+
self.assertEqual(counts["legacy-population-unknown"], 0)
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class UnstatedPopulationMarkerTests(unittest.TestCase):
|
|
447
|
+
"""QE G5 — the sanctioned escape hatch must not be the one path that is punished.
|
|
448
|
+
|
|
449
|
+
`StudyPopulation.unstated(reason)` is the honest way to say the population is
|
|
450
|
+
unknown. Its discrepancy field is the sentinel `(study population)`, and the marker
|
|
451
|
+
rule demanded that literal token appear in the prose, so a report carrying
|
|
452
|
+
`POPULATION_MATCH unknown` plus four natural caveats was STILL flagged
|
|
453
|
+
`POPULATION_UNKNOWN_UNMARKED` (MEASURED).
|
|
454
|
+
"""
|
|
455
|
+
|
|
456
|
+
def setUp(self):
|
|
457
|
+
self.population = pm.StudyPopulation.unstated(
|
|
458
|
+
"the abstract never describes who was enrolled").to_dict()
|
|
459
|
+
self.fact = {"claim": "Testosterone improved erectile function in this group",
|
|
460
|
+
"source_url": "https://pubmed.ncbi.nlm.nih.gov/1",
|
|
461
|
+
"schema_version": 3,
|
|
462
|
+
"study_population": self.population}
|
|
463
|
+
|
|
464
|
+
def _findings(self, text):
|
|
465
|
+
return [f.kind for f in gate.evaluate_population(text, [self.fact], {"bmi": 25})[0]]
|
|
466
|
+
|
|
467
|
+
def test_the_sentinel_has_one_home(self):
|
|
468
|
+
"""The gate recognises the matcher's constant, not a second copy of the string."""
|
|
469
|
+
match = pm.match(pm.StudyPopulation.unstated("r"), pm.PatientProfile({"bmi": 25}))
|
|
470
|
+
self.assertEqual(match.discrepancies[0].field, pm.UNSTATED_POPULATION_FIELD)
|
|
471
|
+
|
|
472
|
+
def test_honest_prose_clears_the_unstated_case(self):
|
|
473
|
+
for caveat in ("Популяция исследования не указана в источнике.",
|
|
474
|
+
"The study population is not stated by the paper.",
|
|
475
|
+
"POPULATION_MATCH unknown — study population unknown.",
|
|
476
|
+
"Популяция исследования неизвестна."):
|
|
477
|
+
with self.subTest(caveat=caveat):
|
|
478
|
+
text = ("Testosterone improved erectile function in this group. " + caveat)
|
|
479
|
+
self.assertEqual(self._findings(text), [],
|
|
480
|
+
"an honest sentence saying the population is unknown must clear it")
|
|
481
|
+
|
|
482
|
+
def test_generic_boilerplate_still_does_not_clear_it(self):
|
|
483
|
+
"""The anti-boilerplate property the axis rule exists for is PRESERVED: saying
|
|
484
|
+
«study population» while saying nothing about it is still not a caveat."""
|
|
485
|
+
text = ("Testosterone improved erectile function in this group. "
|
|
486
|
+
"Note on study population: results may not generalise.")
|
|
487
|
+
self.assertEqual(self._findings(text), ["POPULATION_UNKNOWN_UNMARKED"])
|
|
488
|
+
|
|
489
|
+
def test_bare_text_still_fires(self):
|
|
490
|
+
self.assertEqual(self._findings("Testosterone improved erectile function in this group."),
|
|
491
|
+
["POPULATION_UNKNOWN_UNMARKED"])
|
|
492
|
+
|
|
493
|
+
def test_a_named_axis_still_has_to_be_named(self):
|
|
494
|
+
"""The exception is bounded to the sentinel: when the source DOES state an axis,
|
|
495
|
+
the report must still name THAT axis — an 'unknown population' phrase nearby
|
|
496
|
+
does not clear a bmi mismatch."""
|
|
497
|
+
fact = {"claim": "Erectile dysfunction is reversible in this population",
|
|
498
|
+
"source_url": "u", "schema_version": 3,
|
|
499
|
+
"study_population": {"description": "men with obesity", "criteria": {
|
|
500
|
+
"bmi_min": {"op": ">=", "value": 30, "kind": "eligibility",
|
|
501
|
+
"verbatim": "BMI >= 30"}}}}
|
|
502
|
+
text = ("Erectile dysfunction is reversible in this population. "
|
|
503
|
+
"POPULATION_MATCH: the study population is not stated in detail.")
|
|
504
|
+
findings, _ = gate.evaluate_population(text, [fact], {"bmi": 25})
|
|
505
|
+
self.assertEqual([f.kind for f in findings], ["UNMARKED_POPULATION_MISMATCH"])
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
class ModuleHeaderHonestyTests(unittest.TestCase):
|
|
509
|
+
"""QE G8 — a header must not claim a property its own code falsifies.
|
|
510
|
+
|
|
511
|
+
`population_match.py` said «Pure: no I/O, no network, no global state» while
|
|
512
|
+
`load_field_cases()` opened a file. The SENTENCE was wrong, not the function, so the
|
|
513
|
+
sentence was narrowed — and this test keeps the narrowed claim true by construction.
|
|
514
|
+
"""
|
|
515
|
+
|
|
516
|
+
SOURCE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "population_match.py")
|
|
517
|
+
|
|
518
|
+
def _text(self):
|
|
519
|
+
with open(self.SOURCE, "r", encoding="utf-8") as handle:
|
|
520
|
+
return handle.read()
|
|
521
|
+
|
|
522
|
+
def test_the_header_does_not_claim_blanket_purity(self):
|
|
523
|
+
header = self._text().split('"""')[1]
|
|
524
|
+
self.assertNotIn("Pure: no I/O", header)
|
|
525
|
+
self.assertIn("EVALUATION PATH IS I/O-FREE", header)
|
|
526
|
+
self.assertIn("load_field_cases", header)
|
|
527
|
+
|
|
528
|
+
def test_load_field_cases_is_the_only_file_opener(self):
|
|
529
|
+
"""Layer 1, not reviewer judgment: if a second function ever opens a file, the
|
|
530
|
+
narrowed sentence becomes false and THIS goes red."""
|
|
531
|
+
text = self._text()
|
|
532
|
+
openers = []
|
|
533
|
+
for index, line in enumerate(text.splitlines(), start=1):
|
|
534
|
+
if "open(" not in line or line.strip().startswith("#"):
|
|
535
|
+
continue
|
|
536
|
+
preceding = "\n".join(text.splitlines()[:index])
|
|
537
|
+
owner = [ln for ln in preceding.splitlines() if ln.startswith("def ")]
|
|
538
|
+
openers.append(owner[-1] if owner else f"module level (line {index})")
|
|
539
|
+
self.assertEqual(openers, ["def load_field_cases(path: str) -> Dict[str, Any]:"],
|
|
540
|
+
f"only load_field_cases may perform I/O; found {openers}")
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
if __name__ == "__main__":
|
|
544
|
+
unittest.main()
|