@dzhechkov/p-replicator 1.5.13 → 1.5.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +677 -0
- package/CHANGELOG.md +874 -0
- package/KNOWN_LIMITATIONS.md +327 -0
- package/MULTIPLATFORM_ROADMAP.md +239 -0
- package/README/eng/06_troubleshooting.md +1 -1
- package/README/eng/07_changelog.md +57 -0
- package/README/eng/README.md +2 -2
- package/README/ru/06_troubleshooting.md +1 -1
- package/README/ru/07_changelog.md +58 -0
- package/README/ru/README.md +2 -2
- package/README/ru/html/build.js +7 -7
- package/README/ru/html/index.html +31 -12
- package/README.md +59 -10
- package/package.json +10 -3
- package/sbom.json +1683 -0
- package/templates/.claude/skills/explore/SKILL.md +1 -1
- package/templates/.claude/skills/goap-research-ed25519/SKILL.md +141 -11
- package/templates/.claude/skills/goap-research-ed25519/scripts/check_report_evidence.py +363 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/ed25519_verifier.py +297 -4
- package/templates/.claude/skills/goap-research-ed25519/scripts/evidence_fetch.py +277 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/fixture_legacy_v1_fact.json +21 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/learning_bridge.py +462 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/source_tiers.py +170 -0
- package/templates/.claude/skills/goap-research-ed25519/scripts/test_evidence_provenance.py +784 -0
- package/templates/.claude/skills/problem-solver-enhanced/SKILL.md +1 -1
- package/templates/.claude/skills/reverse-engineering-unicorn/SKILL.md +1 -1
- package/tests/e2e/lifecycle.test.js +973 -0
- package/tests/snapshot/baseline.json +125 -0
- package/tests/snapshot/templates.test.js +89 -0
- package/tests/snapshot/update-baseline.js +68 -0
- package/tests/unit/utils.test.js +636 -0
|
@@ -0,0 +1,784 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
ADR Confirmation tests for the evidence-provenance axis.
|
|
4
|
+
|
|
5
|
+
Every test here names the ADR property it proves. The three tamper tests are the
|
|
6
|
+
core: an evidence class that can be edited with a text editor is not evidence of
|
|
7
|
+
anything, and this feature would be decoration without them.
|
|
8
|
+
|
|
9
|
+
python3 -m unittest test_evidence_provenance -v
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
# Import NOTHING from this directory before this line. Running these tests used to
|
|
15
|
+
# leave a __pycache__ inside the skill directory, and this skill is vendored into 11
|
|
16
|
+
# copies kept byte-identical — so the stray directory read as canonical DRIFT and
|
|
17
|
+
# turned an unrelated repo test red. Bytecode caching buys nothing for a suite that
|
|
18
|
+
# runs in half a second; not writing it removes the failure mode entirely.
|
|
19
|
+
sys.dont_write_bytecode = True
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import threading
|
|
23
|
+
import unittest
|
|
24
|
+
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
25
|
+
|
|
26
|
+
import ed25519_verifier as ev
|
|
27
|
+
import evidence_fetch as ef
|
|
28
|
+
import source_tiers as st
|
|
29
|
+
import check_report_evidence as gate
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _verifier():
|
|
33
|
+
v = ev.Ed25519Verifier()
|
|
34
|
+
v.generate_keypair()
|
|
35
|
+
return v
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _authentic_record(**overrides):
|
|
39
|
+
"""A FetchRecord bearing this process's fetch witness.
|
|
40
|
+
|
|
41
|
+
The test reaches into `evidence_fetch._FETCH_WITNESS` ON PURPOSE: these cases
|
|
42
|
+
exercise SIGNATURE coverage, not fetch authenticity, and they must start from
|
|
43
|
+
a record the API accepts. Authenticity itself is proven separately by
|
|
44
|
+
`test_manual_path_cannot_mint_fetch_verified` (hand-built record REFUSED) and
|
|
45
|
+
by the live local-server fetches in FetchTests.
|
|
46
|
+
"""
|
|
47
|
+
base = dict(
|
|
48
|
+
url="https://pubmed.ncbi.nlm.nih.gov/123",
|
|
49
|
+
final_url="https://pubmed.ncbi.nlm.nih.gov/123",
|
|
50
|
+
status=200,
|
|
51
|
+
sha256_body="a" * 64,
|
|
52
|
+
bytes_len=10,
|
|
53
|
+
fetched_at="2026-08-03T10:00:00Z",
|
|
54
|
+
witness=ef._FETCH_WITNESS,
|
|
55
|
+
)
|
|
56
|
+
base.update(overrides)
|
|
57
|
+
return ef.FetchRecord(**base)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class EvidenceAxisTests(unittest.TestCase):
|
|
61
|
+
"""ADR-001 — the axis is ORTHOGONAL to trust_class."""
|
|
62
|
+
|
|
63
|
+
def test_signed_but_asserted_is_expressible_and_gated(self):
|
|
64
|
+
"""ADR-001 Confirmation: ISSUER_SIGNED + ASSERTED must be REPRESENTABLE,
|
|
65
|
+
verifiable as signed, and still refused by the report gate. Collapsing the
|
|
66
|
+
two axes into one field would make this state impossible to say."""
|
|
67
|
+
v = _verifier()
|
|
68
|
+
fact = v.create_asserted_fact(claim="Testosterone recovers without refeeding", issuer="researcher")
|
|
69
|
+
# Promote the trust axis while leaving the evidence axis untouched, then
|
|
70
|
+
# re-sign: this is a legitimate actor asserting provenance of a record it
|
|
71
|
+
# never read — the dangerous quadrant, and it must be expressible.
|
|
72
|
+
fact.trust_class = ev.TRUST_CLASS_ISSUER_SIGNED
|
|
73
|
+
fact.signature, _ = v.sign_content(ev.canonical_fact_message(fact))
|
|
74
|
+
v.registry.add("researcher", v.get_public_key_b64())
|
|
75
|
+
|
|
76
|
+
result = v.verify_fact(fact)
|
|
77
|
+
self.assertTrue(result.verified, "the signature itself is sound — the axes are independent")
|
|
78
|
+
self.assertEqual(fact.evidence_class, ev.EVIDENCE_ASSERTED)
|
|
79
|
+
self.assertEqual(result.confidence, 0.0, "evidence ceiling wins over trust ceiling (weakest link)")
|
|
80
|
+
|
|
81
|
+
report = "Conclusion: Testosterone recovers without refeeding, so no action is needed."
|
|
82
|
+
findings, _ = gate.evaluate(report, [json.loads(fact.to_json())])
|
|
83
|
+
self.assertTrue(findings, "a signed-but-unread claim must still be refused by the gate")
|
|
84
|
+
self.assertEqual(findings[0].kind, "ASSERTED_IN_REPORT")
|
|
85
|
+
|
|
86
|
+
def test_asserted_confidence_is_zero_not_low(self):
|
|
87
|
+
v = _verifier()
|
|
88
|
+
fact = v.create_asserted_fact(claim="X causes Y")
|
|
89
|
+
self.assertEqual(fact.confidence, 0.0)
|
|
90
|
+
self.assertEqual(ev.evidence_ceiling(ev.EVIDENCE_ASSERTED), 0.0)
|
|
91
|
+
|
|
92
|
+
def test_legacy_fact_evidence_is_unknown_not_guessed(self):
|
|
93
|
+
"""A pre-axis fact must read as UNKNOWN — neither ASSERTED nor VERIFIED."""
|
|
94
|
+
v = _verifier()
|
|
95
|
+
legacy = v.create_signed_fact(claim="c", source_url="https://x.test", source_content="body", issuer="researcher")
|
|
96
|
+
self.assertIsNone(legacy.evidence_class)
|
|
97
|
+
self.assertEqual(ev.fact_schema_version(legacy), 1)
|
|
98
|
+
self.assertEqual(ev.evidence_ceiling(None), 1.0, "unknown contributes no ceiling of its own")
|
|
99
|
+
self.assertTrue(v.verify_fact(legacy).verified)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class SignatureCoverageTests(unittest.TestCase):
|
|
103
|
+
"""ADR-002 — the evidence class is UNDER the signature; both tamper directions fail."""
|
|
104
|
+
|
|
105
|
+
def setUp(self):
|
|
106
|
+
self.v = _verifier()
|
|
107
|
+
self.record = _authentic_record()
|
|
108
|
+
|
|
109
|
+
def test_evidence_class_tamper_swap_fails(self):
|
|
110
|
+
"""Relabel LISTING_ONLY → FETCH_VERIFIED on a signed fact."""
|
|
111
|
+
fact = self.v.create_listing_fact(claim="c", source_url="https://x.test", reason="offline run")
|
|
112
|
+
self.assertTrue(self.v.verify_fact(fact).verified)
|
|
113
|
+
fact.evidence_class = ev.EVIDENCE_FETCH_VERIFIED
|
|
114
|
+
self.assertFalse(self.v.verify_fact(fact).verified, "swapping the class must break the signature")
|
|
115
|
+
|
|
116
|
+
def test_evidence_class_tamper_strip_fails(self):
|
|
117
|
+
"""DOWNGRADE attack: remove the field so verification falls back to v1.
|
|
118
|
+
What breaks it is that the signed v2 text CONTAINS the three evidence keys
|
|
119
|
+
at all — not the schema marker (a discrimination run refuted that)."""
|
|
120
|
+
fact = self.v.create_fetched_fact(claim="c", fetch_record=self.record, issuer="researcher")
|
|
121
|
+
self.assertTrue(self.v.verify_fact(fact).verified)
|
|
122
|
+
fact.evidence_class = None
|
|
123
|
+
self.assertEqual(ev.fact_schema_version(fact), 1, "the fact now looks legacy…")
|
|
124
|
+
self.assertFalse(self.v.verify_fact(fact).verified, "…but its v2 signature must not accept the v1 message")
|
|
125
|
+
|
|
126
|
+
def test_evidence_class_tamper_add_fails(self):
|
|
127
|
+
"""UPGRADE attack: bolt the field onto a legacy v1 fact."""
|
|
128
|
+
legacy = self.v.create_signed_fact(claim="c", source_url="https://x.test", source_content="b", issuer="researcher")
|
|
129
|
+
self.assertTrue(self.v.verify_fact(legacy).verified)
|
|
130
|
+
legacy.evidence_class = ev.EVIDENCE_FETCH_VERIFIED
|
|
131
|
+
self.assertFalse(self.v.verify_fact(legacy).verified, "adding the field must break a v1 signature")
|
|
132
|
+
|
|
133
|
+
def test_fetch_date_and_source_date_are_also_covered(self):
|
|
134
|
+
fact = self.v.create_fetched_fact(claim="c", fetch_record=self.record, issuer="researcher", source_date="2024-01-01")
|
|
135
|
+
self.assertTrue(self.v.verify_fact(fact).verified)
|
|
136
|
+
fact.fetch_date = "2099-01-01T00:00:00Z"
|
|
137
|
+
self.assertFalse(self.v.verify_fact(fact).verified, "fetch_date is signed, not decorative")
|
|
138
|
+
|
|
139
|
+
def test_v1_and_v2_messages_differ_for_identical_shared_fields(self):
|
|
140
|
+
"""The two schemas are distinguishable because v2 carries extra keys; the
|
|
141
|
+
marker is self-description and the v3 growth point, not the protection."""
|
|
142
|
+
fact = self.v.create_listing_fact(claim="c", source_url="https://x.test", reason="r")
|
|
143
|
+
self.assertNotEqual(ev.canonical_fact_message_v1(fact), ev.canonical_fact_message_v2(fact))
|
|
144
|
+
self.assertIn(ev.FACT_SCHEMA_V2, ev.canonical_fact_message_v2(fact))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class EarnedClassTests(unittest.TestCase):
|
|
148
|
+
"""ADR-003 — FETCH_VERIFIED requires an artifact only a real fetch produces."""
|
|
149
|
+
|
|
150
|
+
def test_manual_path_cannot_mint_fetch_verified(self):
|
|
151
|
+
"""The load-bearing property: no manual route yields FETCH_VERIFIED."""
|
|
152
|
+
v = _verifier()
|
|
153
|
+
listing = v.create_listing_fact(claim="c", source_url="https://x.test", reason="hand-supplied body")
|
|
154
|
+
asserted = v.create_asserted_fact(claim="c")
|
|
155
|
+
self.assertEqual(listing.evidence_class, ev.EVIDENCE_LISTING_ONLY)
|
|
156
|
+
self.assertEqual(asserted.evidence_class, ev.EVIDENCE_ASSERTED)
|
|
157
|
+
|
|
158
|
+
# …and the constructor that CAN mint it refuses anything lacking proof.
|
|
159
|
+
class NotAFetch:
|
|
160
|
+
status = 200
|
|
161
|
+
final_url = "https://x.test"
|
|
162
|
+
# no sha256_body, no fetched_at
|
|
163
|
+
with self.assertRaises(ValueError):
|
|
164
|
+
v.create_fetched_fact(claim="c", fetch_record=NotAFetch(), issuer="researcher")
|
|
165
|
+
|
|
166
|
+
# Forging it in the JSON is caught by the signature (ADR-002).
|
|
167
|
+
forged = json.loads(listing.to_json())
|
|
168
|
+
forged["evidence_class"] = ev.EVIDENCE_FETCH_VERIFIED
|
|
169
|
+
self.assertFalse(v.verify_fact(ev.SignedFact.from_dict(forged)).verified)
|
|
170
|
+
|
|
171
|
+
def test_non_2xx_fetch_is_refused_as_evidence(self):
|
|
172
|
+
v = _verifier()
|
|
173
|
+
error_page = _authentic_record(url="https://x.test", final_url="https://x.test", status=404)
|
|
174
|
+
with self.assertRaises(ValueError):
|
|
175
|
+
v.create_fetched_fact(claim="c", fetch_record=error_page, issuer="researcher")
|
|
176
|
+
|
|
177
|
+
def test_listing_fact_demands_a_stated_reason(self):
|
|
178
|
+
v = _verifier()
|
|
179
|
+
with self.assertRaises(ValueError):
|
|
180
|
+
v.create_listing_fact(claim="c", source_url="https://x.test", reason=" ")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class FetchTests(unittest.TestCase):
|
|
184
|
+
"""The network path itself — against a local server, never the internet."""
|
|
185
|
+
|
|
186
|
+
@classmethod
|
|
187
|
+
def setUpClass(cls):
|
|
188
|
+
class Handler(BaseHTTPRequestHandler):
|
|
189
|
+
def do_GET(self): # noqa: N802
|
|
190
|
+
if self.path == "/ok":
|
|
191
|
+
body = b"hello evidence"
|
|
192
|
+
self.send_response(200)
|
|
193
|
+
self.send_header("Content-Type", "text/plain")
|
|
194
|
+
self.send_header("Content-Length", str(len(body)))
|
|
195
|
+
self.end_headers()
|
|
196
|
+
self.wfile.write(body)
|
|
197
|
+
elif self.path == "/redirect":
|
|
198
|
+
self.send_response(302)
|
|
199
|
+
self.send_header("Location", "/ok")
|
|
200
|
+
self.end_headers()
|
|
201
|
+
elif self.path == "/big":
|
|
202
|
+
body = b"x" * 4096
|
|
203
|
+
self.send_response(200)
|
|
204
|
+
self.send_header("Content-Length", str(len(body)))
|
|
205
|
+
self.end_headers()
|
|
206
|
+
self.wfile.write(body)
|
|
207
|
+
else:
|
|
208
|
+
self.send_response(404)
|
|
209
|
+
self.end_headers()
|
|
210
|
+
|
|
211
|
+
def log_message(self, *args): # silence
|
|
212
|
+
pass
|
|
213
|
+
|
|
214
|
+
cls.server = HTTPServer(("127.0.0.1", 0), Handler)
|
|
215
|
+
cls.port = cls.server.server_address[1]
|
|
216
|
+
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
|
217
|
+
cls.thread.start()
|
|
218
|
+
|
|
219
|
+
@classmethod
|
|
220
|
+
def tearDownClass(cls):
|
|
221
|
+
cls.server.shutdown()
|
|
222
|
+
cls.server.server_close()
|
|
223
|
+
|
|
224
|
+
def _url(self, path):
|
|
225
|
+
return f"http://127.0.0.1:{self.port}{path}"
|
|
226
|
+
|
|
227
|
+
def test_successful_fetch_produces_a_real_body_hash(self):
|
|
228
|
+
import hashlib
|
|
229
|
+
record = ef.fetch_source(self._url("/ok"), _allow_private=True)
|
|
230
|
+
self.assertIsInstance(record, ef.FetchRecord)
|
|
231
|
+
self.assertEqual(record.status, 200)
|
|
232
|
+
self.assertEqual(record.sha256_body, hashlib.sha256(b"hello evidence").hexdigest())
|
|
233
|
+
self.assertTrue(record.fetched_at.endswith("Z"))
|
|
234
|
+
|
|
235
|
+
def test_redirect_is_followed_and_final_url_recorded(self):
|
|
236
|
+
record = ef.fetch_source(self._url("/redirect"), _allow_private=True)
|
|
237
|
+
self.assertIsInstance(record, ef.FetchRecord)
|
|
238
|
+
self.assertTrue(record.final_url.endswith("/ok"))
|
|
239
|
+
self.assertNotEqual(record.url, record.final_url)
|
|
240
|
+
|
|
241
|
+
def test_oversize_body_is_refused_not_truncated(self):
|
|
242
|
+
"""A truncated body would hash to something no repeat fetch reproduces."""
|
|
243
|
+
failure = ef.fetch_source(self._url("/big"), max_bytes=100, _allow_private=True)
|
|
244
|
+
self.assertIsInstance(failure, ef.FetchFailure)
|
|
245
|
+
self.assertIn("max_bytes", failure.reason)
|
|
246
|
+
|
|
247
|
+
def test_404_is_a_named_failure_not_a_record(self):
|
|
248
|
+
failure = ef.fetch_source(self._url("/missing"), _allow_private=True)
|
|
249
|
+
self.assertIsInstance(failure, ef.FetchFailure)
|
|
250
|
+
self.assertIn("404", failure.reason)
|
|
251
|
+
|
|
252
|
+
def test_non_http_scheme_is_refused_before_io(self):
|
|
253
|
+
"""file:// would let a local read masquerade as network evidence."""
|
|
254
|
+
failure = ef.fetch_source("file:///etc/passwd")
|
|
255
|
+
self.assertIsInstance(failure, ef.FetchFailure)
|
|
256
|
+
self.assertIn("scheme", failure.reason)
|
|
257
|
+
|
|
258
|
+
def test_offline_degrades_to_listing_with_a_stated_reason(self):
|
|
259
|
+
v = _verifier()
|
|
260
|
+
failure = ef.fetch_source("http://127.0.0.1:1/unreachable", timeout=1, _allow_private=True)
|
|
261
|
+
self.assertIsInstance(failure, ef.FetchFailure)
|
|
262
|
+
fact = v.create_listing_fact(claim="c", source_url=failure.url, reason=failure.reason)
|
|
263
|
+
self.assertEqual(fact.evidence_class, ev.EVIDENCE_LISTING_ONLY)
|
|
264
|
+
self.assertIn("evidence_note", fact.metadata)
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class SourceTierTests(unittest.TestCase):
|
|
268
|
+
def test_known_domains_map_to_their_tier(self):
|
|
269
|
+
self.assertEqual(st.classify_source("https://www.cochrane.org/x").tier, st.TIER_A)
|
|
270
|
+
self.assertEqual(st.classify_source("https://pubmed.ncbi.nlm.nih.gov/123").tier, st.TIER_B)
|
|
271
|
+
self.assertEqual(st.classify_source("https://www.medrxiv.org/y").tier, st.TIER_C)
|
|
272
|
+
|
|
273
|
+
def test_unknown_domain_is_tier_d_not_an_exception(self):
|
|
274
|
+
verdict = st.classify_source("https://some-blog.example/post")
|
|
275
|
+
self.assertEqual(verdict.tier, st.TIER_D)
|
|
276
|
+
self.assertFalse(verdict.known)
|
|
277
|
+
|
|
278
|
+
def test_lookalike_domain_does_not_inherit_a_tier(self):
|
|
279
|
+
"""cochrane.org.evil.com must NOT read as tier A — suffix LABELS, not endswith."""
|
|
280
|
+
self.assertEqual(st.classify_source("https://cochrane.org.evil.com/x").tier, st.TIER_D)
|
|
281
|
+
|
|
282
|
+
def test_missing_source_date_is_flagged_not_waved_through(self):
|
|
283
|
+
stale, reason = st.is_stale(None, kind="guideline")
|
|
284
|
+
self.assertTrue(stale)
|
|
285
|
+
self.assertIn("missing", reason)
|
|
286
|
+
|
|
287
|
+
def test_old_source_is_stale_fresh_one_is_not(self):
|
|
288
|
+
self.assertTrue(st.is_stale("2015-01-01", kind="guideline")[0])
|
|
289
|
+
from datetime import datetime, timezone
|
|
290
|
+
self.assertFalse(st.is_stale(datetime.now(timezone.utc).strftime("%Y-%m-%d"), kind="guideline")[0])
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
class ReportGateTests(unittest.TestCase):
|
|
294
|
+
"""FR-4 — the gate is an exit code, not advice."""
|
|
295
|
+
|
|
296
|
+
def _fact(self, claim, evidence_class, url="https://x.test"):
|
|
297
|
+
return {"claim": claim, "source_url": url, "evidence_class": evidence_class}
|
|
298
|
+
|
|
299
|
+
def test_asserted_claim_used_in_report_fails(self):
|
|
300
|
+
report = "The patient's metabolism recovered without fasting, so we proceed."
|
|
301
|
+
facts = [self._fact("metabolism recovered without fasting", "ASSERTED")]
|
|
302
|
+
findings, counts = gate.evaluate(report, facts)
|
|
303
|
+
self.assertEqual(len(findings), 1)
|
|
304
|
+
self.assertEqual(counts["ASSERTED"], 1)
|
|
305
|
+
|
|
306
|
+
def test_asserted_fact_not_used_in_report_is_not_a_violation(self):
|
|
307
|
+
report = "Unrelated content about iron saturation."
|
|
308
|
+
facts = [self._fact("metabolism recovered without fasting", "ASSERTED")]
|
|
309
|
+
findings, _ = gate.evaluate(report, facts)
|
|
310
|
+
self.assertEqual(findings, [])
|
|
311
|
+
|
|
312
|
+
def test_unmarked_listing_only_fails_marked_passes(self):
|
|
313
|
+
claim = "transferrin saturation rose from 37 to 53 percent"
|
|
314
|
+
unmarked = f"We note that {claim} over six months."
|
|
315
|
+
marked = f"We note that {claim} (LISTING_ONLY: карточка не открывалась — перепроверьте)."
|
|
316
|
+
facts = [self._fact(claim, "LISTING_ONLY")]
|
|
317
|
+
self.assertEqual(len(gate.evaluate(unmarked, facts)[0]), 1)
|
|
318
|
+
self.assertEqual(gate.evaluate(marked, facts)[0], [])
|
|
319
|
+
|
|
320
|
+
def test_marker_far_from_the_claim_does_not_count(self):
|
|
321
|
+
claim = "transferrin saturation rose from 37 to 53 percent"
|
|
322
|
+
far = f"We note that {claim}." + ("filler. " * 200) + "LISTING_ONLY appendix note"
|
|
323
|
+
findings, _ = gate.evaluate(far, [self._fact(claim, "LISTING_ONLY")])
|
|
324
|
+
self.assertEqual(len(findings), 1, "a marker in the appendix does not warn the reader on page 2")
|
|
325
|
+
|
|
326
|
+
def test_legacy_facts_are_counted_and_named_never_folded_into_clean(self):
|
|
327
|
+
report = "Some claim about lipids appears here in the body of the report."
|
|
328
|
+
facts = [{"claim": "Some claim about lipids", "source_url": "u"}] # no evidence_class
|
|
329
|
+
findings, counts = gate.evaluate(report, facts)
|
|
330
|
+
self.assertEqual(findings, [])
|
|
331
|
+
self.assertEqual(counts["UNKNOWN_LEGACY"], 1)
|
|
332
|
+
self.assertIn("predate the evidence axis", gate.render(findings, counts))
|
|
333
|
+
|
|
334
|
+
def test_paraphrased_claim_is_still_detected(self):
|
|
335
|
+
claim = "omega-3 supplementation raised LDL cholesterol by forty percent"
|
|
336
|
+
report = "Reported omega-3 supplementation figures show raised LDL cholesterol by roughly forty percent overall."
|
|
337
|
+
findings, _ = gate.evaluate(report, [self._fact(claim, "ASSERTED")])
|
|
338
|
+
self.assertEqual(len(findings), 1, "over-detection is preferred: a miss is silent, an alarm is arguable")
|
|
339
|
+
|
|
340
|
+
def test_gate_cli_exit_codes(self):
|
|
341
|
+
import tempfile, os
|
|
342
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
343
|
+
report_path = os.path.join(tmp, "r.md")
|
|
344
|
+
facts_path = os.path.join(tmp, "f.json")
|
|
345
|
+
with open(report_path, "w", encoding="utf-8") as fh:
|
|
346
|
+
fh.write("The claim about iron overload appears in this report body.")
|
|
347
|
+
with open(facts_path, "w", encoding="utf-8") as fh:
|
|
348
|
+
json.dump([self._fact("claim about iron overload", "ASSERTED")], fh)
|
|
349
|
+
self.assertEqual(gate.main(["--report", report_path, "--facts", facts_path]), 1)
|
|
350
|
+
self.assertEqual(gate.main(["--report", report_path, "--facts", os.path.join(tmp, "nope.json")]), 2)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class HardeningTests(unittest.TestCase):
|
|
356
|
+
"""Findings from the cross-model QE round — each closed with its own test."""
|
|
357
|
+
|
|
358
|
+
def test_qe1_hand_built_record_cannot_mint_fetch_verified(self):
|
|
359
|
+
"""A duck-typed record with four plausible attributes minted FETCH_VERIFIED
|
|
360
|
+
without any network I/O. The record must be authentic, not merely shaped right."""
|
|
361
|
+
v = _verifier()
|
|
362
|
+
forged = ef.FetchRecord(
|
|
363
|
+
url="https://pubmed.ncbi.nlm.nih.gov/1", final_url="https://pubmed.ncbi.nlm.nih.gov/1",
|
|
364
|
+
status=200, sha256_body="c" * 64, bytes_len=5, fetched_at="2026-08-03T00:00:00Z",
|
|
365
|
+
) # constructed by hand → no witness
|
|
366
|
+
self.assertFalse(forged.is_authentic())
|
|
367
|
+
with self.assertRaises(ValueError) as ctx:
|
|
368
|
+
v.create_fetched_fact(claim="c", fetch_record=forged, issuer="researcher")
|
|
369
|
+
self.assertIn("fetch_source", str(ctx.exception))
|
|
370
|
+
|
|
371
|
+
import types
|
|
372
|
+
ns = types.SimpleNamespace(status=200, final_url="u", sha256_body="d" * 64, fetched_at="t")
|
|
373
|
+
with self.assertRaises(ValueError):
|
|
374
|
+
v.create_fetched_fact(claim="c", fetch_record=ns, issuer="researcher")
|
|
375
|
+
|
|
376
|
+
def test_qe6_tier_ceiling_is_actually_applied_to_v2_facts(self):
|
|
377
|
+
"""The documented min(trust, evidence, tier) was not implemented — tier was
|
|
378
|
+
never consulted, so an unknown-domain fact kept 0.60 instead of 0.40."""
|
|
379
|
+
v = _verifier()
|
|
380
|
+
unknown = v.create_listing_fact(claim="c", source_url="https://some-blog.example/p", reason="listing")
|
|
381
|
+
self.assertEqual(v.verify_fact(unknown).confidence, 0.40, "tier D must cap below the 0.50 evidence ceiling")
|
|
382
|
+
known = v.create_listing_fact(claim="c", source_url="https://www.cochrane.org/x", reason="listing")
|
|
383
|
+
self.assertEqual(v.verify_fact(known).confidence, 0.50, "tier A leaves the evidence ceiling as the binding one")
|
|
384
|
+
|
|
385
|
+
def test_qe6b_legacy_facts_are_not_retroactively_rescored(self):
|
|
386
|
+
"""Wiring a NEW ceiling must not change how OLD records score."""
|
|
387
|
+
v = _verifier()
|
|
388
|
+
legacy = v.create_signed_fact(claim="c", source_url="https://some-blog.example/p",
|
|
389
|
+
source_content="b", issuer="researcher")
|
|
390
|
+
self.assertEqual(v.verify_fact(legacy).confidence, 0.60, "a v1 fact keeps its original semantics")
|
|
391
|
+
|
|
392
|
+
def test_qe16_frozen_v1_fixture_still_verifies(self):
|
|
393
|
+
"""Backward compatibility proven against a STORED artifact, not against the
|
|
394
|
+
current implementation verifying its own current output — a regression that
|
|
395
|
+
changed signing AND verification together would pass that weaker check."""
|
|
396
|
+
import os
|
|
397
|
+
path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "fixture_legacy_v1_fact.json")
|
|
398
|
+
with open(path, "r", encoding="utf-8") as handle:
|
|
399
|
+
fixture = json.load(handle)
|
|
400
|
+
verifier = ev.Ed25519Verifier()
|
|
401
|
+
fact = ev.SignedFact.from_dict(fixture["fact"])
|
|
402
|
+
self.assertIsNone(fact.evidence_class, "the fixture is a genuine pre-axis fact")
|
|
403
|
+
self.assertTrue(verifier.verify_fact(fact).verified, "a frozen v1 fact must verify forever")
|
|
404
|
+
|
|
405
|
+
def test_qe2_empty_ledger_cannot_clear_a_report(self):
|
|
406
|
+
"""Passing an empty facts file made any medical report exit 0."""
|
|
407
|
+
import tempfile, os
|
|
408
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
409
|
+
report = os.path.join(tmp, "r.md"); facts = os.path.join(tmp, "f.json")
|
|
410
|
+
open(report, "w").write("Any claim at all appears in this report body.")
|
|
411
|
+
json.dump([], open(facts, "w"))
|
|
412
|
+
self.assertEqual(gate.main(["--report", report, "--facts", facts]), 2,
|
|
413
|
+
"an empty ledger is unevaluable, never clean")
|
|
414
|
+
|
|
415
|
+
def test_qe3_tampered_fact_is_caught_by_the_gate_not_just_the_verifier(self):
|
|
416
|
+
"""Flipping a signed ASSERTED record to FETCH_VERIFIED in the JSON fooled the
|
|
417
|
+
gate: it read evidence_class as an unsigned string and never verified."""
|
|
418
|
+
v = _verifier()
|
|
419
|
+
asserted = v.create_asserted_fact(claim="the IMEI ban was introduced in Turkey")
|
|
420
|
+
forged = json.loads(asserted.to_json())
|
|
421
|
+
forged["evidence_class"] = ev.EVIDENCE_FETCH_VERIFIED
|
|
422
|
+
findings = gate.verify_ledger_signatures([forged])
|
|
423
|
+
self.assertTrue(findings)
|
|
424
|
+
self.assertEqual(findings[0].kind, "TAMPERED_FACT")
|
|
425
|
+
|
|
426
|
+
def test_qe5_every_occurrence_needs_its_own_marker(self):
|
|
427
|
+
claim = "transferrin saturation rose from 37 to 53 percent"
|
|
428
|
+
text = (f"Early on we note that {claim} (LISTING_ONLY: не открывалась)." + " filler." * 120 +
|
|
429
|
+
f" Later we repeat that {claim} without any warning at all.")
|
|
430
|
+
findings, _ = gate.evaluate(text, [{"claim": claim, "source_url": "u", "evidence_class": "LISTING_ONLY"}])
|
|
431
|
+
self.assertEqual(len(findings), 1, "one marked mention must not clear an unmarked one elsewhere")
|
|
432
|
+
|
|
433
|
+
def test_qe4_short_and_numeric_paraphrase_is_detected(self):
|
|
434
|
+
"""`LDL rose 40%` vs `LDL increased 40%` shares the number, not long words."""
|
|
435
|
+
claim = "LDL rose 40% on the higher dose"
|
|
436
|
+
report = "In that arm LDL increased 40% on the higher dose, which we consider material."
|
|
437
|
+
findings, _ = gate.evaluate(report, [{"claim": claim, "source_url": "u", "evidence_class": "ASSERTED"}])
|
|
438
|
+
self.assertEqual(len(findings), 1)
|
|
439
|
+
|
|
440
|
+
def test_qe4b_claim_wrapped_across_lines_is_detected(self):
|
|
441
|
+
claim = "transferrin saturation above 45 percent warrants attention in men"
|
|
442
|
+
report = "We note transferrin saturation\nabove 45 percent warrants\nattention in men here."
|
|
443
|
+
findings, _ = gate.evaluate(report, [{"claim": claim, "source_url": "u", "evidence_class": "ASSERTED"}])
|
|
444
|
+
self.assertEqual(len(findings), 1, "markdown wrapping must not hide a claim from the gate")
|
|
445
|
+
|
|
446
|
+
def test_qe9_unrecognised_evidence_class_is_a_violation_not_legacy(self):
|
|
447
|
+
claim = "iron overload warrants HFE testing in this context"
|
|
448
|
+
report = f"Our position: {claim}, pending review."
|
|
449
|
+
findings, counts = gate.evaluate(report, [{"claim": claim, "source_url": "u", "evidence_class": "ASSERTED "}])
|
|
450
|
+
self.assertEqual(len(findings), 1)
|
|
451
|
+
self.assertEqual(findings[0].kind, "UNRECOGNISED_EVIDENCE_CLASS")
|
|
452
|
+
|
|
453
|
+
def test_qe9b_malformed_ledger_entry_refuses_evaluation(self):
|
|
454
|
+
import tempfile, os
|
|
455
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
456
|
+
report = os.path.join(tmp, "r.md"); facts = os.path.join(tmp, "f.json")
|
|
457
|
+
open(report, "w").write("body")
|
|
458
|
+
json.dump([{"claim": "ok", "evidence_class": "ASSERTED"}, "not-an-object"], open(facts, "w"))
|
|
459
|
+
self.assertEqual(gate.main(["--report", report, "--facts", facts]), 2)
|
|
460
|
+
|
|
461
|
+
def test_qe8_ssrf_private_addresses_are_refused(self):
|
|
462
|
+
for url in ("http://127.0.0.1/x", "http://localhost/x", "http://169.254.169.254/latest/meta-data/",
|
|
463
|
+
"http://10.0.0.1/x", "http://192.168.1.1/x"):
|
|
464
|
+
failure = ef.fetch_source(url, timeout=2)
|
|
465
|
+
self.assertIsInstance(failure, ef.FetchFailure, f"{url} must be refused")
|
|
466
|
+
self.assertIn("non-public", failure.reason)
|
|
467
|
+
|
|
468
|
+
def test_qe12_unbounded_max_bytes_is_refused(self):
|
|
469
|
+
failure = ef.fetch_source("https://example.com/", max_bytes=10 ** 12)
|
|
470
|
+
self.assertIsInstance(failure, ef.FetchFailure)
|
|
471
|
+
self.assertIn("max_bytes", failure.reason)
|
|
472
|
+
|
|
473
|
+
def test_qe11_malformed_urls_return_named_failures_not_exceptions(self):
|
|
474
|
+
for url in ("http://[::1", "http://exa mple.com/x", "http://host:notaport/x", "notaurl"):
|
|
475
|
+
result = ef.fetch_source(url, timeout=2)
|
|
476
|
+
self.assertIsInstance(result, ef.FetchFailure, f"{url!r} must degrade, not raise")
|
|
477
|
+
|
|
478
|
+
def test_qe13_non_http_scheme_gets_no_tier(self):
|
|
479
|
+
self.assertEqual(st.classify_source("file://who.int/etc/passwd").tier, st.TIER_D)
|
|
480
|
+
|
|
481
|
+
def test_qe14_future_dated_source_is_flagged(self):
|
|
482
|
+
stale, reason = st.is_stale("2099-01-01", kind="guideline")
|
|
483
|
+
self.assertTrue(stale)
|
|
484
|
+
self.assertIn("future", reason)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
class LearningBridgeTests(unittest.TestCase):
|
|
488
|
+
"""Slice H — the privacy invariant, rebuilt after a cross-model review broke the
|
|
489
|
+
first design. The guard is an ALLOWLIST (a method lesson carries no digits and no
|
|
490
|
+
identifiers), because enumerating what personal data looks like is an arms race
|
|
491
|
+
whose every gap is a leaked medical fact."""
|
|
492
|
+
|
|
493
|
+
def setUp(self):
|
|
494
|
+
import learning_bridge
|
|
495
|
+
self.lb = learning_bridge
|
|
496
|
+
|
|
497
|
+
def test_method_lessons_pass(self):
|
|
498
|
+
for good in (
|
|
499
|
+
"total testosterone is uninterpretable without SHBG — always order the companion test",
|
|
500
|
+
"a relative risk without an absolute one misleads; convert before reporting",
|
|
501
|
+
"check the study population before transferring an effect to a patient profile",
|
|
502
|
+
"a prolonged fast lowers total testosterone substantially, with LH down and FSH normal",
|
|
503
|
+
):
|
|
504
|
+
self.assertTrue(self.lb.check_lesson(good).ok, good)
|
|
505
|
+
|
|
506
|
+
def test_every_measured_bypass_from_the_review_is_closed(self):
|
|
507
|
+
"""These EXACT strings were measured passing by the reviewer. Each is a
|
|
508
|
+
regression anchor: they must never be accepted again."""
|
|
509
|
+
for bad in (
|
|
510
|
+
"TSH was 8.04 mIU/L when fasting", # threshold-word laundering
|
|
511
|
+
"testosterone 8.04 nmol/l was below baseline",
|
|
512
|
+
"Alice's BMI 25 was below baseline",
|
|
513
|
+
"Patient John Smith has HIV", # a name
|
|
514
|
+
"MRN 84729163: creatinine elevated", # an identifier
|
|
515
|
+
"reach me at doctor@clinic.example", # an email
|
|
516
|
+
"call +7 916 123 45 67", # a phone
|
|
517
|
+
"05/08/2026 repeat the panel", # a non-ISO date
|
|
518
|
+
"8 мая 2026 пересдать", # a Russian date
|
|
519
|
+
"sodium 128 mEq/L", # a unit outside the old list
|
|
520
|
+
"creatinine 130 µmol/L", # a micro sign
|
|
521
|
+
"8.04 nanomoles per litre", # a spelled-out unit
|
|
522
|
+
):
|
|
523
|
+
self.assertFalse(self.lb.check_lesson(bad).ok, f"must refuse: {bad}")
|
|
524
|
+
|
|
525
|
+
def test_a_number_needs_an_EXPLICIT_human_decision_never_an_inferred_one(self):
|
|
526
|
+
"""The first design inferred 'this number is a threshold' from a nearby word,
|
|
527
|
+
and one stray 'when' laundered a lab value. Now a human passes the flag."""
|
|
528
|
+
threshold = "transferrin saturation above 45% warrants attention in men"
|
|
529
|
+
self.assertFalse(self.lb.check_lesson(threshold).ok, "digits refused by default")
|
|
530
|
+
self.assertTrue(self.lb.check_lesson(threshold, allow_numbers=True).ok)
|
|
531
|
+
|
|
532
|
+
def test_the_flag_cannot_wave_through_a_person_or_an_identifier(self):
|
|
533
|
+
self.assertFalse(self.lb.check_lesson("у пациента 45%", allow_numbers=True).ok)
|
|
534
|
+
self.assertFalse(self.lb.check_lesson("MRN 84729163 note", allow_numbers=True).ok)
|
|
535
|
+
self.assertFalse(self.lb.check_lesson("Patient John Smith 45%", allow_numbers=True).ok)
|
|
536
|
+
|
|
537
|
+
def test_the_accept_message_does_NOT_claim_safety(self):
|
|
538
|
+
"""A shape detector may not issue a safety certificate — saying 'safe to teach'
|
|
539
|
+
transfers responsibility the guard does not have."""
|
|
540
|
+
note = self.lb.check_lesson("a method lesson about companion tests").note
|
|
541
|
+
self.assertNotIn("safe to teach", note)
|
|
542
|
+
self.assertIn("not a safety certificate", note)
|
|
543
|
+
self.assertIn("responsible", note)
|
|
544
|
+
|
|
545
|
+
def test_empty_lesson_is_refused(self):
|
|
546
|
+
self.assertFalse(self.lb.check_lesson(" ").ok)
|
|
547
|
+
|
|
548
|
+
def test_absent_dz_is_a_NOTE_not_a_failure(self):
|
|
549
|
+
original = self.lb.shutil.which
|
|
550
|
+
self.lb.shutil.which = lambda _name: None
|
|
551
|
+
try:
|
|
552
|
+
self.assertIn("self-learning is OFF", self.lb.status())
|
|
553
|
+
self.assertIn("self-learning is OFF", self.lb.recall("anything"))
|
|
554
|
+
code, message = self.lb.teach("a valid method lesson about companion tests")
|
|
555
|
+
self.assertEqual(code, 0, "a missing optional dependency is not an error")
|
|
556
|
+
self.assertIn("self-learning is OFF", message)
|
|
557
|
+
finally:
|
|
558
|
+
self.lb.shutil.which = original
|
|
559
|
+
|
|
560
|
+
def test_privacy_guard_runs_BEFORE_any_dz_call(self):
|
|
561
|
+
calls = []
|
|
562
|
+
original = self.lb._run_dz
|
|
563
|
+
self.lb._run_dz = lambda *a, **k: (calls.append(a) or (0, "", ""))
|
|
564
|
+
try:
|
|
565
|
+
code, message = self.lb.teach("тестостерон 8.04 нмоль/л у пациента")
|
|
566
|
+
self.assertEqual(code, 1)
|
|
567
|
+
self.assertIn("REFUSED", message)
|
|
568
|
+
self.assertEqual(calls, [], "no dz call may happen for a refused lesson")
|
|
569
|
+
finally:
|
|
570
|
+
self.lb._run_dz = original
|
|
571
|
+
|
|
572
|
+
def test_argument_injection_through_the_query_is_refused(self):
|
|
573
|
+
"""`recall("--all")` would become `dz recall --all …` and dump the whole store.
|
|
574
|
+
This is ARGUMENT injection — shell quoting does not address it."""
|
|
575
|
+
calls = []
|
|
576
|
+
original = self.lb._run_dz
|
|
577
|
+
self.lb._run_dz = lambda *a, **k: (calls.append(a) or (0, "", ""))
|
|
578
|
+
try:
|
|
579
|
+
self.assertIn("would be read as an option", self.lb.recall("--all"))
|
|
580
|
+
code, msg = self.lb.teach("--reward 1.0 laundered")
|
|
581
|
+
self.assertEqual(code, 1)
|
|
582
|
+
self.assertIn("read it as an option", msg)
|
|
583
|
+
self.assertEqual(calls, [], "neither call may reach dz")
|
|
584
|
+
finally:
|
|
585
|
+
self.lb._run_dz = original
|
|
586
|
+
|
|
587
|
+
def test_older_cli_is_detected_by_CAPABILITY_not_by_exit_code(self):
|
|
588
|
+
"""The sharpest finding of the review: the previous dz did NOT reject --domain.
|
|
589
|
+
Its parser accepted any `--key value` and cmdRecall ignored it, exiting 0 with
|
|
590
|
+
unfiltered results — so an exit-code test could never fire, and the earlier
|
|
591
|
+
version of THIS test fabricated a failure that no release ever produced. The
|
|
592
|
+
observable difference is the boost note; its absence is what 'too old' means."""
|
|
593
|
+
original = self.lb._run_dz
|
|
594
|
+
# Old CLI: exit 0, results, and NO boost note.
|
|
595
|
+
self.lb._run_dz = lambda *a, **k: (0, "dz recall \"x\" — 1 hit(s)\n [0.90] (general) a lesson", "")
|
|
596
|
+
try:
|
|
597
|
+
out = self.lb.recall("transferrin")
|
|
598
|
+
self.assertIn("a lesson", out, "the results are still returned")
|
|
599
|
+
self.assertIn("WITHOUT the domain boost", out)
|
|
600
|
+
self.assertIn("predates", out)
|
|
601
|
+
finally:
|
|
602
|
+
self.lb._run_dz = original
|
|
603
|
+
|
|
604
|
+
def test_current_cli_with_the_boost_note_reports_no_degradation(self):
|
|
605
|
+
"""The note text here is the REAL renderer output, not a paraphrase. The earlier
|
|
606
|
+
version of this test invented a format the CLI never produced, so it blessed the
|
|
607
|
+
substring check instead of proving the capability. The two tail phrases are
|
|
608
|
+
pinned on the TypeScript side by a matching test."""
|
|
609
|
+
original = self.lb._run_dz
|
|
610
|
+
# Copied VERBATIM from a live run:
|
|
611
|
+
# node dist/bin.js recall "signature" --limit 2 --domain testing
|
|
612
|
+
real_note = (' domain "health-research": among 6 candidate(s) — 1 exact match(es), '
|
|
613
|
+
'3 changed position; foreign-domain lessons kept (a boost, not a filter)')
|
|
614
|
+
self.lb._run_dz = lambda *a, **k: (
|
|
615
|
+
0, 'dz recall "x" — 1 hit(s)\n [0.90] (health-research) a lesson\n' + real_note, "")
|
|
616
|
+
try:
|
|
617
|
+
out = self.lb.recall("transferrin")
|
|
618
|
+
self.assertNotIn("predates", out)
|
|
619
|
+
finally:
|
|
620
|
+
self.lb._run_dz = original
|
|
621
|
+
|
|
622
|
+
def test_the_no_match_note_ALSO_counts_as_capability(self):
|
|
623
|
+
"""A --domain run that matched nothing still proves the CLI understands --domain.
|
|
624
|
+
Reading only the 'a boost, not a filter' tail would have reported a perfectly
|
|
625
|
+
current CLI as too old whenever the domain happened to match nothing."""
|
|
626
|
+
original = self.lb._run_dz
|
|
627
|
+
real_note = (' domain "health-research": no lesson in this result carries it — '
|
|
628
|
+
'order unchanged, nothing was hidden')
|
|
629
|
+
self.lb._run_dz = lambda *a, **k: (0, 'dz recall "x" — 1 hit(s)\n [0.90] (general) x\n' + real_note, "")
|
|
630
|
+
try:
|
|
631
|
+
self.assertNotIn("predates", self.lb.recall("x"))
|
|
632
|
+
finally:
|
|
633
|
+
self.lb._run_dz = original
|
|
634
|
+
|
|
635
|
+
def test_capability_cannot_be_FORGED_by_recalled_lesson_content(self):
|
|
636
|
+
"""Round 2: the marker was the bare substring `domain "`, which a LESSON can
|
|
637
|
+
contain. A probe the payload can forge is not a probe — an old CLI returning
|
|
638
|
+
this ordinary lesson was certified as boost-capable."""
|
|
639
|
+
original = self.lb._run_dz
|
|
640
|
+
self.lb._run_dz = lambda *a, **k: (
|
|
641
|
+
0,
|
|
642
|
+
'dz recall "x" — 1 hit(s)\n'
|
|
643
|
+
' [0.90] (general) remember domain "ownership" before reporting',
|
|
644
|
+
"",
|
|
645
|
+
)
|
|
646
|
+
try:
|
|
647
|
+
self.assertIn("predates", self.lb.recall("x"))
|
|
648
|
+
finally:
|
|
649
|
+
self.lb._run_dz = original
|
|
650
|
+
|
|
651
|
+
# ---------------------------------------------------------------- round 2
|
|
652
|
+
# Every case below is a REPRODUCER the cross-model reviewer ran against the
|
|
653
|
+
# previous guard and got the wrong answer for. They stay here permanently: a
|
|
654
|
+
# privacy guard is only as good as the bypasses it can no longer be talked into.
|
|
655
|
+
|
|
656
|
+
def test_round2_a_name_beside_a_diagnosis_is_REFUSED(self):
|
|
657
|
+
"""The worst miss of the round. The old rule hunted a name — two capitalised
|
|
658
|
+
words in a row, exempting the sentence start — so BOTH of these passed."""
|
|
659
|
+
for leak in ("John Smith has HIV", "Patient John has HIV"):
|
|
660
|
+
with self.subTest(leak=leak):
|
|
661
|
+
self.assertFalse(self.lb.check_lesson(leak).ok)
|
|
662
|
+
|
|
663
|
+
def test_round2_a_capitalised_word_after_the_first_is_refused_whatever_it_is(self):
|
|
664
|
+
"""The replacement rule is a property of the ACCEPTED form (a method lesson is
|
|
665
|
+
written in lower case), not a hunt for names — because names cannot be listed.
|
|
666
|
+
It refuses legitimate title-cased method vocabulary too, deliberately: the fix
|
|
667
|
+
is one keystroke and the false negative it closes is a named person."""
|
|
668
|
+
verdict = self.lb.check_lesson("compare Testosterone Replacement Therapy against placebo")
|
|
669
|
+
self.assertFalse(verdict.ok)
|
|
670
|
+
self.assertIn("lower case", verdict.note)
|
|
671
|
+
# …and the lower-cased form is accepted, so the advice actually works.
|
|
672
|
+
self.assertTrue(self.lb.check_lesson("compare testosterone replacement therapy against placebo").ok)
|
|
673
|
+
|
|
674
|
+
def test_round2_ALL_CAPS_and_mixed_case_acronyms_still_pass(self):
|
|
675
|
+
"""The capitalisation rule must not cost the vocabulary lessons are written in."""
|
|
676
|
+
for ok_text in (
|
|
677
|
+
"total testosterone is uninterpretable without SHBG",
|
|
678
|
+
"a single ferritin cannot separate overload from inflammation, pair it with CRP",
|
|
679
|
+
"apoB tracks risk better than LDL alone",
|
|
680
|
+
):
|
|
681
|
+
with self.subTest(text=ok_text):
|
|
682
|
+
self.assertTrue(self.lb.check_lesson(ok_text).ok, ok_text)
|
|
683
|
+
|
|
684
|
+
def test_round2_a_value_SPELLED_OUT_is_still_a_value(self):
|
|
685
|
+
"""`John's TSH was eight point zero four` carried a reading past a digit-only rule."""
|
|
686
|
+
self.assertFalse(self.lb.check_lesson("his tsh was eight point zero four").ok)
|
|
687
|
+
# A lone number-word is ordinary method prose and must NOT be refused.
|
|
688
|
+
self.assertTrue(self.lb.check_lesson("one marker alone cannot close the question").ok)
|
|
689
|
+
|
|
690
|
+
def test_round2_allow_numbers_never_waves_through_a_person(self):
|
|
691
|
+
"""--allow-numbers is for a guideline threshold, never for a person plus a
|
|
692
|
+
reading. The old person list held only `his`/`her`, so these walked through."""
|
|
693
|
+
for leak in ("she has TSH 8.04 mIU/L", "subject has TSH 8.04 mIU/L",
|
|
694
|
+
"the participant had ferritin 512"):
|
|
695
|
+
with self.subTest(leak=leak):
|
|
696
|
+
self.assertFalse(self.lb.check_lesson(leak, allow_numbers=True).ok)
|
|
697
|
+
|
|
698
|
+
def test_round2_only_a_double_dash_is_refused_as_an_option(self):
|
|
699
|
+
"""`dz` treats `-contrast` as ordinary positional text (measured). Refusing it
|
|
700
|
+
blocked legitimate lessons to defend against nothing, and a guard that refuses
|
|
701
|
+
safe input teaches people to route around the guard."""
|
|
702
|
+
self.assertTrue(self.lb.check_lesson("-negative findings still require confirmation").ok)
|
|
703
|
+
self.assertIn("--", self.lb.teach("--all")[1])
|
|
704
|
+
self.assertIn("--", self.lb.recall("--all"))
|
|
705
|
+
|
|
706
|
+
def test_round2_the_docstring_no_longer_promises_a_retry_that_does_not_exist(self):
|
|
707
|
+
"""The prose claimed recall retries without --domain; the code makes ONE call.
|
|
708
|
+
Documentation of behaviour that does not exist is a defect with no stack trace,
|
|
709
|
+
so it gets a test like any other."""
|
|
710
|
+
calls = []
|
|
711
|
+
original = self.lb._run_dz
|
|
712
|
+
|
|
713
|
+
def record(args, **kwargs):
|
|
714
|
+
calls.append(list(args))
|
|
715
|
+
return 1, "", "unknown option"
|
|
716
|
+
|
|
717
|
+
self.lb._run_dz = record
|
|
718
|
+
try:
|
|
719
|
+
self.lb.recall("x")
|
|
720
|
+
self.assertEqual(len(calls), 1, "exactly one call, no fallback")
|
|
721
|
+
finally:
|
|
722
|
+
self.lb._run_dz = original
|
|
723
|
+
self.assertNotIn("we retry WITHOUT the flag", self.lb.recall.__doc__ or "")
|
|
724
|
+
|
|
725
|
+
|
|
726
|
+
# ---------------------------------------------------------------- round 3
|
|
727
|
+
# The third review found the SAME class of defect a third time: two of the three
|
|
728
|
+
# clauses were still enumerations. These anchor the token-allowlist rewrite.
|
|
729
|
+
|
|
730
|
+
def test_round3_a_short_or_non_ascii_name_is_refused(self):
|
|
731
|
+
"""`Li` was too short for the old 3-letter pattern; `Élodie` starts with a
|
|
732
|
+
Latin-1 capital it never matched; `method—John` hid behind an em dash that the
|
|
733
|
+
whitespace tokenizer did not split, landing the name inside the exempt first
|
|
734
|
+
token. Widening a name pattern buys one round; asking what an ACCEPTED token
|
|
735
|
+
looks like ends the game."""
|
|
736
|
+
for leak in ("patient Li has HIV", "patient Élodie has HIV", "method—John has HIV",
|
|
737
|
+
"however Smith reported otherwise"):
|
|
738
|
+
with self.subTest(leak=leak):
|
|
739
|
+
self.assertFalse(self.lb.check_lesson(leak).ok, leak)
|
|
740
|
+
|
|
741
|
+
def test_round3_a_single_spelled_number_beside_a_person_is_a_reading(self):
|
|
742
|
+
"""`his tsh was twelve` passed: one number word was not treated as a number.
|
|
743
|
+
A lone number word in ordinary prose still must not be refused."""
|
|
744
|
+
self.assertFalse(self.lb.check_lesson("his tsh was twelve").ok)
|
|
745
|
+
self.assertTrue(self.lb.check_lesson("one marker alone cannot close the question").ok)
|
|
746
|
+
|
|
747
|
+
def test_round3_singular_person_is_a_record_PLURAL_is_a_population(self):
|
|
748
|
+
"""The first fix simply added `woman|man|adult|child` and immediately refused
|
|
749
|
+
`…warrants attention in men`, a textbook population lesson. Grammar, not a word
|
|
750
|
+
list, separates a cohort from a patient — and grammar is checkable."""
|
|
751
|
+
self.assertFalse(self.lb.check_lesson("the woman has tsh 8.04", allow_numbers=True).ok)
|
|
752
|
+
self.assertTrue(
|
|
753
|
+
self.lb.check_lesson("transferrin saturation above 45% warrants attention in men",
|
|
754
|
+
allow_numbers=True).ok)
|
|
755
|
+
|
|
756
|
+
def test_round3_a_token_mixing_letters_and_digits_is_an_identifier(self):
|
|
757
|
+
"""`passport ab1234567` walked through --allow-numbers, because the identifier
|
|
758
|
+
list knew only email, phone and medical-record vocabulary. A threshold is a
|
|
759
|
+
number with at most a short unit after it; letters BEFORE digits is a label."""
|
|
760
|
+
for leak in ("passport ab1234567 identifies the holder",
|
|
761
|
+
"trial NCT04368728 enrolled him at 8.04"):
|
|
762
|
+
with self.subTest(leak=leak):
|
|
763
|
+
self.assertFalse(self.lb.check_lesson(leak, allow_numbers=True).ok, leak)
|
|
764
|
+
# …and a real threshold with a unit still passes.
|
|
765
|
+
self.assertTrue(self.lb.check_lesson("a 56h fast lowers total testosterone by a third",
|
|
766
|
+
allow_numbers=True).ok)
|
|
767
|
+
|
|
768
|
+
|
|
769
|
+
def test_a_real_failure_is_not_disguised_as_an_old_cli(self):
|
|
770
|
+
"""A crash, timeout or corrupt store must read as a failure, not as 'upgrade'."""
|
|
771
|
+
original = self.lb._run_dz
|
|
772
|
+
self.lb._run_dz = lambda *a, **k: (1, "", "store corrupt")
|
|
773
|
+
try:
|
|
774
|
+
out = self.lb.recall("x")
|
|
775
|
+
self.assertIn("WITHOUT prior lessons", out)
|
|
776
|
+
self.assertNotIn("predates", out)
|
|
777
|
+
finally:
|
|
778
|
+
self.lb._run_dz = original
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
|
|
783
|
+
if __name__ == "__main__":
|
|
784
|
+
unittest.main()
|