@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,398 +1,524 @@
1
- """
2
- core/reasoning/payload_validation.py
3
-
4
- ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
5
-
6
- §3's retry corrects a response that does not PARSE. Nothing checked that
7
- what parsed was the SHAPE the pass promised, so the same defect appeared
8
- in six places at once (audit #76): a wrapped list crashed with
9
- `'str' object has no attribute 'items'`, a bad `severity` survived two
10
- passes and surfaced in Pass 5's pure-code tiering, and two shapes
11
- completed a run and reported a false outcome.
12
-
13
- WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
14
- this validator -- `review.py:_validated` checks a top-level type, the
15
- required keys per entry with type guards BEFORE the membership tests, an
16
- id cross-check against `{c.id for c in run.claims}`, an enum, and a
17
- duplicate -- and built it once, for the one payload that is NOT a pass.
18
- The passes never got it. Putting the general version in orchestrator.py
19
- would repeat json_retry.py's own mistake in reverse: that module was
20
- extracted precisely so a mechanism a second consumer might want is not
21
- trapped in the orchestrator.
22
-
23
- `review.py` is deliberately NOT migrated onto this module (§30, B1). Its
24
- POLICY is different and correctly so: a finding it cannot apply is
25
- dropped and traced, because the review stage is optional and contained,
26
- and a reviewer that misfires must not cost a completed ten-pass run. A
27
- pass is not optional -- a payload it cannot apply is a run that will
28
- report something untrue -- so a failure here re-enters §3's corrective
29
- retry (B2) and ultimately raises. Same shape, two policies. Merging them
30
- means choosing one, and neither is wrong for its own caller.
31
-
32
- WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
33
- declared type where a value is used arithmetically -- every one has a
34
- demonstrated consequence in #76's table, and nothing else does:
35
-
36
- the top-level type {"claims": [...]} instead of [...]
37
- the required keys per entry a claim with no `id`, a revision with no
38
- `revised_text`, Pass 4 with no
39
- `per_claim_flags`
40
- a declared value type `"severity": "0.0"` -- a string that
41
- fails two passes later, in Pass 5
42
- that entry ids resolve 3a/3b answering about ids Pass 2 never
43
- issued, which grounded nothing while the
44
- trace said it had grounded everything
45
-
46
- Not a schema language, and no new dependency (#145 tracks those). A
47
- richer vocabulary would let a spec describe things no pass has ever got
48
- wrong, at the price of a second place where a payload's meaning lives.
49
-
50
- STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
51
- `Respond with ONLY this JSON structure` block in that pass's own .md, and
52
- NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
53
- field will fail the run rather than silently mis-apply it -- which is the
54
- better failure -- but it will fail it at the model's expense. The parity
55
- that IS mechanised is narrower and is the one that goes wrong silently:
56
- every pass the orchestrator routes through `_run_pass_with_json_retry`
57
- has a spec here (see tests/test_payload_validation.py).
58
- """
59
-
60
- from __future__ import annotations
61
-
62
- from dataclasses import dataclass, field
63
- from typing import Any, Optional
64
-
65
- # Re-exported, not re-derived: base.py defines these beside the Literals
66
- # they come from, and the scorer reads the same two names (§30, B9).
67
- from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
68
-
69
- NUMBER = (int, float)
70
-
71
-
72
- class PayloadShapeError(ValueError):
73
- """A pass's response parsed as JSON but is not the shape it promised.
74
-
75
- SUBCLASSES ValueError deliberately, and that is load-bearing twice:
76
- json_retry.py's corrective loop already catches ValueError (so a shape
77
- failure re-enters the retry with no new except clause), and the
78
- orchestrator's outer try/except -- which turns a failure into a
79
- durable status='failed' record -- is byte-for-byte unchanged.
80
-
81
- The message names the pass, the field and the expected shape, because
82
- the whole complaint in #76's loud half is that
83
- `'str' object has no attribute 'items'` names none of the three.
84
- """
85
-
86
-
87
- @dataclass(frozen=True)
88
- class Spec:
89
- """One pass's promised payload shape.
90
-
91
- top "object" or "array"
92
- required top-level keys that must be present (top="object")
93
- key_types top-level key -> permitted Python type(s)
94
- entry_required keys every entry must carry (top="array")
95
- entry_types entry key -> permitted Python type(s)
96
- enums entry key -> the set of permitted values
97
- ids entry key holding a claim id, cross-checked against
98
- the claims the apply step will resolve against
99
- unique entry key that must not repeat (Pass 2's `id`)
100
- ensemble_required entry keys required only on an ensemble run (B7)
101
- ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
102
- nested top-level key -> Spec for the array beneath it
103
- """
104
-
105
- top: str
106
- required: frozenset = frozenset()
107
- key_types: dict = field(default_factory=dict)
108
- entry_required: frozenset = frozenset()
109
- entry_types: dict = field(default_factory=dict)
110
- enums: dict = field(default_factory=dict)
111
- ids: Optional[str] = None
112
- unique: Optional[str] = None
113
- ensemble_required: frozenset = frozenset()
114
- ids_in_keys: Optional[str] = None
115
- nested: dict = field(default_factory=dict)
116
-
117
-
118
- # The 3a and 3b entry shapes are named, because Pass 6b is documented in
119
- # revalidate.md as returning exactly those two lists under two keys --
120
- # and _apply_grounding/_apply_critic are literally the same two functions.
121
- # Writing them twice would let 6c drift from the passes it re-runs.
122
- _GROUNDING_ENTRY = Spec(
123
- top="array",
124
- entry_required=frozenset({"claim_id", "status"}),
125
- # §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
126
- # asymmetry is the point. B4's rule is that a check earns its place by
127
- # having a demonstrated consequence:
128
- #
129
- # the type §45's scoring stage ITERATES this value. A string
130
- # there scores one source per character, and a null
131
- # raises inside a stage that runs after the model call
132
- # it would need to retry.
133
- # not required source_grounding.md says an ungrounded claim's
134
- # `sources` "may be an empty list", and a model that
135
- # omits the key entirely on a claim it could not
136
- # source has said the same thing. Failing that run
137
- # costs a corrective retry to add `[]` to an entry
138
- # nothing reads.
139
- #
140
- # The case this leaves open -- `grounded` with no sources at all -- is
141
- # NOT unhandled; it is handled where it means something. The scoring
142
- # stage scores it as zero source quality and says so in the trace,
143
- # which is the honest answer to a claim asserted as grounded with
144
- # nothing behind it. Rejecting it here would instead ask the model to
145
- # try again, and a model that had no sources still has none.
146
- entry_types={"claim_id": str, "sources": list},
147
- enums={"status": GROUNDING_STATUSES},
148
- ids="claim_id",
149
- )
150
-
151
- _CRITIC_ENTRY = Spec(
152
- top="array",
153
- entry_required=frozenset({"claim_id", "severity"}),
154
- # `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
155
- # so a string here raises a TypeError two passes downstream, in the
156
- # one stage that makes no model call and has nothing to retry.
157
- entry_types={"claim_id": str, "severity": NUMBER},
158
- ids="claim_id",
159
- )
160
-
161
-
162
- SPECS = {
163
- # preliminary_plan.md names five keys, and the orchestrator reads ONE
164
- # of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
165
- # before handing the plan whole to Pass 3c. So requiring all five
166
- # would fail runs over an omission nothing depends on. The
167
- # demonstrated defect is a bare list, which is what `top` catches.
168
- "Pass 0": Spec(top="object"),
169
-
170
- "Pass 2": Spec(
171
- top="array",
172
- entry_required=frozenset({"id", "text", "type"}),
173
- entry_types={"id": str, "text": str},
174
- enums={"type": CLAIM_TYPES},
175
- # B6. Two claims sharing an id split in half: _apply_grounding
176
- # builds {c.id: c} (last wins) while _apply_assumption_flags uses
177
- # next(...) (first wins), so the grounding landed on one object
178
- # and the assumption flags on the other. claim_extraction.md asks
179
- # for "sequential zero-padded ids" and never says they must be
180
- # unique.
181
- unique="id",
182
- # B7. Only in ensemble mode, and only because claim_extraction.md
183
- # is now asked to supply it on every claim there. An ABSENT field
184
- # is indistinguishable from an empty one at the scorer, and
185
- # `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
186
- # a claim that exists because a candidate asserted it.
187
- ensemble_required=frozenset({"asserted_by_candidates"}),
188
- ),
189
-
190
- "Pass 3a": _GROUNDING_ENTRY,
191
- "Pass 3b": _CRITIC_ENTRY,
192
-
193
- "Pass 3c": Spec(
194
- top="object",
195
- required=frozenset({"gaps", "coverage_score"}),
196
- # `gaps` is iterated by build_coverage_gap_entries; a string there
197
- # would produce one coverage-gap entry per CHARACTER.
198
- key_types={"gaps": list},
199
- ),
200
-
201
- "Pass 4": Spec(
202
- top="object",
203
- required=frozenset({"per_claim_flags"}),
204
- key_types={"per_claim_flags": dict},
205
- ids_in_keys="per_claim_flags",
206
- ),
207
-
208
- "Pass 6a": Spec(
209
- top="array",
210
- entry_required=frozenset({"claim_id", "revised_text"}),
211
- entry_types={"claim_id": str, "revised_text": str},
212
- ids="claim_id",
213
- ),
214
-
215
- "Pass 6b": Spec(
216
- top="object",
217
- required=frozenset({"grounding", "critic"}),
218
- key_types={"grounding": list, "critic": list},
219
- nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
220
- ),
221
- }
222
-
223
-
224
- def _type_name(value: Any) -> str:
225
- """JSON's vocabulary, not Python's -- the message goes back to a model
226
- that was asked for JSON, so `object`/`array` is what it can act on."""
227
- return {dict: "object", list: "array", str: "string",
228
- bool: "boolean", int: "number", float: "number",
229
- type(None): "null"}.get(type(value), type(value).__name__)
230
-
231
-
232
- def _permitted(types) -> str:
233
- if not isinstance(types, tuple):
234
- types = (types,)
235
- return " or ".join(sorted({_type_name(t()) for t in types}))
236
-
237
-
238
- def validate(pass_id: str, payload: Any, claim_ids=None,
239
- ensemble: bool = False) -> None:
240
- """Raise PayloadShapeError unless `payload` is what `pass_id` promised.
241
-
242
- claim_ids the ids the APPLY step will resolve against -- all claims
243
- for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
244
- "every claim in the run": Pass 6a is only asked about
245
- `flagged`, and its own lookup already resolves within that
246
- batch, so validating against a wider set would accept a
247
- revision that then lands on nothing. None skips the check.
248
- ensemble True on an ensemble run, which is the only place Pass 2 is
249
- asked for `asserted_by_candidates` (B7).
250
-
251
- An unknown pass_id RAISES rather than passing through unvalidated. A
252
- pass with no spec is the silent hole this module exists to close, and
253
- the eleventh pass must not be able to arrive by forgetting a line.
254
- """
255
- spec = SPECS.get(pass_id)
256
- if spec is None:
257
- raise PayloadShapeError(
258
- f"{pass_id}: no payload spec is declared for this pass. Every "
259
- f"pass routed through _run_pass_with_json_retry needs one in "
260
- f"core/reasoning/payload_validation.py (§30, B11)."
261
- )
262
- _validate_against(pass_id, payload, spec, claim_ids, ensemble)
263
-
264
-
265
- def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
266
- where: str = "") -> None:
267
- at = f" under {where!r}" if where else ""
268
-
269
- if spec.top == "array":
270
- if not isinstance(payload, list):
271
- raise PayloadShapeError(
272
- f"{pass_id}: expected a JSON array{at}, got "
273
- f"{_type_name(payload)}. Return the array itself, not an "
274
- f"object wrapping it."
275
- )
276
- _validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
277
- return
278
-
279
- if not isinstance(payload, dict):
280
- raise PayloadShapeError(
281
- f"{pass_id}: expected a JSON object{at}, got "
282
- f"{_type_name(payload)}."
283
- )
284
-
285
- missing = sorted(spec.required - set(payload))
286
- if missing:
287
- raise PayloadShapeError(
288
- f"{pass_id}: the response{at} is missing required key(s) "
289
- f"{', '.join(repr(m) for m in missing)}. Expected an object "
290
- f"with {', '.join(sorted(spec.required))}."
291
- )
292
-
293
- for key, types in spec.key_types.items():
294
- if key in payload and not isinstance(payload[key], types):
295
- raise PayloadShapeError(
296
- f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
297
- f"expected {_permitted(types)}."
298
- )
299
-
300
- for key, nested in spec.nested.items():
301
- _validate_against(pass_id, payload[key], nested, claim_ids,
302
- ensemble, where=key)
303
-
304
- if spec.ids_in_keys:
305
- _check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
306
- f"key of {spec.ids_in_keys!r}")
307
-
308
-
309
- def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
310
- required = spec.entry_required | (spec.ensemble_required if ensemble
311
- else frozenset())
312
- seen = {}
313
- for index, entry in enumerate(entries):
314
- if not isinstance(entry, dict):
315
- raise PayloadShapeError(
316
- f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
317
- f"every entry must be an object with "
318
- f"{', '.join(sorted(required))}."
319
- )
320
- missing = sorted(required - set(entry))
321
- if missing:
322
- raise PayloadShapeError(
323
- f"{pass_id}: entry {index}{at} is missing required key(s) "
324
- f"{', '.join(repr(m) for m in missing)}. Every entry needs "
325
- f"{', '.join(sorted(required))}."
326
- )
327
- # TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
328
- # rule and the reason it is worth copying: a JSON array or object
329
- # in an enum-checked field is unhashable, so `value in ALLOWED`
330
- # raises TypeError from inside the validator rather than
331
- # producing a message the model can act on.
332
- for key, types in spec.entry_types.items():
333
- if key in entry and not isinstance(entry[key], types):
334
- raise PayloadShapeError(
335
- f"{pass_id}: entry {index}{at} has {key}="
336
- f"{entry[key]!r}, which is {_type_name(entry[key])}; "
337
- f"expected {_permitted(types)}."
338
- )
339
- for key, allowed in spec.enums.items():
340
- if key not in entry:
341
- continue
342
- value = entry[key]
343
- if not isinstance(value, str) or value not in allowed:
344
- raise PayloadShapeError(
345
- f"{pass_id}: entry {index}{at} has {key}={value!r}; "
346
- f"expected exactly one of "
347
- f"{', '.join(sorted(allowed))} (lower case)."
348
- )
349
- if spec.unique:
350
- value = entry[spec.unique]
351
- if value in seen:
352
- raise PayloadShapeError(
353
- f"{pass_id}: entries {seen[value]} and {index}{at} "
354
- f"share {spec.unique}={value!r}. Ids must be unique -- "
355
- f"two claims with one id are applied to inconsistently "
356
- f"and end up carrying half of each other's results."
357
- )
358
- seen[value] = index
359
-
360
- if spec.ids:
361
- _check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
362
- spec.ids)
363
-
364
-
365
- def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
366
- """B5: a TOTAL mismatch is the error; a partial one is not.
367
-
368
- The demonstrated silent failure is every id being wrong -- 3a/3b
369
- answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
370
- which grounded nothing while the checkpoint line reported that it had
371
- grounded everything. That is a pass with no effect at all, and it is
372
- worth a corrective retry.
373
-
374
- A model that gets nine of ten right is NOT worth failing a run that
375
- has already paid for nine passes: the nine apply, and the checkpoint
376
- line reports the count APPLIED rather than the count asked about, so
377
- the tenth is visible instead of silent (see the orchestrator's
378
- _apply_* return values).
379
-
380
- An EMPTY collection is not a mismatch. assumption_audit.md asks the
381
- model to "omit claims with no issues rather than including them with
382
- an empty list", so no flags at all is the ordinary answer for a clean
383
- run -- and Pass 4's demonstrated defect was the top-level key being
384
- absent, which the required-key check above catches instead.
385
- """
386
- if claim_ids is None or not supplied:
387
- return
388
- known = set(claim_ids)
389
- if any(value in known for value in supplied):
390
- return
391
- sample = ", ".join(repr(v) for v in supplied[:3])
392
- expected = ", ".join(repr(v) for v in sorted(known)[:3])
393
- raise PayloadShapeError(
394
- f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
395
- f"matches a claim you were given (got {sample}; expected ids like "
396
- f"{expected}). Use the claim ids exactly as they appear in the "
397
- f"input."
398
- )
1
+ """
2
+ core/reasoning/payload_validation.py
3
+
4
+ ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
5
+
6
+ §3's retry corrects a response that does not PARSE. Nothing checked that
7
+ what parsed was the SHAPE the pass promised, so the same defect appeared
8
+ in six places at once (audit #76): a wrapped list crashed with
9
+ `'str' object has no attribute 'items'`, a bad `severity` survived two
10
+ passes and surfaced in Pass 5's pure-code tiering, and two shapes
11
+ completed a run and reported a false outcome.
12
+
13
+ WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
14
+ this validator -- `review.py:_validated` checks a top-level type, the
15
+ required keys per entry with type guards BEFORE the membership tests, an
16
+ id cross-check against `{c.id for c in run.claims}`, an enum, and a
17
+ duplicate -- and built it once, for the one payload that is NOT a pass.
18
+ The passes never got it. Putting the general version in orchestrator.py
19
+ would repeat json_retry.py's own mistake in reverse: that module was
20
+ extracted precisely so a mechanism a second consumer might want is not
21
+ trapped in the orchestrator.
22
+
23
+ `review.py` is deliberately NOT migrated onto this module (§30, B1). Its
24
+ POLICY is different and correctly so: a finding it cannot apply is
25
+ dropped and traced, because the review stage is optional and contained,
26
+ and a reviewer that misfires must not cost a completed ten-pass run. A
27
+ pass is not optional -- a payload it cannot apply is a run that will
28
+ report something untrue -- so a failure here re-enters §3's corrective
29
+ retry (B2) and ultimately raises. Same shape, two policies. Merging them
30
+ means choosing one, and neither is wrong for its own caller.
31
+
32
+ WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
33
+ declared type where a value is used arithmetically -- every one has a
34
+ demonstrated consequence in #76's table, and nothing else does:
35
+
36
+ the top-level type {"claims": [...]} instead of [...]
37
+ the required keys per entry a claim with no `id`, a revision with no
38
+ `revised_text`, Pass 4 with no
39
+ `per_claim_flags`
40
+ a declared value type `"severity": "0.0"` -- a string that
41
+ fails two passes later, in Pass 5
42
+ the type of its MEMBERS `"entities": ["madrid", 1947]` -- a list,
43
+ which Pass 3a then sorts
44
+ that entry ids resolve 3a/3b answering about ids Pass 2 never
45
+ issued, which grounded nothing while the
46
+ trace said it had grounded everything
47
+
48
+ THE FOURTH ROW IS BATCH 149, AND IT IS NOT A RICHER VOCABULARY FOR ITS OWN
49
+ SAKE -- it is the same property as the third, asked of the values a
50
+ container holds, because that is where the arithmetic actually happens.
51
+ `entities` is not measured; every element of it is hashed and then
52
+ compared against every other (`sorted({e for c in claims for e in
53
+ c.entities})`), and `asserted_by_candidates` is hashed the same way by
54
+ `len(set(...))`. A declared CONTAINER type closes the three container
55
+ cases and leaves the three member cases open, each driven against the real
56
+ consumer:
57
+
58
+ ["madrid", 1947] TypeError: '<' not supported between
59
+ instances of 'int' and 'str'
60
+ [["madrid"]] TypeError: unhashable type: 'list'
61
+ [{"name": "madrid"}] TypeError: unhashable type: 'dict'
62
+
63
+ A MEMBER is a dict's VALUES or a list's ELEMENTS, deliberately one concept
64
+ and one spec field per level: Pass 4's flag lists live under claim-id keys
65
+ and Pass 2's entities live in an array, and the thing being declared is
66
+ the same in both -- what the pure code downstream will iterate, hash, sort
67
+ or measure. Pass 4's flag lists are typed and their STRINGS are not, which
68
+ is the same rule: `len()` is the only thing that reads them.
69
+
70
+ Not a schema language, and no new dependency (#145 tracks those). A
71
+ richer vocabulary would let a spec describe things no pass has ever got
72
+ wrong, at the price of a second place where a payload's meaning lives.
73
+
74
+ STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
75
+ `Respond with ONLY this JSON structure` block in that pass's own .md, and
76
+ NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
77
+ field will fail the run rather than silently mis-apply it -- which is the
78
+ better failure -- but it will fail it at the model's expense. The parity
79
+ that IS mechanised is narrower and is the one that goes wrong silently:
80
+ every pass the orchestrator routes through `_run_pass_with_json_retry`
81
+ has a spec here (see tests/test_payload_validation.py).
82
+ """
83
+
84
+ from __future__ import annotations
85
+
86
+ from dataclasses import dataclass, field
87
+ from typing import Any, Optional
88
+
89
+ # Re-exported, not re-derived: base.py defines these beside the Literals
90
+ # they come from, and the scorer reads the same two names (§30, B9).
91
+ from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
92
+
93
+ # A BOOLEAN IS NOT A NUMBER, which Python disagrees with: `isinstance(
94
+ # True, int)` is True, so `"severity": true` satisfied this tuple and
95
+ # Pass 5 computed `1.0 - True` as 0.0 -- the MAXIMUM critic penalty, the
96
+ # single harshest value the field can take -- with nothing in the trace
97
+ # and no corrective retry (batch 149). `"severity": false` took the other
98
+ # extreme for the same reason.
99
+ #
100
+ # The exclusion is not new to the codebase, only to the boundary:
101
+ # `source_scoring._as_number` and `confidence_scoring.source_quality` both
102
+ # spell it out already, each guarding a value that is supposed to have
103
+ # been rejected HERE. See `_of_type` below, which is what applies it.
104
+ NUMBER = (int, float)
105
+
106
+
107
+ class PayloadShapeError(ValueError):
108
+ """A pass's response parsed as JSON but is not the shape it promised.
109
+
110
+ SUBCLASSES ValueError deliberately, and that is load-bearing twice:
111
+ json_retry.py's corrective loop already catches ValueError (so a shape
112
+ failure re-enters the retry with no new except clause), and the
113
+ orchestrator's outer try/except -- which turns a failure into a
114
+ durable status='failed' record -- is byte-for-byte unchanged.
115
+
116
+ The message names the pass, the field and the expected shape, because
117
+ the whole complaint in #76's loud half is that
118
+ `'str' object has no attribute 'items'` names none of the three.
119
+ """
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class Spec:
124
+ """One pass's promised payload shape.
125
+
126
+ top "object" or "array"
127
+ required top-level keys that must be present (top="object")
128
+ key_types top-level key -> permitted Python type(s)
129
+ entry_required keys every entry must carry (top="array")
130
+ entry_types entry key -> permitted Python type(s)
131
+ key_member_types top-level key -> permitted type for every MEMBER of
132
+ the container under it (a dict's values, a list's
133
+ elements)
134
+ entry_member_types the same, per entry key
135
+ enums entry key -> the set of permitted values
136
+ ids entry key holding a claim id, cross-checked against
137
+ the claims the apply step will resolve against
138
+ unique entry key that must not repeat (Pass 2's `id`)
139
+ ensemble_required entry keys required only on an ensemble run (B7)
140
+ ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
141
+ nested top-level key -> Spec for the array beneath it
142
+ """
143
+
144
+ top: str
145
+ required: frozenset = frozenset()
146
+ key_types: dict = field(default_factory=dict)
147
+ entry_required: frozenset = frozenset()
148
+ entry_types: dict = field(default_factory=dict)
149
+ key_member_types: dict = field(default_factory=dict)
150
+ entry_member_types: dict = field(default_factory=dict)
151
+ enums: dict = field(default_factory=dict)
152
+ ids: Optional[str] = None
153
+ unique: Optional[str] = None
154
+ ensemble_required: frozenset = frozenset()
155
+ ids_in_keys: Optional[str] = None
156
+ nested: dict = field(default_factory=dict)
157
+
158
+
159
+ # The 3a and 3b entry shapes are named, because Pass 6b is documented in
160
+ # revalidate.md as returning exactly those two lists under two keys --
161
+ # and _apply_grounding/_apply_critic are literally the same two functions.
162
+ # Writing them twice would let 6c drift from the passes it re-runs.
163
+ _GROUNDING_ENTRY = Spec(
164
+ top="array",
165
+ entry_required=frozenset({"claim_id", "status"}),
166
+ # §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
167
+ # asymmetry is the point. B4's rule is that a check earns its place by
168
+ # having a demonstrated consequence:
169
+ #
170
+ # the type §45's scoring stage ITERATES this value. A string
171
+ # there scores one source per character, and a null
172
+ # raises inside a stage that runs after the model call
173
+ # it would need to retry.
174
+ # not required source_grounding.md says an ungrounded claim's
175
+ # `sources` "may be an empty list", and a model that
176
+ # omits the key entirely on a claim it could not
177
+ # source has said the same thing. Failing that run
178
+ # costs a corrective retry to add `[]` to an entry
179
+ # nothing reads.
180
+ #
181
+ # The case this leaves open -- `grounded` with no sources at all -- is
182
+ # NOT unhandled; it is handled where it means something. The scoring
183
+ # stage scores it as zero source quality and says so in the trace,
184
+ # which is the honest answer to a claim asserted as grounded with
185
+ # nothing behind it. Rejecting it here would instead ask the model to
186
+ # try again, and a model that had no sources still has none.
187
+ entry_types={"claim_id": str, "sources": list},
188
+ enums={"status": GROUNDING_STATUSES},
189
+ ids="claim_id",
190
+ )
191
+
192
+ _CRITIC_ENTRY = Spec(
193
+ top="array",
194
+ entry_required=frozenset({"claim_id", "severity"}),
195
+ # `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
196
+ # so a string here raises a TypeError two passes downstream, in the
197
+ # one stage that makes no model call and has nothing to retry.
198
+ entry_types={"claim_id": str, "severity": NUMBER},
199
+ ids="claim_id",
200
+ )
201
+
202
+
203
+ SPECS = {
204
+ # preliminary_plan.md names five keys, and the orchestrator reads ONE
205
+ # of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
206
+ # before handing the plan whole to Pass 3c. So requiring all five
207
+ # would fail runs over an omission nothing depends on. The
208
+ # demonstrated defect is a bare list, which is what `top` catches.
209
+ #
210
+ # THE ONE KEY THE ORCHESTRATOR READS IS TYPED, AND STILL NOT REQUIRED
211
+ # (batch 149) -- `key_types` is checked only `if key in payload`, so
212
+ # the defensive `.get` above keeps its meaning and an omission still
213
+ # costs nothing. What it stops is the read itself: `len(123)` raised
214
+ # inside the Pass 0 checkpoint, and `len("madrid")` did not raise at
215
+ # all -- it reported SIX key entities anticipated, from one word.
216
+ "Pass 0": Spec(top="object",
217
+ key_types={"key_entities_or_subjects": list}),
218
+
219
+ "Pass 2": Spec(
220
+ top="array",
221
+ entry_required=frozenset({"id", "text", "type"}),
222
+ # `entities` and `asserted_by_candidates` are TYPED BUT NOT
223
+ # REQUIRED, which is the same asymmetry `sources` carries below
224
+ # and for the same reason: base.py defaults both to an empty list,
225
+ # a claim with no entities is an ordinary answer, and
226
+ # `asserted_by_candidates` is required only in ensemble mode (B7).
227
+ # What they are not is optional to get RIGHT -- Pass 3a sorts a
228
+ # set built from every claim's entities, and Pass 5 measures
229
+ # `len(set(asserted_by_candidates))`, so both the container and
230
+ # its members are declared (batch 149).
231
+ entry_types={"id": str, "text": str, "entities": list,
232
+ "asserted_by_candidates": list},
233
+ # str and int from base.py's own annotations -- `entities:
234
+ # list[str]` and `asserted_by_candidates: list[int]`. The
235
+ # annotations are not enforced (the comment at the top of base.py
236
+ # says so, and #75 is what that cost); this is where they become a
237
+ # check.
238
+ entry_member_types={"entities": str, "asserted_by_candidates": int},
239
+ enums={"type": CLAIM_TYPES},
240
+ # B6. Two claims sharing an id split in half: _apply_grounding
241
+ # builds {c.id: c} (last wins) while _apply_assumption_flags uses
242
+ # next(...) (first wins), so the grounding landed on one object
243
+ # and the assumption flags on the other. claim_extraction.md asks
244
+ # for "sequential zero-padded ids" and never says they must be
245
+ # unique.
246
+ unique="id",
247
+ # B7. Only in ensemble mode, and only because claim_extraction.md
248
+ # is now asked to supply it on every claim there. An ABSENT field
249
+ # is indistinguishable from an empty one at the scorer, and
250
+ # `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
251
+ # a claim that exists because a candidate asserted it.
252
+ ensemble_required=frozenset({"asserted_by_candidates"}),
253
+ ),
254
+
255
+ "Pass 3a": _GROUNDING_ENTRY,
256
+ "Pass 3b": _CRITIC_ENTRY,
257
+
258
+ "Pass 3c": Spec(
259
+ top="object",
260
+ required=frozenset({"gaps", "coverage_score"}),
261
+ # `gaps` is iterated by build_coverage_gap_entries; a string there
262
+ # would produce one coverage-gap entry per CHARACTER.
263
+ key_types={"gaps": list},
264
+ ),
265
+
266
+ "Pass 4": Spec(
267
+ top="object",
268
+ required=frozenset({"per_claim_flags"}),
269
+ key_types={"per_claim_flags": dict},
270
+ # Every VALUE under those claim ids is a list, because Pass 5
271
+ # measures it: `w["assumption_flag_penalty"] * len(flags)` raised
272
+ # on `{"c001": 123}` and silently charged a THREE-flag penalty for
273
+ # `{"c001": "abc"}` (batch 149). The strings inside are not typed
274
+ # -- `len()` is all that reads them, so a type on the elements
275
+ # would be a check with no demonstrated consequence, which is the
276
+ # rule B4 states.
277
+ key_member_types={"per_claim_flags": list},
278
+ ids_in_keys="per_claim_flags",
279
+ ),
280
+
281
+ "Pass 6a": Spec(
282
+ top="array",
283
+ entry_required=frozenset({"claim_id", "revised_text"}),
284
+ entry_types={"claim_id": str, "revised_text": str},
285
+ ids="claim_id",
286
+ ),
287
+
288
+ "Pass 6b": Spec(
289
+ top="object",
290
+ required=frozenset({"grounding", "critic"}),
291
+ key_types={"grounding": list, "critic": list},
292
+ nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
293
+ ),
294
+ }
295
+
296
+
297
+ def _type_name(value: Any) -> str:
298
+ """JSON's vocabulary, not Python's -- the message goes back to a model
299
+ that was asked for JSON, so `object`/`array` is what it can act on."""
300
+ return {dict: "object", list: "array", str: "string",
301
+ bool: "boolean", int: "number", float: "number",
302
+ type(None): "null"}.get(type(value), type(value).__name__)
303
+
304
+
305
+ def _permitted(types) -> str:
306
+ return " or ".join(sorted({_type_name(t()) for t in _as_tuple(types)}))
307
+
308
+
309
+ def _as_tuple(types) -> tuple:
310
+ return types if isinstance(types, tuple) else (types,)
311
+
312
+
313
+ def _of_type(value, types) -> bool:
314
+ """isinstance, except that a boolean is only ever a boolean.
315
+
316
+ See the comment on NUMBER. Applied to every declared type rather than
317
+ to NUMBER alone, because "a boolean satisfies a type it was not
318
+ declared as" is the defect, and `severity` is simply the only place
319
+ the tree has reached it so far.
320
+ """
321
+ if isinstance(value, bool) and bool not in _as_tuple(types):
322
+ return False
323
+ return isinstance(value, types)
324
+
325
+
326
+ def _members(container):
327
+ """(label, value) for everything a container holds.
328
+
329
+ A dict's VALUES, keyed by the key that named them, and a list's
330
+ elements by position -- so the message can say `'per_claim_flags'
331
+ ['c001']` or `entities[1]` and the model can find the one it got
332
+ wrong.
333
+ """
334
+ if isinstance(container, dict):
335
+ return [(f"[{key!r}]", value) for key, value in container.items()]
336
+ return [(f"[{index}]", value) for index, value in enumerate(container)]
337
+
338
+
339
+ def _check_members(pass_id, container, member_types, at, prefix="") -> None:
340
+ for key, types in member_types.items():
341
+ if key not in container:
342
+ continue
343
+ value = container[key]
344
+ if not isinstance(value, (dict, list)):
345
+ # The declared CONTAINER type has already refused this, or
346
+ # there isn't one and the member rule has nothing to stand on.
347
+ # Either way, iterating here would report the members of a
348
+ # string.
349
+ continue
350
+ for label, member in _members(value):
351
+ if _of_type(member, types):
352
+ continue
353
+ raise PayloadShapeError(
354
+ f"{pass_id}: {prefix}{key}{label}{at} is {member!r}, which "
355
+ f"is {_type_name(member)}; every value in {key!r} must be "
356
+ f"{_permitted(types)}."
357
+ )
358
+
359
+
360
+ def validate(pass_id: str, payload: Any, claim_ids=None,
361
+ ensemble: bool = False) -> None:
362
+ """Raise PayloadShapeError unless `payload` is what `pass_id` promised.
363
+
364
+ claim_ids the ids the APPLY step will resolve against -- all claims
365
+ for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
366
+ "every claim in the run": Pass 6a is only asked about
367
+ `flagged`, and its own lookup already resolves within that
368
+ batch, so validating against a wider set would accept a
369
+ revision that then lands on nothing. None skips the check.
370
+ ensemble True on an ensemble run, which is the only place Pass 2 is
371
+ asked for `asserted_by_candidates` (B7).
372
+
373
+ An unknown pass_id RAISES rather than passing through unvalidated. A
374
+ pass with no spec is the silent hole this module exists to close, and
375
+ the eleventh pass must not be able to arrive by forgetting a line.
376
+ """
377
+ spec = SPECS.get(pass_id)
378
+ if spec is None:
379
+ raise PayloadShapeError(
380
+ f"{pass_id}: no payload spec is declared for this pass. Every "
381
+ f"pass routed through _run_pass_with_json_retry needs one in "
382
+ f"core/reasoning/payload_validation.py (§30, B11)."
383
+ )
384
+ _validate_against(pass_id, payload, spec, claim_ids, ensemble)
385
+
386
+
387
+ def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
388
+ where: str = "") -> None:
389
+ at = f" under {where!r}" if where else ""
390
+
391
+ if spec.top == "array":
392
+ if not isinstance(payload, list):
393
+ raise PayloadShapeError(
394
+ f"{pass_id}: expected a JSON array{at}, got "
395
+ f"{_type_name(payload)}. Return the array itself, not an "
396
+ f"object wrapping it."
397
+ )
398
+ _validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
399
+ return
400
+
401
+ if not isinstance(payload, dict):
402
+ raise PayloadShapeError(
403
+ f"{pass_id}: expected a JSON object{at}, got "
404
+ f"{_type_name(payload)}."
405
+ )
406
+
407
+ missing = sorted(spec.required - set(payload))
408
+ if missing:
409
+ raise PayloadShapeError(
410
+ f"{pass_id}: the response{at} is missing required key(s) "
411
+ f"{', '.join(repr(m) for m in missing)}. Expected an object "
412
+ f"with {', '.join(sorted(spec.required))}."
413
+ )
414
+
415
+ for key, types in spec.key_types.items():
416
+ if key in payload and not _of_type(payload[key], types):
417
+ raise PayloadShapeError(
418
+ f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
419
+ f"expected {_permitted(types)}."
420
+ )
421
+
422
+ _check_members(pass_id, payload, spec.key_member_types, at)
423
+
424
+ for key, nested in spec.nested.items():
425
+ _validate_against(pass_id, payload[key], nested, claim_ids,
426
+ ensemble, where=key)
427
+
428
+ if spec.ids_in_keys:
429
+ _check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
430
+ f"key of {spec.ids_in_keys!r}")
431
+
432
+
433
+ def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
434
+ required = spec.entry_required | (spec.ensemble_required if ensemble
435
+ else frozenset())
436
+ seen = {}
437
+ for index, entry in enumerate(entries):
438
+ if not isinstance(entry, dict):
439
+ raise PayloadShapeError(
440
+ f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
441
+ f"every entry must be an object with "
442
+ f"{', '.join(sorted(required))}."
443
+ )
444
+ missing = sorted(required - set(entry))
445
+ if missing:
446
+ raise PayloadShapeError(
447
+ f"{pass_id}: entry {index}{at} is missing required key(s) "
448
+ f"{', '.join(repr(m) for m in missing)}. Every entry needs "
449
+ f"{', '.join(sorted(required))}."
450
+ )
451
+ # TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
452
+ # rule and the reason it is worth copying: a JSON array or object
453
+ # in an enum-checked field is unhashable, so `value in ALLOWED`
454
+ # raises TypeError from inside the validator rather than
455
+ # producing a message the model can act on.
456
+ for key, types in spec.entry_types.items():
457
+ if key in entry and not _of_type(entry[key], types):
458
+ raise PayloadShapeError(
459
+ f"{pass_id}: entry {index}{at} has {key}="
460
+ f"{entry[key]!r}, which is {_type_name(entry[key])}; "
461
+ f"expected {_permitted(types)}."
462
+ )
463
+ _check_members(pass_id, entry, spec.entry_member_types, at,
464
+ prefix=f"entry {index} ")
465
+ for key, allowed in spec.enums.items():
466
+ if key not in entry:
467
+ continue
468
+ value = entry[key]
469
+ if not isinstance(value, str) or value not in allowed:
470
+ raise PayloadShapeError(
471
+ f"{pass_id}: entry {index}{at} has {key}={value!r}; "
472
+ f"expected exactly one of "
473
+ f"{', '.join(sorted(allowed))} (lower case)."
474
+ )
475
+ if spec.unique:
476
+ value = entry[spec.unique]
477
+ if value in seen:
478
+ raise PayloadShapeError(
479
+ f"{pass_id}: entries {seen[value]} and {index}{at} "
480
+ f"share {spec.unique}={value!r}. Ids must be unique -- "
481
+ f"two claims with one id are applied to inconsistently "
482
+ f"and end up carrying half of each other's results."
483
+ )
484
+ seen[value] = index
485
+
486
+ if spec.ids:
487
+ _check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
488
+ spec.ids)
489
+
490
+
491
+ def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
492
+ """B5: a TOTAL mismatch is the error; a partial one is not.
493
+
494
+ The demonstrated silent failure is every id being wrong -- 3a/3b
495
+ answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
496
+ which grounded nothing while the checkpoint line reported that it had
497
+ grounded everything. That is a pass with no effect at all, and it is
498
+ worth a corrective retry.
499
+
500
+ A model that gets nine of ten right is NOT worth failing a run that
501
+ has already paid for nine passes: the nine apply, and the checkpoint
502
+ line reports the count APPLIED rather than the count asked about, so
503
+ the tenth is visible instead of silent (see the orchestrator's
504
+ _apply_* return values).
505
+
506
+ An EMPTY collection is not a mismatch. assumption_audit.md asks the
507
+ model to "omit claims with no issues rather than including them with
508
+ an empty list", so no flags at all is the ordinary answer for a clean
509
+ run -- and Pass 4's demonstrated defect was the top-level key being
510
+ absent, which the required-key check above catches instead.
511
+ """
512
+ if claim_ids is None or not supplied:
513
+ return
514
+ known = set(claim_ids)
515
+ if any(value in known for value in supplied):
516
+ return
517
+ sample = ", ".join(repr(v) for v in supplied[:3])
518
+ expected = ", ".join(repr(v) for v in sorted(known)[:3])
519
+ raise PayloadShapeError(
520
+ f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
521
+ f"matches a claim you were given (got {sample}; expected ids like "
522
+ f"{expected}). Use the claim ids exactly as they appear in the "
523
+ f"input."
524
+ )