@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,524 +1,524 @@
1
- """
2
- core/reasoning/payload_validation.py
3
-
4
- ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
5
-
6
- §3's retry corrects a response that does not PARSE. Nothing checked that
7
- what parsed was the SHAPE the pass promised, so the same defect appeared
8
- in six places at once (audit #76): a wrapped list crashed with
9
- `'str' object has no attribute 'items'`, a bad `severity` survived two
10
- passes and surfaced in Pass 5's pure-code tiering, and two shapes
11
- completed a run and reported a false outcome.
12
-
13
- WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
14
- this validator -- `review.py:_validated` checks a top-level type, the
15
- required keys per entry with type guards BEFORE the membership tests, an
16
- id cross-check against `{c.id for c in run.claims}`, an enum, and a
17
- duplicate -- and built it once, for the one payload that is NOT a pass.
18
- The passes never got it. Putting the general version in orchestrator.py
19
- would repeat json_retry.py's own mistake in reverse: that module was
20
- extracted precisely so a mechanism a second consumer might want is not
21
- trapped in the orchestrator.
22
-
23
- `review.py` is deliberately NOT migrated onto this module (§30, B1). Its
24
- POLICY is different and correctly so: a finding it cannot apply is
25
- dropped and traced, because the review stage is optional and contained,
26
- and a reviewer that misfires must not cost a completed ten-pass run. A
27
- pass is not optional -- a payload it cannot apply is a run that will
28
- report something untrue -- so a failure here re-enters §3's corrective
29
- retry (B2) and ultimately raises. Same shape, two policies. Merging them
30
- means choosing one, and neither is wrong for its own caller.
31
-
32
- WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
33
- declared type where a value is used arithmetically -- every one has a
34
- demonstrated consequence in #76's table, and nothing else does:
35
-
36
- the top-level type {"claims": [...]} instead of [...]
37
- the required keys per entry a claim with no `id`, a revision with no
38
- `revised_text`, Pass 4 with no
39
- `per_claim_flags`
40
- a declared value type `"severity": "0.0"` -- a string that
41
- fails two passes later, in Pass 5
42
- the type of its MEMBERS `"entities": ["madrid", 1947]` -- a list,
43
- which Pass 3a then sorts
44
- that entry ids resolve 3a/3b answering about ids Pass 2 never
45
- issued, which grounded nothing while the
46
- trace said it had grounded everything
47
-
48
- THE FOURTH ROW IS BATCH 149, AND IT IS NOT A RICHER VOCABULARY FOR ITS OWN
49
- SAKE -- it is the same property as the third, asked of the values a
50
- container holds, because that is where the arithmetic actually happens.
51
- `entities` is not measured; every element of it is hashed and then
52
- compared against every other (`sorted({e for c in claims for e in
53
- c.entities})`), and `asserted_by_candidates` is hashed the same way by
54
- `len(set(...))`. A declared CONTAINER type closes the three container
55
- cases and leaves the three member cases open, each driven against the real
56
- consumer:
57
-
58
- ["madrid", 1947] TypeError: '<' not supported between
59
- instances of 'int' and 'str'
60
- [["madrid"]] TypeError: unhashable type: 'list'
61
- [{"name": "madrid"}] TypeError: unhashable type: 'dict'
62
-
63
- A MEMBER is a dict's VALUES or a list's ELEMENTS, deliberately one concept
64
- and one spec field per level: Pass 4's flag lists live under claim-id keys
65
- and Pass 2's entities live in an array, and the thing being declared is
66
- the same in both -- what the pure code downstream will iterate, hash, sort
67
- or measure. Pass 4's flag lists are typed and their STRINGS are not, which
68
- is the same rule: `len()` is the only thing that reads them.
69
-
70
- Not a schema language, and no new dependency (#145 tracks those). A
71
- richer vocabulary would let a spec describe things no pass has ever got
72
- wrong, at the price of a second place where a payload's meaning lives.
73
-
74
- STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
75
- `Respond with ONLY this JSON structure` block in that pass's own .md, and
76
- NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
77
- field will fail the run rather than silently mis-apply it -- which is the
78
- better failure -- but it will fail it at the model's expense. The parity
79
- that IS mechanised is narrower and is the one that goes wrong silently:
80
- every pass the orchestrator routes through `_run_pass_with_json_retry`
81
- has a spec here (see tests/test_payload_validation.py).
82
- """
83
-
84
- from __future__ import annotations
85
-
86
- from dataclasses import dataclass, field
87
- from typing import Any, Optional
88
-
89
- # Re-exported, not re-derived: base.py defines these beside the Literals
90
- # they come from, and the scorer reads the same two names (§30, B9).
91
- from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
92
-
93
- # A BOOLEAN IS NOT A NUMBER, which Python disagrees with: `isinstance(
94
- # True, int)` is True, so `"severity": true` satisfied this tuple and
95
- # Pass 5 computed `1.0 - True` as 0.0 -- the MAXIMUM critic penalty, the
96
- # single harshest value the field can take -- with nothing in the trace
97
- # and no corrective retry (batch 149). `"severity": false` took the other
98
- # extreme for the same reason.
99
- #
100
- # The exclusion is not new to the codebase, only to the boundary:
101
- # `source_scoring._as_number` and `confidence_scoring.source_quality` both
102
- # spell it out already, each guarding a value that is supposed to have
103
- # been rejected HERE. See `_of_type` below, which is what applies it.
104
- NUMBER = (int, float)
105
-
106
-
107
- class PayloadShapeError(ValueError):
108
- """A pass's response parsed as JSON but is not the shape it promised.
109
-
110
- SUBCLASSES ValueError deliberately, and that is load-bearing twice:
111
- json_retry.py's corrective loop already catches ValueError (so a shape
112
- failure re-enters the retry with no new except clause), and the
113
- orchestrator's outer try/except -- which turns a failure into a
114
- durable status='failed' record -- is byte-for-byte unchanged.
115
-
116
- The message names the pass, the field and the expected shape, because
117
- the whole complaint in #76's loud half is that
118
- `'str' object has no attribute 'items'` names none of the three.
119
- """
120
-
121
-
122
- @dataclass(frozen=True)
123
- class Spec:
124
- """One pass's promised payload shape.
125
-
126
- top "object" or "array"
127
- required top-level keys that must be present (top="object")
128
- key_types top-level key -> permitted Python type(s)
129
- entry_required keys every entry must carry (top="array")
130
- entry_types entry key -> permitted Python type(s)
131
- key_member_types top-level key -> permitted type for every MEMBER of
132
- the container under it (a dict's values, a list's
133
- elements)
134
- entry_member_types the same, per entry key
135
- enums entry key -> the set of permitted values
136
- ids entry key holding a claim id, cross-checked against
137
- the claims the apply step will resolve against
138
- unique entry key that must not repeat (Pass 2's `id`)
139
- ensemble_required entry keys required only on an ensemble run (B7)
140
- ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
141
- nested top-level key -> Spec for the array beneath it
142
- """
143
-
144
- top: str
145
- required: frozenset = frozenset()
146
- key_types: dict = field(default_factory=dict)
147
- entry_required: frozenset = frozenset()
148
- entry_types: dict = field(default_factory=dict)
149
- key_member_types: dict = field(default_factory=dict)
150
- entry_member_types: dict = field(default_factory=dict)
151
- enums: dict = field(default_factory=dict)
152
- ids: Optional[str] = None
153
- unique: Optional[str] = None
154
- ensemble_required: frozenset = frozenset()
155
- ids_in_keys: Optional[str] = None
156
- nested: dict = field(default_factory=dict)
157
-
158
-
159
- # The 3a and 3b entry shapes are named, because Pass 6b is documented in
160
- # revalidate.md as returning exactly those two lists under two keys --
161
- # and _apply_grounding/_apply_critic are literally the same two functions.
162
- # Writing them twice would let 6c drift from the passes it re-runs.
163
- _GROUNDING_ENTRY = Spec(
164
- top="array",
165
- entry_required=frozenset({"claim_id", "status"}),
166
- # §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
167
- # asymmetry is the point. B4's rule is that a check earns its place by
168
- # having a demonstrated consequence:
169
- #
170
- # the type §45's scoring stage ITERATES this value. A string
171
- # there scores one source per character, and a null
172
- # raises inside a stage that runs after the model call
173
- # it would need to retry.
174
- # not required source_grounding.md says an ungrounded claim's
175
- # `sources` "may be an empty list", and a model that
176
- # omits the key entirely on a claim it could not
177
- # source has said the same thing. Failing that run
178
- # costs a corrective retry to add `[]` to an entry
179
- # nothing reads.
180
- #
181
- # The case this leaves open -- `grounded` with no sources at all -- is
182
- # NOT unhandled; it is handled where it means something. The scoring
183
- # stage scores it as zero source quality and says so in the trace,
184
- # which is the honest answer to a claim asserted as grounded with
185
- # nothing behind it. Rejecting it here would instead ask the model to
186
- # try again, and a model that had no sources still has none.
187
- entry_types={"claim_id": str, "sources": list},
188
- enums={"status": GROUNDING_STATUSES},
189
- ids="claim_id",
190
- )
191
-
192
- _CRITIC_ENTRY = Spec(
193
- top="array",
194
- entry_required=frozenset({"claim_id", "severity"}),
195
- # `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
196
- # so a string here raises a TypeError two passes downstream, in the
197
- # one stage that makes no model call and has nothing to retry.
198
- entry_types={"claim_id": str, "severity": NUMBER},
199
- ids="claim_id",
200
- )
201
-
202
-
203
- SPECS = {
204
- # preliminary_plan.md names five keys, and the orchestrator reads ONE
205
- # of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
206
- # before handing the plan whole to Pass 3c. So requiring all five
207
- # would fail runs over an omission nothing depends on. The
208
- # demonstrated defect is a bare list, which is what `top` catches.
209
- #
210
- # THE ONE KEY THE ORCHESTRATOR READS IS TYPED, AND STILL NOT REQUIRED
211
- # (batch 149) -- `key_types` is checked only `if key in payload`, so
212
- # the defensive `.get` above keeps its meaning and an omission still
213
- # costs nothing. What it stops is the read itself: `len(123)` raised
214
- # inside the Pass 0 checkpoint, and `len("madrid")` did not raise at
215
- # all -- it reported SIX key entities anticipated, from one word.
216
- "Pass 0": Spec(top="object",
217
- key_types={"key_entities_or_subjects": list}),
218
-
219
- "Pass 2": Spec(
220
- top="array",
221
- entry_required=frozenset({"id", "text", "type"}),
222
- # `entities` and `asserted_by_candidates` are TYPED BUT NOT
223
- # REQUIRED, which is the same asymmetry `sources` carries below
224
- # and for the same reason: base.py defaults both to an empty list,
225
- # a claim with no entities is an ordinary answer, and
226
- # `asserted_by_candidates` is required only in ensemble mode (B7).
227
- # What they are not is optional to get RIGHT -- Pass 3a sorts a
228
- # set built from every claim's entities, and Pass 5 measures
229
- # `len(set(asserted_by_candidates))`, so both the container and
230
- # its members are declared (batch 149).
231
- entry_types={"id": str, "text": str, "entities": list,
232
- "asserted_by_candidates": list},
233
- # str and int from base.py's own annotations -- `entities:
234
- # list[str]` and `asserted_by_candidates: list[int]`. The
235
- # annotations are not enforced (the comment at the top of base.py
236
- # says so, and #75 is what that cost); this is where they become a
237
- # check.
238
- entry_member_types={"entities": str, "asserted_by_candidates": int},
239
- enums={"type": CLAIM_TYPES},
240
- # B6. Two claims sharing an id split in half: _apply_grounding
241
- # builds {c.id: c} (last wins) while _apply_assumption_flags uses
242
- # next(...) (first wins), so the grounding landed on one object
243
- # and the assumption flags on the other. claim_extraction.md asks
244
- # for "sequential zero-padded ids" and never says they must be
245
- # unique.
246
- unique="id",
247
- # B7. Only in ensemble mode, and only because claim_extraction.md
248
- # is now asked to supply it on every claim there. An ABSENT field
249
- # is indistinguishable from an empty one at the scorer, and
250
- # `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
251
- # a claim that exists because a candidate asserted it.
252
- ensemble_required=frozenset({"asserted_by_candidates"}),
253
- ),
254
-
255
- "Pass 3a": _GROUNDING_ENTRY,
256
- "Pass 3b": _CRITIC_ENTRY,
257
-
258
- "Pass 3c": Spec(
259
- top="object",
260
- required=frozenset({"gaps", "coverage_score"}),
261
- # `gaps` is iterated by build_coverage_gap_entries; a string there
262
- # would produce one coverage-gap entry per CHARACTER.
263
- key_types={"gaps": list},
264
- ),
265
-
266
- "Pass 4": Spec(
267
- top="object",
268
- required=frozenset({"per_claim_flags"}),
269
- key_types={"per_claim_flags": dict},
270
- # Every VALUE under those claim ids is a list, because Pass 5
271
- # measures it: `w["assumption_flag_penalty"] * len(flags)` raised
272
- # on `{"c001": 123}` and silently charged a THREE-flag penalty for
273
- # `{"c001": "abc"}` (batch 149). The strings inside are not typed
274
- # -- `len()` is all that reads them, so a type on the elements
275
- # would be a check with no demonstrated consequence, which is the
276
- # rule B4 states.
277
- key_member_types={"per_claim_flags": list},
278
- ids_in_keys="per_claim_flags",
279
- ),
280
-
281
- "Pass 6a": Spec(
282
- top="array",
283
- entry_required=frozenset({"claim_id", "revised_text"}),
284
- entry_types={"claim_id": str, "revised_text": str},
285
- ids="claim_id",
286
- ),
287
-
288
- "Pass 6b": Spec(
289
- top="object",
290
- required=frozenset({"grounding", "critic"}),
291
- key_types={"grounding": list, "critic": list},
292
- nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
293
- ),
294
- }
295
-
296
-
297
- def _type_name(value: Any) -> str:
298
- """JSON's vocabulary, not Python's -- the message goes back to a model
299
- that was asked for JSON, so `object`/`array` is what it can act on."""
300
- return {dict: "object", list: "array", str: "string",
301
- bool: "boolean", int: "number", float: "number",
302
- type(None): "null"}.get(type(value), type(value).__name__)
303
-
304
-
305
- def _permitted(types) -> str:
306
- return " or ".join(sorted({_type_name(t()) for t in _as_tuple(types)}))
307
-
308
-
309
- def _as_tuple(types) -> tuple:
310
- return types if isinstance(types, tuple) else (types,)
311
-
312
-
313
- def _of_type(value, types) -> bool:
314
- """isinstance, except that a boolean is only ever a boolean.
315
-
316
- See the comment on NUMBER. Applied to every declared type rather than
317
- to NUMBER alone, because "a boolean satisfies a type it was not
318
- declared as" is the defect, and `severity` is simply the only place
319
- the tree has reached it so far.
320
- """
321
- if isinstance(value, bool) and bool not in _as_tuple(types):
322
- return False
323
- return isinstance(value, types)
324
-
325
-
326
- def _members(container):
327
- """(label, value) for everything a container holds.
328
-
329
- A dict's VALUES, keyed by the key that named them, and a list's
330
- elements by position -- so the message can say `'per_claim_flags'
331
- ['c001']` or `entities[1]` and the model can find the one it got
332
- wrong.
333
- """
334
- if isinstance(container, dict):
335
- return [(f"[{key!r}]", value) for key, value in container.items()]
336
- return [(f"[{index}]", value) for index, value in enumerate(container)]
337
-
338
-
339
- def _check_members(pass_id, container, member_types, at, prefix="") -> None:
340
- for key, types in member_types.items():
341
- if key not in container:
342
- continue
343
- value = container[key]
344
- if not isinstance(value, (dict, list)):
345
- # The declared CONTAINER type has already refused this, or
346
- # there isn't one and the member rule has nothing to stand on.
347
- # Either way, iterating here would report the members of a
348
- # string.
349
- continue
350
- for label, member in _members(value):
351
- if _of_type(member, types):
352
- continue
353
- raise PayloadShapeError(
354
- f"{pass_id}: {prefix}{key}{label}{at} is {member!r}, which "
355
- f"is {_type_name(member)}; every value in {key!r} must be "
356
- f"{_permitted(types)}."
357
- )
358
-
359
-
360
- def validate(pass_id: str, payload: Any, claim_ids=None,
361
- ensemble: bool = False) -> None:
362
- """Raise PayloadShapeError unless `payload` is what `pass_id` promised.
363
-
364
- claim_ids the ids the APPLY step will resolve against -- all claims
365
- for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
366
- "every claim in the run": Pass 6a is only asked about
367
- `flagged`, and its own lookup already resolves within that
368
- batch, so validating against a wider set would accept a
369
- revision that then lands on nothing. None skips the check.
370
- ensemble True on an ensemble run, which is the only place Pass 2 is
371
- asked for `asserted_by_candidates` (B7).
372
-
373
- An unknown pass_id RAISES rather than passing through unvalidated. A
374
- pass with no spec is the silent hole this module exists to close, and
375
- the eleventh pass must not be able to arrive by forgetting a line.
376
- """
377
- spec = SPECS.get(pass_id)
378
- if spec is None:
379
- raise PayloadShapeError(
380
- f"{pass_id}: no payload spec is declared for this pass. Every "
381
- f"pass routed through _run_pass_with_json_retry needs one in "
382
- f"core/reasoning/payload_validation.py (§30, B11)."
383
- )
384
- _validate_against(pass_id, payload, spec, claim_ids, ensemble)
385
-
386
-
387
- def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
388
- where: str = "") -> None:
389
- at = f" under {where!r}" if where else ""
390
-
391
- if spec.top == "array":
392
- if not isinstance(payload, list):
393
- raise PayloadShapeError(
394
- f"{pass_id}: expected a JSON array{at}, got "
395
- f"{_type_name(payload)}. Return the array itself, not an "
396
- f"object wrapping it."
397
- )
398
- _validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
399
- return
400
-
401
- if not isinstance(payload, dict):
402
- raise PayloadShapeError(
403
- f"{pass_id}: expected a JSON object{at}, got "
404
- f"{_type_name(payload)}."
405
- )
406
-
407
- missing = sorted(spec.required - set(payload))
408
- if missing:
409
- raise PayloadShapeError(
410
- f"{pass_id}: the response{at} is missing required key(s) "
411
- f"{', '.join(repr(m) for m in missing)}. Expected an object "
412
- f"with {', '.join(sorted(spec.required))}."
413
- )
414
-
415
- for key, types in spec.key_types.items():
416
- if key in payload and not _of_type(payload[key], types):
417
- raise PayloadShapeError(
418
- f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
419
- f"expected {_permitted(types)}."
420
- )
421
-
422
- _check_members(pass_id, payload, spec.key_member_types, at)
423
-
424
- for key, nested in spec.nested.items():
425
- _validate_against(pass_id, payload[key], nested, claim_ids,
426
- ensemble, where=key)
427
-
428
- if spec.ids_in_keys:
429
- _check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
430
- f"key of {spec.ids_in_keys!r}")
431
-
432
-
433
- def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
434
- required = spec.entry_required | (spec.ensemble_required if ensemble
435
- else frozenset())
436
- seen = {}
437
- for index, entry in enumerate(entries):
438
- if not isinstance(entry, dict):
439
- raise PayloadShapeError(
440
- f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
441
- f"every entry must be an object with "
442
- f"{', '.join(sorted(required))}."
443
- )
444
- missing = sorted(required - set(entry))
445
- if missing:
446
- raise PayloadShapeError(
447
- f"{pass_id}: entry {index}{at} is missing required key(s) "
448
- f"{', '.join(repr(m) for m in missing)}. Every entry needs "
449
- f"{', '.join(sorted(required))}."
450
- )
451
- # TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
452
- # rule and the reason it is worth copying: a JSON array or object
453
- # in an enum-checked field is unhashable, so `value in ALLOWED`
454
- # raises TypeError from inside the validator rather than
455
- # producing a message the model can act on.
456
- for key, types in spec.entry_types.items():
457
- if key in entry and not _of_type(entry[key], types):
458
- raise PayloadShapeError(
459
- f"{pass_id}: entry {index}{at} has {key}="
460
- f"{entry[key]!r}, which is {_type_name(entry[key])}; "
461
- f"expected {_permitted(types)}."
462
- )
463
- _check_members(pass_id, entry, spec.entry_member_types, at,
464
- prefix=f"entry {index} ")
465
- for key, allowed in spec.enums.items():
466
- if key not in entry:
467
- continue
468
- value = entry[key]
469
- if not isinstance(value, str) or value not in allowed:
470
- raise PayloadShapeError(
471
- f"{pass_id}: entry {index}{at} has {key}={value!r}; "
472
- f"expected exactly one of "
473
- f"{', '.join(sorted(allowed))} (lower case)."
474
- )
475
- if spec.unique:
476
- value = entry[spec.unique]
477
- if value in seen:
478
- raise PayloadShapeError(
479
- f"{pass_id}: entries {seen[value]} and {index}{at} "
480
- f"share {spec.unique}={value!r}. Ids must be unique -- "
481
- f"two claims with one id are applied to inconsistently "
482
- f"and end up carrying half of each other's results."
483
- )
484
- seen[value] = index
485
-
486
- if spec.ids:
487
- _check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
488
- spec.ids)
489
-
490
-
491
- def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
492
- """B5: a TOTAL mismatch is the error; a partial one is not.
493
-
494
- The demonstrated silent failure is every id being wrong -- 3a/3b
495
- answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
496
- which grounded nothing while the checkpoint line reported that it had
497
- grounded everything. That is a pass with no effect at all, and it is
498
- worth a corrective retry.
499
-
500
- A model that gets nine of ten right is NOT worth failing a run that
501
- has already paid for nine passes: the nine apply, and the checkpoint
502
- line reports the count APPLIED rather than the count asked about, so
503
- the tenth is visible instead of silent (see the orchestrator's
504
- _apply_* return values).
505
-
506
- An EMPTY collection is not a mismatch. assumption_audit.md asks the
507
- model to "omit claims with no issues rather than including them with
508
- an empty list", so no flags at all is the ordinary answer for a clean
509
- run -- and Pass 4's demonstrated defect was the top-level key being
510
- absent, which the required-key check above catches instead.
511
- """
512
- if claim_ids is None or not supplied:
513
- return
514
- known = set(claim_ids)
515
- if any(value in known for value in supplied):
516
- return
517
- sample = ", ".join(repr(v) for v in supplied[:3])
518
- expected = ", ".join(repr(v) for v in sorted(known)[:3])
519
- raise PayloadShapeError(
520
- f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
521
- f"matches a claim you were given (got {sample}; expected ids like "
522
- f"{expected}). Use the claim ids exactly as they appear in the "
523
- f"input."
524
- )
1
+ """
2
+ core/reasoning/payload_validation.py
3
+
4
+ ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
5
+
6
+ §3's retry corrects a response that does not PARSE. Nothing checked that
7
+ what parsed was the SHAPE the pass promised, so the same defect appeared
8
+ in six places at once (audit #76): a wrapped list crashed with
9
+ `'str' object has no attribute 'items'`, a bad `severity` survived two
10
+ passes and surfaced in Pass 5's pure-code tiering, and two shapes
11
+ completed a run and reported a false outcome.
12
+
13
+ WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
14
+ this validator -- `review.py:_validated` checks a top-level type, the
15
+ required keys per entry with type guards BEFORE the membership tests, an
16
+ id cross-check against `{c.id for c in run.claims}`, an enum, and a
17
+ duplicate -- and built it once, for the one payload that is NOT a pass.
18
+ The passes never got it. Putting the general version in orchestrator.py
19
+ would repeat json_retry.py's own mistake in reverse: that module was
20
+ extracted precisely so a mechanism a second consumer might want is not
21
+ trapped in the orchestrator.
22
+
23
+ `review.py` is deliberately NOT migrated onto this module (§30, B1). Its
24
+ POLICY is different and correctly so: a finding it cannot apply is
25
+ dropped and traced, because the review stage is optional and contained,
26
+ and a reviewer that misfires must not cost a completed ten-pass run. A
27
+ pass is not optional -- a payload it cannot apply is a run that will
28
+ report something untrue -- so a failure here re-enters §3's corrective
29
+ retry (B2) and ultimately raises. Same shape, two policies. Merging them
30
+ means choosing one, and neither is wrong for its own caller.
31
+
32
+ WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
33
+ declared type where a value is used arithmetically -- every one has a
34
+ demonstrated consequence in #76's table, and nothing else does:
35
+
36
+ the top-level type {"claims": [...]} instead of [...]
37
+ the required keys per entry a claim with no `id`, a revision with no
38
+ `revised_text`, Pass 4 with no
39
+ `per_claim_flags`
40
+ a declared value type `"severity": "0.0"` -- a string that
41
+ fails two passes later, in Pass 5
42
+ the type of its MEMBERS `"entities": ["madrid", 1947]` -- a list,
43
+ which Pass 3a then sorts
44
+ that entry ids resolve 3a/3b answering about ids Pass 2 never
45
+ issued, which grounded nothing while the
46
+ trace said it had grounded everything
47
+
48
+ THE FOURTH ROW IS BATCH 149, AND IT IS NOT A RICHER VOCABULARY FOR ITS OWN
49
+ SAKE -- it is the same property as the third, asked of the values a
50
+ container holds, because that is where the arithmetic actually happens.
51
+ `entities` is not measured; every element of it is hashed and then
52
+ compared against every other (`sorted({e for c in claims for e in
53
+ c.entities})`), and `asserted_by_candidates` is hashed the same way by
54
+ `len(set(...))`. A declared CONTAINER type closes the three container
55
+ cases and leaves the three member cases open, each driven against the real
56
+ consumer:
57
+
58
+ ["madrid", 1947] TypeError: '<' not supported between
59
+ instances of 'int' and 'str'
60
+ [["madrid"]] TypeError: unhashable type: 'list'
61
+ [{"name": "madrid"}] TypeError: unhashable type: 'dict'
62
+
63
+ A MEMBER is a dict's VALUES or a list's ELEMENTS, deliberately one concept
64
+ and one spec field per level: Pass 4's flag lists live under claim-id keys
65
+ and Pass 2's entities live in an array, and the thing being declared is
66
+ the same in both -- what the pure code downstream will iterate, hash, sort
67
+ or measure. Pass 4's flag lists are typed and their STRINGS are not, which
68
+ is the same rule: `len()` is the only thing that reads them.
69
+
70
+ Not a schema language, and no new dependency (#145 tracks those). A
71
+ richer vocabulary would let a spec describe things no pass has ever got
72
+ wrong, at the price of a second place where a payload's meaning lives.
73
+
74
+ STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
75
+ `Respond with ONLY this JSON structure` block in that pass's own .md, and
76
+ NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
77
+ field will fail the run rather than silently mis-apply it -- which is the
78
+ better failure -- but it will fail it at the model's expense. The parity
79
+ that IS mechanised is narrower and is the one that goes wrong silently:
80
+ every pass the orchestrator routes through `_run_pass_with_json_retry`
81
+ has a spec here (see tests/test_payload_validation.py).
82
+ """
83
+
84
+ from __future__ import annotations
85
+
86
+ from dataclasses import dataclass, field
87
+ from typing import Any, Optional
88
+
89
+ # Re-exported, not re-derived: base.py defines these beside the Literals
90
+ # they come from, and the scorer reads the same two names (§30, B9).
91
+ from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
92
+
93
+ # A BOOLEAN IS NOT A NUMBER, which Python disagrees with: `isinstance(
94
+ # True, int)` is True, so `"severity": true` satisfied this tuple and
95
+ # Pass 5 computed `1.0 - True` as 0.0 -- the MAXIMUM critic penalty, the
96
+ # single harshest value the field can take -- with nothing in the trace
97
+ # and no corrective retry (batch 149). `"severity": false` took the other
98
+ # extreme for the same reason.
99
+ #
100
+ # The exclusion is not new to the codebase, only to the boundary:
101
+ # `source_scoring._as_number` and `confidence_scoring.source_quality` both
102
+ # spell it out already, each guarding a value that is supposed to have
103
+ # been rejected HERE. See `_of_type` below, which is what applies it.
104
+ NUMBER = (int, float)
105
+
106
+
107
+ class PayloadShapeError(ValueError):
108
+ """A pass's response parsed as JSON but is not the shape it promised.
109
+
110
+ SUBCLASSES ValueError deliberately, and that is load-bearing twice:
111
+ json_retry.py's corrective loop already catches ValueError (so a shape
112
+ failure re-enters the retry with no new except clause), and the
113
+ orchestrator's outer try/except -- which turns a failure into a
114
+ durable status='failed' record -- is byte-for-byte unchanged.
115
+
116
+ The message names the pass, the field and the expected shape, because
117
+ the whole complaint in #76's loud half is that
118
+ `'str' object has no attribute 'items'` names none of the three.
119
+ """
120
+
121
+
122
+ @dataclass(frozen=True)
123
+ class Spec:
124
+ """One pass's promised payload shape.
125
+
126
+ top "object" or "array"
127
+ required top-level keys that must be present (top="object")
128
+ key_types top-level key -> permitted Python type(s)
129
+ entry_required keys every entry must carry (top="array")
130
+ entry_types entry key -> permitted Python type(s)
131
+ key_member_types top-level key -> permitted type for every MEMBER of
132
+ the container under it (a dict's values, a list's
133
+ elements)
134
+ entry_member_types the same, per entry key
135
+ enums entry key -> the set of permitted values
136
+ ids entry key holding a claim id, cross-checked against
137
+ the claims the apply step will resolve against
138
+ unique entry key that must not repeat (Pass 2's `id`)
139
+ ensemble_required entry keys required only on an ensemble run (B7)
140
+ ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
141
+ nested top-level key -> Spec for the array beneath it
142
+ """
143
+
144
+ top: str
145
+ required: frozenset = frozenset()
146
+ key_types: dict = field(default_factory=dict)
147
+ entry_required: frozenset = frozenset()
148
+ entry_types: dict = field(default_factory=dict)
149
+ key_member_types: dict = field(default_factory=dict)
150
+ entry_member_types: dict = field(default_factory=dict)
151
+ enums: dict = field(default_factory=dict)
152
+ ids: Optional[str] = None
153
+ unique: Optional[str] = None
154
+ ensemble_required: frozenset = frozenset()
155
+ ids_in_keys: Optional[str] = None
156
+ nested: dict = field(default_factory=dict)
157
+
158
+
159
+ # The 3a and 3b entry shapes are named, because Pass 6b is documented in
160
+ # revalidate.md as returning exactly those two lists under two keys --
161
+ # and _apply_grounding/_apply_critic are literally the same two functions.
162
+ # Writing them twice would let 6c drift from the passes it re-runs.
163
+ _GROUNDING_ENTRY = Spec(
164
+ top="array",
165
+ entry_required=frozenset({"claim_id", "status"}),
166
+ # §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
167
+ # asymmetry is the point. B4's rule is that a check earns its place by
168
+ # having a demonstrated consequence:
169
+ #
170
+ # the type §45's scoring stage ITERATES this value. A string
171
+ # there scores one source per character, and a null
172
+ # raises inside a stage that runs after the model call
173
+ # it would need to retry.
174
+ # not required source_grounding.md says an ungrounded claim's
175
+ # `sources` "may be an empty list", and a model that
176
+ # omits the key entirely on a claim it could not
177
+ # source has said the same thing. Failing that run
178
+ # costs a corrective retry to add `[]` to an entry
179
+ # nothing reads.
180
+ #
181
+ # The case this leaves open -- `grounded` with no sources at all -- is
182
+ # NOT unhandled; it is handled where it means something. The scoring
183
+ # stage scores it as zero source quality and says so in the trace,
184
+ # which is the honest answer to a claim asserted as grounded with
185
+ # nothing behind it. Rejecting it here would instead ask the model to
186
+ # try again, and a model that had no sources still has none.
187
+ entry_types={"claim_id": str, "sources": list},
188
+ enums={"status": GROUNDING_STATUSES},
189
+ ids="claim_id",
190
+ )
191
+
192
+ _CRITIC_ENTRY = Spec(
193
+ top="array",
194
+ entry_required=frozenset({"claim_id", "severity"}),
195
+ # `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
196
+ # so a string here raises a TypeError two passes downstream, in the
197
+ # one stage that makes no model call and has nothing to retry.
198
+ entry_types={"claim_id": str, "severity": NUMBER},
199
+ ids="claim_id",
200
+ )
201
+
202
+
203
+ SPECS = {
204
+ # preliminary_plan.md names five keys, and the orchestrator reads ONE
205
+ # of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
206
+ # before handing the plan whole to Pass 3c. So requiring all five
207
+ # would fail runs over an omission nothing depends on. The
208
+ # demonstrated defect is a bare list, which is what `top` catches.
209
+ #
210
+ # THE ONE KEY THE ORCHESTRATOR READS IS TYPED, AND STILL NOT REQUIRED
211
+ # (batch 149) -- `key_types` is checked only `if key in payload`, so
212
+ # the defensive `.get` above keeps its meaning and an omission still
213
+ # costs nothing. What it stops is the read itself: `len(123)` raised
214
+ # inside the Pass 0 checkpoint, and `len("madrid")` did not raise at
215
+ # all -- it reported SIX key entities anticipated, from one word.
216
+ "Pass 0": Spec(top="object",
217
+ key_types={"key_entities_or_subjects": list}),
218
+
219
+ "Pass 2": Spec(
220
+ top="array",
221
+ entry_required=frozenset({"id", "text", "type"}),
222
+ # `entities` and `asserted_by_candidates` are TYPED BUT NOT
223
+ # REQUIRED, which is the same asymmetry `sources` carries below
224
+ # and for the same reason: base.py defaults both to an empty list,
225
+ # a claim with no entities is an ordinary answer, and
226
+ # `asserted_by_candidates` is required only in ensemble mode (B7).
227
+ # What they are not is optional to get RIGHT -- Pass 3a sorts a
228
+ # set built from every claim's entities, and Pass 5 measures
229
+ # `len(set(asserted_by_candidates))`, so both the container and
230
+ # its members are declared (batch 149).
231
+ entry_types={"id": str, "text": str, "entities": list,
232
+ "asserted_by_candidates": list},
233
+ # str and int from base.py's own annotations -- `entities:
234
+ # list[str]` and `asserted_by_candidates: list[int]`. The
235
+ # annotations are not enforced (the comment at the top of base.py
236
+ # says so, and #75 is what that cost); this is where they become a
237
+ # check.
238
+ entry_member_types={"entities": str, "asserted_by_candidates": int},
239
+ enums={"type": CLAIM_TYPES},
240
+ # B6. Two claims sharing an id split in half: _apply_grounding
241
+ # builds {c.id: c} (last wins) while _apply_assumption_flags uses
242
+ # next(...) (first wins), so the grounding landed on one object
243
+ # and the assumption flags on the other. claim_extraction.md asks
244
+ # for "sequential zero-padded ids" and never says they must be
245
+ # unique.
246
+ unique="id",
247
+ # B7. Only in ensemble mode, and only because claim_extraction.md
248
+ # is now asked to supply it on every claim there. An ABSENT field
249
+ # is indistinguishable from an empty one at the scorer, and
250
+ # `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
251
+ # a claim that exists because a candidate asserted it.
252
+ ensemble_required=frozenset({"asserted_by_candidates"}),
253
+ ),
254
+
255
+ "Pass 3a": _GROUNDING_ENTRY,
256
+ "Pass 3b": _CRITIC_ENTRY,
257
+
258
+ "Pass 3c": Spec(
259
+ top="object",
260
+ required=frozenset({"gaps", "coverage_score"}),
261
+ # `gaps` is iterated by build_coverage_gap_entries; a string there
262
+ # would produce one coverage-gap entry per CHARACTER.
263
+ key_types={"gaps": list},
264
+ ),
265
+
266
+ "Pass 4": Spec(
267
+ top="object",
268
+ required=frozenset({"per_claim_flags"}),
269
+ key_types={"per_claim_flags": dict},
270
+ # Every VALUE under those claim ids is a list, because Pass 5
271
+ # measures it: `w["assumption_flag_penalty"] * len(flags)` raised
272
+ # on `{"c001": 123}` and silently charged a THREE-flag penalty for
273
+ # `{"c001": "abc"}` (batch 149). The strings inside are not typed
274
+ # -- `len()` is all that reads them, so a type on the elements
275
+ # would be a check with no demonstrated consequence, which is the
276
+ # rule B4 states.
277
+ key_member_types={"per_claim_flags": list},
278
+ ids_in_keys="per_claim_flags",
279
+ ),
280
+
281
+ "Pass 6a": Spec(
282
+ top="array",
283
+ entry_required=frozenset({"claim_id", "revised_text"}),
284
+ entry_types={"claim_id": str, "revised_text": str},
285
+ ids="claim_id",
286
+ ),
287
+
288
+ "Pass 6b": Spec(
289
+ top="object",
290
+ required=frozenset({"grounding", "critic"}),
291
+ key_types={"grounding": list, "critic": list},
292
+ nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
293
+ ),
294
+ }
295
+
296
+
297
+ def _type_name(value: Any) -> str:
298
+ """JSON's vocabulary, not Python's -- the message goes back to a model
299
+ that was asked for JSON, so `object`/`array` is what it can act on."""
300
+ return {dict: "object", list: "array", str: "string",
301
+ bool: "boolean", int: "number", float: "number",
302
+ type(None): "null"}.get(type(value), type(value).__name__)
303
+
304
+
305
+ def _permitted(types) -> str:
306
+ return " or ".join(sorted({_type_name(t()) for t in _as_tuple(types)}))
307
+
308
+
309
+ def _as_tuple(types) -> tuple:
310
+ return types if isinstance(types, tuple) else (types,)
311
+
312
+
313
+ def _of_type(value, types) -> bool:
314
+ """isinstance, except that a boolean is only ever a boolean.
315
+
316
+ See the comment on NUMBER. Applied to every declared type rather than
317
+ to NUMBER alone, because "a boolean satisfies a type it was not
318
+ declared as" is the defect, and `severity` is simply the only place
319
+ the tree has reached it so far.
320
+ """
321
+ if isinstance(value, bool) and bool not in _as_tuple(types):
322
+ return False
323
+ return isinstance(value, types)
324
+
325
+
326
+ def _members(container):
327
+ """(label, value) for everything a container holds.
328
+
329
+ A dict's VALUES, keyed by the key that named them, and a list's
330
+ elements by position -- so the message can say `'per_claim_flags'
331
+ ['c001']` or `entities[1]` and the model can find the one it got
332
+ wrong.
333
+ """
334
+ if isinstance(container, dict):
335
+ return [(f"[{key!r}]", value) for key, value in container.items()]
336
+ return [(f"[{index}]", value) for index, value in enumerate(container)]
337
+
338
+
339
+ def _check_members(pass_id, container, member_types, at, prefix="") -> None:
340
+ for key, types in member_types.items():
341
+ if key not in container:
342
+ continue
343
+ value = container[key]
344
+ if not isinstance(value, (dict, list)):
345
+ # The declared CONTAINER type has already refused this, or
346
+ # there isn't one and the member rule has nothing to stand on.
347
+ # Either way, iterating here would report the members of a
348
+ # string.
349
+ continue
350
+ for label, member in _members(value):
351
+ if _of_type(member, types):
352
+ continue
353
+ raise PayloadShapeError(
354
+ f"{pass_id}: {prefix}{key}{label}{at} is {member!r}, which "
355
+ f"is {_type_name(member)}; every value in {key!r} must be "
356
+ f"{_permitted(types)}."
357
+ )
358
+
359
+
360
+ def validate(pass_id: str, payload: Any, claim_ids=None,
361
+ ensemble: bool = False) -> None:
362
+ """Raise PayloadShapeError unless `payload` is what `pass_id` promised.
363
+
364
+ claim_ids the ids the APPLY step will resolve against -- all claims
365
+ for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
366
+ "every claim in the run": Pass 6a is only asked about
367
+ `flagged`, and its own lookup already resolves within that
368
+ batch, so validating against a wider set would accept a
369
+ revision that then lands on nothing. None skips the check.
370
+ ensemble True on an ensemble run, which is the only place Pass 2 is
371
+ asked for `asserted_by_candidates` (B7).
372
+
373
+ An unknown pass_id RAISES rather than passing through unvalidated. A
374
+ pass with no spec is the silent hole this module exists to close, and
375
+ the eleventh pass must not be able to arrive by forgetting a line.
376
+ """
377
+ spec = SPECS.get(pass_id)
378
+ if spec is None:
379
+ raise PayloadShapeError(
380
+ f"{pass_id}: no payload spec is declared for this pass. Every "
381
+ f"pass routed through _run_pass_with_json_retry needs one in "
382
+ f"core/reasoning/payload_validation.py (§30, B11)."
383
+ )
384
+ _validate_against(pass_id, payload, spec, claim_ids, ensemble)
385
+
386
+
387
+ def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
388
+ where: str = "") -> None:
389
+ at = f" under {where!r}" if where else ""
390
+
391
+ if spec.top == "array":
392
+ if not isinstance(payload, list):
393
+ raise PayloadShapeError(
394
+ f"{pass_id}: expected a JSON array{at}, got "
395
+ f"{_type_name(payload)}. Return the array itself, not an "
396
+ f"object wrapping it."
397
+ )
398
+ _validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
399
+ return
400
+
401
+ if not isinstance(payload, dict):
402
+ raise PayloadShapeError(
403
+ f"{pass_id}: expected a JSON object{at}, got "
404
+ f"{_type_name(payload)}."
405
+ )
406
+
407
+ missing = sorted(spec.required - set(payload))
408
+ if missing:
409
+ raise PayloadShapeError(
410
+ f"{pass_id}: the response{at} is missing required key(s) "
411
+ f"{', '.join(repr(m) for m in missing)}. Expected an object "
412
+ f"with {', '.join(sorted(spec.required))}."
413
+ )
414
+
415
+ for key, types in spec.key_types.items():
416
+ if key in payload and not _of_type(payload[key], types):
417
+ raise PayloadShapeError(
418
+ f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
419
+ f"expected {_permitted(types)}."
420
+ )
421
+
422
+ _check_members(pass_id, payload, spec.key_member_types, at)
423
+
424
+ for key, nested in spec.nested.items():
425
+ _validate_against(pass_id, payload[key], nested, claim_ids,
426
+ ensemble, where=key)
427
+
428
+ if spec.ids_in_keys:
429
+ _check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
430
+ f"key of {spec.ids_in_keys!r}")
431
+
432
+
433
+ def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
434
+ required = spec.entry_required | (spec.ensemble_required if ensemble
435
+ else frozenset())
436
+ seen = {}
437
+ for index, entry in enumerate(entries):
438
+ if not isinstance(entry, dict):
439
+ raise PayloadShapeError(
440
+ f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
441
+ f"every entry must be an object with "
442
+ f"{', '.join(sorted(required))}."
443
+ )
444
+ missing = sorted(required - set(entry))
445
+ if missing:
446
+ raise PayloadShapeError(
447
+ f"{pass_id}: entry {index}{at} is missing required key(s) "
448
+ f"{', '.join(repr(m) for m in missing)}. Every entry needs "
449
+ f"{', '.join(sorted(required))}."
450
+ )
451
+ # TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
452
+ # rule and the reason it is worth copying: a JSON array or object
453
+ # in an enum-checked field is unhashable, so `value in ALLOWED`
454
+ # raises TypeError from inside the validator rather than
455
+ # producing a message the model can act on.
456
+ for key, types in spec.entry_types.items():
457
+ if key in entry and not _of_type(entry[key], types):
458
+ raise PayloadShapeError(
459
+ f"{pass_id}: entry {index}{at} has {key}="
460
+ f"{entry[key]!r}, which is {_type_name(entry[key])}; "
461
+ f"expected {_permitted(types)}."
462
+ )
463
+ _check_members(pass_id, entry, spec.entry_member_types, at,
464
+ prefix=f"entry {index} ")
465
+ for key, allowed in spec.enums.items():
466
+ if key not in entry:
467
+ continue
468
+ value = entry[key]
469
+ if not isinstance(value, str) or value not in allowed:
470
+ raise PayloadShapeError(
471
+ f"{pass_id}: entry {index}{at} has {key}={value!r}; "
472
+ f"expected exactly one of "
473
+ f"{', '.join(sorted(allowed))} (lower case)."
474
+ )
475
+ if spec.unique:
476
+ value = entry[spec.unique]
477
+ if value in seen:
478
+ raise PayloadShapeError(
479
+ f"{pass_id}: entries {seen[value]} and {index}{at} "
480
+ f"share {spec.unique}={value!r}. Ids must be unique -- "
481
+ f"two claims with one id are applied to inconsistently "
482
+ f"and end up carrying half of each other's results."
483
+ )
484
+ seen[value] = index
485
+
486
+ if spec.ids:
487
+ _check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
488
+ spec.ids)
489
+
490
+
491
+ def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
492
+ """B5: a TOTAL mismatch is the error; a partial one is not.
493
+
494
+ The demonstrated silent failure is every id being wrong -- 3a/3b
495
+ answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
496
+ which grounded nothing while the checkpoint line reported that it had
497
+ grounded everything. That is a pass with no effect at all, and it is
498
+ worth a corrective retry.
499
+
500
+ A model that gets nine of ten right is NOT worth failing a run that
501
+ has already paid for nine passes: the nine apply, and the checkpoint
502
+ line reports the count APPLIED rather than the count asked about, so
503
+ the tenth is visible instead of silent (see the orchestrator's
504
+ _apply_* return values).
505
+
506
+ An EMPTY collection is not a mismatch. assumption_audit.md asks the
507
+ model to "omit claims with no issues rather than including them with
508
+ an empty list", so no flags at all is the ordinary answer for a clean
509
+ run -- and Pass 4's demonstrated defect was the top-level key being
510
+ absent, which the required-key check above catches instead.
511
+ """
512
+ if claim_ids is None or not supplied:
513
+ return
514
+ known = set(claim_ids)
515
+ if any(value in known for value in supplied):
516
+ return
517
+ sample = ", ".join(repr(v) for v in supplied[:3])
518
+ expected = ", ".join(repr(v) for v in sorted(known)[:3])
519
+ raise PayloadShapeError(
520
+ f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
521
+ f"matches a claim you were given (got {sample}; expected ids like "
522
+ f"{expected}). Use the claim ids exactly as they appear in the "
523
+ f"input."
524
+ )