@yahyahammad/venastine 0.3.8 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2030 -2002
- package/LICENSE +201 -201
- package/README.md +1034 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +528 -528
- package/bin/venastine.py +71 -71
- package/cli_commands.py +838 -835
- package/config.yaml +13 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +971 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2715 -2715
- package/core/commands.py +460 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +234 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +458 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +108 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +733 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +295 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +267 -267
- package/tools/builtin/arxiv.py +681 -681
- package/tools/builtin/fetch_url.py +371 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +328 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1242 -1242
- package/tui/app.py +8318 -8307
- package/tui/app.tcss +747 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3813 -3796
|
@@ -1,524 +1,524 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/payload_validation.py
|
|
3
|
-
|
|
4
|
-
ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
|
|
5
|
-
|
|
6
|
-
§3's retry corrects a response that does not PARSE. Nothing checked that
|
|
7
|
-
what parsed was the SHAPE the pass promised, so the same defect appeared
|
|
8
|
-
in six places at once (audit #76): a wrapped list crashed with
|
|
9
|
-
`'str' object has no attribute 'items'`, a bad `severity` survived two
|
|
10
|
-
passes and surfaced in Pass 5's pure-code tiering, and two shapes
|
|
11
|
-
completed a run and reported a false outcome.
|
|
12
|
-
|
|
13
|
-
WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
|
|
14
|
-
this validator -- `review.py:_validated` checks a top-level type, the
|
|
15
|
-
required keys per entry with type guards BEFORE the membership tests, an
|
|
16
|
-
id cross-check against `{c.id for c in run.claims}`, an enum, and a
|
|
17
|
-
duplicate -- and built it once, for the one payload that is NOT a pass.
|
|
18
|
-
The passes never got it. Putting the general version in orchestrator.py
|
|
19
|
-
would repeat json_retry.py's own mistake in reverse: that module was
|
|
20
|
-
extracted precisely so a mechanism a second consumer might want is not
|
|
21
|
-
trapped in the orchestrator.
|
|
22
|
-
|
|
23
|
-
`review.py` is deliberately NOT migrated onto this module (§30, B1). Its
|
|
24
|
-
POLICY is different and correctly so: a finding it cannot apply is
|
|
25
|
-
dropped and traced, because the review stage is optional and contained,
|
|
26
|
-
and a reviewer that misfires must not cost a completed ten-pass run. A
|
|
27
|
-
pass is not optional -- a payload it cannot apply is a run that will
|
|
28
|
-
report something untrue -- so a failure here re-enters §3's corrective
|
|
29
|
-
retry (B2) and ultimately raises. Same shape, two policies. Merging them
|
|
30
|
-
means choosing one, and neither is wrong for its own caller.
|
|
31
|
-
|
|
32
|
-
WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
|
|
33
|
-
declared type where a value is used arithmetically -- every one has a
|
|
34
|
-
demonstrated consequence in #76's table, and nothing else does:
|
|
35
|
-
|
|
36
|
-
the top-level type {"claims": [...]} instead of [...]
|
|
37
|
-
the required keys per entry a claim with no `id`, a revision with no
|
|
38
|
-
`revised_text`, Pass 4 with no
|
|
39
|
-
`per_claim_flags`
|
|
40
|
-
a declared value type `"severity": "0.0"` -- a string that
|
|
41
|
-
fails two passes later, in Pass 5
|
|
42
|
-
the type of its MEMBERS `"entities": ["madrid", 1947]` -- a list,
|
|
43
|
-
which Pass 3a then sorts
|
|
44
|
-
that entry ids resolve 3a/3b answering about ids Pass 2 never
|
|
45
|
-
issued, which grounded nothing while the
|
|
46
|
-
trace said it had grounded everything
|
|
47
|
-
|
|
48
|
-
THE FOURTH ROW IS BATCH 149, AND IT IS NOT A RICHER VOCABULARY FOR ITS OWN
|
|
49
|
-
SAKE -- it is the same property as the third, asked of the values a
|
|
50
|
-
container holds, because that is where the arithmetic actually happens.
|
|
51
|
-
`entities` is not measured; every element of it is hashed and then
|
|
52
|
-
compared against every other (`sorted({e for c in claims for e in
|
|
53
|
-
c.entities})`), and `asserted_by_candidates` is hashed the same way by
|
|
54
|
-
`len(set(...))`. A declared CONTAINER type closes the three container
|
|
55
|
-
cases and leaves the three member cases open, each driven against the real
|
|
56
|
-
consumer:
|
|
57
|
-
|
|
58
|
-
["madrid", 1947] TypeError: '<' not supported between
|
|
59
|
-
instances of 'int' and 'str'
|
|
60
|
-
[["madrid"]] TypeError: unhashable type: 'list'
|
|
61
|
-
[{"name": "madrid"}] TypeError: unhashable type: 'dict'
|
|
62
|
-
|
|
63
|
-
A MEMBER is a dict's VALUES or a list's ELEMENTS, deliberately one concept
|
|
64
|
-
and one spec field per level: Pass 4's flag lists live under claim-id keys
|
|
65
|
-
and Pass 2's entities live in an array, and the thing being declared is
|
|
66
|
-
the same in both -- what the pure code downstream will iterate, hash, sort
|
|
67
|
-
or measure. Pass 4's flag lists are typed and their STRINGS are not, which
|
|
68
|
-
is the same rule: `len()` is the only thing that reads them.
|
|
69
|
-
|
|
70
|
-
Not a schema language, and no new dependency (#145 tracks those). A
|
|
71
|
-
richer vocabulary would let a spec describe things no pass has ever got
|
|
72
|
-
wrong, at the price of a second place where a payload's meaning lives.
|
|
73
|
-
|
|
74
|
-
STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
|
|
75
|
-
`Respond with ONLY this JSON structure` block in that pass's own .md, and
|
|
76
|
-
NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
|
|
77
|
-
field will fail the run rather than silently mis-apply it -- which is the
|
|
78
|
-
better failure -- but it will fail it at the model's expense. The parity
|
|
79
|
-
that IS mechanised is narrower and is the one that goes wrong silently:
|
|
80
|
-
every pass the orchestrator routes through `_run_pass_with_json_retry`
|
|
81
|
-
has a spec here (see tests/test_payload_validation.py).
|
|
82
|
-
"""
|
|
83
|
-
|
|
84
|
-
from __future__ import annotations
|
|
85
|
-
|
|
86
|
-
from dataclasses import dataclass, field
|
|
87
|
-
from typing import Any, Optional
|
|
88
|
-
|
|
89
|
-
# Re-exported, not re-derived: base.py defines these beside the Literals
|
|
90
|
-
# they come from, and the scorer reads the same two names (§30, B9).
|
|
91
|
-
from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
|
|
92
|
-
|
|
93
|
-
# A BOOLEAN IS NOT A NUMBER, which Python disagrees with: `isinstance(
|
|
94
|
-
# True, int)` is True, so `"severity": true` satisfied this tuple and
|
|
95
|
-
# Pass 5 computed `1.0 - True` as 0.0 -- the MAXIMUM critic penalty, the
|
|
96
|
-
# single harshest value the field can take -- with nothing in the trace
|
|
97
|
-
# and no corrective retry (batch 149). `"severity": false` took the other
|
|
98
|
-
# extreme for the same reason.
|
|
99
|
-
#
|
|
100
|
-
# The exclusion is not new to the codebase, only to the boundary:
|
|
101
|
-
# `source_scoring._as_number` and `confidence_scoring.source_quality` both
|
|
102
|
-
# spell it out already, each guarding a value that is supposed to have
|
|
103
|
-
# been rejected HERE. See `_of_type` below, which is what applies it.
|
|
104
|
-
NUMBER = (int, float)
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
class PayloadShapeError(ValueError):
|
|
108
|
-
"""A pass's response parsed as JSON but is not the shape it promised.
|
|
109
|
-
|
|
110
|
-
SUBCLASSES ValueError deliberately, and that is load-bearing twice:
|
|
111
|
-
json_retry.py's corrective loop already catches ValueError (so a shape
|
|
112
|
-
failure re-enters the retry with no new except clause), and the
|
|
113
|
-
orchestrator's outer try/except -- which turns a failure into a
|
|
114
|
-
durable status='failed' record -- is byte-for-byte unchanged.
|
|
115
|
-
|
|
116
|
-
The message names the pass, the field and the expected shape, because
|
|
117
|
-
the whole complaint in #76's loud half is that
|
|
118
|
-
`'str' object has no attribute 'items'` names none of the three.
|
|
119
|
-
"""
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
@dataclass(frozen=True)
|
|
123
|
-
class Spec:
|
|
124
|
-
"""One pass's promised payload shape.
|
|
125
|
-
|
|
126
|
-
top "object" or "array"
|
|
127
|
-
required top-level keys that must be present (top="object")
|
|
128
|
-
key_types top-level key -> permitted Python type(s)
|
|
129
|
-
entry_required keys every entry must carry (top="array")
|
|
130
|
-
entry_types entry key -> permitted Python type(s)
|
|
131
|
-
key_member_types top-level key -> permitted type for every MEMBER of
|
|
132
|
-
the container under it (a dict's values, a list's
|
|
133
|
-
elements)
|
|
134
|
-
entry_member_types the same, per entry key
|
|
135
|
-
enums entry key -> the set of permitted values
|
|
136
|
-
ids entry key holding a claim id, cross-checked against
|
|
137
|
-
the claims the apply step will resolve against
|
|
138
|
-
unique entry key that must not repeat (Pass 2's `id`)
|
|
139
|
-
ensemble_required entry keys required only on an ensemble run (B7)
|
|
140
|
-
ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
|
|
141
|
-
nested top-level key -> Spec for the array beneath it
|
|
142
|
-
"""
|
|
143
|
-
|
|
144
|
-
top: str
|
|
145
|
-
required: frozenset = frozenset()
|
|
146
|
-
key_types: dict = field(default_factory=dict)
|
|
147
|
-
entry_required: frozenset = frozenset()
|
|
148
|
-
entry_types: dict = field(default_factory=dict)
|
|
149
|
-
key_member_types: dict = field(default_factory=dict)
|
|
150
|
-
entry_member_types: dict = field(default_factory=dict)
|
|
151
|
-
enums: dict = field(default_factory=dict)
|
|
152
|
-
ids: Optional[str] = None
|
|
153
|
-
unique: Optional[str] = None
|
|
154
|
-
ensemble_required: frozenset = frozenset()
|
|
155
|
-
ids_in_keys: Optional[str] = None
|
|
156
|
-
nested: dict = field(default_factory=dict)
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
# The 3a and 3b entry shapes are named, because Pass 6b is documented in
|
|
160
|
-
# revalidate.md as returning exactly those two lists under two keys --
|
|
161
|
-
# and _apply_grounding/_apply_critic are literally the same two functions.
|
|
162
|
-
# Writing them twice would let 6c drift from the passes it re-runs.
|
|
163
|
-
_GROUNDING_ENTRY = Spec(
|
|
164
|
-
top="array",
|
|
165
|
-
entry_required=frozenset({"claim_id", "status"}),
|
|
166
|
-
# §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
|
|
167
|
-
# asymmetry is the point. B4's rule is that a check earns its place by
|
|
168
|
-
# having a demonstrated consequence:
|
|
169
|
-
#
|
|
170
|
-
# the type §45's scoring stage ITERATES this value. A string
|
|
171
|
-
# there scores one source per character, and a null
|
|
172
|
-
# raises inside a stage that runs after the model call
|
|
173
|
-
# it would need to retry.
|
|
174
|
-
# not required source_grounding.md says an ungrounded claim's
|
|
175
|
-
# `sources` "may be an empty list", and a model that
|
|
176
|
-
# omits the key entirely on a claim it could not
|
|
177
|
-
# source has said the same thing. Failing that run
|
|
178
|
-
# costs a corrective retry to add `[]` to an entry
|
|
179
|
-
# nothing reads.
|
|
180
|
-
#
|
|
181
|
-
# The case this leaves open -- `grounded` with no sources at all -- is
|
|
182
|
-
# NOT unhandled; it is handled where it means something. The scoring
|
|
183
|
-
# stage scores it as zero source quality and says so in the trace,
|
|
184
|
-
# which is the honest answer to a claim asserted as grounded with
|
|
185
|
-
# nothing behind it. Rejecting it here would instead ask the model to
|
|
186
|
-
# try again, and a model that had no sources still has none.
|
|
187
|
-
entry_types={"claim_id": str, "sources": list},
|
|
188
|
-
enums={"status": GROUNDING_STATUSES},
|
|
189
|
-
ids="claim_id",
|
|
190
|
-
)
|
|
191
|
-
|
|
192
|
-
_CRITIC_ENTRY = Spec(
|
|
193
|
-
top="array",
|
|
194
|
-
entry_required=frozenset({"claim_id", "severity"}),
|
|
195
|
-
# `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
|
|
196
|
-
# so a string here raises a TypeError two passes downstream, in the
|
|
197
|
-
# one stage that makes no model call and has nothing to retry.
|
|
198
|
-
entry_types={"claim_id": str, "severity": NUMBER},
|
|
199
|
-
ids="claim_id",
|
|
200
|
-
)
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
SPECS = {
|
|
204
|
-
# preliminary_plan.md names five keys, and the orchestrator reads ONE
|
|
205
|
-
# of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
|
|
206
|
-
# before handing the plan whole to Pass 3c. So requiring all five
|
|
207
|
-
# would fail runs over an omission nothing depends on. The
|
|
208
|
-
# demonstrated defect is a bare list, which is what `top` catches.
|
|
209
|
-
#
|
|
210
|
-
# THE ONE KEY THE ORCHESTRATOR READS IS TYPED, AND STILL NOT REQUIRED
|
|
211
|
-
# (batch 149) -- `key_types` is checked only `if key in payload`, so
|
|
212
|
-
# the defensive `.get` above keeps its meaning and an omission still
|
|
213
|
-
# costs nothing. What it stops is the read itself: `len(123)` raised
|
|
214
|
-
# inside the Pass 0 checkpoint, and `len("madrid")` did not raise at
|
|
215
|
-
# all -- it reported SIX key entities anticipated, from one word.
|
|
216
|
-
"Pass 0": Spec(top="object",
|
|
217
|
-
key_types={"key_entities_or_subjects": list}),
|
|
218
|
-
|
|
219
|
-
"Pass 2": Spec(
|
|
220
|
-
top="array",
|
|
221
|
-
entry_required=frozenset({"id", "text", "type"}),
|
|
222
|
-
# `entities` and `asserted_by_candidates` are TYPED BUT NOT
|
|
223
|
-
# REQUIRED, which is the same asymmetry `sources` carries below
|
|
224
|
-
# and for the same reason: base.py defaults both to an empty list,
|
|
225
|
-
# a claim with no entities is an ordinary answer, and
|
|
226
|
-
# `asserted_by_candidates` is required only in ensemble mode (B7).
|
|
227
|
-
# What they are not is optional to get RIGHT -- Pass 3a sorts a
|
|
228
|
-
# set built from every claim's entities, and Pass 5 measures
|
|
229
|
-
# `len(set(asserted_by_candidates))`, so both the container and
|
|
230
|
-
# its members are declared (batch 149).
|
|
231
|
-
entry_types={"id": str, "text": str, "entities": list,
|
|
232
|
-
"asserted_by_candidates": list},
|
|
233
|
-
# str and int from base.py's own annotations -- `entities:
|
|
234
|
-
# list[str]` and `asserted_by_candidates: list[int]`. The
|
|
235
|
-
# annotations are not enforced (the comment at the top of base.py
|
|
236
|
-
# says so, and #75 is what that cost); this is where they become a
|
|
237
|
-
# check.
|
|
238
|
-
entry_member_types={"entities": str, "asserted_by_candidates": int},
|
|
239
|
-
enums={"type": CLAIM_TYPES},
|
|
240
|
-
# B6. Two claims sharing an id split in half: _apply_grounding
|
|
241
|
-
# builds {c.id: c} (last wins) while _apply_assumption_flags uses
|
|
242
|
-
# next(...) (first wins), so the grounding landed on one object
|
|
243
|
-
# and the assumption flags on the other. claim_extraction.md asks
|
|
244
|
-
# for "sequential zero-padded ids" and never says they must be
|
|
245
|
-
# unique.
|
|
246
|
-
unique="id",
|
|
247
|
-
# B7. Only in ensemble mode, and only because claim_extraction.md
|
|
248
|
-
# is now asked to supply it on every claim there. An ABSENT field
|
|
249
|
-
# is indistinguishable from an empty one at the scorer, and
|
|
250
|
-
# `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
|
|
251
|
-
# a claim that exists because a candidate asserted it.
|
|
252
|
-
ensemble_required=frozenset({"asserted_by_candidates"}),
|
|
253
|
-
),
|
|
254
|
-
|
|
255
|
-
"Pass 3a": _GROUNDING_ENTRY,
|
|
256
|
-
"Pass 3b": _CRITIC_ENTRY,
|
|
257
|
-
|
|
258
|
-
"Pass 3c": Spec(
|
|
259
|
-
top="object",
|
|
260
|
-
required=frozenset({"gaps", "coverage_score"}),
|
|
261
|
-
# `gaps` is iterated by build_coverage_gap_entries; a string there
|
|
262
|
-
# would produce one coverage-gap entry per CHARACTER.
|
|
263
|
-
key_types={"gaps": list},
|
|
264
|
-
),
|
|
265
|
-
|
|
266
|
-
"Pass 4": Spec(
|
|
267
|
-
top="object",
|
|
268
|
-
required=frozenset({"per_claim_flags"}),
|
|
269
|
-
key_types={"per_claim_flags": dict},
|
|
270
|
-
# Every VALUE under those claim ids is a list, because Pass 5
|
|
271
|
-
# measures it: `w["assumption_flag_penalty"] * len(flags)` raised
|
|
272
|
-
# on `{"c001": 123}` and silently charged a THREE-flag penalty for
|
|
273
|
-
# `{"c001": "abc"}` (batch 149). The strings inside are not typed
|
|
274
|
-
# -- `len()` is all that reads them, so a type on the elements
|
|
275
|
-
# would be a check with no demonstrated consequence, which is the
|
|
276
|
-
# rule B4 states.
|
|
277
|
-
key_member_types={"per_claim_flags": list},
|
|
278
|
-
ids_in_keys="per_claim_flags",
|
|
279
|
-
),
|
|
280
|
-
|
|
281
|
-
"Pass 6a": Spec(
|
|
282
|
-
top="array",
|
|
283
|
-
entry_required=frozenset({"claim_id", "revised_text"}),
|
|
284
|
-
entry_types={"claim_id": str, "revised_text": str},
|
|
285
|
-
ids="claim_id",
|
|
286
|
-
),
|
|
287
|
-
|
|
288
|
-
"Pass 6b": Spec(
|
|
289
|
-
top="object",
|
|
290
|
-
required=frozenset({"grounding", "critic"}),
|
|
291
|
-
key_types={"grounding": list, "critic": list},
|
|
292
|
-
nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
|
|
293
|
-
),
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
def _type_name(value: Any) -> str:
|
|
298
|
-
"""JSON's vocabulary, not Python's -- the message goes back to a model
|
|
299
|
-
that was asked for JSON, so `object`/`array` is what it can act on."""
|
|
300
|
-
return {dict: "object", list: "array", str: "string",
|
|
301
|
-
bool: "boolean", int: "number", float: "number",
|
|
302
|
-
type(None): "null"}.get(type(value), type(value).__name__)
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
def _permitted(types) -> str:
|
|
306
|
-
return " or ".join(sorted({_type_name(t()) for t in _as_tuple(types)}))
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
def _as_tuple(types) -> tuple:
|
|
310
|
-
return types if isinstance(types, tuple) else (types,)
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
def _of_type(value, types) -> bool:
|
|
314
|
-
"""isinstance, except that a boolean is only ever a boolean.
|
|
315
|
-
|
|
316
|
-
See the comment on NUMBER. Applied to every declared type rather than
|
|
317
|
-
to NUMBER alone, because "a boolean satisfies a type it was not
|
|
318
|
-
declared as" is the defect, and `severity` is simply the only place
|
|
319
|
-
the tree has reached it so far.
|
|
320
|
-
"""
|
|
321
|
-
if isinstance(value, bool) and bool not in _as_tuple(types):
|
|
322
|
-
return False
|
|
323
|
-
return isinstance(value, types)
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
def _members(container):
|
|
327
|
-
"""(label, value) for everything a container holds.
|
|
328
|
-
|
|
329
|
-
A dict's VALUES, keyed by the key that named them, and a list's
|
|
330
|
-
elements by position -- so the message can say `'per_claim_flags'
|
|
331
|
-
['c001']` or `entities[1]` and the model can find the one it got
|
|
332
|
-
wrong.
|
|
333
|
-
"""
|
|
334
|
-
if isinstance(container, dict):
|
|
335
|
-
return [(f"[{key!r}]", value) for key, value in container.items()]
|
|
336
|
-
return [(f"[{index}]", value) for index, value in enumerate(container)]
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def _check_members(pass_id, container, member_types, at, prefix="") -> None:
|
|
340
|
-
for key, types in member_types.items():
|
|
341
|
-
if key not in container:
|
|
342
|
-
continue
|
|
343
|
-
value = container[key]
|
|
344
|
-
if not isinstance(value, (dict, list)):
|
|
345
|
-
# The declared CONTAINER type has already refused this, or
|
|
346
|
-
# there isn't one and the member rule has nothing to stand on.
|
|
347
|
-
# Either way, iterating here would report the members of a
|
|
348
|
-
# string.
|
|
349
|
-
continue
|
|
350
|
-
for label, member in _members(value):
|
|
351
|
-
if _of_type(member, types):
|
|
352
|
-
continue
|
|
353
|
-
raise PayloadShapeError(
|
|
354
|
-
f"{pass_id}: {prefix}{key}{label}{at} is {member!r}, which "
|
|
355
|
-
f"is {_type_name(member)}; every value in {key!r} must be "
|
|
356
|
-
f"{_permitted(types)}."
|
|
357
|
-
)
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def validate(pass_id: str, payload: Any, claim_ids=None,
|
|
361
|
-
ensemble: bool = False) -> None:
|
|
362
|
-
"""Raise PayloadShapeError unless `payload` is what `pass_id` promised.
|
|
363
|
-
|
|
364
|
-
claim_ids the ids the APPLY step will resolve against -- all claims
|
|
365
|
-
for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
|
|
366
|
-
"every claim in the run": Pass 6a is only asked about
|
|
367
|
-
`flagged`, and its own lookup already resolves within that
|
|
368
|
-
batch, so validating against a wider set would accept a
|
|
369
|
-
revision that then lands on nothing. None skips the check.
|
|
370
|
-
ensemble True on an ensemble run, which is the only place Pass 2 is
|
|
371
|
-
asked for `asserted_by_candidates` (B7).
|
|
372
|
-
|
|
373
|
-
An unknown pass_id RAISES rather than passing through unvalidated. A
|
|
374
|
-
pass with no spec is the silent hole this module exists to close, and
|
|
375
|
-
the eleventh pass must not be able to arrive by forgetting a line.
|
|
376
|
-
"""
|
|
377
|
-
spec = SPECS.get(pass_id)
|
|
378
|
-
if spec is None:
|
|
379
|
-
raise PayloadShapeError(
|
|
380
|
-
f"{pass_id}: no payload spec is declared for this pass. Every "
|
|
381
|
-
f"pass routed through _run_pass_with_json_retry needs one in "
|
|
382
|
-
f"core/reasoning/payload_validation.py (§30, B11)."
|
|
383
|
-
)
|
|
384
|
-
_validate_against(pass_id, payload, spec, claim_ids, ensemble)
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
|
|
388
|
-
where: str = "") -> None:
|
|
389
|
-
at = f" under {where!r}" if where else ""
|
|
390
|
-
|
|
391
|
-
if spec.top == "array":
|
|
392
|
-
if not isinstance(payload, list):
|
|
393
|
-
raise PayloadShapeError(
|
|
394
|
-
f"{pass_id}: expected a JSON array{at}, got "
|
|
395
|
-
f"{_type_name(payload)}. Return the array itself, not an "
|
|
396
|
-
f"object wrapping it."
|
|
397
|
-
)
|
|
398
|
-
_validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
|
|
399
|
-
return
|
|
400
|
-
|
|
401
|
-
if not isinstance(payload, dict):
|
|
402
|
-
raise PayloadShapeError(
|
|
403
|
-
f"{pass_id}: expected a JSON object{at}, got "
|
|
404
|
-
f"{_type_name(payload)}."
|
|
405
|
-
)
|
|
406
|
-
|
|
407
|
-
missing = sorted(spec.required - set(payload))
|
|
408
|
-
if missing:
|
|
409
|
-
raise PayloadShapeError(
|
|
410
|
-
f"{pass_id}: the response{at} is missing required key(s) "
|
|
411
|
-
f"{', '.join(repr(m) for m in missing)}. Expected an object "
|
|
412
|
-
f"with {', '.join(sorted(spec.required))}."
|
|
413
|
-
)
|
|
414
|
-
|
|
415
|
-
for key, types in spec.key_types.items():
|
|
416
|
-
if key in payload and not _of_type(payload[key], types):
|
|
417
|
-
raise PayloadShapeError(
|
|
418
|
-
f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
|
|
419
|
-
f"expected {_permitted(types)}."
|
|
420
|
-
)
|
|
421
|
-
|
|
422
|
-
_check_members(pass_id, payload, spec.key_member_types, at)
|
|
423
|
-
|
|
424
|
-
for key, nested in spec.nested.items():
|
|
425
|
-
_validate_against(pass_id, payload[key], nested, claim_ids,
|
|
426
|
-
ensemble, where=key)
|
|
427
|
-
|
|
428
|
-
if spec.ids_in_keys:
|
|
429
|
-
_check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
|
|
430
|
-
f"key of {spec.ids_in_keys!r}")
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
|
|
434
|
-
required = spec.entry_required | (spec.ensemble_required if ensemble
|
|
435
|
-
else frozenset())
|
|
436
|
-
seen = {}
|
|
437
|
-
for index, entry in enumerate(entries):
|
|
438
|
-
if not isinstance(entry, dict):
|
|
439
|
-
raise PayloadShapeError(
|
|
440
|
-
f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
|
|
441
|
-
f"every entry must be an object with "
|
|
442
|
-
f"{', '.join(sorted(required))}."
|
|
443
|
-
)
|
|
444
|
-
missing = sorted(required - set(entry))
|
|
445
|
-
if missing:
|
|
446
|
-
raise PayloadShapeError(
|
|
447
|
-
f"{pass_id}: entry {index}{at} is missing required key(s) "
|
|
448
|
-
f"{', '.join(repr(m) for m in missing)}. Every entry needs "
|
|
449
|
-
f"{', '.join(sorted(required))}."
|
|
450
|
-
)
|
|
451
|
-
# TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
|
|
452
|
-
# rule and the reason it is worth copying: a JSON array or object
|
|
453
|
-
# in an enum-checked field is unhashable, so `value in ALLOWED`
|
|
454
|
-
# raises TypeError from inside the validator rather than
|
|
455
|
-
# producing a message the model can act on.
|
|
456
|
-
for key, types in spec.entry_types.items():
|
|
457
|
-
if key in entry and not _of_type(entry[key], types):
|
|
458
|
-
raise PayloadShapeError(
|
|
459
|
-
f"{pass_id}: entry {index}{at} has {key}="
|
|
460
|
-
f"{entry[key]!r}, which is {_type_name(entry[key])}; "
|
|
461
|
-
f"expected {_permitted(types)}."
|
|
462
|
-
)
|
|
463
|
-
_check_members(pass_id, entry, spec.entry_member_types, at,
|
|
464
|
-
prefix=f"entry {index} ")
|
|
465
|
-
for key, allowed in spec.enums.items():
|
|
466
|
-
if key not in entry:
|
|
467
|
-
continue
|
|
468
|
-
value = entry[key]
|
|
469
|
-
if not isinstance(value, str) or value not in allowed:
|
|
470
|
-
raise PayloadShapeError(
|
|
471
|
-
f"{pass_id}: entry {index}{at} has {key}={value!r}; "
|
|
472
|
-
f"expected exactly one of "
|
|
473
|
-
f"{', '.join(sorted(allowed))} (lower case)."
|
|
474
|
-
)
|
|
475
|
-
if spec.unique:
|
|
476
|
-
value = entry[spec.unique]
|
|
477
|
-
if value in seen:
|
|
478
|
-
raise PayloadShapeError(
|
|
479
|
-
f"{pass_id}: entries {seen[value]} and {index}{at} "
|
|
480
|
-
f"share {spec.unique}={value!r}. Ids must be unique -- "
|
|
481
|
-
f"two claims with one id are applied to inconsistently "
|
|
482
|
-
f"and end up carrying half of each other's results."
|
|
483
|
-
)
|
|
484
|
-
seen[value] = index
|
|
485
|
-
|
|
486
|
-
if spec.ids:
|
|
487
|
-
_check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
|
|
488
|
-
spec.ids)
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
|
|
492
|
-
"""B5: a TOTAL mismatch is the error; a partial one is not.
|
|
493
|
-
|
|
494
|
-
The demonstrated silent failure is every id being wrong -- 3a/3b
|
|
495
|
-
answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
|
|
496
|
-
which grounded nothing while the checkpoint line reported that it had
|
|
497
|
-
grounded everything. That is a pass with no effect at all, and it is
|
|
498
|
-
worth a corrective retry.
|
|
499
|
-
|
|
500
|
-
A model that gets nine of ten right is NOT worth failing a run that
|
|
501
|
-
has already paid for nine passes: the nine apply, and the checkpoint
|
|
502
|
-
line reports the count APPLIED rather than the count asked about, so
|
|
503
|
-
the tenth is visible instead of silent (see the orchestrator's
|
|
504
|
-
_apply_* return values).
|
|
505
|
-
|
|
506
|
-
An EMPTY collection is not a mismatch. assumption_audit.md asks the
|
|
507
|
-
model to "omit claims with no issues rather than including them with
|
|
508
|
-
an empty list", so no flags at all is the ordinary answer for a clean
|
|
509
|
-
run -- and Pass 4's demonstrated defect was the top-level key being
|
|
510
|
-
absent, which the required-key check above catches instead.
|
|
511
|
-
"""
|
|
512
|
-
if claim_ids is None or not supplied:
|
|
513
|
-
return
|
|
514
|
-
known = set(claim_ids)
|
|
515
|
-
if any(value in known for value in supplied):
|
|
516
|
-
return
|
|
517
|
-
sample = ", ".join(repr(v) for v in supplied[:3])
|
|
518
|
-
expected = ", ".join(repr(v) for v in sorted(known)[:3])
|
|
519
|
-
raise PayloadShapeError(
|
|
520
|
-
f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
|
|
521
|
-
f"matches a claim you were given (got {sample}; expected ids like "
|
|
522
|
-
f"{expected}). Use the claim ids exactly as they appear in the "
|
|
523
|
-
f"input."
|
|
524
|
-
)
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/payload_validation.py
|
|
3
|
+
|
|
4
|
+
ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
|
|
5
|
+
|
|
6
|
+
§3's retry corrects a response that does not PARSE. Nothing checked that
|
|
7
|
+
what parsed was the SHAPE the pass promised, so the same defect appeared
|
|
8
|
+
in six places at once (audit #76): a wrapped list crashed with
|
|
9
|
+
`'str' object has no attribute 'items'`, a bad `severity` survived two
|
|
10
|
+
passes and surfaced in Pass 5's pure-code tiering, and two shapes
|
|
11
|
+
completed a run and reported a false outcome.
|
|
12
|
+
|
|
13
|
+
WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
|
|
14
|
+
this validator -- `review.py:_validated` checks a top-level type, the
|
|
15
|
+
required keys per entry with type guards BEFORE the membership tests, an
|
|
16
|
+
id cross-check against `{c.id for c in run.claims}`, an enum, and a
|
|
17
|
+
duplicate -- and built it once, for the one payload that is NOT a pass.
|
|
18
|
+
The passes never got it. Putting the general version in orchestrator.py
|
|
19
|
+
would repeat json_retry.py's own mistake in reverse: that module was
|
|
20
|
+
extracted precisely so a mechanism a second consumer might want is not
|
|
21
|
+
trapped in the orchestrator.
|
|
22
|
+
|
|
23
|
+
`review.py` is deliberately NOT migrated onto this module (§30, B1). Its
|
|
24
|
+
POLICY is different and correctly so: a finding it cannot apply is
|
|
25
|
+
dropped and traced, because the review stage is optional and contained,
|
|
26
|
+
and a reviewer that misfires must not cost a completed ten-pass run. A
|
|
27
|
+
pass is not optional -- a payload it cannot apply is a run that will
|
|
28
|
+
report something untrue -- so a failure here re-enters §3's corrective
|
|
29
|
+
retry (B2) and ultimately raises. Same shape, two policies. Merging them
|
|
30
|
+
means choosing one, and neither is wrong for its own caller.
|
|
31
|
+
|
|
32
|
+
WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
|
|
33
|
+
declared type where a value is used arithmetically -- every one has a
|
|
34
|
+
demonstrated consequence in #76's table, and nothing else does:
|
|
35
|
+
|
|
36
|
+
the top-level type {"claims": [...]} instead of [...]
|
|
37
|
+
the required keys per entry a claim with no `id`, a revision with no
|
|
38
|
+
`revised_text`, Pass 4 with no
|
|
39
|
+
`per_claim_flags`
|
|
40
|
+
a declared value type `"severity": "0.0"` -- a string that
|
|
41
|
+
fails two passes later, in Pass 5
|
|
42
|
+
the type of its MEMBERS `"entities": ["madrid", 1947]` -- a list,
|
|
43
|
+
which Pass 3a then sorts
|
|
44
|
+
that entry ids resolve 3a/3b answering about ids Pass 2 never
|
|
45
|
+
issued, which grounded nothing while the
|
|
46
|
+
trace said it had grounded everything
|
|
47
|
+
|
|
48
|
+
THE FOURTH ROW IS BATCH 149, AND IT IS NOT A RICHER VOCABULARY FOR ITS OWN
|
|
49
|
+
SAKE -- it is the same property as the third, asked of the values a
|
|
50
|
+
container holds, because that is where the arithmetic actually happens.
|
|
51
|
+
`entities` is not measured; every element of it is hashed and then
|
|
52
|
+
compared against every other (`sorted({e for c in claims for e in
|
|
53
|
+
c.entities})`), and `asserted_by_candidates` is hashed the same way by
|
|
54
|
+
`len(set(...))`. A declared CONTAINER type closes the three container
|
|
55
|
+
cases and leaves the three member cases open, each driven against the real
|
|
56
|
+
consumer:
|
|
57
|
+
|
|
58
|
+
["madrid", 1947] TypeError: '<' not supported between
|
|
59
|
+
instances of 'int' and 'str'
|
|
60
|
+
[["madrid"]] TypeError: unhashable type: 'list'
|
|
61
|
+
[{"name": "madrid"}] TypeError: unhashable type: 'dict'
|
|
62
|
+
|
|
63
|
+
A MEMBER is a dict's VALUES or a list's ELEMENTS, deliberately one concept
|
|
64
|
+
and one spec field per level: Pass 4's flag lists live under claim-id keys
|
|
65
|
+
and Pass 2's entities live in an array, and the thing being declared is
|
|
66
|
+
the same in both -- what the pure code downstream will iterate, hash, sort
|
|
67
|
+
or measure. Pass 4's flag lists are typed and their STRINGS are not, which
|
|
68
|
+
is the same rule: `len()` is the only thing that reads them.
|
|
69
|
+
|
|
70
|
+
Not a schema language, and no new dependency (#145 tracks those). A
|
|
71
|
+
richer vocabulary would let a spec describe things no pass has ever got
|
|
72
|
+
wrong, at the price of a second place where a payload's meaning lives.
|
|
73
|
+
|
|
74
|
+
STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
|
|
75
|
+
`Respond with ONLY this JSON structure` block in that pass's own .md, and
|
|
76
|
+
NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
|
|
77
|
+
field will fail the run rather than silently mis-apply it -- which is the
|
|
78
|
+
better failure -- but it will fail it at the model's expense. The parity
|
|
79
|
+
that IS mechanised is narrower and is the one that goes wrong silently:
|
|
80
|
+
every pass the orchestrator routes through `_run_pass_with_json_retry`
|
|
81
|
+
has a spec here (see tests/test_payload_validation.py).
|
|
82
|
+
"""
|
|
83
|
+
|
|
84
|
+
from __future__ import annotations
|
|
85
|
+
|
|
86
|
+
from dataclasses import dataclass, field
|
|
87
|
+
from typing import Any, Optional
|
|
88
|
+
|
|
89
|
+
# Re-exported, not re-derived: base.py defines these beside the Literals
|
|
90
|
+
# they come from, and the scorer reads the same two names (§30, B9).
|
|
91
|
+
from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
|
|
92
|
+
|
|
93
|
+
# A BOOLEAN IS NOT A NUMBER, which Python disagrees with: `isinstance(
|
|
94
|
+
# True, int)` is True, so `"severity": true` satisfied this tuple and
|
|
95
|
+
# Pass 5 computed `1.0 - True` as 0.0 -- the MAXIMUM critic penalty, the
|
|
96
|
+
# single harshest value the field can take -- with nothing in the trace
|
|
97
|
+
# and no corrective retry (batch 149). `"severity": false` took the other
|
|
98
|
+
# extreme for the same reason.
|
|
99
|
+
#
|
|
100
|
+
# The exclusion is not new to the codebase, only to the boundary:
|
|
101
|
+
# `source_scoring._as_number` and `confidence_scoring.source_quality` both
|
|
102
|
+
# spell it out already, each guarding a value that is supposed to have
|
|
103
|
+
# been rejected HERE. See `_of_type` below, which is what applies it.
|
|
104
|
+
NUMBER = (int, float)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
class PayloadShapeError(ValueError):
|
|
108
|
+
"""A pass's response parsed as JSON but is not the shape it promised.
|
|
109
|
+
|
|
110
|
+
SUBCLASSES ValueError deliberately, and that is load-bearing twice:
|
|
111
|
+
json_retry.py's corrective loop already catches ValueError (so a shape
|
|
112
|
+
failure re-enters the retry with no new except clause), and the
|
|
113
|
+
orchestrator's outer try/except -- which turns a failure into a
|
|
114
|
+
durable status='failed' record -- is byte-for-byte unchanged.
|
|
115
|
+
|
|
116
|
+
The message names the pass, the field and the expected shape, because
|
|
117
|
+
the whole complaint in #76's loud half is that
|
|
118
|
+
`'str' object has no attribute 'items'` names none of the three.
|
|
119
|
+
"""
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class Spec:
|
|
124
|
+
"""One pass's promised payload shape.
|
|
125
|
+
|
|
126
|
+
top "object" or "array"
|
|
127
|
+
required top-level keys that must be present (top="object")
|
|
128
|
+
key_types top-level key -> permitted Python type(s)
|
|
129
|
+
entry_required keys every entry must carry (top="array")
|
|
130
|
+
entry_types entry key -> permitted Python type(s)
|
|
131
|
+
key_member_types top-level key -> permitted type for every MEMBER of
|
|
132
|
+
the container under it (a dict's values, a list's
|
|
133
|
+
elements)
|
|
134
|
+
entry_member_types the same, per entry key
|
|
135
|
+
enums entry key -> the set of permitted values
|
|
136
|
+
ids entry key holding a claim id, cross-checked against
|
|
137
|
+
the claims the apply step will resolve against
|
|
138
|
+
unique entry key that must not repeat (Pass 2's `id`)
|
|
139
|
+
ensemble_required entry keys required only on an ensemble run (B7)
|
|
140
|
+
ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
|
|
141
|
+
nested top-level key -> Spec for the array beneath it
|
|
142
|
+
"""
|
|
143
|
+
|
|
144
|
+
top: str
|
|
145
|
+
required: frozenset = frozenset()
|
|
146
|
+
key_types: dict = field(default_factory=dict)
|
|
147
|
+
entry_required: frozenset = frozenset()
|
|
148
|
+
entry_types: dict = field(default_factory=dict)
|
|
149
|
+
key_member_types: dict = field(default_factory=dict)
|
|
150
|
+
entry_member_types: dict = field(default_factory=dict)
|
|
151
|
+
enums: dict = field(default_factory=dict)
|
|
152
|
+
ids: Optional[str] = None
|
|
153
|
+
unique: Optional[str] = None
|
|
154
|
+
ensemble_required: frozenset = frozenset()
|
|
155
|
+
ids_in_keys: Optional[str] = None
|
|
156
|
+
nested: dict = field(default_factory=dict)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# The 3a and 3b entry shapes are named, because Pass 6b is documented in
|
|
160
|
+
# revalidate.md as returning exactly those two lists under two keys --
|
|
161
|
+
# and _apply_grounding/_apply_critic are literally the same two functions.
|
|
162
|
+
# Writing them twice would let 6c drift from the passes it re-runs.
|
|
163
|
+
_GROUNDING_ENTRY = Spec(
|
|
164
|
+
top="array",
|
|
165
|
+
entry_required=frozenset({"claim_id", "status"}),
|
|
166
|
+
# §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
|
|
167
|
+
# asymmetry is the point. B4's rule is that a check earns its place by
|
|
168
|
+
# having a demonstrated consequence:
|
|
169
|
+
#
|
|
170
|
+
# the type §45's scoring stage ITERATES this value. A string
|
|
171
|
+
# there scores one source per character, and a null
|
|
172
|
+
# raises inside a stage that runs after the model call
|
|
173
|
+
# it would need to retry.
|
|
174
|
+
# not required source_grounding.md says an ungrounded claim's
|
|
175
|
+
# `sources` "may be an empty list", and a model that
|
|
176
|
+
# omits the key entirely on a claim it could not
|
|
177
|
+
# source has said the same thing. Failing that run
|
|
178
|
+
# costs a corrective retry to add `[]` to an entry
|
|
179
|
+
# nothing reads.
|
|
180
|
+
#
|
|
181
|
+
# The case this leaves open -- `grounded` with no sources at all -- is
|
|
182
|
+
# NOT unhandled; it is handled where it means something. The scoring
|
|
183
|
+
# stage scores it as zero source quality and says so in the trace,
|
|
184
|
+
# which is the honest answer to a claim asserted as grounded with
|
|
185
|
+
# nothing behind it. Rejecting it here would instead ask the model to
|
|
186
|
+
# try again, and a model that had no sources still has none.
|
|
187
|
+
entry_types={"claim_id": str, "sources": list},
|
|
188
|
+
enums={"status": GROUNDING_STATUSES},
|
|
189
|
+
ids="claim_id",
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
_CRITIC_ENTRY = Spec(
|
|
193
|
+
top="array",
|
|
194
|
+
entry_required=frozenset({"claim_id", "severity"}),
|
|
195
|
+
# `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
|
|
196
|
+
# so a string here raises a TypeError two passes downstream, in the
|
|
197
|
+
# one stage that makes no model call and has nothing to retry.
|
|
198
|
+
entry_types={"claim_id": str, "severity": NUMBER},
|
|
199
|
+
ids="claim_id",
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
SPECS = {
|
|
204
|
+
# preliminary_plan.md names five keys, and the orchestrator reads ONE
|
|
205
|
+
# of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
|
|
206
|
+
# before handing the plan whole to Pass 3c. So requiring all five
|
|
207
|
+
# would fail runs over an omission nothing depends on. The
|
|
208
|
+
# demonstrated defect is a bare list, which is what `top` catches.
|
|
209
|
+
#
|
|
210
|
+
# THE ONE KEY THE ORCHESTRATOR READS IS TYPED, AND STILL NOT REQUIRED
|
|
211
|
+
# (batch 149) -- `key_types` is checked only `if key in payload`, so
|
|
212
|
+
# the defensive `.get` above keeps its meaning and an omission still
|
|
213
|
+
# costs nothing. What it stops is the read itself: `len(123)` raised
|
|
214
|
+
# inside the Pass 0 checkpoint, and `len("madrid")` did not raise at
|
|
215
|
+
# all -- it reported SIX key entities anticipated, from one word.
|
|
216
|
+
"Pass 0": Spec(top="object",
|
|
217
|
+
key_types={"key_entities_or_subjects": list}),
|
|
218
|
+
|
|
219
|
+
"Pass 2": Spec(
|
|
220
|
+
top="array",
|
|
221
|
+
entry_required=frozenset({"id", "text", "type"}),
|
|
222
|
+
# `entities` and `asserted_by_candidates` are TYPED BUT NOT
|
|
223
|
+
# REQUIRED, which is the same asymmetry `sources` carries below
|
|
224
|
+
# and for the same reason: base.py defaults both to an empty list,
|
|
225
|
+
# a claim with no entities is an ordinary answer, and
|
|
226
|
+
# `asserted_by_candidates` is required only in ensemble mode (B7).
|
|
227
|
+
# What they are not is optional to get RIGHT -- Pass 3a sorts a
|
|
228
|
+
# set built from every claim's entities, and Pass 5 measures
|
|
229
|
+
# `len(set(asserted_by_candidates))`, so both the container and
|
|
230
|
+
# its members are declared (batch 149).
|
|
231
|
+
entry_types={"id": str, "text": str, "entities": list,
|
|
232
|
+
"asserted_by_candidates": list},
|
|
233
|
+
# str and int from base.py's own annotations -- `entities:
|
|
234
|
+
# list[str]` and `asserted_by_candidates: list[int]`. The
|
|
235
|
+
# annotations are not enforced (the comment at the top of base.py
|
|
236
|
+
# says so, and #75 is what that cost); this is where they become a
|
|
237
|
+
# check.
|
|
238
|
+
entry_member_types={"entities": str, "asserted_by_candidates": int},
|
|
239
|
+
enums={"type": CLAIM_TYPES},
|
|
240
|
+
# B6. Two claims sharing an id split in half: _apply_grounding
|
|
241
|
+
# builds {c.id: c} (last wins) while _apply_assumption_flags uses
|
|
242
|
+
# next(...) (first wins), so the grounding landed on one object
|
|
243
|
+
# and the assumption flags on the other. claim_extraction.md asks
|
|
244
|
+
# for "sequential zero-padded ids" and never says they must be
|
|
245
|
+
# unique.
|
|
246
|
+
unique="id",
|
|
247
|
+
# B7. Only in ensemble mode, and only because claim_extraction.md
|
|
248
|
+
# is now asked to supply it on every claim there. An ABSENT field
|
|
249
|
+
# is indistinguishable from an empty one at the scorer, and
|
|
250
|
+
# `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
|
|
251
|
+
# a claim that exists because a candidate asserted it.
|
|
252
|
+
ensemble_required=frozenset({"asserted_by_candidates"}),
|
|
253
|
+
),
|
|
254
|
+
|
|
255
|
+
"Pass 3a": _GROUNDING_ENTRY,
|
|
256
|
+
"Pass 3b": _CRITIC_ENTRY,
|
|
257
|
+
|
|
258
|
+
"Pass 3c": Spec(
|
|
259
|
+
top="object",
|
|
260
|
+
required=frozenset({"gaps", "coverage_score"}),
|
|
261
|
+
# `gaps` is iterated by build_coverage_gap_entries; a string there
|
|
262
|
+
# would produce one coverage-gap entry per CHARACTER.
|
|
263
|
+
key_types={"gaps": list},
|
|
264
|
+
),
|
|
265
|
+
|
|
266
|
+
"Pass 4": Spec(
|
|
267
|
+
top="object",
|
|
268
|
+
required=frozenset({"per_claim_flags"}),
|
|
269
|
+
key_types={"per_claim_flags": dict},
|
|
270
|
+
# Every VALUE under those claim ids is a list, because Pass 5
|
|
271
|
+
# measures it: `w["assumption_flag_penalty"] * len(flags)` raised
|
|
272
|
+
# on `{"c001": 123}` and silently charged a THREE-flag penalty for
|
|
273
|
+
# `{"c001": "abc"}` (batch 149). The strings inside are not typed
|
|
274
|
+
# -- `len()` is all that reads them, so a type on the elements
|
|
275
|
+
# would be a check with no demonstrated consequence, which is the
|
|
276
|
+
# rule B4 states.
|
|
277
|
+
key_member_types={"per_claim_flags": list},
|
|
278
|
+
ids_in_keys="per_claim_flags",
|
|
279
|
+
),
|
|
280
|
+
|
|
281
|
+
"Pass 6a": Spec(
|
|
282
|
+
top="array",
|
|
283
|
+
entry_required=frozenset({"claim_id", "revised_text"}),
|
|
284
|
+
entry_types={"claim_id": str, "revised_text": str},
|
|
285
|
+
ids="claim_id",
|
|
286
|
+
),
|
|
287
|
+
|
|
288
|
+
"Pass 6b": Spec(
|
|
289
|
+
top="object",
|
|
290
|
+
required=frozenset({"grounding", "critic"}),
|
|
291
|
+
key_types={"grounding": list, "critic": list},
|
|
292
|
+
nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
|
|
293
|
+
),
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _type_name(value: Any) -> str:
|
|
298
|
+
"""JSON's vocabulary, not Python's -- the message goes back to a model
|
|
299
|
+
that was asked for JSON, so `object`/`array` is what it can act on."""
|
|
300
|
+
return {dict: "object", list: "array", str: "string",
|
|
301
|
+
bool: "boolean", int: "number", float: "number",
|
|
302
|
+
type(None): "null"}.get(type(value), type(value).__name__)
|
|
303
|
+
|
|
304
|
+
|
|
305
|
+
def _permitted(types) -> str:
|
|
306
|
+
return " or ".join(sorted({_type_name(t()) for t in _as_tuple(types)}))
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _as_tuple(types) -> tuple:
|
|
310
|
+
return types if isinstance(types, tuple) else (types,)
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
def _of_type(value, types) -> bool:
|
|
314
|
+
"""isinstance, except that a boolean is only ever a boolean.
|
|
315
|
+
|
|
316
|
+
See the comment on NUMBER. Applied to every declared type rather than
|
|
317
|
+
to NUMBER alone, because "a boolean satisfies a type it was not
|
|
318
|
+
declared as" is the defect, and `severity` is simply the only place
|
|
319
|
+
the tree has reached it so far.
|
|
320
|
+
"""
|
|
321
|
+
if isinstance(value, bool) and bool not in _as_tuple(types):
|
|
322
|
+
return False
|
|
323
|
+
return isinstance(value, types)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _members(container):
|
|
327
|
+
"""(label, value) for everything a container holds.
|
|
328
|
+
|
|
329
|
+
A dict's VALUES, keyed by the key that named them, and a list's
|
|
330
|
+
elements by position -- so the message can say `'per_claim_flags'
|
|
331
|
+
['c001']` or `entities[1]` and the model can find the one it got
|
|
332
|
+
wrong.
|
|
333
|
+
"""
|
|
334
|
+
if isinstance(container, dict):
|
|
335
|
+
return [(f"[{key!r}]", value) for key, value in container.items()]
|
|
336
|
+
return [(f"[{index}]", value) for index, value in enumerate(container)]
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _check_members(pass_id, container, member_types, at, prefix="") -> None:
|
|
340
|
+
for key, types in member_types.items():
|
|
341
|
+
if key not in container:
|
|
342
|
+
continue
|
|
343
|
+
value = container[key]
|
|
344
|
+
if not isinstance(value, (dict, list)):
|
|
345
|
+
# The declared CONTAINER type has already refused this, or
|
|
346
|
+
# there isn't one and the member rule has nothing to stand on.
|
|
347
|
+
# Either way, iterating here would report the members of a
|
|
348
|
+
# string.
|
|
349
|
+
continue
|
|
350
|
+
for label, member in _members(value):
|
|
351
|
+
if _of_type(member, types):
|
|
352
|
+
continue
|
|
353
|
+
raise PayloadShapeError(
|
|
354
|
+
f"{pass_id}: {prefix}{key}{label}{at} is {member!r}, which "
|
|
355
|
+
f"is {_type_name(member)}; every value in {key!r} must be "
|
|
356
|
+
f"{_permitted(types)}."
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def validate(pass_id: str, payload: Any, claim_ids=None,
|
|
361
|
+
ensemble: bool = False) -> None:
|
|
362
|
+
"""Raise PayloadShapeError unless `payload` is what `pass_id` promised.
|
|
363
|
+
|
|
364
|
+
claim_ids the ids the APPLY step will resolve against -- all claims
|
|
365
|
+
for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
|
|
366
|
+
"every claim in the run": Pass 6a is only asked about
|
|
367
|
+
`flagged`, and its own lookup already resolves within that
|
|
368
|
+
batch, so validating against a wider set would accept a
|
|
369
|
+
revision that then lands on nothing. None skips the check.
|
|
370
|
+
ensemble True on an ensemble run, which is the only place Pass 2 is
|
|
371
|
+
asked for `asserted_by_candidates` (B7).
|
|
372
|
+
|
|
373
|
+
An unknown pass_id RAISES rather than passing through unvalidated. A
|
|
374
|
+
pass with no spec is the silent hole this module exists to close, and
|
|
375
|
+
the eleventh pass must not be able to arrive by forgetting a line.
|
|
376
|
+
"""
|
|
377
|
+
spec = SPECS.get(pass_id)
|
|
378
|
+
if spec is None:
|
|
379
|
+
raise PayloadShapeError(
|
|
380
|
+
f"{pass_id}: no payload spec is declared for this pass. Every "
|
|
381
|
+
f"pass routed through _run_pass_with_json_retry needs one in "
|
|
382
|
+
f"core/reasoning/payload_validation.py (§30, B11)."
|
|
383
|
+
)
|
|
384
|
+
_validate_against(pass_id, payload, spec, claim_ids, ensemble)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
|
|
388
|
+
where: str = "") -> None:
|
|
389
|
+
at = f" under {where!r}" if where else ""
|
|
390
|
+
|
|
391
|
+
if spec.top == "array":
|
|
392
|
+
if not isinstance(payload, list):
|
|
393
|
+
raise PayloadShapeError(
|
|
394
|
+
f"{pass_id}: expected a JSON array{at}, got "
|
|
395
|
+
f"{_type_name(payload)}. Return the array itself, not an "
|
|
396
|
+
f"object wrapping it."
|
|
397
|
+
)
|
|
398
|
+
_validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
|
|
399
|
+
return
|
|
400
|
+
|
|
401
|
+
if not isinstance(payload, dict):
|
|
402
|
+
raise PayloadShapeError(
|
|
403
|
+
f"{pass_id}: expected a JSON object{at}, got "
|
|
404
|
+
f"{_type_name(payload)}."
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
missing = sorted(spec.required - set(payload))
|
|
408
|
+
if missing:
|
|
409
|
+
raise PayloadShapeError(
|
|
410
|
+
f"{pass_id}: the response{at} is missing required key(s) "
|
|
411
|
+
f"{', '.join(repr(m) for m in missing)}. Expected an object "
|
|
412
|
+
f"with {', '.join(sorted(spec.required))}."
|
|
413
|
+
)
|
|
414
|
+
|
|
415
|
+
for key, types in spec.key_types.items():
|
|
416
|
+
if key in payload and not _of_type(payload[key], types):
|
|
417
|
+
raise PayloadShapeError(
|
|
418
|
+
f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
|
|
419
|
+
f"expected {_permitted(types)}."
|
|
420
|
+
)
|
|
421
|
+
|
|
422
|
+
_check_members(pass_id, payload, spec.key_member_types, at)
|
|
423
|
+
|
|
424
|
+
for key, nested in spec.nested.items():
|
|
425
|
+
_validate_against(pass_id, payload[key], nested, claim_ids,
|
|
426
|
+
ensemble, where=key)
|
|
427
|
+
|
|
428
|
+
if spec.ids_in_keys:
|
|
429
|
+
_check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
|
|
430
|
+
f"key of {spec.ids_in_keys!r}")
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
|
|
434
|
+
required = spec.entry_required | (spec.ensemble_required if ensemble
|
|
435
|
+
else frozenset())
|
|
436
|
+
seen = {}
|
|
437
|
+
for index, entry in enumerate(entries):
|
|
438
|
+
if not isinstance(entry, dict):
|
|
439
|
+
raise PayloadShapeError(
|
|
440
|
+
f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
|
|
441
|
+
f"every entry must be an object with "
|
|
442
|
+
f"{', '.join(sorted(required))}."
|
|
443
|
+
)
|
|
444
|
+
missing = sorted(required - set(entry))
|
|
445
|
+
if missing:
|
|
446
|
+
raise PayloadShapeError(
|
|
447
|
+
f"{pass_id}: entry {index}{at} is missing required key(s) "
|
|
448
|
+
f"{', '.join(repr(m) for m in missing)}. Every entry needs "
|
|
449
|
+
f"{', '.join(sorted(required))}."
|
|
450
|
+
)
|
|
451
|
+
# TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
|
|
452
|
+
# rule and the reason it is worth copying: a JSON array or object
|
|
453
|
+
# in an enum-checked field is unhashable, so `value in ALLOWED`
|
|
454
|
+
# raises TypeError from inside the validator rather than
|
|
455
|
+
# producing a message the model can act on.
|
|
456
|
+
for key, types in spec.entry_types.items():
|
|
457
|
+
if key in entry and not _of_type(entry[key], types):
|
|
458
|
+
raise PayloadShapeError(
|
|
459
|
+
f"{pass_id}: entry {index}{at} has {key}="
|
|
460
|
+
f"{entry[key]!r}, which is {_type_name(entry[key])}; "
|
|
461
|
+
f"expected {_permitted(types)}."
|
|
462
|
+
)
|
|
463
|
+
_check_members(pass_id, entry, spec.entry_member_types, at,
|
|
464
|
+
prefix=f"entry {index} ")
|
|
465
|
+
for key, allowed in spec.enums.items():
|
|
466
|
+
if key not in entry:
|
|
467
|
+
continue
|
|
468
|
+
value = entry[key]
|
|
469
|
+
if not isinstance(value, str) or value not in allowed:
|
|
470
|
+
raise PayloadShapeError(
|
|
471
|
+
f"{pass_id}: entry {index}{at} has {key}={value!r}; "
|
|
472
|
+
f"expected exactly one of "
|
|
473
|
+
f"{', '.join(sorted(allowed))} (lower case)."
|
|
474
|
+
)
|
|
475
|
+
if spec.unique:
|
|
476
|
+
value = entry[spec.unique]
|
|
477
|
+
if value in seen:
|
|
478
|
+
raise PayloadShapeError(
|
|
479
|
+
f"{pass_id}: entries {seen[value]} and {index}{at} "
|
|
480
|
+
f"share {spec.unique}={value!r}. Ids must be unique -- "
|
|
481
|
+
f"two claims with one id are applied to inconsistently "
|
|
482
|
+
f"and end up carrying half of each other's results."
|
|
483
|
+
)
|
|
484
|
+
seen[value] = index
|
|
485
|
+
|
|
486
|
+
if spec.ids:
|
|
487
|
+
_check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
|
|
488
|
+
spec.ids)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
|
|
492
|
+
"""B5: a TOTAL mismatch is the error; a partial one is not.
|
|
493
|
+
|
|
494
|
+
The demonstrated silent failure is every id being wrong -- 3a/3b
|
|
495
|
+
answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
|
|
496
|
+
which grounded nothing while the checkpoint line reported that it had
|
|
497
|
+
grounded everything. That is a pass with no effect at all, and it is
|
|
498
|
+
worth a corrective retry.
|
|
499
|
+
|
|
500
|
+
A model that gets nine of ten right is NOT worth failing a run that
|
|
501
|
+
has already paid for nine passes: the nine apply, and the checkpoint
|
|
502
|
+
line reports the count APPLIED rather than the count asked about, so
|
|
503
|
+
the tenth is visible instead of silent (see the orchestrator's
|
|
504
|
+
_apply_* return values).
|
|
505
|
+
|
|
506
|
+
An EMPTY collection is not a mismatch. assumption_audit.md asks the
|
|
507
|
+
model to "omit claims with no issues rather than including them with
|
|
508
|
+
an empty list", so no flags at all is the ordinary answer for a clean
|
|
509
|
+
run -- and Pass 4's demonstrated defect was the top-level key being
|
|
510
|
+
absent, which the required-key check above catches instead.
|
|
511
|
+
"""
|
|
512
|
+
if claim_ids is None or not supplied:
|
|
513
|
+
return
|
|
514
|
+
known = set(claim_ids)
|
|
515
|
+
if any(value in known for value in supplied):
|
|
516
|
+
return
|
|
517
|
+
sample = ", ".join(repr(v) for v in supplied[:3])
|
|
518
|
+
expected = ", ".join(repr(v) for v in sorted(known)[:3])
|
|
519
|
+
raise PayloadShapeError(
|
|
520
|
+
f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
|
|
521
|
+
f"matches a claim you were given (got {sample}; expected ids like "
|
|
522
|
+
f"{expected}). Use the claim ids exactly as they appear in the "
|
|
523
|
+
f"input."
|
|
524
|
+
)
|