@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,286 @@
1
+ """
2
+ core/reasoning/output_writer.py
3
+
4
+ Writes every pass's output to OUTPUT_DIR/<run_id>/, matching the
5
+ original pipeline spec's file layout (ROADMAP §12). NOT called
6
+ automatically from the pipeline -- it stays
7
+ a pure computation returning a PipelineRun; writing files (or not) is
8
+ the CALLER's decision (see main.py's research mode, §1). This keeps the
9
+ pipeline function's side effects minimal and makes it trivially testable
10
+ without touching a filesystem.
11
+
12
+ Relationship to §5's pipeline_storage.py: that module is for
13
+ programmatic resume/inspection (a database row, checked by run_id).
14
+ This module is for a human-browsable directory on disk -- a different
15
+ artifact, for a different audience, both keyed by the same run.run_id
16
+ so they correlate.
17
+
18
+ Claims are serialized with vars(c), matching every other
19
+ claim-serialization call site in this codebase (orchestrator.py passes
20
+ 3a/3b/4/6a/6b/final synthesis, pipeline_storage.update_pipeline_run,
21
+ and this module's write_run_artifacts). If a nested-dataclass field is
22
+ ever added to Claim, ALL of those sites migrate to asdict(c) together
23
+ -- see the note on Claim in core/reasoning/base.py.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import json
29
+ import logging
30
+ import os
31
+ from typing import TYPE_CHECKING
32
+
33
+ import config
34
+
35
+ if TYPE_CHECKING:
36
+ from core.reasoning.base import PipelineRun
37
+
38
+ logger = logging.getLogger(__name__)
39
+
40
+
41
+ def _tier_counts(run: PipelineRun) -> tuple[list[str], list[int]]:
42
+ """Returns (tier_labels, counts) for the confidence chart. Extracted
43
+ as a pure function so the counting logic can be unit-tested without
44
+ rendering a PNG."""
45
+ tiers = ["HIGH", "MEDIUM", "LOW", "UNVERIFIED"]
46
+ counts = [sum(1 for c in run.claims if c.confidence_tier == t) for t in tiers]
47
+ tiers.append("UNVERIFIED_COVERAGE")
48
+ counts.append(len(run.coverage_gaps))
49
+ return tiers, counts
50
+
51
+
52
+ def write_run_artifacts(run: PipelineRun) -> str:
53
+ """Writes the full artifact directory for one pipeline run and
54
+ returns the output directory path.
55
+
56
+ Layout under OUTPUT_DIR/<run_id>/:
57
+ 00_plan.md Pass 0 plan (JSON)
58
+ 01_raw_response.md Pass 1 raw generation (single-candidate runs)
59
+ 01_candidate_N.md E13/#77: per-candidate generation, one file
60
+ per SURVIVOR, written instead of
61
+ 01_raw_response.md whenever two or more
62
+ survived -- the N matches the trace's
63
+ candidate-N lines and the claims'
64
+ asserted_by_candidates tags, so the record
65
+ can substantiate its own consistency scores.
66
+ A run degraded to one survivor names its
67
+ file 01_raw_response.md like any other
68
+ single-candidate run.
69
+ 02_claims.json Pass 2 extracted claims (full vars(c))
70
+ 03_grounding.json Pass 3a grounding per claim
71
+ 03_critic.json Pass 3b critic per claim
72
+ 03_completeness.json Pass 3c completeness findings
73
+ 04_assumptions.json Pass 4 assumption audit
74
+ 05_confidence.json Pass 5 tier + score breakdown per claim
75
+ 06_revisions.json 6a/6b revisions (only if any claim was revised)
76
+ 07_review.json §20 review findings + decisions (only if reviewed)
77
+ granted_calls.json §25 audit trail (only if a grant was spent)
78
+ pass_threads.json §27 one entry per pass, {"pass", "thread_id"}
79
+ confidence_chart.png Tier distribution bar chart (matplotlib)
80
+ trace.md Full trace log
81
+ report.md Final synthesis report
82
+ report.pdf PDF rendering (only if markdown+weasyprint installed)
83
+ sources/index.json §45 one entry per URL this run retrieved text
84
+ from: url, title, tool, retrieved_at, sha256,
85
+ chars. Always written when anything was
86
+ retrieved -- provenance, not content.
87
+ sources/<sha256>.txt §45 the text itself, redacted, one file per
88
+ document. Only when config.PERSIST_SOURCE_TEXT
89
+ is on (it is by default); index.json's
90
+ `persisted_text` says which.
91
+ code/ (placeholder for future code artifacts)
92
+ """
93
+ if run.run_id is None:
94
+ raise ValueError(
95
+ "Cannot write artifacts: run.run_id is None. "
96
+ "Persistence must be enabled (the pipeline "
97
+ "sets run_id via create_pipeline_run)."
98
+ )
99
+
100
+ output_dir = os.path.join(config.OUTPUT_DIR, str(run.run_id))
101
+ os.makedirs(output_dir, exist_ok=True)
102
+ os.makedirs(os.path.join(output_dir, "sources"), exist_ok=True)
103
+ os.makedirs(os.path.join(output_dir, "code"), exist_ok=True)
104
+
105
+ def write(filename, content):
106
+ with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as f:
107
+ f.write(content)
108
+
109
+ write("00_plan.md", json.dumps(run.plan, indent=2))
110
+ # E13/#77. Two or more survivors: every candidate is an equal -- none
111
+ # is "the" response, Pass 2 extracted claims from their union and the
112
+ # report is synthesised from those claims -- so naming candidate 1's
113
+ # text `01_raw_response.md` would dress a positional convention up as
114
+ # a distinction the pipeline never made. One numbered file per
115
+ # survivor, matching the numbering everywhere else. One survivor (or
116
+ # none -- ensemble mode never ran): the historical single-candidate
117
+ # shape, unchanged byte for byte.
118
+ if len(getattr(run, "candidates", []) or []) >= 2:
119
+ for entry in run.candidates:
120
+ write(f"01_candidate_{entry['candidate']}.md",
121
+ entry.get("text", ""))
122
+ else:
123
+ write("01_raw_response.md", run.raw_response)
124
+ write("02_claims.json", json.dumps([vars(c) for c in run.claims], indent=2))
125
+ write("03_grounding.json", json.dumps(
126
+ [{"claim_id": c.id, "sources": c.grounding_sources, "status": c.grounding_status}
127
+ for c in run.claims],
128
+ indent=2,
129
+ ))
130
+ write("03_critic.json", json.dumps(
131
+ [{"claim_id": c.id, "fallacies": c.fallacies, "contradictions": c.contradictions,
132
+ "severity": c.critic_severity}
133
+ for c in run.claims],
134
+ indent=2,
135
+ ))
136
+ write("03_completeness.json", json.dumps(run.completeness, indent=2))
137
+ write("04_assumptions.json", json.dumps(run.assumptions, indent=2))
138
+ write("05_confidence.json", json.dumps(
139
+ [{"claim_id": c.id, "tier": c.confidence_tier, "score_breakdown": c.score_breakdown}
140
+ for c in run.claims],
141
+ indent=2,
142
+ ))
143
+
144
+ revised = [c for c in run.claims if c.revision_text is not None]
145
+ if revised:
146
+ write("06_revisions.json", json.dumps(
147
+ [{"claim_id": c.id, "revised_text": c.revision_text, "retry_count": c.retry_count}
148
+ for c in revised],
149
+ indent=2,
150
+ ))
151
+
152
+ # §25: written ONLY when the run actually spent a grant, so its
153
+ # presence in an output directory is itself the signal that this run
154
+ # ran with pre-flight authorization. An always-empty file next to
155
+ # twelve populated ones reads as "nothing happened here" and would be
156
+ # skimmed past, which is the opposite of an audit trail's job.
157
+ if run.granted_calls:
158
+ write("granted_calls.json", json.dumps(run.granted_calls, indent=2))
159
+
160
+ # §20: same rule, same reason. Its presence in an output directory is
161
+ # itself the signal that this run was reviewed -- and on a run with no
162
+ # consent route it is the ONLY durable record of what the reviewer
163
+ # found, since nothing was applied for the claims file to show.
164
+ if run.subagent_reviews:
165
+ write("07_review.json", json.dumps(run.subagent_reviews, indent=2))
166
+
167
+ _write_sources(run, output_dir)
168
+
169
+ # §27 AC6: which thread each pass ran in. Written unconditionally
170
+ # (unlike the two above), because it is not a signal about how the run
171
+ # was configured -- every run has pass threads, and an empty file here
172
+ # would itself be the anomaly worth noticing.
173
+ write("pass_threads.json",
174
+ json.dumps(getattr(run, "pass_threads", []) or [], indent=2))
175
+
176
+ # Essential human-readable artifacts first -- a supplementary-file
177
+ # failure (chart, PDF) must not prevent these from being written.
178
+ write("trace.md", "\n".join(f"- {line}" for line in run.trace))
179
+ write("report.md", run.final_report)
180
+
181
+ # Supplementary artifacts -- failure here is non-fatal.
182
+ _write_confidence_chart(run, output_dir)
183
+ _try_write_pdf(run.final_report, output_dir)
184
+
185
+ return output_dir
186
+
187
+
188
+ def _write_sources(run: PipelineRun, output_dir: str) -> None:
189
+ """Populate sources/ with what each cited page actually served.
190
+
191
+ ROADMAP_v2 §45 (SQ8). This directory has been created and left empty
192
+ since §12, while `fetch_url.py`'s own comment described the artifact
193
+ that would fill it. Without it, `03_grounding.json` is URLs and
194
+ numbers: a reader cannot check a `similarity_score` against anything,
195
+ and neither can anyone re-running the scoring later.
196
+
197
+ ONE FILE PER DOCUMENT, named by the sha256 of its text. The hash is
198
+ also carried in the index, so a source's entry and its file are
199
+ linked, and two runs that fetched the same page produce the same
200
+ filename -- which makes them diffable.
201
+
202
+ `index.json` is written whatever `PERSIST_SOURCE_TEXT` says. Which
203
+ URLs a run retrieved, when, through which tool, and how much text they
204
+ gave is provenance rather than content: it is the part that says
205
+ whether a score COULD have been computed, and withholding it would
206
+ make an opted-out run indistinguishable from one that fetched nothing.
207
+
208
+ Degrades loudly, never fatally (#81): a run that produced a report
209
+ must not fail over an artifact directory.
210
+ """
211
+ documents = getattr(run, "source_documents", None) or []
212
+ if not documents:
213
+ return
214
+
215
+ sources_dir = os.path.join(output_dir, "sources")
216
+ keep_text = getattr(config, "PERSIST_SOURCE_TEXT", True)
217
+ try:
218
+ index = []
219
+ for document in documents:
220
+ entry = {k: v for k, v in document.items() if k != "text"}
221
+ index.append(entry)
222
+ if keep_text:
223
+ with open(os.path.join(sources_dir, f"{document['sha256']}.txt"),
224
+ "w", encoding="utf-8") as f:
225
+ f.write(document.get("text", ""))
226
+ index_payload = {"persisted_text": bool(keep_text), "sources": index}
227
+ with open(os.path.join(sources_dir, "index.json"), "w",
228
+ encoding="utf-8") as f:
229
+ json.dump(index_payload, f, indent=2)
230
+ except Exception as e:
231
+ logger.warning("Source texts skipped (write failed): %s", e)
232
+
233
+
234
+ def _write_confidence_chart(run: PipelineRun, output_dir: str) -> None:
235
+ """Bar chart of confidence tier distribution. Degrades loudly, never
236
+ fatally -- matplotlib is a project dependency (requirements.txt), but
237
+ any failure here (missing import, backend error, rendering error)
238
+ must not crash a completed pipeline run over a chart (#81). Every
239
+ degrade path warns naming the helper and the cause: a chart failing
240
+ on every run is exactly the signal worth shouting, not hiding."""
241
+ try:
242
+ import matplotlib
243
+ matplotlib.use("Agg") # headless, no display needed
244
+ import matplotlib.pyplot as plt
245
+ except Exception as e:
246
+ logger.warning(
247
+ "Confidence chart skipped (matplotlib unavailable): %s", e)
248
+ return
249
+
250
+ # #81. `fig` is bound INSIDE the try, so a failure before the binding
251
+ # used to leave `finally: plt.close(fig)` raising UnboundLocalError --
252
+ # the containment guard inverting into exactly the crash it existed to
253
+ # prevent. Binding None first makes "nothing to clean up" a state
254
+ # instead of an exception.
255
+ fig = None
256
+ try:
257
+ tiers, counts = _tier_counts(run)
258
+ fig, ax = plt.subplots()
259
+ ax.bar(tiers, counts)
260
+ ax.set_ylabel("Number of claims")
261
+ ax.set_title("Confidence tier distribution")
262
+ fig.tight_layout()
263
+ fig.savefig(os.path.join(output_dir, "confidence_chart.png"))
264
+ except Exception as e:
265
+ logger.warning("Confidence chart skipped (render failed): %s", e)
266
+ finally:
267
+ if fig is not None:
268
+ plt.close(fig)
269
+
270
+
271
+ def _try_write_pdf(report_markdown: str, output_dir: str) -> None:
272
+ """Also degrades loudly, never fatally (#81). No markdown-to-PDF
273
+ library is currently a project dependency -- install markdown +
274
+ weasyprint if you want this to actually produce output; until then
275
+ this warns and skips rather than failing the run."""
276
+ try:
277
+ import markdown
278
+ import weasyprint
279
+ except Exception as e:
280
+ logger.warning("PDF report skipped (converters unavailable): %s", e)
281
+ return
282
+ try:
283
+ html = markdown.markdown(report_markdown)
284
+ weasyprint.HTML(string=html).write_pdf(os.path.join(output_dir, "report.pdf"))
285
+ except Exception as e:
286
+ logger.warning("PDF report skipped (render failed): %s", e)
@@ -0,0 +1,398 @@
1
+ """
2
+ core/reasoning/payload_validation.py
3
+
4
+ ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
5
+
6
+ §3's retry corrects a response that does not PARSE. Nothing checked that
7
+ what parsed was the SHAPE the pass promised, so the same defect appeared
8
+ in six places at once (audit #76): a wrapped list crashed with
9
+ `'str' object has no attribute 'items'`, a bad `severity` survived two
10
+ passes and surfaced in Pass 5's pure-code tiering, and two shapes
11
+ completed a run and reported a false outcome.
12
+
13
+ WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
14
+ this validator -- `review.py:_validated` checks a top-level type, the
15
+ required keys per entry with type guards BEFORE the membership tests, an
16
+ id cross-check against `{c.id for c in run.claims}`, an enum, and a
17
+ duplicate -- and built it once, for the one payload that is NOT a pass.
18
+ The passes never got it. Putting the general version in orchestrator.py
19
+ would repeat json_retry.py's own mistake in reverse: that module was
20
+ extracted precisely so a mechanism a second consumer might want is not
21
+ trapped in the orchestrator.
22
+
23
+ `review.py` is deliberately NOT migrated onto this module (§30, B1). Its
24
+ POLICY is different and correctly so: a finding it cannot apply is
25
+ dropped and traced, because the review stage is optional and contained,
26
+ and a reviewer that misfires must not cost a completed ten-pass run. A
27
+ pass is not optional -- a payload it cannot apply is a run that will
28
+ report something untrue -- so a failure here re-enters §3's corrective
29
+ retry (B2) and ultimately raises. Same shape, two policies. Merging them
30
+ means choosing one, and neither is wrong for its own caller.
31
+
32
+ WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
33
+ declared type where a value is used arithmetically -- every one has a
34
+ demonstrated consequence in #76's table, and nothing else does:
35
+
36
+ the top-level type {"claims": [...]} instead of [...]
37
+ the required keys per entry a claim with no `id`, a revision with no
38
+ `revised_text`, Pass 4 with no
39
+ `per_claim_flags`
40
+ a declared value type `"severity": "0.0"` -- a string that
41
+ fails two passes later, in Pass 5
42
+ that entry ids resolve 3a/3b answering about ids Pass 2 never
43
+ issued, which grounded nothing while the
44
+ trace said it had grounded everything
45
+
46
+ Not a schema language, and no new dependency (#145 tracks those). A
47
+ richer vocabulary would let a spec describe things no pass has ever got
48
+ wrong, at the price of a second place where a payload's meaning lives.
49
+
50
+ STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
51
+ `Respond with ONLY this JSON structure` block in that pass's own .md, and
52
+ NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
53
+ field will fail the run rather than silently mis-apply it -- which is the
54
+ better failure -- but it will fail it at the model's expense. The parity
55
+ that IS mechanised is narrower and is the one that goes wrong silently:
56
+ every pass the orchestrator routes through `_run_pass_with_json_retry`
57
+ has a spec here (see tests/test_payload_validation.py).
58
+ """
59
+
60
+ from __future__ import annotations
61
+
62
+ from dataclasses import dataclass, field
63
+ from typing import Any, Optional
64
+
65
+ # Re-exported, not re-derived: base.py defines these beside the Literals
66
+ # they come from, and the scorer reads the same two names (§30, B9).
67
+ from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
68
+
69
+ NUMBER = (int, float)
70
+
71
+
72
+ class PayloadShapeError(ValueError):
73
+ """A pass's response parsed as JSON but is not the shape it promised.
74
+
75
+ SUBCLASSES ValueError deliberately, and that is load-bearing twice:
76
+ json_retry.py's corrective loop already catches ValueError (so a shape
77
+ failure re-enters the retry with no new except clause), and the
78
+ orchestrator's outer try/except -- which turns a failure into a
79
+ durable status='failed' record -- is byte-for-byte unchanged.
80
+
81
+ The message names the pass, the field and the expected shape, because
82
+ the whole complaint in #76's loud half is that
83
+ `'str' object has no attribute 'items'` names none of the three.
84
+ """
85
+
86
+
87
+ @dataclass(frozen=True)
88
+ class Spec:
89
+ """One pass's promised payload shape.
90
+
91
+ top "object" or "array"
92
+ required top-level keys that must be present (top="object")
93
+ key_types top-level key -> permitted Python type(s)
94
+ entry_required keys every entry must carry (top="array")
95
+ entry_types entry key -> permitted Python type(s)
96
+ enums entry key -> the set of permitted values
97
+ ids entry key holding a claim id, cross-checked against
98
+ the claims the apply step will resolve against
99
+ unique entry key that must not repeat (Pass 2's `id`)
100
+ ensemble_required entry keys required only on an ensemble run (B7)
101
+ ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
102
+ nested top-level key -> Spec for the array beneath it
103
+ """
104
+
105
+ top: str
106
+ required: frozenset = frozenset()
107
+ key_types: dict = field(default_factory=dict)
108
+ entry_required: frozenset = frozenset()
109
+ entry_types: dict = field(default_factory=dict)
110
+ enums: dict = field(default_factory=dict)
111
+ ids: Optional[str] = None
112
+ unique: Optional[str] = None
113
+ ensemble_required: frozenset = frozenset()
114
+ ids_in_keys: Optional[str] = None
115
+ nested: dict = field(default_factory=dict)
116
+
117
+
118
+ # The 3a and 3b entry shapes are named, because Pass 6b is documented in
119
+ # revalidate.md as returning exactly those two lists under two keys --
120
+ # and _apply_grounding/_apply_critic are literally the same two functions.
121
+ # Writing them twice would let 6c drift from the passes it re-runs.
122
+ _GROUNDING_ENTRY = Spec(
123
+ top="array",
124
+ entry_required=frozenset({"claim_id", "status"}),
125
+ # §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
126
+ # asymmetry is the point. B4's rule is that a check earns its place by
127
+ # having a demonstrated consequence:
128
+ #
129
+ # the type §45's scoring stage ITERATES this value. A string
130
+ # there scores one source per character, and a null
131
+ # raises inside a stage that runs after the model call
132
+ # it would need to retry.
133
+ # not required source_grounding.md says an ungrounded claim's
134
+ # `sources` "may be an empty list", and a model that
135
+ # omits the key entirely on a claim it could not
136
+ # source has said the same thing. Failing that run
137
+ # costs a corrective retry to add `[]` to an entry
138
+ # nothing reads.
139
+ #
140
+ # The case this leaves open -- `grounded` with no sources at all -- is
141
+ # NOT unhandled; it is handled where it means something. The scoring
142
+ # stage scores it as zero source quality and says so in the trace,
143
+ # which is the honest answer to a claim asserted as grounded with
144
+ # nothing behind it. Rejecting it here would instead ask the model to
145
+ # try again, and a model that had no sources still has none.
146
+ entry_types={"claim_id": str, "sources": list},
147
+ enums={"status": GROUNDING_STATUSES},
148
+ ids="claim_id",
149
+ )
150
+
151
+ _CRITIC_ENTRY = Spec(
152
+ top="array",
153
+ entry_required=frozenset({"claim_id", "severity"}),
154
+ # `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
155
+ # so a string here raises a TypeError two passes downstream, in the
156
+ # one stage that makes no model call and has nothing to retry.
157
+ entry_types={"claim_id": str, "severity": NUMBER},
158
+ ids="claim_id",
159
+ )
160
+
161
+
162
+ SPECS = {
163
+ # preliminary_plan.md names five keys, and the orchestrator reads ONE
164
+ # of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
165
+ # before handing the plan whole to Pass 3c. So requiring all five
166
+ # would fail runs over an omission nothing depends on. The
167
+ # demonstrated defect is a bare list, which is what `top` catches.
168
+ "Pass 0": Spec(top="object"),
169
+
170
+ "Pass 2": Spec(
171
+ top="array",
172
+ entry_required=frozenset({"id", "text", "type"}),
173
+ entry_types={"id": str, "text": str},
174
+ enums={"type": CLAIM_TYPES},
175
+ # B6. Two claims sharing an id split in half: _apply_grounding
176
+ # builds {c.id: c} (last wins) while _apply_assumption_flags uses
177
+ # next(...) (first wins), so the grounding landed on one object
178
+ # and the assumption flags on the other. claim_extraction.md asks
179
+ # for "sequential zero-padded ids" and never says they must be
180
+ # unique.
181
+ unique="id",
182
+ # B7. Only in ensemble mode, and only because claim_extraction.md
183
+ # is now asked to supply it on every claim there. An ABSENT field
184
+ # is indistinguishable from an empty one at the scorer, and
185
+ # `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
186
+ # a claim that exists because a candidate asserted it.
187
+ ensemble_required=frozenset({"asserted_by_candidates"}),
188
+ ),
189
+
190
+ "Pass 3a": _GROUNDING_ENTRY,
191
+ "Pass 3b": _CRITIC_ENTRY,
192
+
193
+ "Pass 3c": Spec(
194
+ top="object",
195
+ required=frozenset({"gaps", "coverage_score"}),
196
+ # `gaps` is iterated by build_coverage_gap_entries; a string there
197
+ # would produce one coverage-gap entry per CHARACTER.
198
+ key_types={"gaps": list},
199
+ ),
200
+
201
+ "Pass 4": Spec(
202
+ top="object",
203
+ required=frozenset({"per_claim_flags"}),
204
+ key_types={"per_claim_flags": dict},
205
+ ids_in_keys="per_claim_flags",
206
+ ),
207
+
208
+ "Pass 6a": Spec(
209
+ top="array",
210
+ entry_required=frozenset({"claim_id", "revised_text"}),
211
+ entry_types={"claim_id": str, "revised_text": str},
212
+ ids="claim_id",
213
+ ),
214
+
215
+ "Pass 6b": Spec(
216
+ top="object",
217
+ required=frozenset({"grounding", "critic"}),
218
+ key_types={"grounding": list, "critic": list},
219
+ nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
220
+ ),
221
+ }
222
+
223
+
224
+ def _type_name(value: Any) -> str:
225
+ """JSON's vocabulary, not Python's -- the message goes back to a model
226
+ that was asked for JSON, so `object`/`array` is what it can act on."""
227
+ return {dict: "object", list: "array", str: "string",
228
+ bool: "boolean", int: "number", float: "number",
229
+ type(None): "null"}.get(type(value), type(value).__name__)
230
+
231
+
232
+ def _permitted(types) -> str:
233
+ if not isinstance(types, tuple):
234
+ types = (types,)
235
+ return " or ".join(sorted({_type_name(t()) for t in types}))
236
+
237
+
238
+ def validate(pass_id: str, payload: Any, claim_ids=None,
239
+ ensemble: bool = False) -> None:
240
+ """Raise PayloadShapeError unless `payload` is what `pass_id` promised.
241
+
242
+ claim_ids the ids the APPLY step will resolve against -- all claims
243
+ for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
244
+ "every claim in the run": Pass 6a is only asked about
245
+ `flagged`, and its own lookup already resolves within that
246
+ batch, so validating against a wider set would accept a
247
+ revision that then lands on nothing. None skips the check.
248
+ ensemble True on an ensemble run, which is the only place Pass 2 is
249
+ asked for `asserted_by_candidates` (B7).
250
+
251
+ An unknown pass_id RAISES rather than passing through unvalidated. A
252
+ pass with no spec is the silent hole this module exists to close, and
253
+ the eleventh pass must not be able to arrive by forgetting a line.
254
+ """
255
+ spec = SPECS.get(pass_id)
256
+ if spec is None:
257
+ raise PayloadShapeError(
258
+ f"{pass_id}: no payload spec is declared for this pass. Every "
259
+ f"pass routed through _run_pass_with_json_retry needs one in "
260
+ f"core/reasoning/payload_validation.py (§30, B11)."
261
+ )
262
+ _validate_against(pass_id, payload, spec, claim_ids, ensemble)
263
+
264
+
265
+ def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
266
+ where: str = "") -> None:
267
+ at = f" under {where!r}" if where else ""
268
+
269
+ if spec.top == "array":
270
+ if not isinstance(payload, list):
271
+ raise PayloadShapeError(
272
+ f"{pass_id}: expected a JSON array{at}, got "
273
+ f"{_type_name(payload)}. Return the array itself, not an "
274
+ f"object wrapping it."
275
+ )
276
+ _validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
277
+ return
278
+
279
+ if not isinstance(payload, dict):
280
+ raise PayloadShapeError(
281
+ f"{pass_id}: expected a JSON object{at}, got "
282
+ f"{_type_name(payload)}."
283
+ )
284
+
285
+ missing = sorted(spec.required - set(payload))
286
+ if missing:
287
+ raise PayloadShapeError(
288
+ f"{pass_id}: the response{at} is missing required key(s) "
289
+ f"{', '.join(repr(m) for m in missing)}. Expected an object "
290
+ f"with {', '.join(sorted(spec.required))}."
291
+ )
292
+
293
+ for key, types in spec.key_types.items():
294
+ if key in payload and not isinstance(payload[key], types):
295
+ raise PayloadShapeError(
296
+ f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
297
+ f"expected {_permitted(types)}."
298
+ )
299
+
300
+ for key, nested in spec.nested.items():
301
+ _validate_against(pass_id, payload[key], nested, claim_ids,
302
+ ensemble, where=key)
303
+
304
+ if spec.ids_in_keys:
305
+ _check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
306
+ f"key of {spec.ids_in_keys!r}")
307
+
308
+
309
+ def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
310
+ required = spec.entry_required | (spec.ensemble_required if ensemble
311
+ else frozenset())
312
+ seen = {}
313
+ for index, entry in enumerate(entries):
314
+ if not isinstance(entry, dict):
315
+ raise PayloadShapeError(
316
+ f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
317
+ f"every entry must be an object with "
318
+ f"{', '.join(sorted(required))}."
319
+ )
320
+ missing = sorted(required - set(entry))
321
+ if missing:
322
+ raise PayloadShapeError(
323
+ f"{pass_id}: entry {index}{at} is missing required key(s) "
324
+ f"{', '.join(repr(m) for m in missing)}. Every entry needs "
325
+ f"{', '.join(sorted(required))}."
326
+ )
327
+ # TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
328
+ # rule and the reason it is worth copying: a JSON array or object
329
+ # in an enum-checked field is unhashable, so `value in ALLOWED`
330
+ # raises TypeError from inside the validator rather than
331
+ # producing a message the model can act on.
332
+ for key, types in spec.entry_types.items():
333
+ if key in entry and not isinstance(entry[key], types):
334
+ raise PayloadShapeError(
335
+ f"{pass_id}: entry {index}{at} has {key}="
336
+ f"{entry[key]!r}, which is {_type_name(entry[key])}; "
337
+ f"expected {_permitted(types)}."
338
+ )
339
+ for key, allowed in spec.enums.items():
340
+ if key not in entry:
341
+ continue
342
+ value = entry[key]
343
+ if not isinstance(value, str) or value not in allowed:
344
+ raise PayloadShapeError(
345
+ f"{pass_id}: entry {index}{at} has {key}={value!r}; "
346
+ f"expected exactly one of "
347
+ f"{', '.join(sorted(allowed))} (lower case)."
348
+ )
349
+ if spec.unique:
350
+ value = entry[spec.unique]
351
+ if value in seen:
352
+ raise PayloadShapeError(
353
+ f"{pass_id}: entries {seen[value]} and {index}{at} "
354
+ f"share {spec.unique}={value!r}. Ids must be unique -- "
355
+ f"two claims with one id are applied to inconsistently "
356
+ f"and end up carrying half of each other's results."
357
+ )
358
+ seen[value] = index
359
+
360
+ if spec.ids:
361
+ _check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
362
+ spec.ids)
363
+
364
+
365
+ def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
366
+ """B5: a TOTAL mismatch is the error; a partial one is not.
367
+
368
+ The demonstrated silent failure is every id being wrong -- 3a/3b
369
+ answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
370
+ which grounded nothing while the checkpoint line reported that it had
371
+ grounded everything. That is a pass with no effect at all, and it is
372
+ worth a corrective retry.
373
+
374
+ A model that gets nine of ten right is NOT worth failing a run that
375
+ has already paid for nine passes: the nine apply, and the checkpoint
376
+ line reports the count APPLIED rather than the count asked about, so
377
+ the tenth is visible instead of silent (see the orchestrator's
378
+ _apply_* return values).
379
+
380
+ An EMPTY collection is not a mismatch. assumption_audit.md asks the
381
+ model to "omit claims with no issues rather than including them with
382
+ an empty list", so no flags at all is the ordinary answer for a clean
383
+ run -- and Pass 4's demonstrated defect was the top-level key being
384
+ absent, which the required-key check above catches instead.
385
+ """
386
+ if claim_ids is None or not supplied:
387
+ return
388
+ known = set(claim_ids)
389
+ if any(value in known for value in supplied):
390
+ return
391
+ sample = ", ".join(repr(v) for v in supplied[:3])
392
+ expected = ", ".join(repr(v) for v in sorted(known)[:3])
393
+ raise PayloadShapeError(
394
+ f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
395
+ f"matches a claim you were given (got {sample}; expected ids like "
396
+ f"{expected}). Use the claim ids exactly as they appear in the "
397
+ f"input."
398
+ )