@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/output_writer.py
|
|
3
|
+
|
|
4
|
+
Writes every pass's output to OUTPUT_DIR/<run_id>/, matching the
|
|
5
|
+
original pipeline spec's file layout (ROADMAP §12). NOT called
|
|
6
|
+
automatically from the pipeline -- it stays
|
|
7
|
+
a pure computation returning a PipelineRun; writing files (or not) is
|
|
8
|
+
the CALLER's decision (see main.py's research mode, §1). This keeps the
|
|
9
|
+
pipeline function's side effects minimal and makes it trivially testable
|
|
10
|
+
without touching a filesystem.
|
|
11
|
+
|
|
12
|
+
Relationship to §5's pipeline_storage.py: that module is for
|
|
13
|
+
programmatic resume/inspection (a database row, checked by run_id).
|
|
14
|
+
This module is for a human-browsable directory on disk -- a different
|
|
15
|
+
artifact, for a different audience, both keyed by the same run.run_id
|
|
16
|
+
so they correlate.
|
|
17
|
+
|
|
18
|
+
Claims are serialized with vars(c), matching every other
|
|
19
|
+
claim-serialization call site in this codebase (orchestrator.py passes
|
|
20
|
+
3a/3b/4/6a/6b/final synthesis, pipeline_storage.update_pipeline_run,
|
|
21
|
+
and this module's write_run_artifacts). If a nested-dataclass field is
|
|
22
|
+
ever added to Claim, ALL of those sites migrate to asdict(c) together
|
|
23
|
+
-- see the note on Claim in core/reasoning/base.py.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
import os
|
|
31
|
+
from typing import TYPE_CHECKING
|
|
32
|
+
|
|
33
|
+
import config
|
|
34
|
+
|
|
35
|
+
if TYPE_CHECKING:
|
|
36
|
+
from core.reasoning.base import PipelineRun
|
|
37
|
+
|
|
38
|
+
logger = logging.getLogger(__name__)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _tier_counts(run: PipelineRun) -> tuple[list[str], list[int]]:
|
|
42
|
+
"""Returns (tier_labels, counts) for the confidence chart. Extracted
|
|
43
|
+
as a pure function so the counting logic can be unit-tested without
|
|
44
|
+
rendering a PNG."""
|
|
45
|
+
tiers = ["HIGH", "MEDIUM", "LOW", "UNVERIFIED"]
|
|
46
|
+
counts = [sum(1 for c in run.claims if c.confidence_tier == t) for t in tiers]
|
|
47
|
+
tiers.append("UNVERIFIED_COVERAGE")
|
|
48
|
+
counts.append(len(run.coverage_gaps))
|
|
49
|
+
return tiers, counts
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def write_run_artifacts(run: PipelineRun) -> str:
|
|
53
|
+
"""Writes the full artifact directory for one pipeline run and
|
|
54
|
+
returns the output directory path.
|
|
55
|
+
|
|
56
|
+
Layout under OUTPUT_DIR/<run_id>/:
|
|
57
|
+
00_plan.md Pass 0 plan (JSON)
|
|
58
|
+
01_raw_response.md Pass 1 raw generation (single-candidate runs)
|
|
59
|
+
01_candidate_N.md E13/#77: per-candidate generation, one file
|
|
60
|
+
per SURVIVOR, written instead of
|
|
61
|
+
01_raw_response.md whenever two or more
|
|
62
|
+
survived -- the N matches the trace's
|
|
63
|
+
candidate-N lines and the claims'
|
|
64
|
+
asserted_by_candidates tags, so the record
|
|
65
|
+
can substantiate its own consistency scores.
|
|
66
|
+
A run degraded to one survivor names its
|
|
67
|
+
file 01_raw_response.md like any other
|
|
68
|
+
single-candidate run.
|
|
69
|
+
02_claims.json Pass 2 extracted claims (full vars(c))
|
|
70
|
+
03_grounding.json Pass 3a grounding per claim
|
|
71
|
+
03_critic.json Pass 3b critic per claim
|
|
72
|
+
03_completeness.json Pass 3c completeness findings
|
|
73
|
+
04_assumptions.json Pass 4 assumption audit
|
|
74
|
+
05_confidence.json Pass 5 tier + score breakdown per claim
|
|
75
|
+
06_revisions.json 6a/6b revisions (only if any claim was revised)
|
|
76
|
+
07_review.json §20 review findings + decisions (only if reviewed)
|
|
77
|
+
granted_calls.json §25 audit trail (only if a grant was spent)
|
|
78
|
+
pass_threads.json §27 one entry per pass, {"pass", "thread_id"}
|
|
79
|
+
confidence_chart.png Tier distribution bar chart (matplotlib)
|
|
80
|
+
trace.md Full trace log
|
|
81
|
+
report.md Final synthesis report
|
|
82
|
+
report.pdf PDF rendering (only if markdown+weasyprint installed)
|
|
83
|
+
sources/index.json §45 one entry per URL this run retrieved text
|
|
84
|
+
from: url, title, tool, retrieved_at, sha256,
|
|
85
|
+
chars. Always written when anything was
|
|
86
|
+
retrieved -- provenance, not content.
|
|
87
|
+
sources/<sha256>.txt §45 the text itself, redacted, one file per
|
|
88
|
+
document. Only when config.PERSIST_SOURCE_TEXT
|
|
89
|
+
is on (it is by default); index.json's
|
|
90
|
+
`persisted_text` says which.
|
|
91
|
+
code/ (placeholder for future code artifacts)
|
|
92
|
+
"""
|
|
93
|
+
if run.run_id is None:
|
|
94
|
+
raise ValueError(
|
|
95
|
+
"Cannot write artifacts: run.run_id is None. "
|
|
96
|
+
"Persistence must be enabled (the pipeline "
|
|
97
|
+
"sets run_id via create_pipeline_run)."
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
output_dir = os.path.join(config.OUTPUT_DIR, str(run.run_id))
|
|
101
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
102
|
+
os.makedirs(os.path.join(output_dir, "sources"), exist_ok=True)
|
|
103
|
+
os.makedirs(os.path.join(output_dir, "code"), exist_ok=True)
|
|
104
|
+
|
|
105
|
+
def write(filename, content):
|
|
106
|
+
with open(os.path.join(output_dir, filename), "w", encoding="utf-8") as f:
|
|
107
|
+
f.write(content)
|
|
108
|
+
|
|
109
|
+
write("00_plan.md", json.dumps(run.plan, indent=2))
|
|
110
|
+
# E13/#77. Two or more survivors: every candidate is an equal -- none
|
|
111
|
+
# is "the" response, Pass 2 extracted claims from their union and the
|
|
112
|
+
# report is synthesised from those claims -- so naming candidate 1's
|
|
113
|
+
# text `01_raw_response.md` would dress a positional convention up as
|
|
114
|
+
# a distinction the pipeline never made. One numbered file per
|
|
115
|
+
# survivor, matching the numbering everywhere else. One survivor (or
|
|
116
|
+
# none -- ensemble mode never ran): the historical single-candidate
|
|
117
|
+
# shape, unchanged byte for byte.
|
|
118
|
+
if len(getattr(run, "candidates", []) or []) >= 2:
|
|
119
|
+
for entry in run.candidates:
|
|
120
|
+
write(f"01_candidate_{entry['candidate']}.md",
|
|
121
|
+
entry.get("text", ""))
|
|
122
|
+
else:
|
|
123
|
+
write("01_raw_response.md", run.raw_response)
|
|
124
|
+
write("02_claims.json", json.dumps([vars(c) for c in run.claims], indent=2))
|
|
125
|
+
write("03_grounding.json", json.dumps(
|
|
126
|
+
[{"claim_id": c.id, "sources": c.grounding_sources, "status": c.grounding_status}
|
|
127
|
+
for c in run.claims],
|
|
128
|
+
indent=2,
|
|
129
|
+
))
|
|
130
|
+
write("03_critic.json", json.dumps(
|
|
131
|
+
[{"claim_id": c.id, "fallacies": c.fallacies, "contradictions": c.contradictions,
|
|
132
|
+
"severity": c.critic_severity}
|
|
133
|
+
for c in run.claims],
|
|
134
|
+
indent=2,
|
|
135
|
+
))
|
|
136
|
+
write("03_completeness.json", json.dumps(run.completeness, indent=2))
|
|
137
|
+
write("04_assumptions.json", json.dumps(run.assumptions, indent=2))
|
|
138
|
+
write("05_confidence.json", json.dumps(
|
|
139
|
+
[{"claim_id": c.id, "tier": c.confidence_tier, "score_breakdown": c.score_breakdown}
|
|
140
|
+
for c in run.claims],
|
|
141
|
+
indent=2,
|
|
142
|
+
))
|
|
143
|
+
|
|
144
|
+
revised = [c for c in run.claims if c.revision_text is not None]
|
|
145
|
+
if revised:
|
|
146
|
+
write("06_revisions.json", json.dumps(
|
|
147
|
+
[{"claim_id": c.id, "revised_text": c.revision_text, "retry_count": c.retry_count}
|
|
148
|
+
for c in revised],
|
|
149
|
+
indent=2,
|
|
150
|
+
))
|
|
151
|
+
|
|
152
|
+
# §25: written ONLY when the run actually spent a grant, so its
|
|
153
|
+
# presence in an output directory is itself the signal that this run
|
|
154
|
+
# ran with pre-flight authorization. An always-empty file next to
|
|
155
|
+
# twelve populated ones reads as "nothing happened here" and would be
|
|
156
|
+
# skimmed past, which is the opposite of an audit trail's job.
|
|
157
|
+
if run.granted_calls:
|
|
158
|
+
write("granted_calls.json", json.dumps(run.granted_calls, indent=2))
|
|
159
|
+
|
|
160
|
+
# §20: same rule, same reason. Its presence in an output directory is
|
|
161
|
+
# itself the signal that this run was reviewed -- and on a run with no
|
|
162
|
+
# consent route it is the ONLY durable record of what the reviewer
|
|
163
|
+
# found, since nothing was applied for the claims file to show.
|
|
164
|
+
if run.subagent_reviews:
|
|
165
|
+
write("07_review.json", json.dumps(run.subagent_reviews, indent=2))
|
|
166
|
+
|
|
167
|
+
_write_sources(run, output_dir)
|
|
168
|
+
|
|
169
|
+
# §27 AC6: which thread each pass ran in. Written unconditionally
|
|
170
|
+
# (unlike the two above), because it is not a signal about how the run
|
|
171
|
+
# was configured -- every run has pass threads, and an empty file here
|
|
172
|
+
# would itself be the anomaly worth noticing.
|
|
173
|
+
write("pass_threads.json",
|
|
174
|
+
json.dumps(getattr(run, "pass_threads", []) or [], indent=2))
|
|
175
|
+
|
|
176
|
+
# Essential human-readable artifacts first -- a supplementary-file
|
|
177
|
+
# failure (chart, PDF) must not prevent these from being written.
|
|
178
|
+
write("trace.md", "\n".join(f"- {line}" for line in run.trace))
|
|
179
|
+
write("report.md", run.final_report)
|
|
180
|
+
|
|
181
|
+
# Supplementary artifacts -- failure here is non-fatal.
|
|
182
|
+
_write_confidence_chart(run, output_dir)
|
|
183
|
+
_try_write_pdf(run.final_report, output_dir)
|
|
184
|
+
|
|
185
|
+
return output_dir
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _write_sources(run: PipelineRun, output_dir: str) -> None:
|
|
189
|
+
"""Populate sources/ with what each cited page actually served.
|
|
190
|
+
|
|
191
|
+
ROADMAP_v2 §45 (SQ8). This directory has been created and left empty
|
|
192
|
+
since §12, while `fetch_url.py`'s own comment described the artifact
|
|
193
|
+
that would fill it. Without it, `03_grounding.json` is URLs and
|
|
194
|
+
numbers: a reader cannot check a `similarity_score` against anything,
|
|
195
|
+
and neither can anyone re-running the scoring later.
|
|
196
|
+
|
|
197
|
+
ONE FILE PER DOCUMENT, named by the sha256 of its text. The hash is
|
|
198
|
+
also carried in the index, so a source's entry and its file are
|
|
199
|
+
linked, and two runs that fetched the same page produce the same
|
|
200
|
+
filename -- which makes them diffable.
|
|
201
|
+
|
|
202
|
+
`index.json` is written whatever `PERSIST_SOURCE_TEXT` says. Which
|
|
203
|
+
URLs a run retrieved, when, through which tool, and how much text they
|
|
204
|
+
gave is provenance rather than content: it is the part that says
|
|
205
|
+
whether a score COULD have been computed, and withholding it would
|
|
206
|
+
make an opted-out run indistinguishable from one that fetched nothing.
|
|
207
|
+
|
|
208
|
+
Degrades loudly, never fatally (#81): a run that produced a report
|
|
209
|
+
must not fail over an artifact directory.
|
|
210
|
+
"""
|
|
211
|
+
documents = getattr(run, "source_documents", None) or []
|
|
212
|
+
if not documents:
|
|
213
|
+
return
|
|
214
|
+
|
|
215
|
+
sources_dir = os.path.join(output_dir, "sources")
|
|
216
|
+
keep_text = getattr(config, "PERSIST_SOURCE_TEXT", True)
|
|
217
|
+
try:
|
|
218
|
+
index = []
|
|
219
|
+
for document in documents:
|
|
220
|
+
entry = {k: v for k, v in document.items() if k != "text"}
|
|
221
|
+
index.append(entry)
|
|
222
|
+
if keep_text:
|
|
223
|
+
with open(os.path.join(sources_dir, f"{document['sha256']}.txt"),
|
|
224
|
+
"w", encoding="utf-8") as f:
|
|
225
|
+
f.write(document.get("text", ""))
|
|
226
|
+
index_payload = {"persisted_text": bool(keep_text), "sources": index}
|
|
227
|
+
with open(os.path.join(sources_dir, "index.json"), "w",
|
|
228
|
+
encoding="utf-8") as f:
|
|
229
|
+
json.dump(index_payload, f, indent=2)
|
|
230
|
+
except Exception as e:
|
|
231
|
+
logger.warning("Source texts skipped (write failed): %s", e)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def _write_confidence_chart(run: PipelineRun, output_dir: str) -> None:
|
|
235
|
+
"""Bar chart of confidence tier distribution. Degrades loudly, never
|
|
236
|
+
fatally -- matplotlib is a project dependency (requirements.txt), but
|
|
237
|
+
any failure here (missing import, backend error, rendering error)
|
|
238
|
+
must not crash a completed pipeline run over a chart (#81). Every
|
|
239
|
+
degrade path warns naming the helper and the cause: a chart failing
|
|
240
|
+
on every run is exactly the signal worth shouting, not hiding."""
|
|
241
|
+
try:
|
|
242
|
+
import matplotlib
|
|
243
|
+
matplotlib.use("Agg") # headless, no display needed
|
|
244
|
+
import matplotlib.pyplot as plt
|
|
245
|
+
except Exception as e:
|
|
246
|
+
logger.warning(
|
|
247
|
+
"Confidence chart skipped (matplotlib unavailable): %s", e)
|
|
248
|
+
return
|
|
249
|
+
|
|
250
|
+
# #81. `fig` is bound INSIDE the try, so a failure before the binding
|
|
251
|
+
# used to leave `finally: plt.close(fig)` raising UnboundLocalError --
|
|
252
|
+
# the containment guard inverting into exactly the crash it existed to
|
|
253
|
+
# prevent. Binding None first makes "nothing to clean up" a state
|
|
254
|
+
# instead of an exception.
|
|
255
|
+
fig = None
|
|
256
|
+
try:
|
|
257
|
+
tiers, counts = _tier_counts(run)
|
|
258
|
+
fig, ax = plt.subplots()
|
|
259
|
+
ax.bar(tiers, counts)
|
|
260
|
+
ax.set_ylabel("Number of claims")
|
|
261
|
+
ax.set_title("Confidence tier distribution")
|
|
262
|
+
fig.tight_layout()
|
|
263
|
+
fig.savefig(os.path.join(output_dir, "confidence_chart.png"))
|
|
264
|
+
except Exception as e:
|
|
265
|
+
logger.warning("Confidence chart skipped (render failed): %s", e)
|
|
266
|
+
finally:
|
|
267
|
+
if fig is not None:
|
|
268
|
+
plt.close(fig)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def _try_write_pdf(report_markdown: str, output_dir: str) -> None:
|
|
272
|
+
"""Also degrades loudly, never fatally (#81). No markdown-to-PDF
|
|
273
|
+
library is currently a project dependency -- install markdown +
|
|
274
|
+
weasyprint if you want this to actually produce output; until then
|
|
275
|
+
this warns and skips rather than failing the run."""
|
|
276
|
+
try:
|
|
277
|
+
import markdown
|
|
278
|
+
import weasyprint
|
|
279
|
+
except Exception as e:
|
|
280
|
+
logger.warning("PDF report skipped (converters unavailable): %s", e)
|
|
281
|
+
return
|
|
282
|
+
try:
|
|
283
|
+
html = markdown.markdown(report_markdown)
|
|
284
|
+
weasyprint.HTML(string=html).write_pdf(os.path.join(output_dir, "report.pdf"))
|
|
285
|
+
except Exception as e:
|
|
286
|
+
logger.warning("PDF report skipped (render failed): %s", e)
|
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/payload_validation.py
|
|
3
|
+
|
|
4
|
+
ROADMAP_v2 §30 (B1). One shape boundary for the ten research passes.
|
|
5
|
+
|
|
6
|
+
§3's retry corrects a response that does not PARSE. Nothing checked that
|
|
7
|
+
what parsed was the SHAPE the pass promised, so the same defect appeared
|
|
8
|
+
in six places at once (audit #76): a wrapped list crashed with
|
|
9
|
+
`'str' object has no attribute 'items'`, a bad `severity` survived two
|
|
10
|
+
passes and surfaced in Pass 5's pure-code tiering, and two shapes
|
|
11
|
+
completed a run and reported a false outcome.
|
|
12
|
+
|
|
13
|
+
WHY THIS IS ITS OWN MODULE, AND WHY IT LOOKS FAMILIAR. §20 already built
|
|
14
|
+
this validator -- `review.py:_validated` checks a top-level type, the
|
|
15
|
+
required keys per entry with type guards BEFORE the membership tests, an
|
|
16
|
+
id cross-check against `{c.id for c in run.claims}`, an enum, and a
|
|
17
|
+
duplicate -- and built it once, for the one payload that is NOT a pass.
|
|
18
|
+
The passes never got it. Putting the general version in orchestrator.py
|
|
19
|
+
would repeat json_retry.py's own mistake in reverse: that module was
|
|
20
|
+
extracted precisely so a mechanism a second consumer might want is not
|
|
21
|
+
trapped in the orchestrator.
|
|
22
|
+
|
|
23
|
+
`review.py` is deliberately NOT migrated onto this module (§30, B1). Its
|
|
24
|
+
POLICY is different and correctly so: a finding it cannot apply is
|
|
25
|
+
dropped and traced, because the review stage is optional and contained,
|
|
26
|
+
and a reviewer that misfires must not cost a completed ten-pass run. A
|
|
27
|
+
pass is not optional -- a payload it cannot apply is a run that will
|
|
28
|
+
report something untrue -- so a failure here re-enters §3's corrective
|
|
29
|
+
retry (B2) and ultimately raises. Same shape, two policies. Merging them
|
|
30
|
+
means choosing one, and neither is wrong for its own caller.
|
|
31
|
+
|
|
32
|
+
WHAT IS CHECKED, AND WHY ONLY THIS (B4). Three properties, plus a
|
|
33
|
+
declared type where a value is used arithmetically -- every one has a
|
|
34
|
+
demonstrated consequence in #76's table, and nothing else does:
|
|
35
|
+
|
|
36
|
+
the top-level type {"claims": [...]} instead of [...]
|
|
37
|
+
the required keys per entry a claim with no `id`, a revision with no
|
|
38
|
+
`revised_text`, Pass 4 with no
|
|
39
|
+
`per_claim_flags`
|
|
40
|
+
a declared value type `"severity": "0.0"` -- a string that
|
|
41
|
+
fails two passes later, in Pass 5
|
|
42
|
+
that entry ids resolve 3a/3b answering about ids Pass 2 never
|
|
43
|
+
issued, which grounded nothing while the
|
|
44
|
+
trace said it had grounded everything
|
|
45
|
+
|
|
46
|
+
Not a schema language, and no new dependency (#145 tracks those). A
|
|
47
|
+
richer vocabulary would let a spec describe things no pass has ever got
|
|
48
|
+
wrong, at the price of a second place where a payload's meaning lives.
|
|
49
|
+
|
|
50
|
+
STATED LIMIT, NOT FIXED (B11). Every spec below is a transcription of the
|
|
51
|
+
`Respond with ONLY this JSON structure` block in that pass's own .md, and
|
|
52
|
+
NOTHING CHECKS THAT THE TWO STILL AGREE. A prompt edit that renames a
|
|
53
|
+
field will fail the run rather than silently mis-apply it -- which is the
|
|
54
|
+
better failure -- but it will fail it at the model's expense. The parity
|
|
55
|
+
that IS mechanised is narrower and is the one that goes wrong silently:
|
|
56
|
+
every pass the orchestrator routes through `_run_pass_with_json_retry`
|
|
57
|
+
has a spec here (see tests/test_payload_validation.py).
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
from __future__ import annotations
|
|
61
|
+
|
|
62
|
+
from dataclasses import dataclass, field
|
|
63
|
+
from typing import Any, Optional
|
|
64
|
+
|
|
65
|
+
# Re-exported, not re-derived: base.py defines these beside the Literals
|
|
66
|
+
# they come from, and the scorer reads the same two names (§30, B9).
|
|
67
|
+
from core.reasoning.base import CLAIM_TYPES, GROUNDING_STATUSES # noqa: F401
|
|
68
|
+
|
|
69
|
+
NUMBER = (int, float)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class PayloadShapeError(ValueError):
|
|
73
|
+
"""A pass's response parsed as JSON but is not the shape it promised.
|
|
74
|
+
|
|
75
|
+
SUBCLASSES ValueError deliberately, and that is load-bearing twice:
|
|
76
|
+
json_retry.py's corrective loop already catches ValueError (so a shape
|
|
77
|
+
failure re-enters the retry with no new except clause), and the
|
|
78
|
+
orchestrator's outer try/except -- which turns a failure into a
|
|
79
|
+
durable status='failed' record -- is byte-for-byte unchanged.
|
|
80
|
+
|
|
81
|
+
The message names the pass, the field and the expected shape, because
|
|
82
|
+
the whole complaint in #76's loud half is that
|
|
83
|
+
`'str' object has no attribute 'items'` names none of the three.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
@dataclass(frozen=True)
|
|
88
|
+
class Spec:
|
|
89
|
+
"""One pass's promised payload shape.
|
|
90
|
+
|
|
91
|
+
top "object" or "array"
|
|
92
|
+
required top-level keys that must be present (top="object")
|
|
93
|
+
key_types top-level key -> permitted Python type(s)
|
|
94
|
+
entry_required keys every entry must carry (top="array")
|
|
95
|
+
entry_types entry key -> permitted Python type(s)
|
|
96
|
+
enums entry key -> the set of permitted values
|
|
97
|
+
ids entry key holding a claim id, cross-checked against
|
|
98
|
+
the claims the apply step will resolve against
|
|
99
|
+
unique entry key that must not repeat (Pass 2's `id`)
|
|
100
|
+
ensemble_required entry keys required only on an ensemble run (B7)
|
|
101
|
+
ids_in_keys top-level key whose KEYS are claim ids (Pass 4)
|
|
102
|
+
nested top-level key -> Spec for the array beneath it
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
top: str
|
|
106
|
+
required: frozenset = frozenset()
|
|
107
|
+
key_types: dict = field(default_factory=dict)
|
|
108
|
+
entry_required: frozenset = frozenset()
|
|
109
|
+
entry_types: dict = field(default_factory=dict)
|
|
110
|
+
enums: dict = field(default_factory=dict)
|
|
111
|
+
ids: Optional[str] = None
|
|
112
|
+
unique: Optional[str] = None
|
|
113
|
+
ensemble_required: frozenset = frozenset()
|
|
114
|
+
ids_in_keys: Optional[str] = None
|
|
115
|
+
nested: dict = field(default_factory=dict)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
# The 3a and 3b entry shapes are named, because Pass 6b is documented in
|
|
119
|
+
# revalidate.md as returning exactly those two lists under two keys --
|
|
120
|
+
# and _apply_grounding/_apply_critic are literally the same two functions.
|
|
121
|
+
# Writing them twice would let 6c drift from the passes it re-runs.
|
|
122
|
+
_GROUNDING_ENTRY = Spec(
|
|
123
|
+
top="array",
|
|
124
|
+
entry_required=frozenset({"claim_id", "status"}),
|
|
125
|
+
# §45 (SQ2). `sources` is TYPE-GUARDED BUT NOT REQUIRED, and the
|
|
126
|
+
# asymmetry is the point. B4's rule is that a check earns its place by
|
|
127
|
+
# having a demonstrated consequence:
|
|
128
|
+
#
|
|
129
|
+
# the type §45's scoring stage ITERATES this value. A string
|
|
130
|
+
# there scores one source per character, and a null
|
|
131
|
+
# raises inside a stage that runs after the model call
|
|
132
|
+
# it would need to retry.
|
|
133
|
+
# not required source_grounding.md says an ungrounded claim's
|
|
134
|
+
# `sources` "may be an empty list", and a model that
|
|
135
|
+
# omits the key entirely on a claim it could not
|
|
136
|
+
# source has said the same thing. Failing that run
|
|
137
|
+
# costs a corrective retry to add `[]` to an entry
|
|
138
|
+
# nothing reads.
|
|
139
|
+
#
|
|
140
|
+
# The case this leaves open -- `grounded` with no sources at all -- is
|
|
141
|
+
# NOT unhandled; it is handled where it means something. The scoring
|
|
142
|
+
# stage scores it as zero source quality and says so in the trace,
|
|
143
|
+
# which is the honest answer to a claim asserted as grounded with
|
|
144
|
+
# nothing behind it. Rejecting it here would instead ask the model to
|
|
145
|
+
# try again, and a model that had no sources still has none.
|
|
146
|
+
entry_types={"claim_id": str, "sources": list},
|
|
147
|
+
enums={"status": GROUNDING_STATUSES},
|
|
148
|
+
ids="claim_id",
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
_CRITIC_ENTRY = Spec(
|
|
152
|
+
top="array",
|
|
153
|
+
entry_required=frozenset({"claim_id", "severity"}),
|
|
154
|
+
# `severity` is the arithmetic one: Pass 5 computes `1.0 - severity`,
|
|
155
|
+
# so a string here raises a TypeError two passes downstream, in the
|
|
156
|
+
# one stage that makes no model call and has nothing to retry.
|
|
157
|
+
entry_types={"claim_id": str, "severity": NUMBER},
|
|
158
|
+
ids="claim_id",
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
SPECS = {
|
|
163
|
+
# preliminary_plan.md names five keys, and the orchestrator reads ONE
|
|
164
|
+
# of them defensively (`run.plan.get("key_entities_or_subjects", [])`)
|
|
165
|
+
# before handing the plan whole to Pass 3c. So requiring all five
|
|
166
|
+
# would fail runs over an omission nothing depends on. The
|
|
167
|
+
# demonstrated defect is a bare list, which is what `top` catches.
|
|
168
|
+
"Pass 0": Spec(top="object"),
|
|
169
|
+
|
|
170
|
+
"Pass 2": Spec(
|
|
171
|
+
top="array",
|
|
172
|
+
entry_required=frozenset({"id", "text", "type"}),
|
|
173
|
+
entry_types={"id": str, "text": str},
|
|
174
|
+
enums={"type": CLAIM_TYPES},
|
|
175
|
+
# B6. Two claims sharing an id split in half: _apply_grounding
|
|
176
|
+
# builds {c.id: c} (last wins) while _apply_assumption_flags uses
|
|
177
|
+
# next(...) (first wins), so the grounding landed on one object
|
|
178
|
+
# and the assumption flags on the other. claim_extraction.md asks
|
|
179
|
+
# for "sequential zero-padded ids" and never says they must be
|
|
180
|
+
# unique.
|
|
181
|
+
unique="id",
|
|
182
|
+
# B7. Only in ensemble mode, and only because claim_extraction.md
|
|
183
|
+
# is now asked to supply it on every claim there. An ABSENT field
|
|
184
|
+
# is indistinguishable from an empty one at the scorer, and
|
|
185
|
+
# `len([]) / n` is the MAXIMUM disagreement penalty -- charged to
|
|
186
|
+
# a claim that exists because a candidate asserted it.
|
|
187
|
+
ensemble_required=frozenset({"asserted_by_candidates"}),
|
|
188
|
+
),
|
|
189
|
+
|
|
190
|
+
"Pass 3a": _GROUNDING_ENTRY,
|
|
191
|
+
"Pass 3b": _CRITIC_ENTRY,
|
|
192
|
+
|
|
193
|
+
"Pass 3c": Spec(
|
|
194
|
+
top="object",
|
|
195
|
+
required=frozenset({"gaps", "coverage_score"}),
|
|
196
|
+
# `gaps` is iterated by build_coverage_gap_entries; a string there
|
|
197
|
+
# would produce one coverage-gap entry per CHARACTER.
|
|
198
|
+
key_types={"gaps": list},
|
|
199
|
+
),
|
|
200
|
+
|
|
201
|
+
"Pass 4": Spec(
|
|
202
|
+
top="object",
|
|
203
|
+
required=frozenset({"per_claim_flags"}),
|
|
204
|
+
key_types={"per_claim_flags": dict},
|
|
205
|
+
ids_in_keys="per_claim_flags",
|
|
206
|
+
),
|
|
207
|
+
|
|
208
|
+
"Pass 6a": Spec(
|
|
209
|
+
top="array",
|
|
210
|
+
entry_required=frozenset({"claim_id", "revised_text"}),
|
|
211
|
+
entry_types={"claim_id": str, "revised_text": str},
|
|
212
|
+
ids="claim_id",
|
|
213
|
+
),
|
|
214
|
+
|
|
215
|
+
"Pass 6b": Spec(
|
|
216
|
+
top="object",
|
|
217
|
+
required=frozenset({"grounding", "critic"}),
|
|
218
|
+
key_types={"grounding": list, "critic": list},
|
|
219
|
+
nested={"grounding": _GROUNDING_ENTRY, "critic": _CRITIC_ENTRY},
|
|
220
|
+
),
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _type_name(value: Any) -> str:
|
|
225
|
+
"""JSON's vocabulary, not Python's -- the message goes back to a model
|
|
226
|
+
that was asked for JSON, so `object`/`array` is what it can act on."""
|
|
227
|
+
return {dict: "object", list: "array", str: "string",
|
|
228
|
+
bool: "boolean", int: "number", float: "number",
|
|
229
|
+
type(None): "null"}.get(type(value), type(value).__name__)
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _permitted(types) -> str:
|
|
233
|
+
if not isinstance(types, tuple):
|
|
234
|
+
types = (types,)
|
|
235
|
+
return " or ".join(sorted({_type_name(t()) for t in types}))
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def validate(pass_id: str, payload: Any, claim_ids=None,
|
|
239
|
+
ensemble: bool = False) -> None:
|
|
240
|
+
"""Raise PayloadShapeError unless `payload` is what `pass_id` promised.
|
|
241
|
+
|
|
242
|
+
claim_ids the ids the APPLY step will resolve against -- all claims
|
|
243
|
+
for 3a/3b/5/6c, the currently-flagged batch for 6a. Not
|
|
244
|
+
"every claim in the run": Pass 6a is only asked about
|
|
245
|
+
`flagged`, and its own lookup already resolves within that
|
|
246
|
+
batch, so validating against a wider set would accept a
|
|
247
|
+
revision that then lands on nothing. None skips the check.
|
|
248
|
+
ensemble True on an ensemble run, which is the only place Pass 2 is
|
|
249
|
+
asked for `asserted_by_candidates` (B7).
|
|
250
|
+
|
|
251
|
+
An unknown pass_id RAISES rather than passing through unvalidated. A
|
|
252
|
+
pass with no spec is the silent hole this module exists to close, and
|
|
253
|
+
the eleventh pass must not be able to arrive by forgetting a line.
|
|
254
|
+
"""
|
|
255
|
+
spec = SPECS.get(pass_id)
|
|
256
|
+
if spec is None:
|
|
257
|
+
raise PayloadShapeError(
|
|
258
|
+
f"{pass_id}: no payload spec is declared for this pass. Every "
|
|
259
|
+
f"pass routed through _run_pass_with_json_retry needs one in "
|
|
260
|
+
f"core/reasoning/payload_validation.py (§30, B11)."
|
|
261
|
+
)
|
|
262
|
+
_validate_against(pass_id, payload, spec, claim_ids, ensemble)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _validate_against(pass_id, payload, spec, claim_ids, ensemble,
|
|
266
|
+
where: str = "") -> None:
|
|
267
|
+
at = f" under {where!r}" if where else ""
|
|
268
|
+
|
|
269
|
+
if spec.top == "array":
|
|
270
|
+
if not isinstance(payload, list):
|
|
271
|
+
raise PayloadShapeError(
|
|
272
|
+
f"{pass_id}: expected a JSON array{at}, got "
|
|
273
|
+
f"{_type_name(payload)}. Return the array itself, not an "
|
|
274
|
+
f"object wrapping it."
|
|
275
|
+
)
|
|
276
|
+
_validate_entries(pass_id, payload, spec, claim_ids, ensemble, at)
|
|
277
|
+
return
|
|
278
|
+
|
|
279
|
+
if not isinstance(payload, dict):
|
|
280
|
+
raise PayloadShapeError(
|
|
281
|
+
f"{pass_id}: expected a JSON object{at}, got "
|
|
282
|
+
f"{_type_name(payload)}."
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
missing = sorted(spec.required - set(payload))
|
|
286
|
+
if missing:
|
|
287
|
+
raise PayloadShapeError(
|
|
288
|
+
f"{pass_id}: the response{at} is missing required key(s) "
|
|
289
|
+
f"{', '.join(repr(m) for m in missing)}. Expected an object "
|
|
290
|
+
f"with {', '.join(sorted(spec.required))}."
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
for key, types in spec.key_types.items():
|
|
294
|
+
if key in payload and not isinstance(payload[key], types):
|
|
295
|
+
raise PayloadShapeError(
|
|
296
|
+
f"{pass_id}: {key!r}{at} is {_type_name(payload[key])}; "
|
|
297
|
+
f"expected {_permitted(types)}."
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
for key, nested in spec.nested.items():
|
|
301
|
+
_validate_against(pass_id, payload[key], nested, claim_ids,
|
|
302
|
+
ensemble, where=key)
|
|
303
|
+
|
|
304
|
+
if spec.ids_in_keys:
|
|
305
|
+
_check_ids(pass_id, list(payload[spec.ids_in_keys]), claim_ids,
|
|
306
|
+
f"key of {spec.ids_in_keys!r}")
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def _validate_entries(pass_id, entries, spec, claim_ids, ensemble, at) -> None:
|
|
310
|
+
required = spec.entry_required | (spec.ensemble_required if ensemble
|
|
311
|
+
else frozenset())
|
|
312
|
+
seen = {}
|
|
313
|
+
for index, entry in enumerate(entries):
|
|
314
|
+
if not isinstance(entry, dict):
|
|
315
|
+
raise PayloadShapeError(
|
|
316
|
+
f"{pass_id}: entry {index}{at} is {_type_name(entry)}; "
|
|
317
|
+
f"every entry must be an object with "
|
|
318
|
+
f"{', '.join(sorted(required))}."
|
|
319
|
+
)
|
|
320
|
+
missing = sorted(required - set(entry))
|
|
321
|
+
if missing:
|
|
322
|
+
raise PayloadShapeError(
|
|
323
|
+
f"{pass_id}: entry {index}{at} is missing required key(s) "
|
|
324
|
+
f"{', '.join(repr(m) for m in missing)}. Every entry needs "
|
|
325
|
+
f"{', '.join(sorted(required))}."
|
|
326
|
+
)
|
|
327
|
+
# TYPE GUARDS BEFORE THE MEMBERSHIP TESTS, which is review.py's
|
|
328
|
+
# rule and the reason it is worth copying: a JSON array or object
|
|
329
|
+
# in an enum-checked field is unhashable, so `value in ALLOWED`
|
|
330
|
+
# raises TypeError from inside the validator rather than
|
|
331
|
+
# producing a message the model can act on.
|
|
332
|
+
for key, types in spec.entry_types.items():
|
|
333
|
+
if key in entry and not isinstance(entry[key], types):
|
|
334
|
+
raise PayloadShapeError(
|
|
335
|
+
f"{pass_id}: entry {index}{at} has {key}="
|
|
336
|
+
f"{entry[key]!r}, which is {_type_name(entry[key])}; "
|
|
337
|
+
f"expected {_permitted(types)}."
|
|
338
|
+
)
|
|
339
|
+
for key, allowed in spec.enums.items():
|
|
340
|
+
if key not in entry:
|
|
341
|
+
continue
|
|
342
|
+
value = entry[key]
|
|
343
|
+
if not isinstance(value, str) or value not in allowed:
|
|
344
|
+
raise PayloadShapeError(
|
|
345
|
+
f"{pass_id}: entry {index}{at} has {key}={value!r}; "
|
|
346
|
+
f"expected exactly one of "
|
|
347
|
+
f"{', '.join(sorted(allowed))} (lower case)."
|
|
348
|
+
)
|
|
349
|
+
if spec.unique:
|
|
350
|
+
value = entry[spec.unique]
|
|
351
|
+
if value in seen:
|
|
352
|
+
raise PayloadShapeError(
|
|
353
|
+
f"{pass_id}: entries {seen[value]} and {index}{at} "
|
|
354
|
+
f"share {spec.unique}={value!r}. Ids must be unique -- "
|
|
355
|
+
f"two claims with one id are applied to inconsistently "
|
|
356
|
+
f"and end up carrying half of each other's results."
|
|
357
|
+
)
|
|
358
|
+
seen[value] = index
|
|
359
|
+
|
|
360
|
+
if spec.ids:
|
|
361
|
+
_check_ids(pass_id, [e[spec.ids] for e in entries], claim_ids,
|
|
362
|
+
spec.ids)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _check_ids(pass_id, supplied, claim_ids, field_name) -> None:
|
|
366
|
+
"""B5: a TOTAL mismatch is the error; a partial one is not.
|
|
367
|
+
|
|
368
|
+
The demonstrated silent failure is every id being wrong -- 3a/3b
|
|
369
|
+
answering about "1", "2", "3" when Pass 2 issued "c001", "c002" --
|
|
370
|
+
which grounded nothing while the checkpoint line reported that it had
|
|
371
|
+
grounded everything. That is a pass with no effect at all, and it is
|
|
372
|
+
worth a corrective retry.
|
|
373
|
+
|
|
374
|
+
A model that gets nine of ten right is NOT worth failing a run that
|
|
375
|
+
has already paid for nine passes: the nine apply, and the checkpoint
|
|
376
|
+
line reports the count APPLIED rather than the count asked about, so
|
|
377
|
+
the tenth is visible instead of silent (see the orchestrator's
|
|
378
|
+
_apply_* return values).
|
|
379
|
+
|
|
380
|
+
An EMPTY collection is not a mismatch. assumption_audit.md asks the
|
|
381
|
+
model to "omit claims with no issues rather than including them with
|
|
382
|
+
an empty list", so no flags at all is the ordinary answer for a clean
|
|
383
|
+
run -- and Pass 4's demonstrated defect was the top-level key being
|
|
384
|
+
absent, which the required-key check above catches instead.
|
|
385
|
+
"""
|
|
386
|
+
if claim_ids is None or not supplied:
|
|
387
|
+
return
|
|
388
|
+
known = set(claim_ids)
|
|
389
|
+
if any(value in known for value in supplied):
|
|
390
|
+
return
|
|
391
|
+
sample = ", ".join(repr(v) for v in supplied[:3])
|
|
392
|
+
expected = ", ".join(repr(v) for v in sorted(known)[:3])
|
|
393
|
+
raise PayloadShapeError(
|
|
394
|
+
f"{pass_id}: not one of the {len(supplied)} {field_name} value(s) "
|
|
395
|
+
f"matches a claim you were given (got {sample}; expected ids like "
|
|
396
|
+
f"{expected}). Use the claim ids exactly as they appear in the "
|
|
397
|
+
f"input."
|
|
398
|
+
)
|