@yahyahammad/venastine 0.3.8 → 0.3.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -19
- package/CONFIG_ARCHITECTURE.md +2139 -2002
- package/LICENSE +201 -201
- package/README.md +1043 -1034
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -106
- package/agents/builtin/review.md +74 -74
- package/agents/manager.py +261 -261
- package/agents/subagent_tool.py +472 -472
- package/agents/tui_commands.py +238 -146
- package/bin/venastine.mjs +584 -528
- package/bin/venastine.py +73 -71
- package/cli_commands.py +848 -835
- package/config.yaml +64 -0
- package/config_edit.py +1301 -1301
- package/config_schema.py +987 -966
- package/config_update.py +749 -749
- package/core/agent_activity.py +256 -256
- package/core/ansi.py +452 -452
- package/core/approval.py +137 -137
- package/core/call_view.py +364 -364
- package/core/cancel.py +121 -121
- package/core/chatgpt_auth.py +747 -747
- package/core/claims_view.py +107 -107
- package/core/client.py +2756 -2715
- package/core/commands.py +464 -454
- package/core/compaction.py +1735 -1735
- package/core/config_loader.py +1982 -1982
- package/core/connect.py +686 -686
- package/core/events.py +123 -123
- package/core/file_mentions.py +280 -280
- package/core/interaction.py +411 -411
- package/core/loop.py +2566 -2566
- package/core/memory.py +482 -482
- package/core/pipeline_models.py +190 -190
- package/core/provider_errors.py +274 -234
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -484
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/orchestrator.py +1561 -1561
- package/core/reasoning/output_writer.py +345 -345
- package/core/reasoning/payload_validation.py +524 -524
- package/core/reasoning/pipeline_storage.py +427 -427
- package/core/reasoning/scholar.py +467 -458
- package/core/reasoning/source_corpus.py +433 -433
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -315
- package/core/shell_sessions.py +1553 -1553
- package/credentials.py +307 -307
- package/database.py +333 -333
- package/docs/PRIVACY.md +78 -78
- package/docs/SECURITY.md +114 -108
- package/docs/THIRD_PARTY_NOTICES.md +148 -148
- package/env_secrets.py +118 -118
- package/json_store.py +343 -343
- package/logging_setup.py +295 -295
- package/main.py +3433 -3433
- package/mcp_client/client.py +780 -733
- package/mcp_client/config.py +484 -484
- package/mcp_client/registration.py +343 -343
- package/memories/manager.py +147 -147
- package/memories/tui_commands.py +319 -239
- package/package.json +303 -303
- package/project_init/config_files.py +282 -282
- package/project_init/generator.py +728 -728
- package/project_init/manifest.py +418 -418
- package/project_init/tui_commands.py +147 -147
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +17 -17
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -316
- package/providers.json.example +94 -94
- package/pyproject.toml +149 -149
- package/requirements.txt +314 -295
- package/safety/policy_enforcement.py +1082 -1082
- package/security/askpass.py +419 -419
- package/security/capability.py +207 -207
- package/security/confine.py +484 -484
- package/security/hosted.py +182 -182
- package/security/permissions.py +237 -237
- package/security/protected_paths.py +501 -501
- package/security/sandbox.py +4108 -4108
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -110
- package/storage.py +1338 -1338
- package/tools/base.py +465 -465
- package/tools/builtin/_math_common.py +667 -667
- package/tools/builtin/_net_common.py +1145 -267
- package/tools/builtin/_page_text.py +490 -0
- package/tools/builtin/_pdf_text.py +113 -0
- package/tools/builtin/arxiv.py +685 -681
- package/tools/builtin/fetch_url.py +639 -371
- package/tools/builtin/file_ops.py +720 -720
- package/tools/builtin/geometry.py +184 -184
- package/tools/builtin/load_skill.py +76 -76
- package/tools/builtin/pin.py +227 -227
- package/tools/builtin/probability_stats.py +224 -224
- package/tools/builtin/project_docs.py +464 -464
- package/tools/builtin/remember.py +175 -175
- package/tools/builtin/shell.py +980 -980
- package/tools/builtin/todo.py +149 -149
- package/tools/builtin/web_search.py +402 -328
- package/tools/context.py +159 -159
- package/tools/isolation.py +387 -387
- package/tools/registry.py +1252 -1242
- package/tui/app.py +8372 -8307
- package/tui/app.tcss +754 -739
- package/tui/commands.py +26 -26
- package/tui/diffs.py +286 -286
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -102
- package/tui/markdown.py +1329 -1329
- package/tui/meters.py +364 -364
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -1197
- package/tui/themes.py +599 -599
- package/tui/widgets.py +3861 -3796
|
@@ -1,1561 +1,1561 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/reasoning/orchestrator.py
|
|
3
|
-
|
|
4
|
-
Sequences the deep-research pipeline: Pass 0 -> 1 -> 2 -> gate D0 -> (3a -> 3b
|
|
5
|
-
for factual claims) -> 3c -> 4 (all claims) -> Pass 5 (code) -> D1 -> the
|
|
6
|
-
6a/6b/D2 retry loop -> 6c (template) -> merge -> final synthesis.
|
|
7
|
-
|
|
8
|
-
SCOPE OF THIS VERSION: core sequential pipeline with optional ensemble
|
|
9
|
-
mode (ROADMAP §10: N-candidate Pass 1 generation + a cross-candidate
|
|
10
|
-
consistency check, whose contested claims D1 routes to the 6a/6b loop,
|
|
11
|
-
ROADMAP_v3 §90). No filesystem output yet
|
|
12
|
-
(returns a PipelineRun object; the /output/<run_id>/ file-writing system
|
|
13
|
-
is a separate, later piece of work).
|
|
14
|
-
|
|
15
|
-
ROADMAP §11 (critic-model routing): when a critic pair is configured,
|
|
16
|
-
Pass 3a, 3b, and 6c (which re-runs 3a/3b logic) use the critic
|
|
17
|
-
provider/model instead of the generator's. Every other pass keeps using
|
|
18
|
-
the main provider_name/model. This prevents a model from checking its
|
|
19
|
-
own output for errors with the same blind spots. §45 (SQ7) moved WHERE
|
|
20
|
-
that pair comes from: core/pipeline_models.resolve("critic") owns the
|
|
21
|
-
precedence -- a `/critic` choice in the user-tier store, else
|
|
22
|
-
config.CRITIC_MODEL -- so the routing is reachable without editing the
|
|
23
|
-
checkout, which for §11's whole life it was not.
|
|
24
|
-
|
|
25
|
-
ROADMAP §3 (malformed-JSON recovery): JSON-emitting passes are wrapped in
|
|
26
|
-
_run_pass_with_json_retry, which on a parse failure re-enters the SAME
|
|
27
|
-
pass-thread via RunAgentLoop.continue_conversation() with a corrective
|
|
28
|
-
follow-up message. The failed assistant turn is already in the thread's
|
|
29
|
-
history (see _run()'s always-persist fix in core/loop.py), so the model
|
|
30
|
-
sees its own prior output in context. Up to config.MAX_JSON_RETRIES
|
|
31
|
-
corrective attempts are made; a final ValueError surfaces unrecoverable
|
|
32
|
-
failures.
|
|
33
|
-
|
|
34
|
-
ROADMAP §5 (durable pipeline persistence): each run is wrapped in a
|
|
35
|
-
try/except that records status='failed' (with a snapshot of the partial
|
|
36
|
-
run) before re-raising, so a mid-pipeline crash still leaves a durable
|
|
37
|
-
record. A data-only update_pipeline_run() checkpoint fires after every
|
|
38
|
-
run.log() trace line, so even a hard kill (KeyboardInterrupt / SystemExit,
|
|
39
|
-
which bypass the except Exception block) leaves the record populated
|
|
40
|
-
through the last completed pass. load_pipeline_run() in
|
|
41
|
-
pipeline_storage.py provides the read API.
|
|
42
|
-
|
|
43
|
-
ROADMAP_v2 §22 (pipeline observability): the pipeline is a GENERATOR
|
|
44
|
-
yielding PipelineEvents (core/reasoning/events.py), so a shell can render
|
|
45
|
-
pass boundaries, trace lines, claim tiers and retries as they happen
|
|
46
|
-
rather than waiting for the whole run. Two names:
|
|
47
|
-
|
|
48
|
-
stream_deep_research_pipeline() the generator
|
|
49
|
-
run_pipeline_to_completion() the drainer, for a caller that wants
|
|
50
|
-
only the finished PipelineRun
|
|
51
|
-
|
|
52
|
-
DECISION P3 IS RETIRED, and it is worth saying why rather than quietly
|
|
53
|
-
dropping a name. P3 kept a third, synchronous `run_deep_research_pipeline()`
|
|
54
|
-
-- the drainer applied to the generator -- so that "every existing caller:
|
|
55
|
-
main.py, tui/app.py and fifteen test sites" would go on working unchanged.
|
|
56
|
-
Both shells then left of their own accord: `main.run_research` and
|
|
57
|
-
`tui/app.py` iterate the generator so they can render progress, each with a
|
|
58
|
-
comment saying it deliberately does not call the wrapper. That left a
|
|
59
|
-
public function with no production caller and fifty call sites in tests,
|
|
60
|
-
while this docstring went on naming it the live entry point. The
|
|
61
|
-
convenience moved to `tests/conftest.run_pipeline`, which is what it had
|
|
62
|
-
become; P3's argument was sound and its premise expired.
|
|
63
|
-
|
|
64
|
-
Decision P2: a pass's own LoopEvents are NOT forwarded up as LoopEvents --
|
|
65
|
-
_run_pass TRANSLATES a chosen subset into PipelineEvents (§26 amended this;
|
|
66
|
-
§22 had kept a pass fully opaque). It iterates
|
|
67
|
-
RunAgentLoop.stream_deep_research_mode() directly. `run_deep_research_mode`,
|
|
68
|
-
the synchronous wrapper this line used to name, is gone for
|
|
69
|
-
`run_deep_research_pipeline`'s reason and one layer down.
|
|
70
|
-
"""
|
|
71
|
-
|
|
72
|
-
from __future__ import annotations
|
|
73
|
-
|
|
74
|
-
import json
|
|
75
|
-
import logging
|
|
76
|
-
|
|
77
|
-
import config
|
|
78
|
-
import prompts.system_prompts as system_prompts
|
|
79
|
-
from core import pipeline_models
|
|
80
|
-
from core.config_loader import effective_confidence, effective_source_scoring
|
|
81
|
-
from core.loop import RunAgentLoop, advertisement_facts
|
|
82
|
-
from core.reasoning.base import Claim, PipelineRun, resolve_by_id
|
|
83
|
-
from core.reasoning.confidence_scoring import is_contested, run_confidence_tiering
|
|
84
|
-
from core.reasoning.events import PipelineEvent
|
|
85
|
-
from core.reasoning.json_retry import parse_json_response as _parse_json_response
|
|
86
|
-
from core.reasoning.json_retry import retry_until_json
|
|
87
|
-
from core.reasoning.payload_validation import validate as _validate_payload
|
|
88
|
-
from core.reasoning.pipeline_storage import create_pipeline_run, update_pipeline_run
|
|
89
|
-
from core.reasoning.scholar import ScholarLookup
|
|
90
|
-
from core.reasoning.source_corpus import SourceCorpus
|
|
91
|
-
from core.reasoning.source_scoring import make_scorer, score_grounding_sources
|
|
92
|
-
from safety.policy_enforcement import redact_diagnostic_text
|
|
93
|
-
from tools.registry import registry
|
|
94
|
-
|
|
95
|
-
logger = logging.getLogger(__name__)
|
|
96
|
-
|
|
97
|
-
DEFAULT_PROVIDER = "ANTHROPIC"
|
|
98
|
-
FLAGGED_TIERS = {"LOW", "UNVERIFIED"}
|
|
99
|
-
MAX_RETRIES = config.MAX_PIPELINE_RETRIES
|
|
100
|
-
# The JSON-retry budget lives in core/reasoning/json_retry.py since the
|
|
101
|
-
# §20 extraction; a second constant here was a dead knob that invited
|
|
102
|
-
# no-op patches (review §19-20 f9).
|
|
103
|
-
|
|
104
|
-
_CLAIM_INPUT_FIELDS = {"id", "text", "type", "entities", "source_span", "asserted_by_candidates"}
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
# ============================================================================
|
|
108
|
-
# ---- Small helpers ----------------------------------------------------------
|
|
109
|
-
# ============================================================================
|
|
110
|
-
|
|
111
|
-
class _Progress:
|
|
112
|
-
"""The ONE place a trace line becomes durable and becomes an event
|
|
113
|
-
(§22 AC3).
|
|
114
|
-
|
|
115
|
-
Before §22 this file paired every `run.log(...)` with an
|
|
116
|
-
`update_pipeline_run(...)` at twenty call sites, and `run.trace` had
|
|
117
|
-
THREE writers: those, review.py's fifteen `run.log()` calls (which
|
|
118
|
-
checkpointed nowhere), and json_retry.py's bare `trace.append()` per
|
|
119
|
-
failed parse. So "§5's persistence fires on every trace line" was
|
|
120
|
-
already untrue, and adding a per-call-site event emission would have
|
|
121
|
-
made it two independent writers of related data -- the bug shape §22
|
|
122
|
-
warns about by name.
|
|
123
|
-
|
|
124
|
-
So both are DERIVED from `run.trace` itself. `checkpoint()` emits
|
|
125
|
-
every line appended since it last ran and then persists. Because the
|
|
126
|
-
watermark reads the list rather than the call site, review.py's and
|
|
127
|
-
json_retry.py's lines are picked up too, WITHOUT either module being
|
|
128
|
-
edited or knowing this exists.
|
|
129
|
-
|
|
130
|
-
The watermark lives here rather than on PipelineRun because base.py
|
|
131
|
-
is data only and §5 serializes that object -- a field meaningful
|
|
132
|
-
only inside a live generator has no business in a persisted row.
|
|
133
|
-
"""
|
|
134
|
-
|
|
135
|
-
def __init__(self, run: PipelineRun) -> None:
|
|
136
|
-
self.run = run
|
|
137
|
-
self._emitted = 0
|
|
138
|
-
|
|
139
|
-
def checkpoint(self, message: str | None = None):
|
|
140
|
-
"""Log (optionally), PERSIST, then emit every un-emitted line.
|
|
141
|
-
|
|
142
|
-
Persist BEFORE emitting, not after. A generator only advances
|
|
143
|
-
while someone iterates it, so a consumer that abandons the run
|
|
144
|
-
between the yield and the persist would take that checkpoint down
|
|
145
|
-
with it -- §5's durability would silently start depending on a
|
|
146
|
-
UI's willingness to keep reading. This way, a line is already in
|
|
147
|
-
the record by the time anyone is told about it.
|
|
148
|
-
|
|
149
|
-
Monotonic: `_emitted` only ever advances, so no line can be
|
|
150
|
-
yielded twice however many times this is called.
|
|
151
|
-
"""
|
|
152
|
-
if message is not None:
|
|
153
|
-
self.run.log(message)
|
|
154
|
-
update_pipeline_run(self.run.run_id, self.run)
|
|
155
|
-
while self._emitted < len(self.run.trace):
|
|
156
|
-
line = self.run.trace[self._emitted]
|
|
157
|
-
self._emitted += 1
|
|
158
|
-
yield PipelineEvent(kind="trace_line", text=line)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
# One pass_activity event per this many characters streamed. A per-token
|
|
162
|
-
# event would be thousands of post_message hand-offs per pass, which is the
|
|
163
|
-
# cost Transcript.stream_delta already exists to avoid on the chat path.
|
|
164
|
-
_ACTIVITY_CHARS = 2000
|
|
165
|
-
|
|
166
|
-
# §26's _param_digest, and its two truncation constants, MOVED to
|
|
167
|
-
# safety/policy_enforcement.py in §27 -- beside the redactors, because
|
|
168
|
-
# §27's replay renderer is its second caller and the redact-before-truncate
|
|
169
|
-
# ordering must not exist twice. Imported under its own name rather than
|
|
170
|
-
# aliased back to the old private one: an alias would leave two spellings
|
|
171
|
-
# for one function, and a reader who greps the old name here should be sent
|
|
172
|
-
# to where it now lives, not handed a local rebinding.
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
def _translate(pass_id: str, stream, corpus=None):
|
|
176
|
-
"""Turn one pass's LoopEvents into PipelineEvents, and return the
|
|
177
|
-
ModelResponse the pass produced (ROADMAP_v2 §26).
|
|
178
|
-
|
|
179
|
-
THE one place a LoopEvent becomes a PipelineEvent. Both _run_pass and
|
|
180
|
-
_run_pass_with_json_retry consume this rather than each iterating the
|
|
181
|
-
stream themselves -- two emission sites for the same translation is how
|
|
182
|
-
they drift, and this file has the twenty-checkpoint version of that
|
|
183
|
-
mistake in its own history (see _Progress).
|
|
184
|
-
|
|
185
|
-
What is translated is a SUBSET, chosen for what a person watching a
|
|
186
|
-
ten-pass run needs:
|
|
187
|
-
|
|
188
|
-
tool_call / tool_result the thing §22 P2 made invisible. A pass
|
|
189
|
-
making fourteen fetch_url calls over
|
|
190
|
-
several minutes looked exactly like one
|
|
191
|
-
that had hung.
|
|
192
|
-
pass_activity for the passes that call no tools at all.
|
|
193
|
-
Throttled: a total, every _ACTIVITY_CHARS.
|
|
194
|
-
|
|
195
|
-
What is NOT translated, and why:
|
|
196
|
-
|
|
197
|
-
token_delta text seven of the ten passes emit raw JSON, so
|
|
198
|
-
streaming pass output into a transcript
|
|
199
|
-
buries the report under it. Only the volume
|
|
200
|
-
escapes, never the content.
|
|
201
|
-
thinking_delta §38 (O5). Not translated AND not counted
|
|
202
|
-
into pass_activity's running total, so
|
|
203
|
-
`chars` keeps meaning one thing -- output
|
|
204
|
-
streamed so far -- rather than quietly
|
|
205
|
-
becoming two instruments the way batch 27
|
|
206
|
-
item 9 describes for the token figures. The
|
|
207
|
-
cost is stated rather than hidden: a pass
|
|
208
|
-
that thinks for minutes before writing
|
|
209
|
-
anything shows no activity, which is the
|
|
210
|
-
shape §26 built pass_activity to cure. Flip
|
|
211
|
-
it here if that turns up in a real run.
|
|
212
|
-
retract batch 113. A retraction tells a surface
|
|
213
|
-
that DREW model text to take it back, and
|
|
214
|
-
this one never drew any -- `chars` is the
|
|
215
|
-
only thing that escaped. Left monotonic
|
|
216
|
-
deliberately: the total measures what the
|
|
217
|
-
pass produced, it is the panel's liveness
|
|
218
|
-
figure rather than a bill, and a running
|
|
219
|
-
total that jumped backwards mid-pass
|
|
220
|
-
would read as the defect instead of the
|
|
221
|
-
honesty. The TUI's own output-char meter
|
|
222
|
-
is left alone for the same reason.
|
|
223
|
-
permission_request §25's ApprovalProvider already carries this
|
|
224
|
-
to the shell and blocks on the answer. A
|
|
225
|
-
second rendering of a question already on
|
|
226
|
-
screen is not observability.
|
|
227
|
-
notice a pass's compaction notices, forwarded as
|
|
228
|
-
trace LINES. M6 makes these backstop-only,
|
|
229
|
-
but "rare" is not "silent" -- §21's rule
|
|
230
|
-
has no exception for the pipeline, and a
|
|
231
|
-
backstop fold rewrites what the REST of
|
|
232
|
-
the pass sees. This used to drop them with
|
|
233
|
-
the claim that they "already reach the
|
|
234
|
-
shell as WARNING log records", which was
|
|
235
|
-
true of blocked and failed compactions and
|
|
236
|
-
FALSE of successful ones (INFO only --
|
|
237
|
-
audit #172). A trace line lands in
|
|
238
|
-
trace.md and both shells print it.
|
|
239
|
-
|
|
240
|
-
§45 (SQ2). `corpus` is a SECOND SINK for tool results, not a change to
|
|
241
|
-
what is translated. The rule above -- a successful result body never
|
|
242
|
-
reaches the UI -- is unchanged; this is a different consumer with a
|
|
243
|
-
different need, and the scoring stage cannot check a quote against a
|
|
244
|
-
page nobody kept. None, the default, collects nothing: that is every
|
|
245
|
-
caller that predates §45 and every test double."""
|
|
246
|
-
names: dict = {} # tool_call id -> name, since tool_result has none
|
|
247
|
-
streamed = 0
|
|
248
|
-
announced = 0
|
|
249
|
-
while True:
|
|
250
|
-
# next() rather than `for event in stream`, because a for loop
|
|
251
|
-
# SWALLOWS the generator's return value -- and the response is
|
|
252
|
-
# returned rather than read off the terminal event precisely
|
|
253
|
-
# because thread_id is attached after the loop finishes. A for
|
|
254
|
-
# loop here would hand every pass a response with no thread_id,
|
|
255
|
-
# and the only symptom would be §3's JSON retry silently starting
|
|
256
|
-
# a new thread instead of correcting the failed one.
|
|
257
|
-
try:
|
|
258
|
-
event = next(stream)
|
|
259
|
-
except StopIteration as stop:
|
|
260
|
-
return stop.value
|
|
261
|
-
|
|
262
|
-
if event.token_delta:
|
|
263
|
-
streamed += len(event.token_delta)
|
|
264
|
-
if streamed - announced >= _ACTIVITY_CHARS:
|
|
265
|
-
announced = streamed
|
|
266
|
-
yield PipelineEvent(kind="pass_activity", pass_id=pass_id,
|
|
267
|
-
chars=streamed)
|
|
268
|
-
if event.notice:
|
|
269
|
-
yield PipelineEvent(
|
|
270
|
-
kind="trace_line",
|
|
271
|
-
text=f"compaction during {pass_id}: {event.notice['text']}")
|
|
272
|
-
if event.tool_call_start:
|
|
273
|
-
name = event.tool_call_start.get("name")
|
|
274
|
-
names[event.tool_call_start.get("id")] = name
|
|
275
|
-
yield PipelineEvent(
|
|
276
|
-
kind="tool_call", pass_id=pass_id, tool=name,
|
|
277
|
-
# §42 (RA3): through the registry, so a tool's
|
|
278
|
-
# declared rationale param is dropped here too -- a
|
|
279
|
-
# pass row is as narrow as a transcript line.
|
|
280
|
-
text=registry.call_digest(
|
|
281
|
-
name, event.tool_call_start.get("input")))
|
|
282
|
-
if event.tool_result:
|
|
283
|
-
result = event.tool_result.get("result")
|
|
284
|
-
failed = isinstance(result, dict) and "error" in result
|
|
285
|
-
name = names.get(event.tool_result.get("id"))
|
|
286
|
-
if corpus is not None and not failed:
|
|
287
|
-
# BEFORE the yield, for #42's reason one layer up: a
|
|
288
|
-
# generator only advances while someone iterates it, so
|
|
289
|
-
# collecting after the yield would make the corpus depend
|
|
290
|
-
# on a consumer continuing to read. `corpus.add` ignores
|
|
291
|
-
# every tool it does not recognise.
|
|
292
|
-
corpus.add(name, result)
|
|
293
|
-
yield PipelineEvent(
|
|
294
|
-
kind="tool_result", pass_id=pass_id,
|
|
295
|
-
tool=name,
|
|
296
|
-
ok=not failed,
|
|
297
|
-
# Redacted here too, though dispatch() already ran the
|
|
298
|
-
# result through check_output_policy. One rule owned at one
|
|
299
|
-
# boundary -- "nothing leaves this function unredacted" --
|
|
300
|
-
# beats depending on a guarantee made two layers away that
|
|
301
|
-
# a later refactor could quietly drop.
|
|
302
|
-
text=(redact_diagnostic_text(str(result.get("error")))
|
|
303
|
-
if failed else None))
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
def _check_not_truncated(pass_id: str, response, trace: list[str] | None):
|
|
307
|
-
"""A pass that ended on a stop condition did NOT finish; say so.
|
|
308
|
-
|
|
309
|
-
`_run_pass` used to return `response.text` and discard `stop_reason`,
|
|
310
|
-
so a pass cut off by the token budget looked exactly like one that
|
|
311
|
-
completed -- and because a budget stop returns the last response AS
|
|
312
|
-
IT STANDS, a pass that was mid-tool-call returned the EMPTY STRING.
|
|
313
|
-
|
|
314
|
-
That is not hypothetical. A Pass 1 making 14 tool calls crossed the
|
|
315
|
-
old 250k ceiling, returned "", and the orchestrator stored it as
|
|
316
|
-
raw_response. Pass 2 was then handed "Response to extract claims
|
|
317
|
-
from:\\n" and correctly answered that there was nothing to extract --
|
|
318
|
-
in a shape that is not a claim list -- and the run finally died in
|
|
319
|
-
Claim's constructor, three passes and one confusing TypeError away
|
|
320
|
-
from the thing that actually went wrong.
|
|
321
|
-
|
|
322
|
-
TWO outcomes, because truncation is not uniformly fatal:
|
|
323
|
-
|
|
324
|
-
* truncated WITH text -- degraded but usable. The pass produced an
|
|
325
|
-
answer and was cut off finishing it; downstream passes can work
|
|
326
|
-
with that. Traced, so the report's own audit log says the answer
|
|
327
|
-
was incomplete, and the run continues.
|
|
328
|
-
* truncated with NO text -- unusable. Everything downstream is
|
|
329
|
-
working from an empty string, and the only question is how many
|
|
330
|
-
passes it takes to produce an incomprehensible error. Raise here,
|
|
331
|
-
naming the pass and the stop reason.
|
|
332
|
-
"""
|
|
333
|
-
stop = getattr(response, "stop_reason", None)
|
|
334
|
-
if stop in (None, "complete"):
|
|
335
|
-
return
|
|
336
|
-
text = (getattr(response, "text", "") or "").strip()
|
|
337
|
-
# #4: the figures turn a bare reason into a diagnosis. A user who
|
|
338
|
-
# reads token_budget_exceeded as a context problem is repeating the
|
|
339
|
-
# exact misreading item 9 recorded.
|
|
340
|
-
billed = getattr(response, "turn_billed_tokens", None)
|
|
341
|
-
figures = (f" billed {billed:,} this turn"
|
|
342
|
-
if isinstance(billed, int) else "")
|
|
343
|
-
if not text:
|
|
344
|
-
raise ValueError(
|
|
345
|
-
f"{pass_id} ended on {stop!r} with no text{figures}. A stop "
|
|
346
|
-
f"condition returns the last response as it stands, so a pass "
|
|
347
|
-
f"cut off mid-tool-call yields an empty string -- and every "
|
|
348
|
-
f"later pass would be working from nothing. "
|
|
349
|
-
f"(config.MAX_ITERATIONS and a configured settings.json "
|
|
350
|
-
f"max_token_budget spend cap are the two ceilings that "
|
|
351
|
-
f"produce this.)"
|
|
352
|
-
)
|
|
353
|
-
message = (f"{pass_id}: TRUNCATED ({stop}{figures}) -- the pass was cut "
|
|
354
|
-
f"off but produced output; continuing with what it returned.")
|
|
355
|
-
logger.warning("%s", message)
|
|
356
|
-
if trace is not None:
|
|
357
|
-
trace.append(message)
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
def _ensemble_roster(ensemble_mode: bool) -> list[dict]:
|
|
361
|
-
"""The (provider_name, model) pairs Pass 1 runs on, or [] when ensemble
|
|
362
|
-
mode is off. Raises ValueError on a roster that cannot produce a
|
|
363
|
-
meaningful consistency signal (ROADMAP §10 revisit, E1/E5).
|
|
364
|
-
|
|
365
|
-
A separate function so every refusal is testable without running a
|
|
366
|
-
pipeline, and so the reasons stay together: this replaces §16's
|
|
367
|
-
sampling-parameter guard, which refused for a mechanism that no longer
|
|
368
|
-
exists.
|
|
369
|
-
|
|
370
|
-
Validated by SHAPE only -- provider names and API keys are not checked
|
|
371
|
-
here (E6). §16's /model warns rather than refuses on an empty API_KEY so
|
|
372
|
-
a local OpenAI-compatible endpoint stays usable, and MCP's posture is
|
|
373
|
-
that validation happens by connecting. A candidate whose provider is
|
|
374
|
-
unreachable is named, traced and skipped at run time instead.
|
|
375
|
-
"""
|
|
376
|
-
if not ensemble_mode:
|
|
377
|
-
return []
|
|
378
|
-
|
|
379
|
-
roster = config.ENSEMBLE_MODELS
|
|
380
|
-
if not roster:
|
|
381
|
-
raise ValueError(
|
|
382
|
-
"Ensemble mode is on but ensemble_models in config.yaml is empty. "
|
|
383
|
-
"Diversity comes from different models now, not from a raised "
|
|
384
|
-
"temperature (ROADMAP §10 revisit), so the roster is required. "
|
|
385
|
-
"Set it in config.yaml to a list of at least two distinct "
|
|
386
|
-
'{"provider_name": ..., "model": ...} entries, or disable '
|
|
387
|
-
"ensemble_mode. There is deliberately no settings.json key for "
|
|
388
|
-
"the roster -- see E2."
|
|
389
|
-
)
|
|
390
|
-
|
|
391
|
-
for entry in roster:
|
|
392
|
-
if not isinstance(entry, dict) or not entry.get("provider_name") or not entry.get("model"):
|
|
393
|
-
raise ValueError(
|
|
394
|
-
f"Malformed ensemble_models entry in config.yaml {entry!r}: every "
|
|
395
|
-
f'entry needs both "provider_name" and "model".'
|
|
396
|
-
)
|
|
397
|
-
|
|
398
|
-
# DISTINCT pairs, not merely two entries. Repeating one model N times
|
|
399
|
-
# would generate N near-identical candidates -- there is no sampling
|
|
400
|
-
# variation left to make them differ -- and then report maximal
|
|
401
|
-
# cross-candidate consistency for every claim, which Pass 5 reads as
|
|
402
|
-
# confidence. That is precisely the failure §16's guard was added to
|
|
403
|
-
# prevent, and this config is the way to recreate it.
|
|
404
|
-
distinct = {(e["provider_name"], e["model"]) for e in roster}
|
|
405
|
-
if len(distinct) < 2:
|
|
406
|
-
raise ValueError(
|
|
407
|
-
f"Ensemble mode needs at least two DISTINCT models; "
|
|
408
|
-
f"ensemble_models in config.yaml resolves to {len(distinct)} "
|
|
409
|
-
f"({sorted(distinct)}). One model cannot disagree with itself, so "
|
|
410
|
-
f"every claim would score maximal consistency and Pass 5 would "
|
|
411
|
-
f"read that as confidence."
|
|
412
|
-
)
|
|
413
|
-
return list(roster)
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
def _pass_failed_event(pass_id: str, exc: BaseException) -> PipelineEvent:
|
|
417
|
-
"""E14/#115. The one shape a pass failure is reported in: a
|
|
418
|
-
pass_complete carrying ok=False, mirroring tool_result's
|
|
419
|
-
outcome-plus-reason contract. A NEW kind was considered and rejected
|
|
420
|
-
-- tool_result already established payload-carried outcomes in this
|
|
421
|
-
very event set, and a second mechanism for 'ended' would make every
|
|
422
|
-
future consumer learn which applies where. The text is TRUNCATED:
|
|
423
|
-
the trace line (run.log at whatever site caught this) carries the
|
|
424
|
-
full cause, and an event payload is not the place to duplicate it
|
|
425
|
-
unboundedly."""
|
|
426
|
-
return PipelineEvent(
|
|
427
|
-
kind="pass_complete", pass_id=pass_id, ok=False,
|
|
428
|
-
text=f"{type(exc).__name__}: {exc}"[:200])
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
def _run_pass(pass_id: str, pass_input: str, model: str, provider_name: str,
|
|
432
|
-
temperature: float | None = None, authorization=None,
|
|
433
|
-
trace: list[str] | None = None, pass_threads: list | None = None,
|
|
434
|
-
effort: str | None = None, corpus=None, activity=None):
|
|
435
|
-
"""One LLM-backed pass. Every actual model call in this file goes
|
|
436
|
-
through this single function.
|
|
437
|
-
|
|
438
|
-
§22: a GENERATOR that yields the pass boundary and returns the
|
|
439
|
-
response text (`text = yield from _run_pass(...)`). The pass id is
|
|
440
|
-
named once here rather than restated beside twelve call sites.
|
|
441
|
-
|
|
442
|
-
§26: the pass is now ITERATED rather than called, through _translate,
|
|
443
|
-
so what it does between its start and its end is visible. Same shape
|
|
444
|
-
as §22's own change to the shells -- the drainer still exists and is
|
|
445
|
-
still what every other caller uses.
|
|
446
|
-
|
|
447
|
-
Batch 25 (#139): effort reaches the passes like authorization does --
|
|
448
|
-
threaded from the shell's setting through the pipeline, validated per
|
|
449
|
-
RECEIVING model by effort_for() inside the loop, so an ensemble roster
|
|
450
|
-
or critic routing needs no effort-specific handling here.
|
|
451
|
-
"""
|
|
452
|
-
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
453
|
-
try:
|
|
454
|
-
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
455
|
-
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
456
|
-
temperature=temperature, authorization=authorization, effort=effort,
|
|
457
|
-
activity=activity,
|
|
458
|
-
), corpus=corpus)
|
|
459
|
-
_record_granted_calls(pass_id, response, authorization)
|
|
460
|
-
_record_pass_thread(pass_id, response, pass_threads)
|
|
461
|
-
_check_not_truncated(pass_id, response, trace)
|
|
462
|
-
except Exception as exc:
|
|
463
|
-
# E14/#115, UNIFORM: a fatal pass reports itself before its
|
|
464
|
-
# exception propagates, so no shell is left holding a row that
|
|
465
|
-
# only ever says "running" -- the same property the ensemble
|
|
466
|
-
# skip path needs, reached from the one site both share.
|
|
467
|
-
yield _pass_failed_event(pass_id, exc)
|
|
468
|
-
raise
|
|
469
|
-
yield PipelineEvent(kind="pass_complete", pass_id=pass_id, ok=True)
|
|
470
|
-
return response.text
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
def _record_pass_thread(pass_id: str, response, pass_threads) -> None:
|
|
474
|
-
"""Note which thread this pass ran in (§27 T2 / AC6).
|
|
475
|
-
|
|
476
|
-
BEFORE _check_not_truncated, which can raise: the thread of a pass that
|
|
477
|
-
died mid-tool-call is the single most useful one to be able to open, and
|
|
478
|
-
recording after the check would lose exactly that one.
|
|
479
|
-
|
|
480
|
-
`pass_threads` IS `run.pass_threads`, shared by reference for the same
|
|
481
|
-
reason §25's granted_calls is: one object means the record is already on
|
|
482
|
-
the run at every §5 checkpoint and on the failure path, rather than
|
|
483
|
-
being assembled at the end by a run that reached the end.
|
|
484
|
-
|
|
485
|
-
A separate optional argument rather than a `run` parameter, because
|
|
486
|
-
`trace` beside it is a list too and _run_pass_with_json_retry takes it
|
|
487
|
-
positionally in twelve call sites -- swapping both for the run object is
|
|
488
|
-
a bigger change than §27 asked for. Default None keeps every test double
|
|
489
|
-
that calls these two functions working.
|
|
490
|
-
"""
|
|
491
|
-
if pass_threads is None:
|
|
492
|
-
return
|
|
493
|
-
thread_id = getattr(response, "thread_id", None)
|
|
494
|
-
if thread_id is None:
|
|
495
|
-
return
|
|
496
|
-
pass_threads.append({"pass": pass_id, "thread_id": str(thread_id)})
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
def _record_granted_calls(pass_id: str, response, authorization) -> None:
|
|
500
|
-
"""Note calls that ran on a pre-flight grant (§25 audit trail).
|
|
501
|
-
|
|
502
|
-
Authorization moved up-front, so accountability moves after: this is
|
|
503
|
-
the record of what one launch-time yes actually authorised, for a run
|
|
504
|
-
nobody was watching.
|
|
505
|
-
|
|
506
|
-
Accumulated ON THE AUTHORIZATION BUNDLE rather than in module state or
|
|
507
|
-
a thirteenth argument. The bundle is already shared by reference across
|
|
508
|
-
every pass -- that is how GrantBudget counts one ceiling rather than
|
|
509
|
-
ten -- and what the authorization was spent on is authorization state.
|
|
510
|
-
A module global would also break the moment two pipelines ran.
|
|
511
|
-
"""
|
|
512
|
-
if authorization is None:
|
|
513
|
-
return
|
|
514
|
-
for call in getattr(response, "granted_calls", ()) or ():
|
|
515
|
-
authorization.granted_calls.append({"pass": pass_id, **call})
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
def _run_pass_with_json_retry(
|
|
519
|
-
pass_id: str,
|
|
520
|
-
pass_input: str,
|
|
521
|
-
model: str,
|
|
522
|
-
provider_name: str,
|
|
523
|
-
trace: list[str] | None = None,
|
|
524
|
-
temperature: float | None = None,
|
|
525
|
-
authorization=None,
|
|
526
|
-
pass_threads: list | None = None,
|
|
527
|
-
claim_ids: list | None = None,
|
|
528
|
-
ensemble: bool = False,
|
|
529
|
-
effort: str | None = None,
|
|
530
|
-
corpus=None,
|
|
531
|
-
activity=None,
|
|
532
|
-
):
|
|
533
|
-
"""Runs one JSON-emitting pass and recovers from malformed JSON.
|
|
534
|
-
|
|
535
|
-
The recovery itself lives in core/reasoning/json_retry.py, shared with
|
|
536
|
-
§20's reviewer. All this function adds is the pass's own first attempt
|
|
537
|
-
and the two pass-specific facts the shared loop cannot know: which
|
|
538
|
-
system prompt to continue under, and that a retry's tool calls belong
|
|
539
|
-
on the same §25 audit trail as the original attempt's.
|
|
540
|
-
|
|
541
|
-
ROADMAP §3: attempts = max_retries + 1 (1 initial +
|
|
542
|
-
config.MAX_JSON_RETRIES corrective, per core/reasoning/json_retry.py).
|
|
543
|
-
Unrecoverable failure raises ValueError with the final parse error
|
|
544
|
-
attached; the caller's try/except turns that into a durable
|
|
545
|
-
status='failed' record.
|
|
546
|
-
|
|
547
|
-
§30 (B3): the payload spec is bound HERE, from the pass id, rather
|
|
548
|
-
than passed in by each of the eight call sites. A call site that
|
|
549
|
-
forgot the argument would be silently unvalidated, which is the exact
|
|
550
|
-
condition this batch exists to remove -- and payload_validation.validate
|
|
551
|
-
raises on a pass id it has no spec for, so an eleventh JSON-emitting
|
|
552
|
-
pass cannot arrive by omission.
|
|
553
|
-
|
|
554
|
-
This function still returns TEXT, deliberately, even though it now
|
|
555
|
-
parses the payload internally to check it. Around twenty test sites
|
|
556
|
-
patch or drain it and assert on the returned string, and §20's
|
|
557
|
-
reviewer shares retry_until_json's contract; re-parsing a short string
|
|
558
|
-
is cheaper than moving twenty seams in the batch that adds the check.
|
|
559
|
-
|
|
560
|
-
`claim_ids` is the set the APPLY step will resolve against -- the
|
|
561
|
-
factual claims for 3a/3b (batch 220), every claim for 4, the
|
|
562
|
-
currently-flagged batch for 6a/6b -- because
|
|
563
|
-
validating against a wider set would accept an entry that then lands
|
|
564
|
-
on nothing.
|
|
565
|
-
|
|
566
|
-
§22: a generator, like _run_pass, and for the same reason. The
|
|
567
|
-
corrective attempts inside retry_until_json need no event kind of
|
|
568
|
-
their own -- that function already appends a trace line per failed
|
|
569
|
-
parse, so _Progress's watermark carries them to the consumer at the
|
|
570
|
-
next checkpoint with json_retry.py untouched. pass_complete fires
|
|
571
|
-
after the retries resolve, so a pass is not reported finished while
|
|
572
|
-
it is still arguing with the model about JSON.
|
|
573
|
-
|
|
574
|
-
§26 limit, stated rather than fixed: the FIRST attempt is translated,
|
|
575
|
-
the retries are not. retry_until_json reaches the model through
|
|
576
|
-
continue_conversation(), which drains its own loop internally, so a
|
|
577
|
-
tool call made while correcting malformed JSON produces no tool_call
|
|
578
|
-
event. Making it visible means a streaming sibling for
|
|
579
|
-
continue_conversation too, which is chat's entry point as well -- a
|
|
580
|
-
larger change than the thing it would show.
|
|
581
|
-
|
|
582
|
-
§45 inherits that limit rather than widening it: a page fetched during
|
|
583
|
-
a corrective retry reaches no corpus either, so a source cited only
|
|
584
|
-
from such a fetch scores with `similarity_method: "llm"` like any
|
|
585
|
-
other source with no captured text. It degrades to the documented
|
|
586
|
-
fallback instead of to a wrong number, which is why the limit is
|
|
587
|
-
stated here and not worked around.
|
|
588
|
-
"""
|
|
589
|
-
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
590
|
-
try:
|
|
591
|
-
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
592
|
-
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
593
|
-
temperature=temperature, authorization=authorization, effort=effort,
|
|
594
|
-
activity=activity,
|
|
595
|
-
), corpus=corpus)
|
|
596
|
-
_record_granted_calls(pass_id, response, authorization)
|
|
597
|
-
# §27: the retries re-enter THIS thread (continue_conversation), so one
|
|
598
|
-
# entry per pass is the whole truth however many corrections it took.
|
|
599
|
-
_record_pass_thread(pass_id, response, pass_threads)
|
|
600
|
-
# BEFORE the retry loop. A truncated pass returned no JSON because it
|
|
601
|
-
# was cut off, not because the model wrote malformed JSON -- sending
|
|
602
|
-
# a "your last response did not parse" nudge into a thread that has
|
|
603
|
-
# already exhausted its budget spends more of it to be told the same
|
|
604
|
-
# thing, and reports a parse failure for something that never parsed
|
|
605
|
-
# because it was never finished.
|
|
606
|
-
_check_not_truncated(pass_id, response, trace)
|
|
607
|
-
|
|
608
|
-
text = retry_until_json(
|
|
609
|
-
response,
|
|
610
|
-
label=pass_id,
|
|
611
|
-
# pass_prompt(), NOT the raw pass file: BOTH the original attempt
|
|
612
|
-
# and its retry must carry the same catalogs, or a retry silently
|
|
613
|
-
# sees a different tool set than the attempt it is correcting.
|
|
614
|
-
# #68 makes that literal -- the same AUTHORIZATION decides the
|
|
615
|
-
# catalogs here as decided them for the attempt.
|
|
616
|
-
system_prompt=system_prompts.pass_prompt(
|
|
617
|
-
pass_id, *advertisement_facts(authorization)),
|
|
618
|
-
model=model,
|
|
619
|
-
provider_name=provider_name,
|
|
620
|
-
trace=trace,
|
|
621
|
-
temperature=temperature,
|
|
622
|
-
authorization=authorization,
|
|
623
|
-
effort=effort,
|
|
624
|
-
on_response=lambda r: _record_granted_calls(pass_id, r, authorization),
|
|
625
|
-
# §47's sink, at the hop batch 73 missed (batch 75). A
|
|
626
|
-
# corrective attempt reaches the model through
|
|
627
|
-
# continue_conversation, which drains its own loop, so a
|
|
628
|
-
# subagent spawned while a pass is being corrected is visible
|
|
629
|
-
# through this channel or not at all.
|
|
630
|
-
activity=activity,
|
|
631
|
-
# The retry re-enters THIS pass's thread. No separate pass
|
|
632
|
-
# ceiling any more (#4): omitting the kwarg lets the wrapper
|
|
633
|
-
# resolve the configured spend cap, exactly as the pass itself
|
|
634
|
-
# did.
|
|
635
|
-
validate=lambda payload: _validate_payload(
|
|
636
|
-
pass_id, payload, claim_ids=claim_ids, ensemble=ensemble),
|
|
637
|
-
)
|
|
638
|
-
except Exception as exc:
|
|
639
|
-
# E14/#115: same uniform failure report as _run_pass -- a JSON
|
|
640
|
-
# pass that exhausts its retries is a pass that ended badly,
|
|
641
|
-
# whatever the exception class.
|
|
642
|
-
yield _pass_failed_event(pass_id, exc)
|
|
643
|
-
raise
|
|
644
|
-
yield PipelineEvent(kind="pass_complete", pass_id=pass_id, ok=True)
|
|
645
|
-
return text
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
def _claim_from_json(raw: dict) -> Claim:
|
|
649
|
-
"""Filters to known input fields so an unexpected extra key from the
|
|
650
|
-
model doesn't crash claim construction; required fields (id/text/type)
|
|
651
|
-
still raise if genuinely missing."""
|
|
652
|
-
filtered = {k: v for k, v in raw.items() if k in _CLAIM_INPUT_FIELDS}
|
|
653
|
-
return Claim(**filtered)
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
# ROADMAP_v2 §30 (B5). THE THREE _apply_* FUNCTIONS RETURN WHAT THEY
|
|
657
|
-
# APPLIED, and their checkpoint lines interpolate that rather than the
|
|
658
|
-
# count the pass was ASKED about.
|
|
659
|
-
#
|
|
660
|
-
# `Pass 3a: grounded {len(unique_entities)} unique entities across
|
|
661
|
-
# {len(factual_claims)} factual claim(s).` is emitted verbatim whether
|
|
662
|
-
# every entry landed or none did -- it is built entirely from this side of
|
|
663
|
-
# the call. Driven with 3a/3b answering about ids Pass 2 never issued, the
|
|
664
|
-
# run reported grounding 4 entities across 3 claims while grounding
|
|
665
|
-
# nothing, then spent four more model calls in the 6a/6c loop and reported
|
|
666
|
-
# that nothing could be verified. base.py:87 calls the trace "at least as
|
|
667
|
-
# trustworthy an artifact as the final report itself"; a line derived from
|
|
668
|
-
# its own input cannot be.
|
|
669
|
-
#
|
|
670
|
-
# A count of CLAIMS TOUCHED, not entries consumed: a pass that answers
|
|
671
|
-
# twice about one claim has affected one claim, and the reader is deciding
|
|
672
|
-
# whether the pass did its job.
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
def _apply_grounding(claims: list[Claim], grounding_entries: list[dict]) -> int:
|
|
676
|
-
by_id = resolve_by_id(claims)
|
|
677
|
-
applied = set()
|
|
678
|
-
for entry in grounding_entries:
|
|
679
|
-
claim = by_id.get(entry.get("claim_id"))
|
|
680
|
-
if claim:
|
|
681
|
-
claim.grounding_sources = entry.get("sources", [])
|
|
682
|
-
claim.grounding_status = entry.get("status")
|
|
683
|
-
applied.add(claim.id)
|
|
684
|
-
return len(applied)
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
def _ground_and_score(run: PipelineRun, grounding_entries: list[dict], corpus,
|
|
688
|
-
scorer=None, knobs=None, scholar=None,
|
|
689
|
-
claims=None) -> int:
|
|
690
|
-
"""Apply Pass 3a/6b's grounding, then everything §45 derives from it.
|
|
691
|
-
|
|
692
|
-
THE ONE SEAM BOTH GROUNDING SITES GO THROUGH. There are two -- Pass 3a
|
|
693
|
-
and Pass 6b's re-validation -- and §45 adds work that has to happen at
|
|
694
|
-
both: a source scored only on the run's first grounding would carry a
|
|
695
|
-
stale number after a revision changed the claim it was scored against.
|
|
696
|
-
Deriving beside each call instead is the shape the "fix at the
|
|
697
|
-
producer" rule exists to prevent, and a third caller could then arrive
|
|
698
|
-
without it.
|
|
699
|
-
|
|
700
|
-
Returns what `_apply_grounding` returned, so the checkpoint lines that
|
|
701
|
-
interpolate it keep reporting what was APPLIED rather than what the
|
|
702
|
-
pass was asked about (§30, B5).
|
|
703
|
-
|
|
704
|
-
`run.source_documents` is refreshed HERE, and this is its only writer.
|
|
705
|
-
Late enough to include everything grounding retrieved, and early
|
|
706
|
-
enough that a run dying in a later pass still carries the evidence it
|
|
707
|
-
had already gathered -- the failure path being where an unattended
|
|
708
|
-
run's record matters most.
|
|
709
|
-
|
|
710
|
-
`claims` NARROWS BOTH HALVES, and defaults to the whole run. Pass 3a
|
|
711
|
-
passes the factual claims since batch 220 (#247), the only ones it is
|
|
712
|
-
shown: the default let an entry naming a speculative claim mark it
|
|
713
|
-
grounded. §66 (A26) passes the 6a/6b loop's current batch, and it has to
|
|
714
|
-
reach the SCORING as well as the apply: `score_grounding_sources`
|
|
715
|
-
"rewrite[s] EVERY claim's grounding_sources with computed scores" by
|
|
716
|
-
default, so a 6b round recomputed the sources of claims D1 had already
|
|
717
|
-
finished with, while the re-tiering beside it was deliberately scoped
|
|
718
|
-
to the batch. Driven with the corpus grown by that round -- which is
|
|
719
|
-
what a grounding round does -- a claim the payload never named had its
|
|
720
|
-
`quote_verified` rewritten from None to True. That one moves no score
|
|
721
|
-
by design (§45 slice A), but the embedding and scholar phases run over
|
|
722
|
-
the same set and do move one, and the claim they would move is not
|
|
723
|
-
re-tiered.
|
|
724
|
-
|
|
725
|
-
A PARAMETER THAT EXISTED AND WAS NEVER PASSED. `score_grounding_sources`
|
|
726
|
-
has had `claims` since §45; the one production caller omitted it, so
|
|
727
|
-
the narrowing it was built for was available and unused.
|
|
728
|
-
"""
|
|
729
|
-
targets = run.claims if claims is None else claims
|
|
730
|
-
applied = _apply_grounding(targets, grounding_entries)
|
|
731
|
-
if corpus is not None:
|
|
732
|
-
run.source_documents = corpus.artifact_entries()
|
|
733
|
-
knobs = knobs or {}
|
|
734
|
-
score_grounding_sources(
|
|
735
|
-
run, claims=claims, corpus=corpus, scorer=scorer, scholar=scholar,
|
|
736
|
-
overrides=knobs.get("domain_overrides"),
|
|
737
|
-
**{k: knobs[k] for k in ("window_chars", "max_windows",
|
|
738
|
-
"claim_min_chars",
|
|
739
|
-
"authority_adjustment_cap") if k in knobs})
|
|
740
|
-
return applied
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
def _sources_were_scored(run: PipelineRun) -> bool:
|
|
744
|
-
"""Whether §45's scoring stage ran for this run.
|
|
745
|
-
|
|
746
|
-
Pass 5 needs this because an empty source list is AMBIGUOUS on its own
|
|
747
|
-
-- "the model claimed grounded and cited nothing" on a scored run,
|
|
748
|
-
"this claim was built without a scoring stage" everywhere else -- and
|
|
749
|
-
only the first should cost a claim its grounding weight.
|
|
750
|
-
|
|
751
|
-
Derived from the run rather than tracked in a flag, so it cannot go
|
|
752
|
-
stale: the stage runs exactly when Pass 3a produced grounding, which
|
|
753
|
-
is exactly when some factual claim carries a status.
|
|
754
|
-
"""
|
|
755
|
-
return any(c.grounding_status is not None for c in run.claims)
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
def _apply_critic(claims: list[Claim], critic_entries: list[dict]) -> int:
|
|
759
|
-
by_id = resolve_by_id(claims)
|
|
760
|
-
applied = set()
|
|
761
|
-
for entry in critic_entries:
|
|
762
|
-
claim = by_id.get(entry.get("claim_id"))
|
|
763
|
-
if claim:
|
|
764
|
-
claim.fallacies = entry.get("fallacies", [])
|
|
765
|
-
claim.contradictions = entry.get("contradictions", [])
|
|
766
|
-
claim.critic_severity = entry.get("severity", 0.0)
|
|
767
|
-
applied.add(claim.id)
|
|
768
|
-
return len(applied)
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
def _note_entries_outside_the_batch(run: PipelineRun, batch: list[Claim],
|
|
772
|
-
*entry_lists, pass_id: str, scope: str,
|
|
773
|
-
asked: str, why: str) -> int:
|
|
774
|
-
"""Say that a pass answered about a claim it was not asked about.
|
|
775
|
-
|
|
776
|
-
§66 (A26). The apply is scoped to the batch, so such an entry lands on
|
|
777
|
-
nothing -- and `review.py` settled the policy for exactly this shape:
|
|
778
|
-
a finding it cannot apply is DROPPED AND TRACED, because dropping in
|
|
779
|
-
silence is indistinguishable from applying. 6b's own checkpoint line
|
|
780
|
-
reports how many claims came out clean, which is an effect and
|
|
781
|
-
therefore cannot report this.
|
|
782
|
-
|
|
783
|
-
FOUR PASSES SINCE BATCH 220. 6a made the same
|
|
784
|
-
drop twenty lines above 6b and said nothing (#248), and 3a and 3b,
|
|
785
|
-
shown only the factual claims, applied to every claim (#247): a 3a
|
|
786
|
-
entry could mark a speculative claim grounded, which gate D0 says cannot
|
|
787
|
-
happen, and a 3b severity on one moved its tier -- measured, MEDIUM to
|
|
788
|
-
LOW at 0.5 and to UNVERIFIED at 0.8, which then cost it 6a/6b rounds.
|
|
789
|
-
Each caller says its own `scope`, what it `asked` about and `why` an
|
|
790
|
-
answer outside it must not land.
|
|
791
|
-
|
|
792
|
-
The id cross-check does not cover it and is not meant to: B5 makes a
|
|
793
|
-
TOTAL mismatch the error and a partial one deliberately not, since a
|
|
794
|
-
model that gets nine of ten right must not cost a run nine passes.
|
|
795
|
-
Measured -- a payload naming the batch PLUS one extra id passes
|
|
796
|
-
`claim_ids=[c.id for c in flagged]` unchanged, which is why the apply
|
|
797
|
-
is where this had to be fixed and the trace is what makes the drop
|
|
798
|
-
visible.
|
|
799
|
-
"""
|
|
800
|
-
known = {c.id for c in batch}
|
|
801
|
-
outside = sorted({entry.get("claim_id") for entries in entry_lists
|
|
802
|
-
for entry in entries if isinstance(entry, dict)}
|
|
803
|
-
- known - {None})
|
|
804
|
-
if outside:
|
|
805
|
-
run.log(
|
|
806
|
-
f"{pass_id}: {len(outside)} claim id(s) in the response name a "
|
|
807
|
-
f"claim outside {scope} "
|
|
808
|
-
f"({', '.join(repr(i) for i in outside[:3])}) -- dropped. "
|
|
809
|
-
f"{pass_id.removeprefix('Pass ')} was asked about {len(known)} "
|
|
810
|
-
f"{asked}; {why}"
|
|
811
|
-
)
|
|
812
|
-
return len(outside)
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
def _apply_assumption_flags(claims: list[Claim], assumptions: dict) -> int:
|
|
816
|
-
by_id = resolve_by_id(claims)
|
|
817
|
-
applied = set()
|
|
818
|
-
for claim_id, flags in assumptions.get("per_claim_flags", {}).items():
|
|
819
|
-
claim = by_id.get(claim_id)
|
|
820
|
-
if claim:
|
|
821
|
-
claim.assumption_flags = flags
|
|
822
|
-
applied.add(claim.id)
|
|
823
|
-
return len(applied)
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
def _synthesis_input(run: PipelineRun, directives: list | None = None) -> str:
|
|
827
|
-
"""What final synthesis reads.
|
|
828
|
-
|
|
829
|
-
A FUNCTION, not a string built once. §20 re-runs synthesis after a
|
|
830
|
-
correction is accepted, and reusing the string built before the review
|
|
831
|
-
would regenerate the report from the UNCORRECTED claims -- producing a
|
|
832
|
-
report that changed for no reason while the correction silently went
|
|
833
|
-
nowhere.
|
|
834
|
-
"""
|
|
835
|
-
text = (
|
|
836
|
-
f"Original query:\n{run.user_query}\n\n"
|
|
837
|
-
f"Final annotated claims:\n{json.dumps([vars(c) for c in run.claims])}\n\n"
|
|
838
|
-
f"Coverage gaps:\n{json.dumps(run.coverage_gaps)}"
|
|
839
|
-
)
|
|
840
|
-
if directives:
|
|
841
|
-
text += (
|
|
842
|
-
"\n\nA reviewer raised the following about the previous draft of "
|
|
843
|
-
"this report, and a human approved each one. Address them:\n"
|
|
844
|
-
+ "\n".join(f"- {d}" for d in directives)
|
|
845
|
-
)
|
|
846
|
-
return text
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
def _review_stage(run: PipelineRun, progress: _Progress, model: str,
|
|
850
|
-
provider_name: str, authorization, review,
|
|
851
|
-
enabled: bool = False, effort: str | None = None,
|
|
852
|
-
activity=None):
|
|
853
|
-
"""ROADMAP_v2 §20. Opt-in review of the finished run, with every
|
|
854
|
-
correction consented to individually.
|
|
855
|
-
|
|
856
|
-
§22: a generator, because it logs, checkpoints and runs a pass. It
|
|
857
|
-
takes the run's _Progress rather than making its own, so review.py's
|
|
858
|
-
trace lines -- which never checkpointed at all before §22 -- land on
|
|
859
|
-
the same watermark as everything else.
|
|
860
|
-
|
|
861
|
-
Called from INSIDE the orchestrator (V3) rather than left to each
|
|
862
|
-
shell. The obvious alternative -- a post-pipeline stage the caller
|
|
863
|
-
invokes, mirroring write_run_artifacts -- is exactly the shape that
|
|
864
|
-
produced this project's live asymmetry: that function has one
|
|
865
|
-
production call site, so /research in the TUI writes no output
|
|
866
|
-
directory at all. The consent OBJECT still comes from the shell,
|
|
867
|
-
because only a shell knows how to reach a human.
|
|
868
|
-
|
|
869
|
-
`enabled` and `review` are SEPARATE on purpose. "The user asked for a
|
|
870
|
-
review" and "someone can be asked about its findings" are different
|
|
871
|
-
facts, and collapsing them loses the case that matters: --review on a
|
|
872
|
-
piped run must still produce findings and still change nothing (V6).
|
|
873
|
-
If the consent object alone enabled the stage, that run would skip the
|
|
874
|
-
review entirely and report nothing at all.
|
|
875
|
-
|
|
876
|
-
A consent object with `enabled` false still runs it, because a caller
|
|
877
|
-
that went to the trouble of building one wants the review.
|
|
878
|
-
|
|
879
|
-
Batch 25 (#139): effort inherits into the reviewer (V7's rule applied
|
|
880
|
-
to one more axis) and into the re-synthesis pass below.
|
|
881
|
-
"""
|
|
882
|
-
if not enabled and review is None:
|
|
883
|
-
return
|
|
884
|
-
|
|
885
|
-
from core.reasoning import review as review_module
|
|
886
|
-
|
|
887
|
-
findings, thread_id = review_module.run_review(
|
|
888
|
-
run, model, provider_name, authorization, effort=effort,
|
|
889
|
-
activity=activity)
|
|
890
|
-
run.log(f"Review: {len(findings)} finding(s) raised.")
|
|
891
|
-
# The findings' CONTENT is durable from this checkpoint on (review
|
|
892
|
-
# r1-2): a kill during the consent walk used to leave only the count,
|
|
893
|
-
# with what the reviewer found unrecoverable. Trace is serialized by
|
|
894
|
-
# every update_pipeline_run, so this needs no new column (V8).
|
|
895
|
-
for f in findings:
|
|
896
|
-
target = f.get("claim_id") or "report"
|
|
897
|
-
excerpt = str(f.get("proposed") or "")[:80]
|
|
898
|
-
run.log(f"Review: finding -- {f.get('kind')} correction to {target}"
|
|
899
|
-
+ (f": {excerpt}" if excerpt else ""))
|
|
900
|
-
yield from progress.checkpoint()
|
|
901
|
-
|
|
902
|
-
decisions = review_module.walk_consent(
|
|
903
|
-
findings, review, run, model=model, provider_name=provider_name,
|
|
904
|
-
thread_id=thread_id, authorization=authorization, effort=effort,
|
|
905
|
-
# §47's sink, for the refinement calls the walk can make. Every
|
|
906
|
-
# hop is a place it can be dropped, and this one was.
|
|
907
|
-
activity=activity)
|
|
908
|
-
run.subagent_reviews = decisions
|
|
909
|
-
yield from progress.checkpoint()
|
|
910
|
-
|
|
911
|
-
# Deferred commit (review f4): corrections land on a copy, and the
|
|
912
|
-
# live claims/report change only after the re-synthesis succeeds. A
|
|
913
|
-
# failed re-synthesis therefore persists a consistent record -- the
|
|
914
|
-
# pre-fix order stored corrected claims beside the stale report.
|
|
915
|
-
changed, corrected, applied = review_module.apply_deferred(run, decisions)
|
|
916
|
-
if changed:
|
|
917
|
-
original_claims = run.claims
|
|
918
|
-
run.claims = corrected
|
|
919
|
-
try:
|
|
920
|
-
run.final_report = yield from _run_pass(
|
|
921
|
-
"Final synthesis",
|
|
922
|
-
_synthesis_input(run, review_module.synthesis_directives(decisions)),
|
|
923
|
-
model, provider_name, authorization=authorization,
|
|
924
|
-
trace=run.trace,
|
|
925
|
-
pass_threads=run.pass_threads,
|
|
926
|
-
effort=effort,
|
|
927
|
-
activity=activity,
|
|
928
|
-
)
|
|
929
|
-
except Exception as e:
|
|
930
|
-
# CONTAINED, not re-raised. A provider error on this one call
|
|
931
|
-
# is the same transient class as a reviewer call failing, and
|
|
932
|
-
# f1 settled that an optional stage must not flip a finished
|
|
933
|
-
# ten-pass run to status='failed'. The deferred commit already
|
|
934
|
-
# guarantees a consistent record: the pre-review claims and the
|
|
935
|
-
# report generated from them, which is a complete run's output
|
|
936
|
-
# minus the corrections -- and the trace says which.
|
|
937
|
-
#
|
|
938
|
-
# A failure that escapes this STAGE (a bug, not a transient
|
|
939
|
-
# provider error) still reaches the pipeline's except and
|
|
940
|
-
# records status='failed'. That distinction is the whole
|
|
941
|
-
# policy, and both halves are test-pinned.
|
|
942
|
-
logger.exception("Review re-synthesis failed; corrections not applied.")
|
|
943
|
-
run.claims = original_claims
|
|
944
|
-
review_module.log_outcomes(run, decisions, applied, committed=False)
|
|
945
|
-
run.log(f"Review: re-synthesis failed ({e}); accepted corrections "
|
|
946
|
-
f"NOT applied -- claims and report left unchanged.")
|
|
947
|
-
yield from progress.checkpoint()
|
|
948
|
-
return
|
|
949
|
-
review_module.log_outcomes(run, decisions, applied, committed=True)
|
|
950
|
-
# NOT re-reviewed. The regress is cut deliberately: a review of the
|
|
951
|
-
# re-synthesised report would need its own consent pass, and so on.
|
|
952
|
-
run.log(f"Review: {len(applied)} correction(s) accepted; final "
|
|
953
|
-
f"synthesis re-run. The re-synthesised report is not "
|
|
954
|
-
f"re-reviewed.")
|
|
955
|
-
else:
|
|
956
|
-
review_module.log_outcomes(run, decisions, applied, committed=False)
|
|
957
|
-
run.log("Review: no correction applied; report unchanged.")
|
|
958
|
-
yield from progress.checkpoint()
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
def _stage(pass_id: str):
|
|
962
|
-
"""One event for a stage that made ZERO model calls (§26).
|
|
963
|
-
|
|
964
|
-
A separate kind from pass_start/pass_complete rather than a pair of
|
|
965
|
-
those, because a code stage cannot meaningfully be "running": it
|
|
966
|
-
completes in the time it takes to yield, and a consumer that showed it
|
|
967
|
-
as in-progress would be showing a state that never exists. Pass 5 and
|
|
968
|
-
Pass 6c are passes by the pipeline's numbering and stages by their
|
|
969
|
-
cost, and this is the field that says which.
|
|
970
|
-
|
|
971
|
-
Before this, every zero-LLM stage was invisible to both shells -- not
|
|
972
|
-
by decision but because _run_pass was the only thing that emitted a
|
|
973
|
-
boundary, and it wraps a model call. So a sidebar showed Pass 4
|
|
974
|
-
followed by Pass 6a with the tiering that decided which claims were
|
|
975
|
-
flagged in between, unmentioned.
|
|
976
|
-
"""
|
|
977
|
-
yield PipelineEvent(kind="stage", pass_id=pass_id)
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
def _tier_events(run: PipelineRun):
|
|
981
|
-
"""One claim_tiered event per claim, after a tiering pass.
|
|
982
|
-
|
|
983
|
-
Called after EVERY run_confidence_tiering() -- Pass 5 and each 6b
|
|
984
|
-
round -- so a consumer sees a claim's tier change rather than only
|
|
985
|
-
its first value. Repeats are by design: the events are a stream of
|
|
986
|
-
"this is the tier now", and a panel re-renders from the latest.
|
|
987
|
-
|
|
988
|
-
Derived from run.claims after the fact rather than emitted inside
|
|
989
|
-
confidence_scoring.py, which makes zero LLM calls and knows nothing
|
|
990
|
-
about events; §22's own warning about two writers of related data
|
|
991
|
-
applies to the tiering just as it does to the trace.
|
|
992
|
-
"""
|
|
993
|
-
for claim in run.claims:
|
|
994
|
-
yield PipelineEvent(kind="claim_tiered", claim_id=claim.id,
|
|
995
|
-
tier=claim.confidence_tier)
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
def _apply_fallback(claim: Claim) -> None:
|
|
999
|
-
"""D2 exhausted / fallback branch -- pure code, no LLM call."""
|
|
1000
|
-
claim.confidence_tier = "UNVERIFIED"
|
|
1001
|
-
claim.annotation = (
|
|
1002
|
-
f"[UNVERIFIED] Could not be adequately verified after {claim.retry_count} "
|
|
1003
|
-
f"revision attempt(s). Treat with caution."
|
|
1004
|
-
)
|
|
1005
|
-
claim.final_text = claim.revision_text or claim.text
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
def _revision_request(claim: Claim) -> dict:
|
|
1009
|
-
"""One claim's entry in Pass 6a's batch: the claim and the feedback
|
|
1010
|
-
that put it there.
|
|
1011
|
-
|
|
1012
|
-
`disagreement_feedback` only on a contested claim (ROADMAP_v3 §90),
|
|
1013
|
-
because D1 may have flagged it for that alone, and 6a cannot address a
|
|
1014
|
-
reason it is not given. `revise.md` says what to do with one."""
|
|
1015
|
-
entry = {
|
|
1016
|
-
"claim": vars(claim),
|
|
1017
|
-
"critic_feedback": {"fallacies": claim.fallacies, "contradictions": claim.contradictions},
|
|
1018
|
-
"grounding_feedback": claim.grounding_sources,
|
|
1019
|
-
"assumption_flags": claim.assumption_flags,
|
|
1020
|
-
}
|
|
1021
|
-
if is_contested(claim):
|
|
1022
|
-
entry["disagreement_feedback"] = {
|
|
1023
|
-
# Deduplicated, as the consistency score it came from is (B8).
|
|
1024
|
-
"asserted_by": len(set(claim.asserted_by_candidates)),
|
|
1025
|
-
"of": claim.score_breakdown["consistency_denominator"],
|
|
1026
|
-
}
|
|
1027
|
-
return entry
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
# ============================================================================
|
|
1031
|
-
# ---- The pipeline ------------------------------------------------------
|
|
1032
|
-
# ============================================================================
|
|
1033
|
-
|
|
1034
|
-
def run_pipeline_to_completion(gen) -> PipelineRun:
|
|
1035
|
-
"""Consumes a stream_deep_research_pipeline() generator fully,
|
|
1036
|
-
discarding intermediate events, and returns the finished PipelineRun.
|
|
1037
|
-
|
|
1038
|
-
The exact counterpart of core.loop.run_to_completion(), including the
|
|
1039
|
-
RuntimeError: a generator that ends without its terminal event has a
|
|
1040
|
-
bug, and returning None instead would push the failure into whichever
|
|
1041
|
-
caller dereferences the run first.
|
|
1042
|
-
"""
|
|
1043
|
-
final = None
|
|
1044
|
-
for event in gen:
|
|
1045
|
-
if event.kind == "run_complete":
|
|
1046
|
-
final = event.run
|
|
1047
|
-
if final is None:
|
|
1048
|
-
raise RuntimeError(
|
|
1049
|
-
"Pipeline generator completed without yielding a run_complete event"
|
|
1050
|
-
)
|
|
1051
|
-
return final
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
def stream_deep_research_pipeline(
|
|
1055
|
-
user_query: str,
|
|
1056
|
-
model: str,
|
|
1057
|
-
provider_name: str = DEFAULT_PROVIDER,
|
|
1058
|
-
ensemble_mode: bool | None = None,
|
|
1059
|
-
ensemble_n: int | None = None,
|
|
1060
|
-
authorization=None,
|
|
1061
|
-
review=None,
|
|
1062
|
-
subagent_review: bool | None = None,
|
|
1063
|
-
effort: str | None = None,
|
|
1064
|
-
# §47. Threaded like `authorization` beside it, and for the same
|
|
1065
|
-
# reason: both are run-scoped values the layers between here and
|
|
1066
|
-
# the model call only forward. Batch 59 stopped at
|
|
1067
|
-
# stream_deep_research_mode's signature; this is the rest of it.
|
|
1068
|
-
activity=None,
|
|
1069
|
-
):
|
|
1070
|
-
"""The pipeline, as a generator of PipelineEvents (§22). The terminal
|
|
1071
|
-
`run_complete` event carries the finished PipelineRun.
|
|
1072
|
-
|
|
1073
|
-
ABANDONMENT (§22, recorded rather than fixed). A consumer that stops
|
|
1074
|
-
iterating early -- a TUI quitting mid-run -- raises GeneratorExit
|
|
1075
|
-
inside the try below, which `except Exception` deliberately does not
|
|
1076
|
-
catch. The record therefore stays status='running', populated through
|
|
1077
|
-
the last checkpoint. That is honest: the run was abandoned, not
|
|
1078
|
-
failed, and forcing 'failed' would make an interrupted run
|
|
1079
|
-
indistinguishable from a broken one. Both shipped consumers drain
|
|
1080
|
-
fully.
|
|
1081
|
-
|
|
1082
|
-
authorization (§25): a core.approval.RunAuthorization built by the
|
|
1083
|
-
shell that launched this run, or None for the pre-§25 behaviour (no
|
|
1084
|
-
grants, nobody to ask, every approval-gated tool hidden from every
|
|
1085
|
-
pass). Carried, not interpreted -- the decisions about WHAT may be
|
|
1086
|
-
granted live in core/reasoning/authorization.py, and the enforcement
|
|
1087
|
-
lives in core/loop.py.
|
|
1088
|
-
|
|
1089
|
-
review (§20): a core.interaction.ResponseChannel, or None (§23 moved
|
|
1090
|
-
it there from core.approval.ReviewConsent). Carried the same
|
|
1091
|
-
way and for the same reason -- only a shell knows how to reach a
|
|
1092
|
-
human. A SECOND parameter rather than a field on the authorization
|
|
1093
|
-
bundle: that bundle answers "may this gated call proceed?" and this
|
|
1094
|
-
answers "may this edit be applied?". None means nothing is applied,
|
|
1095
|
-
even when the review itself runs (V6).
|
|
1096
|
-
|
|
1097
|
-
subagent_review (§20/D9): whether the review stage runs at all. None
|
|
1098
|
-
falls back to config.SUBAGENT_REVIEW, matching ensemble_mode's
|
|
1099
|
-
pattern. Separate from `review` because --review on a piped run must
|
|
1100
|
-
still produce findings while applying nothing."""
|
|
1101
|
-
ensemble_mode = config.ENSEMBLE_MODE if ensemble_mode is None else ensemble_mode
|
|
1102
|
-
subagent_review = (config.SUBAGENT_REVIEW if subagent_review is None
|
|
1103
|
-
else subagent_review)
|
|
1104
|
-
|
|
1105
|
-
# §10 revisit (E4). `ensemble_n` is vestigial: N is len(ENSEMBLE_MODELS)
|
|
1106
|
-
# now, because a separately configured count could disagree with the
|
|
1107
|
-
# roster that produced the candidates, and the denominator of a
|
|
1108
|
-
# confidence score must not be able to lie. The parameter and the
|
|
1109
|
-
# settings.json key are KEPT rather than removed -- unknown settings keys
|
|
1110
|
-
# raise, so deleting the key would make every existing settings.json that
|
|
1111
|
-
# sets it fail to load.
|
|
1112
|
-
if ensemble_n is not None:
|
|
1113
|
-
logger.warning(
|
|
1114
|
-
"Ignoring ensemble_n=%s: the number of candidates is "
|
|
1115
|
-
"len(ensemble_models) now (ROADMAP §10 revisit, E3). "
|
|
1116
|
-
"Change the roster in config.yaml to change N.",
|
|
1117
|
-
ensemble_n,
|
|
1118
|
-
)
|
|
1119
|
-
|
|
1120
|
-
roster = _ensemble_roster(ensemble_mode)
|
|
1121
|
-
|
|
1122
|
-
run = PipelineRun(user_query=user_query)
|
|
1123
|
-
|
|
1124
|
-
# §45 (SQ2). RUN-SCOPED, and attached to every pass rather than only to
|
|
1125
|
-
# the two that ground. The question it answers is "what did this run
|
|
1126
|
-
# retrieve from this URL", which is not a per-pass question: a page
|
|
1127
|
-
# Pass 1 fetched and Pass 3a then cited without re-fetching is still
|
|
1128
|
-
# evidence about that URL, and scoping the corpus to 3a would report
|
|
1129
|
-
# no captured text for it and fall back to the model's own number.
|
|
1130
|
-
corpus = SourceCorpus()
|
|
1131
|
-
|
|
1132
|
-
# §45 (SQ2). Resolved ONCE, before Pass 0, so the warning about
|
|
1133
|
-
# running without one is the first thing the trace says rather than
|
|
1134
|
-
# something a reader meets forty lines in beside a score it already
|
|
1135
|
-
# explains. One scorer for the whole run, because its vector cache and
|
|
1136
|
-
# its give-up flag are both run-scoped: a claim is embedded once
|
|
1137
|
-
# however many sources it has, and a provider that is down stays down.
|
|
1138
|
-
scoring_knobs = effective_source_scoring(warn=True)
|
|
1139
|
-
scorer = make_scorer(run, floor=scoring_knobs["similarity_floor"],
|
|
1140
|
-
ceiling=scoring_knobs["similarity_ceiling"])
|
|
1141
|
-
|
|
1142
|
-
# §45 (SQ9). OFF by default, so an out-of-the-box run makes no call
|
|
1143
|
-
# the user did not ask for. One lookup object for the run, for the
|
|
1144
|
-
# scorer's reason: its cohort cache and its give-up flag are both
|
|
1145
|
-
# run-scoped.
|
|
1146
|
-
scholar = ScholarLookup(knobs=scoring_knobs)
|
|
1147
|
-
|
|
1148
|
-
# §45 (SQ6). Both knob sections resolved ONCE, here, with warn=True so
|
|
1149
|
-
# a mistyped weight is said once per run rather than once per claim --
|
|
1150
|
-
# `effective_compaction`'s gating, for its reason. score_claim stays a
|
|
1151
|
-
# pure function of its arguments, and a run's numbers cannot change
|
|
1152
|
-
# halfway through it.
|
|
1153
|
-
confidence_weights = effective_confidence(warn=True)
|
|
1154
|
-
|
|
1155
|
-
# §25 audit trail: ONE list, shared, not two kept in step. The run and
|
|
1156
|
-
# the authorization bundle now refer to the same object, so a granted
|
|
1157
|
-
# call is visible on the PipelineRun the moment it happens -- including
|
|
1158
|
-
# at every §5 checkpoint and on the FAILURE path, which is where an
|
|
1159
|
-
# unattended run's record matters most. Copying at the end instead
|
|
1160
|
-
# would lose the trail on exactly the runs most worth auditing, and two
|
|
1161
|
-
# writers of related data is the bug shape §22 warns about by name.
|
|
1162
|
-
if authorization is not None:
|
|
1163
|
-
run.granted_calls = authorization.granted_calls
|
|
1164
|
-
|
|
1165
|
-
# ROADMAP §5 minimal core: create the durable record up front. On a
|
|
1166
|
-
# mid-pipeline crash, the except block below flips status to 'failed'
|
|
1167
|
-
# and persists a snapshot of the partial run before re-raising.
|
|
1168
|
-
run.run_id = create_pipeline_run(user_query)
|
|
1169
|
-
|
|
1170
|
-
# ROADMAP §11: resolve critic provider/model once. When no critic is
|
|
1171
|
-
# configured these fall back to the main provider/model — no-op
|
|
1172
|
-
# routing.
|
|
1173
|
-
#
|
|
1174
|
-
# §45 (SQ7): through pipeline_models.resolve rather than
|
|
1175
|
-
# config.CRITIC_MODEL directly, so a `/critic` choice reaches the CLI,
|
|
1176
|
-
# the TUI and a spawned run alike. That function owns the precedence —
|
|
1177
|
-
# the user-tier store outranks config.py — and owning it in one place
|
|
1178
|
-
# is the point: §11 gave this routing to config.py alone, which meant
|
|
1179
|
-
# the methodology's own "a model must not check its own output" was
|
|
1180
|
-
# available only to whoever could edit the checkout.
|
|
1181
|
-
critic = pipeline_models.resolve("critic")
|
|
1182
|
-
critic_provider = critic["provider_name"] if critic else provider_name
|
|
1183
|
-
critic_model = critic["model"] if critic else model
|
|
1184
|
-
|
|
1185
|
-
progress = _Progress(run)
|
|
1186
|
-
|
|
1187
|
-
try:
|
|
1188
|
-
# --- Pass 0: preliminary plan ---
|
|
1189
|
-
run.plan = _parse_json_response((yield from _run_pass_with_json_retry("Pass 0", user_query, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity)))
|
|
1190
|
-
yield from progress.checkpoint(f"Pass 0: plan produced ({len(run.plan.get('key_entities_or_subjects', []))} key entities anticipated).")
|
|
1191
|
-
|
|
1192
|
-
# --- Pass 1: initial generation (ensemble mode: N candidates) ---
|
|
1193
|
-
pass1_input = f"Original query:\n{user_query}\n\nPreliminary plan (loose context, may deviate freely):\n{json.dumps(run.plan)}"
|
|
1194
|
-
effective_n = 0
|
|
1195
|
-
if roster:
|
|
1196
|
-
# A LOOP, not a comprehension: _run_pass is a generator now
|
|
1197
|
-
# (§22), and a comprehension would collect generator objects
|
|
1198
|
-
# without ever running a pass.
|
|
1199
|
-
#
|
|
1200
|
-
# One candidate per roster entry, each on its OWN provider/model
|
|
1201
|
-
# (E1). No new plumbing: _run() calls api_initialization() itself
|
|
1202
|
-
# and effort_for() validates against the receiving model, so a
|
|
1203
|
-
# heterogeneous roster is correct by construction -- the same way
|
|
1204
|
-
# §11 already routes 3a/3b/6c to critic_provider/critic_model.
|
|
1205
|
-
candidates = []
|
|
1206
|
-
for entry in roster:
|
|
1207
|
-
label = f"{entry['provider_name']}/{entry['model']}"
|
|
1208
|
-
try:
|
|
1209
|
-
candidates.append((yield from _run_pass(
|
|
1210
|
-
"Pass 1", pass1_input, entry["model"], entry["provider_name"],
|
|
1211
|
-
authorization=authorization, trace=run.trace,
|
|
1212
|
-
pass_threads=run.pass_threads, effort=effort,
|
|
1213
|
-
activity=activity)))
|
|
1214
|
-
except Exception as e:
|
|
1215
|
-
# NAMED, TRACED, SKIPPED (E7) -- §20's containment rule.
|
|
1216
|
-
# An optional generation strategy must not flip a
|
|
1217
|
-
# ten-pass run to status='failed' because one provider
|
|
1218
|
-
# was unreachable. Same posture MCP takes per server.
|
|
1219
|
-
message = (f"Pass 1: ensemble candidate on {label} failed "
|
|
1220
|
-
f"({type(e).__name__}: {e}) — skipped.")
|
|
1221
|
-
logger.warning("%s", message)
|
|
1222
|
-
run.log(message)
|
|
1223
|
-
continue
|
|
1224
|
-
run.log(f"Pass 1: ensemble candidate {len(candidates)} generated on {label}.")
|
|
1225
|
-
# E13/#77: the record keeps every surviving candidate.
|
|
1226
|
-
# The metadata half sits beside the trace's candidate-N
|
|
1227
|
-
# lines so a stored asserted_by_candidates tag names
|
|
1228
|
-
# models this run can still identify; the text half feeds
|
|
1229
|
-
# the per-candidate artifacts and Pass 3b's input -- the
|
|
1230
|
-
# two places that must be able to check a claim against
|
|
1231
|
-
# what its number actually said. pipeline_storage strips
|
|
1232
|
-
# `text` at serialization; see PipelineRunRecord.
|
|
1233
|
-
run.candidates.append({
|
|
1234
|
-
"candidate": len(candidates),
|
|
1235
|
-
"provider_name": entry["provider_name"],
|
|
1236
|
-
"model": entry["model"],
|
|
1237
|
-
"chars": len(candidates[-1]),
|
|
1238
|
-
"text": candidates[-1],
|
|
1239
|
-
})
|
|
1240
|
-
|
|
1241
|
-
# Labels come from the SURVIVORS, never from position in the
|
|
1242
|
-
# roster (E11). Labelling by roster position leaves a gap when a
|
|
1243
|
-
# candidate fails -- "Candidate 1, Candidate 3" -- so Pass 2
|
|
1244
|
-
# would tag a claim both survivors asserted as [1, 3] while the
|
|
1245
|
-
# denominator said 2, and a unanimous claim would be scored as
|
|
1246
|
-
# dissented-against. A silently deflated confidence score is the
|
|
1247
|
-
# exact failure class this section exists to remove.
|
|
1248
|
-
effective_n = len(candidates)
|
|
1249
|
-
if effective_n >= 2:
|
|
1250
|
-
run.raw_response = candidates[0]
|
|
1251
|
-
pass2_input = "\n\n".join(
|
|
1252
|
-
f"Candidate {i + 1}:\n{c}" for i, c in enumerate(candidates)
|
|
1253
|
-
)
|
|
1254
|
-
run.log(f"Pass 1: ensemble mode — {effective_n} candidate(s) "
|
|
1255
|
-
f"across {effective_n} model(s); consistency is measured "
|
|
1256
|
-
f"out of {effective_n}.")
|
|
1257
|
-
else:
|
|
1258
|
-
# DEGRADE TO THE SINGLE-CANDIDATE PATH (E7), rather than
|
|
1259
|
-
# scoring one candidate as an ensemble of one -- which would
|
|
1260
|
-
# give every claim consistency 1.0, so none could ever be
|
|
1261
|
-
# contested, while reporting that agreement as if it had
|
|
1262
|
-
# been corroborated.
|
|
1263
|
-
# ensemble_n=0 is not a degraded formula, it is the correct
|
|
1264
|
-
# one when there is nothing to compare against.
|
|
1265
|
-
effective_n = 0
|
|
1266
|
-
if candidates:
|
|
1267
|
-
run.raw_response = candidates[0]
|
|
1268
|
-
pass2_input = f"Response to extract claims from:\n{run.raw_response}"
|
|
1269
|
-
run.log("Pass 1: only one ensemble candidate survived — "
|
|
1270
|
-
"no claim can be contested, since one candidate "
|
|
1271
|
-
"cannot disagree with itself.")
|
|
1272
|
-
else:
|
|
1273
|
-
raise RuntimeError(
|
|
1274
|
-
"Pass 1: every ensemble candidate failed; there is no "
|
|
1275
|
-
"response to extract claims from. See the trace for "
|
|
1276
|
-
"each provider's error."
|
|
1277
|
-
)
|
|
1278
|
-
else:
|
|
1279
|
-
run.raw_response = yield from _run_pass("Pass 1", pass1_input, model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity)
|
|
1280
|
-
pass2_input = f"Response to extract claims from:\n{run.raw_response}"
|
|
1281
|
-
run.log("Pass 1: initial generation complete.")
|
|
1282
|
-
yield from progress.checkpoint()
|
|
1283
|
-
|
|
1284
|
-
# --- Pass 2: claim extraction & classification ---
|
|
1285
|
-
claims_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 2", pass2_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, ensemble=effective_n >= 2, effort=effort, activity=activity)))
|
|
1286
|
-
run.claims = [_claim_from_json(c) for c in claims_json]
|
|
1287
|
-
for claim in run.claims:
|
|
1288
|
-
yield PipelineEvent(kind="claim_extracted", claim_id=claim.id,
|
|
1289
|
-
text=claim.text)
|
|
1290
|
-
yield from progress.checkpoint(f"Pass 2: extracted {len(run.claims)} claim(s).")
|
|
1291
|
-
|
|
1292
|
-
# --- gate D0: route by claim type (pure code) ---
|
|
1293
|
-
factual_claims = [c for c in run.claims if c.type == "factual"]
|
|
1294
|
-
non_factual_claims = [c for c in run.claims if c.type != "factual"]
|
|
1295
|
-
yield from _stage("D0")
|
|
1296
|
-
yield from progress.checkpoint(
|
|
1297
|
-
f"D0: {len(factual_claims)} factual claim(s) routed to 3a/3b, "
|
|
1298
|
-
f"{len(non_factual_claims)} synthesis/speculative claim(s) routed directly to Pass 4."
|
|
1299
|
-
)
|
|
1300
|
-
|
|
1301
|
-
# --- Pass 3a + 3b: only for factual claims, batched by deduplicated entity ---
|
|
1302
|
-
if factual_claims:
|
|
1303
|
-
unique_entities = sorted({e for c in factual_claims for e in c.entities})
|
|
1304
|
-
pass3a_input = (
|
|
1305
|
-
f"Factual claims:\n{json.dumps([vars(c) for c in factual_claims])}\n\n"
|
|
1306
|
-
f"Deduplicated entities to research (search each ONCE, map results back "
|
|
1307
|
-
f"to every claim referencing it):\n{json.dumps(unique_entities)}"
|
|
1308
|
-
)
|
|
1309
|
-
# SCOPED TO `factual_claims`, like 6b to its batch (batch 220,
|
|
1310
|
-
# #247): 3a is shown the factual claims and nothing else, so an
|
|
1311
|
-
# entry naming another claim is dropped and traced rather than
|
|
1312
|
-
# marking a claim grounded that gate D0 routed past grounding.
|
|
1313
|
-
grounding_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3a", pass3a_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity)))
|
|
1314
|
-
grounded = _ground_and_score(run, grounding_json, corpus, scorer,
|
|
1315
|
-
scoring_knobs, scholar,
|
|
1316
|
-
claims=factual_claims)
|
|
1317
|
-
yield from progress.checkpoint(f"Pass 3a: grounded {grounded} of {len(factual_claims)} factual claim(s), across {len(unique_entities)} deduplicated entities.")
|
|
1318
|
-
_note_entries_outside_the_batch(
|
|
1319
|
-
run, factual_claims, grounding_json, pass_id="Pass 3a",
|
|
1320
|
-
scope="the factual claims", asked="factual claim(s)",
|
|
1321
|
-
why="Gate D0 routes every other claim past 3a/3b, so grounding "
|
|
1322
|
-
"one would record a check gate D0 says never ran.")
|
|
1323
|
-
|
|
1324
|
-
# E13/#77. Two or more survivors: the claims were extracted
|
|
1325
|
-
# from the UNION of the candidates, so the critic sees every
|
|
1326
|
-
# candidate under the labels Pass 2 used -- a claim only
|
|
1327
|
-
# candidate 3 asserted is checked against text that actually
|
|
1328
|
-
# contains it, and contradictions ACROSS candidates are
|
|
1329
|
-
# findable. Handing it bare `raw_response` (candidate 1)
|
|
1330
|
-
# asked an ill-posed question: find conflicts within a text
|
|
1331
|
-
# much of your input did not come from.
|
|
1332
|
-
if len(run.candidates) >= 2:
|
|
1333
|
-
pass3b_input = (
|
|
1334
|
-
"\n\n".join(
|
|
1335
|
-
f"Candidate {entry['candidate']}:\n{entry['text']}"
|
|
1336
|
-
for entry in run.candidates)
|
|
1337
|
-
+ f"\n\nFactual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1338
|
-
)
|
|
1339
|
-
else:
|
|
1340
|
-
pass3b_input = (
|
|
1341
|
-
f"Raw response:\n{run.raw_response}\n\n"
|
|
1342
|
-
f"Factual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1343
|
-
)
|
|
1344
|
-
critic_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3b", pass3b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity)))
|
|
1345
|
-
# The same scope as 3a, for a stronger reason: a non-factual
|
|
1346
|
-
# claim's score IS its capped critic component, so a severity
|
|
1347
|
-
# landing on one moved its tier (measured, batch 220).
|
|
1348
|
-
critiqued = _apply_critic(factual_claims, critic_json)
|
|
1349
|
-
yield from progress.checkpoint(f"Pass 3b: critique applied to {critiqued} of {len(factual_claims)} factual claim(s).")
|
|
1350
|
-
_note_entries_outside_the_batch(
|
|
1351
|
-
run, factual_claims, critic_json, pass_id="Pass 3b",
|
|
1352
|
-
scope="the factual claims", asked="factual claim(s)",
|
|
1353
|
-
why="Gate D0 routes every other claim past 3a/3b, and a "
|
|
1354
|
-
"severity on one would lower the score Pass 5 gives "
|
|
1355
|
-
"it.")
|
|
1356
|
-
else:
|
|
1357
|
-
# A stage, not a silence. "3a/3b did not run" is a fact about
|
|
1358
|
-
# the run's shape -- it means nothing was factual -- and a
|
|
1359
|
-
# panel that simply skipped from Pass 2 to Pass 3c would leave
|
|
1360
|
-
# the reader to infer it from an absence.
|
|
1361
|
-
yield from _stage("Pass 3a/3b")
|
|
1362
|
-
yield from progress.checkpoint("Pass 3a/3b: skipped -- no factual claims to ground or critique.")
|
|
1363
|
-
|
|
1364
|
-
# --- Pass 3c: completeness, independent of Pass 1's raw_response ---
|
|
1365
|
-
pass3c_input = f"Original query:\n{user_query}\n\nPreliminary plan:\n{json.dumps(run.plan)}"
|
|
1366
|
-
run.completeness = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3c", pass3c_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity)))
|
|
1367
|
-
yield from progress.checkpoint(
|
|
1368
|
-
f"Pass 3c: coverage_score={run.completeness.get('coverage_score')}, "
|
|
1369
|
-
f"{len(run.completeness.get('gaps', []))} gap(s) identified."
|
|
1370
|
-
)
|
|
1371
|
-
|
|
1372
|
-
# --- Pass 4: assumption audit -- ALL claims, plus completeness ---
|
|
1373
|
-
pass4_input = (
|
|
1374
|
-
f"Original query:\n{user_query}\n\n"
|
|
1375
|
-
f"All claims:\n{json.dumps([vars(c) for c in run.claims])}\n\n"
|
|
1376
|
-
f"Completeness findings:\n{json.dumps(run.completeness)}"
|
|
1377
|
-
)
|
|
1378
|
-
run.assumptions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 4", pass4_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in run.claims], effort=effort, activity=activity)))
|
|
1379
|
-
flagged_count = _apply_assumption_flags(run.claims, run.assumptions)
|
|
1380
|
-
yield from progress.checkpoint(f"Pass 4: assumption audit complete, {flagged_count} of {len(run.claims)} claim(s) flagged.")
|
|
1381
|
-
|
|
1382
|
-
# --- Pass 5: confidence tiering (0 LLM calls) ---
|
|
1383
|
-
run_confidence_tiering(run, ensemble_n=effective_n,
|
|
1384
|
-
sources_scored=_sources_were_scored(run),
|
|
1385
|
-
weights=confidence_weights)
|
|
1386
|
-
yield from _stage("Pass 5")
|
|
1387
|
-
yield from _tier_events(run)
|
|
1388
|
-
yield from progress.checkpoint("Pass 5: confidence tiers assigned (pure code, zero LLM calls).")
|
|
1389
|
-
|
|
1390
|
-
# --- D1 + the 6a/6b/D2 retry loop ---
|
|
1391
|
-
# ROADMAP_v3 §90 (A85). D1 also takes the claims most ensemble
|
|
1392
|
-
# candidates did not assert, whatever they scored: disagreement is
|
|
1393
|
-
# a reason to check a claim, and the loop is where claims are
|
|
1394
|
-
# checked. ENTRY ONLY -- `still_flagged` below reads the tier alone.
|
|
1395
|
-
# Who asserted a claim does not change when 6a rewrites it, so a
|
|
1396
|
-
# loop that kept a contested claim for being contested would hold
|
|
1397
|
-
# it to D2's cap and publish it UNVERIFIED, which is the defect the
|
|
1398
|
-
# E8 penalty had.
|
|
1399
|
-
flagged = [c for c in run.claims
|
|
1400
|
-
if c.confidence_tier in FLAGGED_TIERS or is_contested(c)]
|
|
1401
|
-
contested = sum(1 for c in flagged if is_contested(c))
|
|
1402
|
-
yield from _stage("D1")
|
|
1403
|
-
yield from progress.checkpoint(
|
|
1404
|
-
f"D1: {len(flagged)} claim(s) flagged for revision"
|
|
1405
|
-
+ (f" ({contested} contested by the ensemble)" if contested else "")
|
|
1406
|
-
+ f", {len(run.claims) - len(flagged)} already clean.")
|
|
1407
|
-
|
|
1408
|
-
while flagged:
|
|
1409
|
-
# --- Pass 6a: ONE batched call across every currently-flagged claim ---
|
|
1410
|
-
pass6a_input = json.dumps([_revision_request(c) for c in flagged])
|
|
1411
|
-
revisions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6a", pass6a_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity)))
|
|
1412
|
-
|
|
1413
|
-
# COUNT THE ROUND, NOT THE REVISION. The increment used to live
|
|
1414
|
-
# inside the loop below, so only a claim Pass 6a NAMED counted --
|
|
1415
|
-
# and 6a is one batched call across every flagged claim, which is
|
|
1416
|
-
# exactly the shape a model drops an item from. A claim it omitted
|
|
1417
|
-
# could never reach D2's cap, so `while flagged:` had no bound at
|
|
1418
|
-
# all: two model calls per round, for ever. An omission, an empty
|
|
1419
|
-
# list and an
|
|
1420
|
-
# unknown id are indistinguishable here -- the lookup returns None
|
|
1421
|
-
# and the `if claim:` skips in silence -- which is why the bound
|
|
1422
|
-
# cannot be data the model supplies.
|
|
1423
|
-
#
|
|
1424
|
-
# Counting the round makes D2's cap bound the loop by
|
|
1425
|
-
# construction, and since `flagged` only ever shrinks, every claim
|
|
1426
|
-
# still in it exhausts on the same round.
|
|
1427
|
-
for claim in flagged:
|
|
1428
|
-
claim.retry_count += 1
|
|
1429
|
-
|
|
1430
|
-
revised = 0
|
|
1431
|
-
for rev in revisions:
|
|
1432
|
-
# Resolved WITHIN the batch, first-wins like
|
|
1433
|
-
# PipelineRun.claim_by_id: 6a was only asked about `flagged`,
|
|
1434
|
-
# and a revision naming a claim outside it would land on one
|
|
1435
|
-
# already finished -- while carrying a retry_count this round
|
|
1436
|
-
# did not increment, so the event below would report a stale
|
|
1437
|
-
# attempt number.
|
|
1438
|
-
claim = next((c for c in flagged if c.id == rev["claim_id"]), None)
|
|
1439
|
-
if claim:
|
|
1440
|
-
revised += 1
|
|
1441
|
-
claim.revision_text = rev["revised_text"]
|
|
1442
|
-
# §22's "which claim, which attempt" -- emitted from the
|
|
1443
|
-
# claim retry loop, which is the retry the section means.
|
|
1444
|
-
# A JSON-parse retry is a different thing and reaches the
|
|
1445
|
-
# consumer as a trace_line, via json_retry.py's own line.
|
|
1446
|
-
yield PipelineEvent(kind="retry", claim_id=claim.id,
|
|
1447
|
-
attempt=claim.retry_count)
|
|
1448
|
-
yield from progress.checkpoint(f"Pass 6a: revised {revised} of {len(flagged)} claim(s) in this retry round.")
|
|
1449
|
-
# The count above shows that something was dropped, not why
|
|
1450
|
-
# (#248). The comment on the loop bound is about the BOUND,
|
|
1451
|
-
# which the round counter settled; a reader of the trace still
|
|
1452
|
-
# needs to tell an omission from a misdirected revision.
|
|
1453
|
-
_note_entries_outside_the_batch(
|
|
1454
|
-
run, flagged, revisions, pass_id="Pass 6a",
|
|
1455
|
-
scope="this round's batch", asked="flagged claim(s)",
|
|
1456
|
-
why="a revision to a claim D1 had already finished with "
|
|
1457
|
-
"would replace its text without re-validating it.")
|
|
1458
|
-
|
|
1459
|
-
# --- Pass 6b: re-validate the revised subset only (batched, reuses Pass 5's code) ---
|
|
1460
|
-
pass6b_input = json.dumps([vars(c) for c in flagged])
|
|
1461
|
-
# SCOPED TO `flagged`, ALL THREE OF THEM -- the ids validated
|
|
1462
|
-
# against, the claims the entries are applied to, and the
|
|
1463
|
-
# claims whose sources are rescored (§66 A26). 6a was already
|
|
1464
|
-
# resolved within the batch, with the reason written beside it
|
|
1465
|
-
# ("a revision naming a claim outside it would land on one
|
|
1466
|
-
# already finished"); 6b validated against every id in the run
|
|
1467
|
-
# and applied to every claim in it, while the re-tiering below
|
|
1468
|
-
# stayed narrow. So an entry naming a clean claim changed its
|
|
1469
|
-
# grounding and left its tier alone: driven, a HIGH claim
|
|
1470
|
-
# ended up `grounding_status='ungrounded'` with `sources=[]`
|
|
1471
|
-
# beside a breakdown still recording `source_count: 1` -- which
|
|
1472
|
-
# breaks B10's property that the breakdown alone lets a reader
|
|
1473
|
-
# recompute the score.
|
|
1474
|
-
revalidation = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6b", pass6b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity)))
|
|
1475
|
-
grounding_entries = revalidation.get("grounding", [])
|
|
1476
|
-
critic_entries = revalidation.get("critic", [])
|
|
1477
|
-
_ground_and_score(run, grounding_entries, corpus,
|
|
1478
|
-
scorer, scoring_knobs, scholar, claims=flagged)
|
|
1479
|
-
_apply_critic(flagged, critic_entries)
|
|
1480
|
-
_note_entries_outside_the_batch(
|
|
1481
|
-
run, flagged, grounding_entries, critic_entries,
|
|
1482
|
-
pass_id="Pass 6b", scope="this round's batch",
|
|
1483
|
-
asked="flagged claim(s)",
|
|
1484
|
-
why="applying an answer to a claim D1 had already finished "
|
|
1485
|
-
"with would change its grounding without recomputing "
|
|
1486
|
-
"its tier.")
|
|
1487
|
-
# 6b's own checkpoint below already reports an EFFECT -- how
|
|
1488
|
-
# many claims came out clean -- rather than what it was asked
|
|
1489
|
-
# about, so it needs no count from these two.
|
|
1490
|
-
# Pass 5's function again -- code, not a call. Scoped to THIS
|
|
1491
|
-
# round's batch: 6b is only asked about `flagged`, and re-tiering
|
|
1492
|
-
# the whole run once per round recomputes claims that are already
|
|
1493
|
-
# finished, from score data those rounds did not change.
|
|
1494
|
-
run_confidence_tiering(run, ensemble_n=effective_n, claims=flagged,
|
|
1495
|
-
sources_scored=_sources_were_scored(run),
|
|
1496
|
-
weights=confidence_weights)
|
|
1497
|
-
yield from _tier_events(run)
|
|
1498
|
-
|
|
1499
|
-
still_flagged = [c for c in flagged if c.confidence_tier in FLAGGED_TIERS]
|
|
1500
|
-
yield from progress.checkpoint(f"Pass 6b: {len(flagged) - len(still_flagged)} claim(s) now clean, {len(still_flagged)} still flagged.")
|
|
1501
|
-
|
|
1502
|
-
# --- D2: retry cap check (pure code) ---
|
|
1503
|
-
exhausted = [c for c in still_flagged if c.retry_count >= MAX_RETRIES]
|
|
1504
|
-
if exhausted:
|
|
1505
|
-
# ONCE per round, outside the per-claim loop. D2 is one
|
|
1506
|
-
# decision applied to every claim that ran out of retries;
|
|
1507
|
-
# emitting inside the loop would put a D2 row in the panel
|
|
1508
|
-
# per claim, so a round that exhausted six claims would
|
|
1509
|
-
# read as six separate stages.
|
|
1510
|
-
yield from _stage("D2")
|
|
1511
|
-
for c in exhausted:
|
|
1512
|
-
_apply_fallback(c)
|
|
1513
|
-
yield PipelineEvent(kind="claim_tiered", claim_id=c.id,
|
|
1514
|
-
tier=c.confidence_tier)
|
|
1515
|
-
yield from progress.checkpoint(f"D2: claim {c.id} exhausted retries ({c.retry_count}) -> fallback UNVERIFIED.")
|
|
1516
|
-
|
|
1517
|
-
flagged = [c for c in still_flagged if c.retry_count < MAX_RETRIES]
|
|
1518
|
-
|
|
1519
|
-
# --- Pass 6c: annotate (templated, 0 LLM calls) ---
|
|
1520
|
-
for claim in run.claims:
|
|
1521
|
-
if claim.final_text is None:
|
|
1522
|
-
claim.final_text = claim.revision_text or claim.text
|
|
1523
|
-
if claim.annotation is None:
|
|
1524
|
-
claim.annotation = f"[{claim.confidence_tier}]"
|
|
1525
|
-
yield from _stage("Pass 6c")
|
|
1526
|
-
yield from progress.checkpoint("Pass 6c: annotation complete (templated, zero LLM calls).")
|
|
1527
|
-
|
|
1528
|
-
# --- Merge (pure code) ---
|
|
1529
|
-
yield from _stage("Merge")
|
|
1530
|
-
yield from progress.checkpoint(f"Merge: final claim set assembled -- {len(run.claims)} claim(s), {len(run.coverage_gaps)} coverage gap(s).")
|
|
1531
|
-
|
|
1532
|
-
# --- Final synthesis ---
|
|
1533
|
-
run.final_report = yield from _run_pass("Final synthesis", _synthesis_input(run), model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity)
|
|
1534
|
-
yield from progress.checkpoint("Final synthesis complete.")
|
|
1535
|
-
|
|
1536
|
-
# --- §20: review, consent, correct ---
|
|
1537
|
-
yield from _review_stage(run, progress, model, provider_name,
|
|
1538
|
-
authorization, review,
|
|
1539
|
-
enabled=subagent_review, effort=effort,
|
|
1540
|
-
activity=activity)
|
|
1541
|
-
|
|
1542
|
-
update_pipeline_run(run.run_id, run, status="complete")
|
|
1543
|
-
yield PipelineEvent(kind="run_complete", run=run)
|
|
1544
|
-
return
|
|
1545
|
-
except Exception:
|
|
1546
|
-
# Persist the partial state before the exception propagates. The
|
|
1547
|
-
# update is best-effort: if storage itself is what failed, we must
|
|
1548
|
-
# not mask the original error, so a second failure here is logged
|
|
1549
|
-
# (via the real logger -- it reaches stderr through Python's
|
|
1550
|
-
# lastResort handler even before logging_setup is wired in) and the
|
|
1551
|
-
# original exception still re-raises. run.log() would be inert here:
|
|
1552
|
-
# the propagated exception carries no reference back to this local
|
|
1553
|
-
# `run`, and persistence already failed, so a trace entry would
|
|
1554
|
-
# never reach anywhere durable.
|
|
1555
|
-
try:
|
|
1556
|
-
update_pipeline_run(run.run_id, run, status="failed")
|
|
1557
|
-
except Exception as persist_error:
|
|
1558
|
-
logger.error(
|
|
1559
|
-
f"Failed to persist failure state for run {run.run_id}: {persist_error}"
|
|
1560
|
-
)
|
|
1561
|
-
raise
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/orchestrator.py
|
|
3
|
+
|
|
4
|
+
Sequences the deep-research pipeline: Pass 0 -> 1 -> 2 -> gate D0 -> (3a -> 3b
|
|
5
|
+
for factual claims) -> 3c -> 4 (all claims) -> Pass 5 (code) -> D1 -> the
|
|
6
|
+
6a/6b/D2 retry loop -> 6c (template) -> merge -> final synthesis.
|
|
7
|
+
|
|
8
|
+
SCOPE OF THIS VERSION: core sequential pipeline with optional ensemble
|
|
9
|
+
mode (ROADMAP §10: N-candidate Pass 1 generation + a cross-candidate
|
|
10
|
+
consistency check, whose contested claims D1 routes to the 6a/6b loop,
|
|
11
|
+
ROADMAP_v3 §90). No filesystem output yet
|
|
12
|
+
(returns a PipelineRun object; the /output/<run_id>/ file-writing system
|
|
13
|
+
is a separate, later piece of work).
|
|
14
|
+
|
|
15
|
+
ROADMAP §11 (critic-model routing): when a critic pair is configured,
|
|
16
|
+
Pass 3a, 3b, and 6c (which re-runs 3a/3b logic) use the critic
|
|
17
|
+
provider/model instead of the generator's. Every other pass keeps using
|
|
18
|
+
the main provider_name/model. This prevents a model from checking its
|
|
19
|
+
own output for errors with the same blind spots. §45 (SQ7) moved WHERE
|
|
20
|
+
that pair comes from: core/pipeline_models.resolve("critic") owns the
|
|
21
|
+
precedence -- a `/critic` choice in the user-tier store, else
|
|
22
|
+
config.CRITIC_MODEL -- so the routing is reachable without editing the
|
|
23
|
+
checkout, which for §11's whole life it was not.
|
|
24
|
+
|
|
25
|
+
ROADMAP §3 (malformed-JSON recovery): JSON-emitting passes are wrapped in
|
|
26
|
+
_run_pass_with_json_retry, which on a parse failure re-enters the SAME
|
|
27
|
+
pass-thread via RunAgentLoop.continue_conversation() with a corrective
|
|
28
|
+
follow-up message. The failed assistant turn is already in the thread's
|
|
29
|
+
history (see _run()'s always-persist fix in core/loop.py), so the model
|
|
30
|
+
sees its own prior output in context. Up to config.MAX_JSON_RETRIES
|
|
31
|
+
corrective attempts are made; a final ValueError surfaces unrecoverable
|
|
32
|
+
failures.
|
|
33
|
+
|
|
34
|
+
ROADMAP §5 (durable pipeline persistence): each run is wrapped in a
|
|
35
|
+
try/except that records status='failed' (with a snapshot of the partial
|
|
36
|
+
run) before re-raising, so a mid-pipeline crash still leaves a durable
|
|
37
|
+
record. A data-only update_pipeline_run() checkpoint fires after every
|
|
38
|
+
run.log() trace line, so even a hard kill (KeyboardInterrupt / SystemExit,
|
|
39
|
+
which bypass the except Exception block) leaves the record populated
|
|
40
|
+
through the last completed pass. load_pipeline_run() in
|
|
41
|
+
pipeline_storage.py provides the read API.
|
|
42
|
+
|
|
43
|
+
ROADMAP_v2 §22 (pipeline observability): the pipeline is a GENERATOR
|
|
44
|
+
yielding PipelineEvents (core/reasoning/events.py), so a shell can render
|
|
45
|
+
pass boundaries, trace lines, claim tiers and retries as they happen
|
|
46
|
+
rather than waiting for the whole run. Two names:
|
|
47
|
+
|
|
48
|
+
stream_deep_research_pipeline() the generator
|
|
49
|
+
run_pipeline_to_completion() the drainer, for a caller that wants
|
|
50
|
+
only the finished PipelineRun
|
|
51
|
+
|
|
52
|
+
DECISION P3 IS RETIRED, and it is worth saying why rather than quietly
|
|
53
|
+
dropping a name. P3 kept a third, synchronous `run_deep_research_pipeline()`
|
|
54
|
+
-- the drainer applied to the generator -- so that "every existing caller:
|
|
55
|
+
main.py, tui/app.py and fifteen test sites" would go on working unchanged.
|
|
56
|
+
Both shells then left of their own accord: `main.run_research` and
|
|
57
|
+
`tui/app.py` iterate the generator so they can render progress, each with a
|
|
58
|
+
comment saying it deliberately does not call the wrapper. That left a
|
|
59
|
+
public function with no production caller and fifty call sites in tests,
|
|
60
|
+
while this docstring went on naming it the live entry point. The
|
|
61
|
+
convenience moved to `tests/conftest.run_pipeline`, which is what it had
|
|
62
|
+
become; P3's argument was sound and its premise expired.
|
|
63
|
+
|
|
64
|
+
Decision P2: a pass's own LoopEvents are NOT forwarded up as LoopEvents --
|
|
65
|
+
_run_pass TRANSLATES a chosen subset into PipelineEvents (§26 amended this;
|
|
66
|
+
§22 had kept a pass fully opaque). It iterates
|
|
67
|
+
RunAgentLoop.stream_deep_research_mode() directly. `run_deep_research_mode`,
|
|
68
|
+
the synchronous wrapper this line used to name, is gone for
|
|
69
|
+
`run_deep_research_pipeline`'s reason and one layer down.
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
from __future__ import annotations
|
|
73
|
+
|
|
74
|
+
import json
|
|
75
|
+
import logging
|
|
76
|
+
|
|
77
|
+
import config
|
|
78
|
+
import prompts.system_prompts as system_prompts
|
|
79
|
+
from core import pipeline_models
|
|
80
|
+
from core.config_loader import effective_confidence, effective_source_scoring
|
|
81
|
+
from core.loop import RunAgentLoop, advertisement_facts
|
|
82
|
+
from core.reasoning.base import Claim, PipelineRun, resolve_by_id
|
|
83
|
+
from core.reasoning.confidence_scoring import is_contested, run_confidence_tiering
|
|
84
|
+
from core.reasoning.events import PipelineEvent
|
|
85
|
+
from core.reasoning.json_retry import parse_json_response as _parse_json_response
|
|
86
|
+
from core.reasoning.json_retry import retry_until_json
|
|
87
|
+
from core.reasoning.payload_validation import validate as _validate_payload
|
|
88
|
+
from core.reasoning.pipeline_storage import create_pipeline_run, update_pipeline_run
|
|
89
|
+
from core.reasoning.scholar import ScholarLookup
|
|
90
|
+
from core.reasoning.source_corpus import SourceCorpus
|
|
91
|
+
from core.reasoning.source_scoring import make_scorer, score_grounding_sources
|
|
92
|
+
from safety.policy_enforcement import redact_diagnostic_text
|
|
93
|
+
from tools.registry import registry
|
|
94
|
+
|
|
95
|
+
logger = logging.getLogger(__name__)
|
|
96
|
+
|
|
97
|
+
DEFAULT_PROVIDER = "ANTHROPIC"
|
|
98
|
+
FLAGGED_TIERS = {"LOW", "UNVERIFIED"}
|
|
99
|
+
MAX_RETRIES = config.MAX_PIPELINE_RETRIES
|
|
100
|
+
# The JSON-retry budget lives in core/reasoning/json_retry.py since the
|
|
101
|
+
# §20 extraction; a second constant here was a dead knob that invited
|
|
102
|
+
# no-op patches (review §19-20 f9).
|
|
103
|
+
|
|
104
|
+
_CLAIM_INPUT_FIELDS = {"id", "text", "type", "entities", "source_span", "asserted_by_candidates"}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ============================================================================
|
|
108
|
+
# ---- Small helpers ----------------------------------------------------------
|
|
109
|
+
# ============================================================================
|
|
110
|
+
|
|
111
|
+
class _Progress:
|
|
112
|
+
"""The ONE place a trace line becomes durable and becomes an event
|
|
113
|
+
(§22 AC3).
|
|
114
|
+
|
|
115
|
+
Before §22 this file paired every `run.log(...)` with an
|
|
116
|
+
`update_pipeline_run(...)` at twenty call sites, and `run.trace` had
|
|
117
|
+
THREE writers: those, review.py's fifteen `run.log()` calls (which
|
|
118
|
+
checkpointed nowhere), and json_retry.py's bare `trace.append()` per
|
|
119
|
+
failed parse. So "§5's persistence fires on every trace line" was
|
|
120
|
+
already untrue, and adding a per-call-site event emission would have
|
|
121
|
+
made it two independent writers of related data -- the bug shape §22
|
|
122
|
+
warns about by name.
|
|
123
|
+
|
|
124
|
+
So both are DERIVED from `run.trace` itself. `checkpoint()` emits
|
|
125
|
+
every line appended since it last ran and then persists. Because the
|
|
126
|
+
watermark reads the list rather than the call site, review.py's and
|
|
127
|
+
json_retry.py's lines are picked up too, WITHOUT either module being
|
|
128
|
+
edited or knowing this exists.
|
|
129
|
+
|
|
130
|
+
The watermark lives here rather than on PipelineRun because base.py
|
|
131
|
+
is data only and §5 serializes that object -- a field meaningful
|
|
132
|
+
only inside a live generator has no business in a persisted row.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
def __init__(self, run: PipelineRun) -> None:
|
|
136
|
+
self.run = run
|
|
137
|
+
self._emitted = 0
|
|
138
|
+
|
|
139
|
+
def checkpoint(self, message: str | None = None):
|
|
140
|
+
"""Log (optionally), PERSIST, then emit every un-emitted line.
|
|
141
|
+
|
|
142
|
+
Persist BEFORE emitting, not after. A generator only advances
|
|
143
|
+
while someone iterates it, so a consumer that abandons the run
|
|
144
|
+
between the yield and the persist would take that checkpoint down
|
|
145
|
+
with it -- §5's durability would silently start depending on a
|
|
146
|
+
UI's willingness to keep reading. This way, a line is already in
|
|
147
|
+
the record by the time anyone is told about it.
|
|
148
|
+
|
|
149
|
+
Monotonic: `_emitted` only ever advances, so no line can be
|
|
150
|
+
yielded twice however many times this is called.
|
|
151
|
+
"""
|
|
152
|
+
if message is not None:
|
|
153
|
+
self.run.log(message)
|
|
154
|
+
update_pipeline_run(self.run.run_id, self.run)
|
|
155
|
+
while self._emitted < len(self.run.trace):
|
|
156
|
+
line = self.run.trace[self._emitted]
|
|
157
|
+
self._emitted += 1
|
|
158
|
+
yield PipelineEvent(kind="trace_line", text=line)
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# One pass_activity event per this many characters streamed. A per-token
|
|
162
|
+
# event would be thousands of post_message hand-offs per pass, which is the
|
|
163
|
+
# cost Transcript.stream_delta already exists to avoid on the chat path.
|
|
164
|
+
_ACTIVITY_CHARS = 2000
|
|
165
|
+
|
|
166
|
+
# §26's _param_digest, and its two truncation constants, MOVED to
|
|
167
|
+
# safety/policy_enforcement.py in §27 -- beside the redactors, because
|
|
168
|
+
# §27's replay renderer is its second caller and the redact-before-truncate
|
|
169
|
+
# ordering must not exist twice. Imported under its own name rather than
|
|
170
|
+
# aliased back to the old private one: an alias would leave two spellings
|
|
171
|
+
# for one function, and a reader who greps the old name here should be sent
|
|
172
|
+
# to where it now lives, not handed a local rebinding.
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _translate(pass_id: str, stream, corpus=None):
|
|
176
|
+
"""Turn one pass's LoopEvents into PipelineEvents, and return the
|
|
177
|
+
ModelResponse the pass produced (ROADMAP_v2 §26).
|
|
178
|
+
|
|
179
|
+
THE one place a LoopEvent becomes a PipelineEvent. Both _run_pass and
|
|
180
|
+
_run_pass_with_json_retry consume this rather than each iterating the
|
|
181
|
+
stream themselves -- two emission sites for the same translation is how
|
|
182
|
+
they drift, and this file has the twenty-checkpoint version of that
|
|
183
|
+
mistake in its own history (see _Progress).
|
|
184
|
+
|
|
185
|
+
What is translated is a SUBSET, chosen for what a person watching a
|
|
186
|
+
ten-pass run needs:
|
|
187
|
+
|
|
188
|
+
tool_call / tool_result the thing §22 P2 made invisible. A pass
|
|
189
|
+
making fourteen fetch_url calls over
|
|
190
|
+
several minutes looked exactly like one
|
|
191
|
+
that had hung.
|
|
192
|
+
pass_activity for the passes that call no tools at all.
|
|
193
|
+
Throttled: a total, every _ACTIVITY_CHARS.
|
|
194
|
+
|
|
195
|
+
What is NOT translated, and why:
|
|
196
|
+
|
|
197
|
+
token_delta text seven of the ten passes emit raw JSON, so
|
|
198
|
+
streaming pass output into a transcript
|
|
199
|
+
buries the report under it. Only the volume
|
|
200
|
+
escapes, never the content.
|
|
201
|
+
thinking_delta §38 (O5). Not translated AND not counted
|
|
202
|
+
into pass_activity's running total, so
|
|
203
|
+
`chars` keeps meaning one thing -- output
|
|
204
|
+
streamed so far -- rather than quietly
|
|
205
|
+
becoming two instruments the way batch 27
|
|
206
|
+
item 9 describes for the token figures. The
|
|
207
|
+
cost is stated rather than hidden: a pass
|
|
208
|
+
that thinks for minutes before writing
|
|
209
|
+
anything shows no activity, which is the
|
|
210
|
+
shape §26 built pass_activity to cure. Flip
|
|
211
|
+
it here if that turns up in a real run.
|
|
212
|
+
retract batch 113. A retraction tells a surface
|
|
213
|
+
that DREW model text to take it back, and
|
|
214
|
+
this one never drew any -- `chars` is the
|
|
215
|
+
only thing that escaped. Left monotonic
|
|
216
|
+
deliberately: the total measures what the
|
|
217
|
+
pass produced, it is the panel's liveness
|
|
218
|
+
figure rather than a bill, and a running
|
|
219
|
+
total that jumped backwards mid-pass
|
|
220
|
+
would read as the defect instead of the
|
|
221
|
+
honesty. The TUI's own output-char meter
|
|
222
|
+
is left alone for the same reason.
|
|
223
|
+
permission_request §25's ApprovalProvider already carries this
|
|
224
|
+
to the shell and blocks on the answer. A
|
|
225
|
+
second rendering of a question already on
|
|
226
|
+
screen is not observability.
|
|
227
|
+
notice a pass's compaction notices, forwarded as
|
|
228
|
+
trace LINES. M6 makes these backstop-only,
|
|
229
|
+
but "rare" is not "silent" -- §21's rule
|
|
230
|
+
has no exception for the pipeline, and a
|
|
231
|
+
backstop fold rewrites what the REST of
|
|
232
|
+
the pass sees. This used to drop them with
|
|
233
|
+
the claim that they "already reach the
|
|
234
|
+
shell as WARNING log records", which was
|
|
235
|
+
true of blocked and failed compactions and
|
|
236
|
+
FALSE of successful ones (INFO only --
|
|
237
|
+
audit #172). A trace line lands in
|
|
238
|
+
trace.md and both shells print it.
|
|
239
|
+
|
|
240
|
+
§45 (SQ2). `corpus` is a SECOND SINK for tool results, not a change to
|
|
241
|
+
what is translated. The rule above -- a successful result body never
|
|
242
|
+
reaches the UI -- is unchanged; this is a different consumer with a
|
|
243
|
+
different need, and the scoring stage cannot check a quote against a
|
|
244
|
+
page nobody kept. None, the default, collects nothing: that is every
|
|
245
|
+
caller that predates §45 and every test double."""
|
|
246
|
+
names: dict = {} # tool_call id -> name, since tool_result has none
|
|
247
|
+
streamed = 0
|
|
248
|
+
announced = 0
|
|
249
|
+
while True:
|
|
250
|
+
# next() rather than `for event in stream`, because a for loop
|
|
251
|
+
# SWALLOWS the generator's return value -- and the response is
|
|
252
|
+
# returned rather than read off the terminal event precisely
|
|
253
|
+
# because thread_id is attached after the loop finishes. A for
|
|
254
|
+
# loop here would hand every pass a response with no thread_id,
|
|
255
|
+
# and the only symptom would be §3's JSON retry silently starting
|
|
256
|
+
# a new thread instead of correcting the failed one.
|
|
257
|
+
try:
|
|
258
|
+
event = next(stream)
|
|
259
|
+
except StopIteration as stop:
|
|
260
|
+
return stop.value
|
|
261
|
+
|
|
262
|
+
if event.token_delta:
|
|
263
|
+
streamed += len(event.token_delta)
|
|
264
|
+
if streamed - announced >= _ACTIVITY_CHARS:
|
|
265
|
+
announced = streamed
|
|
266
|
+
yield PipelineEvent(kind="pass_activity", pass_id=pass_id,
|
|
267
|
+
chars=streamed)
|
|
268
|
+
if event.notice:
|
|
269
|
+
yield PipelineEvent(
|
|
270
|
+
kind="trace_line",
|
|
271
|
+
text=f"compaction during {pass_id}: {event.notice['text']}")
|
|
272
|
+
if event.tool_call_start:
|
|
273
|
+
name = event.tool_call_start.get("name")
|
|
274
|
+
names[event.tool_call_start.get("id")] = name
|
|
275
|
+
yield PipelineEvent(
|
|
276
|
+
kind="tool_call", pass_id=pass_id, tool=name,
|
|
277
|
+
# §42 (RA3): through the registry, so a tool's
|
|
278
|
+
# declared rationale param is dropped here too -- a
|
|
279
|
+
# pass row is as narrow as a transcript line.
|
|
280
|
+
text=registry.call_digest(
|
|
281
|
+
name, event.tool_call_start.get("input")))
|
|
282
|
+
if event.tool_result:
|
|
283
|
+
result = event.tool_result.get("result")
|
|
284
|
+
failed = isinstance(result, dict) and "error" in result
|
|
285
|
+
name = names.get(event.tool_result.get("id"))
|
|
286
|
+
if corpus is not None and not failed:
|
|
287
|
+
# BEFORE the yield, for #42's reason one layer up: a
|
|
288
|
+
# generator only advances while someone iterates it, so
|
|
289
|
+
# collecting after the yield would make the corpus depend
|
|
290
|
+
# on a consumer continuing to read. `corpus.add` ignores
|
|
291
|
+
# every tool it does not recognise.
|
|
292
|
+
corpus.add(name, result)
|
|
293
|
+
yield PipelineEvent(
|
|
294
|
+
kind="tool_result", pass_id=pass_id,
|
|
295
|
+
tool=name,
|
|
296
|
+
ok=not failed,
|
|
297
|
+
# Redacted here too, though dispatch() already ran the
|
|
298
|
+
# result through check_output_policy. One rule owned at one
|
|
299
|
+
# boundary -- "nothing leaves this function unredacted" --
|
|
300
|
+
# beats depending on a guarantee made two layers away that
|
|
301
|
+
# a later refactor could quietly drop.
|
|
302
|
+
text=(redact_diagnostic_text(str(result.get("error")))
|
|
303
|
+
if failed else None))
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _check_not_truncated(pass_id: str, response, trace: list[str] | None):
|
|
307
|
+
"""A pass that ended on a stop condition did NOT finish; say so.
|
|
308
|
+
|
|
309
|
+
`_run_pass` used to return `response.text` and discard `stop_reason`,
|
|
310
|
+
so a pass cut off by the token budget looked exactly like one that
|
|
311
|
+
completed -- and because a budget stop returns the last response AS
|
|
312
|
+
IT STANDS, a pass that was mid-tool-call returned the EMPTY STRING.
|
|
313
|
+
|
|
314
|
+
That is not hypothetical. A Pass 1 making 14 tool calls crossed the
|
|
315
|
+
old 250k ceiling, returned "", and the orchestrator stored it as
|
|
316
|
+
raw_response. Pass 2 was then handed "Response to extract claims
|
|
317
|
+
from:\\n" and correctly answered that there was nothing to extract --
|
|
318
|
+
in a shape that is not a claim list -- and the run finally died in
|
|
319
|
+
Claim's constructor, three passes and one confusing TypeError away
|
|
320
|
+
from the thing that actually went wrong.
|
|
321
|
+
|
|
322
|
+
TWO outcomes, because truncation is not uniformly fatal:
|
|
323
|
+
|
|
324
|
+
* truncated WITH text -- degraded but usable. The pass produced an
|
|
325
|
+
answer and was cut off finishing it; downstream passes can work
|
|
326
|
+
with that. Traced, so the report's own audit log says the answer
|
|
327
|
+
was incomplete, and the run continues.
|
|
328
|
+
* truncated with NO text -- unusable. Everything downstream is
|
|
329
|
+
working from an empty string, and the only question is how many
|
|
330
|
+
passes it takes to produce an incomprehensible error. Raise here,
|
|
331
|
+
naming the pass and the stop reason.
|
|
332
|
+
"""
|
|
333
|
+
stop = getattr(response, "stop_reason", None)
|
|
334
|
+
if stop in (None, "complete"):
|
|
335
|
+
return
|
|
336
|
+
text = (getattr(response, "text", "") or "").strip()
|
|
337
|
+
# #4: the figures turn a bare reason into a diagnosis. A user who
|
|
338
|
+
# reads token_budget_exceeded as a context problem is repeating the
|
|
339
|
+
# exact misreading item 9 recorded.
|
|
340
|
+
billed = getattr(response, "turn_billed_tokens", None)
|
|
341
|
+
figures = (f" billed {billed:,} this turn"
|
|
342
|
+
if isinstance(billed, int) else "")
|
|
343
|
+
if not text:
|
|
344
|
+
raise ValueError(
|
|
345
|
+
f"{pass_id} ended on {stop!r} with no text{figures}. A stop "
|
|
346
|
+
f"condition returns the last response as it stands, so a pass "
|
|
347
|
+
f"cut off mid-tool-call yields an empty string -- and every "
|
|
348
|
+
f"later pass would be working from nothing. "
|
|
349
|
+
f"(config.MAX_ITERATIONS and a configured settings.json "
|
|
350
|
+
f"max_token_budget spend cap are the two ceilings that "
|
|
351
|
+
f"produce this.)"
|
|
352
|
+
)
|
|
353
|
+
message = (f"{pass_id}: TRUNCATED ({stop}{figures}) -- the pass was cut "
|
|
354
|
+
f"off but produced output; continuing with what it returned.")
|
|
355
|
+
logger.warning("%s", message)
|
|
356
|
+
if trace is not None:
|
|
357
|
+
trace.append(message)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _ensemble_roster(ensemble_mode: bool) -> list[dict]:
|
|
361
|
+
"""The (provider_name, model) pairs Pass 1 runs on, or [] when ensemble
|
|
362
|
+
mode is off. Raises ValueError on a roster that cannot produce a
|
|
363
|
+
meaningful consistency signal (ROADMAP §10 revisit, E1/E5).
|
|
364
|
+
|
|
365
|
+
A separate function so every refusal is testable without running a
|
|
366
|
+
pipeline, and so the reasons stay together: this replaces §16's
|
|
367
|
+
sampling-parameter guard, which refused for a mechanism that no longer
|
|
368
|
+
exists.
|
|
369
|
+
|
|
370
|
+
Validated by SHAPE only -- provider names and API keys are not checked
|
|
371
|
+
here (E6). §16's /model warns rather than refuses on an empty API_KEY so
|
|
372
|
+
a local OpenAI-compatible endpoint stays usable, and MCP's posture is
|
|
373
|
+
that validation happens by connecting. A candidate whose provider is
|
|
374
|
+
unreachable is named, traced and skipped at run time instead.
|
|
375
|
+
"""
|
|
376
|
+
if not ensemble_mode:
|
|
377
|
+
return []
|
|
378
|
+
|
|
379
|
+
roster = config.ENSEMBLE_MODELS
|
|
380
|
+
if not roster:
|
|
381
|
+
raise ValueError(
|
|
382
|
+
"Ensemble mode is on but ensemble_models in config.yaml is empty. "
|
|
383
|
+
"Diversity comes from different models now, not from a raised "
|
|
384
|
+
"temperature (ROADMAP §10 revisit), so the roster is required. "
|
|
385
|
+
"Set it in config.yaml to a list of at least two distinct "
|
|
386
|
+
'{"provider_name": ..., "model": ...} entries, or disable '
|
|
387
|
+
"ensemble_mode. There is deliberately no settings.json key for "
|
|
388
|
+
"the roster -- see E2."
|
|
389
|
+
)
|
|
390
|
+
|
|
391
|
+
for entry in roster:
|
|
392
|
+
if not isinstance(entry, dict) or not entry.get("provider_name") or not entry.get("model"):
|
|
393
|
+
raise ValueError(
|
|
394
|
+
f"Malformed ensemble_models entry in config.yaml {entry!r}: every "
|
|
395
|
+
f'entry needs both "provider_name" and "model".'
|
|
396
|
+
)
|
|
397
|
+
|
|
398
|
+
# DISTINCT pairs, not merely two entries. Repeating one model N times
|
|
399
|
+
# would generate N near-identical candidates -- there is no sampling
|
|
400
|
+
# variation left to make them differ -- and then report maximal
|
|
401
|
+
# cross-candidate consistency for every claim, which Pass 5 reads as
|
|
402
|
+
# confidence. That is precisely the failure §16's guard was added to
|
|
403
|
+
# prevent, and this config is the way to recreate it.
|
|
404
|
+
distinct = {(e["provider_name"], e["model"]) for e in roster}
|
|
405
|
+
if len(distinct) < 2:
|
|
406
|
+
raise ValueError(
|
|
407
|
+
f"Ensemble mode needs at least two DISTINCT models; "
|
|
408
|
+
f"ensemble_models in config.yaml resolves to {len(distinct)} "
|
|
409
|
+
f"({sorted(distinct)}). One model cannot disagree with itself, so "
|
|
410
|
+
f"every claim would score maximal consistency and Pass 5 would "
|
|
411
|
+
f"read that as confidence."
|
|
412
|
+
)
|
|
413
|
+
return list(roster)
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _pass_failed_event(pass_id: str, exc: BaseException) -> PipelineEvent:
|
|
417
|
+
"""E14/#115. The one shape a pass failure is reported in: a
|
|
418
|
+
pass_complete carrying ok=False, mirroring tool_result's
|
|
419
|
+
outcome-plus-reason contract. A NEW kind was considered and rejected
|
|
420
|
+
-- tool_result already established payload-carried outcomes in this
|
|
421
|
+
very event set, and a second mechanism for 'ended' would make every
|
|
422
|
+
future consumer learn which applies where. The text is TRUNCATED:
|
|
423
|
+
the trace line (run.log at whatever site caught this) carries the
|
|
424
|
+
full cause, and an event payload is not the place to duplicate it
|
|
425
|
+
unboundedly."""
|
|
426
|
+
return PipelineEvent(
|
|
427
|
+
kind="pass_complete", pass_id=pass_id, ok=False,
|
|
428
|
+
text=f"{type(exc).__name__}: {exc}"[:200])
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _run_pass(pass_id: str, pass_input: str, model: str, provider_name: str,
|
|
432
|
+
temperature: float | None = None, authorization=None,
|
|
433
|
+
trace: list[str] | None = None, pass_threads: list | None = None,
|
|
434
|
+
effort: str | None = None, corpus=None, activity=None):
|
|
435
|
+
"""One LLM-backed pass. Every actual model call in this file goes
|
|
436
|
+
through this single function.
|
|
437
|
+
|
|
438
|
+
§22: a GENERATOR that yields the pass boundary and returns the
|
|
439
|
+
response text (`text = yield from _run_pass(...)`). The pass id is
|
|
440
|
+
named once here rather than restated beside twelve call sites.
|
|
441
|
+
|
|
442
|
+
§26: the pass is now ITERATED rather than called, through _translate,
|
|
443
|
+
so what it does between its start and its end is visible. Same shape
|
|
444
|
+
as §22's own change to the shells -- the drainer still exists and is
|
|
445
|
+
still what every other caller uses.
|
|
446
|
+
|
|
447
|
+
Batch 25 (#139): effort reaches the passes like authorization does --
|
|
448
|
+
threaded from the shell's setting through the pipeline, validated per
|
|
449
|
+
RECEIVING model by effort_for() inside the loop, so an ensemble roster
|
|
450
|
+
or critic routing needs no effort-specific handling here.
|
|
451
|
+
"""
|
|
452
|
+
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
453
|
+
try:
|
|
454
|
+
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
455
|
+
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
456
|
+
temperature=temperature, authorization=authorization, effort=effort,
|
|
457
|
+
activity=activity,
|
|
458
|
+
), corpus=corpus)
|
|
459
|
+
_record_granted_calls(pass_id, response, authorization)
|
|
460
|
+
_record_pass_thread(pass_id, response, pass_threads)
|
|
461
|
+
_check_not_truncated(pass_id, response, trace)
|
|
462
|
+
except Exception as exc:
|
|
463
|
+
# E14/#115, UNIFORM: a fatal pass reports itself before its
|
|
464
|
+
# exception propagates, so no shell is left holding a row that
|
|
465
|
+
# only ever says "running" -- the same property the ensemble
|
|
466
|
+
# skip path needs, reached from the one site both share.
|
|
467
|
+
yield _pass_failed_event(pass_id, exc)
|
|
468
|
+
raise
|
|
469
|
+
yield PipelineEvent(kind="pass_complete", pass_id=pass_id, ok=True)
|
|
470
|
+
return response.text
|
|
471
|
+
|
|
472
|
+
|
|
473
|
+
def _record_pass_thread(pass_id: str, response, pass_threads) -> None:
|
|
474
|
+
"""Note which thread this pass ran in (§27 T2 / AC6).
|
|
475
|
+
|
|
476
|
+
BEFORE _check_not_truncated, which can raise: the thread of a pass that
|
|
477
|
+
died mid-tool-call is the single most useful one to be able to open, and
|
|
478
|
+
recording after the check would lose exactly that one.
|
|
479
|
+
|
|
480
|
+
`pass_threads` IS `run.pass_threads`, shared by reference for the same
|
|
481
|
+
reason §25's granted_calls is: one object means the record is already on
|
|
482
|
+
the run at every §5 checkpoint and on the failure path, rather than
|
|
483
|
+
being assembled at the end by a run that reached the end.
|
|
484
|
+
|
|
485
|
+
A separate optional argument rather than a `run` parameter, because
|
|
486
|
+
`trace` beside it is a list too and _run_pass_with_json_retry takes it
|
|
487
|
+
positionally in twelve call sites -- swapping both for the run object is
|
|
488
|
+
a bigger change than §27 asked for. Default None keeps every test double
|
|
489
|
+
that calls these two functions working.
|
|
490
|
+
"""
|
|
491
|
+
if pass_threads is None:
|
|
492
|
+
return
|
|
493
|
+
thread_id = getattr(response, "thread_id", None)
|
|
494
|
+
if thread_id is None:
|
|
495
|
+
return
|
|
496
|
+
pass_threads.append({"pass": pass_id, "thread_id": str(thread_id)})
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _record_granted_calls(pass_id: str, response, authorization) -> None:
|
|
500
|
+
"""Note calls that ran on a pre-flight grant (§25 audit trail).
|
|
501
|
+
|
|
502
|
+
Authorization moved up-front, so accountability moves after: this is
|
|
503
|
+
the record of what one launch-time yes actually authorised, for a run
|
|
504
|
+
nobody was watching.
|
|
505
|
+
|
|
506
|
+
Accumulated ON THE AUTHORIZATION BUNDLE rather than in module state or
|
|
507
|
+
a thirteenth argument. The bundle is already shared by reference across
|
|
508
|
+
every pass -- that is how GrantBudget counts one ceiling rather than
|
|
509
|
+
ten -- and what the authorization was spent on is authorization state.
|
|
510
|
+
A module global would also break the moment two pipelines ran.
|
|
511
|
+
"""
|
|
512
|
+
if authorization is None:
|
|
513
|
+
return
|
|
514
|
+
for call in getattr(response, "granted_calls", ()) or ():
|
|
515
|
+
authorization.granted_calls.append({"pass": pass_id, **call})
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _run_pass_with_json_retry(
|
|
519
|
+
pass_id: str,
|
|
520
|
+
pass_input: str,
|
|
521
|
+
model: str,
|
|
522
|
+
provider_name: str,
|
|
523
|
+
trace: list[str] | None = None,
|
|
524
|
+
temperature: float | None = None,
|
|
525
|
+
authorization=None,
|
|
526
|
+
pass_threads: list | None = None,
|
|
527
|
+
claim_ids: list | None = None,
|
|
528
|
+
ensemble: bool = False,
|
|
529
|
+
effort: str | None = None,
|
|
530
|
+
corpus=None,
|
|
531
|
+
activity=None,
|
|
532
|
+
):
|
|
533
|
+
"""Runs one JSON-emitting pass and recovers from malformed JSON.
|
|
534
|
+
|
|
535
|
+
The recovery itself lives in core/reasoning/json_retry.py, shared with
|
|
536
|
+
§20's reviewer. All this function adds is the pass's own first attempt
|
|
537
|
+
and the two pass-specific facts the shared loop cannot know: which
|
|
538
|
+
system prompt to continue under, and that a retry's tool calls belong
|
|
539
|
+
on the same §25 audit trail as the original attempt's.
|
|
540
|
+
|
|
541
|
+
ROADMAP §3: attempts = max_retries + 1 (1 initial +
|
|
542
|
+
config.MAX_JSON_RETRIES corrective, per core/reasoning/json_retry.py).
|
|
543
|
+
Unrecoverable failure raises ValueError with the final parse error
|
|
544
|
+
attached; the caller's try/except turns that into a durable
|
|
545
|
+
status='failed' record.
|
|
546
|
+
|
|
547
|
+
§30 (B3): the payload spec is bound HERE, from the pass id, rather
|
|
548
|
+
than passed in by each of the eight call sites. A call site that
|
|
549
|
+
forgot the argument would be silently unvalidated, which is the exact
|
|
550
|
+
condition this batch exists to remove -- and payload_validation.validate
|
|
551
|
+
raises on a pass id it has no spec for, so an eleventh JSON-emitting
|
|
552
|
+
pass cannot arrive by omission.
|
|
553
|
+
|
|
554
|
+
This function still returns TEXT, deliberately, even though it now
|
|
555
|
+
parses the payload internally to check it. Around twenty test sites
|
|
556
|
+
patch or drain it and assert on the returned string, and §20's
|
|
557
|
+
reviewer shares retry_until_json's contract; re-parsing a short string
|
|
558
|
+
is cheaper than moving twenty seams in the batch that adds the check.
|
|
559
|
+
|
|
560
|
+
`claim_ids` is the set the APPLY step will resolve against -- the
|
|
561
|
+
factual claims for 3a/3b (batch 220), every claim for 4, the
|
|
562
|
+
currently-flagged batch for 6a/6b -- because
|
|
563
|
+
validating against a wider set would accept an entry that then lands
|
|
564
|
+
on nothing.
|
|
565
|
+
|
|
566
|
+
§22: a generator, like _run_pass, and for the same reason. The
|
|
567
|
+
corrective attempts inside retry_until_json need no event kind of
|
|
568
|
+
their own -- that function already appends a trace line per failed
|
|
569
|
+
parse, so _Progress's watermark carries them to the consumer at the
|
|
570
|
+
next checkpoint with json_retry.py untouched. pass_complete fires
|
|
571
|
+
after the retries resolve, so a pass is not reported finished while
|
|
572
|
+
it is still arguing with the model about JSON.
|
|
573
|
+
|
|
574
|
+
§26 limit, stated rather than fixed: the FIRST attempt is translated,
|
|
575
|
+
the retries are not. retry_until_json reaches the model through
|
|
576
|
+
continue_conversation(), which drains its own loop internally, so a
|
|
577
|
+
tool call made while correcting malformed JSON produces no tool_call
|
|
578
|
+
event. Making it visible means a streaming sibling for
|
|
579
|
+
continue_conversation too, which is chat's entry point as well -- a
|
|
580
|
+
larger change than the thing it would show.
|
|
581
|
+
|
|
582
|
+
§45 inherits that limit rather than widening it: a page fetched during
|
|
583
|
+
a corrective retry reaches no corpus either, so a source cited only
|
|
584
|
+
from such a fetch scores with `similarity_method: "llm"` like any
|
|
585
|
+
other source with no captured text. It degrades to the documented
|
|
586
|
+
fallback instead of to a wrong number, which is why the limit is
|
|
587
|
+
stated here and not worked around.
|
|
588
|
+
"""
|
|
589
|
+
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
590
|
+
try:
|
|
591
|
+
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
592
|
+
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
593
|
+
temperature=temperature, authorization=authorization, effort=effort,
|
|
594
|
+
activity=activity,
|
|
595
|
+
), corpus=corpus)
|
|
596
|
+
_record_granted_calls(pass_id, response, authorization)
|
|
597
|
+
# §27: the retries re-enter THIS thread (continue_conversation), so one
|
|
598
|
+
# entry per pass is the whole truth however many corrections it took.
|
|
599
|
+
_record_pass_thread(pass_id, response, pass_threads)
|
|
600
|
+
# BEFORE the retry loop. A truncated pass returned no JSON because it
|
|
601
|
+
# was cut off, not because the model wrote malformed JSON -- sending
|
|
602
|
+
# a "your last response did not parse" nudge into a thread that has
|
|
603
|
+
# already exhausted its budget spends more of it to be told the same
|
|
604
|
+
# thing, and reports a parse failure for something that never parsed
|
|
605
|
+
# because it was never finished.
|
|
606
|
+
_check_not_truncated(pass_id, response, trace)
|
|
607
|
+
|
|
608
|
+
text = retry_until_json(
|
|
609
|
+
response,
|
|
610
|
+
label=pass_id,
|
|
611
|
+
# pass_prompt(), NOT the raw pass file: BOTH the original attempt
|
|
612
|
+
# and its retry must carry the same catalogs, or a retry silently
|
|
613
|
+
# sees a different tool set than the attempt it is correcting.
|
|
614
|
+
# #68 makes that literal -- the same AUTHORIZATION decides the
|
|
615
|
+
# catalogs here as decided them for the attempt.
|
|
616
|
+
system_prompt=system_prompts.pass_prompt(
|
|
617
|
+
pass_id, *advertisement_facts(authorization)),
|
|
618
|
+
model=model,
|
|
619
|
+
provider_name=provider_name,
|
|
620
|
+
trace=trace,
|
|
621
|
+
temperature=temperature,
|
|
622
|
+
authorization=authorization,
|
|
623
|
+
effort=effort,
|
|
624
|
+
on_response=lambda r: _record_granted_calls(pass_id, r, authorization),
|
|
625
|
+
# §47's sink, at the hop batch 73 missed (batch 75). A
|
|
626
|
+
# corrective attempt reaches the model through
|
|
627
|
+
# continue_conversation, which drains its own loop, so a
|
|
628
|
+
# subagent spawned while a pass is being corrected is visible
|
|
629
|
+
# through this channel or not at all.
|
|
630
|
+
activity=activity,
|
|
631
|
+
# The retry re-enters THIS pass's thread. No separate pass
|
|
632
|
+
# ceiling any more (#4): omitting the kwarg lets the wrapper
|
|
633
|
+
# resolve the configured spend cap, exactly as the pass itself
|
|
634
|
+
# did.
|
|
635
|
+
validate=lambda payload: _validate_payload(
|
|
636
|
+
pass_id, payload, claim_ids=claim_ids, ensemble=ensemble),
|
|
637
|
+
)
|
|
638
|
+
except Exception as exc:
|
|
639
|
+
# E14/#115: same uniform failure report as _run_pass -- a JSON
|
|
640
|
+
# pass that exhausts its retries is a pass that ended badly,
|
|
641
|
+
# whatever the exception class.
|
|
642
|
+
yield _pass_failed_event(pass_id, exc)
|
|
643
|
+
raise
|
|
644
|
+
yield PipelineEvent(kind="pass_complete", pass_id=pass_id, ok=True)
|
|
645
|
+
return text
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
def _claim_from_json(raw: dict) -> Claim:
|
|
649
|
+
"""Filters to known input fields so an unexpected extra key from the
|
|
650
|
+
model doesn't crash claim construction; required fields (id/text/type)
|
|
651
|
+
still raise if genuinely missing."""
|
|
652
|
+
filtered = {k: v for k, v in raw.items() if k in _CLAIM_INPUT_FIELDS}
|
|
653
|
+
return Claim(**filtered)
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
# ROADMAP_v2 §30 (B5). THE THREE _apply_* FUNCTIONS RETURN WHAT THEY
|
|
657
|
+
# APPLIED, and their checkpoint lines interpolate that rather than the
|
|
658
|
+
# count the pass was ASKED about.
|
|
659
|
+
#
|
|
660
|
+
# `Pass 3a: grounded {len(unique_entities)} unique entities across
|
|
661
|
+
# {len(factual_claims)} factual claim(s).` is emitted verbatim whether
|
|
662
|
+
# every entry landed or none did -- it is built entirely from this side of
|
|
663
|
+
# the call. Driven with 3a/3b answering about ids Pass 2 never issued, the
|
|
664
|
+
# run reported grounding 4 entities across 3 claims while grounding
|
|
665
|
+
# nothing, then spent four more model calls in the 6a/6c loop and reported
|
|
666
|
+
# that nothing could be verified. base.py:87 calls the trace "at least as
|
|
667
|
+
# trustworthy an artifact as the final report itself"; a line derived from
|
|
668
|
+
# its own input cannot be.
|
|
669
|
+
#
|
|
670
|
+
# A count of CLAIMS TOUCHED, not entries consumed: a pass that answers
|
|
671
|
+
# twice about one claim has affected one claim, and the reader is deciding
|
|
672
|
+
# whether the pass did its job.
|
|
673
|
+
|
|
674
|
+
|
|
675
|
+
def _apply_grounding(claims: list[Claim], grounding_entries: list[dict]) -> int:
|
|
676
|
+
by_id = resolve_by_id(claims)
|
|
677
|
+
applied = set()
|
|
678
|
+
for entry in grounding_entries:
|
|
679
|
+
claim = by_id.get(entry.get("claim_id"))
|
|
680
|
+
if claim:
|
|
681
|
+
claim.grounding_sources = entry.get("sources", [])
|
|
682
|
+
claim.grounding_status = entry.get("status")
|
|
683
|
+
applied.add(claim.id)
|
|
684
|
+
return len(applied)
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _ground_and_score(run: PipelineRun, grounding_entries: list[dict], corpus,
|
|
688
|
+
scorer=None, knobs=None, scholar=None,
|
|
689
|
+
claims=None) -> int:
|
|
690
|
+
"""Apply Pass 3a/6b's grounding, then everything §45 derives from it.
|
|
691
|
+
|
|
692
|
+
THE ONE SEAM BOTH GROUNDING SITES GO THROUGH. There are two -- Pass 3a
|
|
693
|
+
and Pass 6b's re-validation -- and §45 adds work that has to happen at
|
|
694
|
+
both: a source scored only on the run's first grounding would carry a
|
|
695
|
+
stale number after a revision changed the claim it was scored against.
|
|
696
|
+
Deriving beside each call instead is the shape the "fix at the
|
|
697
|
+
producer" rule exists to prevent, and a third caller could then arrive
|
|
698
|
+
without it.
|
|
699
|
+
|
|
700
|
+
Returns what `_apply_grounding` returned, so the checkpoint lines that
|
|
701
|
+
interpolate it keep reporting what was APPLIED rather than what the
|
|
702
|
+
pass was asked about (§30, B5).
|
|
703
|
+
|
|
704
|
+
`run.source_documents` is refreshed HERE, and this is its only writer.
|
|
705
|
+
Late enough to include everything grounding retrieved, and early
|
|
706
|
+
enough that a run dying in a later pass still carries the evidence it
|
|
707
|
+
had already gathered -- the failure path being where an unattended
|
|
708
|
+
run's record matters most.
|
|
709
|
+
|
|
710
|
+
`claims` NARROWS BOTH HALVES, and defaults to the whole run. Pass 3a
|
|
711
|
+
passes the factual claims since batch 220 (#247), the only ones it is
|
|
712
|
+
shown: the default let an entry naming a speculative claim mark it
|
|
713
|
+
grounded. §66 (A26) passes the 6a/6b loop's current batch, and it has to
|
|
714
|
+
reach the SCORING as well as the apply: `score_grounding_sources`
|
|
715
|
+
"rewrite[s] EVERY claim's grounding_sources with computed scores" by
|
|
716
|
+
default, so a 6b round recomputed the sources of claims D1 had already
|
|
717
|
+
finished with, while the re-tiering beside it was deliberately scoped
|
|
718
|
+
to the batch. Driven with the corpus grown by that round -- which is
|
|
719
|
+
what a grounding round does -- a claim the payload never named had its
|
|
720
|
+
`quote_verified` rewritten from None to True. That one moves no score
|
|
721
|
+
by design (§45 slice A), but the embedding and scholar phases run over
|
|
722
|
+
the same set and do move one, and the claim they would move is not
|
|
723
|
+
re-tiered.
|
|
724
|
+
|
|
725
|
+
A PARAMETER THAT EXISTED AND WAS NEVER PASSED. `score_grounding_sources`
|
|
726
|
+
has had `claims` since §45; the one production caller omitted it, so
|
|
727
|
+
the narrowing it was built for was available and unused.
|
|
728
|
+
"""
|
|
729
|
+
targets = run.claims if claims is None else claims
|
|
730
|
+
applied = _apply_grounding(targets, grounding_entries)
|
|
731
|
+
if corpus is not None:
|
|
732
|
+
run.source_documents = corpus.artifact_entries()
|
|
733
|
+
knobs = knobs or {}
|
|
734
|
+
score_grounding_sources(
|
|
735
|
+
run, claims=claims, corpus=corpus, scorer=scorer, scholar=scholar,
|
|
736
|
+
overrides=knobs.get("domain_overrides"),
|
|
737
|
+
**{k: knobs[k] for k in ("window_chars", "max_windows",
|
|
738
|
+
"claim_min_chars",
|
|
739
|
+
"authority_adjustment_cap") if k in knobs})
|
|
740
|
+
return applied
|
|
741
|
+
|
|
742
|
+
|
|
743
|
+
def _sources_were_scored(run: PipelineRun) -> bool:
|
|
744
|
+
"""Whether §45's scoring stage ran for this run.
|
|
745
|
+
|
|
746
|
+
Pass 5 needs this because an empty source list is AMBIGUOUS on its own
|
|
747
|
+
-- "the model claimed grounded and cited nothing" on a scored run,
|
|
748
|
+
"this claim was built without a scoring stage" everywhere else -- and
|
|
749
|
+
only the first should cost a claim its grounding weight.
|
|
750
|
+
|
|
751
|
+
Derived from the run rather than tracked in a flag, so it cannot go
|
|
752
|
+
stale: the stage runs exactly when Pass 3a produced grounding, which
|
|
753
|
+
is exactly when some factual claim carries a status.
|
|
754
|
+
"""
|
|
755
|
+
return any(c.grounding_status is not None for c in run.claims)
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def _apply_critic(claims: list[Claim], critic_entries: list[dict]) -> int:
|
|
759
|
+
by_id = resolve_by_id(claims)
|
|
760
|
+
applied = set()
|
|
761
|
+
for entry in critic_entries:
|
|
762
|
+
claim = by_id.get(entry.get("claim_id"))
|
|
763
|
+
if claim:
|
|
764
|
+
claim.fallacies = entry.get("fallacies", [])
|
|
765
|
+
claim.contradictions = entry.get("contradictions", [])
|
|
766
|
+
claim.critic_severity = entry.get("severity", 0.0)
|
|
767
|
+
applied.add(claim.id)
|
|
768
|
+
return len(applied)
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def _note_entries_outside_the_batch(run: PipelineRun, batch: list[Claim],
|
|
772
|
+
*entry_lists, pass_id: str, scope: str,
|
|
773
|
+
asked: str, why: str) -> int:
|
|
774
|
+
"""Say that a pass answered about a claim it was not asked about.
|
|
775
|
+
|
|
776
|
+
§66 (A26). The apply is scoped to the batch, so such an entry lands on
|
|
777
|
+
nothing -- and `review.py` settled the policy for exactly this shape:
|
|
778
|
+
a finding it cannot apply is DROPPED AND TRACED, because dropping in
|
|
779
|
+
silence is indistinguishable from applying. 6b's own checkpoint line
|
|
780
|
+
reports how many claims came out clean, which is an effect and
|
|
781
|
+
therefore cannot report this.
|
|
782
|
+
|
|
783
|
+
FOUR PASSES SINCE BATCH 220. 6a made the same
|
|
784
|
+
drop twenty lines above 6b and said nothing (#248), and 3a and 3b,
|
|
785
|
+
shown only the factual claims, applied to every claim (#247): a 3a
|
|
786
|
+
entry could mark a speculative claim grounded, which gate D0 says cannot
|
|
787
|
+
happen, and a 3b severity on one moved its tier -- measured, MEDIUM to
|
|
788
|
+
LOW at 0.5 and to UNVERIFIED at 0.8, which then cost it 6a/6b rounds.
|
|
789
|
+
Each caller says its own `scope`, what it `asked` about and `why` an
|
|
790
|
+
answer outside it must not land.
|
|
791
|
+
|
|
792
|
+
The id cross-check does not cover it and is not meant to: B5 makes a
|
|
793
|
+
TOTAL mismatch the error and a partial one deliberately not, since a
|
|
794
|
+
model that gets nine of ten right must not cost a run nine passes.
|
|
795
|
+
Measured -- a payload naming the batch PLUS one extra id passes
|
|
796
|
+
`claim_ids=[c.id for c in flagged]` unchanged, which is why the apply
|
|
797
|
+
is where this had to be fixed and the trace is what makes the drop
|
|
798
|
+
visible.
|
|
799
|
+
"""
|
|
800
|
+
known = {c.id for c in batch}
|
|
801
|
+
outside = sorted({entry.get("claim_id") for entries in entry_lists
|
|
802
|
+
for entry in entries if isinstance(entry, dict)}
|
|
803
|
+
- known - {None})
|
|
804
|
+
if outside:
|
|
805
|
+
run.log(
|
|
806
|
+
f"{pass_id}: {len(outside)} claim id(s) in the response name a "
|
|
807
|
+
f"claim outside {scope} "
|
|
808
|
+
f"({', '.join(repr(i) for i in outside[:3])}) -- dropped. "
|
|
809
|
+
f"{pass_id.removeprefix('Pass ')} was asked about {len(known)} "
|
|
810
|
+
f"{asked}; {why}"
|
|
811
|
+
)
|
|
812
|
+
return len(outside)
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
def _apply_assumption_flags(claims: list[Claim], assumptions: dict) -> int:
|
|
816
|
+
by_id = resolve_by_id(claims)
|
|
817
|
+
applied = set()
|
|
818
|
+
for claim_id, flags in assumptions.get("per_claim_flags", {}).items():
|
|
819
|
+
claim = by_id.get(claim_id)
|
|
820
|
+
if claim:
|
|
821
|
+
claim.assumption_flags = flags
|
|
822
|
+
applied.add(claim.id)
|
|
823
|
+
return len(applied)
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def _synthesis_input(run: PipelineRun, directives: list | None = None) -> str:
|
|
827
|
+
"""What final synthesis reads.
|
|
828
|
+
|
|
829
|
+
A FUNCTION, not a string built once. §20 re-runs synthesis after a
|
|
830
|
+
correction is accepted, and reusing the string built before the review
|
|
831
|
+
would regenerate the report from the UNCORRECTED claims -- producing a
|
|
832
|
+
report that changed for no reason while the correction silently went
|
|
833
|
+
nowhere.
|
|
834
|
+
"""
|
|
835
|
+
text = (
|
|
836
|
+
f"Original query:\n{run.user_query}\n\n"
|
|
837
|
+
f"Final annotated claims:\n{json.dumps([vars(c) for c in run.claims])}\n\n"
|
|
838
|
+
f"Coverage gaps:\n{json.dumps(run.coverage_gaps)}"
|
|
839
|
+
)
|
|
840
|
+
if directives:
|
|
841
|
+
text += (
|
|
842
|
+
"\n\nA reviewer raised the following about the previous draft of "
|
|
843
|
+
"this report, and a human approved each one. Address them:\n"
|
|
844
|
+
+ "\n".join(f"- {d}" for d in directives)
|
|
845
|
+
)
|
|
846
|
+
return text
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _review_stage(run: PipelineRun, progress: _Progress, model: str,
|
|
850
|
+
provider_name: str, authorization, review,
|
|
851
|
+
enabled: bool = False, effort: str | None = None,
|
|
852
|
+
activity=None):
|
|
853
|
+
"""ROADMAP_v2 §20. Opt-in review of the finished run, with every
|
|
854
|
+
correction consented to individually.
|
|
855
|
+
|
|
856
|
+
§22: a generator, because it logs, checkpoints and runs a pass. It
|
|
857
|
+
takes the run's _Progress rather than making its own, so review.py's
|
|
858
|
+
trace lines -- which never checkpointed at all before §22 -- land on
|
|
859
|
+
the same watermark as everything else.
|
|
860
|
+
|
|
861
|
+
Called from INSIDE the orchestrator (V3) rather than left to each
|
|
862
|
+
shell. The obvious alternative -- a post-pipeline stage the caller
|
|
863
|
+
invokes, mirroring write_run_artifacts -- is exactly the shape that
|
|
864
|
+
produced this project's live asymmetry: that function has one
|
|
865
|
+
production call site, so /research in the TUI writes no output
|
|
866
|
+
directory at all. The consent OBJECT still comes from the shell,
|
|
867
|
+
because only a shell knows how to reach a human.
|
|
868
|
+
|
|
869
|
+
`enabled` and `review` are SEPARATE on purpose. "The user asked for a
|
|
870
|
+
review" and "someone can be asked about its findings" are different
|
|
871
|
+
facts, and collapsing them loses the case that matters: --review on a
|
|
872
|
+
piped run must still produce findings and still change nothing (V6).
|
|
873
|
+
If the consent object alone enabled the stage, that run would skip the
|
|
874
|
+
review entirely and report nothing at all.
|
|
875
|
+
|
|
876
|
+
A consent object with `enabled` false still runs it, because a caller
|
|
877
|
+
that went to the trouble of building one wants the review.
|
|
878
|
+
|
|
879
|
+
Batch 25 (#139): effort inherits into the reviewer (V7's rule applied
|
|
880
|
+
to one more axis) and into the re-synthesis pass below.
|
|
881
|
+
"""
|
|
882
|
+
if not enabled and review is None:
|
|
883
|
+
return
|
|
884
|
+
|
|
885
|
+
from core.reasoning import review as review_module
|
|
886
|
+
|
|
887
|
+
findings, thread_id = review_module.run_review(
|
|
888
|
+
run, model, provider_name, authorization, effort=effort,
|
|
889
|
+
activity=activity)
|
|
890
|
+
run.log(f"Review: {len(findings)} finding(s) raised.")
|
|
891
|
+
# The findings' CONTENT is durable from this checkpoint on (review
|
|
892
|
+
# r1-2): a kill during the consent walk used to leave only the count,
|
|
893
|
+
# with what the reviewer found unrecoverable. Trace is serialized by
|
|
894
|
+
# every update_pipeline_run, so this needs no new column (V8).
|
|
895
|
+
for f in findings:
|
|
896
|
+
target = f.get("claim_id") or "report"
|
|
897
|
+
excerpt = str(f.get("proposed") or "")[:80]
|
|
898
|
+
run.log(f"Review: finding -- {f.get('kind')} correction to {target}"
|
|
899
|
+
+ (f": {excerpt}" if excerpt else ""))
|
|
900
|
+
yield from progress.checkpoint()
|
|
901
|
+
|
|
902
|
+
decisions = review_module.walk_consent(
|
|
903
|
+
findings, review, run, model=model, provider_name=provider_name,
|
|
904
|
+
thread_id=thread_id, authorization=authorization, effort=effort,
|
|
905
|
+
# §47's sink, for the refinement calls the walk can make. Every
|
|
906
|
+
# hop is a place it can be dropped, and this one was.
|
|
907
|
+
activity=activity)
|
|
908
|
+
run.subagent_reviews = decisions
|
|
909
|
+
yield from progress.checkpoint()
|
|
910
|
+
|
|
911
|
+
# Deferred commit (review f4): corrections land on a copy, and the
|
|
912
|
+
# live claims/report change only after the re-synthesis succeeds. A
|
|
913
|
+
# failed re-synthesis therefore persists a consistent record -- the
|
|
914
|
+
# pre-fix order stored corrected claims beside the stale report.
|
|
915
|
+
changed, corrected, applied = review_module.apply_deferred(run, decisions)
|
|
916
|
+
if changed:
|
|
917
|
+
original_claims = run.claims
|
|
918
|
+
run.claims = corrected
|
|
919
|
+
try:
|
|
920
|
+
run.final_report = yield from _run_pass(
|
|
921
|
+
"Final synthesis",
|
|
922
|
+
_synthesis_input(run, review_module.synthesis_directives(decisions)),
|
|
923
|
+
model, provider_name, authorization=authorization,
|
|
924
|
+
trace=run.trace,
|
|
925
|
+
pass_threads=run.pass_threads,
|
|
926
|
+
effort=effort,
|
|
927
|
+
activity=activity,
|
|
928
|
+
)
|
|
929
|
+
except Exception as e:
|
|
930
|
+
# CONTAINED, not re-raised. A provider error on this one call
|
|
931
|
+
# is the same transient class as a reviewer call failing, and
|
|
932
|
+
# f1 settled that an optional stage must not flip a finished
|
|
933
|
+
# ten-pass run to status='failed'. The deferred commit already
|
|
934
|
+
# guarantees a consistent record: the pre-review claims and the
|
|
935
|
+
# report generated from them, which is a complete run's output
|
|
936
|
+
# minus the corrections -- and the trace says which.
|
|
937
|
+
#
|
|
938
|
+
# A failure that escapes this STAGE (a bug, not a transient
|
|
939
|
+
# provider error) still reaches the pipeline's except and
|
|
940
|
+
# records status='failed'. That distinction is the whole
|
|
941
|
+
# policy, and both halves are test-pinned.
|
|
942
|
+
logger.exception("Review re-synthesis failed; corrections not applied.")
|
|
943
|
+
run.claims = original_claims
|
|
944
|
+
review_module.log_outcomes(run, decisions, applied, committed=False)
|
|
945
|
+
run.log(f"Review: re-synthesis failed ({e}); accepted corrections "
|
|
946
|
+
f"NOT applied -- claims and report left unchanged.")
|
|
947
|
+
yield from progress.checkpoint()
|
|
948
|
+
return
|
|
949
|
+
review_module.log_outcomes(run, decisions, applied, committed=True)
|
|
950
|
+
# NOT re-reviewed. The regress is cut deliberately: a review of the
|
|
951
|
+
# re-synthesised report would need its own consent pass, and so on.
|
|
952
|
+
run.log(f"Review: {len(applied)} correction(s) accepted; final "
|
|
953
|
+
f"synthesis re-run. The re-synthesised report is not "
|
|
954
|
+
f"re-reviewed.")
|
|
955
|
+
else:
|
|
956
|
+
review_module.log_outcomes(run, decisions, applied, committed=False)
|
|
957
|
+
run.log("Review: no correction applied; report unchanged.")
|
|
958
|
+
yield from progress.checkpoint()
|
|
959
|
+
|
|
960
|
+
|
|
961
|
+
def _stage(pass_id: str):
|
|
962
|
+
"""One event for a stage that made ZERO model calls (§26).
|
|
963
|
+
|
|
964
|
+
A separate kind from pass_start/pass_complete rather than a pair of
|
|
965
|
+
those, because a code stage cannot meaningfully be "running": it
|
|
966
|
+
completes in the time it takes to yield, and a consumer that showed it
|
|
967
|
+
as in-progress would be showing a state that never exists. Pass 5 and
|
|
968
|
+
Pass 6c are passes by the pipeline's numbering and stages by their
|
|
969
|
+
cost, and this is the field that says which.
|
|
970
|
+
|
|
971
|
+
Before this, every zero-LLM stage was invisible to both shells -- not
|
|
972
|
+
by decision but because _run_pass was the only thing that emitted a
|
|
973
|
+
boundary, and it wraps a model call. So a sidebar showed Pass 4
|
|
974
|
+
followed by Pass 6a with the tiering that decided which claims were
|
|
975
|
+
flagged in between, unmentioned.
|
|
976
|
+
"""
|
|
977
|
+
yield PipelineEvent(kind="stage", pass_id=pass_id)
|
|
978
|
+
|
|
979
|
+
|
|
980
|
+
def _tier_events(run: PipelineRun):
|
|
981
|
+
"""One claim_tiered event per claim, after a tiering pass.
|
|
982
|
+
|
|
983
|
+
Called after EVERY run_confidence_tiering() -- Pass 5 and each 6b
|
|
984
|
+
round -- so a consumer sees a claim's tier change rather than only
|
|
985
|
+
its first value. Repeats are by design: the events are a stream of
|
|
986
|
+
"this is the tier now", and a panel re-renders from the latest.
|
|
987
|
+
|
|
988
|
+
Derived from run.claims after the fact rather than emitted inside
|
|
989
|
+
confidence_scoring.py, which makes zero LLM calls and knows nothing
|
|
990
|
+
about events; §22's own warning about two writers of related data
|
|
991
|
+
applies to the tiering just as it does to the trace.
|
|
992
|
+
"""
|
|
993
|
+
for claim in run.claims:
|
|
994
|
+
yield PipelineEvent(kind="claim_tiered", claim_id=claim.id,
|
|
995
|
+
tier=claim.confidence_tier)
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _apply_fallback(claim: Claim) -> None:
|
|
999
|
+
"""D2 exhausted / fallback branch -- pure code, no LLM call."""
|
|
1000
|
+
claim.confidence_tier = "UNVERIFIED"
|
|
1001
|
+
claim.annotation = (
|
|
1002
|
+
f"[UNVERIFIED] Could not be adequately verified after {claim.retry_count} "
|
|
1003
|
+
f"revision attempt(s). Treat with caution."
|
|
1004
|
+
)
|
|
1005
|
+
claim.final_text = claim.revision_text or claim.text
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _revision_request(claim: Claim) -> dict:
|
|
1009
|
+
"""One claim's entry in Pass 6a's batch: the claim and the feedback
|
|
1010
|
+
that put it there.
|
|
1011
|
+
|
|
1012
|
+
`disagreement_feedback` only on a contested claim (ROADMAP_v3 §90),
|
|
1013
|
+
because D1 may have flagged it for that alone, and 6a cannot address a
|
|
1014
|
+
reason it is not given. `revise.md` says what to do with one."""
|
|
1015
|
+
entry = {
|
|
1016
|
+
"claim": vars(claim),
|
|
1017
|
+
"critic_feedback": {"fallacies": claim.fallacies, "contradictions": claim.contradictions},
|
|
1018
|
+
"grounding_feedback": claim.grounding_sources,
|
|
1019
|
+
"assumption_flags": claim.assumption_flags,
|
|
1020
|
+
}
|
|
1021
|
+
if is_contested(claim):
|
|
1022
|
+
entry["disagreement_feedback"] = {
|
|
1023
|
+
# Deduplicated, as the consistency score it came from is (B8).
|
|
1024
|
+
"asserted_by": len(set(claim.asserted_by_candidates)),
|
|
1025
|
+
"of": claim.score_breakdown["consistency_denominator"],
|
|
1026
|
+
}
|
|
1027
|
+
return entry
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
# ============================================================================
|
|
1031
|
+
# ---- The pipeline ------------------------------------------------------
|
|
1032
|
+
# ============================================================================
|
|
1033
|
+
|
|
1034
|
+
def run_pipeline_to_completion(gen) -> PipelineRun:
|
|
1035
|
+
"""Consumes a stream_deep_research_pipeline() generator fully,
|
|
1036
|
+
discarding intermediate events, and returns the finished PipelineRun.
|
|
1037
|
+
|
|
1038
|
+
The exact counterpart of core.loop.run_to_completion(), including the
|
|
1039
|
+
RuntimeError: a generator that ends without its terminal event has a
|
|
1040
|
+
bug, and returning None instead would push the failure into whichever
|
|
1041
|
+
caller dereferences the run first.
|
|
1042
|
+
"""
|
|
1043
|
+
final = None
|
|
1044
|
+
for event in gen:
|
|
1045
|
+
if event.kind == "run_complete":
|
|
1046
|
+
final = event.run
|
|
1047
|
+
if final is None:
|
|
1048
|
+
raise RuntimeError(
|
|
1049
|
+
"Pipeline generator completed without yielding a run_complete event"
|
|
1050
|
+
)
|
|
1051
|
+
return final
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
def stream_deep_research_pipeline(
|
|
1055
|
+
user_query: str,
|
|
1056
|
+
model: str,
|
|
1057
|
+
provider_name: str = DEFAULT_PROVIDER,
|
|
1058
|
+
ensemble_mode: bool | None = None,
|
|
1059
|
+
ensemble_n: int | None = None,
|
|
1060
|
+
authorization=None,
|
|
1061
|
+
review=None,
|
|
1062
|
+
subagent_review: bool | None = None,
|
|
1063
|
+
effort: str | None = None,
|
|
1064
|
+
# §47. Threaded like `authorization` beside it, and for the same
|
|
1065
|
+
# reason: both are run-scoped values the layers between here and
|
|
1066
|
+
# the model call only forward. Batch 59 stopped at
|
|
1067
|
+
# stream_deep_research_mode's signature; this is the rest of it.
|
|
1068
|
+
activity=None,
|
|
1069
|
+
):
|
|
1070
|
+
"""The pipeline, as a generator of PipelineEvents (§22). The terminal
|
|
1071
|
+
`run_complete` event carries the finished PipelineRun.
|
|
1072
|
+
|
|
1073
|
+
ABANDONMENT (§22, recorded rather than fixed). A consumer that stops
|
|
1074
|
+
iterating early -- a TUI quitting mid-run -- raises GeneratorExit
|
|
1075
|
+
inside the try below, which `except Exception` deliberately does not
|
|
1076
|
+
catch. The record therefore stays status='running', populated through
|
|
1077
|
+
the last checkpoint. That is honest: the run was abandoned, not
|
|
1078
|
+
failed, and forcing 'failed' would make an interrupted run
|
|
1079
|
+
indistinguishable from a broken one. Both shipped consumers drain
|
|
1080
|
+
fully.
|
|
1081
|
+
|
|
1082
|
+
authorization (§25): a core.approval.RunAuthorization built by the
|
|
1083
|
+
shell that launched this run, or None for the pre-§25 behaviour (no
|
|
1084
|
+
grants, nobody to ask, every approval-gated tool hidden from every
|
|
1085
|
+
pass). Carried, not interpreted -- the decisions about WHAT may be
|
|
1086
|
+
granted live in core/reasoning/authorization.py, and the enforcement
|
|
1087
|
+
lives in core/loop.py.
|
|
1088
|
+
|
|
1089
|
+
review (§20): a core.interaction.ResponseChannel, or None (§23 moved
|
|
1090
|
+
it there from core.approval.ReviewConsent). Carried the same
|
|
1091
|
+
way and for the same reason -- only a shell knows how to reach a
|
|
1092
|
+
human. A SECOND parameter rather than a field on the authorization
|
|
1093
|
+
bundle: that bundle answers "may this gated call proceed?" and this
|
|
1094
|
+
answers "may this edit be applied?". None means nothing is applied,
|
|
1095
|
+
even when the review itself runs (V6).
|
|
1096
|
+
|
|
1097
|
+
subagent_review (§20/D9): whether the review stage runs at all. None
|
|
1098
|
+
falls back to config.SUBAGENT_REVIEW, matching ensemble_mode's
|
|
1099
|
+
pattern. Separate from `review` because --review on a piped run must
|
|
1100
|
+
still produce findings while applying nothing."""
|
|
1101
|
+
ensemble_mode = config.ENSEMBLE_MODE if ensemble_mode is None else ensemble_mode
|
|
1102
|
+
subagent_review = (config.SUBAGENT_REVIEW if subagent_review is None
|
|
1103
|
+
else subagent_review)
|
|
1104
|
+
|
|
1105
|
+
# §10 revisit (E4). `ensemble_n` is vestigial: N is len(ENSEMBLE_MODELS)
|
|
1106
|
+
# now, because a separately configured count could disagree with the
|
|
1107
|
+
# roster that produced the candidates, and the denominator of a
|
|
1108
|
+
# confidence score must not be able to lie. The parameter and the
|
|
1109
|
+
# settings.json key are KEPT rather than removed -- unknown settings keys
|
|
1110
|
+
# raise, so deleting the key would make every existing settings.json that
|
|
1111
|
+
# sets it fail to load.
|
|
1112
|
+
if ensemble_n is not None:
|
|
1113
|
+
logger.warning(
|
|
1114
|
+
"Ignoring ensemble_n=%s: the number of candidates is "
|
|
1115
|
+
"len(ensemble_models) now (ROADMAP §10 revisit, E3). "
|
|
1116
|
+
"Change the roster in config.yaml to change N.",
|
|
1117
|
+
ensemble_n,
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
roster = _ensemble_roster(ensemble_mode)
|
|
1121
|
+
|
|
1122
|
+
run = PipelineRun(user_query=user_query)
|
|
1123
|
+
|
|
1124
|
+
# §45 (SQ2). RUN-SCOPED, and attached to every pass rather than only to
|
|
1125
|
+
# the two that ground. The question it answers is "what did this run
|
|
1126
|
+
# retrieve from this URL", which is not a per-pass question: a page
|
|
1127
|
+
# Pass 1 fetched and Pass 3a then cited without re-fetching is still
|
|
1128
|
+
# evidence about that URL, and scoping the corpus to 3a would report
|
|
1129
|
+
# no captured text for it and fall back to the model's own number.
|
|
1130
|
+
corpus = SourceCorpus()
|
|
1131
|
+
|
|
1132
|
+
# §45 (SQ2). Resolved ONCE, before Pass 0, so the warning about
|
|
1133
|
+
# running without one is the first thing the trace says rather than
|
|
1134
|
+
# something a reader meets forty lines in beside a score it already
|
|
1135
|
+
# explains. One scorer for the whole run, because its vector cache and
|
|
1136
|
+
# its give-up flag are both run-scoped: a claim is embedded once
|
|
1137
|
+
# however many sources it has, and a provider that is down stays down.
|
|
1138
|
+
scoring_knobs = effective_source_scoring(warn=True)
|
|
1139
|
+
scorer = make_scorer(run, floor=scoring_knobs["similarity_floor"],
|
|
1140
|
+
ceiling=scoring_knobs["similarity_ceiling"])
|
|
1141
|
+
|
|
1142
|
+
# §45 (SQ9). OFF by default, so an out-of-the-box run makes no call
|
|
1143
|
+
# the user did not ask for. One lookup object for the run, for the
|
|
1144
|
+
# scorer's reason: its cohort cache and its give-up flag are both
|
|
1145
|
+
# run-scoped.
|
|
1146
|
+
scholar = ScholarLookup(knobs=scoring_knobs)
|
|
1147
|
+
|
|
1148
|
+
# §45 (SQ6). Both knob sections resolved ONCE, here, with warn=True so
|
|
1149
|
+
# a mistyped weight is said once per run rather than once per claim --
|
|
1150
|
+
# `effective_compaction`'s gating, for its reason. score_claim stays a
|
|
1151
|
+
# pure function of its arguments, and a run's numbers cannot change
|
|
1152
|
+
# halfway through it.
|
|
1153
|
+
confidence_weights = effective_confidence(warn=True)
|
|
1154
|
+
|
|
1155
|
+
# §25 audit trail: ONE list, shared, not two kept in step. The run and
|
|
1156
|
+
# the authorization bundle now refer to the same object, so a granted
|
|
1157
|
+
# call is visible on the PipelineRun the moment it happens -- including
|
|
1158
|
+
# at every §5 checkpoint and on the FAILURE path, which is where an
|
|
1159
|
+
# unattended run's record matters most. Copying at the end instead
|
|
1160
|
+
# would lose the trail on exactly the runs most worth auditing, and two
|
|
1161
|
+
# writers of related data is the bug shape §22 warns about by name.
|
|
1162
|
+
if authorization is not None:
|
|
1163
|
+
run.granted_calls = authorization.granted_calls
|
|
1164
|
+
|
|
1165
|
+
# ROADMAP §5 minimal core: create the durable record up front. On a
|
|
1166
|
+
# mid-pipeline crash, the except block below flips status to 'failed'
|
|
1167
|
+
# and persists a snapshot of the partial run before re-raising.
|
|
1168
|
+
run.run_id = create_pipeline_run(user_query)
|
|
1169
|
+
|
|
1170
|
+
# ROADMAP §11: resolve critic provider/model once. When no critic is
|
|
1171
|
+
# configured these fall back to the main provider/model — no-op
|
|
1172
|
+
# routing.
|
|
1173
|
+
#
|
|
1174
|
+
# §45 (SQ7): through pipeline_models.resolve rather than
|
|
1175
|
+
# config.CRITIC_MODEL directly, so a `/critic` choice reaches the CLI,
|
|
1176
|
+
# the TUI and a spawned run alike. That function owns the precedence —
|
|
1177
|
+
# the user-tier store outranks config.py — and owning it in one place
|
|
1178
|
+
# is the point: §11 gave this routing to config.py alone, which meant
|
|
1179
|
+
# the methodology's own "a model must not check its own output" was
|
|
1180
|
+
# available only to whoever could edit the checkout.
|
|
1181
|
+
critic = pipeline_models.resolve("critic")
|
|
1182
|
+
critic_provider = critic["provider_name"] if critic else provider_name
|
|
1183
|
+
critic_model = critic["model"] if critic else model
|
|
1184
|
+
|
|
1185
|
+
progress = _Progress(run)
|
|
1186
|
+
|
|
1187
|
+
try:
|
|
1188
|
+
# --- Pass 0: preliminary plan ---
|
|
1189
|
+
run.plan = _parse_json_response((yield from _run_pass_with_json_retry("Pass 0", user_query, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity)))
|
|
1190
|
+
yield from progress.checkpoint(f"Pass 0: plan produced ({len(run.plan.get('key_entities_or_subjects', []))} key entities anticipated).")
|
|
1191
|
+
|
|
1192
|
+
# --- Pass 1: initial generation (ensemble mode: N candidates) ---
|
|
1193
|
+
pass1_input = f"Original query:\n{user_query}\n\nPreliminary plan (loose context, may deviate freely):\n{json.dumps(run.plan)}"
|
|
1194
|
+
effective_n = 0
|
|
1195
|
+
if roster:
|
|
1196
|
+
# A LOOP, not a comprehension: _run_pass is a generator now
|
|
1197
|
+
# (§22), and a comprehension would collect generator objects
|
|
1198
|
+
# without ever running a pass.
|
|
1199
|
+
#
|
|
1200
|
+
# One candidate per roster entry, each on its OWN provider/model
|
|
1201
|
+
# (E1). No new plumbing: _run() calls api_initialization() itself
|
|
1202
|
+
# and effort_for() validates against the receiving model, so a
|
|
1203
|
+
# heterogeneous roster is correct by construction -- the same way
|
|
1204
|
+
# §11 already routes 3a/3b/6c to critic_provider/critic_model.
|
|
1205
|
+
candidates = []
|
|
1206
|
+
for entry in roster:
|
|
1207
|
+
label = f"{entry['provider_name']}/{entry['model']}"
|
|
1208
|
+
try:
|
|
1209
|
+
candidates.append((yield from _run_pass(
|
|
1210
|
+
"Pass 1", pass1_input, entry["model"], entry["provider_name"],
|
|
1211
|
+
authorization=authorization, trace=run.trace,
|
|
1212
|
+
pass_threads=run.pass_threads, effort=effort,
|
|
1213
|
+
activity=activity)))
|
|
1214
|
+
except Exception as e:
|
|
1215
|
+
# NAMED, TRACED, SKIPPED (E7) -- §20's containment rule.
|
|
1216
|
+
# An optional generation strategy must not flip a
|
|
1217
|
+
# ten-pass run to status='failed' because one provider
|
|
1218
|
+
# was unreachable. Same posture MCP takes per server.
|
|
1219
|
+
message = (f"Pass 1: ensemble candidate on {label} failed "
|
|
1220
|
+
f"({type(e).__name__}: {e}) — skipped.")
|
|
1221
|
+
logger.warning("%s", message)
|
|
1222
|
+
run.log(message)
|
|
1223
|
+
continue
|
|
1224
|
+
run.log(f"Pass 1: ensemble candidate {len(candidates)} generated on {label}.")
|
|
1225
|
+
# E13/#77: the record keeps every surviving candidate.
|
|
1226
|
+
# The metadata half sits beside the trace's candidate-N
|
|
1227
|
+
# lines so a stored asserted_by_candidates tag names
|
|
1228
|
+
# models this run can still identify; the text half feeds
|
|
1229
|
+
# the per-candidate artifacts and Pass 3b's input -- the
|
|
1230
|
+
# two places that must be able to check a claim against
|
|
1231
|
+
# what its number actually said. pipeline_storage strips
|
|
1232
|
+
# `text` at serialization; see PipelineRunRecord.
|
|
1233
|
+
run.candidates.append({
|
|
1234
|
+
"candidate": len(candidates),
|
|
1235
|
+
"provider_name": entry["provider_name"],
|
|
1236
|
+
"model": entry["model"],
|
|
1237
|
+
"chars": len(candidates[-1]),
|
|
1238
|
+
"text": candidates[-1],
|
|
1239
|
+
})
|
|
1240
|
+
|
|
1241
|
+
# Labels come from the SURVIVORS, never from position in the
|
|
1242
|
+
# roster (E11). Labelling by roster position leaves a gap when a
|
|
1243
|
+
# candidate fails -- "Candidate 1, Candidate 3" -- so Pass 2
|
|
1244
|
+
# would tag a claim both survivors asserted as [1, 3] while the
|
|
1245
|
+
# denominator said 2, and a unanimous claim would be scored as
|
|
1246
|
+
# dissented-against. A silently deflated confidence score is the
|
|
1247
|
+
# exact failure class this section exists to remove.
|
|
1248
|
+
effective_n = len(candidates)
|
|
1249
|
+
if effective_n >= 2:
|
|
1250
|
+
run.raw_response = candidates[0]
|
|
1251
|
+
pass2_input = "\n\n".join(
|
|
1252
|
+
f"Candidate {i + 1}:\n{c}" for i, c in enumerate(candidates)
|
|
1253
|
+
)
|
|
1254
|
+
run.log(f"Pass 1: ensemble mode — {effective_n} candidate(s) "
|
|
1255
|
+
f"across {effective_n} model(s); consistency is measured "
|
|
1256
|
+
f"out of {effective_n}.")
|
|
1257
|
+
else:
|
|
1258
|
+
# DEGRADE TO THE SINGLE-CANDIDATE PATH (E7), rather than
|
|
1259
|
+
# scoring one candidate as an ensemble of one -- which would
|
|
1260
|
+
# give every claim consistency 1.0, so none could ever be
|
|
1261
|
+
# contested, while reporting that agreement as if it had
|
|
1262
|
+
# been corroborated.
|
|
1263
|
+
# ensemble_n=0 is not a degraded formula, it is the correct
|
|
1264
|
+
# one when there is nothing to compare against.
|
|
1265
|
+
effective_n = 0
|
|
1266
|
+
if candidates:
|
|
1267
|
+
run.raw_response = candidates[0]
|
|
1268
|
+
pass2_input = f"Response to extract claims from:\n{run.raw_response}"
|
|
1269
|
+
run.log("Pass 1: only one ensemble candidate survived — "
|
|
1270
|
+
"no claim can be contested, since one candidate "
|
|
1271
|
+
"cannot disagree with itself.")
|
|
1272
|
+
else:
|
|
1273
|
+
raise RuntimeError(
|
|
1274
|
+
"Pass 1: every ensemble candidate failed; there is no "
|
|
1275
|
+
"response to extract claims from. See the trace for "
|
|
1276
|
+
"each provider's error."
|
|
1277
|
+
)
|
|
1278
|
+
else:
|
|
1279
|
+
run.raw_response = yield from _run_pass("Pass 1", pass1_input, model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity)
|
|
1280
|
+
pass2_input = f"Response to extract claims from:\n{run.raw_response}"
|
|
1281
|
+
run.log("Pass 1: initial generation complete.")
|
|
1282
|
+
yield from progress.checkpoint()
|
|
1283
|
+
|
|
1284
|
+
# --- Pass 2: claim extraction & classification ---
|
|
1285
|
+
claims_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 2", pass2_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, ensemble=effective_n >= 2, effort=effort, activity=activity)))
|
|
1286
|
+
run.claims = [_claim_from_json(c) for c in claims_json]
|
|
1287
|
+
for claim in run.claims:
|
|
1288
|
+
yield PipelineEvent(kind="claim_extracted", claim_id=claim.id,
|
|
1289
|
+
text=claim.text)
|
|
1290
|
+
yield from progress.checkpoint(f"Pass 2: extracted {len(run.claims)} claim(s).")
|
|
1291
|
+
|
|
1292
|
+
# --- gate D0: route by claim type (pure code) ---
|
|
1293
|
+
factual_claims = [c for c in run.claims if c.type == "factual"]
|
|
1294
|
+
non_factual_claims = [c for c in run.claims if c.type != "factual"]
|
|
1295
|
+
yield from _stage("D0")
|
|
1296
|
+
yield from progress.checkpoint(
|
|
1297
|
+
f"D0: {len(factual_claims)} factual claim(s) routed to 3a/3b, "
|
|
1298
|
+
f"{len(non_factual_claims)} synthesis/speculative claim(s) routed directly to Pass 4."
|
|
1299
|
+
)
|
|
1300
|
+
|
|
1301
|
+
# --- Pass 3a + 3b: only for factual claims, batched by deduplicated entity ---
|
|
1302
|
+
if factual_claims:
|
|
1303
|
+
unique_entities = sorted({e for c in factual_claims for e in c.entities})
|
|
1304
|
+
pass3a_input = (
|
|
1305
|
+
f"Factual claims:\n{json.dumps([vars(c) for c in factual_claims])}\n\n"
|
|
1306
|
+
f"Deduplicated entities to research (search each ONCE, map results back "
|
|
1307
|
+
f"to every claim referencing it):\n{json.dumps(unique_entities)}"
|
|
1308
|
+
)
|
|
1309
|
+
# SCOPED TO `factual_claims`, like 6b to its batch (batch 220,
|
|
1310
|
+
# #247): 3a is shown the factual claims and nothing else, so an
|
|
1311
|
+
# entry naming another claim is dropped and traced rather than
|
|
1312
|
+
# marking a claim grounded that gate D0 routed past grounding.
|
|
1313
|
+
grounding_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3a", pass3a_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity)))
|
|
1314
|
+
grounded = _ground_and_score(run, grounding_json, corpus, scorer,
|
|
1315
|
+
scoring_knobs, scholar,
|
|
1316
|
+
claims=factual_claims)
|
|
1317
|
+
yield from progress.checkpoint(f"Pass 3a: grounded {grounded} of {len(factual_claims)} factual claim(s), across {len(unique_entities)} deduplicated entities.")
|
|
1318
|
+
_note_entries_outside_the_batch(
|
|
1319
|
+
run, factual_claims, grounding_json, pass_id="Pass 3a",
|
|
1320
|
+
scope="the factual claims", asked="factual claim(s)",
|
|
1321
|
+
why="Gate D0 routes every other claim past 3a/3b, so grounding "
|
|
1322
|
+
"one would record a check gate D0 says never ran.")
|
|
1323
|
+
|
|
1324
|
+
# E13/#77. Two or more survivors: the claims were extracted
|
|
1325
|
+
# from the UNION of the candidates, so the critic sees every
|
|
1326
|
+
# candidate under the labels Pass 2 used -- a claim only
|
|
1327
|
+
# candidate 3 asserted is checked against text that actually
|
|
1328
|
+
# contains it, and contradictions ACROSS candidates are
|
|
1329
|
+
# findable. Handing it bare `raw_response` (candidate 1)
|
|
1330
|
+
# asked an ill-posed question: find conflicts within a text
|
|
1331
|
+
# much of your input did not come from.
|
|
1332
|
+
if len(run.candidates) >= 2:
|
|
1333
|
+
pass3b_input = (
|
|
1334
|
+
"\n\n".join(
|
|
1335
|
+
f"Candidate {entry['candidate']}:\n{entry['text']}"
|
|
1336
|
+
for entry in run.candidates)
|
|
1337
|
+
+ f"\n\nFactual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1338
|
+
)
|
|
1339
|
+
else:
|
|
1340
|
+
pass3b_input = (
|
|
1341
|
+
f"Raw response:\n{run.raw_response}\n\n"
|
|
1342
|
+
f"Factual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1343
|
+
)
|
|
1344
|
+
critic_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3b", pass3b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity)))
|
|
1345
|
+
# The same scope as 3a, for a stronger reason: a non-factual
|
|
1346
|
+
# claim's score IS its capped critic component, so a severity
|
|
1347
|
+
# landing on one moved its tier (measured, batch 220).
|
|
1348
|
+
critiqued = _apply_critic(factual_claims, critic_json)
|
|
1349
|
+
yield from progress.checkpoint(f"Pass 3b: critique applied to {critiqued} of {len(factual_claims)} factual claim(s).")
|
|
1350
|
+
_note_entries_outside_the_batch(
|
|
1351
|
+
run, factual_claims, critic_json, pass_id="Pass 3b",
|
|
1352
|
+
scope="the factual claims", asked="factual claim(s)",
|
|
1353
|
+
why="Gate D0 routes every other claim past 3a/3b, and a "
|
|
1354
|
+
"severity on one would lower the score Pass 5 gives "
|
|
1355
|
+
"it.")
|
|
1356
|
+
else:
|
|
1357
|
+
# A stage, not a silence. "3a/3b did not run" is a fact about
|
|
1358
|
+
# the run's shape -- it means nothing was factual -- and a
|
|
1359
|
+
# panel that simply skipped from Pass 2 to Pass 3c would leave
|
|
1360
|
+
# the reader to infer it from an absence.
|
|
1361
|
+
yield from _stage("Pass 3a/3b")
|
|
1362
|
+
yield from progress.checkpoint("Pass 3a/3b: skipped -- no factual claims to ground or critique.")
|
|
1363
|
+
|
|
1364
|
+
# --- Pass 3c: completeness, independent of Pass 1's raw_response ---
|
|
1365
|
+
pass3c_input = f"Original query:\n{user_query}\n\nPreliminary plan:\n{json.dumps(run.plan)}"
|
|
1366
|
+
run.completeness = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3c", pass3c_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity)))
|
|
1367
|
+
yield from progress.checkpoint(
|
|
1368
|
+
f"Pass 3c: coverage_score={run.completeness.get('coverage_score')}, "
|
|
1369
|
+
f"{len(run.completeness.get('gaps', []))} gap(s) identified."
|
|
1370
|
+
)
|
|
1371
|
+
|
|
1372
|
+
# --- Pass 4: assumption audit -- ALL claims, plus completeness ---
|
|
1373
|
+
pass4_input = (
|
|
1374
|
+
f"Original query:\n{user_query}\n\n"
|
|
1375
|
+
f"All claims:\n{json.dumps([vars(c) for c in run.claims])}\n\n"
|
|
1376
|
+
f"Completeness findings:\n{json.dumps(run.completeness)}"
|
|
1377
|
+
)
|
|
1378
|
+
run.assumptions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 4", pass4_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in run.claims], effort=effort, activity=activity)))
|
|
1379
|
+
flagged_count = _apply_assumption_flags(run.claims, run.assumptions)
|
|
1380
|
+
yield from progress.checkpoint(f"Pass 4: assumption audit complete, {flagged_count} of {len(run.claims)} claim(s) flagged.")
|
|
1381
|
+
|
|
1382
|
+
# --- Pass 5: confidence tiering (0 LLM calls) ---
|
|
1383
|
+
run_confidence_tiering(run, ensemble_n=effective_n,
|
|
1384
|
+
sources_scored=_sources_were_scored(run),
|
|
1385
|
+
weights=confidence_weights)
|
|
1386
|
+
yield from _stage("Pass 5")
|
|
1387
|
+
yield from _tier_events(run)
|
|
1388
|
+
yield from progress.checkpoint("Pass 5: confidence tiers assigned (pure code, zero LLM calls).")
|
|
1389
|
+
|
|
1390
|
+
# --- D1 + the 6a/6b/D2 retry loop ---
|
|
1391
|
+
# ROADMAP_v3 §90 (A85). D1 also takes the claims most ensemble
|
|
1392
|
+
# candidates did not assert, whatever they scored: disagreement is
|
|
1393
|
+
# a reason to check a claim, and the loop is where claims are
|
|
1394
|
+
# checked. ENTRY ONLY -- `still_flagged` below reads the tier alone.
|
|
1395
|
+
# Who asserted a claim does not change when 6a rewrites it, so a
|
|
1396
|
+
# loop that kept a contested claim for being contested would hold
|
|
1397
|
+
# it to D2's cap and publish it UNVERIFIED, which is the defect the
|
|
1398
|
+
# E8 penalty had.
|
|
1399
|
+
flagged = [c for c in run.claims
|
|
1400
|
+
if c.confidence_tier in FLAGGED_TIERS or is_contested(c)]
|
|
1401
|
+
contested = sum(1 for c in flagged if is_contested(c))
|
|
1402
|
+
yield from _stage("D1")
|
|
1403
|
+
yield from progress.checkpoint(
|
|
1404
|
+
f"D1: {len(flagged)} claim(s) flagged for revision"
|
|
1405
|
+
+ (f" ({contested} contested by the ensemble)" if contested else "")
|
|
1406
|
+
+ f", {len(run.claims) - len(flagged)} already clean.")
|
|
1407
|
+
|
|
1408
|
+
while flagged:
|
|
1409
|
+
# --- Pass 6a: ONE batched call across every currently-flagged claim ---
|
|
1410
|
+
pass6a_input = json.dumps([_revision_request(c) for c in flagged])
|
|
1411
|
+
revisions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6a", pass6a_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity)))
|
|
1412
|
+
|
|
1413
|
+
# COUNT THE ROUND, NOT THE REVISION. The increment used to live
|
|
1414
|
+
# inside the loop below, so only a claim Pass 6a NAMED counted --
|
|
1415
|
+
# and 6a is one batched call across every flagged claim, which is
|
|
1416
|
+
# exactly the shape a model drops an item from. A claim it omitted
|
|
1417
|
+
# could never reach D2's cap, so `while flagged:` had no bound at
|
|
1418
|
+
# all: two model calls per round, for ever. An omission, an empty
|
|
1419
|
+
# list and an
|
|
1420
|
+
# unknown id are indistinguishable here -- the lookup returns None
|
|
1421
|
+
# and the `if claim:` skips in silence -- which is why the bound
|
|
1422
|
+
# cannot be data the model supplies.
|
|
1423
|
+
#
|
|
1424
|
+
# Counting the round makes D2's cap bound the loop by
|
|
1425
|
+
# construction, and since `flagged` only ever shrinks, every claim
|
|
1426
|
+
# still in it exhausts on the same round.
|
|
1427
|
+
for claim in flagged:
|
|
1428
|
+
claim.retry_count += 1
|
|
1429
|
+
|
|
1430
|
+
revised = 0
|
|
1431
|
+
for rev in revisions:
|
|
1432
|
+
# Resolved WITHIN the batch, first-wins like
|
|
1433
|
+
# PipelineRun.claim_by_id: 6a was only asked about `flagged`,
|
|
1434
|
+
# and a revision naming a claim outside it would land on one
|
|
1435
|
+
# already finished -- while carrying a retry_count this round
|
|
1436
|
+
# did not increment, so the event below would report a stale
|
|
1437
|
+
# attempt number.
|
|
1438
|
+
claim = next((c for c in flagged if c.id == rev["claim_id"]), None)
|
|
1439
|
+
if claim:
|
|
1440
|
+
revised += 1
|
|
1441
|
+
claim.revision_text = rev["revised_text"]
|
|
1442
|
+
# §22's "which claim, which attempt" -- emitted from the
|
|
1443
|
+
# claim retry loop, which is the retry the section means.
|
|
1444
|
+
# A JSON-parse retry is a different thing and reaches the
|
|
1445
|
+
# consumer as a trace_line, via json_retry.py's own line.
|
|
1446
|
+
yield PipelineEvent(kind="retry", claim_id=claim.id,
|
|
1447
|
+
attempt=claim.retry_count)
|
|
1448
|
+
yield from progress.checkpoint(f"Pass 6a: revised {revised} of {len(flagged)} claim(s) in this retry round.")
|
|
1449
|
+
# The count above shows that something was dropped, not why
|
|
1450
|
+
# (#248). The comment on the loop bound is about the BOUND,
|
|
1451
|
+
# which the round counter settled; a reader of the trace still
|
|
1452
|
+
# needs to tell an omission from a misdirected revision.
|
|
1453
|
+
_note_entries_outside_the_batch(
|
|
1454
|
+
run, flagged, revisions, pass_id="Pass 6a",
|
|
1455
|
+
scope="this round's batch", asked="flagged claim(s)",
|
|
1456
|
+
why="a revision to a claim D1 had already finished with "
|
|
1457
|
+
"would replace its text without re-validating it.")
|
|
1458
|
+
|
|
1459
|
+
# --- Pass 6b: re-validate the revised subset only (batched, reuses Pass 5's code) ---
|
|
1460
|
+
pass6b_input = json.dumps([vars(c) for c in flagged])
|
|
1461
|
+
# SCOPED TO `flagged`, ALL THREE OF THEM -- the ids validated
|
|
1462
|
+
# against, the claims the entries are applied to, and the
|
|
1463
|
+
# claims whose sources are rescored (§66 A26). 6a was already
|
|
1464
|
+
# resolved within the batch, with the reason written beside it
|
|
1465
|
+
# ("a revision naming a claim outside it would land on one
|
|
1466
|
+
# already finished"); 6b validated against every id in the run
|
|
1467
|
+
# and applied to every claim in it, while the re-tiering below
|
|
1468
|
+
# stayed narrow. So an entry naming a clean claim changed its
|
|
1469
|
+
# grounding and left its tier alone: driven, a HIGH claim
|
|
1470
|
+
# ended up `grounding_status='ungrounded'` with `sources=[]`
|
|
1471
|
+
# beside a breakdown still recording `source_count: 1` -- which
|
|
1472
|
+
# breaks B10's property that the breakdown alone lets a reader
|
|
1473
|
+
# recompute the score.
|
|
1474
|
+
revalidation = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6b", pass6b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity)))
|
|
1475
|
+
grounding_entries = revalidation.get("grounding", [])
|
|
1476
|
+
critic_entries = revalidation.get("critic", [])
|
|
1477
|
+
_ground_and_score(run, grounding_entries, corpus,
|
|
1478
|
+
scorer, scoring_knobs, scholar, claims=flagged)
|
|
1479
|
+
_apply_critic(flagged, critic_entries)
|
|
1480
|
+
_note_entries_outside_the_batch(
|
|
1481
|
+
run, flagged, grounding_entries, critic_entries,
|
|
1482
|
+
pass_id="Pass 6b", scope="this round's batch",
|
|
1483
|
+
asked="flagged claim(s)",
|
|
1484
|
+
why="applying an answer to a claim D1 had already finished "
|
|
1485
|
+
"with would change its grounding without recomputing "
|
|
1486
|
+
"its tier.")
|
|
1487
|
+
# 6b's own checkpoint below already reports an EFFECT -- how
|
|
1488
|
+
# many claims came out clean -- rather than what it was asked
|
|
1489
|
+
# about, so it needs no count from these two.
|
|
1490
|
+
# Pass 5's function again -- code, not a call. Scoped to THIS
|
|
1491
|
+
# round's batch: 6b is only asked about `flagged`, and re-tiering
|
|
1492
|
+
# the whole run once per round recomputes claims that are already
|
|
1493
|
+
# finished, from score data those rounds did not change.
|
|
1494
|
+
run_confidence_tiering(run, ensemble_n=effective_n, claims=flagged,
|
|
1495
|
+
sources_scored=_sources_were_scored(run),
|
|
1496
|
+
weights=confidence_weights)
|
|
1497
|
+
yield from _tier_events(run)
|
|
1498
|
+
|
|
1499
|
+
still_flagged = [c for c in flagged if c.confidence_tier in FLAGGED_TIERS]
|
|
1500
|
+
yield from progress.checkpoint(f"Pass 6b: {len(flagged) - len(still_flagged)} claim(s) now clean, {len(still_flagged)} still flagged.")
|
|
1501
|
+
|
|
1502
|
+
# --- D2: retry cap check (pure code) ---
|
|
1503
|
+
exhausted = [c for c in still_flagged if c.retry_count >= MAX_RETRIES]
|
|
1504
|
+
if exhausted:
|
|
1505
|
+
# ONCE per round, outside the per-claim loop. D2 is one
|
|
1506
|
+
# decision applied to every claim that ran out of retries;
|
|
1507
|
+
# emitting inside the loop would put a D2 row in the panel
|
|
1508
|
+
# per claim, so a round that exhausted six claims would
|
|
1509
|
+
# read as six separate stages.
|
|
1510
|
+
yield from _stage("D2")
|
|
1511
|
+
for c in exhausted:
|
|
1512
|
+
_apply_fallback(c)
|
|
1513
|
+
yield PipelineEvent(kind="claim_tiered", claim_id=c.id,
|
|
1514
|
+
tier=c.confidence_tier)
|
|
1515
|
+
yield from progress.checkpoint(f"D2: claim {c.id} exhausted retries ({c.retry_count}) -> fallback UNVERIFIED.")
|
|
1516
|
+
|
|
1517
|
+
flagged = [c for c in still_flagged if c.retry_count < MAX_RETRIES]
|
|
1518
|
+
|
|
1519
|
+
# --- Pass 6c: annotate (templated, 0 LLM calls) ---
|
|
1520
|
+
for claim in run.claims:
|
|
1521
|
+
if claim.final_text is None:
|
|
1522
|
+
claim.final_text = claim.revision_text or claim.text
|
|
1523
|
+
if claim.annotation is None:
|
|
1524
|
+
claim.annotation = f"[{claim.confidence_tier}]"
|
|
1525
|
+
yield from _stage("Pass 6c")
|
|
1526
|
+
yield from progress.checkpoint("Pass 6c: annotation complete (templated, zero LLM calls).")
|
|
1527
|
+
|
|
1528
|
+
# --- Merge (pure code) ---
|
|
1529
|
+
yield from _stage("Merge")
|
|
1530
|
+
yield from progress.checkpoint(f"Merge: final claim set assembled -- {len(run.claims)} claim(s), {len(run.coverage_gaps)} coverage gap(s).")
|
|
1531
|
+
|
|
1532
|
+
# --- Final synthesis ---
|
|
1533
|
+
run.final_report = yield from _run_pass("Final synthesis", _synthesis_input(run), model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity)
|
|
1534
|
+
yield from progress.checkpoint("Final synthesis complete.")
|
|
1535
|
+
|
|
1536
|
+
# --- §20: review, consent, correct ---
|
|
1537
|
+
yield from _review_stage(run, progress, model, provider_name,
|
|
1538
|
+
authorization, review,
|
|
1539
|
+
enabled=subagent_review, effort=effort,
|
|
1540
|
+
activity=activity)
|
|
1541
|
+
|
|
1542
|
+
update_pipeline_run(run.run_id, run, status="complete")
|
|
1543
|
+
yield PipelineEvent(kind="run_complete", run=run)
|
|
1544
|
+
return
|
|
1545
|
+
except Exception:
|
|
1546
|
+
# Persist the partial state before the exception propagates. The
|
|
1547
|
+
# update is best-effort: if storage itself is what failed, we must
|
|
1548
|
+
# not mask the original error, so a second failure here is logged
|
|
1549
|
+
# (via the real logger -- it reaches stderr through Python's
|
|
1550
|
+
# lastResort handler even before logging_setup is wired in) and the
|
|
1551
|
+
# original exception still re-raises. run.log() would be inert here:
|
|
1552
|
+
# the propagated exception carries no reference back to this local
|
|
1553
|
+
# `run`, and persistence already failed, so a trace entry would
|
|
1554
|
+
# never reach anywhere durable.
|
|
1555
|
+
try:
|
|
1556
|
+
update_pipeline_run(run.run_id, run, status="failed")
|
|
1557
|
+
except Exception as persist_error:
|
|
1558
|
+
logger.error(
|
|
1559
|
+
f"Failed to persist failure state for run {run.run_id}: {persist_error}"
|
|
1560
|
+
)
|
|
1561
|
+
raise
|