@yahyahammad/venastine 0.3.17 → 0.3.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -4
- package/core/commands.py +3 -2
- package/core/loop.py +66 -10
- package/core/reasoning/events.py +6 -0
- package/core/reasoning/json_retry.py +12 -1
- package/core/reasoning/orchestrator.py +562 -270
- package/core/reasoning/pipeline_storage.py +86 -5
- package/core/reasoning/resume.py +557 -0
- package/core/reasoning/source_corpus.py +25 -10
- package/main.py +108 -6
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/safety/policy_enforcement.py +55 -14
- package/storage.py +13 -0
- package/tui/app.py +114 -8
- package/tui/app.tcss +5 -1
- package/tui/screens.py +61 -0
|
@@ -87,7 +87,12 @@ from core.reasoning.events import PipelineEvent
|
|
|
87
87
|
from core.reasoning.json_retry import parse_json_response as _parse_json_response
|
|
88
88
|
from core.reasoning.json_retry import retry_until_json, retry_until_text
|
|
89
89
|
from core.reasoning.payload_validation import validate as _validate_payload
|
|
90
|
-
from core.reasoning.pipeline_storage import
|
|
90
|
+
from core.reasoning.pipeline_storage import (
|
|
91
|
+
create_pipeline_run,
|
|
92
|
+
reopen_pipeline_run,
|
|
93
|
+
update_pipeline_run,
|
|
94
|
+
)
|
|
95
|
+
from core.reasoning.pipeline_storage import save_resume_point as _save_resume_point
|
|
91
96
|
from core.reasoning.scholar import ScholarLookup
|
|
92
97
|
from core.reasoning.source_corpus import SourceCorpus
|
|
93
98
|
from core.reasoning.source_scoring import make_scorer, score_grounding_sources
|
|
@@ -134,11 +139,13 @@ class _Progress:
|
|
|
134
139
|
only inside a live generator has no business in a persisted row.
|
|
135
140
|
"""
|
|
136
141
|
|
|
137
|
-
def __init__(self, run: PipelineRun) -> None:
|
|
142
|
+
def __init__(self, run: PipelineRun, emitted: int = 0) -> None:
|
|
138
143
|
self.run = run
|
|
139
|
-
|
|
144
|
+
# A resumed run starts past the trace it already has (ROADMAP_v3
|
|
145
|
+
# §98): those lines were shown when they happened.
|
|
146
|
+
self._emitted = emitted
|
|
140
147
|
|
|
141
|
-
def checkpoint(self, message: str | None = None):
|
|
148
|
+
def checkpoint(self, message: str | None = None, done: dict | None = None):
|
|
142
149
|
"""Log (optionally), PERSIST, then emit every un-emitted line.
|
|
143
150
|
|
|
144
151
|
Persist BEFORE emitting, not after. A generator only advances
|
|
@@ -150,15 +157,47 @@ class _Progress:
|
|
|
150
157
|
|
|
151
158
|
Monotonic: `_emitted` only ever advances, so no line can be
|
|
152
159
|
yielded twice however many times this is called.
|
|
160
|
+
|
|
161
|
+
`done` (ROADMAP_v3 §98, RS2) is the resume point of a stage this
|
|
162
|
+
checkpoint ENDS, built by resume.snapshot. Saved after the data, so
|
|
163
|
+
a run killed between the two resumes one stage early rather than
|
|
164
|
+
one stage late. A point that cannot be saved is WARNED and the run
|
|
165
|
+
goes on: it is an aid to a later resume, not the run's record, and
|
|
166
|
+
the cost of losing one is a resume that starts a stage earlier --
|
|
167
|
+
never a reason to fail a run that is working.
|
|
153
168
|
"""
|
|
154
169
|
if message is not None:
|
|
155
170
|
self.run.log(message)
|
|
156
171
|
update_pipeline_run(self.run.run_id, self.run)
|
|
172
|
+
if done is not None:
|
|
173
|
+
self.save_point(done)
|
|
157
174
|
while self._emitted < len(self.run.trace):
|
|
158
175
|
line = self.run.trace[self._emitted]
|
|
159
176
|
self._emitted += 1
|
|
160
177
|
yield PipelineEvent(kind="trace_line", text=line)
|
|
161
178
|
|
|
179
|
+
def save_point(self, point: dict) -> None:
|
|
180
|
+
"""Save a resume point, warning rather than raising (ROADMAP_v3 §98,
|
|
181
|
+
RS2) -- `checkpoint`'s docstring says why a point is not worth a
|
|
182
|
+
working run."""
|
|
183
|
+
try:
|
|
184
|
+
_save_resume_point(self.run.run_id, point)
|
|
185
|
+
except Exception: # noqa: BLE001 -- see checkpoint's docstring
|
|
186
|
+
logger.warning("Could not save the resume point after %s for "
|
|
187
|
+
"run %s.", point.get("stage"), self.run.run_id,
|
|
188
|
+
exc_info=True)
|
|
189
|
+
|
|
190
|
+
def recorder(self, pass_id: str):
|
|
191
|
+
"""The `on_thread` a pass is given (ROADMAP_v3 §98, RS3): note the
|
|
192
|
+
pass's thread on the run the moment it exists, and persist it then
|
|
193
|
+
and there. A pass that raises, or a run a quit abandons, would
|
|
194
|
+
otherwise leave a thread its record cannot name -- the abandoned
|
|
195
|
+
case reaches no `except` at all, so nothing later writes it."""
|
|
196
|
+
def record(thread_id) -> None:
|
|
197
|
+
if _note_thread(pass_id, thread_id, self.run.pass_threads):
|
|
198
|
+
update_pipeline_run(self.run.run_id, self.run)
|
|
199
|
+
return record
|
|
200
|
+
|
|
162
201
|
|
|
163
202
|
# One pass_activity event per this many characters streamed. A per-token
|
|
164
203
|
# event would be thousands of post_message hand-offs per pass, which is the
|
|
@@ -439,10 +478,71 @@ def _pass_failed_event(pass_id: str, exc: BaseException) -> PipelineEvent:
|
|
|
439
478
|
text=f"{type(exc).__name__}: {exc}"[:200])
|
|
440
479
|
|
|
441
480
|
|
|
481
|
+
def _pass_kwargs(on_thread, resume_thread) -> dict:
|
|
482
|
+
"""The two §98 arguments a pass forwards to the loop, only when set, so
|
|
483
|
+
a pass that is neither recorded early nor resumed calls
|
|
484
|
+
`stream_deep_research_mode` exactly as it did before §98."""
|
|
485
|
+
kwargs = {}
|
|
486
|
+
if on_thread is not None:
|
|
487
|
+
kwargs["on_thread"] = on_thread
|
|
488
|
+
if resume_thread is not None:
|
|
489
|
+
kwargs["resume_thread"] = resume_thread
|
|
490
|
+
return kwargs
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _resumed_attempt(pass_id: str, thread_id, model: str, provider_name: str,
|
|
494
|
+
temperature, authorization, effort, activity, corpus,
|
|
495
|
+
trace, on_thread):
|
|
496
|
+
"""The first attempt of a pass a resume hands its old thread (ROADMAP_v3
|
|
497
|
+
§98, RS4). Returns the response to carry on with, or None when the
|
|
498
|
+
thread cannot be continued and the pass must start again.
|
|
499
|
+
|
|
500
|
+
THREE OUTCOMES, by what the thread's last row is (resume.thread_tail):
|
|
501
|
+
|
|
502
|
+
an answer the response the pass returned before the run died. It is
|
|
503
|
+
handed back AS the first attempt, with no model call, so
|
|
504
|
+
everything after it -- the truncation check, the JSON
|
|
505
|
+
re-check and its corrective retries, the apply -- runs on
|
|
506
|
+
it exactly as it ran the first time. The run that
|
|
507
|
+
prompted §98 failed on an answer the parser now accepts
|
|
508
|
+
(RS7), so its Pass 3a resumes with no call at all.
|
|
509
|
+
a request the run stopped before the model replied. The loop
|
|
510
|
+
resends it in the same thread, which keeps every tool
|
|
511
|
+
result the pass had gathered.
|
|
512
|
+
nothing the thread is gone. The pass starts again.
|
|
513
|
+
"""
|
|
514
|
+
from core.client import ModelResponse
|
|
515
|
+
from core.reasoning import resume as resume_module
|
|
516
|
+
|
|
517
|
+
kind, text = resume_module.thread_tail(thread_id)
|
|
518
|
+
if kind == "answer":
|
|
519
|
+
if trace is not None:
|
|
520
|
+
trace.append(f"{pass_id}: resumed from the answer stored in its "
|
|
521
|
+
f"thread; no model call.")
|
|
522
|
+
response = ModelResponse(text=text, usage={})
|
|
523
|
+
response.thread_id = thread_id
|
|
524
|
+
return response
|
|
525
|
+
if kind == "pending":
|
|
526
|
+
if trace is not None:
|
|
527
|
+
trace.append(f"{pass_id}: resumed in its own thread, resending "
|
|
528
|
+
f"the request the run stopped on.")
|
|
529
|
+
return (yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
530
|
+
pass_input=None, model=model, pass_id=pass_id,
|
|
531
|
+
provider_name=provider_name, temperature=temperature,
|
|
532
|
+
authorization=authorization, effort=effort, activity=activity,
|
|
533
|
+
**_pass_kwargs(on_thread, thread_id),
|
|
534
|
+
), corpus=corpus))
|
|
535
|
+
if trace is not None:
|
|
536
|
+
trace.append(f"{pass_id}: its thread could not be read back; "
|
|
537
|
+
f"starting the pass again.")
|
|
538
|
+
return None
|
|
539
|
+
|
|
540
|
+
|
|
442
541
|
def _run_pass(pass_id: str, pass_input: str, model: str, provider_name: str,
|
|
443
542
|
temperature: float | None = None, authorization=None,
|
|
444
543
|
trace: list[str] | None = None, pass_threads: list | None = None,
|
|
445
|
-
effort: str | None = None, corpus=None, activity=None
|
|
544
|
+
effort: str | None = None, corpus=None, activity=None,
|
|
545
|
+
on_thread=None, resume_thread=None):
|
|
446
546
|
"""One LLM-backed pass. Every actual model call in this file goes
|
|
447
547
|
through this single function.
|
|
448
548
|
|
|
@@ -471,14 +571,25 @@ def _run_pass(pass_id: str, pass_input: str, model: str, provider_name: str,
|
|
|
471
571
|
The retries share the JSON passes' two §26 limits, stated at
|
|
472
572
|
`_run_pass_with_json_retry`: a tool call a corrective attempt makes
|
|
473
573
|
produces no tool_call event, and a page it fetches reaches no corpus.
|
|
574
|
+
|
|
575
|
+
ROADMAP_v3 §98: `on_thread` records the pass's thread as it opens
|
|
576
|
+
(RS3), and `resume_thread` is a failed pass's thread for a resume to
|
|
577
|
+
continue (RS4, `_resumed_attempt`). A resumed answer goes through the
|
|
578
|
+
same text retry as a fresh one.
|
|
474
579
|
"""
|
|
475
580
|
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
476
581
|
try:
|
|
477
|
-
response =
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
582
|
+
response = None
|
|
583
|
+
if resume_thread is not None:
|
|
584
|
+
response = yield from _resumed_attempt(
|
|
585
|
+
pass_id, resume_thread, model, provider_name, temperature,
|
|
586
|
+
authorization, effort, activity, corpus, trace, on_thread)
|
|
587
|
+
if response is None:
|
|
588
|
+
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
589
|
+
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
590
|
+
temperature=temperature, authorization=authorization, effort=effort,
|
|
591
|
+
activity=activity, **_pass_kwargs(on_thread, None),
|
|
592
|
+
), corpus=corpus)
|
|
482
593
|
_record_granted_calls(pass_id, response, authorization)
|
|
483
594
|
_record_pass_thread(pass_id, response, pass_threads)
|
|
484
595
|
_check_not_truncated(pass_id, response, trace)
|
|
@@ -545,10 +656,24 @@ def _record_pass_thread(pass_id: str, response, pass_threads) -> None:
|
|
|
545
656
|
"""
|
|
546
657
|
if pass_threads is None:
|
|
547
658
|
return
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
659
|
+
_note_thread(pass_id, getattr(response, "thread_id", None), pass_threads)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def _note_thread(pass_id: str, thread_id, pass_threads) -> bool:
|
|
663
|
+
"""Add one pass thread to the run's list, once. Returns whether it was
|
|
664
|
+
new. Since ROADMAP_v3 §98 (RS3) a pass reports its thread when it
|
|
665
|
+
STARTS and again when it returns, and a resumed pass reports the thread
|
|
666
|
+
it continues, so the second report of an id must not be a second
|
|
667
|
+
entry: §27 promises one entry per pass however many corrections it
|
|
668
|
+
took."""
|
|
669
|
+
if pass_threads is None or thread_id is None:
|
|
670
|
+
return False
|
|
671
|
+
key = str(thread_id)
|
|
672
|
+
if any(isinstance(e, dict) and e.get("thread_id") == key
|
|
673
|
+
for e in pass_threads):
|
|
674
|
+
return False
|
|
675
|
+
pass_threads.append({"pass": pass_id, "thread_id": key})
|
|
676
|
+
return True
|
|
552
677
|
|
|
553
678
|
|
|
554
679
|
def _record_granted_calls(pass_id: str, response, authorization) -> None:
|
|
@@ -584,6 +709,8 @@ def _run_pass_with_json_retry(
|
|
|
584
709
|
effort: str | None = None,
|
|
585
710
|
corpus=None,
|
|
586
711
|
activity=None,
|
|
712
|
+
on_thread=None,
|
|
713
|
+
resume_thread=None,
|
|
587
714
|
):
|
|
588
715
|
"""Runs one JSON-emitting pass and recovers from malformed JSON.
|
|
589
716
|
|
|
@@ -640,14 +767,25 @@ def _run_pass_with_json_retry(
|
|
|
640
767
|
other source with no captured text. It degrades to the documented
|
|
641
768
|
fallback instead of to a wrong number, which is why the limit is
|
|
642
769
|
stated here and not worked around.
|
|
770
|
+
|
|
771
|
+
ROADMAP_v3 §98: `on_thread` and `resume_thread` as for `_run_pass`. A
|
|
772
|
+
resumed answer goes through the same retry loop as a fresh one, so the
|
|
773
|
+
re-check with no model call and the corrective retries after it are
|
|
774
|
+
this function's existing path rather than a second one.
|
|
643
775
|
"""
|
|
644
776
|
yield PipelineEvent(kind="pass_start", pass_id=pass_id)
|
|
645
777
|
try:
|
|
646
|
-
response =
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
778
|
+
response = None
|
|
779
|
+
if resume_thread is not None:
|
|
780
|
+
response = yield from _resumed_attempt(
|
|
781
|
+
pass_id, resume_thread, model, provider_name, temperature,
|
|
782
|
+
authorization, effort, activity, corpus, trace, on_thread)
|
|
783
|
+
if response is None:
|
|
784
|
+
response = yield from _translate(pass_id, RunAgentLoop.stream_deep_research_mode(
|
|
785
|
+
pass_input=pass_input, model=model, pass_id=pass_id, provider_name=provider_name,
|
|
786
|
+
temperature=temperature, authorization=authorization, effort=effort,
|
|
787
|
+
activity=activity, **_pass_kwargs(on_thread, None),
|
|
788
|
+
), corpus=corpus)
|
|
651
789
|
_record_granted_calls(pass_id, response, authorization)
|
|
652
790
|
# §27: the retries re-enter THIS thread (continue_conversation), so one
|
|
653
791
|
# entry per pass is the whole truth however many corrections it took.
|
|
@@ -783,8 +921,7 @@ def _ground_and_score(run: PipelineRun, grounding_entries: list[dict], corpus,
|
|
|
783
921
|
"""
|
|
784
922
|
targets = run.claims if claims is None else claims
|
|
785
923
|
applied = _apply_grounding(targets, grounding_entries)
|
|
786
|
-
|
|
787
|
-
run.source_documents = corpus.artifact_entries()
|
|
924
|
+
_collect_sources(run, corpus)
|
|
788
925
|
knobs = knobs or {}
|
|
789
926
|
score_grounding_sources(
|
|
790
927
|
run, claims=claims, corpus=corpus, scorer=scorer, scholar=scholar,
|
|
@@ -795,6 +932,20 @@ def _ground_and_score(run: PipelineRun, grounding_entries: list[dict], corpus,
|
|
|
795
932
|
return applied
|
|
796
933
|
|
|
797
934
|
|
|
935
|
+
def _collect_sources(run: PipelineRun, corpus) -> None:
|
|
936
|
+
"""THE writer of `run.source_documents`: the evidence the run gathered,
|
|
937
|
+
as the corpus holds it now.
|
|
938
|
+
|
|
939
|
+
Two callers. `_ground_and_score`, as before, for both grounding sites.
|
|
940
|
+
And a resumed run that is past Pass 3a (ROADMAP_v3 §98, RS6): the field
|
|
941
|
+
is in-memory only, so a resume starts without it, and its corpus has
|
|
942
|
+
just been rebuilt from the stored tool results -- without this its
|
|
943
|
+
artifacts would list no sources unless a 6b round happened to run. It
|
|
944
|
+
is the same call both times, which is what keeps "one writer" true."""
|
|
945
|
+
if corpus is not None:
|
|
946
|
+
run.source_documents = corpus.artifact_entries()
|
|
947
|
+
|
|
948
|
+
|
|
798
949
|
def _sources_were_scored(run: PipelineRun) -> bool:
|
|
799
950
|
"""Whether §45's scoring stage ran for this run.
|
|
800
951
|
|
|
@@ -1107,7 +1258,7 @@ def run_pipeline_to_completion(gen) -> PipelineRun:
|
|
|
1107
1258
|
|
|
1108
1259
|
|
|
1109
1260
|
def stream_deep_research_pipeline(
|
|
1110
|
-
user_query: str,
|
|
1261
|
+
user_query: str | None,
|
|
1111
1262
|
model: str,
|
|
1112
1263
|
provider_name: str = DEFAULT_PROVIDER,
|
|
1113
1264
|
ensemble_mode: bool | None = None,
|
|
@@ -1121,6 +1272,7 @@ def stream_deep_research_pipeline(
|
|
|
1121
1272
|
# the model call only forward. Batch 59 stopped at
|
|
1122
1273
|
# stream_deep_research_mode's signature; this is the rest of it.
|
|
1123
1274
|
activity=None,
|
|
1275
|
+
resume=None,
|
|
1124
1276
|
):
|
|
1125
1277
|
"""The pipeline, as a generator of PipelineEvents (§22). The terminal
|
|
1126
1278
|
`run_complete` event carries the finished PipelineRun.
|
|
@@ -1152,7 +1304,22 @@ def stream_deep_research_pipeline(
|
|
|
1152
1304
|
subagent_review (§20/D9): whether the review stage runs at all. None
|
|
1153
1305
|
falls back to config.SUBAGENT_REVIEW, matching ensemble_mode's
|
|
1154
1306
|
pattern. Separate from `review` because --review on a piped run must
|
|
1155
|
-
still produce findings while applying nothing.
|
|
1307
|
+
still produce findings while applying nothing.
|
|
1308
|
+
|
|
1309
|
+
resume (ROADMAP_v3 §98): the id of a failed or abandoned run to pick
|
|
1310
|
+
up instead of starting one; `user_query` is then the record's own and
|
|
1311
|
+
the argument is ignored. core/reasoning/resume.py rebuilds the run and
|
|
1312
|
+
says where it stopped, BEFORE anything changes -- a run it refuses
|
|
1313
|
+
raises ResumeRefused here and keeps its status. After that the run
|
|
1314
|
+
goes through this same body: a stage it completed is skipped (RS1),
|
|
1315
|
+
the pass it failed in continues its own thread (RS4), and the run ends
|
|
1316
|
+
'complete' or 'failed' like any other. The model, effort and critic
|
|
1317
|
+
are the ones passed now (RS8), and authorization is the shell's again,
|
|
1318
|
+
because a grant is never stored (R12).
|
|
1319
|
+
|
|
1320
|
+
A RESUME POINT is saved at the end of every stage (RS2), through
|
|
1321
|
+
`_Progress.checkpoint(done=...)`. That is the only place it is written,
|
|
1322
|
+
so it always names a boundary the run reached in full."""
|
|
1156
1323
|
ensemble_mode = config.ENSEMBLE_MODE if ensemble_mode is None else ensemble_mode
|
|
1157
1324
|
subagent_review = (config.SUBAGENT_REVIEW if subagent_review is None
|
|
1158
1325
|
else subagent_review)
|
|
@@ -1174,7 +1341,20 @@ def stream_deep_research_pipeline(
|
|
|
1174
1341
|
|
|
1175
1342
|
roster = _ensemble_roster(ensemble_mode)
|
|
1176
1343
|
|
|
1177
|
-
|
|
1344
|
+
# ROADMAP_v3 §98. A resume is prepared BEFORE the record is touched, so a
|
|
1345
|
+
# refusal leaves a failed run failed. `point` stays None for a new run,
|
|
1346
|
+
# and every `done()` below is then False.
|
|
1347
|
+
point = None
|
|
1348
|
+
if resume is not None:
|
|
1349
|
+
from core.reasoning import resume as resume_module
|
|
1350
|
+
point = resume_module.prepare(resume)
|
|
1351
|
+
run = point.run
|
|
1352
|
+
user_query = run.user_query
|
|
1353
|
+
else:
|
|
1354
|
+
run = PipelineRun(user_query=user_query)
|
|
1355
|
+
|
|
1356
|
+
def done(stage: str, round_no: int = 0) -> bool:
|
|
1357
|
+
return point is not None and point.done(stage, round_no)
|
|
1178
1358
|
|
|
1179
1359
|
# §45 (SQ2). RUN-SCOPED, and attached to every pass rather than only to
|
|
1180
1360
|
# the two that ground. The question it answers is "what did this run
|
|
@@ -1214,13 +1394,20 @@ def stream_deep_research_pipeline(
|
|
|
1214
1394
|
# unattended run's record matters most. Copying at the end instead
|
|
1215
1395
|
# would lose the trail on exactly the runs most worth auditing, and two
|
|
1216
1396
|
# writers of related data is the bug shape §22 warns about by name.
|
|
1397
|
+
# A resumed run's earlier calls go to the FRONT of the shared list, so
|
|
1398
|
+
# its granted_calls.json is the whole run's trail, in order.
|
|
1217
1399
|
if authorization is not None:
|
|
1400
|
+
authorization.granted_calls[:0] = run.granted_calls
|
|
1218
1401
|
run.granted_calls = authorization.granted_calls
|
|
1219
1402
|
|
|
1220
1403
|
# ROADMAP §5 minimal core: create the durable record up front. On a
|
|
1221
1404
|
# mid-pipeline crash, the except block below flips status to 'failed'
|
|
1222
|
-
# and persists a snapshot of the partial run before re-raising.
|
|
1223
|
-
run
|
|
1405
|
+
# and persists a snapshot of the partial run before re-raising. A
|
|
1406
|
+
# resumed run reuses its record, back at 'running' (§98).
|
|
1407
|
+
if point is None:
|
|
1408
|
+
run.run_id = create_pipeline_run(user_query)
|
|
1409
|
+
else:
|
|
1410
|
+
reopen_pipeline_run(run.run_id)
|
|
1224
1411
|
|
|
1225
1412
|
# ROADMAP §11: resolve critic provider/model once. When no critic is
|
|
1226
1413
|
# configured these fall back to the main provider/model — no-op
|
|
@@ -1237,210 +1424,268 @@ def stream_deep_research_pipeline(
|
|
|
1237
1424
|
critic_provider = critic["provider_name"] if critic else provider_name
|
|
1238
1425
|
critic_model = critic["model"] if critic else model
|
|
1239
1426
|
|
|
1240
|
-
progress = _Progress(run)
|
|
1427
|
+
progress = _Progress(run, emitted=point.trace_length if point else 0)
|
|
1428
|
+
|
|
1429
|
+
# §98. What a resume point needs that is not on the run: how many
|
|
1430
|
+
# ensemble candidates the scores count out of, and which thread holds
|
|
1431
|
+
# Pass 1's text (one per candidate). Both are restored on a resume.
|
|
1432
|
+
effective_n = point.effective_n if point else 0
|
|
1433
|
+
pass1_threads = list(point.pass1_threads) if point else []
|
|
1434
|
+
|
|
1435
|
+
def mark(stage: str, round_no: int = 0, batch=()) -> dict:
|
|
1436
|
+
from core.reasoning.resume import snapshot
|
|
1437
|
+
return snapshot(run, stage, round_no=round_no, flagged=batch,
|
|
1438
|
+
effective_n=effective_n, pass1_threads=pass1_threads)
|
|
1439
|
+
|
|
1440
|
+
def via(pass_id: str) -> dict:
|
|
1441
|
+
"""The §98 arguments one pass is given: its thread is recorded as
|
|
1442
|
+
it opens (RS3), and a resume hands the failed pass its own (RS4)."""
|
|
1443
|
+
return {"on_thread": progress.recorder(pass_id),
|
|
1444
|
+
"resume_thread": point.take_thread(pass_id) if point else None}
|
|
1445
|
+
|
|
1446
|
+
# §98. A new run has a resume point before its first pass: "nothing
|
|
1447
|
+
# completed yet". Without it a run that failed in Pass 0 would carry no
|
|
1448
|
+
# point, which is what marks a run recorded before §98, and would be
|
|
1449
|
+
# read from its trace instead.
|
|
1450
|
+
if point is None:
|
|
1451
|
+
progress.save_point(mark(None))
|
|
1241
1452
|
|
|
1242
1453
|
try:
|
|
1454
|
+
# --- §98: a resume says where it picks up, before anything runs ---
|
|
1455
|
+
yield PipelineEvent(kind="run_started", text=str(run.run_id))
|
|
1456
|
+
if point is not None:
|
|
1457
|
+
run.log(resume_module.describe(point, provider_name, model))
|
|
1458
|
+
for note in point.notes:
|
|
1459
|
+
run.log(note)
|
|
1460
|
+
added, skipped = resume_module.rebuild_corpus(corpus, run.pass_threads)
|
|
1461
|
+
run.log(f"Resume: rebuilt the source corpus from {added} stored "
|
|
1462
|
+
f"tool result(s)"
|
|
1463
|
+
+ (f"; {skipped} could not be read back and were skipped"
|
|
1464
|
+
if skipped else "") + ".")
|
|
1465
|
+
if done("Pass 3a"):
|
|
1466
|
+
_collect_sources(run, corpus)
|
|
1467
|
+
# The claims the run already has, so a panel and /claims show
|
|
1468
|
+
# them from the start rather than only those a later stage
|
|
1469
|
+
# touches.
|
|
1470
|
+
for claim in run.claims:
|
|
1471
|
+
yield PipelineEvent(kind="claim_extracted", claim_id=claim.id,
|
|
1472
|
+
text=claim.text)
|
|
1473
|
+
if claim.confidence_tier is not None:
|
|
1474
|
+
yield PipelineEvent(kind="claim_tiered", claim_id=claim.id,
|
|
1475
|
+
tier=claim.confidence_tier)
|
|
1476
|
+
yield from progress.checkpoint()
|
|
1477
|
+
|
|
1243
1478
|
# --- Pass 0: preliminary plan ---
|
|
1244
|
-
|
|
1245
|
-
|
|
1479
|
+
if not done("Pass 0"):
|
|
1480
|
+
run.plan = _parse_json_response((yield from _run_pass_with_json_retry("Pass 0", user_query, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity, **via("Pass 0"))))
|
|
1481
|
+
yield from progress.checkpoint(f"Pass 0: plan produced ({len(run.plan.get('key_entities_or_subjects', []))} key entities anticipated).", done=mark("Pass 0"))
|
|
1246
1482
|
|
|
1247
1483
|
# --- Pass 1: initial generation (ensemble mode: N candidates) ---
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
f"out of {effective_n}.")
|
|
1312
|
-
else:
|
|
1313
|
-
# DEGRADE TO THE SINGLE-CANDIDATE PATH (E7), rather than
|
|
1314
|
-
# scoring one candidate as an ensemble of one -- which would
|
|
1315
|
-
# give every claim consistency 1.0, so none could ever be
|
|
1316
|
-
# contested, while reporting that agreement as if it had
|
|
1317
|
-
# been corroborated.
|
|
1318
|
-
# ensemble_n=0 is not a degraded formula, it is the correct
|
|
1319
|
-
# one when there is nothing to compare against.
|
|
1320
|
-
effective_n = 0
|
|
1321
|
-
if candidates:
|
|
1484
|
+
if not done("Pass 1"):
|
|
1485
|
+
pass1_input = f"Original query:\n{user_query}\n\nPreliminary plan (loose context, may deviate freely):\n{json.dumps(run.plan)}"
|
|
1486
|
+
effective_n = 0
|
|
1487
|
+
pass1_threads = []
|
|
1488
|
+
if roster:
|
|
1489
|
+
# A LOOP, not a comprehension: _run_pass is a generator now
|
|
1490
|
+
# (§22), and a comprehension would collect generator objects
|
|
1491
|
+
# without ever running a pass.
|
|
1492
|
+
#
|
|
1493
|
+
# One candidate per roster entry, each on its OWN provider/model
|
|
1494
|
+
# (E1). No new plumbing: _run() calls api_initialization() itself
|
|
1495
|
+
# and effort_for() validates against the receiving model, so a
|
|
1496
|
+
# heterogeneous roster is correct by construction -- the same way
|
|
1497
|
+
# §11 already routes 3a/3b/6c to critic_provider/critic_model.
|
|
1498
|
+
candidates = []
|
|
1499
|
+
for entry in roster:
|
|
1500
|
+
label = f"{entry['provider_name']}/{entry['model']}"
|
|
1501
|
+
try:
|
|
1502
|
+
candidates.append((yield from _run_pass(
|
|
1503
|
+
"Pass 1", pass1_input, entry["model"], entry["provider_name"],
|
|
1504
|
+
authorization=authorization, trace=run.trace,
|
|
1505
|
+
pass_threads=run.pass_threads, effort=effort,
|
|
1506
|
+
activity=activity, **via("Pass 1"))))
|
|
1507
|
+
except Exception as e:
|
|
1508
|
+
# NAMED, TRACED, SKIPPED (E7) -- §20's containment rule.
|
|
1509
|
+
# An optional generation strategy must not flip a
|
|
1510
|
+
# ten-pass run to status='failed' because one provider
|
|
1511
|
+
# was unreachable. Same posture MCP takes per server.
|
|
1512
|
+
message = (f"Pass 1: ensemble candidate on {label} failed "
|
|
1513
|
+
f"({type(e).__name__}: {e}) — skipped.")
|
|
1514
|
+
logger.warning("%s", message)
|
|
1515
|
+
run.log(message)
|
|
1516
|
+
continue
|
|
1517
|
+
run.log(f"Pass 1: ensemble candidate {len(candidates)} generated on {label}.")
|
|
1518
|
+
# §98: which thread holds this candidate's text, for a
|
|
1519
|
+
# resume to read back. The newest Pass 1 entry is this
|
|
1520
|
+
# candidate's: it was recorded as the pass opened.
|
|
1521
|
+
pass1_threads.append(_last_thread(run.pass_threads, "Pass 1"))
|
|
1522
|
+
# E13/#77: the record keeps every surviving candidate.
|
|
1523
|
+
# The metadata half sits beside the trace's candidate-N
|
|
1524
|
+
# lines so a stored asserted_by_candidates tag names
|
|
1525
|
+
# models this run can still identify; the text half feeds
|
|
1526
|
+
# the per-candidate artifacts and Pass 3b's input -- the
|
|
1527
|
+
# two places that must be able to check a claim against
|
|
1528
|
+
# what its number actually said. pipeline_storage strips
|
|
1529
|
+
# `text` at serialization; see PipelineRunRecord.
|
|
1530
|
+
run.candidates.append({
|
|
1531
|
+
"candidate": len(candidates),
|
|
1532
|
+
"provider_name": entry["provider_name"],
|
|
1533
|
+
"model": entry["model"],
|
|
1534
|
+
"chars": len(candidates[-1]),
|
|
1535
|
+
"text": candidates[-1],
|
|
1536
|
+
})
|
|
1537
|
+
|
|
1538
|
+
# Labels come from the SURVIVORS, never from position in the
|
|
1539
|
+
# roster (E11). Labelling by roster position leaves a gap when a
|
|
1540
|
+
# candidate fails -- "Candidate 1, Candidate 3" -- so Pass 2
|
|
1541
|
+
# would tag a claim both survivors asserted as [1, 3] while the
|
|
1542
|
+
# denominator said 2, and a unanimous claim would be scored as
|
|
1543
|
+
# dissented-against. A silently deflated confidence score is the
|
|
1544
|
+
# exact failure class this section exists to remove.
|
|
1545
|
+
effective_n = len(candidates)
|
|
1546
|
+
if effective_n >= 2:
|
|
1322
1547
|
run.raw_response = candidates[0]
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
"
|
|
1326
|
-
"cannot disagree with itself.")
|
|
1548
|
+
run.log(f"Pass 1: ensemble mode — {effective_n} candidate(s) "
|
|
1549
|
+
f"across {effective_n} model(s); consistency is measured "
|
|
1550
|
+
f"out of {effective_n}.")
|
|
1327
1551
|
else:
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1552
|
+
# DEGRADE TO THE SINGLE-CANDIDATE PATH (E7), rather than
|
|
1553
|
+
# scoring one candidate as an ensemble of one -- which would
|
|
1554
|
+
# give every claim consistency 1.0, so none could ever be
|
|
1555
|
+
# contested, while reporting that agreement as if it had
|
|
1556
|
+
# been corroborated.
|
|
1557
|
+
# ensemble_n=0 is not a degraded formula, it is the correct
|
|
1558
|
+
# one when there is nothing to compare against.
|
|
1559
|
+
effective_n = 0
|
|
1560
|
+
if candidates:
|
|
1561
|
+
run.raw_response = candidates[0]
|
|
1562
|
+
run.log("Pass 1: only one ensemble candidate survived — "
|
|
1563
|
+
"no claim can be contested, since one candidate "
|
|
1564
|
+
"cannot disagree with itself.")
|
|
1565
|
+
else:
|
|
1566
|
+
raise RuntimeError(
|
|
1567
|
+
"Pass 1: every ensemble candidate failed; there is no "
|
|
1568
|
+
"response to extract claims from. See the trace for "
|
|
1569
|
+
"each provider's error."
|
|
1570
|
+
)
|
|
1571
|
+
else:
|
|
1572
|
+
run.raw_response = yield from _run_pass("Pass 1", pass1_input, model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity, **via("Pass 1"))
|
|
1573
|
+
pass1_threads = [_last_thread(run.pass_threads, "Pass 1")]
|
|
1574
|
+
run.log("Pass 1: initial generation complete.")
|
|
1575
|
+
yield from progress.checkpoint(done=mark("Pass 1"))
|
|
1338
1576
|
|
|
1339
1577
|
# --- Pass 2: claim extraction & classification ---
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1578
|
+
if not done("Pass 2"):
|
|
1579
|
+
claims_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 2", _pass2_input(run, effective_n), model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, ensemble=effective_n >= 2, effort=effort, activity=activity, **via("Pass 2"))))
|
|
1580
|
+
run.claims = [_claim_from_json(c) for c in claims_json]
|
|
1581
|
+
for claim in run.claims:
|
|
1582
|
+
yield PipelineEvent(kind="claim_extracted", claim_id=claim.id,
|
|
1583
|
+
text=claim.text)
|
|
1584
|
+
yield from progress.checkpoint(f"Pass 2: extracted {len(run.claims)} claim(s).", done=mark("Pass 2"))
|
|
1346
1585
|
|
|
1347
1586
|
# --- gate D0: route by claim type (pure code) ---
|
|
1348
1587
|
factual_claims = [c for c in run.claims if c.type == "factual"]
|
|
1349
1588
|
non_factual_claims = [c for c in run.claims if c.type != "factual"]
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1589
|
+
if not done("D0"):
|
|
1590
|
+
yield from _stage("D0")
|
|
1591
|
+
yield from progress.checkpoint(
|
|
1592
|
+
f"D0: {len(factual_claims)} factual claim(s) routed to 3a/3b, "
|
|
1593
|
+
f"{len(non_factual_claims)} synthesis/speculative claim(s) routed directly to Pass 4.",
|
|
1594
|
+
done=mark("D0"))
|
|
1355
1595
|
|
|
1356
1596
|
# --- Pass 3a + 3b: only for factual claims, batched by deduplicated entity ---
|
|
1357
1597
|
if factual_claims:
|
|
1358
1598
|
unique_entities = sorted({e for c in factual_claims for e in c.entities})
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
# SCOPED TO `factual_claims`, like 6b to its batch (batch 220,
|
|
1365
|
-
# #247): 3a is shown the factual claims and nothing else, so an
|
|
1366
|
-
# entry naming another claim is dropped and traced rather than
|
|
1367
|
-
# marking a claim grounded that gate D0 routed past grounding.
|
|
1368
|
-
grounding_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3a", pass3a_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity)))
|
|
1369
|
-
grounded = _ground_and_score(run, grounding_json, corpus, scorer,
|
|
1370
|
-
scoring_knobs, scholar,
|
|
1371
|
-
claims=factual_claims)
|
|
1372
|
-
yield from progress.checkpoint(f"Pass 3a: grounded {grounded} of {len(factual_claims)} factual claim(s), across {len(unique_entities)} deduplicated entities.")
|
|
1373
|
-
_note_entries_outside_the_batch(
|
|
1374
|
-
run, factual_claims, grounding_json, pass_id="Pass 3a",
|
|
1375
|
-
scope="the factual claims", asked="factual claim(s)",
|
|
1376
|
-
why="Gate D0 routes every other claim past 3a/3b, so grounding "
|
|
1377
|
-
"one would record a check gate D0 says never ran.")
|
|
1378
|
-
|
|
1379
|
-
# E13/#77. Two or more survivors: the claims were extracted
|
|
1380
|
-
# from the UNION of the candidates, so the critic sees every
|
|
1381
|
-
# candidate under the labels Pass 2 used -- a claim only
|
|
1382
|
-
# candidate 3 asserted is checked against text that actually
|
|
1383
|
-
# contains it, and contradictions ACROSS candidates are
|
|
1384
|
-
# findable. Handing it bare `raw_response` (candidate 1)
|
|
1385
|
-
# asked an ill-posed question: find conflicts within a text
|
|
1386
|
-
# much of your input did not come from.
|
|
1387
|
-
if len(run.candidates) >= 2:
|
|
1388
|
-
pass3b_input = (
|
|
1389
|
-
"\n\n".join(
|
|
1390
|
-
f"Candidate {entry['candidate']}:\n{entry['text']}"
|
|
1391
|
-
for entry in run.candidates)
|
|
1392
|
-
+ f"\n\nFactual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1393
|
-
)
|
|
1394
|
-
else:
|
|
1395
|
-
pass3b_input = (
|
|
1396
|
-
f"Raw response:\n{run.raw_response}\n\n"
|
|
1397
|
-
f"Factual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1599
|
+
if not done("Pass 3a"):
|
|
1600
|
+
pass3a_input = (
|
|
1601
|
+
f"Factual claims:\n{json.dumps([vars(c) for c in factual_claims])}\n\n"
|
|
1602
|
+
f"Deduplicated entities to research (search each ONCE, map results back "
|
|
1603
|
+
f"to every claim referencing it):\n{json.dumps(unique_entities)}"
|
|
1398
1604
|
)
|
|
1399
|
-
|
|
1400
|
-
|
|
1401
|
-
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
"
|
|
1411
|
-
|
|
1605
|
+
# SCOPED TO `factual_claims`, like 6b to its batch (batch 220,
|
|
1606
|
+
# #247): 3a is shown the factual claims and nothing else, so an
|
|
1607
|
+
# entry naming another claim is dropped and traced rather than
|
|
1608
|
+
# marking a claim grounded that gate D0 routed past grounding.
|
|
1609
|
+
grounding_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3a", pass3a_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity, **via("Pass 3a"))))
|
|
1610
|
+
grounded = _ground_and_score(run, grounding_json, corpus, scorer,
|
|
1611
|
+
scoring_knobs, scholar,
|
|
1612
|
+
claims=factual_claims)
|
|
1613
|
+
yield from progress.checkpoint(f"Pass 3a: grounded {grounded} of {len(factual_claims)} factual claim(s), across {len(unique_entities)} deduplicated entities.", done=mark("Pass 3a"))
|
|
1614
|
+
_note_entries_outside_the_batch(
|
|
1615
|
+
run, factual_claims, grounding_json, pass_id="Pass 3a",
|
|
1616
|
+
scope="the factual claims", asked="factual claim(s)",
|
|
1617
|
+
why="Gate D0 routes every other claim past 3a/3b, so grounding "
|
|
1618
|
+
"one would record a check gate D0 says never ran.")
|
|
1619
|
+
|
|
1620
|
+
if not done("Pass 3b"):
|
|
1621
|
+
# E13/#77. Two or more survivors: the claims were extracted
|
|
1622
|
+
# from the UNION of the candidates, so the critic sees every
|
|
1623
|
+
# candidate under the labels Pass 2 used -- a claim only
|
|
1624
|
+
# candidate 3 asserted is checked against text that actually
|
|
1625
|
+
# contains it, and contradictions ACROSS candidates are
|
|
1626
|
+
# findable. Handing it bare `raw_response` (candidate 1)
|
|
1627
|
+
# asked an ill-posed question: find conflicts within a text
|
|
1628
|
+
# much of your input did not come from.
|
|
1629
|
+
if len(run.candidates) >= 2:
|
|
1630
|
+
pass3b_input = (
|
|
1631
|
+
"\n\n".join(
|
|
1632
|
+
f"Candidate {entry['candidate']}:\n{entry['text']}"
|
|
1633
|
+
for entry in run.candidates)
|
|
1634
|
+
+ f"\n\nFactual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1635
|
+
)
|
|
1636
|
+
else:
|
|
1637
|
+
pass3b_input = (
|
|
1638
|
+
f"Raw response:\n{run.raw_response}\n\n"
|
|
1639
|
+
f"Factual claims with grounding:\n{json.dumps([vars(c) for c in factual_claims])}"
|
|
1640
|
+
)
|
|
1641
|
+
critic_json = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3b", pass3b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in factual_claims], effort=effort, activity=activity, **via("Pass 3b"))))
|
|
1642
|
+
# The same scope as 3a, for a stronger reason: a non-factual
|
|
1643
|
+
# claim's score IS its capped critic component, so a severity
|
|
1644
|
+
# landing on one moved its tier (measured, batch 220).
|
|
1645
|
+
critiqued = _apply_critic(factual_claims, critic_json)
|
|
1646
|
+
yield from progress.checkpoint(f"Pass 3b: critique applied to {critiqued} of {len(factual_claims)} factual claim(s).", done=mark("Pass 3b"))
|
|
1647
|
+
_note_entries_outside_the_batch(
|
|
1648
|
+
run, factual_claims, critic_json, pass_id="Pass 3b",
|
|
1649
|
+
scope="the factual claims", asked="factual claim(s)",
|
|
1650
|
+
why="Gate D0 routes every other claim past 3a/3b, and a "
|
|
1651
|
+
"severity on one would lower the score Pass 5 gives "
|
|
1652
|
+
"it.")
|
|
1653
|
+
elif not done("Pass 3b"):
|
|
1412
1654
|
# A stage, not a silence. "3a/3b did not run" is a fact about
|
|
1413
1655
|
# the run's shape -- it means nothing was factual -- and a
|
|
1414
1656
|
# panel that simply skipped from Pass 2 to Pass 3c would leave
|
|
1415
1657
|
# the reader to infer it from an absence.
|
|
1416
1658
|
yield from _stage("Pass 3a/3b")
|
|
1417
|
-
yield from progress.checkpoint("Pass 3a/3b: skipped -- no factual claims to ground or critique.")
|
|
1659
|
+
yield from progress.checkpoint("Pass 3a/3b: skipped -- no factual claims to ground or critique.", done=mark("Pass 3b"))
|
|
1418
1660
|
|
|
1419
1661
|
# --- Pass 3c: completeness, independent of Pass 1's raw_response ---
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1662
|
+
if not done("Pass 3c"):
|
|
1663
|
+
pass3c_input = f"Original query:\n{user_query}\n\nPreliminary plan:\n{json.dumps(run.plan)}"
|
|
1664
|
+
run.completeness = _parse_json_response((yield from _run_pass_with_json_retry("Pass 3c", pass3c_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, effort=effort, activity=activity, **via("Pass 3c"))))
|
|
1665
|
+
yield from progress.checkpoint(
|
|
1666
|
+
f"Pass 3c: coverage_score={run.completeness.get('coverage_score')}, "
|
|
1667
|
+
f"{len(run.completeness.get('gaps', []))} gap(s) identified.",
|
|
1668
|
+
done=mark("Pass 3c"))
|
|
1426
1669
|
|
|
1427
1670
|
# --- Pass 4: assumption audit -- ALL claims, plus completeness ---
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1671
|
+
if not done("Pass 4"):
|
|
1672
|
+
pass4_input = (
|
|
1673
|
+
f"Original query:\n{user_query}\n\n"
|
|
1674
|
+
f"All claims:\n{json.dumps([vars(c) for c in run.claims])}\n\n"
|
|
1675
|
+
f"Completeness findings:\n{json.dumps(run.completeness)}"
|
|
1676
|
+
)
|
|
1677
|
+
run.assumptions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 4", pass4_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in run.claims], effort=effort, activity=activity, **via("Pass 4"))))
|
|
1678
|
+
flagged_count = _apply_assumption_flags(run.claims, run.assumptions)
|
|
1679
|
+
yield from progress.checkpoint(f"Pass 4: assumption audit complete, {flagged_count} of {len(run.claims)} claim(s) flagged.", done=mark("Pass 4"))
|
|
1436
1680
|
|
|
1437
1681
|
# --- Pass 5: confidence tiering (0 LLM calls) ---
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1682
|
+
if not done("Pass 5"):
|
|
1683
|
+
run_confidence_tiering(run, ensemble_n=effective_n,
|
|
1684
|
+
sources_scored=_sources_were_scored(run),
|
|
1685
|
+
weights=confidence_weights)
|
|
1686
|
+
yield from _stage("Pass 5")
|
|
1687
|
+
yield from _tier_events(run)
|
|
1688
|
+
yield from progress.checkpoint("Pass 5: confidence tiers assigned (pure code, zero LLM calls).", done=mark("Pass 5"))
|
|
1444
1689
|
|
|
1445
1690
|
# --- D1 + the 6a/6b/D2 retry loop ---
|
|
1446
1691
|
# ROADMAP_v3 §90 (A85). D1 also takes the claims most ensemble
|
|
@@ -1451,65 +1696,85 @@ def stream_deep_research_pipeline(
|
|
|
1451
1696
|
# loop that kept a contested claim for being contested would hold
|
|
1452
1697
|
# it to D2's cap and publish it UNVERIFIED, which is the defect the
|
|
1453
1698
|
# E8 penalty had.
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1699
|
+
if not done("D1"):
|
|
1700
|
+
flagged = [c for c in run.claims
|
|
1701
|
+
if c.confidence_tier in FLAGGED_TIERS or is_contested(c)]
|
|
1702
|
+
contested = sum(1 for c in flagged if is_contested(c))
|
|
1703
|
+
yield from _stage("D1")
|
|
1704
|
+
yield from progress.checkpoint(
|
|
1705
|
+
f"D1: {len(flagged)} claim(s) flagged for revision"
|
|
1706
|
+
+ (f" ({contested} contested by the ensemble)" if contested else "")
|
|
1707
|
+
+ f", {len(run.claims) - len(flagged)} already clean.",
|
|
1708
|
+
done=mark("D1", batch=flagged))
|
|
1709
|
+
elif done("Pass 6c"):
|
|
1710
|
+
flagged = []
|
|
1711
|
+
else:
|
|
1712
|
+
# §98: the batch the loop's next step works on, as the resume
|
|
1713
|
+
# point recorded it -- after D1, after a 6a, or after a round.
|
|
1714
|
+
flagged = list(point.flagged)
|
|
1715
|
+
|
|
1716
|
+
# The round number, so a resume point inside the loop names its
|
|
1717
|
+
# round. A resume after 6a of round k re-enters round k at 6b; one
|
|
1718
|
+
# after a whole round starts the next.
|
|
1719
|
+
round_no = 0
|
|
1720
|
+
if point is not None and point.stage in ("Pass 6a", "Pass 6b"):
|
|
1721
|
+
round_no = point.round - (1 if point.stage == "Pass 6a" else 0)
|
|
1462
1722
|
|
|
1463
1723
|
while flagged:
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
claim
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1724
|
+
round_no += 1
|
|
1725
|
+
if not done("Pass 6a", round_no):
|
|
1726
|
+
# --- Pass 6a: ONE batched call across every currently-flagged claim ---
|
|
1727
|
+
pass6a_input = json.dumps([_revision_request(c) for c in flagged])
|
|
1728
|
+
revisions = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6a", pass6a_input, model, provider_name, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity, **via("Pass 6a"))))
|
|
1729
|
+
|
|
1730
|
+
# COUNT THE ROUND, NOT THE REVISION. The increment used to live
|
|
1731
|
+
# inside the loop below, so only a claim Pass 6a NAMED counted --
|
|
1732
|
+
# and 6a is one batched call across every flagged claim, which is
|
|
1733
|
+
# exactly the shape a model drops an item from. A claim it omitted
|
|
1734
|
+
# could never reach D2's cap, so `while flagged:` had no bound at
|
|
1735
|
+
# all: two model calls per round, for ever. An omission, an empty
|
|
1736
|
+
# list and an
|
|
1737
|
+
# unknown id are indistinguishable here -- the lookup returns None
|
|
1738
|
+
# and the `if claim:` skips in silence -- which is why the bound
|
|
1739
|
+
# cannot be data the model supplies.
|
|
1740
|
+
#
|
|
1741
|
+
# Counting the round makes D2's cap bound the loop by
|
|
1742
|
+
# construction, and since `flagged` only ever shrinks, every claim
|
|
1743
|
+
# still in it exhausts on the same round.
|
|
1744
|
+
for claim in flagged:
|
|
1745
|
+
claim.retry_count += 1
|
|
1746
|
+
|
|
1747
|
+
revised = 0
|
|
1748
|
+
for rev in revisions:
|
|
1749
|
+
# Resolved WITHIN the batch, first-wins like
|
|
1750
|
+
# PipelineRun.claim_by_id: 6a was only asked about `flagged`,
|
|
1751
|
+
# and a revision naming a claim outside it would land on one
|
|
1752
|
+
# already finished -- while carrying a retry_count this round
|
|
1753
|
+
# did not increment, so the event below would report a stale
|
|
1754
|
+
# attempt number.
|
|
1755
|
+
claim = next((c for c in flagged if c.id == rev["claim_id"]), None)
|
|
1756
|
+
if claim:
|
|
1757
|
+
revised += 1
|
|
1758
|
+
claim.revision_text = rev["revised_text"]
|
|
1759
|
+
# §22's "which claim, which attempt" -- emitted from the
|
|
1760
|
+
# claim retry loop, which is the retry the section means.
|
|
1761
|
+
# A JSON-parse retry is a different thing and reaches the
|
|
1762
|
+
# consumer as a trace_line, via json_retry.py's own line.
|
|
1763
|
+
yield PipelineEvent(kind="retry", claim_id=claim.id,
|
|
1764
|
+
attempt=claim.retry_count)
|
|
1765
|
+
# §98: 6a has changed the claims (their retry counts above all),
|
|
1766
|
+
# so this boundary is saved. A resume from here goes to 6b with
|
|
1767
|
+
# this batch rather than running 6a again and counting twice.
|
|
1768
|
+
yield from progress.checkpoint(f"Pass 6a: revised {revised} of {len(flagged)} claim(s) in this retry round.", done=mark("Pass 6a", round_no, flagged))
|
|
1769
|
+
# The count above shows that something was dropped, not why
|
|
1770
|
+
# (#248). The comment on the loop bound is about the BOUND,
|
|
1771
|
+
# which the round counter settled; a reader of the trace still
|
|
1772
|
+
# needs to tell an omission from a misdirected revision.
|
|
1773
|
+
_note_entries_outside_the_batch(
|
|
1774
|
+
run, flagged, revisions, pass_id="Pass 6a",
|
|
1775
|
+
scope="this round's batch", asked="flagged claim(s)",
|
|
1776
|
+
why="a revision to a claim D1 had already finished with "
|
|
1777
|
+
"would replace its text without re-validating it.")
|
|
1513
1778
|
|
|
1514
1779
|
# --- Pass 6b: re-validate the revised subset only (batched, reuses Pass 5's code) ---
|
|
1515
1780
|
pass6b_input = json.dumps([vars(c) for c in flagged])
|
|
@@ -1526,7 +1791,7 @@ def stream_deep_research_pipeline(
|
|
|
1526
1791
|
# beside a breakdown still recording `source_count: 1` -- which
|
|
1527
1792
|
# breaks B10's property that the breakdown alone lets a reader
|
|
1528
1793
|
# recompute the score.
|
|
1529
|
-
revalidation = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6b", pass6b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity)))
|
|
1794
|
+
revalidation = _parse_json_response((yield from _run_pass_with_json_retry("Pass 6b", pass6b_input, critic_model, critic_provider, run.trace, authorization=authorization, corpus=corpus, pass_threads=run.pass_threads, claim_ids=[c.id for c in flagged], effort=effort, activity=activity, **via("Pass 6b"))))
|
|
1530
1795
|
grounding_entries = revalidation.get("grounding", [])
|
|
1531
1796
|
critic_entries = revalidation.get("critic", [])
|
|
1532
1797
|
_ground_and_score(run, grounding_entries, corpus,
|
|
@@ -1570,23 +1835,29 @@ def stream_deep_research_pipeline(
|
|
|
1570
1835
|
yield from progress.checkpoint(f"D2: claim {c.id} exhausted retries ({c.retry_count}) -> fallback UNVERIFIED.")
|
|
1571
1836
|
|
|
1572
1837
|
flagged = [c for c in still_flagged if c.retry_count < MAX_RETRIES]
|
|
1838
|
+
# §98: the round is over, D2 included. The point names the batch
|
|
1839
|
+
# the next round starts with, which may be none.
|
|
1840
|
+
yield from progress.checkpoint(done=mark("Pass 6b", round_no, flagged))
|
|
1573
1841
|
|
|
1574
1842
|
# --- Pass 6c: annotate (templated, 0 LLM calls) ---
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
claim.final_text
|
|
1578
|
-
|
|
1579
|
-
claim.annotation
|
|
1580
|
-
|
|
1581
|
-
|
|
1843
|
+
if not done("Pass 6c"):
|
|
1844
|
+
for claim in run.claims:
|
|
1845
|
+
if claim.final_text is None:
|
|
1846
|
+
claim.final_text = claim.revision_text or claim.text
|
|
1847
|
+
if claim.annotation is None:
|
|
1848
|
+
claim.annotation = f"[{claim.confidence_tier}]"
|
|
1849
|
+
yield from _stage("Pass 6c")
|
|
1850
|
+
yield from progress.checkpoint("Pass 6c: annotation complete (templated, zero LLM calls).", done=mark("Pass 6c"))
|
|
1582
1851
|
|
|
1583
1852
|
# --- Merge (pure code) ---
|
|
1584
|
-
|
|
1585
|
-
|
|
1853
|
+
if not done("Merge"):
|
|
1854
|
+
yield from _stage("Merge")
|
|
1855
|
+
yield from progress.checkpoint(f"Merge: final claim set assembled -- {len(run.claims)} claim(s), {len(run.coverage_gaps)} coverage gap(s).", done=mark("Merge"))
|
|
1586
1856
|
|
|
1587
1857
|
# --- Final synthesis ---
|
|
1588
|
-
|
|
1589
|
-
|
|
1858
|
+
if not done("Final synthesis"):
|
|
1859
|
+
run.final_report = yield from _run_pass("Final synthesis", _synthesis_input(run), model, provider_name, authorization=authorization, corpus=corpus, trace=run.trace, pass_threads=run.pass_threads, effort=effort, activity=activity, **via("Final synthesis"))
|
|
1860
|
+
yield from progress.checkpoint("Final synthesis complete.", done=mark("Final synthesis"))
|
|
1590
1861
|
|
|
1591
1862
|
# --- §20: review, consent, correct ---
|
|
1592
1863
|
yield from _review_stage(run, progress, model, provider_name,
|
|
@@ -1614,3 +1885,24 @@ def stream_deep_research_pipeline(
|
|
|
1614
1885
|
f"Failed to persist failure state for run {run.run_id}: {persist_error}"
|
|
1615
1886
|
)
|
|
1616
1887
|
raise
|
|
1888
|
+
|
|
1889
|
+
|
|
1890
|
+
def _pass2_input(run: PipelineRun, effective_n: int) -> str:
|
|
1891
|
+
"""What Pass 2 extracts claims from: every surviving candidate under
|
|
1892
|
+
the label Pass 2 tags with, or the one response. A function rather than
|
|
1893
|
+
the inline string it was, because a resume at Pass 2 builds it from
|
|
1894
|
+
the restored run (ROADMAP_v3 §98) and two spellings of one input drift.
|
|
1895
|
+
`entry['candidate']` is the survivor's 1-based number, which is the
|
|
1896
|
+
label E11 requires."""
|
|
1897
|
+
if effective_n >= 2:
|
|
1898
|
+
return "\n\n".join(
|
|
1899
|
+
f"Candidate {entry['candidate']}:\n{entry['text']}"
|
|
1900
|
+
for entry in run.candidates)
|
|
1901
|
+
return f"Response to extract claims from:\n{run.raw_response}"
|
|
1902
|
+
|
|
1903
|
+
|
|
1904
|
+
def _last_thread(pass_threads, pass_id: str):
|
|
1905
|
+
"""The newest thread recorded for `pass_id`, or None (a test double
|
|
1906
|
+
that reports no thread)."""
|
|
1907
|
+
return next((e.get("thread_id") for e in reversed(pass_threads or ())
|
|
1908
|
+
if isinstance(e, dict) and e.get("pass") == pass_id), None)
|