@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,361 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/pipeline_storage.py
|
|
3
|
+
|
|
4
|
+
Persistence layer for deep-research pipeline runs (ROADMAP §5). A
|
|
5
|
+
PipelineRunRecord row is created at the start of
|
|
6
|
+
stream_deep_research_pipeline(), checkpointed with a data-only
|
|
7
|
+
update_pipeline_run() call after every pass's trace-log line, and
|
|
8
|
+
flipped to a terminal status (complete / failed) at the end. The
|
|
9
|
+
load_pipeline_run() read API returns the stored state as a plain dict
|
|
10
|
+
for inspection and debugging.
|
|
11
|
+
|
|
12
|
+
Schema deliberately mirrors the codebase convention established by
|
|
13
|
+
storage.py's MessageLog: semantically distinct data lives in separate
|
|
14
|
+
typed columns (claims_json / trace_json / coverage_gaps_json /
|
|
15
|
+
final_report), NOT one catch-all blob. The *_json columns hold
|
|
16
|
+
json.dumps'd strings (same pattern as MessageLog.content); final_report
|
|
17
|
+
is plain text. This keeps a serialization problem in one field from
|
|
18
|
+
corrupting the others -- a persistence layer built for crash-resilience
|
|
19
|
+
shouldn't itself be a single point of failure.
|
|
20
|
+
|
|
21
|
+
Tables created here are registered in SQLModel.metadata and built by
|
|
22
|
+
database.create_db_and_tables(), the same startup hook that creates
|
|
23
|
+
ConversationThread / MessageLog.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import json
|
|
29
|
+
import logging
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
from typing import Optional
|
|
32
|
+
from uuid import UUID, uuid4
|
|
33
|
+
|
|
34
|
+
from sqlmodel import Field, Session, SQLModel
|
|
35
|
+
|
|
36
|
+
from database import engine
|
|
37
|
+
# §27: the thread-kind vocabulary lives beside the table that stores it.
|
|
38
|
+
# This import direction is the only one that exists -- storage.py knows
|
|
39
|
+
# nothing about pipeline runs, and classify_legacy_pass_threads() below is
|
|
40
|
+
# where the two meet.
|
|
41
|
+
from storage import THREAD_KIND_CHAT, THREAD_KIND_RESEARCH_PASS
|
|
42
|
+
|
|
43
|
+
logger = logging.getLogger(__name__)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class PipelineRunRecord(SQLModel, table=True):
|
|
47
|
+
"""One row per stream_deep_research_pipeline() invocation. `status` is
|
|
48
|
+
'running' from creation until the wrapping try/except in the
|
|
49
|
+
orchestrator flips it to 'complete' or 'failed'. The four data
|
|
50
|
+
columns carry the run's structured output (or the partial state
|
|
51
|
+
available at failure time) so an operator can inspect what happened
|
|
52
|
+
without a separate trace log."""
|
|
53
|
+
|
|
54
|
+
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
55
|
+
user_query: str = Field(index=True)
|
|
56
|
+
status: str = Field(default="running") # "running" | "complete" | "failed"
|
|
57
|
+
claims_json: str = "[]"
|
|
58
|
+
trace_json: str = "[]"
|
|
59
|
+
coverage_gaps_json: str = "[]"
|
|
60
|
+
# §27 (T2): [{"pass": ..., "thread_id": ...}] for this run's pass
|
|
61
|
+
# threads. Persisted rather than left to the output directory alone
|
|
62
|
+
# (where granted_calls and subagent_reviews live) because those are
|
|
63
|
+
# written once, at the end, by a run that finished -- and a run whose
|
|
64
|
+
# passes are worth chasing through their threads is frequently one
|
|
65
|
+
# that did not. Checkpointed like the other data columns, so an
|
|
66
|
+
# abandoned run keeps the ids it had reached.
|
|
67
|
+
pass_threads_json: str = "[]"
|
|
68
|
+
# E13/#77: [{"candidate", "provider_name", "model", "chars"}] per
|
|
69
|
+
# ensemble-Pass-1 SURVIVOR, so a stored claim's
|
|
70
|
+
# asserted_by_candidates names models this row can still identify.
|
|
71
|
+
# Metadata only: the full texts are per-candidate artifacts and pass
|
|
72
|
+
# threads, and update_pipeline_run strips the in-memory `text` key
|
|
73
|
+
# here rather than duplicating bodies into the database.
|
|
74
|
+
# Checkpointed like the other data columns, so an abandoned run keeps
|
|
75
|
+
# what it had reached.
|
|
76
|
+
candidates_json: str = "[]"
|
|
77
|
+
final_report: str = ""
|
|
78
|
+
started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
|
79
|
+
finished_at: Optional[datetime] = None
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
# ---------------------------------------------------------------------------
|
|
83
|
+
# ---- Public write API ------------------------------------------------------
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
|
|
86
|
+
def create_pipeline_run(user_query: str) -> UUID:
|
|
87
|
+
"""Inserts a 'running' PipelineRunRecord row and returns its id.
|
|
88
|
+
The orchestrator assigns this id to PipelineRun.run_id so later
|
|
89
|
+
updates can find the row."""
|
|
90
|
+
with Session(engine) as session:
|
|
91
|
+
record = PipelineRunRecord(user_query=user_query, status="running")
|
|
92
|
+
session.add(record)
|
|
93
|
+
session.commit()
|
|
94
|
+
session.refresh(record)
|
|
95
|
+
return record.id
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def update_pipeline_run(
|
|
99
|
+
run_id: UUID,
|
|
100
|
+
run,
|
|
101
|
+
status: Optional[str] = None,
|
|
102
|
+
) -> None:
|
|
103
|
+
"""Updates an existing row's data columns (claims / trace /
|
|
104
|
+
coverage_gaps / final_report) from the live PipelineRun, and -- only
|
|
105
|
+
when `status` is given -- flips status and sets finished_at on a
|
|
106
|
+
terminal transition. Raises ValueError if run_id doesn't exist (the
|
|
107
|
+
record was never created or was deleted), which is a real bug, not an
|
|
108
|
+
expected path.
|
|
109
|
+
|
|
110
|
+
`status` is Optional by design: the minimal core only ever passes a
|
|
111
|
+
terminal status, but §5 proper's deferred per-pass checkpointing will
|
|
112
|
+
call this for plain data snapshots WITHOUT a status. Defaulting status
|
|
113
|
+
to a value here would silently reset a 'complete'/'failed' record back
|
|
114
|
+
to that default on a data-only update; Optional=None means "don't touch
|
|
115
|
+
status unless told to," which makes that future extension safe by
|
|
116
|
+
construction.
|
|
117
|
+
|
|
118
|
+
Claims are serialized with vars(c), matching every other
|
|
119
|
+
claim-serialization call site in orchestrator.py (3a/3b/5/6a/6c/final
|
|
120
|
+
synthesis). If a nested-dataclass field is ever added to Claim, ALL of
|
|
121
|
+
those sites migrate to asdict(c) together -- see the note on Claim in
|
|
122
|
+
core/reasoning/base.py.
|
|
123
|
+
"""
|
|
124
|
+
with Session(engine) as session:
|
|
125
|
+
record = session.get(PipelineRunRecord, run_id)
|
|
126
|
+
if record is None:
|
|
127
|
+
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
128
|
+
if status is not None:
|
|
129
|
+
record.status = status
|
|
130
|
+
if status in ("complete", "failed"):
|
|
131
|
+
record.finished_at = datetime.now(timezone.utc)
|
|
132
|
+
record.claims_json = json.dumps([vars(c) for c in run.claims])
|
|
133
|
+
record.trace_json = json.dumps(run.trace)
|
|
134
|
+
record.coverage_gaps_json = json.dumps(run.coverage_gaps)
|
|
135
|
+
record.pass_threads_json = json.dumps(run.pass_threads)
|
|
136
|
+
# getattr, not run.candidates: this function is called with test
|
|
137
|
+
# doubles and with runs reconstructed elsewhere, and a checkpoint
|
|
138
|
+
# that raises on a missing optional field would take down the pass
|
|
139
|
+
# that was only trying to record its progress.
|
|
140
|
+
# `text` rides the in-memory entry (it feeds the artifacts and
|
|
141
|
+
# Pass 3b) and is stripped HERE -- the database carries metadata
|
|
142
|
+
# only; see PipelineRunRecord's column comment.
|
|
143
|
+
record.candidates_json = json.dumps([
|
|
144
|
+
{k: v for k, v in entry.items() if k != "text"}
|
|
145
|
+
for entry in getattr(run, "candidates", []) or []
|
|
146
|
+
])
|
|
147
|
+
record.final_report = run.final_report
|
|
148
|
+
session.add(record)
|
|
149
|
+
session.commit()
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# ---------------------------------------------------------------------------
|
|
153
|
+
# ---- Public read API -------------------------------------------------------
|
|
154
|
+
# ---------------------------------------------------------------------------
|
|
155
|
+
|
|
156
|
+
def load_pipeline_run(run_id: UUID) -> dict:
|
|
157
|
+
"""Loads a PipelineRunRecord by id and returns its state as a plain
|
|
158
|
+
dict. Raises ValueError if run_id doesn't exist (matching
|
|
159
|
+
update_pipeline_run's convention -- a missing record is a real bug,
|
|
160
|
+
not an expected path).
|
|
161
|
+
|
|
162
|
+
Claims are returned as raw dicts (the shape vars(c) produced at write
|
|
163
|
+
time), not reconstructed Claim dataclass instances. If/when
|
|
164
|
+
programmatic pipeline resume is built, that function reconstructs
|
|
165
|
+
Claim objects itself, using the same field-filtering pattern
|
|
166
|
+
_claim_from_json already established in orchestrator.py.
|
|
167
|
+
"""
|
|
168
|
+
with Session(engine) as session:
|
|
169
|
+
record = session.get(PipelineRunRecord, run_id)
|
|
170
|
+
if record is None:
|
|
171
|
+
raise ValueError(f"No pipeline run found with id {run_id}")
|
|
172
|
+
return {
|
|
173
|
+
"id": record.id,
|
|
174
|
+
"user_query": record.user_query,
|
|
175
|
+
"status": record.status,
|
|
176
|
+
"started_at": record.started_at,
|
|
177
|
+
"finished_at": record.finished_at,
|
|
178
|
+
"claims": json.loads(record.claims_json),
|
|
179
|
+
"trace": json.loads(record.trace_json),
|
|
180
|
+
"coverage_gaps": json.loads(record.coverage_gaps_json),
|
|
181
|
+
# §27 AC6. Defaulted rather than read straight off the record:
|
|
182
|
+
# a row written before this column existed has it as NULL after
|
|
183
|
+
# ensure_columns' ALTER (the migration is additive and never
|
|
184
|
+
# backfills), and json.loads(None) raises.
|
|
185
|
+
"pass_threads": json.loads(record.pass_threads_json or "[]"),
|
|
186
|
+
# Same NULL guard as pass_threads above: ensure_columns'
|
|
187
|
+
# ALTER never backfills, so a row written before this column
|
|
188
|
+
# existed reads as NULL, and json.loads(None) raises.
|
|
189
|
+
"candidates": json.loads(record.candidates_json or "[]"),
|
|
190
|
+
"final_report": record.final_report,
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
# ---------------------------------------------------------------------------
|
|
195
|
+
# ---- One-time classification of pre-§27 threads ---------------------------
|
|
196
|
+
# ---------------------------------------------------------------------------
|
|
197
|
+
|
|
198
|
+
def classify_legacy_pass_threads(connection) -> int:
|
|
199
|
+
"""Label pass threads that predate ConversationThread.kind. Returns the
|
|
200
|
+
number of rows relabelled.
|
|
201
|
+
|
|
202
|
+
WHY THIS IS NOT PART OF ensure_columns(). That function is additive
|
|
203
|
+
only and explicitly never backfills (#21 M7) -- so every row already on
|
|
204
|
+
disk takes `kind='chat'` from the column default, and the hundreds of
|
|
205
|
+
pass threads in an existing database would still fill the picker #27
|
|
206
|
+
exists to clear. Widening ensure_columns' contract to backfill would
|
|
207
|
+
put data policy inside a schema migrator; if that function ever grows a
|
|
208
|
+
DROP or an UPDATE, the project has outgrown it.
|
|
209
|
+
|
|
210
|
+
WHY IT LIVES HERE rather than in storage.py. The signal is a pipeline
|
|
211
|
+
fact -- which threads belong to which run -- and storage.py owns schema
|
|
212
|
+
and CRUD without knowing what a run is. The dependency runs this way
|
|
213
|
+
round only; storage.py does not import this module.
|
|
214
|
+
|
|
215
|
+
TWO TIERS, EXACT FIRST (#82). Since T2, every run records the exact
|
|
216
|
+
thread ids of its passes in `pass_threads_json`. That record is
|
|
217
|
+
authoritative over any clock comparison -- so:
|
|
218
|
+
|
|
219
|
+
* TIER 1 (exact): every thread id named in a populated
|
|
220
|
+
`pass_threads_json` is a pass thread, whatever its timestamp says
|
|
221
|
+
and whether or not the run ever finished. Membership was recorded
|
|
222
|
+
at run time; there is nothing to infer.
|
|
223
|
+
* TIER 2 (window): only runs whose `pass_threads_json` is absent,
|
|
224
|
+
NULL, empty, or unusable fall back to the time-window heuristic --
|
|
225
|
+
"created between started_at and finished_at". That set is the
|
|
226
|
+
pre-T2 population, WHICH CANNOT GROW: a run recorded after T2
|
|
227
|
+
always names its own passes, so its window is never consulted and
|
|
228
|
+
a chat thread another session creates during it is safe (#82's
|
|
229
|
+
headline hazard).
|
|
230
|
+
|
|
231
|
+
CORRUPTED JSON FALLS BACK, LOUDLY. An unparseable `pass_threads_json`
|
|
232
|
+
puts THAT run in the window tier rather than crashing the launch --
|
|
233
|
+
the under-classification direction, which this docstring already
|
|
234
|
+
prefers -- and warns naming the run, because silently ignoring data
|
|
235
|
+
the writer did write is how the corruption spreads unnoticed.
|
|
236
|
+
|
|
237
|
+
A RUN WITH finished_at IS NULL GETS ONLY TIER 1. The window needs a
|
|
238
|
+
closing bound: #22's abandoned-generator case deliberately leaves
|
|
239
|
+
status='running', so an open window would swallow every thread created
|
|
240
|
+
since -- including real conversations. Tier 1 needs no bound (exact
|
|
241
|
+
ids), so an abandoned run's recorded passes ARE relabelled.
|
|
242
|
+
Under-classifying leaves a straggler in the picker; over-classifying
|
|
243
|
+
HIDES A REAL CONVERSATION, and only one of those is recoverable by the
|
|
244
|
+
user.
|
|
245
|
+
|
|
246
|
+
IDEMPOTENT BY CONSTRUCTION. It only ever moves rows still 'chat', so
|
|
247
|
+
running it at every launch is a no-op after the first -- and a database
|
|
248
|
+
last opened by an older build still gets classified, both for its
|
|
249
|
+
pre-T2 rows (window tier) and its post-T2 rows (whose ids were recorded
|
|
250
|
+
all along).
|
|
251
|
+
|
|
252
|
+
Raw SQL over a DBAPI connection (`engine.raw_connection()` in
|
|
253
|
+
production, stdlib `sqlite3.connect(...)` in tests), the same seam
|
|
254
|
+
database.ensure_columns() uses and for the same reason: the suite's
|
|
255
|
+
fake `sqlmodel` cannot execute DDL, and a seam that only works in
|
|
256
|
+
production is not a seam. Does NOT close the connection -- the caller
|
|
257
|
+
owns its lifetime. Computation happens in Python (fetch runs once,
|
|
258
|
+
fetch chat rows once, one bounded UPDATE per chunk) so the two tiers'
|
|
259
|
+
conditions are ordinary code a mutation can target, not a correlated
|
|
260
|
+
subquery whose bounds the suite could only probe end-to-end.
|
|
261
|
+
|
|
262
|
+
Threads a legacy compactor or subagent created are relabelled
|
|
263
|
+
'research_pass' here if they happen to fall inside a PRE-T2 run's
|
|
264
|
+
window, and left as 'chat' otherwise. Both acceptable: the first is
|
|
265
|
+
hidden either way, and the second is under-classification.
|
|
266
|
+
"""
|
|
267
|
+
cursor = connection.cursor()
|
|
268
|
+
cursor.execute("PRAGMA table_info(conversationthread)")
|
|
269
|
+
if not any(row[1] == "kind" for row in cursor.fetchall()):
|
|
270
|
+
# No column yet, so nothing to classify. create_all/ensure_columns
|
|
271
|
+
# run before this in production; a caller that skipped them gets a
|
|
272
|
+
# no-op rather than "no such column".
|
|
273
|
+
return 0
|
|
274
|
+
cursor.execute("PRAGMA table_info(pipelinerunrecord)")
|
|
275
|
+
run_columns = {row[1] for row in cursor.fetchall()}
|
|
276
|
+
if not run_columns:
|
|
277
|
+
# A database that has never run research. Nothing to classify, and
|
|
278
|
+
# neither query below would find its table.
|
|
279
|
+
return 0
|
|
280
|
+
has_recorded = "pass_threads_json" in run_columns
|
|
281
|
+
|
|
282
|
+
# ---- Fetch the runs once, and sort each into a tier -------------------
|
|
283
|
+
cursor.execute(
|
|
284
|
+
"SELECT id, started_at, finished_at"
|
|
285
|
+
+ (", pass_threads_json" if has_recorded else "")
|
|
286
|
+
+ " FROM pipelinerunrecord")
|
|
287
|
+
run_rows = cursor.fetchall()
|
|
288
|
+
|
|
289
|
+
exact_ids = set() # tier 1: named in some run's pass_threads_json
|
|
290
|
+
window_runs = [] # tier 2: (started_at, finished_at) pairs
|
|
291
|
+
for row in run_rows:
|
|
292
|
+
run_id, started_at, finished_at = row[0], row[1], row[2]
|
|
293
|
+
raw = row[3] if has_recorded else None
|
|
294
|
+
usable, corrupted = False, False
|
|
295
|
+
if raw:
|
|
296
|
+
try:
|
|
297
|
+
entries = json.loads(raw)
|
|
298
|
+
except ValueError:
|
|
299
|
+
corrupted = True
|
|
300
|
+
else:
|
|
301
|
+
if not isinstance(entries, list):
|
|
302
|
+
corrupted = True
|
|
303
|
+
else:
|
|
304
|
+
# T2's shape: [{"pass": ..., "thread_id": ...}]. Entries
|
|
305
|
+
# without a usable thread_id contribute nothing either
|
|
306
|
+
# way; an entry list that is merely EMPTY is the
|
|
307
|
+
# legitimate pre-T2 shape, i.e. window tier.
|
|
308
|
+
for entry in entries:
|
|
309
|
+
if (isinstance(entry, dict)
|
|
310
|
+
and isinstance(entry.get("thread_id"), str)):
|
|
311
|
+
exact_ids.add(entry["thread_id"])
|
|
312
|
+
usable = True
|
|
313
|
+
# raw None/"" (or a missing column) falls through with usable=False:
|
|
314
|
+
# the legitimate pre-T2 shape, i.e. window tier -- not corruption.
|
|
315
|
+
if corrupted:
|
|
316
|
+
# #82, owner decision B3: fall back for THIS run, and say so --
|
|
317
|
+
# data the writer did write must not vanish without a trace.
|
|
318
|
+
logger.warning(
|
|
319
|
+
"Classify pass threads: run %s has unparseable "
|
|
320
|
+
"pass_threads_json (%r); using its time window instead.",
|
|
321
|
+
run_id, raw[:80] if isinstance(raw, str) else raw)
|
|
322
|
+
if not usable:
|
|
323
|
+
# Window tier -- but only a CLOSED window. The intent guard
|
|
324
|
+
# against a future edit "kindly" treating NULL finished_at as
|
|
325
|
+
# open-ended lives here, in ordinary code, where removing it
|
|
326
|
+
# turns a test red (it never could under the old correlated
|
|
327
|
+
# subquery: SQL's x <= NULL was already not true).
|
|
328
|
+
if started_at is not None and finished_at is not None:
|
|
329
|
+
window_runs.append((started_at, finished_at))
|
|
330
|
+
|
|
331
|
+
# ---- Fetch the still-chat rows once ------------------------------------
|
|
332
|
+
cursor.execute(
|
|
333
|
+
"SELECT id, created_at FROM conversationthread WHERE kind = ?",
|
|
334
|
+
(THREAD_KIND_CHAT,))
|
|
335
|
+
chat_rows = cursor.fetchall()
|
|
336
|
+
|
|
337
|
+
to_relabel = []
|
|
338
|
+
for thread_id, created_at in chat_rows:
|
|
339
|
+
if thread_id in exact_ids:
|
|
340
|
+
to_relabel.append(thread_id)
|
|
341
|
+
elif created_at is not None and any(
|
|
342
|
+
start <= created_at <= finish for start, finish in window_runs):
|
|
343
|
+
to_relabel.append(thread_id)
|
|
344
|
+
|
|
345
|
+
relabelled = 0
|
|
346
|
+
# Chunked: SQLite's host-parameter ceiling is finite, and a long-lived
|
|
347
|
+
# database can carry thousands of pass threads.
|
|
348
|
+
for offset in range(0, len(to_relabel), 500):
|
|
349
|
+
chunk = to_relabel[offset:offset + 500]
|
|
350
|
+
placeholders = ",".join("?" * len(chunk))
|
|
351
|
+
cursor.execute(
|
|
352
|
+
f"UPDATE conversationthread SET kind = ? "
|
|
353
|
+
f" WHERE kind = ? AND id IN ({placeholders})",
|
|
354
|
+
(THREAD_KIND_RESEARCH_PASS, THREAD_KIND_CHAT, *chunk))
|
|
355
|
+
relabelled += cursor.rowcount if cursor.rowcount > 0 else 0
|
|
356
|
+
connection.commit()
|
|
357
|
+
if relabelled:
|
|
358
|
+
logger.info(
|
|
359
|
+
"Classified %d pre-\u00a727 thread(s) as research passes; they no "
|
|
360
|
+
"longer appear in the thread picker.", relabelled)
|
|
361
|
+
return relabelled
|