@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -12,9 +12,9 @@ for inspection and debugging.
|
|
|
12
12
|
Schema deliberately mirrors the codebase convention established by
|
|
13
13
|
storage.py's MessageLog: semantically distinct data lives in separate
|
|
14
14
|
typed columns (claims_json / trace_json / coverage_gaps_json /
|
|
15
|
-
final_report), NOT one catch-all
|
|
16
|
-
json.dumps'd strings (same pattern as
|
|
17
|
-
is plain text. This keeps a serialization problem in one field from
|
|
15
|
+
pass_threads_json / candidates_json / final_report), NOT one catch-all
|
|
16
|
+
blob. The *_json columns hold json.dumps'd strings (same pattern as
|
|
17
|
+
MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
|
|
18
18
|
corrupting the others -- a persistence layer built for crash-resilience
|
|
19
19
|
shouldn't itself be a single point of failure.
|
|
20
20
|
|
|
@@ -27,13 +27,14 @@ from __future__ import annotations
|
|
|
27
27
|
|
|
28
28
|
import json
|
|
29
29
|
import logging
|
|
30
|
-
from datetime import
|
|
30
|
+
from datetime import UTC, datetime
|
|
31
31
|
from typing import Optional
|
|
32
32
|
from uuid import UUID, uuid4
|
|
33
33
|
|
|
34
34
|
from sqlmodel import Field, Session, SQLModel
|
|
35
35
|
|
|
36
36
|
from database import engine
|
|
37
|
+
|
|
37
38
|
# §27: the thread-kind vocabulary lives beside the table that stores it.
|
|
38
39
|
# This import direction is the only one that exists -- storage.py knows
|
|
39
40
|
# nothing about pipeline runs, and classify_legacy_pass_threads() below is
|
|
@@ -46,10 +47,16 @@ logger = logging.getLogger(__name__)
|
|
|
46
47
|
class PipelineRunRecord(SQLModel, table=True):
|
|
47
48
|
"""One row per stream_deep_research_pipeline() invocation. `status` is
|
|
48
49
|
'running' from creation until the wrapping try/except in the
|
|
49
|
-
orchestrator flips it to 'complete' or 'failed'. The
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
orchestrator flips it to 'complete' or 'failed'. The SIX data columns
|
|
51
|
+
carry the run's structured output (or the partial state available at
|
|
52
|
+
failure time) so an operator can inspect what happened without a
|
|
53
|
+
separate trace log.
|
|
54
|
+
|
|
55
|
+
SIX, not the four this paragraph said until batch 154: T2 added
|
|
56
|
+
pass_threads_json and E13 added candidates_json, and neither batch
|
|
57
|
+
found the enumerations written about them. The count is pinned by
|
|
58
|
+
test_the_documented_record_shape_matches_what_load_returns, which
|
|
59
|
+
reads it out of the prose rather than out of a second list here."""
|
|
53
60
|
|
|
54
61
|
id: UUID = Field(default_factory=uuid4, primary_key=True)
|
|
55
62
|
user_query: str = Field(index=True)
|
|
@@ -75,7 +82,7 @@ class PipelineRunRecord(SQLModel, table=True):
|
|
|
75
82
|
# what it had reached.
|
|
76
83
|
candidates_json: str = "[]"
|
|
77
84
|
final_report: str = ""
|
|
78
|
-
started_at: datetime = Field(default_factory=lambda: datetime.now(
|
|
85
|
+
started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
|
79
86
|
finished_at: Optional[datetime] = None
|
|
80
87
|
|
|
81
88
|
|
|
@@ -101,9 +108,9 @@ def update_pipeline_run(
|
|
|
101
108
|
status: Optional[str] = None,
|
|
102
109
|
) -> None:
|
|
103
110
|
"""Updates an existing row's data columns (claims / trace /
|
|
104
|
-
coverage_gaps / final_report) from the
|
|
105
|
-
when `status` is given -- flips status
|
|
106
|
-
terminal transition. Raises ValueError if run_id doesn't exist (the
|
|
111
|
+
coverage_gaps / pass_threads / candidates / final_report) from the
|
|
112
|
+
live PipelineRun, and -- only when `status` is given -- flips status
|
|
113
|
+
and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
|
|
107
114
|
record was never created or was deleted), which is a real bug, not an
|
|
108
115
|
expected path.
|
|
109
116
|
|
|
@@ -128,7 +135,7 @@ def update_pipeline_run(
|
|
|
128
135
|
if status is not None:
|
|
129
136
|
record.status = status
|
|
130
137
|
if status in ("complete", "failed"):
|
|
131
|
-
record.finished_at = datetime.now(
|
|
138
|
+
record.finished_at = datetime.now(UTC)
|
|
132
139
|
record.claims_json = json.dumps([vars(c) for c in run.claims])
|
|
133
140
|
record.trace_json = json.dumps(run.trace)
|
|
134
141
|
record.coverage_gaps_json = json.dumps(run.coverage_gaps)
|
|
@@ -178,14 +185,18 @@ def load_pipeline_run(run_id: UUID) -> dict:
|
|
|
178
185
|
"claims": json.loads(record.claims_json),
|
|
179
186
|
"trace": json.loads(record.trace_json),
|
|
180
187
|
"coverage_gaps": json.loads(record.coverage_gaps_json),
|
|
181
|
-
# §27 AC6.
|
|
182
|
-
#
|
|
183
|
-
# ensure_columns
|
|
184
|
-
#
|
|
188
|
+
# §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
|
|
189
|
+
# until #258 this said a migrated row was one. It is not:
|
|
190
|
+
# ensure_columns adds the column with DEFAULT '[]', and SQLite
|
|
191
|
+
# reads every row older than the ALTER as that default. A NULL
|
|
192
|
+
# takes a database first migrated before #26 carried NOT NULL
|
|
193
|
+
# (2026-08-06 to 08-22), where the column is nullable, and SQL
|
|
194
|
+
# nobody here wrote -- on which json.loads(None) would raise.
|
|
185
195
|
"pass_threads": json.loads(record.pass_threads_json or "[]"),
|
|
186
|
-
#
|
|
187
|
-
#
|
|
188
|
-
#
|
|
196
|
+
# E13 arrived after #26, so this column is NOT NULL on every
|
|
197
|
+
# database this project has built, and SQLite refuses the NULL
|
|
198
|
+
# this would read past. Kept as the reader's own floor beside
|
|
199
|
+
# its sibling's; tests/test_storage_e2e.py pins both facts.
|
|
189
200
|
"candidates": json.loads(record.candidates_json or "[]"),
|
|
190
201
|
"final_report": record.final_report,
|
|
191
202
|
}
|
|
@@ -195,6 +206,47 @@ def load_pipeline_run(run_id: UUID) -> dict:
|
|
|
195
206
|
# ---- One-time classification of pre-§27 threads ---------------------------
|
|
196
207
|
# ---------------------------------------------------------------------------
|
|
197
208
|
|
|
209
|
+
def _as_uuid_key(value) -> Optional[str]:
|
|
210
|
+
"""One id's single spelling, or None for anything that is not a UUID.
|
|
211
|
+
|
|
212
|
+
THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
|
|
213
|
+
they were compared literally. `pass_threads_json` carries
|
|
214
|
+
`str(thread_id)` -- the dashed 36-character form, which is also what
|
|
215
|
+
output_writer.py ships as an artifact and what an operator reads --
|
|
216
|
+
while `conversationthread.id` comes back from the raw DBAPI as the
|
|
217
|
+
32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
|
|
218
|
+
every id there has ever been, so the tier #82 calls authoritative
|
|
219
|
+
matched nothing, ever. Measured end to end on real SQLite (the
|
|
220
|
+
suite's fake sqlmodel cannot show a storage format): recorded
|
|
221
|
+
'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
|
|
222
|
+
|
|
223
|
+
NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
|
|
224
|
+
thread id IS a UUID (`ConversationThread.id`), so a record holding
|
|
225
|
+
something else is not an id this function can speak about, and
|
|
226
|
+
deciding that two unparseable strings denote one thread is the
|
|
227
|
+
over-classifying direction -- the one that hides a real conversation.
|
|
228
|
+
The caller treats None as "no tier-1 answer", which is how an entry
|
|
229
|
+
with no usable thread_id has always been treated.
|
|
230
|
+
|
|
231
|
+
ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
|
|
232
|
+
RECORDED ids only; the row ids are compared as the database hands
|
|
233
|
+
them over. Normalising both was written first, and the control run
|
|
234
|
+
as mutation C1 found it unearned -- removing the row-side call left
|
|
235
|
+
every test green, because `conversationthread.id` is hex and so is
|
|
236
|
+
this. The remaining assumption is that the column holds hex, which
|
|
237
|
+
is a fact about sqlmodel's UUID type rather than about this project,
|
|
238
|
+
so it is ASSERTED where it can be:
|
|
239
|
+
`test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
|
|
240
|
+
tests/test_storage_e2e.py fails naming this function if the library
|
|
241
|
+
ever starts storing the dashed form. A pinned dependency beats a
|
|
242
|
+
branch for a spelling nothing here has ever written.
|
|
243
|
+
"""
|
|
244
|
+
try:
|
|
245
|
+
return UUID(value).hex
|
|
246
|
+
except (ValueError, AttributeError, TypeError):
|
|
247
|
+
return None
|
|
248
|
+
|
|
249
|
+
|
|
198
250
|
def classify_legacy_pass_threads(connection) -> int:
|
|
199
251
|
"""Label pass threads that predate ConversationThread.kind. Returns the
|
|
200
252
|
number of rows relabelled.
|
|
@@ -219,7 +271,9 @@ def classify_legacy_pass_threads(connection) -> int:
|
|
|
219
271
|
* TIER 1 (exact): every thread id named in a populated
|
|
220
272
|
`pass_threads_json` is a pass thread, whatever its timestamp says
|
|
221
273
|
and whether or not the run ever finished. Membership was recorded
|
|
222
|
-
at run time; there is nothing to infer.
|
|
274
|
+
at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
|
|
275
|
+
STRINGS -- the two sides are spelt differently and `_as_uuid_key`
|
|
276
|
+
below is where they are made one thing (batch 154).
|
|
223
277
|
* TIER 2 (window): only runs whose `pass_threads_json` is absent,
|
|
224
278
|
NULL, empty, or unusable fall back to the time-window heuristic --
|
|
225
279
|
"created between started_at and finished_at". That set is the
|
|
@@ -308,8 +362,17 @@ def classify_legacy_pass_threads(connection) -> int:
|
|
|
308
362
|
for entry in entries:
|
|
309
363
|
if (isinstance(entry, dict)
|
|
310
364
|
and isinstance(entry.get("thread_id"), str)):
|
|
311
|
-
|
|
365
|
+
# POPULATED is what puts this run in tier 1,
|
|
366
|
+
# whatever its ids turn out to parse as. An id
|
|
367
|
+
# this function cannot read contributes no
|
|
368
|
+
# membership, but it must not drop the run to
|
|
369
|
+
# the window either: that would consult a
|
|
370
|
+
# modern run's clock and hide the concurrent
|
|
371
|
+
# chat thread #82's headline is about.
|
|
312
372
|
usable = True
|
|
373
|
+
key = _as_uuid_key(entry["thread_id"])
|
|
374
|
+
if key is not None:
|
|
375
|
+
exact_ids.add(key)
|
|
313
376
|
# raw None/"" (or a missing column) falls through with usable=False:
|
|
314
377
|
# the legitimate pre-T2 shape, i.e. window tier -- not corruption.
|
|
315
378
|
if corrupted:
|
|
@@ -336,6 +399,9 @@ def classify_legacy_pass_threads(connection) -> int:
|
|
|
336
399
|
|
|
337
400
|
to_relabel = []
|
|
338
401
|
for thread_id, created_at in chat_rows:
|
|
402
|
+
# `exact_ids` holds hex, which is the spelling this column is in.
|
|
403
|
+
# ONE normalisation, on the recorded side, and the assumption on
|
|
404
|
+
# this side is PINNED rather than branched on -- see _as_uuid_key.
|
|
339
405
|
if thread_id in exact_ids:
|
|
340
406
|
to_relabel.append(thread_id)
|
|
341
407
|
elif created_at is not None and any(
|