@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -12,9 +12,9 @@ for inspection and debugging.
12
12
  Schema deliberately mirrors the codebase convention established by
13
13
  storage.py's MessageLog: semantically distinct data lives in separate
14
14
  typed columns (claims_json / trace_json / coverage_gaps_json /
15
- final_report), NOT one catch-all blob. The *_json columns hold
16
- json.dumps'd strings (same pattern as MessageLog.content); final_report
17
- is plain text. This keeps a serialization problem in one field from
15
+ pass_threads_json / candidates_json / final_report), NOT one catch-all
16
+ blob. The *_json columns hold json.dumps'd strings (same pattern as
17
+ MessageLog.content); final_report is plain text. This keeps a serialization problem in one field from
18
18
  corrupting the others -- a persistence layer built for crash-resilience
19
19
  shouldn't itself be a single point of failure.
20
20
 
@@ -27,13 +27,14 @@ from __future__ import annotations
27
27
 
28
28
  import json
29
29
  import logging
30
- from datetime import datetime, timezone
30
+ from datetime import UTC, datetime
31
31
  from typing import Optional
32
32
  from uuid import UUID, uuid4
33
33
 
34
34
  from sqlmodel import Field, Session, SQLModel
35
35
 
36
36
  from database import engine
37
+
37
38
  # §27: the thread-kind vocabulary lives beside the table that stores it.
38
39
  # This import direction is the only one that exists -- storage.py knows
39
40
  # nothing about pipeline runs, and classify_legacy_pass_threads() below is
@@ -46,10 +47,16 @@ logger = logging.getLogger(__name__)
46
47
  class PipelineRunRecord(SQLModel, table=True):
47
48
  """One row per stream_deep_research_pipeline() invocation. `status` is
48
49
  'running' from creation until the wrapping try/except in the
49
- orchestrator flips it to 'complete' or 'failed'. The four data
50
- columns carry the run's structured output (or the partial state
51
- available at failure time) so an operator can inspect what happened
52
- without a separate trace log."""
50
+ orchestrator flips it to 'complete' or 'failed'. The SIX data columns
51
+ carry the run's structured output (or the partial state available at
52
+ failure time) so an operator can inspect what happened without a
53
+ separate trace log.
54
+
55
+ SIX, not the four this paragraph said until batch 154: T2 added
56
+ pass_threads_json and E13 added candidates_json, and neither batch
57
+ found the enumerations written about them. The count is pinned by
58
+ test_the_documented_record_shape_matches_what_load_returns, which
59
+ reads it out of the prose rather than out of a second list here."""
53
60
 
54
61
  id: UUID = Field(default_factory=uuid4, primary_key=True)
55
62
  user_query: str = Field(index=True)
@@ -75,7 +82,7 @@ class PipelineRunRecord(SQLModel, table=True):
75
82
  # what it had reached.
76
83
  candidates_json: str = "[]"
77
84
  final_report: str = ""
78
- started_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
85
+ started_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
79
86
  finished_at: Optional[datetime] = None
80
87
 
81
88
 
@@ -101,9 +108,9 @@ def update_pipeline_run(
101
108
  status: Optional[str] = None,
102
109
  ) -> None:
103
110
  """Updates an existing row's data columns (claims / trace /
104
- coverage_gaps / final_report) from the live PipelineRun, and -- only
105
- when `status` is given -- flips status and sets finished_at on a
106
- terminal transition. Raises ValueError if run_id doesn't exist (the
111
+ coverage_gaps / pass_threads / candidates / final_report) from the
112
+ live PipelineRun, and -- only when `status` is given -- flips status
113
+ and sets finished_at on a terminal transition. Raises ValueError if run_id doesn't exist (the
107
114
  record was never created or was deleted), which is a real bug, not an
108
115
  expected path.
109
116
 
@@ -128,7 +135,7 @@ def update_pipeline_run(
128
135
  if status is not None:
129
136
  record.status = status
130
137
  if status in ("complete", "failed"):
131
- record.finished_at = datetime.now(timezone.utc)
138
+ record.finished_at = datetime.now(UTC)
132
139
  record.claims_json = json.dumps([vars(c) for c in run.claims])
133
140
  record.trace_json = json.dumps(run.trace)
134
141
  record.coverage_gaps_json = json.dumps(run.coverage_gaps)
@@ -178,14 +185,18 @@ def load_pipeline_run(run_id: UUID) -> dict:
178
185
  "claims": json.loads(record.claims_json),
179
186
  "trace": json.loads(record.trace_json),
180
187
  "coverage_gaps": json.loads(record.coverage_gaps_json),
181
- # §27 AC6. Defaulted rather than read straight off the record:
182
- # a row written before this column existed has it as NULL after
183
- # ensure_columns' ALTER (the migration is additive and never
184
- # backfills), and json.loads(None) raises.
188
+ # §27 AC6. The `or "[]"` is for a NULL written BY HAND, and
189
+ # until #258 this said a migrated row was one. It is not:
190
+ # ensure_columns adds the column with DEFAULT '[]', and SQLite
191
+ # reads every row older than the ALTER as that default. A NULL
192
+ # takes a database first migrated before #26 carried NOT NULL
193
+ # (2026-08-06 to 08-22), where the column is nullable, and SQL
194
+ # nobody here wrote -- on which json.loads(None) would raise.
185
195
  "pass_threads": json.loads(record.pass_threads_json or "[]"),
186
- # Same NULL guard as pass_threads above: ensure_columns'
187
- # ALTER never backfills, so a row written before this column
188
- # existed reads as NULL, and json.loads(None) raises.
196
+ # E13 arrived after #26, so this column is NOT NULL on every
197
+ # database this project has built, and SQLite refuses the NULL
198
+ # this would read past. Kept as the reader's own floor beside
199
+ # its sibling's; tests/test_storage_e2e.py pins both facts.
189
200
  "candidates": json.loads(record.candidates_json or "[]"),
190
201
  "final_report": record.final_report,
191
202
  }
@@ -195,6 +206,47 @@ def load_pipeline_run(run_id: UUID) -> dict:
195
206
  # ---- One-time classification of pre-§27 threads ---------------------------
196
207
  # ---------------------------------------------------------------------------
197
208
 
209
+ def _as_uuid_key(value) -> Optional[str]:
210
+ """One id's single spelling, or None for anything that is not a UUID.
211
+
212
+ THE TWO SIDES OF TIER 1 ARE SPELT DIFFERENTLY, and until batch 154
213
+ they were compared literally. `pass_threads_json` carries
214
+ `str(thread_id)` -- the dashed 36-character form, which is also what
215
+ output_writer.py ships as an artifact and what an operator reads --
216
+ while `conversationthread.id` comes back from the raw DBAPI as the
217
+ 32-character hex sqlmodel stores. `'a-b' in {'ab'}` is False for
218
+ every id there has ever been, so the tier #82 calls authoritative
219
+ matched nothing, ever. Measured end to end on real SQLite (the
220
+ suite's fake sqlmodel cannot show a storage format): recorded
221
+ 'acb451c5-9c50-...', stored 'acb451c59c50...', relabelled 0.
222
+
223
+ NONE RATHER THAN THE STRING ITSELF for a value that does not parse. A
224
+ thread id IS a UUID (`ConversationThread.id`), so a record holding
225
+ something else is not an id this function can speak about, and
226
+ deciding that two unparseable strings denote one thread is the
227
+ over-classifying direction -- the one that hides a real conversation.
228
+ The caller treats None as "no tier-1 answer", which is how an entry
229
+ with no usable thread_id has always been treated.
230
+
231
+ ONE SIDE, NOT TWO, AND THE OTHER SIDE IS PINNED. This normalises the
232
+ RECORDED ids only; the row ids are compared as the database hands
233
+ them over. Normalising both was written first, and the control run
234
+ as mutation C1 found it unearned -- removing the row-side call left
235
+ every test green, because `conversationthread.id` is hex and so is
236
+ this. The remaining assumption is that the column holds hex, which
237
+ is a fact about sqlmodel's UUID type rather than about this project,
238
+ so it is ASSERTED where it can be:
239
+ `test_a_thread_id_is_stored_in_a_spelling_str_does_not_produce` in
240
+ tests/test_storage_e2e.py fails naming this function if the library
241
+ ever starts storing the dashed form. A pinned dependency beats a
242
+ branch for a spelling nothing here has ever written.
243
+ """
244
+ try:
245
+ return UUID(value).hex
246
+ except (ValueError, AttributeError, TypeError):
247
+ return None
248
+
249
+
198
250
  def classify_legacy_pass_threads(connection) -> int:
199
251
  """Label pass threads that predate ConversationThread.kind. Returns the
200
252
  number of rows relabelled.
@@ -219,7 +271,9 @@ def classify_legacy_pass_threads(connection) -> int:
219
271
  * TIER 1 (exact): every thread id named in a populated
220
272
  `pass_threads_json` is a pass thread, whatever its timestamp says
221
273
  and whether or not the run ever finished. Membership was recorded
222
- at run time; there is nothing to infer.
274
+ at run time; there is nothing to infer. COMPARED AS UUIDS, NOT AS
275
+ STRINGS -- the two sides are spelt differently and `_as_uuid_key`
276
+ below is where they are made one thing (batch 154).
223
277
  * TIER 2 (window): only runs whose `pass_threads_json` is absent,
224
278
  NULL, empty, or unusable fall back to the time-window heuristic --
225
279
  "created between started_at and finished_at". That set is the
@@ -308,8 +362,17 @@ def classify_legacy_pass_threads(connection) -> int:
308
362
  for entry in entries:
309
363
  if (isinstance(entry, dict)
310
364
  and isinstance(entry.get("thread_id"), str)):
311
- exact_ids.add(entry["thread_id"])
365
+ # POPULATED is what puts this run in tier 1,
366
+ # whatever its ids turn out to parse as. An id
367
+ # this function cannot read contributes no
368
+ # membership, but it must not drop the run to
369
+ # the window either: that would consult a
370
+ # modern run's clock and hide the concurrent
371
+ # chat thread #82's headline is about.
312
372
  usable = True
373
+ key = _as_uuid_key(entry["thread_id"])
374
+ if key is not None:
375
+ exact_ids.add(key)
313
376
  # raw None/"" (or a missing column) falls through with usable=False:
314
377
  # the legitimate pre-T2 shape, i.e. window tier -- not corruption.
315
378
  if corrupted:
@@ -336,6 +399,9 @@ def classify_legacy_pass_threads(connection) -> int:
336
399
 
337
400
  to_relabel = []
338
401
  for thread_id, created_at in chat_rows:
402
+ # `exact_ids` holds hex, which is the spelling this column is in.
403
+ # ONE normalisation, on the recorded side, and the assumption on
404
+ # this side is PINNED rather than branched on -- see _as_uuid_key.
339
405
  if thread_id in exact_ids:
340
406
  to_relabel.append(thread_id)
341
407
  elif created_at is not None and any(