@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -92,10 +92,29 @@ def doi_for(url: str) -> str | None:
92
92
  # ---- Pure scoring ---------------------------------------------------------
93
93
  # ---------------------------------------------------------------------------
94
94
 
95
+ def _obj(value) -> dict:
96
+ """`value` if OpenAlex sent an object there, else an empty one.
97
+
98
+ Batch 219 (#218). Every read of a record used to trust its shape --
99
+ `(x or {}).get(...)` survives a null and nothing else -- so one
100
+ non-object where an object was expected raised AttributeError out of
101
+ an optional stage and failed a run that had already grounded and
102
+ scored. Driven: 104 malformed shapes across ten keys. A shape this
103
+ module cannot read is a signal that is absent, and every term here
104
+ already knows how to drop an absent one.
105
+ """
106
+ return value if isinstance(value, dict) else {}
107
+
108
+
109
+ def _text(value) -> str:
110
+ """`value` if it is a string, else "" -- `_obj`'s rule for a leaf."""
111
+ return value if isinstance(value, str) else ""
112
+
113
+
95
114
  def _location_type(work: dict) -> str:
96
- location = work.get("primary_location") or {}
97
- source = location.get("source") or {}
98
- return (source.get("type") or "").lower()
115
+ location = _obj(work.get("primary_location"))
116
+ source = _obj(location.get("source"))
117
+ return _text(source.get("type")).lower()
99
118
 
100
119
 
101
120
  def is_peer_reviewed(work: dict) -> bool:
@@ -108,8 +127,8 @@ def is_peer_reviewed(work: dict) -> bool:
108
127
  publisher's own answer where there is one. A record has to survive all
109
128
  three to count.
110
129
  """
111
- location = work.get("primary_location") or {}
112
- if (work.get("type") or "").lower() == "preprint":
130
+ location = _obj(work.get("primary_location"))
131
+ if _text(work.get("type")).lower() == "preprint":
113
132
  return False
114
133
  if _location_type(work) not in ("journal", "conference", "book series"):
115
134
  return False
@@ -277,7 +296,7 @@ class ScholarLookup:
277
296
  "select": "id"})
278
297
  if not isinstance(payload, dict):
279
298
  return None
280
- count = (payload.get("meta") or {}).get("count")
299
+ count = _obj(payload.get("meta")).get("count")
281
300
  return count if isinstance(count, int) else None
282
301
 
283
302
  def citation_percentile(self, work: dict, today: date | None = None):
@@ -298,9 +317,10 @@ class ScholarLookup:
298
317
  if not isinstance(cites, int) or isinstance(cites, bool):
299
318
  return None, "insufficient_cohort"
300
319
 
301
- field = ((work.get("primary_topic") or {}).get("field") or {}).get("id")
302
- domain = ((work.get("primary_topic") or {}).get("domain") or {}).get("id")
303
- work_type = (work.get("type") or "").lower() or "article"
320
+ topic = _obj(work.get("primary_topic"))
321
+ field = _text(_obj(topic.get("field")).get("id")) or None
322
+ domain = _text(_obj(topic.get("domain")).get("id")) or None
323
+ work_type = _text(work.get("type")).lower() or "article"
304
324
  published = date.fromisoformat(work["publication_date"][:10])
305
325
 
306
326
  minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
@@ -327,9 +347,12 @@ class ScholarLookup:
327
347
  pass someone is waiting on.
328
348
  """
329
349
  ids = []
330
- for authorship in (work.get("authorships") or [])[:_MAX_AUTHORS]:
331
- author = (authorship or {}).get("author") or {}
332
- identifier = (author.get("id") or "").rsplit("/", 1)[-1]
350
+ authorships = work.get("authorships")
351
+ if not isinstance(authorships, list):
352
+ authorships = []
353
+ for authorship in authorships[:_MAX_AUTHORS]:
354
+ author = _obj(_obj(authorship).get("author"))
355
+ identifier = _text(author.get("id")).rsplit("/", 1)[-1]
333
356
  if identifier.startswith("A"):
334
357
  ids.append(identifier)
335
358
  if not ids:
@@ -339,8 +362,9 @@ class ScholarLookup:
339
362
  "select": "id,summary_stats", "per-page": len(ids)})
340
363
  if not isinstance(payload, dict):
341
364
  return None
342
- indexes = [((entry or {}).get("summary_stats") or {}).get("h_index")
343
- for entry in payload.get("results") or []]
365
+ results = payload.get("results")
366
+ indexes = [_obj(_obj(entry).get("summary_stats")).get("h_index")
367
+ for entry in (results if isinstance(results, list) else [])]
344
368
  return author_standing(
345
369
  indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
346
370
 
@@ -28,6 +28,17 @@ boundary -- nothing is stored by this module unredacted -- beats depending
28
28
  on a guarantee made two layers away that a later refactor could quietly
29
29
  drop. The second pass is a no-op on already-clean text.
30
30
 
31
+ AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
32
+ §51 made this module the one place where two pages of a document become
33
+ adjacent, so a credential straddling a page boundary was rejoined HERE --
34
+ reproduced end to end in batch 106, into `artifact_entries()` and therefore
35
+ into `sources/<sha256>.txt` and the window sent to the embedding provider.
36
+ The register's proposed fix was to re-redact the whole merged text on every
37
+ append. It was declined: `fetch_url` redacts before it cuts now, so the
38
+ halves never exist, and re-scanning up to 20,000 characters per page would
39
+ buy nothing this module can see -- while leaving unfixed the model's own two
40
+ tool results, which the producer-side fix does close.
41
+
31
42
  THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
32
43
  paper are one source, and a model that cites `/abs/2005.14165v3` after
33
44
  fetching `/abs/2005.14165` must not be told its own citation has no text.
@@ -43,7 +54,7 @@ import hashlib
43
54
  import logging
44
55
  import re
45
56
  from dataclasses import dataclass, field
46
- from datetime import datetime, timezone
57
+ from datetime import UTC, datetime
47
58
  from urllib.parse import urlsplit, urlunsplit
48
59
 
49
60
  from safety.policy_enforcement import redact_output_text
@@ -99,6 +110,26 @@ class SourceDocument:
99
110
  tool: str = ""
100
111
  retrieved_at: str = ""
101
112
 
113
+ #: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
114
+ #: reaches, in the source's own character positions -- which is what
115
+ #: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
116
+ #: document no paging tool produced.
117
+ #:
118
+ #: §52 (RB1) removed ONE of the two reasons those numbers differ.
119
+ #: Redaction used to happen here, on each page, so a page's held length
120
+ #: could be shorter than the span it covered; `fetch_url` now redacts
121
+ #: before it cuts, so its pages arrive clean and that gap is gone. What
122
+ #: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
123
+ #: of which change how much is HELD and neither of which moves where a
124
+ #: page sat in the document it came from.
125
+ #:
126
+ #: It is what stops a page being absorbed twice. Without it, a model
127
+ #: that re-reads page 2 -- which it will, because re-reading is how a
128
+ #: model checks itself -- would append it again and the corpus would
129
+ #: hold the same paragraph twice, which a similarity score then counts
130
+ #: twice.
131
+ next_offset: int = 0
132
+
102
133
  @property
103
134
  def sha256(self) -> str:
104
135
  return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
@@ -116,7 +147,7 @@ class SourceDocument:
116
147
 
117
148
 
118
149
  def _now() -> str:
119
- return datetime.now(timezone.utc).isoformat(timespec="seconds")
150
+ return datetime.now(UTC).isoformat(timespec="seconds")
120
151
 
121
152
 
122
153
  def arxiv_id_from_url(url: str) -> str | None:
@@ -224,11 +255,19 @@ class SourceCorpus:
224
255
  # `url` is the URL that ANSWERED, after redirects -- which is what
225
256
  # fetch_url.py:89-95 says the grounding pass attributes the text
226
257
  # to, and therefore what the model will cite.
258
+ #
259
+ # `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
260
+ # pre-§51 shape or a hand-built fixture, and 0 is the right reading
261
+ # of both: a whole document starting at the beginning.
262
+ offset = result.get("offset")
263
+ if not isinstance(offset, int) or offset < 0:
264
+ offset = 0
227
265
  return self._store(
228
266
  url=result.get("url"),
229
267
  text=result.get("content"),
230
268
  title="",
231
269
  tool="fetch_url",
270
+ offset=offset,
232
271
  )
233
272
 
234
273
  def _add_arxiv(self, result: dict) -> int:
@@ -270,19 +309,81 @@ class SourceCorpus:
270
309
  )
271
310
  return added
272
311
 
273
- def _store(self, url, text, title: str, tool: str) -> int:
312
+ def _store(self, url, text, title: str, tool: str, offset=None) -> int:
313
+ """Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
314
+
315
+ `offset` is the source position this text starts at, or None from a
316
+ tool that does not page. None and 0 are NOT the same thing here:
317
+ None means "this is the whole of what that tool returns", 0 means
318
+ "this is the first page of something that may have more".
319
+
320
+ WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
321
+ before any of this was written: a corpus given page 1 and then page
322
+ 2 of one URL returned 1 then **0** -- `len(existing.text) >=
323
+ len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
324
+ text stayed page 1. Paging would then have made the model better
325
+ informed and the grounding pass no better, and the divergence is
326
+ invisible from both sides: the model reads four pages and the score
327
+ is computed against the first. The near miss is worse than the
328
+ miss -- a page 2 one character longer REPLACED page 1 outright,
329
+ leaving the corpus holding the middle of a document and attributing
330
+ it to the whole URL.
331
+ """
274
332
  if not isinstance(url, str) or not isinstance(text, str):
275
333
  return 0
276
334
  key = normalize_url(url)
277
335
  if not key:
278
336
  return 0
279
337
 
338
+ existing = self.documents.get(key)
339
+
340
+ # A continuation extends what is held; anything else competes on
341
+ # length exactly as it did before §51.
342
+ continuing = (
343
+ offset is not None
344
+ and offset > 0
345
+ and existing is not None
346
+ and existing.tool == tool
347
+ )
348
+
349
+ if continuing:
350
+ # How much of this page is already held. Covers all three
351
+ # shapes with one subtraction: an exact continuation overlaps
352
+ # by nothing, a re-read of the same page overlaps entirely and
353
+ # contributes the empty string, and a page that starts inside
354
+ # what is held contributes only its tail. A page starting
355
+ # BEYOND the held text leaves a gap -- appended anyway, because
356
+ # the text is real and the alternative is losing it, and an
357
+ # incomplete document is what this corpus already holds for
358
+ # every source the byte cap cut.
359
+ #
360
+ # There is deliberately NO `if overlap >= len(text): return 0`
361
+ # above this. One was written, and a mutation pass showed it
362
+ # could be deleted with no test failing anywhere -- because the
363
+ # slice already yields "" in exactly the cases it guarded, and
364
+ # the empty-addition check below catches them. Batch 103's
365
+ # lesson applies to a guard that cannot fire as much as to one
366
+ # that fires in the wrong place: it reads as the thing
367
+ # protecting you.
368
+ overlap = existing.next_offset - offset
369
+ addition = redact_output_text(text[max(0, overlap):])
370
+ if not addition.strip():
371
+ return 0
372
+ merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
373
+ if merged == existing.text:
374
+ return 0
375
+ self.documents[key] = SourceDocument(
376
+ url=key, text=merged, title=existing.title,
377
+ tool=tool, retrieved_at=_now(),
378
+ next_offset=max(existing.next_offset, offset + len(text)),
379
+ )
380
+ return 1
381
+
280
382
  cleaned = redact_output_text(text).strip()
281
383
  if len(cleaned) < MIN_DOCUMENT_CHARS:
282
384
  return 0
283
385
  cleaned = cleaned[:MAX_DOCUMENT_CHARS]
284
386
 
285
- existing = self.documents.get(key)
286
387
  if existing is not None and len(existing.text) >= len(cleaned):
287
388
  return 0
288
389
  if existing is None and len(self.documents) >= MAX_DOCUMENTS:
@@ -295,6 +396,11 @@ class SourceCorpus:
295
396
  self.documents[key] = SourceDocument(
296
397
  url=key, text=cleaned, title=title or (existing.title if existing else ""),
297
398
  tool=tool, retrieved_at=_now(),
399
+ # In the SOURCE's positions, so the next page's offset can be
400
+ # compared against it. `len(text)` and not `len(cleaned)`:
401
+ # redaction and stripping change how much is HELD, never where
402
+ # the page sat in the document it came from.
403
+ next_offset=(offset + len(text)) if offset is not None else 0,
298
404
  )
299
405
  return 1
300
406