@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -92,10 +92,29 @@ def doi_for(url: str) -> str | None:
|
|
|
92
92
|
# ---- Pure scoring ---------------------------------------------------------
|
|
93
93
|
# ---------------------------------------------------------------------------
|
|
94
94
|
|
|
95
|
+
def _obj(value) -> dict:
|
|
96
|
+
"""`value` if OpenAlex sent an object there, else an empty one.
|
|
97
|
+
|
|
98
|
+
Batch 219 (#218). Every read of a record used to trust its shape --
|
|
99
|
+
`(x or {}).get(...)` survives a null and nothing else -- so one
|
|
100
|
+
non-object where an object was expected raised AttributeError out of
|
|
101
|
+
an optional stage and failed a run that had already grounded and
|
|
102
|
+
scored. Driven: 104 malformed shapes across ten keys. A shape this
|
|
103
|
+
module cannot read is a signal that is absent, and every term here
|
|
104
|
+
already knows how to drop an absent one.
|
|
105
|
+
"""
|
|
106
|
+
return value if isinstance(value, dict) else {}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _text(value) -> str:
|
|
110
|
+
"""`value` if it is a string, else "" -- `_obj`'s rule for a leaf."""
|
|
111
|
+
return value if isinstance(value, str) else ""
|
|
112
|
+
|
|
113
|
+
|
|
95
114
|
def _location_type(work: dict) -> str:
|
|
96
|
-
location = work.get("primary_location")
|
|
97
|
-
source = location.get("source")
|
|
98
|
-
return (source.get("type")
|
|
115
|
+
location = _obj(work.get("primary_location"))
|
|
116
|
+
source = _obj(location.get("source"))
|
|
117
|
+
return _text(source.get("type")).lower()
|
|
99
118
|
|
|
100
119
|
|
|
101
120
|
def is_peer_reviewed(work: dict) -> bool:
|
|
@@ -108,8 +127,8 @@ def is_peer_reviewed(work: dict) -> bool:
|
|
|
108
127
|
publisher's own answer where there is one. A record has to survive all
|
|
109
128
|
three to count.
|
|
110
129
|
"""
|
|
111
|
-
location = work.get("primary_location")
|
|
112
|
-
if (work.get("type")
|
|
130
|
+
location = _obj(work.get("primary_location"))
|
|
131
|
+
if _text(work.get("type")).lower() == "preprint":
|
|
113
132
|
return False
|
|
114
133
|
if _location_type(work) not in ("journal", "conference", "book series"):
|
|
115
134
|
return False
|
|
@@ -277,7 +296,7 @@ class ScholarLookup:
|
|
|
277
296
|
"select": "id"})
|
|
278
297
|
if not isinstance(payload, dict):
|
|
279
298
|
return None
|
|
280
|
-
count = (payload.get("meta")
|
|
299
|
+
count = _obj(payload.get("meta")).get("count")
|
|
281
300
|
return count if isinstance(count, int) else None
|
|
282
301
|
|
|
283
302
|
def citation_percentile(self, work: dict, today: date | None = None):
|
|
@@ -298,9 +317,10 @@ class ScholarLookup:
|
|
|
298
317
|
if not isinstance(cites, int) or isinstance(cites, bool):
|
|
299
318
|
return None, "insufficient_cohort"
|
|
300
319
|
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
320
|
+
topic = _obj(work.get("primary_topic"))
|
|
321
|
+
field = _text(_obj(topic.get("field")).get("id")) or None
|
|
322
|
+
domain = _text(_obj(topic.get("domain")).get("id")) or None
|
|
323
|
+
work_type = _text(work.get("type")).lower() or "article"
|
|
304
324
|
published = date.fromisoformat(work["publication_date"][:10])
|
|
305
325
|
|
|
306
326
|
minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
|
|
@@ -327,9 +347,12 @@ class ScholarLookup:
|
|
|
327
347
|
pass someone is waiting on.
|
|
328
348
|
"""
|
|
329
349
|
ids = []
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
350
|
+
authorships = work.get("authorships")
|
|
351
|
+
if not isinstance(authorships, list):
|
|
352
|
+
authorships = []
|
|
353
|
+
for authorship in authorships[:_MAX_AUTHORS]:
|
|
354
|
+
author = _obj(_obj(authorship).get("author"))
|
|
355
|
+
identifier = _text(author.get("id")).rsplit("/", 1)[-1]
|
|
333
356
|
if identifier.startswith("A"):
|
|
334
357
|
ids.append(identifier)
|
|
335
358
|
if not ids:
|
|
@@ -339,8 +362,9 @@ class ScholarLookup:
|
|
|
339
362
|
"select": "id,summary_stats", "per-page": len(ids)})
|
|
340
363
|
if not isinstance(payload, dict):
|
|
341
364
|
return None
|
|
342
|
-
|
|
343
|
-
|
|
365
|
+
results = payload.get("results")
|
|
366
|
+
indexes = [_obj(_obj(entry).get("summary_stats")).get("h_index")
|
|
367
|
+
for entry in (results if isinstance(results, list) else [])]
|
|
344
368
|
return author_standing(
|
|
345
369
|
indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
|
|
346
370
|
|
|
@@ -28,6 +28,17 @@ boundary -- nothing is stored by this module unredacted -- beats depending
|
|
|
28
28
|
on a guarantee made two layers away that a later refactor could quietly
|
|
29
29
|
drop. The second pass is a no-op on already-clean text.
|
|
30
30
|
|
|
31
|
+
AND IT IS NOT WHERE §52's SPLIT-CREDENTIAL FIX WENT, deliberately (RB4).
|
|
32
|
+
§51 made this module the one place where two pages of a document become
|
|
33
|
+
adjacent, so a credential straddling a page boundary was rejoined HERE --
|
|
34
|
+
reproduced end to end in batch 106, into `artifact_entries()` and therefore
|
|
35
|
+
into `sources/<sha256>.txt` and the window sent to the embedding provider.
|
|
36
|
+
The register's proposed fix was to re-redact the whole merged text on every
|
|
37
|
+
append. It was declined: `fetch_url` redacts before it cuts now, so the
|
|
38
|
+
halves never exist, and re-scanning up to 20,000 characters per page would
|
|
39
|
+
buy nothing this module can see -- while leaving unfixed the model's own two
|
|
40
|
+
tool results, which the producer-side fix does close.
|
|
41
|
+
|
|
31
42
|
THIS MODULE OWNS URL IDENTITY for the pipeline. Two spellings of one arXiv
|
|
32
43
|
paper are one source, and a model that cites `/abs/2005.14165v3` after
|
|
33
44
|
fetching `/abs/2005.14165` must not be told its own citation has no text.
|
|
@@ -43,7 +54,7 @@ import hashlib
|
|
|
43
54
|
import logging
|
|
44
55
|
import re
|
|
45
56
|
from dataclasses import dataclass, field
|
|
46
|
-
from datetime import
|
|
57
|
+
from datetime import UTC, datetime
|
|
47
58
|
from urllib.parse import urlsplit, urlunsplit
|
|
48
59
|
|
|
49
60
|
from safety.policy_enforcement import redact_output_text
|
|
@@ -99,6 +110,26 @@ class SourceDocument:
|
|
|
99
110
|
tool: str = ""
|
|
100
111
|
retrieved_at: str = ""
|
|
101
112
|
|
|
113
|
+
#: ROADMAP_v3 §51 (PG7). How far into the SOURCE document the held text
|
|
114
|
+
#: reaches, in the source's own character positions -- which is what
|
|
115
|
+
#: `fetch_url` pages by, and not the same as `len(text)`. 0 for a
|
|
116
|
+
#: document no paging tool produced.
|
|
117
|
+
#:
|
|
118
|
+
#: §52 (RB1) removed ONE of the two reasons those numbers differ.
|
|
119
|
+
#: Redaction used to happen here, on each page, so a page's held length
|
|
120
|
+
#: could be shorter than the span it covered; `fetch_url` now redacts
|
|
121
|
+
#: before it cuts, so its pages arrive clean and that gap is gone. What
|
|
122
|
+
#: still separates the two is `.strip()` and `MAX_DOCUMENT_CHARS`, both
|
|
123
|
+
#: of which change how much is HELD and neither of which moves where a
|
|
124
|
+
#: page sat in the document it came from.
|
|
125
|
+
#:
|
|
126
|
+
#: It is what stops a page being absorbed twice. Without it, a model
|
|
127
|
+
#: that re-reads page 2 -- which it will, because re-reading is how a
|
|
128
|
+
#: model checks itself -- would append it again and the corpus would
|
|
129
|
+
#: hold the same paragraph twice, which a similarity score then counts
|
|
130
|
+
#: twice.
|
|
131
|
+
next_offset: int = 0
|
|
132
|
+
|
|
102
133
|
@property
|
|
103
134
|
def sha256(self) -> str:
|
|
104
135
|
return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
|
|
@@ -116,7 +147,7 @@ class SourceDocument:
|
|
|
116
147
|
|
|
117
148
|
|
|
118
149
|
def _now() -> str:
|
|
119
|
-
return datetime.now(
|
|
150
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
120
151
|
|
|
121
152
|
|
|
122
153
|
def arxiv_id_from_url(url: str) -> str | None:
|
|
@@ -224,11 +255,19 @@ class SourceCorpus:
|
|
|
224
255
|
# `url` is the URL that ANSWERED, after redirects -- which is what
|
|
225
256
|
# fetch_url.py:89-95 says the grounding pass attributes the text
|
|
226
257
|
# to, and therefore what the model will cite.
|
|
258
|
+
#
|
|
259
|
+
# `offset` is ROADMAP_v3 §51 (PG7). A result without one is a
|
|
260
|
+
# pre-§51 shape or a hand-built fixture, and 0 is the right reading
|
|
261
|
+
# of both: a whole document starting at the beginning.
|
|
262
|
+
offset = result.get("offset")
|
|
263
|
+
if not isinstance(offset, int) or offset < 0:
|
|
264
|
+
offset = 0
|
|
227
265
|
return self._store(
|
|
228
266
|
url=result.get("url"),
|
|
229
267
|
text=result.get("content"),
|
|
230
268
|
title="",
|
|
231
269
|
tool="fetch_url",
|
|
270
|
+
offset=offset,
|
|
232
271
|
)
|
|
233
272
|
|
|
234
273
|
def _add_arxiv(self, result: dict) -> int:
|
|
@@ -270,19 +309,81 @@ class SourceCorpus:
|
|
|
270
309
|
)
|
|
271
310
|
return added
|
|
272
311
|
|
|
273
|
-
def _store(self, url, text, title: str, tool: str) -> int:
|
|
312
|
+
def _store(self, url, text, title: str, tool: str, offset=None) -> int:
|
|
313
|
+
"""Absorb one document, or one PAGE of one (ROADMAP_v3 §51, PG7).
|
|
314
|
+
|
|
315
|
+
`offset` is the source position this text starts at, or None from a
|
|
316
|
+
tool that does not page. None and 0 are NOT the same thing here:
|
|
317
|
+
None means "this is the whole of what that tool returns", 0 means
|
|
318
|
+
"this is the first page of something that may have more".
|
|
319
|
+
|
|
320
|
+
WHY THE PAGED CASE CANNOT USE LONGEST-WINS. Measured, batch 105,
|
|
321
|
+
before any of this was written: a corpus given page 1 and then page
|
|
322
|
+
2 of one URL returned 1 then **0** -- `len(existing.text) >=
|
|
323
|
+
len(cleaned)` is 5000 >= 5000, so page 2 was DROPPED and the held
|
|
324
|
+
text stayed page 1. Paging would then have made the model better
|
|
325
|
+
informed and the grounding pass no better, and the divergence is
|
|
326
|
+
invisible from both sides: the model reads four pages and the score
|
|
327
|
+
is computed against the first. The near miss is worse than the
|
|
328
|
+
miss -- a page 2 one character longer REPLACED page 1 outright,
|
|
329
|
+
leaving the corpus holding the middle of a document and attributing
|
|
330
|
+
it to the whole URL.
|
|
331
|
+
"""
|
|
274
332
|
if not isinstance(url, str) or not isinstance(text, str):
|
|
275
333
|
return 0
|
|
276
334
|
key = normalize_url(url)
|
|
277
335
|
if not key:
|
|
278
336
|
return 0
|
|
279
337
|
|
|
338
|
+
existing = self.documents.get(key)
|
|
339
|
+
|
|
340
|
+
# A continuation extends what is held; anything else competes on
|
|
341
|
+
# length exactly as it did before §51.
|
|
342
|
+
continuing = (
|
|
343
|
+
offset is not None
|
|
344
|
+
and offset > 0
|
|
345
|
+
and existing is not None
|
|
346
|
+
and existing.tool == tool
|
|
347
|
+
)
|
|
348
|
+
|
|
349
|
+
if continuing:
|
|
350
|
+
# How much of this page is already held. Covers all three
|
|
351
|
+
# shapes with one subtraction: an exact continuation overlaps
|
|
352
|
+
# by nothing, a re-read of the same page overlaps entirely and
|
|
353
|
+
# contributes the empty string, and a page that starts inside
|
|
354
|
+
# what is held contributes only its tail. A page starting
|
|
355
|
+
# BEYOND the held text leaves a gap -- appended anyway, because
|
|
356
|
+
# the text is real and the alternative is losing it, and an
|
|
357
|
+
# incomplete document is what this corpus already holds for
|
|
358
|
+
# every source the byte cap cut.
|
|
359
|
+
#
|
|
360
|
+
# There is deliberately NO `if overlap >= len(text): return 0`
|
|
361
|
+
# above this. One was written, and a mutation pass showed it
|
|
362
|
+
# could be deleted with no test failing anywhere -- because the
|
|
363
|
+
# slice already yields "" in exactly the cases it guarded, and
|
|
364
|
+
# the empty-addition check below catches them. Batch 103's
|
|
365
|
+
# lesson applies to a guard that cannot fire as much as to one
|
|
366
|
+
# that fires in the wrong place: it reads as the thing
|
|
367
|
+
# protecting you.
|
|
368
|
+
overlap = existing.next_offset - offset
|
|
369
|
+
addition = redact_output_text(text[max(0, overlap):])
|
|
370
|
+
if not addition.strip():
|
|
371
|
+
return 0
|
|
372
|
+
merged = (existing.text + addition)[:MAX_DOCUMENT_CHARS]
|
|
373
|
+
if merged == existing.text:
|
|
374
|
+
return 0
|
|
375
|
+
self.documents[key] = SourceDocument(
|
|
376
|
+
url=key, text=merged, title=existing.title,
|
|
377
|
+
tool=tool, retrieved_at=_now(),
|
|
378
|
+
next_offset=max(existing.next_offset, offset + len(text)),
|
|
379
|
+
)
|
|
380
|
+
return 1
|
|
381
|
+
|
|
280
382
|
cleaned = redact_output_text(text).strip()
|
|
281
383
|
if len(cleaned) < MIN_DOCUMENT_CHARS:
|
|
282
384
|
return 0
|
|
283
385
|
cleaned = cleaned[:MAX_DOCUMENT_CHARS]
|
|
284
386
|
|
|
285
|
-
existing = self.documents.get(key)
|
|
286
387
|
if existing is not None and len(existing.text) >= len(cleaned):
|
|
287
388
|
return 0
|
|
288
389
|
if existing is None and len(self.documents) >= MAX_DOCUMENTS:
|
|
@@ -295,6 +396,11 @@ class SourceCorpus:
|
|
|
295
396
|
self.documents[key] = SourceDocument(
|
|
296
397
|
url=key, text=cleaned, title=title or (existing.title if existing else ""),
|
|
297
398
|
tool=tool, retrieved_at=_now(),
|
|
399
|
+
# In the SOURCE's positions, so the next page's offset can be
|
|
400
|
+
# compared against it. `len(text)` and not `len(cleaned)`:
|
|
401
|
+
# redaction and stripping change how much is HELD, never where
|
|
402
|
+
# the page sat in the document it came from.
|
|
403
|
+
next_offset=(offset + len(text)) if offset is not None else 0,
|
|
298
404
|
)
|
|
299
405
|
return 1
|
|
300
406
|
|