@yahyahammad/venastine 0.2.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +233 -94
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +277 -43
- package/agents/tui_commands.py +7 -12
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +158 -20
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1238 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +830 -66
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +1516 -440
- package/core/memory.py +120 -8
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +158 -26
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1671 -199
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +7 -12
- package/package.json +234 -4
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +51 -2
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +353 -17
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +368 -35
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +347 -30
- package/tui/app.py +4645 -438
- package/tui/app.tcss +185 -2
- package/tui/commands.py +10 -154
- package/tui/diffs.py +286 -265
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +60 -10
- package/tui/meters.py +103 -7
- package/tui/screens.py +346 -92
- package/tui/themes.py +59 -19
- package/tui/widgets.py +1575 -111
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
|
@@ -18,7 +18,11 @@ flag it if you intended something different.
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
20
|
from core.reasoning.base import (
|
|
21
|
-
CLAIM_TYPES,
|
|
21
|
+
CLAIM_TYPES,
|
|
22
|
+
GROUNDING_STATUSES,
|
|
23
|
+
Claim,
|
|
24
|
+
ConfidenceTier,
|
|
25
|
+
PipelineRun,
|
|
22
26
|
)
|
|
23
27
|
|
|
24
28
|
# --- Tunable weights -- change these, not the logic below, to retune scoring ---
|
|
@@ -28,27 +32,32 @@ CRITIC_WEIGHT_FACTOR = 0.35
|
|
|
28
32
|
ASSUMPTION_FLAG_PENALTY = 0.15 # per flag
|
|
29
33
|
NON_FACTUAL_SCORE_CAP = 0.65 # claims never grounded can't reach HIGH
|
|
30
34
|
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
35
|
+
# ROADMAP_v3 §90 (A85, issue #12). Cross-candidate disagreement is ROUTED,
|
|
36
|
+
# NOT SCORED. A factual claim asserted by at most this share of the
|
|
37
|
+
# candidates is `contested`, and D1 sends it to the 6a/6b loop to be
|
|
38
|
+
# checked; nothing is subtracted for it.
|
|
34
39
|
#
|
|
35
|
-
# §10
|
|
36
|
-
#
|
|
40
|
+
# It was a penalty before (§10's revisit, E8): 0.15 * (1 - consistency),
|
|
41
|
+
# itself the repair of §10's first shape, a fourth weighted term that made
|
|
42
|
+
# grounding count for less whenever ensemble mode was on. Two things were
|
|
43
|
+
# wrong with the penalty, and the second was measured:
|
|
37
44
|
#
|
|
38
|
-
# 1.
|
|
39
|
-
# the pipeline
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
45
|
+
# 1. It answered the wrong question. Disagreement is the one signal in
|
|
46
|
+
# the pipeline that points at a specific claim and says "the models do
|
|
47
|
+
# not concur here", which is what a verification pass wants as input,
|
|
48
|
+
# and a deduction spends it on a number instead.
|
|
49
|
+
# 2. It could send a claim into the loop that no revision could get out
|
|
50
|
+
# of. A partially grounded claim scores 0.60 (MEDIUM); at 1 of 3 the
|
|
51
|
+
# penalty took it to 0.50 (LOW), so D1 flagged it, 6a was never told
|
|
52
|
+
# why, and 6b re-scored it under the same penalty -- who asserted a
|
|
53
|
+
# claim does not change when it is reworded -- until D2 published it
|
|
54
|
+
# UNVERIFIED.
|
|
44
55
|
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
|
|
50
|
-
# byte-for-byte regression guard holds by construction.
|
|
51
|
-
DISAGREEMENT_PENALTY_FACTOR = 0.15
|
|
56
|
+
# AT MOST HALF, so 1 of 2, 1 of 3 and 2 of 4 are contested and 2 of 3 is
|
|
57
|
+
# not: exactly the shapes where the penalty used to change a tier. The
|
|
58
|
+
# non-ensemble formula is untouched, and the ensemble one is now the same
|
|
59
|
+
# formula, so §10's byte-for-byte regression guard holds for both.
|
|
60
|
+
CONTESTED_SHARE = 0.5
|
|
52
61
|
|
|
53
62
|
# ROADMAP_v2 §45 (SQ3). What a grounding is WORTH.
|
|
54
63
|
#
|
|
@@ -113,7 +122,6 @@ def default_weights() -> dict:
|
|
|
113
122
|
"grounding_weight_factor": GROUNDING_WEIGHT_FACTOR,
|
|
114
123
|
"critic_weight_factor": CRITIC_WEIGHT_FACTOR,
|
|
115
124
|
"assumption_flag_penalty": ASSUMPTION_FLAG_PENALTY,
|
|
116
|
-
"disagreement_penalty_factor": DISAGREEMENT_PENALTY_FACTOR,
|
|
117
125
|
"non_factual_score_cap": NON_FACTUAL_SCORE_CAP,
|
|
118
126
|
"source_quality_floor": SOURCE_QUALITY_FLOOR,
|
|
119
127
|
"authority_full_credit": AUTHORITY_FULL_CREDIT,
|
|
@@ -194,9 +202,10 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
194
202
|
"""Returns (tier, score_breakdown). Pure function -- no side effects,
|
|
195
203
|
easy to unit test claim-by-claim without running any pass.
|
|
196
204
|
|
|
197
|
-
When ensemble_n > 0 (ROADMAP §10),
|
|
198
|
-
|
|
199
|
-
|
|
205
|
+
When ensemble_n > 0 (ROADMAP §10), the share of candidates that
|
|
206
|
+
asserted this claim is recorded, and a factual claim asserted by at
|
|
207
|
+
most CONTESTED_SHARE of them is marked `contested` for D1 to route
|
|
208
|
+
(ROADMAP_v3 §90). It changes no score. ensemble_n is the number of
|
|
200
209
|
candidates that actually produced text, which on a run where a
|
|
201
210
|
candidate's provider failed is fewer than the configured roster.
|
|
202
211
|
|
|
@@ -251,29 +260,25 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
251
260
|
critic_component = 1.0 - claim.critic_severity
|
|
252
261
|
assumption_penalty = w["assumption_flag_penalty"] * len(claim.assumption_flags)
|
|
253
262
|
|
|
254
|
-
# None when ensemble mode is off
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
263
|
+
# None when ensemble mode is off -- and (B7) also when ensemble mode is
|
|
264
|
+
# ON and Pass 2 reported nothing for this claim, which is a different
|
|
265
|
+
# fact from "no candidate asserted it". An unreported claim is never
|
|
266
|
+
# contested.
|
|
258
267
|
consistency_score = None
|
|
259
268
|
consistency_reported = None
|
|
260
|
-
disagreement_penalty = 0.0
|
|
261
269
|
if ensemble_n > 0 and claim.asserted_by_candidates:
|
|
262
|
-
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made
|
|
263
|
-
#
|
|
264
|
-
# claim
|
|
265
|
-
#
|
|
266
|
-
#
|
|
267
|
-
#
|
|
270
|
+
# B8. Deduplicated and clamped. `[1, 2, 3, 3]` made consistency
|
|
271
|
+
# 1.3333, which under E8 made the penalty NEGATIVE and scored a
|
|
272
|
+
# MEDIUM claim HIGH; here a duplicate would make one candidate's
|
|
273
|
+
# claim read as two agreeing, and `[1, 1]` of 3 would escape
|
|
274
|
+
# routing. `consistency_reported` keeps the unclamped value when
|
|
275
|
+
# it fired, because a numerator above the denominator is itself
|
|
276
|
+
# evidence that Pass 2 misread the candidate labelling, and
|
|
268
277
|
# clamping silently would hide the evidence along with the bug.
|
|
269
278
|
reported = len(set(claim.asserted_by_candidates)) / ensemble_n
|
|
270
279
|
consistency_score = min(1.0, reported)
|
|
271
280
|
if reported > 1.0:
|
|
272
281
|
consistency_reported = round(reported, 4)
|
|
273
|
-
# E9: computed only where it is applied, so the breakdown records
|
|
274
|
-
# what was subtracted rather than what was available (B10).
|
|
275
|
-
if claim_type == "factual":
|
|
276
|
-
disagreement_penalty = w["disagreement_penalty_factor"] * (1.0 - consistency_score)
|
|
277
282
|
|
|
278
283
|
if claim_type != "factual":
|
|
279
284
|
# Cap applies to the critic component BEFORE subtracting the
|
|
@@ -283,11 +288,6 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
283
288
|
# claim that was already clean under critique. Found this via a
|
|
284
289
|
# direct test case (two speculative claims, one flagged one not,
|
|
285
290
|
# both landing at the same tier) before it shipped.
|
|
286
|
-
#
|
|
287
|
-
# E9: no disagreement penalty here. §10 decided consistency does not
|
|
288
|
-
# RESCUE a non-factual claim from the cap; whether dissent should
|
|
289
|
-
# penalise a speculative claim is a separate judgment about what
|
|
290
|
-
# candidates declining to speculate means, and it was not made.
|
|
291
291
|
capped_critic = min(critic_component, w["non_factual_score_cap"])
|
|
292
292
|
raw_score = capped_critic - assumption_penalty
|
|
293
293
|
formula = "non_factual"
|
|
@@ -298,7 +298,6 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
298
298
|
(w["grounding_weight_factor"] * grounding_component)
|
|
299
299
|
+ (w["critic_weight_factor"] * critic_component)
|
|
300
300
|
- assumption_penalty
|
|
301
|
-
- disagreement_penalty
|
|
302
301
|
)
|
|
303
302
|
|
|
304
303
|
# ROUND ONCE, then use that one value for BOTH the tier comparison and
|
|
@@ -309,8 +308,9 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
309
308
|
# auditing a run artifact read a number that contradicted the published
|
|
310
309
|
# thresholds, and the tier was the one that mattered.
|
|
311
310
|
#
|
|
312
|
-
# Binary floating point
|
|
313
|
-
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3)
|
|
311
|
+
# Binary floating point made this reachable from ordinary weights, not
|
|
312
|
+
# from an exotic input: 0.5 + 0.35 - 0.15 * (1/3), E8's 2-of-3 penalty,
|
|
313
|
+
# was exactly that value.
|
|
314
314
|
# Sweeping the whole non-ensemble input space, exactly ONE shape changes
|
|
315
315
|
# tier under this fix -- a speculative claim at severity 0.55 with one
|
|
316
316
|
# assumption flag, which records 0.3 and now tiers LOW instead of
|
|
@@ -336,7 +336,7 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
336
336
|
# extra key: "anyone auditing a run artifact read a number that
|
|
337
337
|
# contradicted the published thresholds."
|
|
338
338
|
#
|
|
339
|
-
# factual 0.5*grounding + 0.35*critic - assumption
|
|
339
|
+
# factual 0.5*grounding + 0.35*critic - assumption
|
|
340
340
|
# non_factual capped_critic - assumption
|
|
341
341
|
breakdown = {
|
|
342
342
|
"formula": formula,
|
|
@@ -371,11 +371,18 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
371
371
|
# B7. None means PASS 2 REPORTED NOTHING, which is not 0.0 -- and
|
|
372
372
|
# the two were indistinguishable, so a claim one candidate raised
|
|
373
373
|
# uncontested took the maximum 0.15 penalty rather than the 0.05 it
|
|
374
|
-
# earned
|
|
375
|
-
# apart too.
|
|
374
|
+
# earned under E8. `contested` needs them apart for the same reason.
|
|
376
375
|
breakdown["consistency_score"] = (
|
|
377
376
|
None if consistency_score is None else round(consistency_score, 4))
|
|
378
|
-
|
|
377
|
+
# §90: what D1 routes on, and the record of why a claim that scored
|
|
378
|
+
# HIGH went through 6a/6b anyway. E9 LIVES HERE, and only here:
|
|
379
|
+
# whether candidates declining to speculate says anything about a
|
|
380
|
+
# speculative claim is a judgment §10 never made, so the non-factual
|
|
381
|
+
# branch is never asked and carries no key (B10: a field that did
|
|
382
|
+
# nothing is never present).
|
|
383
|
+
if formula == "factual":
|
|
384
|
+
breakdown["contested"] = (consistency_score is not None
|
|
385
|
+
and consistency_score <= CONTESTED_SHARE)
|
|
379
386
|
if consistency_reported is not None:
|
|
380
387
|
breakdown["consistency_reported"] = consistency_reported
|
|
381
388
|
if unrecognised_status is not None:
|
|
@@ -385,6 +392,14 @@ def score_claim(claim: Claim, ensemble_n: int = 0,
|
|
|
385
392
|
return tier, breakdown
|
|
386
393
|
|
|
387
394
|
|
|
395
|
+
def is_contested(claim: Claim) -> bool:
|
|
396
|
+
"""Whether Pass 5 marked this claim contested (ROADMAP_v3 §90).
|
|
397
|
+
|
|
398
|
+
Read off the breakdown rather than recomputed, so D1 routes on exactly
|
|
399
|
+
what the stored claim says and the type coercion above happens once."""
|
|
400
|
+
return claim.score_breakdown.get("contested") is True
|
|
401
|
+
|
|
402
|
+
|
|
388
403
|
def build_coverage_gap_entries(completeness: dict) -> list[dict]:
|
|
389
404
|
"""Gaps from Pass 3c become separate coverage-gap entries, not Claim
|
|
390
405
|
objects -- they were never asserted, so there's nothing to re-tier."""
|
|
@@ -398,8 +413,9 @@ def run_confidence_tiering(
|
|
|
398
413
|
weights: dict | None = None,
|
|
399
414
|
) -> None:
|
|
400
415
|
"""The actual Pass 5 entry point the orchestrator calls. Mutates
|
|
401
|
-
the claims in place; zero LLM calls. Pass ensemble_n > 0 to
|
|
402
|
-
the
|
|
416
|
+
the claims in place; zero LLM calls. Pass ensemble_n > 0 to record
|
|
417
|
+
each claim's consistency and mark the contested ones (ROADMAP §10,
|
|
418
|
+
ROADMAP_v3 §90); the formula is the same either way.
|
|
403
419
|
|
|
404
420
|
`claims` narrows WHICH claims are re-tiered, and defaults to all of
|
|
405
421
|
them -- which is Pass 5 itself, where every claim is in play. The
|
|
@@ -449,8 +465,8 @@ def _report_coercions(run: PipelineRun, claim: Claim, breakdown: dict,
|
|
|
449
465
|
if ensemble_n > 0 and breakdown.get("consistency_score") is None:
|
|
450
466
|
run.log(
|
|
451
467
|
f"Pass 5: claim {claim.id} carries no asserted_by_candidates on "
|
|
452
|
-
f"an ensemble run --
|
|
453
|
-
f"
|
|
468
|
+
f"an ensemble run -- not treated as contested, since an "
|
|
469
|
+
f"unreported numerator is not a numerator of zero."
|
|
454
470
|
)
|
|
455
471
|
if "unrecognised_grounding_status" in breakdown:
|
|
456
472
|
run.log(
|
|
@@ -44,7 +44,8 @@ status='failed' record.
|
|
|
44
44
|
from __future__ import annotations
|
|
45
45
|
|
|
46
46
|
import json
|
|
47
|
-
from
|
|
47
|
+
from collections.abc import Callable
|
|
48
|
+
from typing import Any, Optional
|
|
48
49
|
|
|
49
50
|
import config
|
|
50
51
|
from core.reasoning.payload_validation import PayloadShapeError
|
|
@@ -85,6 +86,7 @@ def retry_until_json(
|
|
|
85
86
|
max_total_tokens: Optional[int] = None,
|
|
86
87
|
validate: Optional[Callable] = None,
|
|
87
88
|
effort: Optional[str] = None,
|
|
89
|
+
activity=None,
|
|
88
90
|
) -> str:
|
|
89
91
|
"""Returns the first response text that parses as JSON, retrying by
|
|
90
92
|
continuing `response`'s thread.
|
|
@@ -121,6 +123,16 @@ def retry_until_json(
|
|
|
121
123
|
PayloadShapeError re-enters this same loop. None means
|
|
122
124
|
"parse is the only contract", which is what the reviewer
|
|
123
125
|
wants and what every caller had before §30.
|
|
126
|
+
activity §47's agent-activity sink, forwarded for the same reason
|
|
127
|
+
as everything above it: a retry is the same run
|
|
128
|
+
continuing. It matters MORE here than elsewhere, not
|
|
129
|
+
less -- `continue_conversation` drains its own loop, so
|
|
130
|
+
a subagent spawned while the model is being corrected
|
|
131
|
+
reaches a shell through this channel or through nothing
|
|
132
|
+
(`tui/app.py`'s one-shot path carries it for exactly
|
|
133
|
+
that reason). No span is opened here: the caller's own
|
|
134
|
+
brackets the whole attempt-and-retry sequence, and a row
|
|
135
|
+
per correction would report one run as several.
|
|
124
136
|
"""
|
|
125
137
|
from core.loop import RunAgentLoop
|
|
126
138
|
|
|
@@ -165,6 +177,12 @@ def retry_until_json(
|
|
|
165
177
|
# continuing, so it carries the pass's effort too --
|
|
166
178
|
# effort_for re-validates against this thread's model.
|
|
167
179
|
effort=effort,
|
|
180
|
+
# And the same rule again for §47's sink (batch 75). This
|
|
181
|
+
# hop dropped it, so a spawn inside a corrective attempt
|
|
182
|
+
# was invisible -- and this is the one path where the
|
|
183
|
+
# channel is the ONLY route, because continue_conversation
|
|
184
|
+
# drains its own loop.
|
|
185
|
+
activity=activity,
|
|
168
186
|
**({} if max_total_tokens is None
|
|
169
187
|
else {"max_total_tokens": max_total_tokens}),
|
|
170
188
|
)
|