@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
package/credentials.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
# #24. Resolved at IMPORT time, exactly like its four siblings --
|
|
5
|
+
# config.DB_PATH / OUTPUT_DIR / WORKSPACE_DIR and logging_setup's log
|
|
6
|
+
# file are all `os.environ.get(NAME, default)` evaluated once -- so a
|
|
7
|
+
# test can still redirect it by monkeypatching the attribute, and a
|
|
8
|
+
# user can point it anywhere with AGENT_PROVIDERS_FILE before launch.
|
|
9
|
+
# The default stays CWD-relative on purpose: for a repo-local tool,
|
|
10
|
+
# per-checkout state is what you want, and repo-relative would share
|
|
11
|
+
# one key file across every clone on the machine.
|
|
12
|
+
LLM_PROVIDERS_FILE = os.environ.get("AGENT_PROVIDERS_FILE", "providers.json")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def no_providers_message() -> str:
|
|
16
|
+
"""The error for "the file itself is not there" (#24).
|
|
17
|
+
|
|
18
|
+
ONE wording, used by both raisers -- api_initialization (the one
|
|
19
|
+
every user actually meets) and load_credentials below -- so the two
|
|
20
|
+
cannot drift apart the way their old messages already had.
|
|
21
|
+
|
|
22
|
+
Batch 35: the LOCATION clause is conditional now, because the path is
|
|
23
|
+
no longer always cwd-relative. AGENT_PROVIDERS_FILE has always accepted
|
|
24
|
+
an absolute path, and the npm launcher sets one when the working
|
|
25
|
+
directory has no providers.json of its own. Naming os.getcwd()
|
|
26
|
+
unconditionally then reported a directory the code never looked in and
|
|
27
|
+
sent the user to create the file in the wrong place -- the exact class
|
|
28
|
+
of defect #24 was filed about, reintroduced by a path it did not
|
|
29
|
+
anticipate. An absolute path is already the whole answer and needs no
|
|
30
|
+
second location.
|
|
31
|
+
"""
|
|
32
|
+
if os.path.isabs(LLM_PROVIDERS_FILE):
|
|
33
|
+
found = f"No providers configured: {LLM_PROVIDERS_FILE} was not found."
|
|
34
|
+
else:
|
|
35
|
+
found = (
|
|
36
|
+
f"No providers configured: {LLM_PROVIDERS_FILE} was not found in "
|
|
37
|
+
f"{os.getcwd()!r}."
|
|
38
|
+
)
|
|
39
|
+
return (
|
|
40
|
+
f"{found} Copy providers.json.example to "
|
|
41
|
+
f"{LLM_PROVIDERS_FILE} and add your API key."
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def unknown_provider_message(provider_name: str, providers: dict) -> str:
|
|
46
|
+
"""The error for "the file is there, this provider is not in it" (#24).
|
|
47
|
+
|
|
48
|
+
Names the configured set exactly like /model's refusal does, so the
|
|
49
|
+
typo is answered where it was made rather than two layers later.
|
|
50
|
+
"""
|
|
51
|
+
configured = ", ".join(sorted(providers)) or "none"
|
|
52
|
+
return (
|
|
53
|
+
f"Unknown provider: {provider_name}. Configured: {configured} "
|
|
54
|
+
f"(file: {LLM_PROVIDERS_FILE})."
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def load_provider_data() -> dict:
|
|
59
|
+
if not os.path.exists(LLM_PROVIDERS_FILE):
|
|
60
|
+
return {}
|
|
61
|
+
with open(LLM_PROVIDERS_FILE, "r", encoding="utf-8") as file:
|
|
62
|
+
return json.load(file)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# 0600 -- owner read/write, nobody else. This file holds every LLM
|
|
66
|
+
# provider's API key in cleartext, and a bare open(path, "w") left its
|
|
67
|
+
# protection to the process umask: 0644 under the common default, i.e.
|
|
68
|
+
# readable by every other account on the machine (#19).
|
|
69
|
+
#
|
|
70
|
+
# The blast radius of a credential file is not the file; it is the paid
|
|
71
|
+
# API account behind it. Every comparable tool treats one as 0600
|
|
72
|
+
# (~/.aws/credentials, ~/.docker/config.json, ~/.ssh/id_*), and OpenSSH
|
|
73
|
+
# goes further and REFUSES a group-readable private key.
|
|
74
|
+
_SECRET_FILE_MODE = 0o600
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _write_secret_json(path: str, data: dict) -> None:
|
|
78
|
+
"""Write JSON to a file created 0600, never world-readable even briefly.
|
|
79
|
+
|
|
80
|
+
os.open with the mode, NOT open() followed by os.chmod. A post-write
|
|
81
|
+
chmod leaves a window in which the key is on disk with the umask's
|
|
82
|
+
permissions -- short, but it is exactly the window an attacker with
|
|
83
|
+
local read access is waiting for, and closing it costs one line.
|
|
84
|
+
|
|
85
|
+
WINDOWS: POSIX mode bits do not apply, and os.open's mode argument
|
|
86
|
+
only controls the read-only flag there. This is a no-op on Windows
|
|
87
|
+
rather than a wrong-op; the file's protection comes from the ACL on
|
|
88
|
+
the user profile directory instead. The tests skip accordingly, and
|
|
89
|
+
the CI container is where the assertion actually runs.
|
|
90
|
+
"""
|
|
91
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
|
|
92
|
+
_SECRET_FILE_MODE)
|
|
93
|
+
with os.fdopen(fd, "w", encoding="utf-8") as file:
|
|
94
|
+
json.dump(data, file, indent=2)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _write_provider_data(provider_data: dict) -> None:
|
|
98
|
+
_write_secret_json(LLM_PROVIDERS_FILE, provider_data)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def save_credentials(
|
|
102
|
+
provider_name: str,
|
|
103
|
+
api_key: str,
|
|
104
|
+
api_url: str = "",
|
|
105
|
+
is_v1_compatible: bool = True,
|
|
106
|
+
supports_stream_usage: bool = False,
|
|
107
|
+
) -> None:
|
|
108
|
+
provider_data = load_provider_data()
|
|
109
|
+
|
|
110
|
+
if provider_name in provider_data:
|
|
111
|
+
provider_data[provider_name]["API_KEY"] = api_key
|
|
112
|
+
if api_url:
|
|
113
|
+
provider_data[provider_name]["API_URL"] = api_url
|
|
114
|
+
provider_data[provider_name]["is_v1_compatible"] = is_v1_compatible
|
|
115
|
+
provider_data[provider_name]["supports_stream_usage"] = supports_stream_usage
|
|
116
|
+
else:
|
|
117
|
+
provider_data[provider_name] = {
|
|
118
|
+
"API_KEY": api_key,
|
|
119
|
+
"API_URL": api_url,
|
|
120
|
+
"is_v1_compatible": is_v1_compatible,
|
|
121
|
+
"supports_stream_usage": supports_stream_usage,
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
_write_provider_data(provider_data)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def load_credentials(provider_name: str) -> tuple[str, str]:
|
|
128
|
+
provider_data = load_provider_data()
|
|
129
|
+
# #24. The two failure modes say different things now -- "the file is
|
|
130
|
+
# not there" is a setup problem with a one-line remedy; "the provider
|
|
131
|
+
# is not in it" is a typo. Same builders api_initialization uses.
|
|
132
|
+
if not provider_data and not os.path.exists(LLM_PROVIDERS_FILE):
|
|
133
|
+
raise ValueError(no_providers_message())
|
|
134
|
+
if provider_name not in provider_data:
|
|
135
|
+
raise ValueError(unknown_provider_message(provider_name, provider_data))
|
|
136
|
+
|
|
137
|
+
entry = provider_data[provider_name]
|
|
138
|
+
return entry["API_KEY"], entry.get("API_URL", "")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def missing_key_message(provider_name: str) -> str:
|
|
142
|
+
"""The one wording for "configured, but with an empty API_KEY".
|
|
143
|
+
|
|
144
|
+
Extracted in batch 44 for the reason no_providers_message() was: /model
|
|
145
|
+
carried its own copy, naming providers.json literally where this file
|
|
146
|
+
names LLM_PROVIDERS_FILE, so the same fact read differently at launch
|
|
147
|
+
and at a switch -- while tui/app.py's comment beside the launch render
|
|
148
|
+
already claimed they were "the same fact at both moments". WARN-only at
|
|
149
|
+
both call sites: a local OpenAI-compatible endpoint legitimately takes
|
|
150
|
+
no key, and refusing would block a real configuration.
|
|
151
|
+
"""
|
|
152
|
+
return (f"{provider_name} has no API_KEY in {LLM_PROVIDERS_FILE} "
|
|
153
|
+
f"— calls will fail unless it is a local endpoint that "
|
|
154
|
+
f"needs none.")
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def provider_startup_issues(provider_name: str) -> list:
|
|
158
|
+
"""What launch should say about the provider setup, before either
|
|
159
|
+
shell starts (#138).
|
|
160
|
+
|
|
161
|
+
The same three checks /model performs on a switch -- file present,
|
|
162
|
+
provider configured, key non-empty -- applied at the moment the user
|
|
163
|
+
chose the provider, instead of at the first model call, which is a
|
|
164
|
+
dead end away from the mistake that caused it. WARN-ONLY, never a
|
|
165
|
+
refusal: an OpenAI-compatible endpoint running locally legitimately
|
|
166
|
+
takes no key (the reason /model warns there too), and refusing would
|
|
167
|
+
block a real configuration.
|
|
168
|
+
|
|
169
|
+
Empty list means healthy and says NOTHING -- silence is the healthy
|
|
170
|
+
case's whole UX. Plain sentences, no prefix: rendering is the
|
|
171
|
+
shell's (CLI prints, TUI write_error), so one wording reaches both.
|
|
172
|
+
"""
|
|
173
|
+
providers = load_provider_data()
|
|
174
|
+
if not providers and not os.path.exists(LLM_PROVIDERS_FILE):
|
|
175
|
+
return [no_providers_message()]
|
|
176
|
+
if provider_name not in providers:
|
|
177
|
+
return [unknown_provider_message(provider_name, providers)]
|
|
178
|
+
if not providers[provider_name].get("API_KEY"):
|
|
179
|
+
return [missing_key_message(provider_name)]
|
|
180
|
+
return []
|
package/database.py
ADDED
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""
|
|
2
|
+
database.py
|
|
3
|
+
|
|
4
|
+
Owns the actual database connection -- the engine, where the DB file
|
|
5
|
+
lives, and how tables get created. storage.py imports `engine` from here;
|
|
6
|
+
nothing here knows about ConversationThread, MessageLog, or any other
|
|
7
|
+
table schema.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from sqlmodel import create_engine, SQLModel
|
|
13
|
+
|
|
14
|
+
import config
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
DATABASE_URL = f"sqlite:///{config.DB_PATH}"
|
|
19
|
+
|
|
20
|
+
engine = create_engine(
|
|
21
|
+
DATABASE_URL,
|
|
22
|
+
echo=False,
|
|
23
|
+
connect_args={"check_same_thread": False},
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def create_db_and_tables() -> None:
|
|
28
|
+
"""Creates any table that doesn't exist yet. Safe to call every startup.
|
|
29
|
+
|
|
30
|
+
IMPORTANT: this creates only the tables REGISTERED ON SQLModel.metadata
|
|
31
|
+
when it runs, and a table class registers when its module is imported.
|
|
32
|
+
This file knows nothing about which tables exist (§4.4), so the caller
|
|
33
|
+
must have imported the modules defining them first -- see main.py, which
|
|
34
|
+
imports storage and pipeline_storage explicitly for that side effect.
|
|
35
|
+
|
|
36
|
+
Getting this wrong is silent: create_all() happily creates nothing and
|
|
37
|
+
returns, and the failure surfaces much later as "no such table" at the
|
|
38
|
+
first write. That is exactly how a fresh database ended up with no
|
|
39
|
+
pipelinerunrecord table while research mode looked fine on an existing
|
|
40
|
+
one. The guard below turns the total-failure case into an immediate,
|
|
41
|
+
named error; a partially-registered set can't be detected from here
|
|
42
|
+
without this file learning what data exists, which is the boundary §4.4
|
|
43
|
+
exists to hold.
|
|
44
|
+
"""
|
|
45
|
+
if not SQLModel.metadata.tables:
|
|
46
|
+
raise RuntimeError(
|
|
47
|
+
"create_db_and_tables() found no table classes registered. Import "
|
|
48
|
+
"the modules that define them (storage, core.reasoning.pipeline_storage) "
|
|
49
|
+
"before calling this, or the database will be created empty."
|
|
50
|
+
)
|
|
51
|
+
SQLModel.metadata.create_all(engine)
|
|
52
|
+
# AFTER create_all, deliberately: a brand-new database gets every table
|
|
53
|
+
# built complete and needs no ALTER at all. This pass exists only for
|
|
54
|
+
# databases that predate a column.
|
|
55
|
+
connection = engine.raw_connection()
|
|
56
|
+
try:
|
|
57
|
+
added = ensure_columns(connection, _declared_columns())
|
|
58
|
+
finally:
|
|
59
|
+
connection.close()
|
|
60
|
+
if added:
|
|
61
|
+
logger.info("Added missing columns to an existing database: %s",
|
|
62
|
+
", ".join(added))
|
|
63
|
+
logger.debug("Ensured tables exist: %s", sorted(SQLModel.metadata.tables))
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
# ---- Additive column migration (ROADMAP_v2 §21, M7) -----------------------
|
|
68
|
+
# ---------------------------------------------------------------------------
|
|
69
|
+
#
|
|
70
|
+
# create_all() creates missing TABLES and never touches an existing one, so
|
|
71
|
+
# adding a field to a table class that is already on disk produces "no such
|
|
72
|
+
# column" on the next SELECT -- for every database that predates the change,
|
|
73
|
+
# silently, at read time rather than at startup. §21's `MessageLog.pinned` is
|
|
74
|
+
# the first field this project has ever added to a shipped table; it will not
|
|
75
|
+
# be the last.
|
|
76
|
+
#
|
|
77
|
+
# ADDITIVE ONLY, and that is the whole contract. This will add a column that
|
|
78
|
+
# is declared and missing. It will never rename, drop, retype, backfill, or
|
|
79
|
+
# reorder, and it does not track a schema version. Anything beyond "a new
|
|
80
|
+
# nullable-or-defaulted column appeared" needs a real migration and should
|
|
81
|
+
# not be smuggled in here -- if this file ever grows a `DROP`, that is the
|
|
82
|
+
# signal the project has outgrown it.
|
|
83
|
+
#
|
|
84
|
+
# Nullability rides along where SQLite allows it (#26): `NOT NULL DEFAULT x`
|
|
85
|
+
# is expressible in one ALTER and matches create_all exactly; a not-null
|
|
86
|
+
# column with no SQL-spellable default cannot be added NOT NULL at all and
|
|
87
|
+
# lands nullable under a named WARNING. The two columns migrated before this
|
|
88
|
+
# (`pinned`, `kind`) remain nullable on databases that predate #26 -- accepted,
|
|
89
|
+
# not repaired: flipping them needs a table rebuild, which is outside this
|
|
90
|
+
# contract, and no writer can produce a NULL for them.
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _default_literal(column):
|
|
94
|
+
"""A SQL literal for the column's default, or None if it has none we
|
|
95
|
+
can express.
|
|
96
|
+
|
|
97
|
+
SQLite fills existing rows with NULL when a column is added, so a
|
|
98
|
+
column the Python side declares non-optional would read back as None
|
|
99
|
+
on every pre-existing row. Emitting DEFAULT closes that: `pinned`
|
|
100
|
+
arrives as 0 on rows written before pinning existed, which is what
|
|
101
|
+
"this message was never pinned" should mean.
|
|
102
|
+
|
|
103
|
+
Only scalar defaults qualify. A default_factory (uuid4, utcnow) is
|
|
104
|
+
per-row Python and has no SQL spelling, so those columns are added
|
|
105
|
+
nullable and left to the application -- which is correct, because a
|
|
106
|
+
factory-defaulted column on an existing row has no right answer.
|
|
107
|
+
"""
|
|
108
|
+
default = getattr(column, "default", None)
|
|
109
|
+
if default is None or getattr(default, "is_callable", False):
|
|
110
|
+
return None
|
|
111
|
+
value = getattr(default, "arg", None)
|
|
112
|
+
if value is None or callable(value):
|
|
113
|
+
return None
|
|
114
|
+
if isinstance(value, bool):
|
|
115
|
+
return "1" if value else "0"
|
|
116
|
+
if isinstance(value, (int, float)):
|
|
117
|
+
return str(value)
|
|
118
|
+
if isinstance(value, str):
|
|
119
|
+
escaped = value.replace("'", "''")
|
|
120
|
+
return f"'{escaped}'"
|
|
121
|
+
return None
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _declared_columns() -> dict:
|
|
125
|
+
"""table name -> {column name: (sql type, default literal or None,
|
|
126
|
+
not-null flag)}, read off whatever SQLModel currently has registered.
|
|
127
|
+
|
|
128
|
+
This keeps §4.4's boundary intact: the file still knows nothing about
|
|
129
|
+
WHICH tables exist, it asks the metadata -- the same source create_all
|
|
130
|
+
above is already driven by. A table class that was never imported is
|
|
131
|
+
invisible here exactly as it is there.
|
|
132
|
+
|
|
133
|
+
Everything SQLAlchemy-shaped is resolved HERE, into plain strings, so
|
|
134
|
+
ensure_columns() below is pure SQL over a DBAPI connection with no
|
|
135
|
+
ORM knowledge at all. That is what makes it testable against stdlib
|
|
136
|
+
sqlite3 while the suite's fake `sqlmodel` is installed -- the fake's
|
|
137
|
+
engine has no dialect to compile a type against, and a seam that only
|
|
138
|
+
works in production is not a seam.
|
|
139
|
+
|
|
140
|
+
The third slot is the nullability the model declares (#26): SQLModel
|
|
141
|
+
derives it from the annotation, so `pinned: bool` arrives not-null
|
|
142
|
+
and `name: Optional[str]` does not. Without carrying it, every column
|
|
143
|
+
this migration adds lands nullable where create_all would have built
|
|
144
|
+
NOT NULL -- a migrated database silently diverging from a fresh one.
|
|
145
|
+
"""
|
|
146
|
+
dialect = engine.dialect
|
|
147
|
+
return {
|
|
148
|
+
name: {
|
|
149
|
+
c.name: (
|
|
150
|
+
c.type.compile(dialect),
|
|
151
|
+
_default_literal(c),
|
|
152
|
+
not bool(getattr(c, "nullable", True)),
|
|
153
|
+
)
|
|
154
|
+
for c in table.columns
|
|
155
|
+
}
|
|
156
|
+
for name, table in SQLModel.metadata.tables.items()
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _affinity(sql_type: str) -> str:
|
|
161
|
+
"""Normalized type name for comparison -- uppercased, with any length
|
|
162
|
+
or precision specifier dropped. VARCHAR(255) and varchar compare
|
|
163
|
+
equal; BOOLEAN and VARCHAR do not."""
|
|
164
|
+
return sql_type.split("(")[0].strip().upper()
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def ensure_columns(connection, declared: dict) -> list:
|
|
168
|
+
"""Add every declared column missing from an existing table.
|
|
169
|
+
|
|
170
|
+
`declared` is _declared_columns()' plain-string shape --
|
|
171
|
+
table -> {column: (sql type, default literal or None, not-null flag)}.
|
|
172
|
+
`connection` is a DBAPI connection (`engine.raw_connection()` in
|
|
173
|
+
production, a plain `sqlite3.connect(...)` in tests) rather than a
|
|
174
|
+
SQLAlchemy Connection. Between them this function touches no ORM at
|
|
175
|
+
all, which is what makes it testable against stdlib sqlite3 while the
|
|
176
|
+
suite's fake `sqlmodel` is installed. Returns the "table.column" names
|
|
177
|
+
added, for logging.
|
|
178
|
+
|
|
179
|
+
A table with no rows in `PRAGMA table_info` does not exist yet;
|
|
180
|
+
create_all built it or will, so there is nothing to migrate.
|
|
181
|
+
|
|
182
|
+
Nullability is carried where SQLite can express it (#26). ALTER TABLE
|
|
183
|
+
ADD COLUMN accepts NOT NULL only together with a non-NULL DEFAULT, so
|
|
184
|
+
a not-null column WITH a scalar default migrates as `NOT NULL DEFAULT
|
|
185
|
+
x` -- matching what create_all builds on a fresh database in every
|
|
186
|
+
flag that changes what the schema permits (type, NOT NULL, pk). A
|
|
187
|
+
not-null column WITHOUT an expressible default (a default_factory:
|
|
188
|
+
uuid4, utcnow) cannot be added NOT NULL at all -- SQLite would have
|
|
189
|
+
to invent a value for every existing row -- so it lands nullable and
|
|
190
|
+
the divergence is named at WARNING rather than left silent.
|
|
191
|
+
|
|
192
|
+
One residual difference from a fresh database is inherent to the
|
|
193
|
+
mechanism and accepted: the migrated column carries a DEFAULT literal
|
|
194
|
+
while create_all emits none, because ALTER cannot backfill existing
|
|
195
|
+
rows without one and the ORM needs none. Only raw-SQL inserts into a
|
|
196
|
+
migrated database can observe the fill.
|
|
197
|
+
|
|
198
|
+
A column that exists with a DIFFERENT declared type is reported at
|
|
199
|
+
WARNING and left alone. It is deliberately not fatal: SQLite columns
|
|
200
|
+
have type affinity rather than enforcement, so a mismatch there
|
|
201
|
+
usually stores and reads exactly the same values -- and an additive
|
|
202
|
+
migrator cannot fix it either way. Refusing to start over a
|
|
203
|
+
cosmetic dialect difference would be a self-inflicted outage in the
|
|
204
|
+
one code path every launch runs through.
|
|
205
|
+
|
|
206
|
+
The two columns migrated before nullability was carried (`pinned`,
|
|
207
|
+
`kind`) exist as nullable on every database that predates #26, and
|
|
208
|
+
stay that way: repairing them would take a table rebuild, which is
|
|
209
|
+
outside this migration's additive contract, and no writer can put a
|
|
210
|
+
NULL there (the ORM always supplies its Python-side default), so the
|
|
211
|
+
residual risk is hand-written SQL only. Accepted and recorded in the
|
|
212
|
+
batch-18 decisions rather than papered over with machinery.
|
|
213
|
+
|
|
214
|
+
Does NOT close the connection -- the caller opened it and decides its
|
|
215
|
+
lifetime. In production that matters: engine.raw_connection() checks a
|
|
216
|
+
pooled connection out, and closing is how it goes back.
|
|
217
|
+
"""
|
|
218
|
+
added = []
|
|
219
|
+
cursor = connection.cursor()
|
|
220
|
+
for table, columns in declared.items():
|
|
221
|
+
cursor.execute(f"PRAGMA table_info({table})")
|
|
222
|
+
rows = cursor.fetchall()
|
|
223
|
+
if not rows:
|
|
224
|
+
continue
|
|
225
|
+
existing = {row[1]: row[2] for row in rows}
|
|
226
|
+
for name, (declared_type, literal, not_null) in columns.items():
|
|
227
|
+
if name in existing:
|
|
228
|
+
if _affinity(existing[name]) != _affinity(declared_type):
|
|
229
|
+
logger.warning(
|
|
230
|
+
"%s.%s exists as %s but is declared %s. Left "
|
|
231
|
+
"unchanged -- this migration is additive only "
|
|
232
|
+
"and cannot retype a column.",
|
|
233
|
+
table, name, existing[name], declared_type)
|
|
234
|
+
continue
|
|
235
|
+
ddl = f"ALTER TABLE {table} ADD COLUMN {name} {declared_type}"
|
|
236
|
+
if literal is not None:
|
|
237
|
+
if not_null:
|
|
238
|
+
ddl = f"{ddl} NOT NULL"
|
|
239
|
+
ddl = f"{ddl} DEFAULT {literal}"
|
|
240
|
+
elif not_null:
|
|
241
|
+
logger.warning(
|
|
242
|
+
"%s.%s is declared NOT NULL but its Python default "
|
|
243
|
+
"(default_factory) has no SQL spelling; added nullable. "
|
|
244
|
+
"SQLite cannot add a NOT NULL column without a default, "
|
|
245
|
+
"so a migrated database differs from a fresh one here "
|
|
246
|
+
"until a real migration exists.",
|
|
247
|
+
table, name)
|
|
248
|
+
cursor.execute(ddl)
|
|
249
|
+
added.append(f"{table}.{name}")
|
|
250
|
+
cursor.close()
|
|
251
|
+
connection.commit()
|
|
252
|
+
return added
|
package/env_secrets.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""
|
|
2
|
+
env_secrets.py
|
|
3
|
+
|
|
4
|
+
Miscellaneous API keys for individual TOOLS (e.g. a GitHub token, an
|
|
5
|
+
NVD/NIST API key) -- NOT LLM provider keys. Loaded from a .env file via
|
|
6
|
+
python-dotenv.
|
|
7
|
+
|
|
8
|
+
This is deliberately separate from credentials.py. The distinction:
|
|
9
|
+
|
|
10
|
+
credentials.py -- LLM PROVIDER keys (OpenAI, Anthropic, ...). Structured
|
|
11
|
+
per-provider (URL + key + compatibility flag), stored
|
|
12
|
+
in providers.json, read/written through
|
|
13
|
+
load_provider_data()/save_credentials().
|
|
14
|
+
|
|
15
|
+
env_secrets.py -- Flat, single-value secrets that one specific TOOL
|
|
16
|
+
needs to call some third-party API. No structure
|
|
17
|
+
beyond name -> value, stored in .env (never committed
|
|
18
|
+
-- see .gitignore), read through get_env_secret().
|
|
19
|
+
|
|
20
|
+
If you're adding a new tool that needs its own API key (a weather API, a
|
|
21
|
+
GitHub token, an NVD key, etc.), it belongs here, not in credentials.py or
|
|
22
|
+
providers.json -- those are exclusively for the LLM provider abstraction
|
|
23
|
+
in core/client.py.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import os
|
|
27
|
+
|
|
28
|
+
from dotenv import find_dotenv, load_dotenv
|
|
29
|
+
|
|
30
|
+
# Batch 35. Resolved at IMPORT time like credentials.LLM_PROVIDERS_FILE and
|
|
31
|
+
# for the same reason -- a test can monkeypatch the attribute, and a user can
|
|
32
|
+
# point it anywhere before launch.
|
|
33
|
+
#
|
|
34
|
+
# THE usecwd=True IS LOAD-BEARING, not a style choice. A bare load_dotenv()
|
|
35
|
+
# calls find_dotenv(usecwd=False), which walks up from THE CALLING FRAME'S
|
|
36
|
+
# FILE -- this module -- and not from the working directory. In a checkout
|
|
37
|
+
# those are the same directory, which is the only reason this has ever looked
|
|
38
|
+
# like cwd resolution. Install the harness anywhere else (npm puts it under
|
|
39
|
+
# node_modules/) and the walk starts in the wrong tree: it never reaches the
|
|
40
|
+
# user's .env, and it can pick up a stray one sitting above the install
|
|
41
|
+
# prefix. Measured before the fix, with a .env in the working directory and
|
|
42
|
+
# the harness imported from elsewhere: get_env_secret() saw None.
|
|
43
|
+
#
|
|
44
|
+
# Do not "simplify" this back to load_dotenv(). The defect is invisible from
|
|
45
|
+
# a checkout and presents as a tool reporting a missing key that is sitting
|
|
46
|
+
# in the .env right next to you.
|
|
47
|
+
#
|
|
48
|
+
# AGENT_ENV_FILE is the escape hatch, named to match AGENT_PROVIDERS_FILE.
|
|
49
|
+
# There is deliberately NO user-level fallback to ~/.config/venastine/ here,
|
|
50
|
+
# though providers.json has one: a provider key is machine-wide by nature,
|
|
51
|
+
# while .env holds the per-project tool tokens described above, and the two
|
|
52
|
+
# stores are kept apart on purpose. find_dotenv returns "" when it finds
|
|
53
|
+
# nothing, and load_dotenv("") is a no-op, so the unconfigured case is still
|
|
54
|
+
# safe to import.
|
|
55
|
+
ENV_FILE = os.environ.get("AGENT_ENV_FILE")
|
|
56
|
+
|
|
57
|
+
load_dotenv(ENV_FILE or find_dotenv(usecwd=True))
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_env_secret(key: str, required: bool = True) -> str:
|
|
61
|
+
"""
|
|
62
|
+
Fetch a misc API key/secret populated from .env.
|
|
63
|
+
|
|
64
|
+
Raises a clear, actionable error if a REQUIRED secret is missing,
|
|
65
|
+
rather than letting a tool proceed with an empty string and fail
|
|
66
|
+
later with a confusing 401 from some third-party API. Pass
|
|
67
|
+
required=False for genuinely optional keys (e.g. a key that only
|
|
68
|
+
unlocks a higher rate limit but isn't needed for the tool to work
|
|
69
|
+
at all).
|
|
70
|
+
"""
|
|
71
|
+
value = os.environ.get(key)
|
|
72
|
+
if required and not value:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"Missing required environment variable '{key}'. "
|
|
75
|
+
f"Copy .env.example to .env and fill in {key}."
|
|
76
|
+
)
|
|
77
|
+
return value or ""
|
package/json_store.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
"""
|
|
2
|
+
json_store.py
|
|
3
|
+
|
|
4
|
+
The mechanics four user-tier JSON stores had four copies of.
|
|
5
|
+
|
|
6
|
+
WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
|
|
7
|
+
written -- the temp-file-plus-replace dance, the version guard, the
|
|
8
|
+
fail-soft read -- and nothing about what any of them MEAN. Each store keeps
|
|
9
|
+
its own module, its own path, its own STORE_VERSION and its own posture,
|
|
10
|
+
because those are the parts that were decided separately and are argued at
|
|
11
|
+
length where they live:
|
|
12
|
+
|
|
13
|
+
core/workspace_trust.py trusted_projects.json fails CLOSED, 0600,
|
|
14
|
+
unversioned (a flat path -> hash mapping)
|
|
15
|
+
mcp_client/config.py known_mcp_servers.json versioned, with F3's
|
|
16
|
+
legacy-v1 re-ask arm
|
|
17
|
+
core/model_windows.py model_windows.json versioned, fail soft
|
|
18
|
+
tui/preferences.py ui_preferences.json versioned, fail soft,
|
|
19
|
+
read-modify-write (RM4)
|
|
20
|
+
|
|
21
|
+
WHY IT IS AT THE ROOT. `core/model_windows.py` is read by
|
|
22
|
+
`core/compaction.py`; `tui/preferences.py` states that it imports nothing
|
|
23
|
+
from `core/`; `mcp_client/` and `security/` each document a reason not to
|
|
24
|
+
reach into `core/` either. A root module is the one place all four can
|
|
25
|
+
already see -- `config`, `storage`, `database` and `credentials` are
|
|
26
|
+
imported from everywhere for the same reason -- so sharing this costs no
|
|
27
|
+
new edge in any direction.
|
|
28
|
+
|
|
29
|
+
WHAT THE COPIES HAD ALREADY DRIFTED INTO. Three of the four wrote through a
|
|
30
|
+
temp file and `os.replace`, each spending four to eight docstring lines on
|
|
31
|
+
why a truncate-then-write is unacceptable: it leaves the file empty for the
|
|
32
|
+
length of the write and permanently damaged if the process dies inside it.
|
|
33
|
+
The fourth -- `mcp_client`'s, which stores a record of CONSENT, the same
|
|
34
|
+
class of data as the trust store -- wrote with a bare `open(path, "w")` and
|
|
35
|
+
said nothing. That is what a comment-enforced convention decays into, and
|
|
36
|
+
extracting the write is what makes it stop being a convention.
|
|
37
|
+
|
|
38
|
+
ERRORS BELONG TO THE CALLER. `write_json_atomic` raises; it does not decide
|
|
39
|
+
whether a failed write is a WARNING the user should see (model_windows,
|
|
40
|
+
preferences: yes, and return False) or an exception (workspace_trust: yes,
|
|
41
|
+
because a trust grant that silently did not persist is worse than a crash).
|
|
42
|
+
Each caller keeps the policy it already had.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
from __future__ import annotations
|
|
46
|
+
|
|
47
|
+
import json
|
|
48
|
+
import logging
|
|
49
|
+
import os
|
|
50
|
+
from typing import Optional
|
|
51
|
+
|
|
52
|
+
logger = logging.getLogger(__name__)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
|
|
56
|
+
indent: int = 2, sort_keys: bool = False,
|
|
57
|
+
trailing_newline: bool = False) -> None:
|
|
58
|
+
"""Write `payload` as JSON to `path`, atomically. Raises OSError.
|
|
59
|
+
|
|
60
|
+
Temp file plus `os.replace`, which is atomic on POSIX and on Windows.
|
|
61
|
+
Parent directories are created.
|
|
62
|
+
|
|
63
|
+
`mode` sets permissions on the TEMP FILE, not on the destination (#19).
|
|
64
|
+
`os.replace` carries the source file's mode across, so setting it
|
|
65
|
+
afterwards would be both a no-op and a lie, and setting it on the
|
|
66
|
+
destination beforehand is worse -- replace overwrites that too. Only
|
|
67
|
+
the trust store passes one today, and its docstring says why it is for
|
|
68
|
+
consistency with credentials.py rather than because the file holds a
|
|
69
|
+
secret.
|
|
70
|
+
|
|
71
|
+
`trailing_newline` is off by default so every existing store writes
|
|
72
|
+
the same bytes it always did. The four here are machine state nobody
|
|
73
|
+
opens; §24's `/init --config` scaffold is the first file written
|
|
74
|
+
through this that a human is meant to edit and COMMIT, and a
|
|
75
|
+
committed file with no final newline carries git's "No newline at end
|
|
76
|
+
of file" marker in every diff of it, forever after.
|
|
77
|
+
"""
|
|
78
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
79
|
+
tmp = f"{path}.tmp"
|
|
80
|
+
|
|
81
|
+
def _dump(handle) -> None:
|
|
82
|
+
json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
|
|
83
|
+
if trailing_newline:
|
|
84
|
+
handle.write("\n")
|
|
85
|
+
|
|
86
|
+
if mode is None:
|
|
87
|
+
with open(tmp, "w", encoding="utf-8") as f:
|
|
88
|
+
_dump(f)
|
|
89
|
+
else:
|
|
90
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
|
91
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
92
|
+
_dump(f)
|
|
93
|
+
os.replace(tmp, path)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
|
|
97
|
+
"""The store at `path` if this build can use it, `{}` otherwise.
|
|
98
|
+
|
|
99
|
+
`{}` covers every unhappy case -- a missing file, an unreadable one, a
|
|
100
|
+
non-object, and a version this build does not know. Fail SOFT, and that
|
|
101
|
+
is a property of the callers this serves rather than of JSON: forgetting
|
|
102
|
+
a remembered window or a remembered theme costs one launch on the
|
|
103
|
+
derived value, and there is no security property to preserve. The trust
|
|
104
|
+
store also fails soft in the *sense* that a damaged file reads as empty,
|
|
105
|
+
but empty means "nothing is trusted" there, so it is fail-CLOSED and
|
|
106
|
+
keeps its own reader.
|
|
107
|
+
|
|
108
|
+
`fallback_note` is the caller's own clause for the debug line, so
|
|
109
|
+
"using derived context windows" and "starting on the configured
|
|
110
|
+
preferences" survive being one function.
|
|
111
|
+
"""
|
|
112
|
+
if not os.path.exists(path):
|
|
113
|
+
return {}
|
|
114
|
+
try:
|
|
115
|
+
with open(path, "r", encoding="utf-8-sig") as f:
|
|
116
|
+
data = json.load(f)
|
|
117
|
+
except (OSError, ValueError):
|
|
118
|
+
logger.debug("Could not read %s; %s", path,
|
|
119
|
+
fallback_note or "falling back to defaults.")
|
|
120
|
+
return {}
|
|
121
|
+
if not isinstance(data, dict) or data.get("version") != version:
|
|
122
|
+
return {}
|
|
123
|
+
return data
|