@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
package/credentials.py ADDED
@@ -0,0 +1,180 @@
1
+ import json
2
+ import os
3
+
4
+ # #24. Resolved at IMPORT time, exactly like its four siblings --
5
+ # config.DB_PATH / OUTPUT_DIR / WORKSPACE_DIR and logging_setup's log
6
+ # file are all `os.environ.get(NAME, default)` evaluated once -- so a
7
+ # test can still redirect it by monkeypatching the attribute, and a
8
+ # user can point it anywhere with AGENT_PROVIDERS_FILE before launch.
9
+ # The default stays CWD-relative on purpose: for a repo-local tool,
10
+ # per-checkout state is what you want, and repo-relative would share
11
+ # one key file across every clone on the machine.
12
+ LLM_PROVIDERS_FILE = os.environ.get("AGENT_PROVIDERS_FILE", "providers.json")
13
+
14
+
15
+ def no_providers_message() -> str:
16
+ """The error for "the file itself is not there" (#24).
17
+
18
+ ONE wording, used by both raisers -- api_initialization (the one
19
+ every user actually meets) and load_credentials below -- so the two
20
+ cannot drift apart the way their old messages already had.
21
+
22
+ Batch 35: the LOCATION clause is conditional now, because the path is
23
+ no longer always cwd-relative. AGENT_PROVIDERS_FILE has always accepted
24
+ an absolute path, and the npm launcher sets one when the working
25
+ directory has no providers.json of its own. Naming os.getcwd()
26
+ unconditionally then reported a directory the code never looked in and
27
+ sent the user to create the file in the wrong place -- the exact class
28
+ of defect #24 was filed about, reintroduced by a path it did not
29
+ anticipate. An absolute path is already the whole answer and needs no
30
+ second location.
31
+ """
32
+ if os.path.isabs(LLM_PROVIDERS_FILE):
33
+ found = f"No providers configured: {LLM_PROVIDERS_FILE} was not found."
34
+ else:
35
+ found = (
36
+ f"No providers configured: {LLM_PROVIDERS_FILE} was not found in "
37
+ f"{os.getcwd()!r}."
38
+ )
39
+ return (
40
+ f"{found} Copy providers.json.example to "
41
+ f"{LLM_PROVIDERS_FILE} and add your API key."
42
+ )
43
+
44
+
45
+ def unknown_provider_message(provider_name: str, providers: dict) -> str:
46
+ """The error for "the file is there, this provider is not in it" (#24).
47
+
48
+ Names the configured set exactly like /model's refusal does, so the
49
+ typo is answered where it was made rather than two layers later.
50
+ """
51
+ configured = ", ".join(sorted(providers)) or "none"
52
+ return (
53
+ f"Unknown provider: {provider_name}. Configured: {configured} "
54
+ f"(file: {LLM_PROVIDERS_FILE})."
55
+ )
56
+
57
+
58
+ def load_provider_data() -> dict:
59
+ if not os.path.exists(LLM_PROVIDERS_FILE):
60
+ return {}
61
+ with open(LLM_PROVIDERS_FILE, "r", encoding="utf-8") as file:
62
+ return json.load(file)
63
+
64
+
65
+ # 0600 -- owner read/write, nobody else. This file holds every LLM
66
+ # provider's API key in cleartext, and a bare open(path, "w") left its
67
+ # protection to the process umask: 0644 under the common default, i.e.
68
+ # readable by every other account on the machine (#19).
69
+ #
70
+ # The blast radius of a credential file is not the file; it is the paid
71
+ # API account behind it. Every comparable tool treats one as 0600
72
+ # (~/.aws/credentials, ~/.docker/config.json, ~/.ssh/id_*), and OpenSSH
73
+ # goes further and REFUSES a group-readable private key.
74
+ _SECRET_FILE_MODE = 0o600
75
+
76
+
77
+ def _write_secret_json(path: str, data: dict) -> None:
78
+ """Write JSON to a file created 0600, never world-readable even briefly.
79
+
80
+ os.open with the mode, NOT open() followed by os.chmod. A post-write
81
+ chmod leaves a window in which the key is on disk with the umask's
82
+ permissions -- short, but it is exactly the window an attacker with
83
+ local read access is waiting for, and closing it costs one line.
84
+
85
+ WINDOWS: POSIX mode bits do not apply, and os.open's mode argument
86
+ only controls the read-only flag there. This is a no-op on Windows
87
+ rather than a wrong-op; the file's protection comes from the ACL on
88
+ the user profile directory instead. The tests skip accordingly, and
89
+ the CI container is where the assertion actually runs.
90
+ """
91
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC,
92
+ _SECRET_FILE_MODE)
93
+ with os.fdopen(fd, "w", encoding="utf-8") as file:
94
+ json.dump(data, file, indent=2)
95
+
96
+
97
+ def _write_provider_data(provider_data: dict) -> None:
98
+ _write_secret_json(LLM_PROVIDERS_FILE, provider_data)
99
+
100
+
101
+ def save_credentials(
102
+ provider_name: str,
103
+ api_key: str,
104
+ api_url: str = "",
105
+ is_v1_compatible: bool = True,
106
+ supports_stream_usage: bool = False,
107
+ ) -> None:
108
+ provider_data = load_provider_data()
109
+
110
+ if provider_name in provider_data:
111
+ provider_data[provider_name]["API_KEY"] = api_key
112
+ if api_url:
113
+ provider_data[provider_name]["API_URL"] = api_url
114
+ provider_data[provider_name]["is_v1_compatible"] = is_v1_compatible
115
+ provider_data[provider_name]["supports_stream_usage"] = supports_stream_usage
116
+ else:
117
+ provider_data[provider_name] = {
118
+ "API_KEY": api_key,
119
+ "API_URL": api_url,
120
+ "is_v1_compatible": is_v1_compatible,
121
+ "supports_stream_usage": supports_stream_usage,
122
+ }
123
+
124
+ _write_provider_data(provider_data)
125
+
126
+
127
+ def load_credentials(provider_name: str) -> tuple[str, str]:
128
+ provider_data = load_provider_data()
129
+ # #24. The two failure modes say different things now -- "the file is
130
+ # not there" is a setup problem with a one-line remedy; "the provider
131
+ # is not in it" is a typo. Same builders api_initialization uses.
132
+ if not provider_data and not os.path.exists(LLM_PROVIDERS_FILE):
133
+ raise ValueError(no_providers_message())
134
+ if provider_name not in provider_data:
135
+ raise ValueError(unknown_provider_message(provider_name, provider_data))
136
+
137
+ entry = provider_data[provider_name]
138
+ return entry["API_KEY"], entry.get("API_URL", "")
139
+
140
+
141
+ def missing_key_message(provider_name: str) -> str:
142
+ """The one wording for "configured, but with an empty API_KEY".
143
+
144
+ Extracted in batch 44 for the reason no_providers_message() was: /model
145
+ carried its own copy, naming providers.json literally where this file
146
+ names LLM_PROVIDERS_FILE, so the same fact read differently at launch
147
+ and at a switch -- while tui/app.py's comment beside the launch render
148
+ already claimed they were "the same fact at both moments". WARN-only at
149
+ both call sites: a local OpenAI-compatible endpoint legitimately takes
150
+ no key, and refusing would block a real configuration.
151
+ """
152
+ return (f"{provider_name} has no API_KEY in {LLM_PROVIDERS_FILE} "
153
+ f"— calls will fail unless it is a local endpoint that "
154
+ f"needs none.")
155
+
156
+
157
+ def provider_startup_issues(provider_name: str) -> list:
158
+ """What launch should say about the provider setup, before either
159
+ shell starts (#138).
160
+
161
+ The same three checks /model performs on a switch -- file present,
162
+ provider configured, key non-empty -- applied at the moment the user
163
+ chose the provider, instead of at the first model call, which is a
164
+ dead end away from the mistake that caused it. WARN-ONLY, never a
165
+ refusal: an OpenAI-compatible endpoint running locally legitimately
166
+ takes no key (the reason /model warns there too), and refusing would
167
+ block a real configuration.
168
+
169
+ Empty list means healthy and says NOTHING -- silence is the healthy
170
+ case's whole UX. Plain sentences, no prefix: rendering is the
171
+ shell's (CLI prints, TUI write_error), so one wording reaches both.
172
+ """
173
+ providers = load_provider_data()
174
+ if not providers and not os.path.exists(LLM_PROVIDERS_FILE):
175
+ return [no_providers_message()]
176
+ if provider_name not in providers:
177
+ return [unknown_provider_message(provider_name, providers)]
178
+ if not providers[provider_name].get("API_KEY"):
179
+ return [missing_key_message(provider_name)]
180
+ return []
package/database.py ADDED
@@ -0,0 +1,252 @@
1
+ """
2
+ database.py
3
+
4
+ Owns the actual database connection -- the engine, where the DB file
5
+ lives, and how tables get created. storage.py imports `engine` from here;
6
+ nothing here knows about ConversationThread, MessageLog, or any other
7
+ table schema.
8
+ """
9
+
10
+ import logging
11
+
12
+ from sqlmodel import create_engine, SQLModel
13
+
14
+ import config
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ DATABASE_URL = f"sqlite:///{config.DB_PATH}"
19
+
20
+ engine = create_engine(
21
+ DATABASE_URL,
22
+ echo=False,
23
+ connect_args={"check_same_thread": False},
24
+ )
25
+
26
+
27
+ def create_db_and_tables() -> None:
28
+ """Creates any table that doesn't exist yet. Safe to call every startup.
29
+
30
+ IMPORTANT: this creates only the tables REGISTERED ON SQLModel.metadata
31
+ when it runs, and a table class registers when its module is imported.
32
+ This file knows nothing about which tables exist (§4.4), so the caller
33
+ must have imported the modules defining them first -- see main.py, which
34
+ imports storage and pipeline_storage explicitly for that side effect.
35
+
36
+ Getting this wrong is silent: create_all() happily creates nothing and
37
+ returns, and the failure surfaces much later as "no such table" at the
38
+ first write. That is exactly how a fresh database ended up with no
39
+ pipelinerunrecord table while research mode looked fine on an existing
40
+ one. The guard below turns the total-failure case into an immediate,
41
+ named error; a partially-registered set can't be detected from here
42
+ without this file learning what data exists, which is the boundary §4.4
43
+ exists to hold.
44
+ """
45
+ if not SQLModel.metadata.tables:
46
+ raise RuntimeError(
47
+ "create_db_and_tables() found no table classes registered. Import "
48
+ "the modules that define them (storage, core.reasoning.pipeline_storage) "
49
+ "before calling this, or the database will be created empty."
50
+ )
51
+ SQLModel.metadata.create_all(engine)
52
+ # AFTER create_all, deliberately: a brand-new database gets every table
53
+ # built complete and needs no ALTER at all. This pass exists only for
54
+ # databases that predate a column.
55
+ connection = engine.raw_connection()
56
+ try:
57
+ added = ensure_columns(connection, _declared_columns())
58
+ finally:
59
+ connection.close()
60
+ if added:
61
+ logger.info("Added missing columns to an existing database: %s",
62
+ ", ".join(added))
63
+ logger.debug("Ensured tables exist: %s", sorted(SQLModel.metadata.tables))
64
+
65
+
66
+ # ---------------------------------------------------------------------------
67
+ # ---- Additive column migration (ROADMAP_v2 §21, M7) -----------------------
68
+ # ---------------------------------------------------------------------------
69
+ #
70
+ # create_all() creates missing TABLES and never touches an existing one, so
71
+ # adding a field to a table class that is already on disk produces "no such
72
+ # column" on the next SELECT -- for every database that predates the change,
73
+ # silently, at read time rather than at startup. §21's `MessageLog.pinned` is
74
+ # the first field this project has ever added to a shipped table; it will not
75
+ # be the last.
76
+ #
77
+ # ADDITIVE ONLY, and that is the whole contract. This will add a column that
78
+ # is declared and missing. It will never rename, drop, retype, backfill, or
79
+ # reorder, and it does not track a schema version. Anything beyond "a new
80
+ # nullable-or-defaulted column appeared" needs a real migration and should
81
+ # not be smuggled in here -- if this file ever grows a `DROP`, that is the
82
+ # signal the project has outgrown it.
83
+ #
84
+ # Nullability rides along where SQLite allows it (#26): `NOT NULL DEFAULT x`
85
+ # is expressible in one ALTER and matches create_all exactly; a not-null
86
+ # column with no SQL-spellable default cannot be added NOT NULL at all and
87
+ # lands nullable under a named WARNING. The two columns migrated before this
88
+ # (`pinned`, `kind`) remain nullable on databases that predate #26 -- accepted,
89
+ # not repaired: flipping them needs a table rebuild, which is outside this
90
+ # contract, and no writer can produce a NULL for them.
91
+
92
+
93
+ def _default_literal(column):
94
+ """A SQL literal for the column's default, or None if it has none we
95
+ can express.
96
+
97
+ SQLite fills existing rows with NULL when a column is added, so a
98
+ column the Python side declares non-optional would read back as None
99
+ on every pre-existing row. Emitting DEFAULT closes that: `pinned`
100
+ arrives as 0 on rows written before pinning existed, which is what
101
+ "this message was never pinned" should mean.
102
+
103
+ Only scalar defaults qualify. A default_factory (uuid4, utcnow) is
104
+ per-row Python and has no SQL spelling, so those columns are added
105
+ nullable and left to the application -- which is correct, because a
106
+ factory-defaulted column on an existing row has no right answer.
107
+ """
108
+ default = getattr(column, "default", None)
109
+ if default is None or getattr(default, "is_callable", False):
110
+ return None
111
+ value = getattr(default, "arg", None)
112
+ if value is None or callable(value):
113
+ return None
114
+ if isinstance(value, bool):
115
+ return "1" if value else "0"
116
+ if isinstance(value, (int, float)):
117
+ return str(value)
118
+ if isinstance(value, str):
119
+ escaped = value.replace("'", "''")
120
+ return f"'{escaped}'"
121
+ return None
122
+
123
+
124
+ def _declared_columns() -> dict:
125
+ """table name -> {column name: (sql type, default literal or None,
126
+ not-null flag)}, read off whatever SQLModel currently has registered.
127
+
128
+ This keeps §4.4's boundary intact: the file still knows nothing about
129
+ WHICH tables exist, it asks the metadata -- the same source create_all
130
+ above is already driven by. A table class that was never imported is
131
+ invisible here exactly as it is there.
132
+
133
+ Everything SQLAlchemy-shaped is resolved HERE, into plain strings, so
134
+ ensure_columns() below is pure SQL over a DBAPI connection with no
135
+ ORM knowledge at all. That is what makes it testable against stdlib
136
+ sqlite3 while the suite's fake `sqlmodel` is installed -- the fake's
137
+ engine has no dialect to compile a type against, and a seam that only
138
+ works in production is not a seam.
139
+
140
+ The third slot is the nullability the model declares (#26): SQLModel
141
+ derives it from the annotation, so `pinned: bool` arrives not-null
142
+ and `name: Optional[str]` does not. Without carrying it, every column
143
+ this migration adds lands nullable where create_all would have built
144
+ NOT NULL -- a migrated database silently diverging from a fresh one.
145
+ """
146
+ dialect = engine.dialect
147
+ return {
148
+ name: {
149
+ c.name: (
150
+ c.type.compile(dialect),
151
+ _default_literal(c),
152
+ not bool(getattr(c, "nullable", True)),
153
+ )
154
+ for c in table.columns
155
+ }
156
+ for name, table in SQLModel.metadata.tables.items()
157
+ }
158
+
159
+
160
+ def _affinity(sql_type: str) -> str:
161
+ """Normalized type name for comparison -- uppercased, with any length
162
+ or precision specifier dropped. VARCHAR(255) and varchar compare
163
+ equal; BOOLEAN and VARCHAR do not."""
164
+ return sql_type.split("(")[0].strip().upper()
165
+
166
+
167
+ def ensure_columns(connection, declared: dict) -> list:
168
+ """Add every declared column missing from an existing table.
169
+
170
+ `declared` is _declared_columns()' plain-string shape --
171
+ table -> {column: (sql type, default literal or None, not-null flag)}.
172
+ `connection` is a DBAPI connection (`engine.raw_connection()` in
173
+ production, a plain `sqlite3.connect(...)` in tests) rather than a
174
+ SQLAlchemy Connection. Between them this function touches no ORM at
175
+ all, which is what makes it testable against stdlib sqlite3 while the
176
+ suite's fake `sqlmodel` is installed. Returns the "table.column" names
177
+ added, for logging.
178
+
179
+ A table with no rows in `PRAGMA table_info` does not exist yet;
180
+ create_all built it or will, so there is nothing to migrate.
181
+
182
+ Nullability is carried where SQLite can express it (#26). ALTER TABLE
183
+ ADD COLUMN accepts NOT NULL only together with a non-NULL DEFAULT, so
184
+ a not-null column WITH a scalar default migrates as `NOT NULL DEFAULT
185
+ x` -- matching what create_all builds on a fresh database in every
186
+ flag that changes what the schema permits (type, NOT NULL, pk). A
187
+ not-null column WITHOUT an expressible default (a default_factory:
188
+ uuid4, utcnow) cannot be added NOT NULL at all -- SQLite would have
189
+ to invent a value for every existing row -- so it lands nullable and
190
+ the divergence is named at WARNING rather than left silent.
191
+
192
+ One residual difference from a fresh database is inherent to the
193
+ mechanism and accepted: the migrated column carries a DEFAULT literal
194
+ while create_all emits none, because ALTER cannot backfill existing
195
+ rows without one and the ORM needs none. Only raw-SQL inserts into a
196
+ migrated database can observe the fill.
197
+
198
+ A column that exists with a DIFFERENT declared type is reported at
199
+ WARNING and left alone. It is deliberately not fatal: SQLite columns
200
+ have type affinity rather than enforcement, so a mismatch there
201
+ usually stores and reads exactly the same values -- and an additive
202
+ migrator cannot fix it either way. Refusing to start over a
203
+ cosmetic dialect difference would be a self-inflicted outage in the
204
+ one code path every launch runs through.
205
+
206
+ The two columns migrated before nullability was carried (`pinned`,
207
+ `kind`) exist as nullable on every database that predates #26, and
208
+ stay that way: repairing them would take a table rebuild, which is
209
+ outside this migration's additive contract, and no writer can put a
210
+ NULL there (the ORM always supplies its Python-side default), so the
211
+ residual risk is hand-written SQL only. Accepted and recorded in the
212
+ batch-18 decisions rather than papered over with machinery.
213
+
214
+ Does NOT close the connection -- the caller opened it and decides its
215
+ lifetime. In production that matters: engine.raw_connection() checks a
216
+ pooled connection out, and closing is how it goes back.
217
+ """
218
+ added = []
219
+ cursor = connection.cursor()
220
+ for table, columns in declared.items():
221
+ cursor.execute(f"PRAGMA table_info({table})")
222
+ rows = cursor.fetchall()
223
+ if not rows:
224
+ continue
225
+ existing = {row[1]: row[2] for row in rows}
226
+ for name, (declared_type, literal, not_null) in columns.items():
227
+ if name in existing:
228
+ if _affinity(existing[name]) != _affinity(declared_type):
229
+ logger.warning(
230
+ "%s.%s exists as %s but is declared %s. Left "
231
+ "unchanged -- this migration is additive only "
232
+ "and cannot retype a column.",
233
+ table, name, existing[name], declared_type)
234
+ continue
235
+ ddl = f"ALTER TABLE {table} ADD COLUMN {name} {declared_type}"
236
+ if literal is not None:
237
+ if not_null:
238
+ ddl = f"{ddl} NOT NULL"
239
+ ddl = f"{ddl} DEFAULT {literal}"
240
+ elif not_null:
241
+ logger.warning(
242
+ "%s.%s is declared NOT NULL but its Python default "
243
+ "(default_factory) has no SQL spelling; added nullable. "
244
+ "SQLite cannot add a NOT NULL column without a default, "
245
+ "so a migrated database differs from a fresh one here "
246
+ "until a real migration exists.",
247
+ table, name)
248
+ cursor.execute(ddl)
249
+ added.append(f"{table}.{name}")
250
+ cursor.close()
251
+ connection.commit()
252
+ return added
package/env_secrets.py ADDED
@@ -0,0 +1,77 @@
1
+ """
2
+ env_secrets.py
3
+
4
+ Miscellaneous API keys for individual TOOLS (e.g. a GitHub token, an
5
+ NVD/NIST API key) -- NOT LLM provider keys. Loaded from a .env file via
6
+ python-dotenv.
7
+
8
+ This is deliberately separate from credentials.py. The distinction:
9
+
10
+ credentials.py -- LLM PROVIDER keys (OpenAI, Anthropic, ...). Structured
11
+ per-provider (URL + key + compatibility flag), stored
12
+ in providers.json, read/written through
13
+ load_provider_data()/save_credentials().
14
+
15
+ env_secrets.py -- Flat, single-value secrets that one specific TOOL
16
+ needs to call some third-party API. No structure
17
+ beyond name -> value, stored in .env (never committed
18
+ -- see .gitignore), read through get_env_secret().
19
+
20
+ If you're adding a new tool that needs its own API key (a weather API, a
21
+ GitHub token, an NVD key, etc.), it belongs here, not in credentials.py or
22
+ providers.json -- those are exclusively for the LLM provider abstraction
23
+ in core/client.py.
24
+ """
25
+
26
+ import os
27
+
28
+ from dotenv import find_dotenv, load_dotenv
29
+
30
+ # Batch 35. Resolved at IMPORT time like credentials.LLM_PROVIDERS_FILE and
31
+ # for the same reason -- a test can monkeypatch the attribute, and a user can
32
+ # point it anywhere before launch.
33
+ #
34
+ # THE usecwd=True IS LOAD-BEARING, not a style choice. A bare load_dotenv()
35
+ # calls find_dotenv(usecwd=False), which walks up from THE CALLING FRAME'S
36
+ # FILE -- this module -- and not from the working directory. In a checkout
37
+ # those are the same directory, which is the only reason this has ever looked
38
+ # like cwd resolution. Install the harness anywhere else (npm puts it under
39
+ # node_modules/) and the walk starts in the wrong tree: it never reaches the
40
+ # user's .env, and it can pick up a stray one sitting above the install
41
+ # prefix. Measured before the fix, with a .env in the working directory and
42
+ # the harness imported from elsewhere: get_env_secret() saw None.
43
+ #
44
+ # Do not "simplify" this back to load_dotenv(). The defect is invisible from
45
+ # a checkout and presents as a tool reporting a missing key that is sitting
46
+ # in the .env right next to you.
47
+ #
48
+ # AGENT_ENV_FILE is the escape hatch, named to match AGENT_PROVIDERS_FILE.
49
+ # There is deliberately NO user-level fallback to ~/.config/venastine/ here,
50
+ # though providers.json has one: a provider key is machine-wide by nature,
51
+ # while .env holds the per-project tool tokens described above, and the two
52
+ # stores are kept apart on purpose. find_dotenv returns "" when it finds
53
+ # nothing, and load_dotenv("") is a no-op, so the unconfigured case is still
54
+ # safe to import.
55
+ ENV_FILE = os.environ.get("AGENT_ENV_FILE")
56
+
57
+ load_dotenv(ENV_FILE or find_dotenv(usecwd=True))
58
+
59
+
60
+ def get_env_secret(key: str, required: bool = True) -> str:
61
+ """
62
+ Fetch a misc API key/secret populated from .env.
63
+
64
+ Raises a clear, actionable error if a REQUIRED secret is missing,
65
+ rather than letting a tool proceed with an empty string and fail
66
+ later with a confusing 401 from some third-party API. Pass
67
+ required=False for genuinely optional keys (e.g. a key that only
68
+ unlocks a higher rate limit but isn't needed for the tool to work
69
+ at all).
70
+ """
71
+ value = os.environ.get(key)
72
+ if required and not value:
73
+ raise ValueError(
74
+ f"Missing required environment variable '{key}'. "
75
+ f"Copy .env.example to .env and fill in {key}."
76
+ )
77
+ return value or ""
package/json_store.py ADDED
@@ -0,0 +1,123 @@
1
+ """
2
+ json_store.py
3
+
4
+ The mechanics four user-tier JSON stores had four copies of.
5
+
6
+ WHAT THIS IS AND IS NOT. It owns *how* a small JSON file is read and
7
+ written -- the temp-file-plus-replace dance, the version guard, the
8
+ fail-soft read -- and nothing about what any of them MEAN. Each store keeps
9
+ its own module, its own path, its own STORE_VERSION and its own posture,
10
+ because those are the parts that were decided separately and are argued at
11
+ length where they live:
12
+
13
+ core/workspace_trust.py trusted_projects.json fails CLOSED, 0600,
14
+ unversioned (a flat path -> hash mapping)
15
+ mcp_client/config.py known_mcp_servers.json versioned, with F3's
16
+ legacy-v1 re-ask arm
17
+ core/model_windows.py model_windows.json versioned, fail soft
18
+ tui/preferences.py ui_preferences.json versioned, fail soft,
19
+ read-modify-write (RM4)
20
+
21
+ WHY IT IS AT THE ROOT. `core/model_windows.py` is read by
22
+ `core/compaction.py`; `tui/preferences.py` states that it imports nothing
23
+ from `core/`; `mcp_client/` and `security/` each document a reason not to
24
+ reach into `core/` either. A root module is the one place all four can
25
+ already see -- `config`, `storage`, `database` and `credentials` are
26
+ imported from everywhere for the same reason -- so sharing this costs no
27
+ new edge in any direction.
28
+
29
+ WHAT THE COPIES HAD ALREADY DRIFTED INTO. Three of the four wrote through a
30
+ temp file and `os.replace`, each spending four to eight docstring lines on
31
+ why a truncate-then-write is unacceptable: it leaves the file empty for the
32
+ length of the write and permanently damaged if the process dies inside it.
33
+ The fourth -- `mcp_client`'s, which stores a record of CONSENT, the same
34
+ class of data as the trust store -- wrote with a bare `open(path, "w")` and
35
+ said nothing. That is what a comment-enforced convention decays into, and
36
+ extracting the write is what makes it stop being a convention.
37
+
38
+ ERRORS BELONG TO THE CALLER. `write_json_atomic` raises; it does not decide
39
+ whether a failed write is a WARNING the user should see (model_windows,
40
+ preferences: yes, and return False) or an exception (workspace_trust: yes,
41
+ because a trust grant that silently did not persist is worse than a crash).
42
+ Each caller keeps the policy it already had.
43
+ """
44
+
45
+ from __future__ import annotations
46
+
47
+ import json
48
+ import logging
49
+ import os
50
+ from typing import Optional
51
+
52
+ logger = logging.getLogger(__name__)
53
+
54
+
55
+ def write_json_atomic(path: str, payload, *, mode: Optional[int] = None,
56
+ indent: int = 2, sort_keys: bool = False,
57
+ trailing_newline: bool = False) -> None:
58
+ """Write `payload` as JSON to `path`, atomically. Raises OSError.
59
+
60
+ Temp file plus `os.replace`, which is atomic on POSIX and on Windows.
61
+ Parent directories are created.
62
+
63
+ `mode` sets permissions on the TEMP FILE, not on the destination (#19).
64
+ `os.replace` carries the source file's mode across, so setting it
65
+ afterwards would be both a no-op and a lie, and setting it on the
66
+ destination beforehand is worse -- replace overwrites that too. Only
67
+ the trust store passes one today, and its docstring says why it is for
68
+ consistency with credentials.py rather than because the file holds a
69
+ secret.
70
+
71
+ `trailing_newline` is off by default so every existing store writes
72
+ the same bytes it always did. The four here are machine state nobody
73
+ opens; §24's `/init --config` scaffold is the first file written
74
+ through this that a human is meant to edit and COMMIT, and a
75
+ committed file with no final newline carries git's "No newline at end
76
+ of file" marker in every diff of it, forever after.
77
+ """
78
+ os.makedirs(os.path.dirname(path), exist_ok=True)
79
+ tmp = f"{path}.tmp"
80
+
81
+ def _dump(handle) -> None:
82
+ json.dump(payload, handle, indent=indent, sort_keys=sort_keys)
83
+ if trailing_newline:
84
+ handle.write("\n")
85
+
86
+ if mode is None:
87
+ with open(tmp, "w", encoding="utf-8") as f:
88
+ _dump(f)
89
+ else:
90
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
91
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
92
+ _dump(f)
93
+ os.replace(tmp, path)
94
+
95
+
96
+ def read_versioned(path: str, version: int, fallback_note: str = "") -> dict:
97
+ """The store at `path` if this build can use it, `{}` otherwise.
98
+
99
+ `{}` covers every unhappy case -- a missing file, an unreadable one, a
100
+ non-object, and a version this build does not know. Fail SOFT, and that
101
+ is a property of the callers this serves rather than of JSON: forgetting
102
+ a remembered window or a remembered theme costs one launch on the
103
+ derived value, and there is no security property to preserve. The trust
104
+ store also fails soft in the *sense* that a damaged file reads as empty,
105
+ but empty means "nothing is trusted" there, so it is fail-CLOSED and
106
+ keeps its own reader.
107
+
108
+ `fallback_note` is the caller's own clause for the debug line, so
109
+ "using derived context windows" and "starting on the configured
110
+ preferences" survive being one function.
111
+ """
112
+ if not os.path.exists(path):
113
+ return {}
114
+ try:
115
+ with open(path, "r", encoding="utf-8-sig") as f:
116
+ data = json.load(f)
117
+ except (OSError, ValueError):
118
+ logger.debug("Could not read %s; %s", path,
119
+ fallback_note or "falling back to defaults.")
120
+ return {}
121
+ if not isinstance(data, dict) or data.get("version") != version:
122
+ return {}
123
+ return data