@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/database.py
CHANGED
|
@@ -1,252 +1,333 @@
|
|
|
1
|
-
"""
|
|
2
|
-
database.py
|
|
3
|
-
|
|
4
|
-
Owns the actual database connection -- the engine, where the DB file
|
|
5
|
-
lives, and how tables get created. storage.py imports `engine` from here;
|
|
6
|
-
nothing here knows about ConversationThread, MessageLog, or any other
|
|
7
|
-
table schema.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
|
-
import logging
|
|
11
|
-
|
|
12
|
-
from sqlmodel import
|
|
13
|
-
|
|
14
|
-
import config
|
|
15
|
-
|
|
16
|
-
logger = logging.getLogger(__name__)
|
|
17
|
-
|
|
18
|
-
DATABASE_URL = f"sqlite:///{config.DB_PATH}"
|
|
19
|
-
|
|
20
|
-
engine = create_engine(
|
|
21
|
-
DATABASE_URL,
|
|
22
|
-
echo=False,
|
|
23
|
-
connect_args={"check_same_thread": False},
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
#
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
#
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
"""
|
|
126
|
-
not
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
1
|
+
"""
|
|
2
|
+
database.py
|
|
3
|
+
|
|
4
|
+
Owns the actual database connection -- the engine, where the DB file
|
|
5
|
+
lives, and how tables get created. storage.py imports `engine` from here;
|
|
6
|
+
nothing here knows about ConversationThread, MessageLog, or any other
|
|
7
|
+
table schema.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
|
|
12
|
+
from sqlmodel import SQLModel, create_engine
|
|
13
|
+
|
|
14
|
+
import config
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
DATABASE_URL = f"sqlite:///{config.DB_PATH}"
|
|
19
|
+
|
|
20
|
+
engine = create_engine(
|
|
21
|
+
DATABASE_URL,
|
|
22
|
+
echo=False,
|
|
23
|
+
connect_args={"check_same_thread": False},
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
# ---------------------------------------------------------------------------
|
|
27
|
+
# ---- Concurrent writers (ROADMAP_v2 §47 slice 8) --------------------------
|
|
28
|
+
# ---------------------------------------------------------------------------
|
|
29
|
+
#
|
|
30
|
+
# `check_same_thread: False` has always been here, because the TUI drains the
|
|
31
|
+
# loop on a worker thread while the UI thread reads. What was never true
|
|
32
|
+
# until NA9 is that two writers could be in flight at once: one agent ran at
|
|
33
|
+
# a time, so the pool handed out one connection that mattered and SQLite's
|
|
34
|
+
# single-writer rule cost nothing.
|
|
35
|
+
#
|
|
36
|
+
# MEASURED, at the real engine against a real database, because this is the
|
|
37
|
+
# one hazard in the slice it would have been easiest to assume. Each shape is
|
|
38
|
+
# N threads creating a subagent thread and writing 200 messages to it, which
|
|
39
|
+
# is a child run's shape:
|
|
40
|
+
#
|
|
41
|
+
# writers journal=delete journal=WAL
|
|
42
|
+
# 3 0 errors, 2.94s 0 errors, 1.31s
|
|
43
|
+
# 8 1 "database is 0 errors, 3.21s
|
|
44
|
+
# locked", 6.78s
|
|
45
|
+
# 16 -- 1 "database is locked", 6.00s
|
|
46
|
+
#
|
|
47
|
+
# So at SUBAGENT_MAX_PARALLEL = 3 the shipped configuration does NOT lock,
|
|
48
|
+
# and WAL is enabled anyway for the two things the table shows: it removes
|
|
49
|
+
# the failure at 8, which is where a raised ceiling lands, and it is a little
|
|
50
|
+
# over twice as fast at 3, which is the configuration everyone runs. A
|
|
51
|
+
# concurrency ceiling that is a tunable constant invites being tuned, and the
|
|
52
|
+
# failure it walks into is a lost message rather than a slow one.
|
|
53
|
+
#
|
|
54
|
+
# THE BUSY TIMEOUT WAS ALREADY 5000ms and this does not change it. Measured
|
|
55
|
+
# too, via `PRAGMA busy_timeout`: SQLAlchemy's pysqlite dialect passes
|
|
56
|
+
# `timeout=5.0` to sqlite3.connect by default. It is restated here rather
|
|
57
|
+
# than left implicit because it is load-bearing for the numbers above, and a
|
|
58
|
+
# default nobody wrote down is a number that can change under us.
|
|
59
|
+
#
|
|
60
|
+
# ON CONNECT rather than once at import: `journal_mode` is persistent in the
|
|
61
|
+
# file but the pool opens connections over the life of the process and a
|
|
62
|
+
# fresh database file arrives with none of this, so the reliable place to
|
|
63
|
+
# state it is every connection. Contained and logged for the reason
|
|
64
|
+
# everything at this layer is: WAL needs shared memory, which some network
|
|
65
|
+
# filesystems do not provide, and a harness that refuses to start because it
|
|
66
|
+
# could not make its journal faster would be a self-inflicted outage.
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _apply_sqlite_pragmas(dbapi_connection, _record) -> None:
|
|
70
|
+
"""WAL and an explicit busy timeout, per connection. See above."""
|
|
71
|
+
try:
|
|
72
|
+
cursor = dbapi_connection.cursor()
|
|
73
|
+
try:
|
|
74
|
+
cursor.execute("PRAGMA journal_mode=WAL")
|
|
75
|
+
cursor.execute("PRAGMA busy_timeout=5000")
|
|
76
|
+
finally:
|
|
77
|
+
cursor.close()
|
|
78
|
+
except Exception: # noqa: BLE001 -- a slower journal is not an outage
|
|
79
|
+
logger.warning(
|
|
80
|
+
"Could not set journal_mode=WAL on this database; continuing on "
|
|
81
|
+
"whatever journal it has. Concurrent subagent writes may "
|
|
82
|
+
"contend. This is expected on filesystems without shared "
|
|
83
|
+
"memory support.", exc_info=True)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# Imported here rather than at the top so the listener registration reads as
|
|
87
|
+
# one unit with the reasoning above it.
|
|
88
|
+
from sqlalchemy import event as _sqlalchemy_event # noqa: E402
|
|
89
|
+
from sqlalchemy.engine import Engine as _SqlAlchemyEngine # noqa: E402
|
|
90
|
+
|
|
91
|
+
# ONLY ON A REAL ENGINE, and the condition is named rather than caught.
|
|
92
|
+
#
|
|
93
|
+
# The suite's root conftest stubs `sqlmodel` into sys.modules before
|
|
94
|
+
# collection, so `create_engine` above returns a plain namespace there and
|
|
95
|
+
# `listens_for` raises InvalidRequestError on it -- at import, which is a
|
|
96
|
+
# collection error for every test that touches storage. An `except` around
|
|
97
|
+
# the registration would fix that and would also swallow a genuine
|
|
98
|
+
# misregistration in production, silently, leaving the journal on `delete`
|
|
99
|
+
# with nothing said. An isinstance check cannot: a real engine always
|
|
100
|
+
# passes it, so a production failure still raises.
|
|
101
|
+
#
|
|
102
|
+
# Same seam `ensure_columns` already uses one function down, for the same
|
|
103
|
+
# stated reason -- a boundary that only works in production is not a seam.
|
|
104
|
+
if isinstance(engine, _SqlAlchemyEngine):
|
|
105
|
+
_sqlalchemy_event.listens_for(engine, "connect")(_apply_sqlite_pragmas)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def create_db_and_tables() -> None:
|
|
109
|
+
"""Creates any table that doesn't exist yet. Safe to call every startup.
|
|
110
|
+
|
|
111
|
+
IMPORTANT: this creates only the tables REGISTERED ON SQLModel.metadata
|
|
112
|
+
when it runs, and a table class registers when its module is imported.
|
|
113
|
+
This file knows nothing about which tables exist (§4.4), so the caller
|
|
114
|
+
must have imported the modules defining them first -- see main.py, which
|
|
115
|
+
imports storage and pipeline_storage explicitly for that side effect.
|
|
116
|
+
|
|
117
|
+
Getting this wrong is silent: create_all() happily creates nothing and
|
|
118
|
+
returns, and the failure surfaces much later as "no such table" at the
|
|
119
|
+
first write. That is exactly how a fresh database ended up with no
|
|
120
|
+
pipelinerunrecord table while research mode looked fine on an existing
|
|
121
|
+
one. The guard below turns the total-failure case into an immediate,
|
|
122
|
+
named error; a partially-registered set can't be detected from here
|
|
123
|
+
without this file learning what data exists, which is the boundary §4.4
|
|
124
|
+
exists to hold.
|
|
125
|
+
"""
|
|
126
|
+
if not SQLModel.metadata.tables:
|
|
127
|
+
raise RuntimeError(
|
|
128
|
+
"create_db_and_tables() found no table classes registered. Import "
|
|
129
|
+
"the modules that define them (storage, core.reasoning.pipeline_storage) "
|
|
130
|
+
"before calling this, or the database will be created empty."
|
|
131
|
+
)
|
|
132
|
+
SQLModel.metadata.create_all(engine)
|
|
133
|
+
# AFTER create_all, deliberately: a brand-new database gets every table
|
|
134
|
+
# built complete and needs no ALTER at all. This pass exists only for
|
|
135
|
+
# databases that predate a column.
|
|
136
|
+
connection = engine.raw_connection()
|
|
137
|
+
try:
|
|
138
|
+
added = ensure_columns(connection, _declared_columns())
|
|
139
|
+
finally:
|
|
140
|
+
connection.close()
|
|
141
|
+
if added:
|
|
142
|
+
logger.info("Added missing columns to an existing database: %s",
|
|
143
|
+
", ".join(added))
|
|
144
|
+
logger.debug("Ensured tables exist: %s", sorted(SQLModel.metadata.tables))
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
# ---------------------------------------------------------------------------
|
|
148
|
+
# ---- Additive column migration (ROADMAP_v2 §21, M7) -----------------------
|
|
149
|
+
# ---------------------------------------------------------------------------
|
|
150
|
+
#
|
|
151
|
+
# create_all() creates missing TABLES and never touches an existing one, so
|
|
152
|
+
# adding a field to a table class that is already on disk produces "no such
|
|
153
|
+
# column" on the next SELECT -- for every database that predates the change,
|
|
154
|
+
# silently, at read time rather than at startup. §21's `MessageLog.pinned` is
|
|
155
|
+
# the first field this project has ever added to a shipped table; it will not
|
|
156
|
+
# be the last.
|
|
157
|
+
#
|
|
158
|
+
# ADDITIVE ONLY, and that is the whole contract. This will add a column that
|
|
159
|
+
# is declared and missing. It will never rename, drop, retype, backfill, or
|
|
160
|
+
# reorder, and it does not track a schema version. Anything beyond "a new
|
|
161
|
+
# nullable-or-defaulted column appeared" needs a real migration and should
|
|
162
|
+
# not be smuggled in here -- if this file ever grows a `DROP`, that is the
|
|
163
|
+
# signal the project has outgrown it.
|
|
164
|
+
#
|
|
165
|
+
# Nullability rides along where SQLite allows it (#26): `NOT NULL DEFAULT x`
|
|
166
|
+
# is expressible in one ALTER and matches create_all exactly; a not-null
|
|
167
|
+
# column with no SQL-spellable default cannot be added NOT NULL at all and
|
|
168
|
+
# lands nullable under a named WARNING. The two columns migrated before this
|
|
169
|
+
# (`pinned`, `kind`) remain nullable on databases that predate #26 -- accepted,
|
|
170
|
+
# not repaired: flipping them needs a table rebuild, which is outside this
|
|
171
|
+
# contract, and no writer can produce a NULL for them.
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def _default_literal(column):
|
|
175
|
+
"""A SQL literal for the column's default, or None if it has none we
|
|
176
|
+
can express.
|
|
177
|
+
|
|
178
|
+
SQLite fills existing rows with NULL when a column is added, so a
|
|
179
|
+
column the Python side declares non-optional would read back as None
|
|
180
|
+
on every pre-existing row. Emitting DEFAULT closes that: `pinned`
|
|
181
|
+
arrives as 0 on rows written before pinning existed, which is what
|
|
182
|
+
"this message was never pinned" should mean.
|
|
183
|
+
|
|
184
|
+
Only scalar defaults qualify. A default_factory (uuid4, utcnow) is
|
|
185
|
+
per-row Python and has no SQL spelling, so those columns are added
|
|
186
|
+
nullable and left to the application -- which is correct, because a
|
|
187
|
+
factory-defaulted column on an existing row has no right answer.
|
|
188
|
+
"""
|
|
189
|
+
default = getattr(column, "default", None)
|
|
190
|
+
if default is None or getattr(default, "is_callable", False):
|
|
191
|
+
return None
|
|
192
|
+
value = getattr(default, "arg", None)
|
|
193
|
+
if value is None or callable(value):
|
|
194
|
+
return None
|
|
195
|
+
if isinstance(value, bool):
|
|
196
|
+
return "1" if value else "0"
|
|
197
|
+
if isinstance(value, (int, float)):
|
|
198
|
+
return str(value)
|
|
199
|
+
if isinstance(value, str):
|
|
200
|
+
escaped = value.replace("'", "''")
|
|
201
|
+
return f"'{escaped}'"
|
|
202
|
+
return None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _declared_columns() -> dict:
|
|
206
|
+
"""table name -> {column name: (sql type, default literal or None,
|
|
207
|
+
not-null flag)}, read off whatever SQLModel currently has registered.
|
|
208
|
+
|
|
209
|
+
This keeps §4.4's boundary intact: the file still knows nothing about
|
|
210
|
+
WHICH tables exist, it asks the metadata -- the same source create_all
|
|
211
|
+
above is already driven by. A table class that was never imported is
|
|
212
|
+
invisible here exactly as it is there.
|
|
213
|
+
|
|
214
|
+
Everything SQLAlchemy-shaped is resolved HERE, into plain strings, so
|
|
215
|
+
ensure_columns() below is pure SQL over a DBAPI connection with no
|
|
216
|
+
ORM knowledge at all. That is what makes it testable against stdlib
|
|
217
|
+
sqlite3 while the suite's fake `sqlmodel` is installed -- the fake's
|
|
218
|
+
engine has no dialect to compile a type against, and a seam that only
|
|
219
|
+
works in production is not a seam.
|
|
220
|
+
|
|
221
|
+
The third slot is the nullability the model declares (#26): SQLModel
|
|
222
|
+
derives it from the annotation, so `pinned: bool` arrives not-null
|
|
223
|
+
and `name: Optional[str]` does not. Without carrying it, every column
|
|
224
|
+
this migration adds lands nullable where create_all would have built
|
|
225
|
+
NOT NULL -- a migrated database silently diverging from a fresh one.
|
|
226
|
+
"""
|
|
227
|
+
dialect = engine.dialect
|
|
228
|
+
return {
|
|
229
|
+
name: {
|
|
230
|
+
c.name: (
|
|
231
|
+
c.type.compile(dialect),
|
|
232
|
+
_default_literal(c),
|
|
233
|
+
not bool(getattr(c, "nullable", True)),
|
|
234
|
+
)
|
|
235
|
+
for c in table.columns
|
|
236
|
+
}
|
|
237
|
+
for name, table in SQLModel.metadata.tables.items()
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _affinity(sql_type: str) -> str:
|
|
242
|
+
"""Normalized type name for comparison -- uppercased, with any length
|
|
243
|
+
or precision specifier dropped. VARCHAR(255) and varchar compare
|
|
244
|
+
equal; BOOLEAN and VARCHAR do not."""
|
|
245
|
+
return sql_type.split("(")[0].strip().upper()
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def ensure_columns(connection, declared: dict) -> list:
|
|
249
|
+
"""Add every declared column missing from an existing table.
|
|
250
|
+
|
|
251
|
+
`declared` is _declared_columns()' plain-string shape --
|
|
252
|
+
table -> {column: (sql type, default literal or None, not-null flag)}.
|
|
253
|
+
`connection` is a DBAPI connection (`engine.raw_connection()` in
|
|
254
|
+
production, a plain `sqlite3.connect(...)` in tests) rather than a
|
|
255
|
+
SQLAlchemy Connection. Between them this function touches no ORM at
|
|
256
|
+
all, which is what makes it testable against stdlib sqlite3 while the
|
|
257
|
+
suite's fake `sqlmodel` is installed. Returns the "table.column" names
|
|
258
|
+
added, for logging.
|
|
259
|
+
|
|
260
|
+
A table with no rows in `PRAGMA table_info` does not exist yet;
|
|
261
|
+
create_all built it or will, so there is nothing to migrate.
|
|
262
|
+
|
|
263
|
+
Nullability is carried where SQLite can express it (#26). ALTER TABLE
|
|
264
|
+
ADD COLUMN accepts NOT NULL only together with a non-NULL DEFAULT, so
|
|
265
|
+
a not-null column WITH a scalar default migrates as `NOT NULL DEFAULT
|
|
266
|
+
x` -- matching what create_all builds on a fresh database in every
|
|
267
|
+
flag that changes what the schema permits (type, NOT NULL, pk). A
|
|
268
|
+
not-null column WITHOUT an expressible default (a default_factory:
|
|
269
|
+
uuid4, utcnow) cannot be added NOT NULL at all -- SQLite would have
|
|
270
|
+
to invent a value for every existing row -- so it lands nullable and
|
|
271
|
+
the divergence is named at WARNING rather than left silent.
|
|
272
|
+
|
|
273
|
+
One residual difference from a fresh database is inherent to the
|
|
274
|
+
mechanism and accepted: the migrated column carries a DEFAULT literal
|
|
275
|
+
while create_all emits none, because ALTER cannot backfill existing
|
|
276
|
+
rows without one and the ORM needs none. Only raw-SQL inserts into a
|
|
277
|
+
migrated database can observe the fill.
|
|
278
|
+
|
|
279
|
+
A column that exists with a DIFFERENT declared type is reported at
|
|
280
|
+
WARNING and left alone. It is deliberately not fatal: SQLite columns
|
|
281
|
+
have type affinity rather than enforcement, so a mismatch there
|
|
282
|
+
usually stores and reads exactly the same values -- and an additive
|
|
283
|
+
migrator cannot fix it either way. Refusing to start over a
|
|
284
|
+
cosmetic dialect difference would be a self-inflicted outage in the
|
|
285
|
+
one code path every launch runs through.
|
|
286
|
+
|
|
287
|
+
The two columns migrated before nullability was carried (`pinned`,
|
|
288
|
+
`kind`) exist as nullable on every database that predates #26, and
|
|
289
|
+
stay that way: repairing them would take a table rebuild, which is
|
|
290
|
+
outside this migration's additive contract, and no writer can put a
|
|
291
|
+
NULL there (the ORM always supplies its Python-side default), so the
|
|
292
|
+
residual risk is hand-written SQL only. Accepted and recorded in the
|
|
293
|
+
batch-18 decisions rather than papered over with machinery.
|
|
294
|
+
|
|
295
|
+
Does NOT close the connection -- the caller opened it and decides its
|
|
296
|
+
lifetime. In production that matters: engine.raw_connection() checks a
|
|
297
|
+
pooled connection out, and closing is how it goes back.
|
|
298
|
+
"""
|
|
299
|
+
added = []
|
|
300
|
+
cursor = connection.cursor()
|
|
301
|
+
for table, columns in declared.items():
|
|
302
|
+
cursor.execute(f"PRAGMA table_info({table})")
|
|
303
|
+
rows = cursor.fetchall()
|
|
304
|
+
if not rows:
|
|
305
|
+
continue
|
|
306
|
+
existing = {row[1]: row[2] for row in rows}
|
|
307
|
+
for name, (declared_type, literal, not_null) in columns.items():
|
|
308
|
+
if name in existing:
|
|
309
|
+
if _affinity(existing[name]) != _affinity(declared_type):
|
|
310
|
+
logger.warning(
|
|
311
|
+
"%s.%s exists as %s but is declared %s. Left "
|
|
312
|
+
"unchanged -- this migration is additive only "
|
|
313
|
+
"and cannot retype a column.",
|
|
314
|
+
table, name, existing[name], declared_type)
|
|
315
|
+
continue
|
|
316
|
+
ddl = f"ALTER TABLE {table} ADD COLUMN {name} {declared_type}"
|
|
317
|
+
if literal is not None:
|
|
318
|
+
if not_null:
|
|
319
|
+
ddl = f"{ddl} NOT NULL"
|
|
320
|
+
ddl = f"{ddl} DEFAULT {literal}"
|
|
321
|
+
elif not_null:
|
|
322
|
+
logger.warning(
|
|
323
|
+
"%s.%s is declared NOT NULL but its Python default "
|
|
324
|
+
"(default_factory) has no SQL spelling; added nullable. "
|
|
325
|
+
"SQLite cannot add a NOT NULL column without a default, "
|
|
326
|
+
"so a migrated database differs from a fresh one here "
|
|
327
|
+
"until a real migration exists.",
|
|
328
|
+
table, name)
|
|
329
|
+
cursor.execute(ddl)
|
|
330
|
+
added.append(f"{table}.{name}")
|
|
331
|
+
cursor.close()
|
|
332
|
+
connection.commit()
|
|
333
|
+
return added
|