@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
package/database.py CHANGED
@@ -1,252 +1,333 @@
1
- """
2
- database.py
3
-
4
- Owns the actual database connection -- the engine, where the DB file
5
- lives, and how tables get created. storage.py imports `engine` from here;
6
- nothing here knows about ConversationThread, MessageLog, or any other
7
- table schema.
8
- """
9
-
10
- import logging
11
-
12
- from sqlmodel import create_engine, SQLModel
13
-
14
- import config
15
-
16
- logger = logging.getLogger(__name__)
17
-
18
- DATABASE_URL = f"sqlite:///{config.DB_PATH}"
19
-
20
- engine = create_engine(
21
- DATABASE_URL,
22
- echo=False,
23
- connect_args={"check_same_thread": False},
24
- )
25
-
26
-
27
- def create_db_and_tables() -> None:
28
- """Creates any table that doesn't exist yet. Safe to call every startup.
29
-
30
- IMPORTANT: this creates only the tables REGISTERED ON SQLModel.metadata
31
- when it runs, and a table class registers when its module is imported.
32
- This file knows nothing about which tables exist (§4.4), so the caller
33
- must have imported the modules defining them first -- see main.py, which
34
- imports storage and pipeline_storage explicitly for that side effect.
35
-
36
- Getting this wrong is silent: create_all() happily creates nothing and
37
- returns, and the failure surfaces much later as "no such table" at the
38
- first write. That is exactly how a fresh database ended up with no
39
- pipelinerunrecord table while research mode looked fine on an existing
40
- one. The guard below turns the total-failure case into an immediate,
41
- named error; a partially-registered set can't be detected from here
42
- without this file learning what data exists, which is the boundary §4.4
43
- exists to hold.
44
- """
45
- if not SQLModel.metadata.tables:
46
- raise RuntimeError(
47
- "create_db_and_tables() found no table classes registered. Import "
48
- "the modules that define them (storage, core.reasoning.pipeline_storage) "
49
- "before calling this, or the database will be created empty."
50
- )
51
- SQLModel.metadata.create_all(engine)
52
- # AFTER create_all, deliberately: a brand-new database gets every table
53
- # built complete and needs no ALTER at all. This pass exists only for
54
- # databases that predate a column.
55
- connection = engine.raw_connection()
56
- try:
57
- added = ensure_columns(connection, _declared_columns())
58
- finally:
59
- connection.close()
60
- if added:
61
- logger.info("Added missing columns to an existing database: %s",
62
- ", ".join(added))
63
- logger.debug("Ensured tables exist: %s", sorted(SQLModel.metadata.tables))
64
-
65
-
66
- # ---------------------------------------------------------------------------
67
- # ---- Additive column migration (ROADMAP_v2 §21, M7) -----------------------
68
- # ---------------------------------------------------------------------------
69
- #
70
- # create_all() creates missing TABLES and never touches an existing one, so
71
- # adding a field to a table class that is already on disk produces "no such
72
- # column" on the next SELECT -- for every database that predates the change,
73
- # silently, at read time rather than at startup. §21's `MessageLog.pinned` is
74
- # the first field this project has ever added to a shipped table; it will not
75
- # be the last.
76
- #
77
- # ADDITIVE ONLY, and that is the whole contract. This will add a column that
78
- # is declared and missing. It will never rename, drop, retype, backfill, or
79
- # reorder, and it does not track a schema version. Anything beyond "a new
80
- # nullable-or-defaulted column appeared" needs a real migration and should
81
- # not be smuggled in here -- if this file ever grows a `DROP`, that is the
82
- # signal the project has outgrown it.
83
- #
84
- # Nullability rides along where SQLite allows it (#26): `NOT NULL DEFAULT x`
85
- # is expressible in one ALTER and matches create_all exactly; a not-null
86
- # column with no SQL-spellable default cannot be added NOT NULL at all and
87
- # lands nullable under a named WARNING. The two columns migrated before this
88
- # (`pinned`, `kind`) remain nullable on databases that predate #26 -- accepted,
89
- # not repaired: flipping them needs a table rebuild, which is outside this
90
- # contract, and no writer can produce a NULL for them.
91
-
92
-
93
- def _default_literal(column):
94
- """A SQL literal for the column's default, or None if it has none we
95
- can express.
96
-
97
- SQLite fills existing rows with NULL when a column is added, so a
98
- column the Python side declares non-optional would read back as None
99
- on every pre-existing row. Emitting DEFAULT closes that: `pinned`
100
- arrives as 0 on rows written before pinning existed, which is what
101
- "this message was never pinned" should mean.
102
-
103
- Only scalar defaults qualify. A default_factory (uuid4, utcnow) is
104
- per-row Python and has no SQL spelling, so those columns are added
105
- nullable and left to the application -- which is correct, because a
106
- factory-defaulted column on an existing row has no right answer.
107
- """
108
- default = getattr(column, "default", None)
109
- if default is None or getattr(default, "is_callable", False):
110
- return None
111
- value = getattr(default, "arg", None)
112
- if value is None or callable(value):
113
- return None
114
- if isinstance(value, bool):
115
- return "1" if value else "0"
116
- if isinstance(value, (int, float)):
117
- return str(value)
118
- if isinstance(value, str):
119
- escaped = value.replace("'", "''")
120
- return f"'{escaped}'"
121
- return None
122
-
123
-
124
- def _declared_columns() -> dict:
125
- """table name -> {column name: (sql type, default literal or None,
126
- not-null flag)}, read off whatever SQLModel currently has registered.
127
-
128
- This keeps §4.4's boundary intact: the file still knows nothing about
129
- WHICH tables exist, it asks the metadata -- the same source create_all
130
- above is already driven by. A table class that was never imported is
131
- invisible here exactly as it is there.
132
-
133
- Everything SQLAlchemy-shaped is resolved HERE, into plain strings, so
134
- ensure_columns() below is pure SQL over a DBAPI connection with no
135
- ORM knowledge at all. That is what makes it testable against stdlib
136
- sqlite3 while the suite's fake `sqlmodel` is installed -- the fake's
137
- engine has no dialect to compile a type against, and a seam that only
138
- works in production is not a seam.
139
-
140
- The third slot is the nullability the model declares (#26): SQLModel
141
- derives it from the annotation, so `pinned: bool` arrives not-null
142
- and `name: Optional[str]` does not. Without carrying it, every column
143
- this migration adds lands nullable where create_all would have built
144
- NOT NULL -- a migrated database silently diverging from a fresh one.
145
- """
146
- dialect = engine.dialect
147
- return {
148
- name: {
149
- c.name: (
150
- c.type.compile(dialect),
151
- _default_literal(c),
152
- not bool(getattr(c, "nullable", True)),
153
- )
154
- for c in table.columns
155
- }
156
- for name, table in SQLModel.metadata.tables.items()
157
- }
158
-
159
-
160
- def _affinity(sql_type: str) -> str:
161
- """Normalized type name for comparison -- uppercased, with any length
162
- or precision specifier dropped. VARCHAR(255) and varchar compare
163
- equal; BOOLEAN and VARCHAR do not."""
164
- return sql_type.split("(")[0].strip().upper()
165
-
166
-
167
- def ensure_columns(connection, declared: dict) -> list:
168
- """Add every declared column missing from an existing table.
169
-
170
- `declared` is _declared_columns()' plain-string shape --
171
- table -> {column: (sql type, default literal or None, not-null flag)}.
172
- `connection` is a DBAPI connection (`engine.raw_connection()` in
173
- production, a plain `sqlite3.connect(...)` in tests) rather than a
174
- SQLAlchemy Connection. Between them this function touches no ORM at
175
- all, which is what makes it testable against stdlib sqlite3 while the
176
- suite's fake `sqlmodel` is installed. Returns the "table.column" names
177
- added, for logging.
178
-
179
- A table with no rows in `PRAGMA table_info` does not exist yet;
180
- create_all built it or will, so there is nothing to migrate.
181
-
182
- Nullability is carried where SQLite can express it (#26). ALTER TABLE
183
- ADD COLUMN accepts NOT NULL only together with a non-NULL DEFAULT, so
184
- a not-null column WITH a scalar default migrates as `NOT NULL DEFAULT
185
- x` -- matching what create_all builds on a fresh database in every
186
- flag that changes what the schema permits (type, NOT NULL, pk). A
187
- not-null column WITHOUT an expressible default (a default_factory:
188
- uuid4, utcnow) cannot be added NOT NULL at all -- SQLite would have
189
- to invent a value for every existing row -- so it lands nullable and
190
- the divergence is named at WARNING rather than left silent.
191
-
192
- One residual difference from a fresh database is inherent to the
193
- mechanism and accepted: the migrated column carries a DEFAULT literal
194
- while create_all emits none, because ALTER cannot backfill existing
195
- rows without one and the ORM needs none. Only raw-SQL inserts into a
196
- migrated database can observe the fill.
197
-
198
- A column that exists with a DIFFERENT declared type is reported at
199
- WARNING and left alone. It is deliberately not fatal: SQLite columns
200
- have type affinity rather than enforcement, so a mismatch there
201
- usually stores and reads exactly the same values -- and an additive
202
- migrator cannot fix it either way. Refusing to start over a
203
- cosmetic dialect difference would be a self-inflicted outage in the
204
- one code path every launch runs through.
205
-
206
- The two columns migrated before nullability was carried (`pinned`,
207
- `kind`) exist as nullable on every database that predates #26, and
208
- stay that way: repairing them would take a table rebuild, which is
209
- outside this migration's additive contract, and no writer can put a
210
- NULL there (the ORM always supplies its Python-side default), so the
211
- residual risk is hand-written SQL only. Accepted and recorded in the
212
- batch-18 decisions rather than papered over with machinery.
213
-
214
- Does NOT close the connection -- the caller opened it and decides its
215
- lifetime. In production that matters: engine.raw_connection() checks a
216
- pooled connection out, and closing is how it goes back.
217
- """
218
- added = []
219
- cursor = connection.cursor()
220
- for table, columns in declared.items():
221
- cursor.execute(f"PRAGMA table_info({table})")
222
- rows = cursor.fetchall()
223
- if not rows:
224
- continue
225
- existing = {row[1]: row[2] for row in rows}
226
- for name, (declared_type, literal, not_null) in columns.items():
227
- if name in existing:
228
- if _affinity(existing[name]) != _affinity(declared_type):
229
- logger.warning(
230
- "%s.%s exists as %s but is declared %s. Left "
231
- "unchanged -- this migration is additive only "
232
- "and cannot retype a column.",
233
- table, name, existing[name], declared_type)
234
- continue
235
- ddl = f"ALTER TABLE {table} ADD COLUMN {name} {declared_type}"
236
- if literal is not None:
237
- if not_null:
238
- ddl = f"{ddl} NOT NULL"
239
- ddl = f"{ddl} DEFAULT {literal}"
240
- elif not_null:
241
- logger.warning(
242
- "%s.%s is declared NOT NULL but its Python default "
243
- "(default_factory) has no SQL spelling; added nullable. "
244
- "SQLite cannot add a NOT NULL column without a default, "
245
- "so a migrated database differs from a fresh one here "
246
- "until a real migration exists.",
247
- table, name)
248
- cursor.execute(ddl)
249
- added.append(f"{table}.{name}")
250
- cursor.close()
251
- connection.commit()
252
- return added
1
+ """
2
+ database.py
3
+
4
+ Owns the actual database connection -- the engine, where the DB file
5
+ lives, and how tables get created. storage.py imports `engine` from here;
6
+ nothing here knows about ConversationThread, MessageLog, or any other
7
+ table schema.
8
+ """
9
+
10
+ import logging
11
+
12
+ from sqlmodel import SQLModel, create_engine
13
+
14
+ import config
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+ DATABASE_URL = f"sqlite:///{config.DB_PATH}"
19
+
20
+ engine = create_engine(
21
+ DATABASE_URL,
22
+ echo=False,
23
+ connect_args={"check_same_thread": False},
24
+ )
25
+
26
+ # ---------------------------------------------------------------------------
27
+ # ---- Concurrent writers (ROADMAP_v2 §47 slice 8) --------------------------
28
+ # ---------------------------------------------------------------------------
29
+ #
30
+ # `check_same_thread: False` has always been here, because the TUI drains the
31
+ # loop on a worker thread while the UI thread reads. What was never true
32
+ # until NA9 is that two writers could be in flight at once: one agent ran at
33
+ # a time, so the pool handed out one connection that mattered and SQLite's
34
+ # single-writer rule cost nothing.
35
+ #
36
+ # MEASURED, at the real engine against a real database, because this is the
37
+ # one hazard in the slice it would have been easiest to assume. Each shape is
38
+ # N threads creating a subagent thread and writing 200 messages to it, which
39
+ # is a child run's shape:
40
+ #
41
+ # writers journal=delete journal=WAL
42
+ # 3 0 errors, 2.94s 0 errors, 1.31s
43
+ # 8 1 "database is 0 errors, 3.21s
44
+ # locked", 6.78s
45
+ # 16 -- 1 "database is locked", 6.00s
46
+ #
47
+ # So at SUBAGENT_MAX_PARALLEL = 3 the shipped configuration does NOT lock,
48
+ # and WAL is enabled anyway for the two things the table shows: it removes
49
+ # the failure at 8, which is where a raised ceiling lands, and it is a little
50
+ # over twice as fast at 3, which is the configuration everyone runs. A
51
+ # concurrency ceiling that is a tunable constant invites being tuned, and the
52
+ # failure it walks into is a lost message rather than a slow one.
53
+ #
54
+ # THE BUSY TIMEOUT WAS ALREADY 5000ms and this does not change it. Measured
55
+ # too, via `PRAGMA busy_timeout`: SQLAlchemy's pysqlite dialect passes
56
+ # `timeout=5.0` to sqlite3.connect by default. It is restated here rather
57
+ # than left implicit because it is load-bearing for the numbers above, and a
58
+ # default nobody wrote down is a number that can change under us.
59
+ #
60
+ # ON CONNECT rather than once at import: `journal_mode` is persistent in the
61
+ # file but the pool opens connections over the life of the process and a
62
+ # fresh database file arrives with none of this, so the reliable place to
63
+ # state it is every connection. Contained and logged for the reason
64
+ # everything at this layer is: WAL needs shared memory, which some network
65
+ # filesystems do not provide, and a harness that refuses to start because it
66
+ # could not make its journal faster would be a self-inflicted outage.
67
+
68
+
69
+ def _apply_sqlite_pragmas(dbapi_connection, _record) -> None:
70
+ """WAL and an explicit busy timeout, per connection. See above."""
71
+ try:
72
+ cursor = dbapi_connection.cursor()
73
+ try:
74
+ cursor.execute("PRAGMA journal_mode=WAL")
75
+ cursor.execute("PRAGMA busy_timeout=5000")
76
+ finally:
77
+ cursor.close()
78
+ except Exception: # noqa: BLE001 -- a slower journal is not an outage
79
+ logger.warning(
80
+ "Could not set journal_mode=WAL on this database; continuing on "
81
+ "whatever journal it has. Concurrent subagent writes may "
82
+ "contend. This is expected on filesystems without shared "
83
+ "memory support.", exc_info=True)
84
+
85
+
86
+ # Imported here rather than at the top so the listener registration reads as
87
+ # one unit with the reasoning above it.
88
+ from sqlalchemy import event as _sqlalchemy_event # noqa: E402
89
+ from sqlalchemy.engine import Engine as _SqlAlchemyEngine # noqa: E402
90
+
91
+ # ONLY ON A REAL ENGINE, and the condition is named rather than caught.
92
+ #
93
+ # The suite's root conftest stubs `sqlmodel` into sys.modules before
94
+ # collection, so `create_engine` above returns a plain namespace there and
95
+ # `listens_for` raises InvalidRequestError on it -- at import, which is a
96
+ # collection error for every test that touches storage. An `except` around
97
+ # the registration would fix that and would also swallow a genuine
98
+ # misregistration in production, silently, leaving the journal on `delete`
99
+ # with nothing said. An isinstance check cannot: a real engine always
100
+ # passes it, so a production failure still raises.
101
+ #
102
+ # Same seam `ensure_columns` already uses one function down, for the same
103
+ # stated reason -- a boundary that only works in production is not a seam.
104
+ if isinstance(engine, _SqlAlchemyEngine):
105
+ _sqlalchemy_event.listens_for(engine, "connect")(_apply_sqlite_pragmas)
106
+
107
+
108
+ def create_db_and_tables() -> None:
109
+ """Creates any table that doesn't exist yet. Safe to call every startup.
110
+
111
+ IMPORTANT: this creates only the tables REGISTERED ON SQLModel.metadata
112
+ when it runs, and a table class registers when its module is imported.
113
+ This file knows nothing about which tables exist (§4.4), so the caller
114
+ must have imported the modules defining them first -- see main.py, which
115
+ imports storage and pipeline_storage explicitly for that side effect.
116
+
117
+ Getting this wrong is silent: create_all() happily creates nothing and
118
+ returns, and the failure surfaces much later as "no such table" at the
119
+ first write. That is exactly how a fresh database ended up with no
120
+ pipelinerunrecord table while research mode looked fine on an existing
121
+ one. The guard below turns the total-failure case into an immediate,
122
+ named error; a partially-registered set can't be detected from here
123
+ without this file learning what data exists, which is the boundary §4.4
124
+ exists to hold.
125
+ """
126
+ if not SQLModel.metadata.tables:
127
+ raise RuntimeError(
128
+ "create_db_and_tables() found no table classes registered. Import "
129
+ "the modules that define them (storage, core.reasoning.pipeline_storage) "
130
+ "before calling this, or the database will be created empty."
131
+ )
132
+ SQLModel.metadata.create_all(engine)
133
+ # AFTER create_all, deliberately: a brand-new database gets every table
134
+ # built complete and needs no ALTER at all. This pass exists only for
135
+ # databases that predate a column.
136
+ connection = engine.raw_connection()
137
+ try:
138
+ added = ensure_columns(connection, _declared_columns())
139
+ finally:
140
+ connection.close()
141
+ if added:
142
+ logger.info("Added missing columns to an existing database: %s",
143
+ ", ".join(added))
144
+ logger.debug("Ensured tables exist: %s", sorted(SQLModel.metadata.tables))
145
+
146
+
147
+ # ---------------------------------------------------------------------------
148
+ # ---- Additive column migration (ROADMAP_v2 §21, M7) -----------------------
149
+ # ---------------------------------------------------------------------------
150
+ #
151
+ # create_all() creates missing TABLES and never touches an existing one, so
152
+ # adding a field to a table class that is already on disk produces "no such
153
+ # column" on the next SELECT -- for every database that predates the change,
154
+ # silently, at read time rather than at startup. §21's `MessageLog.pinned` is
155
+ # the first field this project has ever added to a shipped table; it will not
156
+ # be the last.
157
+ #
158
+ # ADDITIVE ONLY, and that is the whole contract. This will add a column that
159
+ # is declared and missing. It will never rename, drop, retype, backfill, or
160
+ # reorder, and it does not track a schema version. Anything beyond "a new
161
+ # nullable-or-defaulted column appeared" needs a real migration and should
162
+ # not be smuggled in here -- if this file ever grows a `DROP`, that is the
163
+ # signal the project has outgrown it.
164
+ #
165
+ # Nullability rides along where SQLite allows it (#26): `NOT NULL DEFAULT x`
166
+ # is expressible in one ALTER and matches create_all exactly; a not-null
167
+ # column with no SQL-spellable default cannot be added NOT NULL at all and
168
+ # lands nullable under a named WARNING. The two columns migrated before this
169
+ # (`pinned`, `kind`) remain nullable on databases that predate #26 -- accepted,
170
+ # not repaired: flipping them needs a table rebuild, which is outside this
171
+ # contract, and no writer can produce a NULL for them.
172
+
173
+
174
+ def _default_literal(column):
175
+ """A SQL literal for the column's default, or None if it has none we
176
+ can express.
177
+
178
+ SQLite fills existing rows with NULL when a column is added, so a
179
+ column the Python side declares non-optional would read back as None
180
+ on every pre-existing row. Emitting DEFAULT closes that: `pinned`
181
+ arrives as 0 on rows written before pinning existed, which is what
182
+ "this message was never pinned" should mean.
183
+
184
+ Only scalar defaults qualify. A default_factory (uuid4, utcnow) is
185
+ per-row Python and has no SQL spelling, so those columns are added
186
+ nullable and left to the application -- which is correct, because a
187
+ factory-defaulted column on an existing row has no right answer.
188
+ """
189
+ default = getattr(column, "default", None)
190
+ if default is None or getattr(default, "is_callable", False):
191
+ return None
192
+ value = getattr(default, "arg", None)
193
+ if value is None or callable(value):
194
+ return None
195
+ if isinstance(value, bool):
196
+ return "1" if value else "0"
197
+ if isinstance(value, (int, float)):
198
+ return str(value)
199
+ if isinstance(value, str):
200
+ escaped = value.replace("'", "''")
201
+ return f"'{escaped}'"
202
+ return None
203
+
204
+
205
+ def _declared_columns() -> dict:
206
+ """table name -> {column name: (sql type, default literal or None,
207
+ not-null flag)}, read off whatever SQLModel currently has registered.
208
+
209
+ This keeps §4.4's boundary intact: the file still knows nothing about
210
+ WHICH tables exist, it asks the metadata -- the same source create_all
211
+ above is already driven by. A table class that was never imported is
212
+ invisible here exactly as it is there.
213
+
214
+ Everything SQLAlchemy-shaped is resolved HERE, into plain strings, so
215
+ ensure_columns() below is pure SQL over a DBAPI connection with no
216
+ ORM knowledge at all. That is what makes it testable against stdlib
217
+ sqlite3 while the suite's fake `sqlmodel` is installed -- the fake's
218
+ engine has no dialect to compile a type against, and a seam that only
219
+ works in production is not a seam.
220
+
221
+ The third slot is the nullability the model declares (#26): SQLModel
222
+ derives it from the annotation, so `pinned: bool` arrives not-null
223
+ and `name: Optional[str]` does not. Without carrying it, every column
224
+ this migration adds lands nullable where create_all would have built
225
+ NOT NULL -- a migrated database silently diverging from a fresh one.
226
+ """
227
+ dialect = engine.dialect
228
+ return {
229
+ name: {
230
+ c.name: (
231
+ c.type.compile(dialect),
232
+ _default_literal(c),
233
+ not bool(getattr(c, "nullable", True)),
234
+ )
235
+ for c in table.columns
236
+ }
237
+ for name, table in SQLModel.metadata.tables.items()
238
+ }
239
+
240
+
241
+ def _affinity(sql_type: str) -> str:
242
+ """Normalized type name for comparison -- uppercased, with any length
243
+ or precision specifier dropped. VARCHAR(255) and varchar compare
244
+ equal; BOOLEAN and VARCHAR do not."""
245
+ return sql_type.split("(")[0].strip().upper()
246
+
247
+
248
+ def ensure_columns(connection, declared: dict) -> list:
249
+ """Add every declared column missing from an existing table.
250
+
251
+ `declared` is _declared_columns()' plain-string shape --
252
+ table -> {column: (sql type, default literal or None, not-null flag)}.
253
+ `connection` is a DBAPI connection (`engine.raw_connection()` in
254
+ production, a plain `sqlite3.connect(...)` in tests) rather than a
255
+ SQLAlchemy Connection. Between them this function touches no ORM at
256
+ all, which is what makes it testable against stdlib sqlite3 while the
257
+ suite's fake `sqlmodel` is installed. Returns the "table.column" names
258
+ added, for logging.
259
+
260
+ A table with no rows in `PRAGMA table_info` does not exist yet;
261
+ create_all built it or will, so there is nothing to migrate.
262
+
263
+ Nullability is carried where SQLite can express it (#26). ALTER TABLE
264
+ ADD COLUMN accepts NOT NULL only together with a non-NULL DEFAULT, so
265
+ a not-null column WITH a scalar default migrates as `NOT NULL DEFAULT
266
+ x` -- matching what create_all builds on a fresh database in every
267
+ flag that changes what the schema permits (type, NOT NULL, pk). A
268
+ not-null column WITHOUT an expressible default (a default_factory:
269
+ uuid4, utcnow) cannot be added NOT NULL at all -- SQLite would have
270
+ to invent a value for every existing row -- so it lands nullable and
271
+ the divergence is named at WARNING rather than left silent.
272
+
273
+ One residual difference from a fresh database is inherent to the
274
+ mechanism and accepted: the migrated column carries a DEFAULT literal
275
+ while create_all emits none, because ALTER cannot backfill existing
276
+ rows without one and the ORM needs none. Only raw-SQL inserts into a
277
+ migrated database can observe the fill.
278
+
279
+ A column that exists with a DIFFERENT declared type is reported at
280
+ WARNING and left alone. It is deliberately not fatal: SQLite columns
281
+ have type affinity rather than enforcement, so a mismatch there
282
+ usually stores and reads exactly the same values -- and an additive
283
+ migrator cannot fix it either way. Refusing to start over a
284
+ cosmetic dialect difference would be a self-inflicted outage in the
285
+ one code path every launch runs through.
286
+
287
+ The two columns migrated before nullability was carried (`pinned`,
288
+ `kind`) exist as nullable on every database that predates #26, and
289
+ stay that way: repairing them would take a table rebuild, which is
290
+ outside this migration's additive contract, and no writer can put a
291
+ NULL there (the ORM always supplies its Python-side default), so the
292
+ residual risk is hand-written SQL only. Accepted and recorded in the
293
+ batch-18 decisions rather than papered over with machinery.
294
+
295
+ Does NOT close the connection -- the caller opened it and decides its
296
+ lifetime. In production that matters: engine.raw_connection() checks a
297
+ pooled connection out, and closing is how it goes back.
298
+ """
299
+ added = []
300
+ cursor = connection.cursor()
301
+ for table, columns in declared.items():
302
+ cursor.execute(f"PRAGMA table_info({table})")
303
+ rows = cursor.fetchall()
304
+ if not rows:
305
+ continue
306
+ existing = {row[1]: row[2] for row in rows}
307
+ for name, (declared_type, literal, not_null) in columns.items():
308
+ if name in existing:
309
+ if _affinity(existing[name]) != _affinity(declared_type):
310
+ logger.warning(
311
+ "%s.%s exists as %s but is declared %s. Left "
312
+ "unchanged -- this migration is additive only "
313
+ "and cannot retype a column.",
314
+ table, name, existing[name], declared_type)
315
+ continue
316
+ ddl = f"ALTER TABLE {table} ADD COLUMN {name} {declared_type}"
317
+ if literal is not None:
318
+ if not_null:
319
+ ddl = f"{ddl} NOT NULL"
320
+ ddl = f"{ddl} DEFAULT {literal}"
321
+ elif not_null:
322
+ logger.warning(
323
+ "%s.%s is declared NOT NULL but its Python default "
324
+ "(default_factory) has no SQL spelling; added nullable. "
325
+ "SQLite cannot add a NOT NULL column without a default, "
326
+ "so a migrated database differs from a fresh one here "
327
+ "until a real migration exists.",
328
+ table, name)
329
+ cursor.execute(ddl)
330
+ added.append(f"{table}.{name}")
331
+ cursor.close()
332
+ connection.commit()
333
+ return added