@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,367 @@
1
+ """The SQLite engine of the world database."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import re
7
+ import sqlite3
8
+ import sys
9
+ import time
10
+ from contextlib import closing
11
+ from pathlib import Path
12
+ from typing import Any, Callable, Iterable
13
+
14
+ from ..compiler import json_pointer_parts, quote, scalar_types
15
+ from ..validation import ContractError, fail
16
+ from .base import STORED_LOCK_QUERY, WorldDatabase
17
+ from .budget import QueryBudget
18
+ from .guard import STAGING, Access, AccessKind
19
+
20
+ ACCESS_KINDS = {
21
+ sqlite3.SQLITE_SELECT: AccessKind.SELECT,
22
+ sqlite3.SQLITE_READ: AccessKind.READ,
23
+ sqlite3.SQLITE_FUNCTION: AccessKind.FUNCTION,
24
+ sqlite3.SQLITE_RECURSIVE: AccessKind.RECURSIVE,
25
+ sqlite3.SQLITE_INSERT: AccessKind.WRITE,
26
+ sqlite3.SQLITE_UPDATE: AccessKind.WRITE,
27
+ sqlite3.SQLITE_DELETE: AccessKind.WRITE,
28
+ sqlite3.SQLITE_CREATE_TEMP_TABLE: AccessKind.STAGE,
29
+ sqlite3.SQLITE_CREATE_TEMP_INDEX: AccessKind.STAGE,
30
+ sqlite3.SQLITE_CREATE_TEMP_VIEW: AccessKind.STAGE,
31
+ sqlite3.SQLITE_DROP_TEMP_TABLE: AccessKind.STAGE,
32
+ sqlite3.SQLITE_DROP_TEMP_INDEX: AccessKind.STAGE,
33
+ sqlite3.SQLITE_DROP_TEMP_VIEW: AccessKind.STAGE,
34
+ sqlite3.SQLITE_SAVEPOINT: AccessKind.SAVEPOINT,
35
+ }
36
+ TEMP_SCHEMA = "temp"
37
+ UNSAFE_FUNCTIONS = frozenset({"load_extension", "writefile", "readfile"})
38
+ #: Virtual-machine steps between deadline checks: frequent enough to stop within milliseconds.
39
+ PROGRESS_STEPS = 10_000
40
+ MISSING_TABLE = re.compile(r"no such table: (.+?)\s*$", re.MULTILINE)
41
+ CARRIED_TABLE = "_gateway_calls"
42
+ SNAPSHOT_STEP_PAGES = 256
43
+ SNAPSHOT_STEP_PAUSE_S = 0.001
44
+
45
+
46
+ def access(action: int, first: str | None, second: str | None, database: str | None) -> Access:
47
+ """SQLite's authorizer arguments as an :class:`Access`: a function call names
48
+ the function in ``second``; every other action names its object in ``first``."""
49
+ kind = ACCESS_KINDS.get(action, AccessKind.OTHER)
50
+ target = second if kind is AccessKind.FUNCTION else first
51
+ return Access(kind, target, STAGING if database == TEMP_SCHEMA else database)
52
+
53
+
54
+ def _without_comments(sql: str) -> str:
55
+ """``sql`` with ``--`` line comments and ``/* */`` blocks removed, string
56
+ literals left alone (a ``--`` inside quotes is text, not a comment)."""
57
+ out: list[str] = []
58
+ i, n = 0, len(sql)
59
+ quoted: str | None = None
60
+ while i < n:
61
+ ch = sql[i]
62
+ if quoted is not None:
63
+ out.append(ch)
64
+ quoted = None if ch == quoted else quoted
65
+ i += 1
66
+ continue
67
+ if ch in ("'", '"', "`"):
68
+ quoted = ch
69
+ out.append(ch)
70
+ i += 1
71
+ continue
72
+ if sql.startswith("--", i):
73
+ end = sql.find("\n", i)
74
+ i = n if end == -1 else end
75
+ continue
76
+ if sql.startswith("/*", i):
77
+ end = sql.find("*/", i + 2)
78
+ i = n if end == -1 else end + 2
79
+ continue
80
+ out.append(ch)
81
+ i += 1
82
+ return "".join(out)
83
+
84
+
85
+ def _append_statement(statements: list[str], text: str) -> None:
86
+ stripped = text.strip()
87
+ if _without_comments(stripped).strip(" \t\r\n;"):
88
+ statements.append(stripped)
89
+
90
+
91
+ def _json_path(field: str) -> str:
92
+ parts = json_pointer_parts(field, field, code="key")
93
+ return "$" + "".join(f".{json.dumps(part, ensure_ascii=False)}" for part in parts)
94
+
95
+
96
+ def quote_sql(value: str) -> str:
97
+ return "'" + value.replace("'", "''") + "'"
98
+
99
+
100
+ def _projection(field: str) -> str:
101
+ return f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", {quote_sql(_json_path(field))})) VIRTUAL'
102
+
103
+
104
+ def _existing_file(path: Path) -> sqlite3.Connection:
105
+ """A connection to ``path`` that refuses a missing file instead of creating one."""
106
+ return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=rw", uri=True)
107
+
108
+
109
+ class SQLiteWorldDatabase(WorldDatabase):
110
+ engine = "sqlite"
111
+ engine_label = "SQLite"
112
+ target = "sqlite-json/1"
113
+ snapshot_name = "state.sqlite"
114
+ content_type = "application/vnd.sqlite3"
115
+ native_error = sqlite3.Error
116
+ unsafe_functions = UNSAFE_FUNCTIONS
117
+ insertion_order = "rowid"
118
+
119
+ @classmethod
120
+ def ddl(cls, entities: dict) -> str:
121
+ """The tables a contract's entities compile to: the JSON payload is the source
122
+ of truth; each scalar field and every constrained field is a virtual column, so
123
+ verifier SQL can name it and SQLite enforces UNIQUE and foreign keys without
124
+ affinity coercion."""
125
+ lines = [
126
+ "-- Generated by gateway-world-compiler/1; do not edit.",
127
+ "PRAGMA foreign_keys = ON;",
128
+ ]
129
+ for name, entity in entities.items():
130
+ lines.append(f"CREATE TABLE {quote(name)} (\n " + ",\n ".join(cls._columns(entities, entity)) + "\n);")
131
+ return "\n".join(lines) + "\n"
132
+
133
+ @staticmethod
134
+ def _columns(entities: dict, entity: dict) -> list[str]:
135
+ properties = entity["schema"].get("properties", {})
136
+ columns = [
137
+ '"_key" TEXT PRIMARY KEY NOT NULL',
138
+ '"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
139
+ ]
140
+ columns.extend(_projection(field) for field, prop in sorted(properties.items()) if scalar_types(prop))
141
+ constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
142
+ field for relationship in entity["relationships"] for field in relationship["fields"]
143
+ }
144
+ columns.extend(_projection(field) for field in sorted(constrained - set(properties)))
145
+ columns.extend(
146
+ f"UNIQUE ({', '.join(quote(f) for f in fields)})" for fields in [entity["primary_key"], *entity["unique"]]
147
+ )
148
+ for rel in entity["relationships"]:
149
+ target = entities[rel["target"]]
150
+ columns.append(
151
+ f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
152
+ f"REFERENCES {quote(rel['target'])} "
153
+ f"({', '.join(quote(f) for f in target['primary_key'])}) "
154
+ "ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
155
+ )
156
+ return columns
157
+
158
+ @staticmethod
159
+ def split_statements(script: str) -> list[str]:
160
+ """Complete statements by ``sqlite3.complete_statement`` accumulation, so a
161
+ semicolon inside a literal or a trigger body never splits. Text that is only
162
+ comments is not a statement and is dropped, so every reader of a verifier
163
+ (check, grade, the host) sees the same list."""
164
+ statements: list[str] = []
165
+ buffer = ""
166
+ for line in script.splitlines(keepends=True):
167
+ buffer += line
168
+ if sqlite3.complete_statement(buffer):
169
+ _append_statement(statements, buffer)
170
+ buffer = ""
171
+ _append_statement(statements, buffer)
172
+ return statements
173
+
174
+ @staticmethod
175
+ def exceeds_limit(error: BaseException) -> bool:
176
+ return getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_TOOBIG
177
+
178
+ @staticmethod
179
+ def missing_table(error: BaseException) -> str | None:
180
+ match = MISSING_TABLE.search(str(error))
181
+ return match.group(1) if match else None
182
+
183
+ def _open_connection(self) -> None:
184
+ # cached_statements=0: a statement served from the cache is not re-authorized, so a text first
185
+ # prepared through one door would run unchecked through another.
186
+ self._conn = sqlite3.connect(
187
+ self.path,
188
+ isolation_level=None,
189
+ check_same_thread=False,
190
+ cached_statements=0,
191
+ )
192
+ self._conn.row_factory = sqlite3.Row
193
+ self._saved_limits: dict[int, int] = {}
194
+
195
+ def _enforce_strict_dialect(self) -> None:
196
+ self.forbid_double_quoted_literals(self._conn)
197
+ self._conn.execute("PRAGMA foreign_keys = ON")
198
+ if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
199
+ fail("sqlite", "Foreign-key enforcement is required")
200
+
201
+ @staticmethod
202
+ def forbid_double_quoted_literals(conn: Any) -> None:
203
+ """SQLite reads a double-quoted name that is not a column as a string literal, so a
204
+ verifier naming a missing column compares two literals and scores 0 without a word.
205
+ Switch the fallback off, or refuse to open where Python cannot (``setconfig`` is 3.12+)."""
206
+ if not hasattr(conn, "setconfig"):
207
+ fail(
208
+ "python_version",
209
+ f"Python {sys.version_info.major}.{sys.version_info.minor} cannot turn off SQLite's "
210
+ "double-quoted string literals; the world store needs Python 3.12 or newer.",
211
+ )
212
+ conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
213
+
214
+ def _execute(self, sql: str, params: Any = ()) -> sqlite3.Cursor:
215
+ return self._conn.execute(sql, params)
216
+
217
+ def _executemany(self, sql: str, rows: Any) -> sqlite3.Cursor:
218
+ return self._conn.executemany(sql, rows)
219
+
220
+ def _execute_script(self, script: str) -> None:
221
+ self._conn.executescript(script)
222
+
223
+ def _schema_objects(self) -> dict[str, str]:
224
+ return self._master(self._conn)
225
+
226
+ def _expected_schema_objects(self) -> dict[str, str]:
227
+ expected = sqlite3.connect(":memory:")
228
+ try:
229
+ expected.executescript(self._schema_script())
230
+ return self._master(expected)
231
+ finally:
232
+ expected.close()
233
+
234
+ @staticmethod
235
+ def _master(conn: sqlite3.Connection) -> dict[str, str]:
236
+ return dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
237
+
238
+ def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
239
+ self._conn.create_function("gateway_key", 2, function, deterministic=True)
240
+
241
+ def _install_guard(self) -> None:
242
+ self._conn.set_authorizer(self._authorize)
243
+
244
+ def _authorize(self, action, first, second, database, trigger):
245
+ if self._guard_allows(access(action, first, second, database)):
246
+ return sqlite3.SQLITE_OK
247
+ return sqlite3.SQLITE_DENY
248
+
249
+ def _begin_write(self) -> None:
250
+ self._conn.execute("BEGIN IMMEDIATE")
251
+
252
+ def _commit(self) -> None:
253
+ self._conn.execute("COMMIT")
254
+
255
+ def _rollback(self) -> None:
256
+ self._conn.execute("ROLLBACK")
257
+
258
+ def _savepoint(self, name: str) -> None:
259
+ self._conn.execute(f"SAVEPOINT {name}")
260
+
261
+ def _release(self, name: str) -> None:
262
+ self._conn.execute(f"RELEASE {name}")
263
+
264
+ def _rollback_to(self, name: str) -> None:
265
+ self._conn.execute(f"ROLLBACK TO {name}")
266
+
267
+ def _page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
268
+ return " LIMIT ? OFFSET ?", (-1 if limit is None else limit, offset)
269
+
270
+ def _relationships_broken(self) -> bool:
271
+ return self._conn.execute("PRAGMA foreign_key_check").fetchone() is not None
272
+
273
+ def _is_constraint_violation(self, error: BaseException) -> bool:
274
+ return isinstance(error, sqlite3.IntegrityError)
275
+
276
+ def _drop_staging(self) -> None:
277
+ objects = self._conn.execute(
278
+ "SELECT type, name FROM sqlite_temp_master WHERE type IN ('view', 'table') ORDER BY type DESC"
279
+ ).fetchall()
280
+ for kind, name in objects:
281
+ self._conn.execute(f"DROP {kind} IF EXISTS temp.{quote(name)}")
282
+
283
+ def _attach_file(self, name: str, path: str | Path) -> None:
284
+ self._conn.execute("ATTACH DATABASE ? AS ?", (str(path), name))
285
+
286
+ @staticmethod
287
+ def carry_entries(path: Path, entries: list[str]) -> None:
288
+ with closing(_existing_file(path)) as connection, connection:
289
+ connection.execute(f"CREATE TABLE {CARRIED_TABLE} (ordinal INTEGER PRIMARY KEY, entry TEXT NOT NULL)")
290
+ connection.executemany(f"INSERT INTO {CARRIED_TABLE} VALUES (?, ?)", list(enumerate(entries)))
291
+
292
+ @staticmethod
293
+ def take_entries(path: Path) -> list[str]:
294
+ with closing(_existing_file(path)) as connection, connection:
295
+ carried = connection.execute(
296
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (CARRIED_TABLE,)
297
+ ).fetchone()
298
+ if carried is None:
299
+ return []
300
+ entries = [entry for (entry,) in connection.execute(f"SELECT entry FROM {CARRIED_TABLE} ORDER BY ordinal")]
301
+ connection.execute(f"DROP TABLE {CARRIED_TABLE}")
302
+ return entries
303
+
304
+ @staticmethod
305
+ def stored_lock(path: Path) -> dict:
306
+ connection = sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
307
+ try:
308
+ rows = connection.execute(STORED_LOCK_QUERY).fetchall()
309
+ except sqlite3.Error as error:
310
+ raise ContractError("state_contract", "State has no compatible schema contract") from error
311
+ finally:
312
+ connection.close()
313
+ return WorldDatabase.lock_from(rows)
314
+
315
+ @staticmethod
316
+ def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
317
+ connection = sqlite3.connect(destination, isolation_level=None)
318
+ try:
319
+ connection.execute("PRAGMA foreign_keys = ON")
320
+ connection.execute("ATTACH DATABASE ? AS stored", (str(source),))
321
+ connection.execute("BEGIN IMMEDIATE")
322
+ for table in ledger:
323
+ connection.execute(f"INSERT INTO main.{quote(table)} SELECT * FROM stored.{quote(table)}")
324
+ for entity in entities:
325
+ connection.execute(
326
+ f"INSERT INTO main.{quote(entity)} (_key, payload) SELECT _key, payload FROM stored.{quote(entity)}"
327
+ )
328
+ connection.execute("COMMIT")
329
+ except sqlite3.Error as error:
330
+ raise ContractError("state_contract", f"The stored rows could not be migrated ({error})") from None
331
+ finally:
332
+ connection.close()
333
+
334
+ def _write_copy(self, destination: Path) -> None:
335
+ """SQLite's online backup, :data:`SNAPSHOT_STEP_PAGES` at a time. Between steps the
336
+ lock is let go for :data:`SNAPSHOT_STEP_PAUSE_S`, long enough for a waiting request
337
+ to take it; the backup runs on this connection, so a write made then lands in the copy."""
338
+ target = sqlite3.connect(destination)
339
+ try:
340
+ with self._lock:
341
+ self._conn.backup(target, pages=SNAPSHOT_STEP_PAGES, progress=self._between_steps)
342
+ finally:
343
+ target.close()
344
+
345
+ def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
346
+ if budget is None:
347
+ self._conn.set_progress_handler(None, 0)
348
+ for limit, value in self._saved_limits.items():
349
+ self._conn.setlimit(limit, value)
350
+ self._saved_limits = {}
351
+ return
352
+ self._saved_limits[sqlite3.SQLITE_LIMIT_LENGTH] = self._conn.setlimit(
353
+ sqlite3.SQLITE_LIMIT_LENGTH, budget.value_bytes
354
+ )
355
+ self._conn.set_progress_handler(lambda: int(time.monotonic() >= deadline), PROGRESS_STEPS)
356
+
357
+ def _between_steps(self, _status: int, remaining: int, _total: int) -> None:
358
+ if not remaining:
359
+ return
360
+ self._lock.release()
361
+ try:
362
+ time.sleep(SNAPSHOT_STEP_PAUSE_S)
363
+ finally:
364
+ self._lock.acquire()
365
+
366
+ def _close_connection(self) -> None:
367
+ self._conn.close()
@@ -0,0 +1,236 @@
1
+ """Rows stored under one domain, carried to a contract that only adds to it.
2
+
3
+ Stored rows (a data snapshot, a journal) were validated against the domain they were written
4
+ under. A contract that only adds to that domain — a new entity, a new field no row has to carry,
5
+ another ``domain_version`` — takes every one of them as it is, so they carry over: a journal is
6
+ re-targeted, a store is copied row for row (its batch ledger and lineage with it) into a store of
7
+ the new contract. Any other change to the stored domain is refused, naming each change.
8
+
9
+ The stored domain is proven by its compiled entities, which must hash to the domain the rows name:
10
+ a tree records them in ``schema/runtime.json`` until its next compile, and ``db/snapshot.json``
11
+ keeps them, beside the ``lock``, once the contract has moved off it.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import json
17
+ from dataclasses import dataclass
18
+ from pathlib import Path
19
+ from typing import Any, Iterable
20
+
21
+ from .compiler import STORAGE_PARTS, Contract, StorageContract, _domain_hash
22
+ from .database import WorldDatabase
23
+ from .database.guard import RUNTIME_TABLES
24
+ from .validation import canonical, fail
25
+
26
+ RUNTIME = "schema/runtime.json"
27
+ PIN = "db/snapshot.json"
28
+ FORMAT_PARTS = frozenset({"language", "annotations", "compiler", "target"})
29
+ ENTITY_KEYS = (
30
+ ("primary_key", "the primary key"),
31
+ ("unique", "the unique keys"),
32
+ ("relationships", "the relationships"),
33
+ )
34
+ SCHEMA_PARTS = frozenset({"properties", "required", "$defs"})
35
+ LEDGER_TABLES = ("_gateway_batches", "_gateway_lineage", *sorted(RUNTIME_TABLES))
36
+ ADDITIVE = "a new entity, a new field that is not required, or another domain_version"
37
+
38
+
39
+ def _proves(target: dict, entities: Any, storage: str) -> bool:
40
+ return isinstance(entities, dict) and _domain_hash(entities, storage) == target.get("hash")
41
+
42
+
43
+ def _read_entities(path: Path, key: str) -> Any:
44
+ try:
45
+ document = json.loads(path.read_text(encoding="utf-8"))
46
+ except (OSError, ValueError):
47
+ return None
48
+ return document.get(key) if isinstance(document, dict) else None
49
+
50
+
51
+ @dataclass(frozen=True)
52
+ class DomainMigration:
53
+ """The one rule for whether rows stored under ``target`` (a domain's ``id``, ``version`` and
54
+ entity ``hash``) carry over to ``contract``: only when the change adds to that domain. The
55
+ stored ``entities`` are None when nothing proves them; ``lock`` is the stored lock when the
56
+ rows live in a store, which is then the only kind that can be migrated."""
57
+
58
+ target: dict
59
+ entities: dict | None
60
+ contract: Contract
61
+ lock: dict | None = None
62
+ format: tuple[str, ...] = ()
63
+
64
+ @staticmethod
65
+ def recorded(world: str | Path) -> tuple[Any, ...]:
66
+ """Where ``world`` may still record the entities its stored rows were written under."""
67
+ world = Path(world)
68
+ return (_read_entities(world / PIN, "entities"), _read_entities(world / RUNTIME, "entities"))
69
+
70
+ @classmethod
71
+ def of_target(
72
+ cls, target: dict, contract: Contract, candidates: Iterable[Any], storage: str | None = None
73
+ ) -> "DomainMigration":
74
+ storage = storage or contract.lock["target"]
75
+ proven = next((entities for entities in candidates if _proves(target, entities, storage)), None)
76
+ return cls(target, proven, contract)
77
+
78
+ @classmethod
79
+ def of_lock(cls, lock: dict, contract: Contract, candidates: Iterable[Any]) -> "DomainMigration":
80
+ stored, current = StorageContract.of(lock), contract.storage
81
+ changed = tuple(
82
+ f"{words} ({min(getattr(stored, name))} → {min(getattr(current, name))})"
83
+ for name, words, _shown in STORAGE_PARTS
84
+ if name in FORMAT_PARTS and getattr(current, name).isdisjoint(getattr(stored, name))
85
+ )
86
+ migration = cls.of_target(lock["domain"], contract, candidates, lock.get("target"))
87
+ return cls(migration.target, migration.entities, contract, lock, changed)
88
+
89
+ @classmethod
90
+ def carry(cls, store: Path, contract: Contract, world: str | Path) -> dict[str, int] | None:
91
+ """The store at ``store`` carried in place to ``contract`` when its rows were stored under
92
+ a domain the contract only adds to (its counts); None when it already opens under it."""
93
+ lock = WorldDatabase.for_contract(contract).stored_lock(store)
94
+ migration = cls.of_lock(lock, contract, cls.recorded(world))
95
+ if not migration.needed:
96
+ return None
97
+ migration.require(
98
+ "state_contract",
99
+ f"/db/{store.name}",
100
+ "Import the rows into the world again, or restore the domain they were stored under",
101
+ )
102
+ fresh = store.with_name(f".{store.name}.migrating")
103
+ fresh.unlink(missing_ok=True)
104
+ counts = migration.migrate(store, fresh)
105
+ fresh.replace(store)
106
+ return counts
107
+
108
+ @property
109
+ def needed(self) -> bool:
110
+ return bool(self.format) or not self.contract.targets(self.target)
111
+
112
+ @property
113
+ def carries(self) -> bool:
114
+ return self.entities is not None and not self.refusals
115
+
116
+ @property
117
+ def additions(self) -> list[str]:
118
+ if self.entities is None:
119
+ return []
120
+ current = self.contract.entities
121
+ found = [f"entity {name}" for name in sorted(set(current) - set(self.entities))]
122
+ for name, stored in sorted(self.entities.items()):
123
+ now = current.get(name)
124
+ if now is None:
125
+ continue
126
+ held = stored["schema"].get("properties", {})
127
+ found.extend(f"field {name}.{f}" for f in sorted(set(now["schema"].get("properties", {})) - set(held)))
128
+ version = self.contract.target["version"]
129
+ if self.target.get("version") != version:
130
+ found.append(f"domain_version {self.target.get('version')} → {version}")
131
+ return found
132
+
133
+ @property
134
+ def refusals(self) -> list[str]:
135
+ found = [f"changes {words}" for words in self.format]
136
+ if self.target.get("id") != self.contract.target["id"]:
137
+ found.append(f"changes the domain $id ({self.target.get('id')} → {self.contract.target['id']})")
138
+ if self.entities is None:
139
+ return found
140
+ current = self.contract.entities
141
+ for name, stored in sorted(self.entities.items()):
142
+ found.extend(_entity_refusals(name, stored, current.get(name)))
143
+ return found
144
+
145
+ def require(self, code: str, pointer: str, remedy: str) -> None:
146
+ """Refuse unless the stored rows carry over, saying what to do about it."""
147
+ if not self.needed:
148
+ return
149
+ found = self.refusals
150
+ if found:
151
+ fail(
152
+ code,
153
+ f"The rows here were stored under the previous domain, and this change {'; '.join(found)}. "
154
+ f"Stored rows carry over only when a change adds to their domain ({ADDITIVE}): "
155
+ f"keep the rest as it was, or: {remedy}",
156
+ pointer,
157
+ changed=found,
158
+ )
159
+ if self.entities is None:
160
+ fail(
161
+ code,
162
+ f"This change moves the domain the rows here were stored under, and nothing in the tree "
163
+ f"records that domain's entities any more, so the runtime cannot tell whether the change only "
164
+ f"adds to it ({ADDITIVE}). Compile the change on the tree the stored rows came from (a fresh "
165
+ f"`gateway worlds sandbox open <slug>`, or `gateway bench pull <slug>`), or: {remedy}",
166
+ pointer,
167
+ )
168
+
169
+ def view(self) -> dict:
170
+ return {"from": self.lock_hash, "adds": self.additions}
171
+
172
+ @property
173
+ def lock_hash(self) -> str:
174
+ from .snapshot import lock_hash
175
+
176
+ return lock_hash(self.lock) if self.lock is not None else self.target.get("hash", "")
177
+
178
+ def migrate(self, source: Path, destination: Path) -> dict[str, int]:
179
+ """``source`` (a store of the stored lock) copied into a new store at ``destination`` under
180
+ the contract: every row, the batch ledger and the lineage, byte for byte."""
181
+ if self.lock is None or not self.carries:
182
+ fail("state_contract", "Only a store whose stored domain is proven and only added to can be migrated")
183
+ engine = WorldDatabase.for_contract(self.contract)
184
+ with WorldDatabase.open(source, self._stored_contract(), audit=False) as held:
185
+ counts = held.counts()
186
+ try:
187
+ WorldDatabase.create(destination, self.contract).close()
188
+ engine.copy_rows(source, destination, LEDGER_TABLES, sorted(self.entities))
189
+ with WorldDatabase.open(destination, self.contract, audit=False) as moved:
190
+ carried = moved.counts()
191
+ except BaseException:
192
+ destination.unlink(missing_ok=True)
193
+ raise
194
+ if any(carried.get(name) != count for name, count in counts.items()):
195
+ destination.unlink(missing_ok=True)
196
+ fail("state_contract", "The migrated store does not hold every stored row")
197
+ return carried
198
+
199
+ def _stored_contract(self) -> Contract:
200
+ target = self.lock["target"]
201
+ metadata = {"target": target, "domain": self.lock["domain"], "entities": self.entities, "tools": {}}
202
+ ddl = WorldDatabase.for_contract(Contract(metadata, {}, self.lock)).ddl(self.entities)
203
+ return Contract(metadata, {"db/schema.sql": ddl.encode()}, self.lock)
204
+
205
+
206
+ def _entity_refusals(name: str, stored: dict, current: dict | None) -> list[str]:
207
+ if current is None:
208
+ return [f"removes entity {name}"]
209
+ found = [
210
+ f"changes {words} of {name}" for key, words in ENTITY_KEYS if canonical(stored[key]) != canonical(current[key])
211
+ ]
212
+ return found + _schema_refusals(name, stored["schema"], current["schema"])
213
+
214
+
215
+ def _schema_refusals(name: str, stored: dict, current: dict) -> list[str]:
216
+ held, now = stored.get("properties", {}), current.get("properties", {})
217
+ required, requiring = set(stored.get("required", [])), set(current.get("required", []))
218
+ found = [f"removes field {name}.{f}" for f in sorted(set(held) - set(now))]
219
+ found += [
220
+ f"changes field {name}.{f}" for f in sorted(set(held) & set(now)) if canonical(held[f]) != canonical(now[f])
221
+ ]
222
+ found += [f"requires field {name}.{f}" for f in sorted(requiring - required)]
223
+ found += [f"stops requiring field {name}.{f}" for f in sorted((required - requiring) & set(now))]
224
+ defs, defined = stored.get("$defs", {}), current.get("$defs", {})
225
+ found += [
226
+ f"changes the definition {d} {name} stores"
227
+ for d in sorted(defs)
228
+ if canonical(defs[d]) != canonical(defined.get(d))
229
+ ]
230
+ rest = {key: value for key, value in stored.items() if key not in SCHEMA_PARTS}
231
+ if canonical(rest) != canonical({key: value for key, value in current.items() if key not in SCHEMA_PARTS}):
232
+ found.append(f"changes the schema of {name}")
233
+ return found
234
+
235
+
236
+ __all__ = ["DomainMigration"]