@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,367 @@
|
|
|
1
|
+
"""The SQLite engine of the world database."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from contextlib import closing
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any, Callable, Iterable
|
|
13
|
+
|
|
14
|
+
from ..compiler import json_pointer_parts, quote, scalar_types
|
|
15
|
+
from ..validation import ContractError, fail
|
|
16
|
+
from .base import STORED_LOCK_QUERY, WorldDatabase
|
|
17
|
+
from .budget import QueryBudget
|
|
18
|
+
from .guard import STAGING, Access, AccessKind
|
|
19
|
+
|
|
20
|
+
ACCESS_KINDS = {
|
|
21
|
+
sqlite3.SQLITE_SELECT: AccessKind.SELECT,
|
|
22
|
+
sqlite3.SQLITE_READ: AccessKind.READ,
|
|
23
|
+
sqlite3.SQLITE_FUNCTION: AccessKind.FUNCTION,
|
|
24
|
+
sqlite3.SQLITE_RECURSIVE: AccessKind.RECURSIVE,
|
|
25
|
+
sqlite3.SQLITE_INSERT: AccessKind.WRITE,
|
|
26
|
+
sqlite3.SQLITE_UPDATE: AccessKind.WRITE,
|
|
27
|
+
sqlite3.SQLITE_DELETE: AccessKind.WRITE,
|
|
28
|
+
sqlite3.SQLITE_CREATE_TEMP_TABLE: AccessKind.STAGE,
|
|
29
|
+
sqlite3.SQLITE_CREATE_TEMP_INDEX: AccessKind.STAGE,
|
|
30
|
+
sqlite3.SQLITE_CREATE_TEMP_VIEW: AccessKind.STAGE,
|
|
31
|
+
sqlite3.SQLITE_DROP_TEMP_TABLE: AccessKind.STAGE,
|
|
32
|
+
sqlite3.SQLITE_DROP_TEMP_INDEX: AccessKind.STAGE,
|
|
33
|
+
sqlite3.SQLITE_DROP_TEMP_VIEW: AccessKind.STAGE,
|
|
34
|
+
sqlite3.SQLITE_SAVEPOINT: AccessKind.SAVEPOINT,
|
|
35
|
+
}
|
|
36
|
+
TEMP_SCHEMA = "temp"
|
|
37
|
+
UNSAFE_FUNCTIONS = frozenset({"load_extension", "writefile", "readfile"})
|
|
38
|
+
#: Virtual-machine steps between deadline checks: frequent enough to stop within milliseconds.
|
|
39
|
+
PROGRESS_STEPS = 10_000
|
|
40
|
+
MISSING_TABLE = re.compile(r"no such table: (.+?)\s*$", re.MULTILINE)
|
|
41
|
+
CARRIED_TABLE = "_gateway_calls"
|
|
42
|
+
SNAPSHOT_STEP_PAGES = 256
|
|
43
|
+
SNAPSHOT_STEP_PAUSE_S = 0.001
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def access(action: int, first: str | None, second: str | None, database: str | None) -> Access:
|
|
47
|
+
"""SQLite's authorizer arguments as an :class:`Access`: a function call names
|
|
48
|
+
the function in ``second``; every other action names its object in ``first``."""
|
|
49
|
+
kind = ACCESS_KINDS.get(action, AccessKind.OTHER)
|
|
50
|
+
target = second if kind is AccessKind.FUNCTION else first
|
|
51
|
+
return Access(kind, target, STAGING if database == TEMP_SCHEMA else database)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _without_comments(sql: str) -> str:
|
|
55
|
+
"""``sql`` with ``--`` line comments and ``/* */`` blocks removed, string
|
|
56
|
+
literals left alone (a ``--`` inside quotes is text, not a comment)."""
|
|
57
|
+
out: list[str] = []
|
|
58
|
+
i, n = 0, len(sql)
|
|
59
|
+
quoted: str | None = None
|
|
60
|
+
while i < n:
|
|
61
|
+
ch = sql[i]
|
|
62
|
+
if quoted is not None:
|
|
63
|
+
out.append(ch)
|
|
64
|
+
quoted = None if ch == quoted else quoted
|
|
65
|
+
i += 1
|
|
66
|
+
continue
|
|
67
|
+
if ch in ("'", '"', "`"):
|
|
68
|
+
quoted = ch
|
|
69
|
+
out.append(ch)
|
|
70
|
+
i += 1
|
|
71
|
+
continue
|
|
72
|
+
if sql.startswith("--", i):
|
|
73
|
+
end = sql.find("\n", i)
|
|
74
|
+
i = n if end == -1 else end
|
|
75
|
+
continue
|
|
76
|
+
if sql.startswith("/*", i):
|
|
77
|
+
end = sql.find("*/", i + 2)
|
|
78
|
+
i = n if end == -1 else end + 2
|
|
79
|
+
continue
|
|
80
|
+
out.append(ch)
|
|
81
|
+
i += 1
|
|
82
|
+
return "".join(out)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _append_statement(statements: list[str], text: str) -> None:
|
|
86
|
+
stripped = text.strip()
|
|
87
|
+
if _without_comments(stripped).strip(" \t\r\n;"):
|
|
88
|
+
statements.append(stripped)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _json_path(field: str) -> str:
|
|
92
|
+
parts = json_pointer_parts(field, field, code="key")
|
|
93
|
+
return "$" + "".join(f".{json.dumps(part, ensure_ascii=False)}" for part in parts)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def quote_sql(value: str) -> str:
|
|
97
|
+
return "'" + value.replace("'", "''") + "'"
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _projection(field: str) -> str:
|
|
101
|
+
return f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", {quote_sql(_json_path(field))})) VIRTUAL'
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _existing_file(path: Path) -> sqlite3.Connection:
|
|
105
|
+
"""A connection to ``path`` that refuses a missing file instead of creating one."""
|
|
106
|
+
return sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=rw", uri=True)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class SQLiteWorldDatabase(WorldDatabase):
|
|
110
|
+
engine = "sqlite"
|
|
111
|
+
engine_label = "SQLite"
|
|
112
|
+
target = "sqlite-json/1"
|
|
113
|
+
snapshot_name = "state.sqlite"
|
|
114
|
+
content_type = "application/vnd.sqlite3"
|
|
115
|
+
native_error = sqlite3.Error
|
|
116
|
+
unsafe_functions = UNSAFE_FUNCTIONS
|
|
117
|
+
insertion_order = "rowid"
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def ddl(cls, entities: dict) -> str:
|
|
121
|
+
"""The tables a contract's entities compile to: the JSON payload is the source
|
|
122
|
+
of truth; each scalar field and every constrained field is a virtual column, so
|
|
123
|
+
verifier SQL can name it and SQLite enforces UNIQUE and foreign keys without
|
|
124
|
+
affinity coercion."""
|
|
125
|
+
lines = [
|
|
126
|
+
"-- Generated by gateway-world-compiler/1; do not edit.",
|
|
127
|
+
"PRAGMA foreign_keys = ON;",
|
|
128
|
+
]
|
|
129
|
+
for name, entity in entities.items():
|
|
130
|
+
lines.append(f"CREATE TABLE {quote(name)} (\n " + ",\n ".join(cls._columns(entities, entity)) + "\n);")
|
|
131
|
+
return "\n".join(lines) + "\n"
|
|
132
|
+
|
|
133
|
+
@staticmethod
|
|
134
|
+
def _columns(entities: dict, entity: dict) -> list[str]:
|
|
135
|
+
properties = entity["schema"].get("properties", {})
|
|
136
|
+
columns = [
|
|
137
|
+
'"_key" TEXT PRIMARY KEY NOT NULL',
|
|
138
|
+
'"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
|
|
139
|
+
]
|
|
140
|
+
columns.extend(_projection(field) for field, prop in sorted(properties.items()) if scalar_types(prop))
|
|
141
|
+
constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
|
|
142
|
+
field for relationship in entity["relationships"] for field in relationship["fields"]
|
|
143
|
+
}
|
|
144
|
+
columns.extend(_projection(field) for field in sorted(constrained - set(properties)))
|
|
145
|
+
columns.extend(
|
|
146
|
+
f"UNIQUE ({', '.join(quote(f) for f in fields)})" for fields in [entity["primary_key"], *entity["unique"]]
|
|
147
|
+
)
|
|
148
|
+
for rel in entity["relationships"]:
|
|
149
|
+
target = entities[rel["target"]]
|
|
150
|
+
columns.append(
|
|
151
|
+
f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
|
|
152
|
+
f"REFERENCES {quote(rel['target'])} "
|
|
153
|
+
f"({', '.join(quote(f) for f in target['primary_key'])}) "
|
|
154
|
+
"ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
|
|
155
|
+
)
|
|
156
|
+
return columns
|
|
157
|
+
|
|
158
|
+
@staticmethod
|
|
159
|
+
def split_statements(script: str) -> list[str]:
|
|
160
|
+
"""Complete statements by ``sqlite3.complete_statement`` accumulation, so a
|
|
161
|
+
semicolon inside a literal or a trigger body never splits. Text that is only
|
|
162
|
+
comments is not a statement and is dropped, so every reader of a verifier
|
|
163
|
+
(check, grade, the host) sees the same list."""
|
|
164
|
+
statements: list[str] = []
|
|
165
|
+
buffer = ""
|
|
166
|
+
for line in script.splitlines(keepends=True):
|
|
167
|
+
buffer += line
|
|
168
|
+
if sqlite3.complete_statement(buffer):
|
|
169
|
+
_append_statement(statements, buffer)
|
|
170
|
+
buffer = ""
|
|
171
|
+
_append_statement(statements, buffer)
|
|
172
|
+
return statements
|
|
173
|
+
|
|
174
|
+
@staticmethod
|
|
175
|
+
def exceeds_limit(error: BaseException) -> bool:
|
|
176
|
+
return getattr(error, "sqlite_errorcode", None) == sqlite3.SQLITE_TOOBIG
|
|
177
|
+
|
|
178
|
+
@staticmethod
|
|
179
|
+
def missing_table(error: BaseException) -> str | None:
|
|
180
|
+
match = MISSING_TABLE.search(str(error))
|
|
181
|
+
return match.group(1) if match else None
|
|
182
|
+
|
|
183
|
+
def _open_connection(self) -> None:
|
|
184
|
+
# cached_statements=0: a statement served from the cache is not re-authorized, so a text first
|
|
185
|
+
# prepared through one door would run unchecked through another.
|
|
186
|
+
self._conn = sqlite3.connect(
|
|
187
|
+
self.path,
|
|
188
|
+
isolation_level=None,
|
|
189
|
+
check_same_thread=False,
|
|
190
|
+
cached_statements=0,
|
|
191
|
+
)
|
|
192
|
+
self._conn.row_factory = sqlite3.Row
|
|
193
|
+
self._saved_limits: dict[int, int] = {}
|
|
194
|
+
|
|
195
|
+
def _enforce_strict_dialect(self) -> None:
|
|
196
|
+
self.forbid_double_quoted_literals(self._conn)
|
|
197
|
+
self._conn.execute("PRAGMA foreign_keys = ON")
|
|
198
|
+
if self._conn.execute("PRAGMA foreign_keys").fetchone()[0] != 1:
|
|
199
|
+
fail("sqlite", "Foreign-key enforcement is required")
|
|
200
|
+
|
|
201
|
+
@staticmethod
|
|
202
|
+
def forbid_double_quoted_literals(conn: Any) -> None:
|
|
203
|
+
"""SQLite reads a double-quoted name that is not a column as a string literal, so a
|
|
204
|
+
verifier naming a missing column compares two literals and scores 0 without a word.
|
|
205
|
+
Switch the fallback off, or refuse to open where Python cannot (``setconfig`` is 3.12+)."""
|
|
206
|
+
if not hasattr(conn, "setconfig"):
|
|
207
|
+
fail(
|
|
208
|
+
"python_version",
|
|
209
|
+
f"Python {sys.version_info.major}.{sys.version_info.minor} cannot turn off SQLite's "
|
|
210
|
+
"double-quoted string literals; the world store needs Python 3.12 or newer.",
|
|
211
|
+
)
|
|
212
|
+
conn.setconfig(sqlite3.SQLITE_DBCONFIG_DQS_DML, False)
|
|
213
|
+
|
|
214
|
+
def _execute(self, sql: str, params: Any = ()) -> sqlite3.Cursor:
|
|
215
|
+
return self._conn.execute(sql, params)
|
|
216
|
+
|
|
217
|
+
def _executemany(self, sql: str, rows: Any) -> sqlite3.Cursor:
|
|
218
|
+
return self._conn.executemany(sql, rows)
|
|
219
|
+
|
|
220
|
+
def _execute_script(self, script: str) -> None:
|
|
221
|
+
self._conn.executescript(script)
|
|
222
|
+
|
|
223
|
+
def _schema_objects(self) -> dict[str, str]:
|
|
224
|
+
return self._master(self._conn)
|
|
225
|
+
|
|
226
|
+
def _expected_schema_objects(self) -> dict[str, str]:
|
|
227
|
+
expected = sqlite3.connect(":memory:")
|
|
228
|
+
try:
|
|
229
|
+
expected.executescript(self._schema_script())
|
|
230
|
+
return self._master(expected)
|
|
231
|
+
finally:
|
|
232
|
+
expected.close()
|
|
233
|
+
|
|
234
|
+
@staticmethod
|
|
235
|
+
def _master(conn: sqlite3.Connection) -> dict[str, str]:
|
|
236
|
+
return dict(conn.execute("SELECT name, sql FROM sqlite_master WHERE sql IS NOT NULL"))
|
|
237
|
+
|
|
238
|
+
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
|
|
239
|
+
self._conn.create_function("gateway_key", 2, function, deterministic=True)
|
|
240
|
+
|
|
241
|
+
def _install_guard(self) -> None:
|
|
242
|
+
self._conn.set_authorizer(self._authorize)
|
|
243
|
+
|
|
244
|
+
def _authorize(self, action, first, second, database, trigger):
|
|
245
|
+
if self._guard_allows(access(action, first, second, database)):
|
|
246
|
+
return sqlite3.SQLITE_OK
|
|
247
|
+
return sqlite3.SQLITE_DENY
|
|
248
|
+
|
|
249
|
+
def _begin_write(self) -> None:
|
|
250
|
+
self._conn.execute("BEGIN IMMEDIATE")
|
|
251
|
+
|
|
252
|
+
def _commit(self) -> None:
|
|
253
|
+
self._conn.execute("COMMIT")
|
|
254
|
+
|
|
255
|
+
def _rollback(self) -> None:
|
|
256
|
+
self._conn.execute("ROLLBACK")
|
|
257
|
+
|
|
258
|
+
def _savepoint(self, name: str) -> None:
|
|
259
|
+
self._conn.execute(f"SAVEPOINT {name}")
|
|
260
|
+
|
|
261
|
+
def _release(self, name: str) -> None:
|
|
262
|
+
self._conn.execute(f"RELEASE {name}")
|
|
263
|
+
|
|
264
|
+
def _rollback_to(self, name: str) -> None:
|
|
265
|
+
self._conn.execute(f"ROLLBACK TO {name}")
|
|
266
|
+
|
|
267
|
+
def _page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
268
|
+
return " LIMIT ? OFFSET ?", (-1 if limit is None else limit, offset)
|
|
269
|
+
|
|
270
|
+
def _relationships_broken(self) -> bool:
|
|
271
|
+
return self._conn.execute("PRAGMA foreign_key_check").fetchone() is not None
|
|
272
|
+
|
|
273
|
+
def _is_constraint_violation(self, error: BaseException) -> bool:
|
|
274
|
+
return isinstance(error, sqlite3.IntegrityError)
|
|
275
|
+
|
|
276
|
+
def _drop_staging(self) -> None:
|
|
277
|
+
objects = self._conn.execute(
|
|
278
|
+
"SELECT type, name FROM sqlite_temp_master WHERE type IN ('view', 'table') ORDER BY type DESC"
|
|
279
|
+
).fetchall()
|
|
280
|
+
for kind, name in objects:
|
|
281
|
+
self._conn.execute(f"DROP {kind} IF EXISTS temp.{quote(name)}")
|
|
282
|
+
|
|
283
|
+
def _attach_file(self, name: str, path: str | Path) -> None:
|
|
284
|
+
self._conn.execute("ATTACH DATABASE ? AS ?", (str(path), name))
|
|
285
|
+
|
|
286
|
+
@staticmethod
|
|
287
|
+
def carry_entries(path: Path, entries: list[str]) -> None:
|
|
288
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
289
|
+
connection.execute(f"CREATE TABLE {CARRIED_TABLE} (ordinal INTEGER PRIMARY KEY, entry TEXT NOT NULL)")
|
|
290
|
+
connection.executemany(f"INSERT INTO {CARRIED_TABLE} VALUES (?, ?)", list(enumerate(entries)))
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def take_entries(path: Path) -> list[str]:
|
|
294
|
+
with closing(_existing_file(path)) as connection, connection:
|
|
295
|
+
carried = connection.execute(
|
|
296
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (CARRIED_TABLE,)
|
|
297
|
+
).fetchone()
|
|
298
|
+
if carried is None:
|
|
299
|
+
return []
|
|
300
|
+
entries = [entry for (entry,) in connection.execute(f"SELECT entry FROM {CARRIED_TABLE} ORDER BY ordinal")]
|
|
301
|
+
connection.execute(f"DROP TABLE {CARRIED_TABLE}")
|
|
302
|
+
return entries
|
|
303
|
+
|
|
304
|
+
@staticmethod
|
|
305
|
+
def stored_lock(path: Path) -> dict:
|
|
306
|
+
connection = sqlite3.connect(f"{Path(path).resolve().as_uri()}?mode=ro", uri=True)
|
|
307
|
+
try:
|
|
308
|
+
rows = connection.execute(STORED_LOCK_QUERY).fetchall()
|
|
309
|
+
except sqlite3.Error as error:
|
|
310
|
+
raise ContractError("state_contract", "State has no compatible schema contract") from error
|
|
311
|
+
finally:
|
|
312
|
+
connection.close()
|
|
313
|
+
return WorldDatabase.lock_from(rows)
|
|
314
|
+
|
|
315
|
+
@staticmethod
|
|
316
|
+
def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
|
|
317
|
+
connection = sqlite3.connect(destination, isolation_level=None)
|
|
318
|
+
try:
|
|
319
|
+
connection.execute("PRAGMA foreign_keys = ON")
|
|
320
|
+
connection.execute("ATTACH DATABASE ? AS stored", (str(source),))
|
|
321
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
322
|
+
for table in ledger:
|
|
323
|
+
connection.execute(f"INSERT INTO main.{quote(table)} SELECT * FROM stored.{quote(table)}")
|
|
324
|
+
for entity in entities:
|
|
325
|
+
connection.execute(
|
|
326
|
+
f"INSERT INTO main.{quote(entity)} (_key, payload) SELECT _key, payload FROM stored.{quote(entity)}"
|
|
327
|
+
)
|
|
328
|
+
connection.execute("COMMIT")
|
|
329
|
+
except sqlite3.Error as error:
|
|
330
|
+
raise ContractError("state_contract", f"The stored rows could not be migrated ({error})") from None
|
|
331
|
+
finally:
|
|
332
|
+
connection.close()
|
|
333
|
+
|
|
334
|
+
def _write_copy(self, destination: Path) -> None:
|
|
335
|
+
"""SQLite's online backup, :data:`SNAPSHOT_STEP_PAGES` at a time. Between steps the
|
|
336
|
+
lock is let go for :data:`SNAPSHOT_STEP_PAUSE_S`, long enough for a waiting request
|
|
337
|
+
to take it; the backup runs on this connection, so a write made then lands in the copy."""
|
|
338
|
+
target = sqlite3.connect(destination)
|
|
339
|
+
try:
|
|
340
|
+
with self._lock:
|
|
341
|
+
self._conn.backup(target, pages=SNAPSHOT_STEP_PAGES, progress=self._between_steps)
|
|
342
|
+
finally:
|
|
343
|
+
target.close()
|
|
344
|
+
|
|
345
|
+
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
|
|
346
|
+
if budget is None:
|
|
347
|
+
self._conn.set_progress_handler(None, 0)
|
|
348
|
+
for limit, value in self._saved_limits.items():
|
|
349
|
+
self._conn.setlimit(limit, value)
|
|
350
|
+
self._saved_limits = {}
|
|
351
|
+
return
|
|
352
|
+
self._saved_limits[sqlite3.SQLITE_LIMIT_LENGTH] = self._conn.setlimit(
|
|
353
|
+
sqlite3.SQLITE_LIMIT_LENGTH, budget.value_bytes
|
|
354
|
+
)
|
|
355
|
+
self._conn.set_progress_handler(lambda: int(time.monotonic() >= deadline), PROGRESS_STEPS)
|
|
356
|
+
|
|
357
|
+
def _between_steps(self, _status: int, remaining: int, _total: int) -> None:
|
|
358
|
+
if not remaining:
|
|
359
|
+
return
|
|
360
|
+
self._lock.release()
|
|
361
|
+
try:
|
|
362
|
+
time.sleep(SNAPSHOT_STEP_PAUSE_S)
|
|
363
|
+
finally:
|
|
364
|
+
self._lock.acquire()
|
|
365
|
+
|
|
366
|
+
def _close_connection(self) -> None:
|
|
367
|
+
self._conn.close()
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Rows stored under one domain, carried to a contract that only adds to it.
|
|
2
|
+
|
|
3
|
+
Stored rows (a data snapshot, a journal) were validated against the domain they were written
|
|
4
|
+
under. A contract that only adds to that domain — a new entity, a new field no row has to carry,
|
|
5
|
+
another ``domain_version`` — takes every one of them as it is, so they carry over: a journal is
|
|
6
|
+
re-targeted, a store is copied row for row (its batch ledger and lineage with it) into a store of
|
|
7
|
+
the new contract. Any other change to the stored domain is refused, naming each change.
|
|
8
|
+
|
|
9
|
+
The stored domain is proven by its compiled entities, which must hash to the domain the rows name:
|
|
10
|
+
a tree records them in ``schema/runtime.json`` until its next compile, and ``db/snapshot.json``
|
|
11
|
+
keeps them, beside the ``lock``, once the contract has moved off it.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Iterable
|
|
20
|
+
|
|
21
|
+
from .compiler import STORAGE_PARTS, Contract, StorageContract, _domain_hash
|
|
22
|
+
from .database import WorldDatabase
|
|
23
|
+
from .database.guard import RUNTIME_TABLES
|
|
24
|
+
from .validation import canonical, fail
|
|
25
|
+
|
|
26
|
+
RUNTIME = "schema/runtime.json"
|
|
27
|
+
PIN = "db/snapshot.json"
|
|
28
|
+
FORMAT_PARTS = frozenset({"language", "annotations", "compiler", "target"})
|
|
29
|
+
ENTITY_KEYS = (
|
|
30
|
+
("primary_key", "the primary key"),
|
|
31
|
+
("unique", "the unique keys"),
|
|
32
|
+
("relationships", "the relationships"),
|
|
33
|
+
)
|
|
34
|
+
SCHEMA_PARTS = frozenset({"properties", "required", "$defs"})
|
|
35
|
+
LEDGER_TABLES = ("_gateway_batches", "_gateway_lineage", *sorted(RUNTIME_TABLES))
|
|
36
|
+
ADDITIVE = "a new entity, a new field that is not required, or another domain_version"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _proves(target: dict, entities: Any, storage: str) -> bool:
|
|
40
|
+
return isinstance(entities, dict) and _domain_hash(entities, storage) == target.get("hash")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read_entities(path: Path, key: str) -> Any:
|
|
44
|
+
try:
|
|
45
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
46
|
+
except (OSError, ValueError):
|
|
47
|
+
return None
|
|
48
|
+
return document.get(key) if isinstance(document, dict) else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class DomainMigration:
|
|
53
|
+
"""The one rule for whether rows stored under ``target`` (a domain's ``id``, ``version`` and
|
|
54
|
+
entity ``hash``) carry over to ``contract``: only when the change adds to that domain. The
|
|
55
|
+
stored ``entities`` are None when nothing proves them; ``lock`` is the stored lock when the
|
|
56
|
+
rows live in a store, which is then the only kind that can be migrated."""
|
|
57
|
+
|
|
58
|
+
target: dict
|
|
59
|
+
entities: dict | None
|
|
60
|
+
contract: Contract
|
|
61
|
+
lock: dict | None = None
|
|
62
|
+
format: tuple[str, ...] = ()
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def recorded(world: str | Path) -> tuple[Any, ...]:
|
|
66
|
+
"""Where ``world`` may still record the entities its stored rows were written under."""
|
|
67
|
+
world = Path(world)
|
|
68
|
+
return (_read_entities(world / PIN, "entities"), _read_entities(world / RUNTIME, "entities"))
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def of_target(
|
|
72
|
+
cls, target: dict, contract: Contract, candidates: Iterable[Any], storage: str | None = None
|
|
73
|
+
) -> "DomainMigration":
|
|
74
|
+
storage = storage or contract.lock["target"]
|
|
75
|
+
proven = next((entities for entities in candidates if _proves(target, entities, storage)), None)
|
|
76
|
+
return cls(target, proven, contract)
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def of_lock(cls, lock: dict, contract: Contract, candidates: Iterable[Any]) -> "DomainMigration":
|
|
80
|
+
stored, current = StorageContract.of(lock), contract.storage
|
|
81
|
+
changed = tuple(
|
|
82
|
+
f"{words} ({min(getattr(stored, name))} → {min(getattr(current, name))})"
|
|
83
|
+
for name, words, _shown in STORAGE_PARTS
|
|
84
|
+
if name in FORMAT_PARTS and getattr(current, name).isdisjoint(getattr(stored, name))
|
|
85
|
+
)
|
|
86
|
+
migration = cls.of_target(lock["domain"], contract, candidates, lock.get("target"))
|
|
87
|
+
return cls(migration.target, migration.entities, contract, lock, changed)
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def carry(cls, store: Path, contract: Contract, world: str | Path) -> dict[str, int] | None:
|
|
91
|
+
"""The store at ``store`` carried in place to ``contract`` when its rows were stored under
|
|
92
|
+
a domain the contract only adds to (its counts); None when it already opens under it."""
|
|
93
|
+
lock = WorldDatabase.for_contract(contract).stored_lock(store)
|
|
94
|
+
migration = cls.of_lock(lock, contract, cls.recorded(world))
|
|
95
|
+
if not migration.needed:
|
|
96
|
+
return None
|
|
97
|
+
migration.require(
|
|
98
|
+
"state_contract",
|
|
99
|
+
f"/db/{store.name}",
|
|
100
|
+
"Import the rows into the world again, or restore the domain they were stored under",
|
|
101
|
+
)
|
|
102
|
+
fresh = store.with_name(f".{store.name}.migrating")
|
|
103
|
+
fresh.unlink(missing_ok=True)
|
|
104
|
+
counts = migration.migrate(store, fresh)
|
|
105
|
+
fresh.replace(store)
|
|
106
|
+
return counts
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def needed(self) -> bool:
|
|
110
|
+
return bool(self.format) or not self.contract.targets(self.target)
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def carries(self) -> bool:
|
|
114
|
+
return self.entities is not None and not self.refusals
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def additions(self) -> list[str]:
|
|
118
|
+
if self.entities is None:
|
|
119
|
+
return []
|
|
120
|
+
current = self.contract.entities
|
|
121
|
+
found = [f"entity {name}" for name in sorted(set(current) - set(self.entities))]
|
|
122
|
+
for name, stored in sorted(self.entities.items()):
|
|
123
|
+
now = current.get(name)
|
|
124
|
+
if now is None:
|
|
125
|
+
continue
|
|
126
|
+
held = stored["schema"].get("properties", {})
|
|
127
|
+
found.extend(f"field {name}.{f}" for f in sorted(set(now["schema"].get("properties", {})) - set(held)))
|
|
128
|
+
version = self.contract.target["version"]
|
|
129
|
+
if self.target.get("version") != version:
|
|
130
|
+
found.append(f"domain_version {self.target.get('version')} → {version}")
|
|
131
|
+
return found
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def refusals(self) -> list[str]:
|
|
135
|
+
found = [f"changes {words}" for words in self.format]
|
|
136
|
+
if self.target.get("id") != self.contract.target["id"]:
|
|
137
|
+
found.append(f"changes the domain $id ({self.target.get('id')} → {self.contract.target['id']})")
|
|
138
|
+
if self.entities is None:
|
|
139
|
+
return found
|
|
140
|
+
current = self.contract.entities
|
|
141
|
+
for name, stored in sorted(self.entities.items()):
|
|
142
|
+
found.extend(_entity_refusals(name, stored, current.get(name)))
|
|
143
|
+
return found
|
|
144
|
+
|
|
145
|
+
def require(self, code: str, pointer: str, remedy: str) -> None:
|
|
146
|
+
"""Refuse unless the stored rows carry over, saying what to do about it."""
|
|
147
|
+
if not self.needed:
|
|
148
|
+
return
|
|
149
|
+
found = self.refusals
|
|
150
|
+
if found:
|
|
151
|
+
fail(
|
|
152
|
+
code,
|
|
153
|
+
f"The rows here were stored under the previous domain, and this change {'; '.join(found)}. "
|
|
154
|
+
f"Stored rows carry over only when a change adds to their domain ({ADDITIVE}): "
|
|
155
|
+
f"keep the rest as it was, or: {remedy}",
|
|
156
|
+
pointer,
|
|
157
|
+
changed=found,
|
|
158
|
+
)
|
|
159
|
+
if self.entities is None:
|
|
160
|
+
fail(
|
|
161
|
+
code,
|
|
162
|
+
f"This change moves the domain the rows here were stored under, and nothing in the tree "
|
|
163
|
+
f"records that domain's entities any more, so the runtime cannot tell whether the change only "
|
|
164
|
+
f"adds to it ({ADDITIVE}). Compile the change on the tree the stored rows came from (a fresh "
|
|
165
|
+
f"`gateway worlds sandbox open <slug>`, or `gateway bench pull <slug>`), or: {remedy}",
|
|
166
|
+
pointer,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def view(self) -> dict:
|
|
170
|
+
return {"from": self.lock_hash, "adds": self.additions}
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def lock_hash(self) -> str:
|
|
174
|
+
from .snapshot import lock_hash
|
|
175
|
+
|
|
176
|
+
return lock_hash(self.lock) if self.lock is not None else self.target.get("hash", "")
|
|
177
|
+
|
|
178
|
+
def migrate(self, source: Path, destination: Path) -> dict[str, int]:
|
|
179
|
+
"""``source`` (a store of the stored lock) copied into a new store at ``destination`` under
|
|
180
|
+
the contract: every row, the batch ledger and the lineage, byte for byte."""
|
|
181
|
+
if self.lock is None or not self.carries:
|
|
182
|
+
fail("state_contract", "Only a store whose stored domain is proven and only added to can be migrated")
|
|
183
|
+
engine = WorldDatabase.for_contract(self.contract)
|
|
184
|
+
with WorldDatabase.open(source, self._stored_contract(), audit=False) as held:
|
|
185
|
+
counts = held.counts()
|
|
186
|
+
try:
|
|
187
|
+
WorldDatabase.create(destination, self.contract).close()
|
|
188
|
+
engine.copy_rows(source, destination, LEDGER_TABLES, sorted(self.entities))
|
|
189
|
+
with WorldDatabase.open(destination, self.contract, audit=False) as moved:
|
|
190
|
+
carried = moved.counts()
|
|
191
|
+
except BaseException:
|
|
192
|
+
destination.unlink(missing_ok=True)
|
|
193
|
+
raise
|
|
194
|
+
if any(carried.get(name) != count for name, count in counts.items()):
|
|
195
|
+
destination.unlink(missing_ok=True)
|
|
196
|
+
fail("state_contract", "The migrated store does not hold every stored row")
|
|
197
|
+
return carried
|
|
198
|
+
|
|
199
|
+
def _stored_contract(self) -> Contract:
|
|
200
|
+
target = self.lock["target"]
|
|
201
|
+
metadata = {"target": target, "domain": self.lock["domain"], "entities": self.entities, "tools": {}}
|
|
202
|
+
ddl = WorldDatabase.for_contract(Contract(metadata, {}, self.lock)).ddl(self.entities)
|
|
203
|
+
return Contract(metadata, {"db/schema.sql": ddl.encode()}, self.lock)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _entity_refusals(name: str, stored: dict, current: dict | None) -> list[str]:
|
|
207
|
+
if current is None:
|
|
208
|
+
return [f"removes entity {name}"]
|
|
209
|
+
found = [
|
|
210
|
+
f"changes {words} of {name}" for key, words in ENTITY_KEYS if canonical(stored[key]) != canonical(current[key])
|
|
211
|
+
]
|
|
212
|
+
return found + _schema_refusals(name, stored["schema"], current["schema"])
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _schema_refusals(name: str, stored: dict, current: dict) -> list[str]:
|
|
216
|
+
held, now = stored.get("properties", {}), current.get("properties", {})
|
|
217
|
+
required, requiring = set(stored.get("required", [])), set(current.get("required", []))
|
|
218
|
+
found = [f"removes field {name}.{f}" for f in sorted(set(held) - set(now))]
|
|
219
|
+
found += [
|
|
220
|
+
f"changes field {name}.{f}" for f in sorted(set(held) & set(now)) if canonical(held[f]) != canonical(now[f])
|
|
221
|
+
]
|
|
222
|
+
found += [f"requires field {name}.{f}" for f in sorted(requiring - required)]
|
|
223
|
+
found += [f"stops requiring field {name}.{f}" for f in sorted((required - requiring) & set(now))]
|
|
224
|
+
defs, defined = stored.get("$defs", {}), current.get("$defs", {})
|
|
225
|
+
found += [
|
|
226
|
+
f"changes the definition {d} {name} stores"
|
|
227
|
+
for d in sorted(defs)
|
|
228
|
+
if canonical(defs[d]) != canonical(defined.get(d))
|
|
229
|
+
]
|
|
230
|
+
rest = {key: value for key, value in stored.items() if key not in SCHEMA_PARTS}
|
|
231
|
+
if canonical(rest) != canonical({key: value for key, value in current.items() if key not in SCHEMA_PARTS}):
|
|
232
|
+
found.append(f"changes the schema of {name}")
|
|
233
|
+
return found
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
__all__ = ["DomainMigration"]
|