@withgateway/sdk 0.3.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/dist/cli.js +3971 -2338
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +3950 -2318
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +214 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +206 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
- package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1456 -564
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1433 -557
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +211 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +203 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +96 -21
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +96 -21
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +300 -123
- package/dist/worlds/index.d.ts +300 -123
- package/dist/worlds/index.js +1385 -508
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1377 -501
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -515
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +22 -14
- package/skills/connector-schema-authoring/connector-worlds.md +26 -10
- package/skills/connector-schema-authoring/world-schema.md +111 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +16 -16
- package/skills/worlds-getting-started/SKILL.md +18 -18
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,995 @@
|
|
|
1
|
+
"""The world database: one contract-bound store, whatever engine holds the rows.
|
|
2
|
+
|
|
3
|
+
:class:`WorldDatabase` owns everything a world's state promises — primary-key
|
|
4
|
+
identity, schema validation, atomic batches with lineage, the commit-time audit,
|
|
5
|
+
the guard deciding who may do what — and asks its engine subclass only for the
|
|
6
|
+
primitives: open a connection, run a statement, begin and end a transaction,
|
|
7
|
+
list the schema objects, check relationships, copy the file.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import shutil
|
|
14
|
+
import stat
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import uuid
|
|
18
|
+
from abc import ABC, abstractmethod
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any, Callable, Iterable, Iterator
|
|
22
|
+
|
|
23
|
+
from ..compiler import Contract, StorageContract, field_schema, quote, scalar_types
|
|
24
|
+
from ..primary_key import PrimaryKey
|
|
25
|
+
from ..validation import (
|
|
26
|
+
ContractError,
|
|
27
|
+
canonical,
|
|
28
|
+
digest,
|
|
29
|
+
encoded,
|
|
30
|
+
fail,
|
|
31
|
+
members,
|
|
32
|
+
nonempty,
|
|
33
|
+
parse_json,
|
|
34
|
+
validate,
|
|
35
|
+
)
|
|
36
|
+
from .audit import ContractAudit
|
|
37
|
+
from .budget import QueryBudget
|
|
38
|
+
from .connections import IngestConnection, ReadConnection
|
|
39
|
+
from .guard import PRIVATE_TABLE_ORDER, Guard
|
|
40
|
+
from .ingest import IngestSession
|
|
41
|
+
from .matching import RowMatch
|
|
42
|
+
from .provenance import validate_source
|
|
43
|
+
from .rows import Row, Rows
|
|
44
|
+
from .runtime_rows import RuntimeRows, Statement
|
|
45
|
+
|
|
46
|
+
STORED_LOCK_QUERY = "SELECT contract FROM _gateway_contract"
|
|
47
|
+
|
|
48
|
+
BOOKKEEPING_DDL = """
|
|
49
|
+
CREATE TABLE _gateway_contract (contract TEXT NOT NULL);
|
|
50
|
+
CREATE TABLE _gateway_batches (id TEXT PRIMARY KEY, digest TEXT NOT NULL, metadata TEXT NOT NULL);
|
|
51
|
+
CREATE TABLE _gateway_lineage (batch TEXT NOT NULL REFERENCES _gateway_batches(id),
|
|
52
|
+
ordinal INTEGER NOT NULL, entity TEXT NOT NULL, key TEXT NOT NULL, source TEXT NOT NULL,
|
|
53
|
+
PRIMARY KEY(batch, ordinal));
|
|
54
|
+
CREATE TABLE benchmark_runtime_context (context_key TEXT PRIMARY KEY, scenario_id TEXT NOT NULL);
|
|
55
|
+
CREATE TABLE benchmark_runtime_parameters (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
56
|
+
CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn_index INTEGER NOT NULL,
|
|
57
|
+
event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
|
|
58
|
+
|
|
59
|
+
"""
|
|
60
|
+
#: The scalar types whose stored text equals Python's ``str`` of the value, so a
|
|
61
|
+
#: ``list_where`` filter on such a column can run in SQL and answer as :class:`RowMatch` does.
|
|
62
|
+
#: A boolean is stored as 1/0 and read as True/False; it filters in Python.
|
|
63
|
+
TEXT_SCALARS = {"string", "integer", "number"}
|
|
64
|
+
_FICLONE = 0x40049409
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _reflink(source: Path, destination: Path) -> bool:
|
|
68
|
+
try:
|
|
69
|
+
import fcntl
|
|
70
|
+
|
|
71
|
+
with open(source, "rb") as src, open(destination, "wb") as dst:
|
|
72
|
+
fcntl.ioctl(dst.fileno(), _FICLONE, src.fileno())
|
|
73
|
+
return True
|
|
74
|
+
except (ImportError, OSError):
|
|
75
|
+
return False
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _sql_filterable(prop: dict) -> bool:
|
|
79
|
+
types = scalar_types(prop)
|
|
80
|
+
return bool(types) and types <= TEXT_SCALARS
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class WorldDatabase(ABC):
|
|
84
|
+
"""Use create/open as context managers; every public mutation is atomic.
|
|
85
|
+
|
|
86
|
+
Payloads are JSON values, keys are dicts of exactly the primary-key fields.
|
|
87
|
+
Updates shallow-merge provided properties and validate the complete result.
|
|
88
|
+
"""
|
|
89
|
+
|
|
90
|
+
#: The engine's name; also the key under which an ingest refusal carries the engine's own words.
|
|
91
|
+
engine: str
|
|
92
|
+
engine_label: str
|
|
93
|
+
#: The storage target a contract compiles to for this engine (the lock's ``target``).
|
|
94
|
+
target: str
|
|
95
|
+
#: The file name of a snapshot this engine writes, and its media type on the wire.
|
|
96
|
+
snapshot_name: str
|
|
97
|
+
content_type: str
|
|
98
|
+
#: The engine's error type: what a statement the engine itself refused raises.
|
|
99
|
+
native_error: type[BaseException]
|
|
100
|
+
#: Functions that reach outside the database (files, native extensions): every door but the trusted one
|
|
101
|
+
#: refuses them by name.
|
|
102
|
+
unsafe_functions: frozenset[str]
|
|
103
|
+
|
|
104
|
+
def __init__(self, path: Path, contract: Contract, *, create: bool = False, audit: bool = True):
|
|
105
|
+
self.path = Path(path)
|
|
106
|
+
self.contract = contract
|
|
107
|
+
self._lock = threading.RLock()
|
|
108
|
+
self._copies = 0
|
|
109
|
+
self._copied = threading.Condition(self._lock)
|
|
110
|
+
self._depth = 0
|
|
111
|
+
self._touched: set[tuple[str, str]] = set()
|
|
112
|
+
self._guard = Guard(frozenset(contract.entities), self.unsafe_functions)
|
|
113
|
+
self._audit = ContractAudit(self)
|
|
114
|
+
self._key_error: dict | None = None
|
|
115
|
+
self._budget: QueryBudget | None = None
|
|
116
|
+
self._deadline: float | None = None
|
|
117
|
+
self._closed = False
|
|
118
|
+
if create and self.path.exists():
|
|
119
|
+
fail(
|
|
120
|
+
"state_exists",
|
|
121
|
+
"Refusing to replace existing state; reset/rebuild explicitly",
|
|
122
|
+
)
|
|
123
|
+
if not create and not self.path.is_file():
|
|
124
|
+
fail("state_missing", "Cannot open missing state")
|
|
125
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
self._open_connection()
|
|
127
|
+
try:
|
|
128
|
+
self._enforce_strict_dialect()
|
|
129
|
+
if create:
|
|
130
|
+
self._create_schema()
|
|
131
|
+
self._check_contract()
|
|
132
|
+
self._register_key_function(self._sql_key)
|
|
133
|
+
self._install_guard()
|
|
134
|
+
if audit:
|
|
135
|
+
self.audit()
|
|
136
|
+
except BaseException:
|
|
137
|
+
self._close_connection()
|
|
138
|
+
if create:
|
|
139
|
+
self.path.unlink(missing_ok=True)
|
|
140
|
+
raise
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def engines() -> tuple[type["WorldDatabase"], ...]:
|
|
144
|
+
from .engines import ENGINES
|
|
145
|
+
|
|
146
|
+
return ENGINES
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def default_engine() -> type["WorldDatabase"]:
|
|
150
|
+
from .engines import DEFAULT_ENGINE
|
|
151
|
+
|
|
152
|
+
return DEFAULT_ENGINE
|
|
153
|
+
|
|
154
|
+
@classmethod
|
|
155
|
+
def requirement(cls) -> str:
|
|
156
|
+
return "Schema worlds require " + ", ".join(engine.engine_label for engine in cls.engines())
|
|
157
|
+
|
|
158
|
+
@classmethod
|
|
159
|
+
def engine_named(cls, name: str) -> type["WorldDatabase"]:
|
|
160
|
+
"""The engine a world's manifest names (``[db] engine``)."""
|
|
161
|
+
found = [engine for engine in cls.engines() if engine.engine == name]
|
|
162
|
+
if not found:
|
|
163
|
+
fail("target", f"{cls.requirement()}; db engine {name!r} is not one")
|
|
164
|
+
return found[0]
|
|
165
|
+
|
|
166
|
+
@classmethod
|
|
167
|
+
def for_contract(cls, contract: Contract) -> type["WorldDatabase"]:
|
|
168
|
+
"""The engine whose storage target the contract was compiled to."""
|
|
169
|
+
target = contract.metadata.get("target")
|
|
170
|
+
found = [engine for engine in cls.engines() if engine.target == target]
|
|
171
|
+
if not found:
|
|
172
|
+
fail("target", f"No world database engine stores contracts compiled to {target!r}")
|
|
173
|
+
return found[0]
|
|
174
|
+
|
|
175
|
+
@classmethod
|
|
176
|
+
def create(cls, path: str | Path, contract: Contract) -> "WorldDatabase":
|
|
177
|
+
return cls.for_contract(contract)(Path(path), contract, create=True)
|
|
178
|
+
|
|
179
|
+
@classmethod
|
|
180
|
+
def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "WorldDatabase":
|
|
181
|
+
"""Open existing state. ``audit=False`` skips the full row audit: for a snapshot
|
|
182
|
+
the store itself wrote and whose bytes a sidecar digest has just verified,
|
|
183
|
+
every row was audited when it was written, so opening is then a matter of
|
|
184
|
+
milliseconds however many rows the file holds. The contract check always runs."""
|
|
185
|
+
return cls.for_contract(contract)(Path(path), contract, audit=audit)
|
|
186
|
+
|
|
187
|
+
@classmethod
|
|
188
|
+
def open_copy(cls, source: str | Path, contract: Contract, destination: str | Path) -> "WorldDatabase":
|
|
189
|
+
"""Open a private copy of a snapshot the store wrote and a digest verified."""
|
|
190
|
+
cls.clone_file(Path(source), Path(destination))
|
|
191
|
+
return cls.open(destination, contract, audit=False)
|
|
192
|
+
|
|
193
|
+
@staticmethod
|
|
194
|
+
def clone_file(source: Path, destination: Path) -> None:
|
|
195
|
+
"""Copy a database file cheaply: a reflink (``FICLONE``) shares the source's
|
|
196
|
+
blocks copy-on-write, so a session costs the same whatever the snapshot's size;
|
|
197
|
+
a file system or platform without reflink falls back to a plain copy."""
|
|
198
|
+
if not stat.S_ISREG(source.stat().st_mode):
|
|
199
|
+
raise shutil.SpecialFileError(f"{source} is not a regular file")
|
|
200
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
201
|
+
if destination.exists() and destination.samefile(source):
|
|
202
|
+
raise shutil.SameFileError(f"{source} and {destination} are the same file")
|
|
203
|
+
if not _reflink(source, destination):
|
|
204
|
+
shutil.copyfile(source, destination)
|
|
205
|
+
|
|
206
|
+
@classmethod
|
|
207
|
+
@abstractmethod
|
|
208
|
+
def ddl(cls, entities: dict) -> str:
|
|
209
|
+
"""The storage DDL a contract's compiled entities become (``db/schema.sql``)."""
|
|
210
|
+
|
|
211
|
+
@staticmethod
|
|
212
|
+
@abstractmethod
|
|
213
|
+
def split_statements(script: str) -> list[str]:
|
|
214
|
+
"""A script as its complete statements, comment-only text dropped."""
|
|
215
|
+
|
|
216
|
+
@staticmethod
|
|
217
|
+
@abstractmethod
|
|
218
|
+
def missing_table(error: BaseException) -> str | None:
|
|
219
|
+
"""The table an engine error says does not exist, or ``None``."""
|
|
220
|
+
|
|
221
|
+
@staticmethod
|
|
222
|
+
@abstractmethod
|
|
223
|
+
def exceeds_limit(error: BaseException) -> bool:
|
|
224
|
+
"""Whether an engine error says a value or a row passed the engine's size limits."""
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
@abstractmethod
|
|
228
|
+
def carry_entries(path: Path, entries: list[str]) -> None:
|
|
229
|
+
"""Write ``entries`` into a closed database file beside its rows, in a table the store never opens."""
|
|
230
|
+
|
|
231
|
+
@staticmethod
|
|
232
|
+
@abstractmethod
|
|
233
|
+
def stored_lock(path: Path) -> dict:
|
|
234
|
+
"""The lock a database file records its rows under, read without opening it as a store."""
|
|
235
|
+
|
|
236
|
+
@staticmethod
|
|
237
|
+
@abstractmethod
|
|
238
|
+
def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
|
|
239
|
+
"""Every row of ``source`` copied into the new database at ``destination`` in one transaction:
|
|
240
|
+
the ``ledger`` tables whole, each entity's keys and payloads, relationships checked at commit."""
|
|
241
|
+
|
|
242
|
+
@staticmethod
|
|
243
|
+
@abstractmethod
|
|
244
|
+
def take_entries(path: Path) -> list[str]:
|
|
245
|
+
"""The entries :meth:`carry_entries` wrote into ``path``, in order, dropped from it; empty when none."""
|
|
246
|
+
|
|
247
|
+
@abstractmethod
|
|
248
|
+
def _open_connection(self) -> None: ...
|
|
249
|
+
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def _enforce_strict_dialect(self) -> None:
|
|
252
|
+
"""Refuse what would let a verifier score silently wrong (a missing column
|
|
253
|
+
read as a literal) and turn on relationship enforcement."""
|
|
254
|
+
|
|
255
|
+
@abstractmethod
|
|
256
|
+
def _execute(self, sql: str, params: Any = ()) -> Any: ...
|
|
257
|
+
|
|
258
|
+
@abstractmethod
|
|
259
|
+
def _executemany(self, sql: str, rows: Any) -> Any: ...
|
|
260
|
+
|
|
261
|
+
@abstractmethod
|
|
262
|
+
def _execute_script(self, script: str) -> None: ...
|
|
263
|
+
|
|
264
|
+
@abstractmethod
|
|
265
|
+
def _schema_objects(self) -> dict[str, str]:
|
|
266
|
+
"""Every schema object the file holds, by name, with its DDL."""
|
|
267
|
+
|
|
268
|
+
@abstractmethod
|
|
269
|
+
def _expected_schema_objects(self) -> dict[str, str]:
|
|
270
|
+
"""The schema objects a fresh file built from :meth:`_schema_script` holds."""
|
|
271
|
+
|
|
272
|
+
@abstractmethod
|
|
273
|
+
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
|
|
274
|
+
"""Make ``gateway_key(entity, payload)`` callable from SQL."""
|
|
275
|
+
|
|
276
|
+
@abstractmethod
|
|
277
|
+
def _install_guard(self) -> None:
|
|
278
|
+
"""Route every access the engine makes through :meth:`_guard_allows`."""
|
|
279
|
+
|
|
280
|
+
@abstractmethod
|
|
281
|
+
def _begin_write(self) -> None:
|
|
282
|
+
"""Begin a transaction that holds the write lock from its first statement."""
|
|
283
|
+
|
|
284
|
+
@abstractmethod
|
|
285
|
+
def _commit(self) -> None: ...
|
|
286
|
+
|
|
287
|
+
@abstractmethod
|
|
288
|
+
def _rollback(self) -> None: ...
|
|
289
|
+
|
|
290
|
+
@abstractmethod
|
|
291
|
+
def _savepoint(self, name: str) -> None: ...
|
|
292
|
+
|
|
293
|
+
@abstractmethod
|
|
294
|
+
def _release(self, name: str) -> None: ...
|
|
295
|
+
|
|
296
|
+
@abstractmethod
|
|
297
|
+
def _rollback_to(self, name: str) -> None: ...
|
|
298
|
+
|
|
299
|
+
@abstractmethod
|
|
300
|
+
def _page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
|
|
301
|
+
"""``LIMIT``/``OFFSET`` text and its parameters; ``limit=None`` is every row."""
|
|
302
|
+
|
|
303
|
+
@abstractmethod
|
|
304
|
+
def _relationships_broken(self) -> bool:
|
|
305
|
+
"""Whether any persisted row references a row that does not exist."""
|
|
306
|
+
|
|
307
|
+
@abstractmethod
|
|
308
|
+
def _is_constraint_violation(self, error: BaseException) -> bool:
|
|
309
|
+
"""Whether a row write failed on a declared key or constraint."""
|
|
310
|
+
|
|
311
|
+
@abstractmethod
|
|
312
|
+
def _drop_staging(self) -> None:
|
|
313
|
+
"""Drop the TEMP tables and views an ingest session left behind."""
|
|
314
|
+
|
|
315
|
+
@abstractmethod
|
|
316
|
+
def _attach_file(self, name: str, path: str | Path) -> None: ...
|
|
317
|
+
|
|
318
|
+
@abstractmethod
|
|
319
|
+
def _write_copy(self, destination: Path) -> None:
|
|
320
|
+
"""A consistent copy of the committed state at ``destination``."""
|
|
321
|
+
|
|
322
|
+
@abstractmethod
|
|
323
|
+
def _close_connection(self) -> None: ...
|
|
324
|
+
|
|
325
|
+
@abstractmethod
|
|
326
|
+
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
|
|
327
|
+
"""Interrupt any statement still running at ``deadline`` (``time.monotonic``) and refuse a value
|
|
328
|
+
or a row wider than ``budget`` allows; ``None`` restores what the connection had before."""
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
@abstractmethod
|
|
332
|
+
def insertion_order(self) -> str:
|
|
333
|
+
"""An expression ordering a bookkeeping table's rows as they were inserted."""
|
|
334
|
+
|
|
335
|
+
def _schema_script(self) -> str:
|
|
336
|
+
return self.contract.artifacts["db/schema.sql"].decode() + BOOKKEEPING_DDL
|
|
337
|
+
|
|
338
|
+
def _create_schema(self) -> None:
|
|
339
|
+
self._execute_script(self._schema_script())
|
|
340
|
+
self._execute("INSERT INTO _gateway_contract VALUES (?)", (canonical(self.contract.lock),))
|
|
341
|
+
|
|
342
|
+
def _check_contract(self) -> None:
|
|
343
|
+
try:
|
|
344
|
+
# The storage contract, never the whole lock: a tool or package edit touches no row.
|
|
345
|
+
changed = self.contract.storage.changes_from(self._stored_contract())
|
|
346
|
+
if changed:
|
|
347
|
+
message = (
|
|
348
|
+
f"State belongs to a different schema: {' and '.join(changed)} changed; no automatic migration"
|
|
349
|
+
)
|
|
350
|
+
fail("state_contract", message, changed=changed)
|
|
351
|
+
# The live DDL is compared with freshly compiled DDL, not a stored hash a tampered file could carry.
|
|
352
|
+
expected = self._expected_schema_objects()
|
|
353
|
+
actual = self._schema_objects()
|
|
354
|
+
if any(actual.get(k) != v for k, v in expected.items()) or set(actual) - set(expected):
|
|
355
|
+
fail(
|
|
356
|
+
"state_contract",
|
|
357
|
+
"State DDL differs from the compiled storage contract",
|
|
358
|
+
)
|
|
359
|
+
except self.native_error as error:
|
|
360
|
+
raise ContractError("state_contract", "State has no compatible schema contract") from error
|
|
361
|
+
|
|
362
|
+
def _stored_contract(self) -> StorageContract:
|
|
363
|
+
return StorageContract.of(self.lock_from(self._execute(STORED_LOCK_QUERY).fetchall()))
|
|
364
|
+
|
|
365
|
+
@staticmethod
|
|
366
|
+
def lock_from(rows: Iterable[Any]) -> dict:
|
|
367
|
+
"""The one lock a database's ``_gateway_contract`` rows record."""
|
|
368
|
+
try:
|
|
369
|
+
(lock,) = [json.loads(text) for (text,) in rows]
|
|
370
|
+
except (TypeError, ValueError):
|
|
371
|
+
fail("state_contract", "State records no single readable contract")
|
|
372
|
+
return lock
|
|
373
|
+
|
|
374
|
+
def _guard_allows(self, access) -> bool:
|
|
375
|
+
return self._guard.allows(access)
|
|
376
|
+
|
|
377
|
+
def _sql_key(self, entity: Any, payload: Any) -> str | None:
|
|
378
|
+
"""``gateway_key(entity, payload)``: the encoded ``_key`` for a payload.
|
|
379
|
+
|
|
380
|
+
Returns NULL on an invalid entity or payload so the NOT NULL constraint
|
|
381
|
+
rejects the row; the reason is attached to the resulting error.
|
|
382
|
+
"""
|
|
383
|
+
try:
|
|
384
|
+
value = payload
|
|
385
|
+
if isinstance(payload, (str, bytes)):
|
|
386
|
+
value = parse_json(payload)
|
|
387
|
+
if not isinstance(value, dict):
|
|
388
|
+
fail("key", "gateway_key needs a JSON object payload", entity=entity)
|
|
389
|
+
return self._key(entity, value)
|
|
390
|
+
except (ContractError, ValueError, TypeError) as error:
|
|
391
|
+
detail = getattr(error, "detail", None)
|
|
392
|
+
self._key_error = detail or {"code": "key", "message": str(error)}
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
def _trusted(self):
|
|
396
|
+
return self._guard.trusted()
|
|
397
|
+
|
|
398
|
+
@contextmanager
|
|
399
|
+
def transaction(self) -> Iterator["WorldDatabase"]:
|
|
400
|
+
with self._lock:
|
|
401
|
+
if self._guard.ingesting:
|
|
402
|
+
fail(
|
|
403
|
+
"transaction",
|
|
404
|
+
"Use SQL inside an ingest session; store mutations belong outside",
|
|
405
|
+
)
|
|
406
|
+
outer = self._depth == 0
|
|
407
|
+
savepoint = f"gateway_batch_{self._depth}"
|
|
408
|
+
with self._trusted():
|
|
409
|
+
self._enter_level(outer, savepoint)
|
|
410
|
+
if outer:
|
|
411
|
+
self._touched = set()
|
|
412
|
+
self._depth += 1
|
|
413
|
+
try:
|
|
414
|
+
yield self
|
|
415
|
+
if outer and self._touched:
|
|
416
|
+
self._audit.touched(self._touched)
|
|
417
|
+
with self._trusted():
|
|
418
|
+
self._leave_level(outer, savepoint)
|
|
419
|
+
except BaseException:
|
|
420
|
+
with self._trusted():
|
|
421
|
+
self._abort_level(outer, savepoint)
|
|
422
|
+
raise
|
|
423
|
+
finally:
|
|
424
|
+
self._depth -= 1
|
|
425
|
+
|
|
426
|
+
def _enter_level(self, outer: bool, savepoint: str) -> None:
|
|
427
|
+
if outer:
|
|
428
|
+
self._begin_write()
|
|
429
|
+
return
|
|
430
|
+
self._savepoint(savepoint)
|
|
431
|
+
|
|
432
|
+
def _leave_level(self, outer: bool, savepoint: str) -> None:
|
|
433
|
+
if outer:
|
|
434
|
+
self._commit()
|
|
435
|
+
return
|
|
436
|
+
self._release(savepoint)
|
|
437
|
+
|
|
438
|
+
def _abort_level(self, outer: bool, savepoint: str) -> None:
|
|
439
|
+
if outer:
|
|
440
|
+
self._rollback()
|
|
441
|
+
return
|
|
442
|
+
self._rollback_to(savepoint)
|
|
443
|
+
self._release(savepoint)
|
|
444
|
+
|
|
445
|
+
def _entity(self, name: str) -> dict:
|
|
446
|
+
if not isinstance(name, str) or name not in self.contract.entities:
|
|
447
|
+
fail("entity", "Unknown entity type", entity=name)
|
|
448
|
+
return self.contract.entities[name]
|
|
449
|
+
|
|
450
|
+
def _key(self, name: str, value: dict, *, exact: bool = False) -> str:
|
|
451
|
+
entity = self._entity(name)
|
|
452
|
+
key = PrimaryKey.of(entity)
|
|
453
|
+
if key.missing(value) is not None or (exact and set(value) != set(key.fields)):
|
|
454
|
+
fail("key", "Supply exactly the declared primary-key fields", entity=name)
|
|
455
|
+
values = key.values(value)
|
|
456
|
+
for field, item in zip(key.fields, values):
|
|
457
|
+
validate(
|
|
458
|
+
field_schema(entity["schema"], field, entity["location"]),
|
|
459
|
+
item,
|
|
460
|
+
location=f"{entity['location']}/schema/properties/{field}",
|
|
461
|
+
)
|
|
462
|
+
# JSON Schema integers include 1.0: the key's identity normalizes integral floats, never the payload.
|
|
463
|
+
return canonical([int(item) if type(item) is float else item for item in values])
|
|
464
|
+
|
|
465
|
+
def get(self, entity: str, key: dict) -> dict | None:
|
|
466
|
+
with self._lock:
|
|
467
|
+
encoded_key = self._key(entity, key, exact=True)
|
|
468
|
+
with self._trusted():
|
|
469
|
+
row = self._execute(
|
|
470
|
+
f"SELECT payload FROM {quote(entity)} WHERE _key = ?",
|
|
471
|
+
(encoded_key,),
|
|
472
|
+
).fetchone()
|
|
473
|
+
return json.loads(row[0]) if row else None
|
|
474
|
+
|
|
475
|
+
def list(self, entity: str) -> list[dict]:
|
|
476
|
+
with self._lock:
|
|
477
|
+
self._entity(entity)
|
|
478
|
+
with self._trusted():
|
|
479
|
+
rows = self._execute(f"SELECT payload FROM {quote(entity)} ORDER BY _key").fetchall()
|
|
480
|
+
return [json.loads(row[0]) for row in rows]
|
|
481
|
+
|
|
482
|
+
def list_where(
|
|
483
|
+
self,
|
|
484
|
+
entity: str,
|
|
485
|
+
where: dict[str, list],
|
|
486
|
+
limit: int | None = None,
|
|
487
|
+
offset: int = 0,
|
|
488
|
+
) -> tuple[list[dict], int]:
|
|
489
|
+
"""A page of the rows of ``entity`` whose every ``where`` field holds one of
|
|
490
|
+
the listed values, and how many rows match in all: ``(rows, total)``.
|
|
491
|
+
|
|
492
|
+
Equality is the routing rule of :class:`RowMatch`: the stored value as
|
|
493
|
+
text, case-insensitively; a list of wanted values matches when any does
|
|
494
|
+
(IN); a list field matches when any element does. A field projected to a
|
|
495
|
+
column (string, integer, number) filters in SQL through the read-only
|
|
496
|
+
guard, so a large world pages without loading; a field that is not
|
|
497
|
+
(an array, an object, a boolean) filters the listed rows in Python with the
|
|
498
|
+
same rule, so both paths answer alike. A missing field never matches."""
|
|
499
|
+
with self._lock:
|
|
500
|
+
spec = self._entity(entity)
|
|
501
|
+
properties = spec["schema"].get("properties", {})
|
|
502
|
+
unknown = sorted(set(where) - set(properties))
|
|
503
|
+
if unknown:
|
|
504
|
+
fail(
|
|
505
|
+
"field",
|
|
506
|
+
f"Unknown field {', '.join(unknown)} on {entity}; its fields are {', '.join(sorted(properties))}",
|
|
507
|
+
entity=entity,
|
|
508
|
+
fields=sorted(properties),
|
|
509
|
+
)
|
|
510
|
+
if offset < 0:
|
|
511
|
+
fail("input", "offset must be zero or a positive integer")
|
|
512
|
+
wanted = {field: [str(value) for value in values] for field, values in where.items()}
|
|
513
|
+
if any(not values for values in wanted.values()):
|
|
514
|
+
return [], 0
|
|
515
|
+
if all(_sql_filterable(properties[field]) for field in wanted):
|
|
516
|
+
return self._list_where_sql(entity, wanted, limit, offset)
|
|
517
|
+
matched = RowMatch.on_fields(wanted).select(self.list(entity))
|
|
518
|
+
end = None if limit is None else offset + limit
|
|
519
|
+
return matched[offset:end], len(matched)
|
|
520
|
+
|
|
521
|
+
def _list_where_sql(
|
|
522
|
+
self, entity: str, wanted: dict[str, list[str]], limit: int | None, offset: int
|
|
523
|
+
) -> tuple[list[dict], int]:
|
|
524
|
+
clauses = []
|
|
525
|
+
params: list = []
|
|
526
|
+
for field, values in wanted.items():
|
|
527
|
+
clauses.append(f"lower(CAST({quote(field)} AS TEXT)) IN ({', '.join('?' for _ in values)})")
|
|
528
|
+
params.extend(value.lower() for value in values)
|
|
529
|
+
condition = f" WHERE {' AND '.join(clauses)}" if clauses else ""
|
|
530
|
+
table = quote(entity)
|
|
531
|
+
total = self.query(f"SELECT COUNT(*) FROM {table}{condition}", tuple(params))[0][0]
|
|
532
|
+
clause, paging = self._page_clause(limit, offset)
|
|
533
|
+
page = self.query(f"SELECT payload FROM {table}{condition} ORDER BY _key{clause}", (*params, *paging))
|
|
534
|
+
return [json.loads(row[0]) for row in page], total
|
|
535
|
+
|
|
536
|
+
def attach(self, name: str, path: str | Path, entities: Iterable[str]) -> None:
|
|
537
|
+
"""Read another store's file as ``<name>.<table>`` from :meth:`query` and
|
|
538
|
+
the verifiers that run through it: a cross-world check joins its own
|
|
539
|
+
rows to a dependency world's (``SELECT ... FROM slack.slack_messages``).
|
|
540
|
+
Only the ``entities`` named are readable and nothing is writable, so a
|
|
541
|
+
tool of this world cannot reach into another's store."""
|
|
542
|
+
if self._guard.is_attached(name):
|
|
543
|
+
fail("attach", f"'{name}' is attached already")
|
|
544
|
+
with self._lock, self._trusted():
|
|
545
|
+
self._attach_file(name, path)
|
|
546
|
+
self._guard.attach(name, frozenset(entities))
|
|
547
|
+
|
|
548
|
+
@property
|
|
549
|
+
def attached(self) -> dict[str, frozenset[str]]:
|
|
550
|
+
"""The attached stores by name, each with the entities readable there."""
|
|
551
|
+
return dict(self._guard.attached)
|
|
552
|
+
|
|
553
|
+
def read_connection(self) -> ReadConnection:
|
|
554
|
+
"""What a tool handler receives as ``conn``: reads through the read-only guard, no writes."""
|
|
555
|
+
return ReadConnection(self)
|
|
556
|
+
|
|
557
|
+
def key_of(self, entity: str, row: dict) -> str:
|
|
558
|
+
"""The encoded identity of ``row`` in ``entity``: its primary-key fields, validated."""
|
|
559
|
+
with self._lock:
|
|
560
|
+
return self._key(entity, row)
|
|
561
|
+
|
|
562
|
+
def keys(self, entity: str) -> set[str]:
|
|
563
|
+
"""The encoded identity of every row ``entity`` holds."""
|
|
564
|
+
with self._lock:
|
|
565
|
+
self._entity(entity)
|
|
566
|
+
with self._trusted():
|
|
567
|
+
return {row[0] for row in self._execute(f"SELECT _key FROM {quote(entity)}")}
|
|
568
|
+
|
|
569
|
+
def set_task_context(self, scenario_id: str) -> None:
|
|
570
|
+
"""Name the active task in ``benchmark_runtime_context``, so a verifier keys off its scenario."""
|
|
571
|
+
self._write_runtime(RuntimeRows.task_context(scenario_id))
|
|
572
|
+
|
|
573
|
+
def set_run_parameters(self, parameters: dict[str, str]) -> None:
|
|
574
|
+
"""Replace ``benchmark_runtime_parameters`` with the run's condition bindings."""
|
|
575
|
+
self._write_runtime(RuntimeRows.run_parameters(parameters))
|
|
576
|
+
|
|
577
|
+
def record_conversation_event(
|
|
578
|
+
self, scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
|
|
579
|
+
) -> None:
|
|
580
|
+
"""Append one turn event to ``benchmark_conversation_events``, read only by a verifier."""
|
|
581
|
+
self._write_runtime(
|
|
582
|
+
RuntimeRows.conversation_event(scenario_id, user_turn_index, event_type, actor, name, payload_json)
|
|
583
|
+
)
|
|
584
|
+
|
|
585
|
+
def _write_runtime(self, statements: list[Statement]) -> None:
|
|
586
|
+
"""Store-authored writes to the runtime tables, all or none. They are not
|
|
587
|
+
entities, so no audit runs; inside an open transaction they ride a savepoint."""
|
|
588
|
+
with self._lock:
|
|
589
|
+
if self._guard.ingesting:
|
|
590
|
+
fail("transaction", "Runtime rows are written outside an ingest session")
|
|
591
|
+
outer = self._depth == 0
|
|
592
|
+
savepoint = f"gateway_runtime_{self._depth}"
|
|
593
|
+
with self._trusted():
|
|
594
|
+
self._enter_level(outer, savepoint)
|
|
595
|
+
try:
|
|
596
|
+
for sql, params in statements:
|
|
597
|
+
self._execute(sql, params)
|
|
598
|
+
self._leave_level(outer, savepoint)
|
|
599
|
+
except BaseException:
|
|
600
|
+
self._abort_level(outer, savepoint)
|
|
601
|
+
raise
|
|
602
|
+
|
|
603
|
+
def query(self, sql: str, params: tuple = (), *, private: bool = False, limit: int | None = None) -> Rows:
|
|
604
|
+
"""Read-only SQL over domain tables, enforced by the guard; rows are :class:`Row`, never the
|
|
605
|
+
engine's own. With ``limit``, at most that many rows are read from the engine."""
|
|
606
|
+
with self._lock, self._guard.reading(private):
|
|
607
|
+
cursor = None
|
|
608
|
+
try:
|
|
609
|
+
cursor = self._execute(sql, params)
|
|
610
|
+
return Row.all(cursor.description, self._fetch(cursor, limit))
|
|
611
|
+
except self.native_error as error:
|
|
612
|
+
raise self._refusal(error) from error
|
|
613
|
+
finally:
|
|
614
|
+
if cursor is not None:
|
|
615
|
+
cursor.close()
|
|
616
|
+
|
|
617
|
+
def _fetch(self, cursor: Any, limit: int | None) -> list:
|
|
618
|
+
if self._budget is None:
|
|
619
|
+
return cursor.fetchall() if limit is None else cursor.fetchmany(limit)
|
|
620
|
+
rows: list = []
|
|
621
|
+
spent = 0
|
|
622
|
+
while limit is None or len(rows) < limit:
|
|
623
|
+
row = cursor.fetchone()
|
|
624
|
+
if row is None:
|
|
625
|
+
break
|
|
626
|
+
spent += QueryBudget.size_of(row)
|
|
627
|
+
if spent > self._budget.answer_bytes:
|
|
628
|
+
fail("size", f"The answer passes {self._budget.answer_bytes} bytes; select fewer columns or rows")
|
|
629
|
+
rows.append(row)
|
|
630
|
+
return rows
|
|
631
|
+
|
|
632
|
+
def _refusal(self, error: BaseException) -> ContractError:
|
|
633
|
+
if self._deadline is not None and time.monotonic() >= self._deadline:
|
|
634
|
+
return ContractError("time", "The query ran past its time budget; narrow it or add a LIMIT")
|
|
635
|
+
if self.exceeds_limit(error):
|
|
636
|
+
cap = f" of {self._budget.value_bytes} bytes" if self._budget is not None else ""
|
|
637
|
+
return ContractError("size", f"A value passes the size limit{cap}; select fewer or smaller columns")
|
|
638
|
+
# The engine's words ("no such column") reach the author; a bare refusal grades as a silent zero.
|
|
639
|
+
return ContractError("read_only", f"Query failed or attempted a forbidden operation: {error}")
|
|
640
|
+
|
|
641
|
+
@contextmanager
|
|
642
|
+
def bounded(self, budget: QueryBudget) -> Iterator["WorldDatabase"]:
|
|
643
|
+
"""Every :meth:`query` inside runs within ``budget``: past its time the statement is stopped
|
|
644
|
+
(``time``), past its answer bytes the read is refused (``size``), and the engine refuses a
|
|
645
|
+
value larger than it allows (``size``). One budget at a time per connection."""
|
|
646
|
+
with self._lock:
|
|
647
|
+
if self._budget is not None:
|
|
648
|
+
fail("budget", "A query budget is already in force on this connection")
|
|
649
|
+
try:
|
|
650
|
+
self._budget = budget
|
|
651
|
+
self._deadline = time.monotonic() + budget.seconds
|
|
652
|
+
self._set_budget(budget, self._deadline)
|
|
653
|
+
yield self
|
|
654
|
+
finally:
|
|
655
|
+
self._budget = None
|
|
656
|
+
self._deadline = None
|
|
657
|
+
self._set_budget(None, None)
|
|
658
|
+
|
|
659
|
+
def public_copy(self, destination: str | Path) -> "WorldDatabase":
|
|
660
|
+
"""A copy of the committed state at ``destination`` holding no bookkeeping or private runtime
|
|
661
|
+
rows (contract, batches, lineage, task context, conversation events): what a caller's own SQL
|
|
662
|
+
may open, so a read the engine's guard misses finds nothing private to return."""
|
|
663
|
+
with self._lock:
|
|
664
|
+
if self._depth:
|
|
665
|
+
fail("transaction", "Cannot copy uncommitted state")
|
|
666
|
+
dest = Path(destination)
|
|
667
|
+
if dest.exists():
|
|
668
|
+
fail("state_exists", "Copy destination already exists")
|
|
669
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
670
|
+
self._write_copy(dest)
|
|
671
|
+
copy = type(self)(dest, self.contract, audit=False)
|
|
672
|
+
try:
|
|
673
|
+
copy._forget_private()
|
|
674
|
+
except BaseException:
|
|
675
|
+
copy._close_connection()
|
|
676
|
+
raise
|
|
677
|
+
return copy
|
|
678
|
+
|
|
679
|
+
def _forget_private(self) -> None:
|
|
680
|
+
with self._lock, self._trusted():
|
|
681
|
+
self._begin_write()
|
|
682
|
+
for table in PRIVATE_TABLE_ORDER:
|
|
683
|
+
self._execute(f"DELETE FROM {quote(table)}")
|
|
684
|
+
self._commit()
|
|
685
|
+
|
|
686
|
+
def _mutate(self, mutation: dict) -> str:
|
|
687
|
+
name = mutation["entity"]
|
|
688
|
+
entity = self._entity(name)
|
|
689
|
+
operation = mutation["op"]
|
|
690
|
+
if operation == "insert":
|
|
691
|
+
value = mutation["value"]
|
|
692
|
+
validate(entity["schema"], value, location=f"{entity['location']}/schema")
|
|
693
|
+
key = self._key(name, value)
|
|
694
|
+
else:
|
|
695
|
+
key = self._key(name, mutation["key"], exact=True)
|
|
696
|
+
value = self._merged(name, entity, key, mutation)
|
|
697
|
+
self._touched.add((name, key))
|
|
698
|
+
try:
|
|
699
|
+
with self._trusted():
|
|
700
|
+
self._write_row(operation, name, key, value)
|
|
701
|
+
except self.native_error as error:
|
|
702
|
+
if not self._is_constraint_violation(error):
|
|
703
|
+
raise
|
|
704
|
+
raise ContractError(
|
|
705
|
+
"unique",
|
|
706
|
+
"Primary or unique key conflicts with existing state",
|
|
707
|
+
entity=name,
|
|
708
|
+
key=json.loads(key),
|
|
709
|
+
schema_path=f"{entity['location']}/unique",
|
|
710
|
+
) from error
|
|
711
|
+
return key
|
|
712
|
+
|
|
713
|
+
def _merged(self, name: str, entity: dict, key: str, mutation: dict) -> dict | None:
|
|
714
|
+
"""The row an update leaves (validated, key unchanged); ``None`` for a delete."""
|
|
715
|
+
current = self.get(name, mutation["key"])
|
|
716
|
+
if current is None:
|
|
717
|
+
fail(
|
|
718
|
+
"not_found",
|
|
719
|
+
"No entity exists at this key",
|
|
720
|
+
entity=name,
|
|
721
|
+
key=mutation["key"],
|
|
722
|
+
)
|
|
723
|
+
if mutation["op"] != "update":
|
|
724
|
+
return None
|
|
725
|
+
if not isinstance(mutation["value"], dict):
|
|
726
|
+
fail("type", "An update must be an object")
|
|
727
|
+
value = {**current, **mutation["value"]}
|
|
728
|
+
validate(entity["schema"], value, location=f"{entity['location']}/schema")
|
|
729
|
+
if self._key(name, value) != key:
|
|
730
|
+
fail(
|
|
731
|
+
"key_immutable",
|
|
732
|
+
"Updates cannot change primary keys",
|
|
733
|
+
entity=name,
|
|
734
|
+
)
|
|
735
|
+
return value
|
|
736
|
+
|
|
737
|
+
def _write_row(self, operation: str, name: str, key: str, value: dict | None) -> None:
|
|
738
|
+
table = quote(name)
|
|
739
|
+
if operation == "insert":
|
|
740
|
+
self._execute(f"INSERT INTO {table} (_key, payload) VALUES (?, ?)", (key, canonical(value)))
|
|
741
|
+
return
|
|
742
|
+
if operation == "update":
|
|
743
|
+
self._execute(f"UPDATE {table} SET payload = ? WHERE _key = ?", (canonical(value), key))
|
|
744
|
+
return
|
|
745
|
+
self._execute(f"DELETE FROM {table} WHERE _key = ?", (key,))
|
|
746
|
+
|
|
747
|
+
def apply_batch(self, batch: dict) -> dict:
|
|
748
|
+
"""Atomic rejection; exact retries no-op, conflicting ID reuse rejects."""
|
|
749
|
+
members(
|
|
750
|
+
batch,
|
|
751
|
+
{"id", "target", "mapping", "coverage", "mutations"},
|
|
752
|
+
{"id", "target", "mapping", "coverage", "mutations"},
|
|
753
|
+
"/batch",
|
|
754
|
+
)
|
|
755
|
+
nonempty(batch["id"], "/batch/id")
|
|
756
|
+
if not self.contract.targets(batch["target"]):
|
|
757
|
+
fail(
|
|
758
|
+
"batch_target",
|
|
759
|
+
"Batch targets a different domain identity/version/hash",
|
|
760
|
+
"/batch/target",
|
|
761
|
+
)
|
|
762
|
+
members(batch["mapping"], {"id", "version"}, {"id", "version"}, "/batch/mapping")
|
|
763
|
+
for key, value in batch["mapping"].items():
|
|
764
|
+
nonempty(value, f"/batch/mapping/{key}")
|
|
765
|
+
if batch["coverage"] not in ("partial", "complete"):
|
|
766
|
+
fail(
|
|
767
|
+
"coverage",
|
|
768
|
+
"coverage must be partial or complete (neither implies deletion)",
|
|
769
|
+
"/batch/coverage",
|
|
770
|
+
)
|
|
771
|
+
if not isinstance(batch["mutations"], list):
|
|
772
|
+
fail("batch", "mutations must be an array", "/batch/mutations")
|
|
773
|
+
batch_hash = digest(encoded(batch))
|
|
774
|
+
with self.transaction():
|
|
775
|
+
with self._trusted():
|
|
776
|
+
prior = self._execute("SELECT digest FROM _gateway_batches WHERE id = ?", (batch["id"],)).fetchone()
|
|
777
|
+
if prior:
|
|
778
|
+
if prior[0] != batch_hash:
|
|
779
|
+
fail(
|
|
780
|
+
"idempotency_conflict",
|
|
781
|
+
"Batch ID was already applied with different content",
|
|
782
|
+
"/batch/id",
|
|
783
|
+
batch=batch["id"],
|
|
784
|
+
)
|
|
785
|
+
return {"status": "already_applied", "batch": batch["id"], "applied": 0}
|
|
786
|
+
with self._trusted():
|
|
787
|
+
self._execute(
|
|
788
|
+
"INSERT INTO _gateway_batches VALUES (?, ?, ?)",
|
|
789
|
+
(
|
|
790
|
+
batch["id"],
|
|
791
|
+
batch_hash,
|
|
792
|
+
canonical({k: v for k, v in batch.items() if k != "mutations"}),
|
|
793
|
+
),
|
|
794
|
+
)
|
|
795
|
+
for index, mutation in enumerate(batch["mutations"]):
|
|
796
|
+
self._apply_mutation(batch["id"], index, mutation)
|
|
797
|
+
return {
|
|
798
|
+
"status": "applied",
|
|
799
|
+
"batch": batch["id"],
|
|
800
|
+
"applied": len(batch["mutations"]),
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
def _apply_mutation(self, batch_id: str, index: int, mutation: Any) -> None:
|
|
804
|
+
location = f"/batch/mutations/{index}"
|
|
805
|
+
try:
|
|
806
|
+
members(
|
|
807
|
+
mutation,
|
|
808
|
+
{"op", "entity", "key", "value", "source"},
|
|
809
|
+
{"op", "entity", "source"},
|
|
810
|
+
location,
|
|
811
|
+
)
|
|
812
|
+
op = mutation["op"]
|
|
813
|
+
if not isinstance(op, str):
|
|
814
|
+
fail(
|
|
815
|
+
"operation",
|
|
816
|
+
"Operation must be insert, update or delete",
|
|
817
|
+
location,
|
|
818
|
+
)
|
|
819
|
+
required = {
|
|
820
|
+
"insert": {"value"},
|
|
821
|
+
"update": {"key", "value"},
|
|
822
|
+
"delete": {"key"},
|
|
823
|
+
}.get(op)
|
|
824
|
+
if required is None:
|
|
825
|
+
fail(
|
|
826
|
+
"operation",
|
|
827
|
+
"Only insert, update and delete are supported",
|
|
828
|
+
location,
|
|
829
|
+
)
|
|
830
|
+
members(
|
|
831
|
+
mutation,
|
|
832
|
+
{"op", "entity", "source"} | required,
|
|
833
|
+
{"op", "entity", "source"} | required,
|
|
834
|
+
location,
|
|
835
|
+
)
|
|
836
|
+
validate_source(mutation["source"], f"{location}/source")
|
|
837
|
+
key = self._mutate(mutation)
|
|
838
|
+
with self._trusted():
|
|
839
|
+
self._execute(
|
|
840
|
+
"INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
|
|
841
|
+
(
|
|
842
|
+
batch_id,
|
|
843
|
+
index,
|
|
844
|
+
mutation["entity"],
|
|
845
|
+
key,
|
|
846
|
+
canonical(mutation["source"]),
|
|
847
|
+
),
|
|
848
|
+
)
|
|
849
|
+
except ContractError as error:
|
|
850
|
+
raise error.contextualize(**self._mutation_context(batch_id, index, mutation, location))
|
|
851
|
+
|
|
852
|
+
@staticmethod
|
|
853
|
+
def _mutation_context(batch_id: str, index: int, mutation: Any, location: str) -> dict:
|
|
854
|
+
context = {"batch": batch_id, "index": index}
|
|
855
|
+
if not isinstance(mutation, dict):
|
|
856
|
+
return context
|
|
857
|
+
context["entity"] = mutation.get("entity")
|
|
858
|
+
if "key" in mutation:
|
|
859
|
+
context["key"] = mutation["key"]
|
|
860
|
+
# A source is echoed back only once it has passed metadata validation.
|
|
861
|
+
try:
|
|
862
|
+
validate_source(mutation.get("source"), location)
|
|
863
|
+
context["source"] = mutation["source"]
|
|
864
|
+
except ContractError:
|
|
865
|
+
pass
|
|
866
|
+
return context
|
|
867
|
+
|
|
868
|
+
def _single(self, op: str, entity: str, *, source: dict | None = None, **values):
|
|
869
|
+
self.apply_batch(
|
|
870
|
+
{
|
|
871
|
+
"id": uuid.uuid4().hex,
|
|
872
|
+
"target": self.contract.target,
|
|
873
|
+
"mapping": {"id": "gateway-store", "version": "1"},
|
|
874
|
+
"coverage": "partial",
|
|
875
|
+
"mutations": [
|
|
876
|
+
{
|
|
877
|
+
"op": op,
|
|
878
|
+
"entity": entity,
|
|
879
|
+
**values,
|
|
880
|
+
"source": (source if source is not None else {"record": f"store.{op}"}),
|
|
881
|
+
}
|
|
882
|
+
],
|
|
883
|
+
}
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
def insert(self, entity: str, value: dict, *, source: dict | None = None) -> dict:
|
|
887
|
+
self._single("insert", entity, value=value, source=source)
|
|
888
|
+
return json.loads(canonical(value))
|
|
889
|
+
|
|
890
|
+
def update(self, entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict:
|
|
891
|
+
with self._lock:
|
|
892
|
+
self._single("update", entity, key=key, value=patch, source=source)
|
|
893
|
+
return self.get(entity, key)
|
|
894
|
+
|
|
895
|
+
def delete(self, entity: str, key: dict, *, source: dict | None = None) -> None:
|
|
896
|
+
self._single("delete", entity, key=key, source=source)
|
|
897
|
+
|
|
898
|
+
@contextmanager
|
|
899
|
+
def ingest(
|
|
900
|
+
self,
|
|
901
|
+
*,
|
|
902
|
+
source: dict,
|
|
903
|
+
mapping: dict,
|
|
904
|
+
batch_id: str | None = None,
|
|
905
|
+
limit: int = 50,
|
|
906
|
+
) -> Iterator[IngestConnection]:
|
|
907
|
+
"""Schema-gated raw SQL: write freely, commit only if every row conforms.
|
|
908
|
+
|
|
909
|
+
The session yields a connection that may INSERT/UPDATE/DELETE domain
|
|
910
|
+
tables and create TEMP staging objects. ``gateway_key(entity, payload)``
|
|
911
|
+
computes ``_key`` in SQL. On exit the store validates all persisted
|
|
912
|
+
state; any violation (up to ``limit`` are reported together) rolls the
|
|
913
|
+
whole session back. Committed changes are recorded as one batch with a
|
|
914
|
+
lineage row per touched key, so raw ingestion keeps provenance. Known gap:
|
|
915
|
+
a TEMP object named like a main table still shadows it (open bug
|
|
916
|
+
``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, to be fixed by #474).
|
|
917
|
+
"""
|
|
918
|
+
session = IngestSession(self, source=source, mapping=mapping, batch_id=batch_id, limit=limit)
|
|
919
|
+
yield from session.run()
|
|
920
|
+
|
|
921
|
+
def lineage(self, entity: str, key: dict) -> list[dict]:
|
|
922
|
+
with self._lock:
|
|
923
|
+
encoded_key = self._key(entity, key, exact=True)
|
|
924
|
+
with self._trusted():
|
|
925
|
+
rows = self._execute(
|
|
926
|
+
"SELECT b.metadata, l.ordinal, l.source "
|
|
927
|
+
"FROM _gateway_lineage l "
|
|
928
|
+
"JOIN _gateway_batches b ON b.id=l.batch "
|
|
929
|
+
"WHERE l.entity=? AND l.key=? "
|
|
930
|
+
f"ORDER BY b.{self.insertion_order}, l.ordinal",
|
|
931
|
+
(entity, encoded_key),
|
|
932
|
+
).fetchall()
|
|
933
|
+
return [
|
|
934
|
+
{
|
|
935
|
+
"batch": json.loads(row[0]),
|
|
936
|
+
"index": row[1],
|
|
937
|
+
"source": json.loads(row[2]),
|
|
938
|
+
}
|
|
939
|
+
for row in rows
|
|
940
|
+
]
|
|
941
|
+
|
|
942
|
+
def audit(self) -> None:
|
|
943
|
+
"""Validate real persisted payloads, key identity and strict references."""
|
|
944
|
+
with self._lock:
|
|
945
|
+
violations = self._audit.violations(limit=1)
|
|
946
|
+
if violations:
|
|
947
|
+
raise violations[0]
|
|
948
|
+
|
|
949
|
+
def counts(self) -> dict[str, int]:
|
|
950
|
+
"""Rows per entity, counted by the engine: no payload is decoded."""
|
|
951
|
+
with self._lock, self._trusted():
|
|
952
|
+
return {
|
|
953
|
+
name: self._execute(f"SELECT COUNT(*) FROM {quote(name)}").fetchone()[0]
|
|
954
|
+
for name in self.contract.entities
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
def snapshot(self, destination: str | Path) -> Path:
|
|
958
|
+
"""A consistent copy of the committed state, written by the engine's :meth:`_write_copy`.
|
|
959
|
+
It is not audited again: every transaction was audited as it committed
|
|
960
|
+
(:meth:`_audit_touched`). The engine may let the lock go between copy steps;
|
|
961
|
+
:meth:`close` waits for a copy in progress."""
|
|
962
|
+
dest = Path(destination)
|
|
963
|
+
with self._lock:
|
|
964
|
+
if self._depth:
|
|
965
|
+
fail("transaction", "Cannot snapshot uncommitted state")
|
|
966
|
+
if self._closed:
|
|
967
|
+
fail("state_closed", "Cannot snapshot a closed store")
|
|
968
|
+
if dest.exists():
|
|
969
|
+
fail("state_exists", "Snapshot destination already exists")
|
|
970
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
971
|
+
self._copies += 1
|
|
972
|
+
try:
|
|
973
|
+
self._write_copy(dest)
|
|
974
|
+
except BaseException:
|
|
975
|
+
dest.unlink(missing_ok=True)
|
|
976
|
+
raise
|
|
977
|
+
finally:
|
|
978
|
+
with self._lock:
|
|
979
|
+
self._copies -= 1
|
|
980
|
+
self._copied.notify_all()
|
|
981
|
+
return dest
|
|
982
|
+
|
|
983
|
+
def close(self) -> None:
|
|
984
|
+
with self._lock:
|
|
985
|
+
if self._depth:
|
|
986
|
+
fail("transaction", "Cannot close an active tool/batch transaction")
|
|
987
|
+
self._copied.wait_for(lambda: not self._copies)
|
|
988
|
+
self._close_connection()
|
|
989
|
+
self._closed = True
|
|
990
|
+
|
|
991
|
+
def __enter__(self):
|
|
992
|
+
return self
|
|
993
|
+
|
|
994
|
+
def __exit__(self, *args):
|
|
995
|
+
self.close()
|