@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,995 @@
1
+ """The world database: one contract-bound store, whatever engine holds the rows.
2
+
3
+ :class:`WorldDatabase` owns everything a world's state promises — primary-key
4
+ identity, schema validation, atomic batches with lineage, the commit-time audit,
5
+ the guard deciding who may do what — and asks its engine subclass only for the
6
+ primitives: open a connection, run a statement, begin and end a transaction,
7
+ list the schema objects, check relationships, copy the file.
8
+ """
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import shutil
14
+ import stat
15
+ import threading
16
+ import time
17
+ import uuid
18
+ from abc import ABC, abstractmethod
19
+ from contextlib import contextmanager
20
+ from pathlib import Path
21
+ from typing import Any, Callable, Iterable, Iterator
22
+
23
+ from ..compiler import Contract, StorageContract, field_schema, quote, scalar_types
24
+ from ..primary_key import PrimaryKey
25
+ from ..validation import (
26
+ ContractError,
27
+ canonical,
28
+ digest,
29
+ encoded,
30
+ fail,
31
+ members,
32
+ nonempty,
33
+ parse_json,
34
+ validate,
35
+ )
36
+ from .audit import ContractAudit
37
+ from .budget import QueryBudget
38
+ from .connections import IngestConnection, ReadConnection
39
+ from .guard import PRIVATE_TABLE_ORDER, Guard
40
+ from .ingest import IngestSession
41
+ from .matching import RowMatch
42
+ from .provenance import validate_source
43
+ from .rows import Row, Rows
44
+ from .runtime_rows import RuntimeRows, Statement
45
+
46
+ STORED_LOCK_QUERY = "SELECT contract FROM _gateway_contract"
47
+
48
+ BOOKKEEPING_DDL = """
49
+ CREATE TABLE _gateway_contract (contract TEXT NOT NULL);
50
+ CREATE TABLE _gateway_batches (id TEXT PRIMARY KEY, digest TEXT NOT NULL, metadata TEXT NOT NULL);
51
+ CREATE TABLE _gateway_lineage (batch TEXT NOT NULL REFERENCES _gateway_batches(id),
52
+ ordinal INTEGER NOT NULL, entity TEXT NOT NULL, key TEXT NOT NULL, source TEXT NOT NULL,
53
+ PRIMARY KEY(batch, ordinal));
54
+ CREATE TABLE benchmark_runtime_context (context_key TEXT PRIMARY KEY, scenario_id TEXT NOT NULL);
55
+ CREATE TABLE benchmark_runtime_parameters (key TEXT PRIMARY KEY, value TEXT NOT NULL);
56
+ CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn_index INTEGER NOT NULL,
57
+ event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
58
+
59
+ """
60
+ #: The scalar types whose stored text equals Python's ``str`` of the value, so a
61
+ #: ``list_where`` filter on such a column can run in SQL and answer as :class:`RowMatch` does.
62
+ #: A boolean is stored as 1/0 and read as True/False; it filters in Python.
63
+ TEXT_SCALARS = {"string", "integer", "number"}
64
+ _FICLONE = 0x40049409
65
+
66
+
67
+ def _reflink(source: Path, destination: Path) -> bool:
68
+ try:
69
+ import fcntl
70
+
71
+ with open(source, "rb") as src, open(destination, "wb") as dst:
72
+ fcntl.ioctl(dst.fileno(), _FICLONE, src.fileno())
73
+ return True
74
+ except (ImportError, OSError):
75
+ return False
76
+
77
+
78
+ def _sql_filterable(prop: dict) -> bool:
79
+ types = scalar_types(prop)
80
+ return bool(types) and types <= TEXT_SCALARS
81
+
82
+
83
+ class WorldDatabase(ABC):
84
+ """Use create/open as context managers; every public mutation is atomic.
85
+
86
+ Payloads are JSON values, keys are dicts of exactly the primary-key fields.
87
+ Updates shallow-merge provided properties and validate the complete result.
88
+ """
89
+
90
+ #: The engine's name; also the key under which an ingest refusal carries the engine's own words.
91
+ engine: str
92
+ engine_label: str
93
+ #: The storage target a contract compiles to for this engine (the lock's ``target``).
94
+ target: str
95
+ #: The file name of a snapshot this engine writes, and its media type on the wire.
96
+ snapshot_name: str
97
+ content_type: str
98
+ #: The engine's error type: what a statement the engine itself refused raises.
99
+ native_error: type[BaseException]
100
+ #: Functions that reach outside the database (files, native extensions): every door but the trusted one
101
+ #: refuses them by name.
102
+ unsafe_functions: frozenset[str]
103
+
104
+ def __init__(self, path: Path, contract: Contract, *, create: bool = False, audit: bool = True):
105
+ self.path = Path(path)
106
+ self.contract = contract
107
+ self._lock = threading.RLock()
108
+ self._copies = 0
109
+ self._copied = threading.Condition(self._lock)
110
+ self._depth = 0
111
+ self._touched: set[tuple[str, str]] = set()
112
+ self._guard = Guard(frozenset(contract.entities), self.unsafe_functions)
113
+ self._audit = ContractAudit(self)
114
+ self._key_error: dict | None = None
115
+ self._budget: QueryBudget | None = None
116
+ self._deadline: float | None = None
117
+ self._closed = False
118
+ if create and self.path.exists():
119
+ fail(
120
+ "state_exists",
121
+ "Refusing to replace existing state; reset/rebuild explicitly",
122
+ )
123
+ if not create and not self.path.is_file():
124
+ fail("state_missing", "Cannot open missing state")
125
+ self.path.parent.mkdir(parents=True, exist_ok=True)
126
+ self._open_connection()
127
+ try:
128
+ self._enforce_strict_dialect()
129
+ if create:
130
+ self._create_schema()
131
+ self._check_contract()
132
+ self._register_key_function(self._sql_key)
133
+ self._install_guard()
134
+ if audit:
135
+ self.audit()
136
+ except BaseException:
137
+ self._close_connection()
138
+ if create:
139
+ self.path.unlink(missing_ok=True)
140
+ raise
141
+
142
+ @staticmethod
143
+ def engines() -> tuple[type["WorldDatabase"], ...]:
144
+ from .engines import ENGINES
145
+
146
+ return ENGINES
147
+
148
+ @staticmethod
149
+ def default_engine() -> type["WorldDatabase"]:
150
+ from .engines import DEFAULT_ENGINE
151
+
152
+ return DEFAULT_ENGINE
153
+
154
+ @classmethod
155
+ def requirement(cls) -> str:
156
+ return "Schema worlds require " + ", ".join(engine.engine_label for engine in cls.engines())
157
+
158
+ @classmethod
159
+ def engine_named(cls, name: str) -> type["WorldDatabase"]:
160
+ """The engine a world's manifest names (``[db] engine``)."""
161
+ found = [engine for engine in cls.engines() if engine.engine == name]
162
+ if not found:
163
+ fail("target", f"{cls.requirement()}; db engine {name!r} is not one")
164
+ return found[0]
165
+
166
+ @classmethod
167
+ def for_contract(cls, contract: Contract) -> type["WorldDatabase"]:
168
+ """The engine whose storage target the contract was compiled to."""
169
+ target = contract.metadata.get("target")
170
+ found = [engine for engine in cls.engines() if engine.target == target]
171
+ if not found:
172
+ fail("target", f"No world database engine stores contracts compiled to {target!r}")
173
+ return found[0]
174
+
175
+ @classmethod
176
+ def create(cls, path: str | Path, contract: Contract) -> "WorldDatabase":
177
+ return cls.for_contract(contract)(Path(path), contract, create=True)
178
+
179
+ @classmethod
180
+ def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "WorldDatabase":
181
+ """Open existing state. ``audit=False`` skips the full row audit: for a snapshot
182
+ the store itself wrote and whose bytes a sidecar digest has just verified,
183
+ every row was audited when it was written, so opening is then a matter of
184
+ milliseconds however many rows the file holds. The contract check always runs."""
185
+ return cls.for_contract(contract)(Path(path), contract, audit=audit)
186
+
187
+ @classmethod
188
+ def open_copy(cls, source: str | Path, contract: Contract, destination: str | Path) -> "WorldDatabase":
189
+ """Open a private copy of a snapshot the store wrote and a digest verified."""
190
+ cls.clone_file(Path(source), Path(destination))
191
+ return cls.open(destination, contract, audit=False)
192
+
193
+ @staticmethod
194
+ def clone_file(source: Path, destination: Path) -> None:
195
+ """Copy a database file cheaply: a reflink (``FICLONE``) shares the source's
196
+ blocks copy-on-write, so a session costs the same whatever the snapshot's size;
197
+ a file system or platform without reflink falls back to a plain copy."""
198
+ if not stat.S_ISREG(source.stat().st_mode):
199
+ raise shutil.SpecialFileError(f"{source} is not a regular file")
200
+ destination.parent.mkdir(parents=True, exist_ok=True)
201
+ if destination.exists() and destination.samefile(source):
202
+ raise shutil.SameFileError(f"{source} and {destination} are the same file")
203
+ if not _reflink(source, destination):
204
+ shutil.copyfile(source, destination)
205
+
206
+ @classmethod
207
+ @abstractmethod
208
+ def ddl(cls, entities: dict) -> str:
209
+ """The storage DDL a contract's compiled entities become (``db/schema.sql``)."""
210
+
211
+ @staticmethod
212
+ @abstractmethod
213
+ def split_statements(script: str) -> list[str]:
214
+ """A script as its complete statements, comment-only text dropped."""
215
+
216
+ @staticmethod
217
+ @abstractmethod
218
+ def missing_table(error: BaseException) -> str | None:
219
+ """The table an engine error says does not exist, or ``None``."""
220
+
221
+ @staticmethod
222
+ @abstractmethod
223
+ def exceeds_limit(error: BaseException) -> bool:
224
+ """Whether an engine error says a value or a row passed the engine's size limits."""
225
+
226
+ @staticmethod
227
+ @abstractmethod
228
+ def carry_entries(path: Path, entries: list[str]) -> None:
229
+ """Write ``entries`` into a closed database file beside its rows, in a table the store never opens."""
230
+
231
+ @staticmethod
232
+ @abstractmethod
233
+ def stored_lock(path: Path) -> dict:
234
+ """The lock a database file records its rows under, read without opening it as a store."""
235
+
236
+ @staticmethod
237
+ @abstractmethod
238
+ def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
239
+ """Every row of ``source`` copied into the new database at ``destination`` in one transaction:
240
+ the ``ledger`` tables whole, each entity's keys and payloads, relationships checked at commit."""
241
+
242
+ @staticmethod
243
+ @abstractmethod
244
+ def take_entries(path: Path) -> list[str]:
245
+ """The entries :meth:`carry_entries` wrote into ``path``, in order, dropped from it; empty when none."""
246
+
247
+ @abstractmethod
248
+ def _open_connection(self) -> None: ...
249
+
250
+ @abstractmethod
251
+ def _enforce_strict_dialect(self) -> None:
252
+ """Refuse what would let a verifier score silently wrong (a missing column
253
+ read as a literal) and turn on relationship enforcement."""
254
+
255
+ @abstractmethod
256
+ def _execute(self, sql: str, params: Any = ()) -> Any: ...
257
+
258
+ @abstractmethod
259
+ def _executemany(self, sql: str, rows: Any) -> Any: ...
260
+
261
+ @abstractmethod
262
+ def _execute_script(self, script: str) -> None: ...
263
+
264
+ @abstractmethod
265
+ def _schema_objects(self) -> dict[str, str]:
266
+ """Every schema object the file holds, by name, with its DDL."""
267
+
268
+ @abstractmethod
269
+ def _expected_schema_objects(self) -> dict[str, str]:
270
+ """The schema objects a fresh file built from :meth:`_schema_script` holds."""
271
+
272
+ @abstractmethod
273
+ def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
274
+ """Make ``gateway_key(entity, payload)`` callable from SQL."""
275
+
276
+ @abstractmethod
277
+ def _install_guard(self) -> None:
278
+ """Route every access the engine makes through :meth:`_guard_allows`."""
279
+
280
+ @abstractmethod
281
+ def _begin_write(self) -> None:
282
+ """Begin a transaction that holds the write lock from its first statement."""
283
+
284
+ @abstractmethod
285
+ def _commit(self) -> None: ...
286
+
287
+ @abstractmethod
288
+ def _rollback(self) -> None: ...
289
+
290
+ @abstractmethod
291
+ def _savepoint(self, name: str) -> None: ...
292
+
293
+ @abstractmethod
294
+ def _release(self, name: str) -> None: ...
295
+
296
+ @abstractmethod
297
+ def _rollback_to(self, name: str) -> None: ...
298
+
299
+ @abstractmethod
300
+ def _page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]:
301
+ """``LIMIT``/``OFFSET`` text and its parameters; ``limit=None`` is every row."""
302
+
303
+ @abstractmethod
304
+ def _relationships_broken(self) -> bool:
305
+ """Whether any persisted row references a row that does not exist."""
306
+
307
+ @abstractmethod
308
+ def _is_constraint_violation(self, error: BaseException) -> bool:
309
+ """Whether a row write failed on a declared key or constraint."""
310
+
311
+ @abstractmethod
312
+ def _drop_staging(self) -> None:
313
+ """Drop the TEMP tables and views an ingest session left behind."""
314
+
315
+ @abstractmethod
316
+ def _attach_file(self, name: str, path: str | Path) -> None: ...
317
+
318
+ @abstractmethod
319
+ def _write_copy(self, destination: Path) -> None:
320
+ """A consistent copy of the committed state at ``destination``."""
321
+
322
+ @abstractmethod
323
+ def _close_connection(self) -> None: ...
324
+
325
+ @abstractmethod
326
+ def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
327
+ """Interrupt any statement still running at ``deadline`` (``time.monotonic``) and refuse a value
328
+ or a row wider than ``budget`` allows; ``None`` restores what the connection had before."""
329
+
330
+ @property
331
+ @abstractmethod
332
+ def insertion_order(self) -> str:
333
+ """An expression ordering a bookkeeping table's rows as they were inserted."""
334
+
335
+ def _schema_script(self) -> str:
336
+ return self.contract.artifacts["db/schema.sql"].decode() + BOOKKEEPING_DDL
337
+
338
+ def _create_schema(self) -> None:
339
+ self._execute_script(self._schema_script())
340
+ self._execute("INSERT INTO _gateway_contract VALUES (?)", (canonical(self.contract.lock),))
341
+
342
+ def _check_contract(self) -> None:
343
+ try:
344
+ # The storage contract, never the whole lock: a tool or package edit touches no row.
345
+ changed = self.contract.storage.changes_from(self._stored_contract())
346
+ if changed:
347
+ message = (
348
+ f"State belongs to a different schema: {' and '.join(changed)} changed; no automatic migration"
349
+ )
350
+ fail("state_contract", message, changed=changed)
351
+ # The live DDL is compared with freshly compiled DDL, not a stored hash a tampered file could carry.
352
+ expected = self._expected_schema_objects()
353
+ actual = self._schema_objects()
354
+ if any(actual.get(k) != v for k, v in expected.items()) or set(actual) - set(expected):
355
+ fail(
356
+ "state_contract",
357
+ "State DDL differs from the compiled storage contract",
358
+ )
359
+ except self.native_error as error:
360
+ raise ContractError("state_contract", "State has no compatible schema contract") from error
361
+
362
+ def _stored_contract(self) -> StorageContract:
363
+ return StorageContract.of(self.lock_from(self._execute(STORED_LOCK_QUERY).fetchall()))
364
+
365
+ @staticmethod
366
+ def lock_from(rows: Iterable[Any]) -> dict:
367
+ """The one lock a database's ``_gateway_contract`` rows record."""
368
+ try:
369
+ (lock,) = [json.loads(text) for (text,) in rows]
370
+ except (TypeError, ValueError):
371
+ fail("state_contract", "State records no single readable contract")
372
+ return lock
373
+
374
+ def _guard_allows(self, access) -> bool:
375
+ return self._guard.allows(access)
376
+
377
+ def _sql_key(self, entity: Any, payload: Any) -> str | None:
378
+ """``gateway_key(entity, payload)``: the encoded ``_key`` for a payload.
379
+
380
+ Returns NULL on an invalid entity or payload so the NOT NULL constraint
381
+ rejects the row; the reason is attached to the resulting error.
382
+ """
383
+ try:
384
+ value = payload
385
+ if isinstance(payload, (str, bytes)):
386
+ value = parse_json(payload)
387
+ if not isinstance(value, dict):
388
+ fail("key", "gateway_key needs a JSON object payload", entity=entity)
389
+ return self._key(entity, value)
390
+ except (ContractError, ValueError, TypeError) as error:
391
+ detail = getattr(error, "detail", None)
392
+ self._key_error = detail or {"code": "key", "message": str(error)}
393
+ return None
394
+
395
+ def _trusted(self):
396
+ return self._guard.trusted()
397
+
398
+ @contextmanager
399
+ def transaction(self) -> Iterator["WorldDatabase"]:
400
+ with self._lock:
401
+ if self._guard.ingesting:
402
+ fail(
403
+ "transaction",
404
+ "Use SQL inside an ingest session; store mutations belong outside",
405
+ )
406
+ outer = self._depth == 0
407
+ savepoint = f"gateway_batch_{self._depth}"
408
+ with self._trusted():
409
+ self._enter_level(outer, savepoint)
410
+ if outer:
411
+ self._touched = set()
412
+ self._depth += 1
413
+ try:
414
+ yield self
415
+ if outer and self._touched:
416
+ self._audit.touched(self._touched)
417
+ with self._trusted():
418
+ self._leave_level(outer, savepoint)
419
+ except BaseException:
420
+ with self._trusted():
421
+ self._abort_level(outer, savepoint)
422
+ raise
423
+ finally:
424
+ self._depth -= 1
425
+
426
+ def _enter_level(self, outer: bool, savepoint: str) -> None:
427
+ if outer:
428
+ self._begin_write()
429
+ return
430
+ self._savepoint(savepoint)
431
+
432
+ def _leave_level(self, outer: bool, savepoint: str) -> None:
433
+ if outer:
434
+ self._commit()
435
+ return
436
+ self._release(savepoint)
437
+
438
+ def _abort_level(self, outer: bool, savepoint: str) -> None:
439
+ if outer:
440
+ self._rollback()
441
+ return
442
+ self._rollback_to(savepoint)
443
+ self._release(savepoint)
444
+
445
+ def _entity(self, name: str) -> dict:
446
+ if not isinstance(name, str) or name not in self.contract.entities:
447
+ fail("entity", "Unknown entity type", entity=name)
448
+ return self.contract.entities[name]
449
+
450
+ def _key(self, name: str, value: dict, *, exact: bool = False) -> str:
451
+ entity = self._entity(name)
452
+ key = PrimaryKey.of(entity)
453
+ if key.missing(value) is not None or (exact and set(value) != set(key.fields)):
454
+ fail("key", "Supply exactly the declared primary-key fields", entity=name)
455
+ values = key.values(value)
456
+ for field, item in zip(key.fields, values):
457
+ validate(
458
+ field_schema(entity["schema"], field, entity["location"]),
459
+ item,
460
+ location=f"{entity['location']}/schema/properties/{field}",
461
+ )
462
+ # JSON Schema integers include 1.0: the key's identity normalizes integral floats, never the payload.
463
+ return canonical([int(item) if type(item) is float else item for item in values])
464
+
465
+ def get(self, entity: str, key: dict) -> dict | None:
466
+ with self._lock:
467
+ encoded_key = self._key(entity, key, exact=True)
468
+ with self._trusted():
469
+ row = self._execute(
470
+ f"SELECT payload FROM {quote(entity)} WHERE _key = ?",
471
+ (encoded_key,),
472
+ ).fetchone()
473
+ return json.loads(row[0]) if row else None
474
+
475
+ def list(self, entity: str) -> list[dict]:
476
+ with self._lock:
477
+ self._entity(entity)
478
+ with self._trusted():
479
+ rows = self._execute(f"SELECT payload FROM {quote(entity)} ORDER BY _key").fetchall()
480
+ return [json.loads(row[0]) for row in rows]
481
+
482
+ def list_where(
483
+ self,
484
+ entity: str,
485
+ where: dict[str, list],
486
+ limit: int | None = None,
487
+ offset: int = 0,
488
+ ) -> tuple[list[dict], int]:
489
+ """A page of the rows of ``entity`` whose every ``where`` field holds one of
490
+ the listed values, and how many rows match in all: ``(rows, total)``.
491
+
492
+ Equality is the routing rule of :class:`RowMatch`: the stored value as
493
+ text, case-insensitively; a list of wanted values matches when any does
494
+ (IN); a list field matches when any element does. A field projected to a
495
+ column (string, integer, number) filters in SQL through the read-only
496
+ guard, so a large world pages without loading; a field that is not
497
+ (an array, an object, a boolean) filters the listed rows in Python with the
498
+ same rule, so both paths answer alike. A missing field never matches."""
499
+ with self._lock:
500
+ spec = self._entity(entity)
501
+ properties = spec["schema"].get("properties", {})
502
+ unknown = sorted(set(where) - set(properties))
503
+ if unknown:
504
+ fail(
505
+ "field",
506
+ f"Unknown field {', '.join(unknown)} on {entity}; its fields are {', '.join(sorted(properties))}",
507
+ entity=entity,
508
+ fields=sorted(properties),
509
+ )
510
+ if offset < 0:
511
+ fail("input", "offset must be zero or a positive integer")
512
+ wanted = {field: [str(value) for value in values] for field, values in where.items()}
513
+ if any(not values for values in wanted.values()):
514
+ return [], 0
515
+ if all(_sql_filterable(properties[field]) for field in wanted):
516
+ return self._list_where_sql(entity, wanted, limit, offset)
517
+ matched = RowMatch.on_fields(wanted).select(self.list(entity))
518
+ end = None if limit is None else offset + limit
519
+ return matched[offset:end], len(matched)
520
+
521
+ def _list_where_sql(
522
+ self, entity: str, wanted: dict[str, list[str]], limit: int | None, offset: int
523
+ ) -> tuple[list[dict], int]:
524
+ clauses = []
525
+ params: list = []
526
+ for field, values in wanted.items():
527
+ clauses.append(f"lower(CAST({quote(field)} AS TEXT)) IN ({', '.join('?' for _ in values)})")
528
+ params.extend(value.lower() for value in values)
529
+ condition = f" WHERE {' AND '.join(clauses)}" if clauses else ""
530
+ table = quote(entity)
531
+ total = self.query(f"SELECT COUNT(*) FROM {table}{condition}", tuple(params))[0][0]
532
+ clause, paging = self._page_clause(limit, offset)
533
+ page = self.query(f"SELECT payload FROM {table}{condition} ORDER BY _key{clause}", (*params, *paging))
534
+ return [json.loads(row[0]) for row in page], total
535
+
536
+ def attach(self, name: str, path: str | Path, entities: Iterable[str]) -> None:
537
+ """Read another store's file as ``<name>.<table>`` from :meth:`query` and
538
+ the verifiers that run through it: a cross-world check joins its own
539
+ rows to a dependency world's (``SELECT ... FROM slack.slack_messages``).
540
+ Only the ``entities`` named are readable and nothing is writable, so a
541
+ tool of this world cannot reach into another's store."""
542
+ if self._guard.is_attached(name):
543
+ fail("attach", f"'{name}' is attached already")
544
+ with self._lock, self._trusted():
545
+ self._attach_file(name, path)
546
+ self._guard.attach(name, frozenset(entities))
547
+
548
+ @property
549
+ def attached(self) -> dict[str, frozenset[str]]:
550
+ """The attached stores by name, each with the entities readable there."""
551
+ return dict(self._guard.attached)
552
+
553
+ def read_connection(self) -> ReadConnection:
554
+ """What a tool handler receives as ``conn``: reads through the read-only guard, no writes."""
555
+ return ReadConnection(self)
556
+
557
+ def key_of(self, entity: str, row: dict) -> str:
558
+ """The encoded identity of ``row`` in ``entity``: its primary-key fields, validated."""
559
+ with self._lock:
560
+ return self._key(entity, row)
561
+
562
+ def keys(self, entity: str) -> set[str]:
563
+ """The encoded identity of every row ``entity`` holds."""
564
+ with self._lock:
565
+ self._entity(entity)
566
+ with self._trusted():
567
+ return {row[0] for row in self._execute(f"SELECT _key FROM {quote(entity)}")}
568
+
569
+ def set_task_context(self, scenario_id: str) -> None:
570
+ """Name the active task in ``benchmark_runtime_context``, so a verifier keys off its scenario."""
571
+ self._write_runtime(RuntimeRows.task_context(scenario_id))
572
+
573
+ def set_run_parameters(self, parameters: dict[str, str]) -> None:
574
+ """Replace ``benchmark_runtime_parameters`` with the run's condition bindings."""
575
+ self._write_runtime(RuntimeRows.run_parameters(parameters))
576
+
577
+ def record_conversation_event(
578
+ self, scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
579
+ ) -> None:
580
+ """Append one turn event to ``benchmark_conversation_events``, read only by a verifier."""
581
+ self._write_runtime(
582
+ RuntimeRows.conversation_event(scenario_id, user_turn_index, event_type, actor, name, payload_json)
583
+ )
584
+
585
+ def _write_runtime(self, statements: list[Statement]) -> None:
586
+ """Store-authored writes to the runtime tables, all or none. They are not
587
+ entities, so no audit runs; inside an open transaction they ride a savepoint."""
588
+ with self._lock:
589
+ if self._guard.ingesting:
590
+ fail("transaction", "Runtime rows are written outside an ingest session")
591
+ outer = self._depth == 0
592
+ savepoint = f"gateway_runtime_{self._depth}"
593
+ with self._trusted():
594
+ self._enter_level(outer, savepoint)
595
+ try:
596
+ for sql, params in statements:
597
+ self._execute(sql, params)
598
+ self._leave_level(outer, savepoint)
599
+ except BaseException:
600
+ self._abort_level(outer, savepoint)
601
+ raise
602
+
603
+ def query(self, sql: str, params: tuple = (), *, private: bool = False, limit: int | None = None) -> Rows:
604
+ """Read-only SQL over domain tables, enforced by the guard; rows are :class:`Row`, never the
605
+ engine's own. With ``limit``, at most that many rows are read from the engine."""
606
+ with self._lock, self._guard.reading(private):
607
+ cursor = None
608
+ try:
609
+ cursor = self._execute(sql, params)
610
+ return Row.all(cursor.description, self._fetch(cursor, limit))
611
+ except self.native_error as error:
612
+ raise self._refusal(error) from error
613
+ finally:
614
+ if cursor is not None:
615
+ cursor.close()
616
+
617
+ def _fetch(self, cursor: Any, limit: int | None) -> list:
618
+ if self._budget is None:
619
+ return cursor.fetchall() if limit is None else cursor.fetchmany(limit)
620
+ rows: list = []
621
+ spent = 0
622
+ while limit is None or len(rows) < limit:
623
+ row = cursor.fetchone()
624
+ if row is None:
625
+ break
626
+ spent += QueryBudget.size_of(row)
627
+ if spent > self._budget.answer_bytes:
628
+ fail("size", f"The answer passes {self._budget.answer_bytes} bytes; select fewer columns or rows")
629
+ rows.append(row)
630
+ return rows
631
+
632
+ def _refusal(self, error: BaseException) -> ContractError:
633
+ if self._deadline is not None and time.monotonic() >= self._deadline:
634
+ return ContractError("time", "The query ran past its time budget; narrow it or add a LIMIT")
635
+ if self.exceeds_limit(error):
636
+ cap = f" of {self._budget.value_bytes} bytes" if self._budget is not None else ""
637
+ return ContractError("size", f"A value passes the size limit{cap}; select fewer or smaller columns")
638
+ # The engine's words ("no such column") reach the author; a bare refusal grades as a silent zero.
639
+ return ContractError("read_only", f"Query failed or attempted a forbidden operation: {error}")
640
+
641
+ @contextmanager
642
+ def bounded(self, budget: QueryBudget) -> Iterator["WorldDatabase"]:
643
+ """Every :meth:`query` inside runs within ``budget``: past its time the statement is stopped
644
+ (``time``), past its answer bytes the read is refused (``size``), and the engine refuses a
645
+ value larger than it allows (``size``). One budget at a time per connection."""
646
+ with self._lock:
647
+ if self._budget is not None:
648
+ fail("budget", "A query budget is already in force on this connection")
649
+ try:
650
+ self._budget = budget
651
+ self._deadline = time.monotonic() + budget.seconds
652
+ self._set_budget(budget, self._deadline)
653
+ yield self
654
+ finally:
655
+ self._budget = None
656
+ self._deadline = None
657
+ self._set_budget(None, None)
658
+
659
+ def public_copy(self, destination: str | Path) -> "WorldDatabase":
660
+ """A copy of the committed state at ``destination`` holding no bookkeeping or private runtime
661
+ rows (contract, batches, lineage, task context, conversation events): what a caller's own SQL
662
+ may open, so a read the engine's guard misses finds nothing private to return."""
663
+ with self._lock:
664
+ if self._depth:
665
+ fail("transaction", "Cannot copy uncommitted state")
666
+ dest = Path(destination)
667
+ if dest.exists():
668
+ fail("state_exists", "Copy destination already exists")
669
+ dest.parent.mkdir(parents=True, exist_ok=True)
670
+ self._write_copy(dest)
671
+ copy = type(self)(dest, self.contract, audit=False)
672
+ try:
673
+ copy._forget_private()
674
+ except BaseException:
675
+ copy._close_connection()
676
+ raise
677
+ return copy
678
+
679
+ def _forget_private(self) -> None:
680
+ with self._lock, self._trusted():
681
+ self._begin_write()
682
+ for table in PRIVATE_TABLE_ORDER:
683
+ self._execute(f"DELETE FROM {quote(table)}")
684
+ self._commit()
685
+
686
+ def _mutate(self, mutation: dict) -> str:
687
+ name = mutation["entity"]
688
+ entity = self._entity(name)
689
+ operation = mutation["op"]
690
+ if operation == "insert":
691
+ value = mutation["value"]
692
+ validate(entity["schema"], value, location=f"{entity['location']}/schema")
693
+ key = self._key(name, value)
694
+ else:
695
+ key = self._key(name, mutation["key"], exact=True)
696
+ value = self._merged(name, entity, key, mutation)
697
+ self._touched.add((name, key))
698
+ try:
699
+ with self._trusted():
700
+ self._write_row(operation, name, key, value)
701
+ except self.native_error as error:
702
+ if not self._is_constraint_violation(error):
703
+ raise
704
+ raise ContractError(
705
+ "unique",
706
+ "Primary or unique key conflicts with existing state",
707
+ entity=name,
708
+ key=json.loads(key),
709
+ schema_path=f"{entity['location']}/unique",
710
+ ) from error
711
+ return key
712
+
713
+ def _merged(self, name: str, entity: dict, key: str, mutation: dict) -> dict | None:
714
+ """The row an update leaves (validated, key unchanged); ``None`` for a delete."""
715
+ current = self.get(name, mutation["key"])
716
+ if current is None:
717
+ fail(
718
+ "not_found",
719
+ "No entity exists at this key",
720
+ entity=name,
721
+ key=mutation["key"],
722
+ )
723
+ if mutation["op"] != "update":
724
+ return None
725
+ if not isinstance(mutation["value"], dict):
726
+ fail("type", "An update must be an object")
727
+ value = {**current, **mutation["value"]}
728
+ validate(entity["schema"], value, location=f"{entity['location']}/schema")
729
+ if self._key(name, value) != key:
730
+ fail(
731
+ "key_immutable",
732
+ "Updates cannot change primary keys",
733
+ entity=name,
734
+ )
735
+ return value
736
+
737
+ def _write_row(self, operation: str, name: str, key: str, value: dict | None) -> None:
738
+ table = quote(name)
739
+ if operation == "insert":
740
+ self._execute(f"INSERT INTO {table} (_key, payload) VALUES (?, ?)", (key, canonical(value)))
741
+ return
742
+ if operation == "update":
743
+ self._execute(f"UPDATE {table} SET payload = ? WHERE _key = ?", (canonical(value), key))
744
+ return
745
+ self._execute(f"DELETE FROM {table} WHERE _key = ?", (key,))
746
+
747
+ def apply_batch(self, batch: dict) -> dict:
748
+ """Atomic rejection; exact retries no-op, conflicting ID reuse rejects."""
749
+ members(
750
+ batch,
751
+ {"id", "target", "mapping", "coverage", "mutations"},
752
+ {"id", "target", "mapping", "coverage", "mutations"},
753
+ "/batch",
754
+ )
755
+ nonempty(batch["id"], "/batch/id")
756
+ if not self.contract.targets(batch["target"]):
757
+ fail(
758
+ "batch_target",
759
+ "Batch targets a different domain identity/version/hash",
760
+ "/batch/target",
761
+ )
762
+ members(batch["mapping"], {"id", "version"}, {"id", "version"}, "/batch/mapping")
763
+ for key, value in batch["mapping"].items():
764
+ nonempty(value, f"/batch/mapping/{key}")
765
+ if batch["coverage"] not in ("partial", "complete"):
766
+ fail(
767
+ "coverage",
768
+ "coverage must be partial or complete (neither implies deletion)",
769
+ "/batch/coverage",
770
+ )
771
+ if not isinstance(batch["mutations"], list):
772
+ fail("batch", "mutations must be an array", "/batch/mutations")
773
+ batch_hash = digest(encoded(batch))
774
+ with self.transaction():
775
+ with self._trusted():
776
+ prior = self._execute("SELECT digest FROM _gateway_batches WHERE id = ?", (batch["id"],)).fetchone()
777
+ if prior:
778
+ if prior[0] != batch_hash:
779
+ fail(
780
+ "idempotency_conflict",
781
+ "Batch ID was already applied with different content",
782
+ "/batch/id",
783
+ batch=batch["id"],
784
+ )
785
+ return {"status": "already_applied", "batch": batch["id"], "applied": 0}
786
+ with self._trusted():
787
+ self._execute(
788
+ "INSERT INTO _gateway_batches VALUES (?, ?, ?)",
789
+ (
790
+ batch["id"],
791
+ batch_hash,
792
+ canonical({k: v for k, v in batch.items() if k != "mutations"}),
793
+ ),
794
+ )
795
+ for index, mutation in enumerate(batch["mutations"]):
796
+ self._apply_mutation(batch["id"], index, mutation)
797
+ return {
798
+ "status": "applied",
799
+ "batch": batch["id"],
800
+ "applied": len(batch["mutations"]),
801
+ }
802
+
803
+ def _apply_mutation(self, batch_id: str, index: int, mutation: Any) -> None:
804
+ location = f"/batch/mutations/{index}"
805
+ try:
806
+ members(
807
+ mutation,
808
+ {"op", "entity", "key", "value", "source"},
809
+ {"op", "entity", "source"},
810
+ location,
811
+ )
812
+ op = mutation["op"]
813
+ if not isinstance(op, str):
814
+ fail(
815
+ "operation",
816
+ "Operation must be insert, update or delete",
817
+ location,
818
+ )
819
+ required = {
820
+ "insert": {"value"},
821
+ "update": {"key", "value"},
822
+ "delete": {"key"},
823
+ }.get(op)
824
+ if required is None:
825
+ fail(
826
+ "operation",
827
+ "Only insert, update and delete are supported",
828
+ location,
829
+ )
830
+ members(
831
+ mutation,
832
+ {"op", "entity", "source"} | required,
833
+ {"op", "entity", "source"} | required,
834
+ location,
835
+ )
836
+ validate_source(mutation["source"], f"{location}/source")
837
+ key = self._mutate(mutation)
838
+ with self._trusted():
839
+ self._execute(
840
+ "INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
841
+ (
842
+ batch_id,
843
+ index,
844
+ mutation["entity"],
845
+ key,
846
+ canonical(mutation["source"]),
847
+ ),
848
+ )
849
+ except ContractError as error:
850
+ raise error.contextualize(**self._mutation_context(batch_id, index, mutation, location))
851
+
852
+ @staticmethod
853
+ def _mutation_context(batch_id: str, index: int, mutation: Any, location: str) -> dict:
854
+ context = {"batch": batch_id, "index": index}
855
+ if not isinstance(mutation, dict):
856
+ return context
857
+ context["entity"] = mutation.get("entity")
858
+ if "key" in mutation:
859
+ context["key"] = mutation["key"]
860
+ # A source is echoed back only once it has passed metadata validation.
861
+ try:
862
+ validate_source(mutation.get("source"), location)
863
+ context["source"] = mutation["source"]
864
+ except ContractError:
865
+ pass
866
+ return context
867
+
868
+ def _single(self, op: str, entity: str, *, source: dict | None = None, **values):
869
+ self.apply_batch(
870
+ {
871
+ "id": uuid.uuid4().hex,
872
+ "target": self.contract.target,
873
+ "mapping": {"id": "gateway-store", "version": "1"},
874
+ "coverage": "partial",
875
+ "mutations": [
876
+ {
877
+ "op": op,
878
+ "entity": entity,
879
+ **values,
880
+ "source": (source if source is not None else {"record": f"store.{op}"}),
881
+ }
882
+ ],
883
+ }
884
+ )
885
+
886
+ def insert(self, entity: str, value: dict, *, source: dict | None = None) -> dict:
887
+ self._single("insert", entity, value=value, source=source)
888
+ return json.loads(canonical(value))
889
+
890
+ def update(self, entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict:
891
+ with self._lock:
892
+ self._single("update", entity, key=key, value=patch, source=source)
893
+ return self.get(entity, key)
894
+
895
+ def delete(self, entity: str, key: dict, *, source: dict | None = None) -> None:
896
+ self._single("delete", entity, key=key, source=source)
897
+
898
+ @contextmanager
899
+ def ingest(
900
+ self,
901
+ *,
902
+ source: dict,
903
+ mapping: dict,
904
+ batch_id: str | None = None,
905
+ limit: int = 50,
906
+ ) -> Iterator[IngestConnection]:
907
+ """Schema-gated raw SQL: write freely, commit only if every row conforms.
908
+
909
+ The session yields a connection that may INSERT/UPDATE/DELETE domain
910
+ tables and create TEMP staging objects. ``gateway_key(entity, payload)``
911
+ computes ``_key`` in SQL. On exit the store validates all persisted
912
+ state; any violation (up to ``limit`` are reported together) rolls the
913
+ whole session back. Committed changes are recorded as one batch with a
914
+ lineage row per touched key, so raw ingestion keeps provenance. Known gap:
915
+ a TEMP object named like a main table still shadows it (open bug
916
+ ``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, to be fixed by #474).
917
+ """
918
+ session = IngestSession(self, source=source, mapping=mapping, batch_id=batch_id, limit=limit)
919
+ yield from session.run()
920
+
921
+ def lineage(self, entity: str, key: dict) -> list[dict]:
922
+ with self._lock:
923
+ encoded_key = self._key(entity, key, exact=True)
924
+ with self._trusted():
925
+ rows = self._execute(
926
+ "SELECT b.metadata, l.ordinal, l.source "
927
+ "FROM _gateway_lineage l "
928
+ "JOIN _gateway_batches b ON b.id=l.batch "
929
+ "WHERE l.entity=? AND l.key=? "
930
+ f"ORDER BY b.{self.insertion_order}, l.ordinal",
931
+ (entity, encoded_key),
932
+ ).fetchall()
933
+ return [
934
+ {
935
+ "batch": json.loads(row[0]),
936
+ "index": row[1],
937
+ "source": json.loads(row[2]),
938
+ }
939
+ for row in rows
940
+ ]
941
+
942
+ def audit(self) -> None:
943
+ """Validate real persisted payloads, key identity and strict references."""
944
+ with self._lock:
945
+ violations = self._audit.violations(limit=1)
946
+ if violations:
947
+ raise violations[0]
948
+
949
+ def counts(self) -> dict[str, int]:
950
+ """Rows per entity, counted by the engine: no payload is decoded."""
951
+ with self._lock, self._trusted():
952
+ return {
953
+ name: self._execute(f"SELECT COUNT(*) FROM {quote(name)}").fetchone()[0]
954
+ for name in self.contract.entities
955
+ }
956
+
957
+ def snapshot(self, destination: str | Path) -> Path:
958
+ """A consistent copy of the committed state, written by the engine's :meth:`_write_copy`.
959
+ It is not audited again: every transaction was audited as it committed
960
+ (:meth:`_audit_touched`). The engine may let the lock go between copy steps;
961
+ :meth:`close` waits for a copy in progress."""
962
+ dest = Path(destination)
963
+ with self._lock:
964
+ if self._depth:
965
+ fail("transaction", "Cannot snapshot uncommitted state")
966
+ if self._closed:
967
+ fail("state_closed", "Cannot snapshot a closed store")
968
+ if dest.exists():
969
+ fail("state_exists", "Snapshot destination already exists")
970
+ dest.parent.mkdir(parents=True, exist_ok=True)
971
+ self._copies += 1
972
+ try:
973
+ self._write_copy(dest)
974
+ except BaseException:
975
+ dest.unlink(missing_ok=True)
976
+ raise
977
+ finally:
978
+ with self._lock:
979
+ self._copies -= 1
980
+ self._copied.notify_all()
981
+ return dest
982
+
983
+ def close(self) -> None:
984
+ with self._lock:
985
+ if self._depth:
986
+ fail("transaction", "Cannot close an active tool/batch transaction")
987
+ self._copied.wait_for(lambda: not self._copies)
988
+ self._close_connection()
989
+ self._closed = True
990
+
991
+ def __enter__(self):
992
+ return self
993
+
994
+ def __exit__(self, *args):
995
+ self.close()