@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
"""The world database: one contract-bound store, whatever engine holds the rows.
|
|
2
|
-
|
|
3
|
-
:class:`WorldDatabase` owns everything a world's state promises — primary-key
|
|
4
|
-
identity, schema validation, atomic batches with lineage, the commit-time audit,
|
|
5
|
-
the guard deciding who may do what — and asks its engine subclass only for the
|
|
6
|
-
primitives: open a connection, run a statement, begin and end a transaction,
|
|
7
|
-
list the schema objects, check relationships, copy the file.
|
|
8
|
-
"""
|
|
9
|
-
|
|
10
1
|
from __future__ import annotations
|
|
11
2
|
|
|
12
3
|
import json
|
|
@@ -17,10 +8,11 @@ import time
|
|
|
17
8
|
import uuid
|
|
18
9
|
from abc import ABC, abstractmethod
|
|
19
10
|
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
20
12
|
from pathlib import Path
|
|
21
13
|
from typing import Any, Callable, Iterable, Iterator
|
|
22
14
|
|
|
23
|
-
from ..compiler import Contract, StorageContract, field_schema, quote
|
|
15
|
+
from ..compiler import Contract, StorageContract, field_schema, quote
|
|
24
16
|
from ..primary_key import PrimaryKey
|
|
25
17
|
from ..validation import (
|
|
26
18
|
ContractError,
|
|
@@ -40,6 +32,7 @@ from .guard import PRIVATE_TABLE_ORDER, Guard
|
|
|
40
32
|
from .ingest import IngestSession
|
|
41
33
|
from .matching import RowMatch
|
|
42
34
|
from .provenance import validate_source
|
|
35
|
+
from .reads import ReadIndexes, Selection
|
|
43
36
|
from .rows import Row, Rows
|
|
44
37
|
from .runtime_rows import RuntimeRows, Statement
|
|
45
38
|
|
|
@@ -57,10 +50,6 @@ CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn
|
|
|
57
50
|
event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
|
|
58
51
|
|
|
59
52
|
"""
|
|
60
|
-
#: The scalar types whose stored text equals Python's ``str`` of the value, so a
|
|
61
|
-
#: ``list_where`` filter on such a column can run in SQL and answer as :class:`RowMatch` does.
|
|
62
|
-
#: A boolean is stored as 1/0 and read as True/False; it filters in Python.
|
|
63
|
-
TEXT_SCALARS = {"string", "integer", "number"}
|
|
64
53
|
_FICLONE = 0x40049409
|
|
65
54
|
|
|
66
55
|
|
|
@@ -75,27 +64,21 @@ def _reflink(source: Path, destination: Path) -> bool:
|
|
|
75
64
|
return False
|
|
76
65
|
|
|
77
66
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class SideTable:
|
|
69
|
+
name: str
|
|
70
|
+
columns: tuple[tuple[str, str], ...]
|
|
71
|
+
rows: tuple[tuple, ...]
|
|
81
72
|
|
|
82
73
|
|
|
83
74
|
class WorldDatabase(ABC):
|
|
84
|
-
"""
|
|
75
|
+
"""Every public mutation is atomic; ``update`` shallow-merges ``patch`` and validates the whole row."""
|
|
85
76
|
|
|
86
|
-
Payloads are JSON values, keys are dicts of exactly the primary-key fields.
|
|
87
|
-
Updates shallow-merge provided properties and validate the complete result.
|
|
88
|
-
"""
|
|
89
|
-
|
|
90
|
-
#: The engine's name; also the key under which an ingest refusal carries the engine's own words.
|
|
91
77
|
engine: str
|
|
92
78
|
engine_label: str
|
|
93
|
-
#: The storage target a contract compiles to for this engine (the lock's ``target``).
|
|
94
79
|
target: str
|
|
95
|
-
#: The file name of a snapshot this engine writes, and its media type on the wire.
|
|
96
80
|
snapshot_name: str
|
|
97
81
|
content_type: str
|
|
98
|
-
#: The engine's error type: what a statement the engine itself refused raises.
|
|
99
82
|
native_error: type[BaseException]
|
|
100
83
|
#: Functions that reach outside the database (files, native extensions): every door but the trusted one
|
|
101
84
|
#: refuses them by name.
|
|
@@ -157,7 +140,6 @@ class WorldDatabase(ABC):
|
|
|
157
140
|
|
|
158
141
|
@classmethod
|
|
159
142
|
def engine_named(cls, name: str) -> type["WorldDatabase"]:
|
|
160
|
-
"""The engine a world's manifest names (``[db] engine``)."""
|
|
161
143
|
found = [engine for engine in cls.engines() if engine.engine == name]
|
|
162
144
|
if not found:
|
|
163
145
|
fail("target", f"{cls.requirement()}; db engine {name!r} is not one")
|
|
@@ -165,7 +147,6 @@ class WorldDatabase(ABC):
|
|
|
165
147
|
|
|
166
148
|
@classmethod
|
|
167
149
|
def for_contract(cls, contract: Contract) -> type["WorldDatabase"]:
|
|
168
|
-
"""The engine whose storage target the contract was compiled to."""
|
|
169
150
|
target = contract.metadata.get("target")
|
|
170
151
|
found = [engine for engine in cls.engines() if engine.target == target]
|
|
171
152
|
if not found:
|
|
@@ -178,23 +159,17 @@ class WorldDatabase(ABC):
|
|
|
178
159
|
|
|
179
160
|
@classmethod
|
|
180
161
|
def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "WorldDatabase":
|
|
181
|
-
"""
|
|
182
|
-
the store itself wrote and whose bytes a sidecar digest has just verified,
|
|
183
|
-
every row was audited when it was written, so opening is then a matter of
|
|
184
|
-
milliseconds however many rows the file holds. The contract check always runs."""
|
|
162
|
+
"""``audit=False`` only for a file the store wrote and a digest verified; the contract check always runs."""
|
|
185
163
|
return cls.for_contract(contract)(Path(path), contract, audit=audit)
|
|
186
164
|
|
|
187
165
|
@classmethod
|
|
188
166
|
def open_copy(cls, source: str | Path, contract: Contract, destination: str | Path) -> "WorldDatabase":
|
|
189
|
-
"""
|
|
167
|
+
"""Only for a snapshot the store wrote and a digest verified: the copy opens unaudited."""
|
|
190
168
|
cls.clone_file(Path(source), Path(destination))
|
|
191
169
|
return cls.open(destination, contract, audit=False)
|
|
192
170
|
|
|
193
171
|
@staticmethod
|
|
194
172
|
def clone_file(source: Path, destination: Path) -> None:
|
|
195
|
-
"""Copy a database file cheaply: a reflink (``FICLONE``) shares the source's
|
|
196
|
-
blocks copy-on-write, so a session costs the same whatever the snapshot's size;
|
|
197
|
-
a file system or platform without reflink falls back to a plain copy."""
|
|
198
173
|
if not stat.S_ISREG(source.stat().st_mode):
|
|
199
174
|
raise shutil.SpecialFileError(f"{source} is not a regular file")
|
|
200
175
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -205,52 +180,63 @@ class WorldDatabase(ABC):
|
|
|
205
180
|
|
|
206
181
|
@classmethod
|
|
207
182
|
@abstractmethod
|
|
208
|
-
def ddl(cls, entities: dict) -> str:
|
|
209
|
-
"""The storage DDL a contract's compiled entities become (``db/schema.sql``)."""
|
|
183
|
+
def ddl(cls, entities: dict) -> str: ...
|
|
210
184
|
|
|
211
185
|
@staticmethod
|
|
212
186
|
@abstractmethod
|
|
213
|
-
def split_statements(script: str) -> list[str]:
|
|
214
|
-
"""A script as its complete statements, comment-only text dropped."""
|
|
187
|
+
def split_statements(script: str) -> list[str]: ...
|
|
215
188
|
|
|
216
189
|
@staticmethod
|
|
217
190
|
@abstractmethod
|
|
218
|
-
def missing_table(error: BaseException) -> str | None:
|
|
219
|
-
"""The table an engine error says does not exist, or ``None``."""
|
|
191
|
+
def missing_table(error: BaseException) -> str | None: ...
|
|
220
192
|
|
|
221
193
|
@staticmethod
|
|
222
194
|
@abstractmethod
|
|
223
|
-
def exceeds_limit(error: BaseException) -> bool:
|
|
224
|
-
"""Whether an engine error says a value or a row passed the engine's size limits."""
|
|
195
|
+
def exceeds_limit(error: BaseException) -> bool: ...
|
|
225
196
|
|
|
226
197
|
@staticmethod
|
|
227
198
|
@abstractmethod
|
|
228
|
-
def carry_entries(path: Path, entries: list[str]) -> None:
|
|
229
|
-
"""Write ``entries`` into a closed database file beside its rows, in a table the store never opens."""
|
|
199
|
+
def carry_entries(path: Path, entries: list[str]) -> None: ...
|
|
230
200
|
|
|
231
201
|
@staticmethod
|
|
232
202
|
@abstractmethod
|
|
233
|
-
def stored_lock(path: Path) -> dict:
|
|
234
|
-
"""The lock a database file records its rows under, read without opening it as a store."""
|
|
203
|
+
def stored_lock(path: Path) -> dict: ...
|
|
235
204
|
|
|
236
205
|
@staticmethod
|
|
237
206
|
@abstractmethod
|
|
238
207
|
def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
|
|
239
|
-
"""
|
|
240
|
-
|
|
208
|
+
"""In one transaction, relationships checked at commit."""
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
@abstractmethod
|
|
212
|
+
def take_entries(path: Path) -> list[str]: ...
|
|
213
|
+
|
|
214
|
+
@staticmethod
|
|
215
|
+
@abstractmethod
|
|
216
|
+
def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None: ...
|
|
241
217
|
|
|
242
218
|
@staticmethod
|
|
243
219
|
@abstractmethod
|
|
244
|
-
def
|
|
245
|
-
"""The
|
|
220
|
+
def identifiers(sql: str) -> frozenset[str]:
|
|
221
|
+
"""The names a statement uses outside its comments and string literals, lowercased."""
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
@abstractmethod
|
|
225
|
+
def drop_indexes(path: Path, names: Iterable[str]) -> None: ...
|
|
226
|
+
|
|
227
|
+
@staticmethod
|
|
228
|
+
@abstractmethod
|
|
229
|
+
def recover_copy(source: Path, destination: Path) -> None: ...
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
@abstractmethod
|
|
233
|
+
def match_text(cls, column: str) -> str: ...
|
|
246
234
|
|
|
247
235
|
@abstractmethod
|
|
248
236
|
def _open_connection(self) -> None: ...
|
|
249
237
|
|
|
250
238
|
@abstractmethod
|
|
251
|
-
def _enforce_strict_dialect(self) -> None:
|
|
252
|
-
"""Refuse what would let a verifier score silently wrong (a missing column
|
|
253
|
-
read as a literal) and turn on relationship enforcement."""
|
|
239
|
+
def _enforce_strict_dialect(self) -> None: ...
|
|
254
240
|
|
|
255
241
|
@abstractmethod
|
|
256
242
|
def _execute(self, sql: str, params: Any = ()) -> Any: ...
|
|
@@ -262,16 +248,13 @@ class WorldDatabase(ABC):
|
|
|
262
248
|
def _execute_script(self, script: str) -> None: ...
|
|
263
249
|
|
|
264
250
|
@abstractmethod
|
|
265
|
-
def _schema_objects(self) -> dict[str, str]:
|
|
266
|
-
"""Every schema object the file holds, by name, with its DDL."""
|
|
251
|
+
def _schema_objects(self) -> dict[str, str]: ...
|
|
267
252
|
|
|
268
253
|
@abstractmethod
|
|
269
|
-
def
|
|
270
|
-
"""The schema objects a fresh file built from :meth:`_schema_script` holds."""
|
|
254
|
+
def _objects_of(self, script: str) -> dict[str, str]: ...
|
|
271
255
|
|
|
272
256
|
@abstractmethod
|
|
273
|
-
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None:
|
|
274
|
-
"""Make ``gateway_key(entity, payload)`` callable from SQL."""
|
|
257
|
+
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None: ...
|
|
275
258
|
|
|
276
259
|
@abstractmethod
|
|
277
260
|
def _install_guard(self) -> None:
|
|
@@ -297,40 +280,35 @@ class WorldDatabase(ABC):
|
|
|
297
280
|
def _rollback_to(self, name: str) -> None: ...
|
|
298
281
|
|
|
299
282
|
@abstractmethod
|
|
300
|
-
def
|
|
301
|
-
"""``LIMIT``/``OFFSET`` text and its parameters; ``limit=None`` is every row."""
|
|
283
|
+
def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]: ...
|
|
302
284
|
|
|
303
285
|
@abstractmethod
|
|
304
|
-
def _relationships_broken(self) -> bool:
|
|
305
|
-
"""Whether any persisted row references a row that does not exist."""
|
|
286
|
+
def _relationships_broken(self) -> bool: ...
|
|
306
287
|
|
|
307
288
|
@abstractmethod
|
|
308
|
-
def _is_constraint_violation(self, error: BaseException) -> bool:
|
|
309
|
-
"""Whether a row write failed on a declared key or constraint."""
|
|
289
|
+
def _is_constraint_violation(self, error: BaseException) -> bool: ...
|
|
310
290
|
|
|
311
291
|
@abstractmethod
|
|
312
|
-
def _drop_staging(self) -> None:
|
|
313
|
-
"""Drop the TEMP tables and views an ingest session left behind."""
|
|
292
|
+
def _drop_staging(self) -> None: ...
|
|
314
293
|
|
|
315
294
|
@abstractmethod
|
|
316
295
|
def _attach_file(self, name: str, path: str | Path) -> None: ...
|
|
317
296
|
|
|
318
297
|
@abstractmethod
|
|
319
|
-
def
|
|
320
|
-
|
|
298
|
+
def _detach_file(self, name: str) -> None: ...
|
|
299
|
+
|
|
300
|
+
@abstractmethod
|
|
301
|
+
def _write_copy(self, destination: Path) -> None: ...
|
|
321
302
|
|
|
322
303
|
@abstractmethod
|
|
323
304
|
def _close_connection(self) -> None: ...
|
|
324
305
|
|
|
325
306
|
@abstractmethod
|
|
326
|
-
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None:
|
|
327
|
-
"""Interrupt any statement still running at ``deadline`` (``time.monotonic``) and refuse a value
|
|
328
|
-
or a row wider than ``budget`` allows; ``None`` restores what the connection had before."""
|
|
307
|
+
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None: ...
|
|
329
308
|
|
|
330
309
|
@property
|
|
331
310
|
@abstractmethod
|
|
332
|
-
def insertion_order(self) -> str:
|
|
333
|
-
"""An expression ordering a bookkeeping table's rows as they were inserted."""
|
|
311
|
+
def insertion_order(self) -> str: ...
|
|
334
312
|
|
|
335
313
|
def _schema_script(self) -> str:
|
|
336
314
|
return self.contract.artifacts["db/schema.sql"].decode() + BOOKKEEPING_DDL
|
|
@@ -349,9 +327,12 @@ class WorldDatabase(ABC):
|
|
|
349
327
|
)
|
|
350
328
|
fail("state_contract", message, changed=changed)
|
|
351
329
|
# The live DDL is compared with freshly compiled DDL, not a stored hash a tampered file could carry.
|
|
352
|
-
expected = self._expected_schema_objects()
|
|
353
330
|
actual = self._schema_objects()
|
|
354
|
-
|
|
331
|
+
reads = ReadIndexes.of(self, ReadIndexes.parse(actual)).by_name()
|
|
332
|
+
built = self._objects_of(self._schema_script() + "".join(f"{sql};\n" for sql in reads.values()))
|
|
333
|
+
expected = {name: sql for name, sql in built.items() if name not in reads}
|
|
334
|
+
extra = {name: sql for name, sql in actual.items() if name not in expected}
|
|
335
|
+
if any(actual.get(k) != v for k, v in expected.items()) or any(built.get(n) != v for n, v in extra.items()):
|
|
355
336
|
fail(
|
|
356
337
|
"state_contract",
|
|
357
338
|
"State DDL differs from the compiled storage contract",
|
|
@@ -364,7 +345,6 @@ class WorldDatabase(ABC):
|
|
|
364
345
|
|
|
365
346
|
@staticmethod
|
|
366
347
|
def lock_from(rows: Iterable[Any]) -> dict:
|
|
367
|
-
"""The one lock a database's ``_gateway_contract`` rows record."""
|
|
368
348
|
try:
|
|
369
349
|
(lock,) = [json.loads(text) for (text,) in rows]
|
|
370
350
|
except (TypeError, ValueError):
|
|
@@ -375,11 +355,7 @@ class WorldDatabase(ABC):
|
|
|
375
355
|
return self._guard.allows(access)
|
|
376
356
|
|
|
377
357
|
def _sql_key(self, entity: Any, payload: Any) -> str | None:
|
|
378
|
-
"""
|
|
379
|
-
|
|
380
|
-
Returns NULL on an invalid entity or payload so the NOT NULL constraint
|
|
381
|
-
rejects the row; the reason is attached to the resulting error.
|
|
382
|
-
"""
|
|
358
|
+
"""NULL on an invalid entity or payload, so NOT NULL rejects the row; the reason is attached to its error."""
|
|
383
359
|
try:
|
|
384
360
|
value = payload
|
|
385
361
|
if isinstance(payload, (str, bytes)):
|
|
@@ -480,25 +456,12 @@ class WorldDatabase(ABC):
|
|
|
480
456
|
return [json.loads(row[0]) for row in rows]
|
|
481
457
|
|
|
482
458
|
def list_where(
|
|
483
|
-
self,
|
|
484
|
-
entity: str,
|
|
485
|
-
where: dict[str, list],
|
|
486
|
-
limit: int | None = None,
|
|
487
|
-
offset: int = 0,
|
|
459
|
+
self, entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0
|
|
488
460
|
) -> tuple[list[dict], int]:
|
|
489
|
-
"""A page of
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
Equality is the routing rule of :class:`RowMatch`: the stored value as
|
|
493
|
-
text, case-insensitively; a list of wanted values matches when any does
|
|
494
|
-
(IN); a list field matches when any element does. A field projected to a
|
|
495
|
-
column (string, integer, number) filters in SQL through the read-only
|
|
496
|
-
guard, so a large world pages without loading; a field that is not
|
|
497
|
-
(an array, an object, a boolean) filters the listed rows in Python with the
|
|
498
|
-
same rule, so both paths answer alike. A missing field never matches."""
|
|
461
|
+
"""A page of ``entity``'s rows whose every ``where`` field holds one of its values, and the total match count.
|
|
462
|
+
Values compare as text, case-insensitively; a list field matches when any element does, a missing one never."""
|
|
499
463
|
with self._lock:
|
|
500
|
-
|
|
501
|
-
properties = spec["schema"].get("properties", {})
|
|
464
|
+
properties = self._entity(entity)["schema"].get("properties", {})
|
|
502
465
|
unknown = sorted(set(where) - set(properties))
|
|
503
466
|
if unknown:
|
|
504
467
|
fail(
|
|
@@ -510,81 +473,75 @@ class WorldDatabase(ABC):
|
|
|
510
473
|
if offset < 0:
|
|
511
474
|
fail("input", "offset must be zero or a positive integer")
|
|
512
475
|
wanted = {field: [str(value) for value in values] for field, values in where.items()}
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
return
|
|
520
|
-
|
|
521
|
-
def
|
|
522
|
-
self,
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
table = quote(entity)
|
|
531
|
-
total = self.query(f"SELECT COUNT(*) FROM {table}{condition}", tuple(params))[0][0]
|
|
532
|
-
clause, paging = self._page_clause(limit, offset)
|
|
533
|
-
page = self.query(f"SELECT payload FROM {table}{condition} ORDER BY _key{clause}", (*params, *paging))
|
|
534
|
-
return [json.loads(row[0]) for row in page], total
|
|
535
|
-
|
|
536
|
-
def attach(self, name: str, path: str | Path, entities: Iterable[str]) -> None:
|
|
537
|
-
"""Read another store's file as ``<name>.<table>`` from :meth:`query` and
|
|
538
|
-
the verifiers that run through it: a cross-world check joins its own
|
|
539
|
-
rows to a dependency world's (``SELECT ... FROM slack.slack_messages``).
|
|
540
|
-
Only the ``entities`` named are readable and nothing is writable, so a
|
|
541
|
-
tool of this world cannot reach into another's store."""
|
|
476
|
+
return self.select(entity, RowMatch.on_fields(wanted), limit, offset)
|
|
477
|
+
|
|
478
|
+
def select(self, entity: str, match: RowMatch, limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]:
|
|
479
|
+
"""Equals ``match.select(self.list(entity))`` sliced to a page, and its length, read by the engine."""
|
|
480
|
+
with self._lock:
|
|
481
|
+
self._entity(entity)
|
|
482
|
+
return Selection(self, entity, match).page(limit, offset)
|
|
483
|
+
|
|
484
|
+
def index_reads(self, extra: Iterable[tuple[str, tuple[str, ...]]] = ()) -> None:
|
|
485
|
+
with self._lock, self._trusted():
|
|
486
|
+
held = self._schema_objects()
|
|
487
|
+
for name, statement in ReadIndexes.of(self, extra).by_name().items():
|
|
488
|
+
if name not in held:
|
|
489
|
+
self._execute(statement)
|
|
490
|
+
|
|
491
|
+
def attach(self, name: str, path: str | Path, entities: Iterable[str], *, bare: bool = False) -> None:
|
|
492
|
+
"""Only ``entities`` are readable and nothing is writable: a tool cannot reach into another world's store."""
|
|
542
493
|
if self._guard.is_attached(name):
|
|
543
494
|
fail("attach", f"'{name}' is attached already")
|
|
544
495
|
with self._lock, self._trusted():
|
|
545
496
|
self._attach_file(name, path)
|
|
546
|
-
self._guard.attach(name, frozenset(entities))
|
|
497
|
+
self._guard.attach(name, frozenset(entities), bare=bare)
|
|
498
|
+
|
|
499
|
+
def detach(self, name: str) -> None:
|
|
500
|
+
if not self._guard.is_attached(name):
|
|
501
|
+
return
|
|
502
|
+
self._guard.detach(name)
|
|
503
|
+
with self._lock, self._trusted():
|
|
504
|
+
self._detach_file(name)
|
|
505
|
+
|
|
506
|
+
def reads_of(self, name: str) -> int:
|
|
507
|
+
return self._guard.reads(name)
|
|
508
|
+
|
|
509
|
+
def columns_read_of(self, name: str) -> frozenset[str]:
|
|
510
|
+
return self._guard.columns(name)
|
|
547
511
|
|
|
548
512
|
@property
|
|
549
513
|
def attached(self) -> dict[str, frozenset[str]]:
|
|
550
|
-
"""The attached stores by name, each with the entities readable there."""
|
|
551
514
|
return dict(self._guard.attached)
|
|
552
515
|
|
|
553
516
|
def read_connection(self) -> ReadConnection:
|
|
554
|
-
"""
|
|
517
|
+
"""A tool handler's ``conn``: reads through the read-only guard, no writes."""
|
|
555
518
|
return ReadConnection(self)
|
|
556
519
|
|
|
557
520
|
def key_of(self, entity: str, row: dict) -> str:
|
|
558
|
-
"""The encoded identity of ``row`` in ``entity``: its primary-key fields, validated."""
|
|
559
521
|
with self._lock:
|
|
560
522
|
return self._key(entity, row)
|
|
561
523
|
|
|
562
524
|
def keys(self, entity: str) -> set[str]:
|
|
563
|
-
"""The encoded identity of every row ``entity`` holds."""
|
|
564
525
|
with self._lock:
|
|
565
526
|
self._entity(entity)
|
|
566
527
|
with self._trusted():
|
|
567
528
|
return {row[0] for row in self._execute(f"SELECT _key FROM {quote(entity)}")}
|
|
568
529
|
|
|
569
530
|
def set_task_context(self, scenario_id: str) -> None:
|
|
570
|
-
"""Name the active task in ``benchmark_runtime_context``, so a verifier keys off its scenario."""
|
|
571
531
|
self._write_runtime(RuntimeRows.task_context(scenario_id))
|
|
572
532
|
|
|
573
533
|
def set_run_parameters(self, parameters: dict[str, str]) -> None:
|
|
574
|
-
"""Replace ``benchmark_runtime_parameters`` with the run's condition bindings."""
|
|
575
534
|
self._write_runtime(RuntimeRows.run_parameters(parameters))
|
|
576
535
|
|
|
577
536
|
def record_conversation_event(
|
|
578
537
|
self, scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
|
|
579
538
|
) -> None:
|
|
580
|
-
"""Append one turn event to ``benchmark_conversation_events``, read only by a verifier."""
|
|
581
539
|
self._write_runtime(
|
|
582
540
|
RuntimeRows.conversation_event(scenario_id, user_turn_index, event_type, actor, name, payload_json)
|
|
583
541
|
)
|
|
584
542
|
|
|
585
543
|
def _write_runtime(self, statements: list[Statement]) -> None:
|
|
586
|
-
"""
|
|
587
|
-
entities, so no audit runs; inside an open transaction they ride a savepoint."""
|
|
544
|
+
"""Runtime rows are not entities, so no audit runs."""
|
|
588
545
|
with self._lock:
|
|
589
546
|
if self._guard.ingesting:
|
|
590
547
|
fail("transaction", "Runtime rows are written outside an ingest session")
|
|
@@ -601,8 +558,7 @@ class WorldDatabase(ABC):
|
|
|
601
558
|
raise
|
|
602
559
|
|
|
603
560
|
def query(self, sql: str, params: tuple = (), *, private: bool = False, limit: int | None = None) -> Rows:
|
|
604
|
-
"""Read-only SQL over domain tables, enforced by the guard
|
|
605
|
-
engine's own. With ``limit``, at most that many rows are read from the engine."""
|
|
561
|
+
"""Read-only SQL over domain tables, enforced by the guard."""
|
|
606
562
|
with self._lock, self._guard.reading(private):
|
|
607
563
|
cursor = None
|
|
608
564
|
try:
|
|
@@ -640,9 +596,6 @@ class WorldDatabase(ABC):
|
|
|
640
596
|
|
|
641
597
|
@contextmanager
|
|
642
598
|
def bounded(self, budget: QueryBudget) -> Iterator["WorldDatabase"]:
|
|
643
|
-
"""Every :meth:`query` inside runs within ``budget``: past its time the statement is stopped
|
|
644
|
-
(``time``), past its answer bytes the read is refused (``size``), and the engine refuses a
|
|
645
|
-
value larger than it allows (``size``). One budget at a time per connection."""
|
|
646
599
|
with self._lock:
|
|
647
600
|
if self._budget is not None:
|
|
648
601
|
fail("budget", "A query budget is already in force on this connection")
|
|
@@ -657,9 +610,7 @@ class WorldDatabase(ABC):
|
|
|
657
610
|
self._set_budget(None, None)
|
|
658
611
|
|
|
659
612
|
def public_copy(self, destination: str | Path) -> "WorldDatabase":
|
|
660
|
-
"""
|
|
661
|
-
rows (contract, batches, lineage, task context, conversation events): what a caller's own SQL
|
|
662
|
-
may open, so a read the engine's guard misses finds nothing private to return."""
|
|
613
|
+
"""Holds no bookkeeping or private runtime rows, so a read the guard misses finds nothing private to return."""
|
|
663
614
|
with self._lock:
|
|
664
615
|
if self._depth:
|
|
665
616
|
fail("transaction", "Cannot copy uncommitted state")
|
|
@@ -711,7 +662,6 @@ class WorldDatabase(ABC):
|
|
|
711
662
|
return key
|
|
712
663
|
|
|
713
664
|
def _merged(self, name: str, entity: dict, key: str, mutation: dict) -> dict | None:
|
|
714
|
-
"""The row an update leaves (validated, key unchanged); ``None`` for a delete."""
|
|
715
665
|
current = self.get(name, mutation["key"])
|
|
716
666
|
if current is None:
|
|
717
667
|
fail(
|
|
@@ -904,17 +854,8 @@ class WorldDatabase(ABC):
|
|
|
904
854
|
batch_id: str | None = None,
|
|
905
855
|
limit: int = 50,
|
|
906
856
|
) -> Iterator[IngestConnection]:
|
|
907
|
-
"""
|
|
908
|
-
|
|
909
|
-
The session yields a connection that may INSERT/UPDATE/DELETE domain
|
|
910
|
-
tables and create TEMP staging objects. ``gateway_key(entity, payload)``
|
|
911
|
-
computes ``_key`` in SQL. On exit the store validates all persisted
|
|
912
|
-
state; any violation (up to ``limit`` are reported together) rolls the
|
|
913
|
-
whole session back. Committed changes are recorded as one batch with a
|
|
914
|
-
lineage row per touched key, so raw ingestion keeps provenance. Known gap:
|
|
915
|
-
a TEMP object named like a main table still shadows it (open bug
|
|
916
|
-
``2026-09-28-world-store-ingest-temp-shadow-bypasses-audit``, to be fixed by #474).
|
|
917
|
-
"""
|
|
857
|
+
"""Commits only if every row conforms (up to ``limit`` violations reported together). Known gap (#474): a TEMP
|
|
858
|
+
object named like a main table still shadows it."""
|
|
918
859
|
session = IngestSession(self, source=source, mapping=mapping, batch_id=batch_id, limit=limit)
|
|
919
860
|
yield from session.run()
|
|
920
861
|
|
|
@@ -940,7 +881,6 @@ class WorldDatabase(ABC):
|
|
|
940
881
|
]
|
|
941
882
|
|
|
942
883
|
def audit(self) -> None:
|
|
943
|
-
"""Validate real persisted payloads, key identity and strict references."""
|
|
944
884
|
with self._lock:
|
|
945
885
|
violations = self._audit.violations(limit=1)
|
|
946
886
|
if violations:
|
|
@@ -955,10 +895,8 @@ class WorldDatabase(ABC):
|
|
|
955
895
|
}
|
|
956
896
|
|
|
957
897
|
def snapshot(self, destination: str | Path) -> Path:
|
|
958
|
-
"""
|
|
959
|
-
|
|
960
|
-
(:meth:`_audit_touched`). The engine may let the lock go between copy steps;
|
|
961
|
-
:meth:`close` waits for a copy in progress."""
|
|
898
|
+
"""Drops the :class:`ReadIndexes`, so its bytes do not depend on where it was taken. The engine may let the
|
|
899
|
+
lock go between copy steps; :meth:`close` waits for a copy in progress."""
|
|
962
900
|
dest = Path(destination)
|
|
963
901
|
with self._lock:
|
|
964
902
|
if self._depth:
|
|
@@ -971,6 +909,9 @@ class WorldDatabase(ABC):
|
|
|
971
909
|
self._copies += 1
|
|
972
910
|
try:
|
|
973
911
|
self._write_copy(dest)
|
|
912
|
+
with self._lock:
|
|
913
|
+
held = self._schema_objects()
|
|
914
|
+
self.drop_indexes(dest, ReadIndexes.named(held))
|
|
974
915
|
except BaseException:
|
|
975
916
|
dest.unlink(missing_ok=True)
|
|
976
917
|
raise
|