@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,936 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import shutil
|
|
5
|
+
import stat
|
|
6
|
+
import threading
|
|
7
|
+
import time
|
|
8
|
+
import uuid
|
|
9
|
+
from abc import ABC, abstractmethod
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from typing import Any, Callable, Iterable, Iterator
|
|
14
|
+
|
|
15
|
+
from ..compiler import Contract, StorageContract, field_schema, quote
|
|
16
|
+
from ..primary_key import PrimaryKey
|
|
17
|
+
from ..validation import (
|
|
18
|
+
ContractError,
|
|
19
|
+
canonical,
|
|
20
|
+
digest,
|
|
21
|
+
encoded,
|
|
22
|
+
fail,
|
|
23
|
+
members,
|
|
24
|
+
nonempty,
|
|
25
|
+
parse_json,
|
|
26
|
+
validate,
|
|
27
|
+
)
|
|
28
|
+
from .audit import ContractAudit
|
|
29
|
+
from .budget import QueryBudget
|
|
30
|
+
from .connections import IngestConnection, ReadConnection
|
|
31
|
+
from .guard import PRIVATE_TABLE_ORDER, Guard
|
|
32
|
+
from .ingest import IngestSession
|
|
33
|
+
from .matching import RowMatch
|
|
34
|
+
from .provenance import validate_source
|
|
35
|
+
from .reads import ReadIndexes, Selection
|
|
36
|
+
from .rows import Row, Rows
|
|
37
|
+
from .runtime_rows import RuntimeRows, Statement
|
|
38
|
+
|
|
39
|
+
STORED_LOCK_QUERY = "SELECT contract FROM _gateway_contract"
|
|
40
|
+
|
|
41
|
+
BOOKKEEPING_DDL = """
|
|
42
|
+
CREATE TABLE _gateway_contract (contract TEXT NOT NULL);
|
|
43
|
+
CREATE TABLE _gateway_batches (id TEXT PRIMARY KEY, digest TEXT NOT NULL, metadata TEXT NOT NULL);
|
|
44
|
+
CREATE TABLE _gateway_lineage (batch TEXT NOT NULL REFERENCES _gateway_batches(id),
|
|
45
|
+
ordinal INTEGER NOT NULL, entity TEXT NOT NULL, key TEXT NOT NULL, source TEXT NOT NULL,
|
|
46
|
+
PRIMARY KEY(batch, ordinal));
|
|
47
|
+
CREATE TABLE benchmark_runtime_context (context_key TEXT PRIMARY KEY, scenario_id TEXT NOT NULL);
|
|
48
|
+
CREATE TABLE benchmark_runtime_parameters (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
49
|
+
CREATE TABLE benchmark_conversation_events (scenario_id TEXT NOT NULL, user_turn_index INTEGER NOT NULL,
|
|
50
|
+
event_type TEXT NOT NULL, actor TEXT NOT NULL, name TEXT, payload_json TEXT NOT NULL);
|
|
51
|
+
|
|
52
|
+
"""
|
|
53
|
+
_FICLONE = 0x40049409
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _reflink(source: Path, destination: Path) -> bool:
|
|
57
|
+
try:
|
|
58
|
+
import fcntl
|
|
59
|
+
|
|
60
|
+
with open(source, "rb") as src, open(destination, "wb") as dst:
|
|
61
|
+
fcntl.ioctl(dst.fileno(), _FICLONE, src.fileno())
|
|
62
|
+
return True
|
|
63
|
+
except (ImportError, OSError):
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@dataclass(frozen=True)
|
|
68
|
+
class SideTable:
|
|
69
|
+
name: str
|
|
70
|
+
columns: tuple[tuple[str, str], ...]
|
|
71
|
+
rows: tuple[tuple, ...]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
class WorldDatabase(ABC):
|
|
75
|
+
"""Every public mutation is atomic; ``update`` shallow-merges ``patch`` and validates the whole row."""
|
|
76
|
+
|
|
77
|
+
engine: str
|
|
78
|
+
engine_label: str
|
|
79
|
+
target: str
|
|
80
|
+
snapshot_name: str
|
|
81
|
+
content_type: str
|
|
82
|
+
native_error: type[BaseException]
|
|
83
|
+
#: Functions that reach outside the database (files, native extensions): every door but the trusted one
|
|
84
|
+
#: refuses them by name.
|
|
85
|
+
unsafe_functions: frozenset[str]
|
|
86
|
+
|
|
87
|
+
def __init__(self, path: Path, contract: Contract, *, create: bool = False, audit: bool = True):
|
|
88
|
+
self.path = Path(path)
|
|
89
|
+
self.contract = contract
|
|
90
|
+
self._lock = threading.RLock()
|
|
91
|
+
self._copies = 0
|
|
92
|
+
self._copied = threading.Condition(self._lock)
|
|
93
|
+
self._depth = 0
|
|
94
|
+
self._touched: set[tuple[str, str]] = set()
|
|
95
|
+
self._guard = Guard(frozenset(contract.entities), self.unsafe_functions)
|
|
96
|
+
self._audit = ContractAudit(self)
|
|
97
|
+
self._key_error: dict | None = None
|
|
98
|
+
self._budget: QueryBudget | None = None
|
|
99
|
+
self._deadline: float | None = None
|
|
100
|
+
self._closed = False
|
|
101
|
+
if create and self.path.exists():
|
|
102
|
+
fail(
|
|
103
|
+
"state_exists",
|
|
104
|
+
"Refusing to replace existing state; reset/rebuild explicitly",
|
|
105
|
+
)
|
|
106
|
+
if not create and not self.path.is_file():
|
|
107
|
+
fail("state_missing", "Cannot open missing state")
|
|
108
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
109
|
+
self._open_connection()
|
|
110
|
+
try:
|
|
111
|
+
self._enforce_strict_dialect()
|
|
112
|
+
if create:
|
|
113
|
+
self._create_schema()
|
|
114
|
+
self._check_contract()
|
|
115
|
+
self._register_key_function(self._sql_key)
|
|
116
|
+
self._install_guard()
|
|
117
|
+
if audit:
|
|
118
|
+
self.audit()
|
|
119
|
+
except BaseException:
|
|
120
|
+
self._close_connection()
|
|
121
|
+
if create:
|
|
122
|
+
self.path.unlink(missing_ok=True)
|
|
123
|
+
raise
|
|
124
|
+
|
|
125
|
+
@staticmethod
|
|
126
|
+
def engines() -> tuple[type["WorldDatabase"], ...]:
|
|
127
|
+
from .engines import ENGINES
|
|
128
|
+
|
|
129
|
+
return ENGINES
|
|
130
|
+
|
|
131
|
+
@staticmethod
|
|
132
|
+
def default_engine() -> type["WorldDatabase"]:
|
|
133
|
+
from .engines import DEFAULT_ENGINE
|
|
134
|
+
|
|
135
|
+
return DEFAULT_ENGINE
|
|
136
|
+
|
|
137
|
+
@classmethod
|
|
138
|
+
def requirement(cls) -> str:
|
|
139
|
+
return "Schema worlds require " + ", ".join(engine.engine_label for engine in cls.engines())
|
|
140
|
+
|
|
141
|
+
@classmethod
|
|
142
|
+
def engine_named(cls, name: str) -> type["WorldDatabase"]:
|
|
143
|
+
found = [engine for engine in cls.engines() if engine.engine == name]
|
|
144
|
+
if not found:
|
|
145
|
+
fail("target", f"{cls.requirement()}; db engine {name!r} is not one")
|
|
146
|
+
return found[0]
|
|
147
|
+
|
|
148
|
+
@classmethod
|
|
149
|
+
def for_contract(cls, contract: Contract) -> type["WorldDatabase"]:
|
|
150
|
+
target = contract.metadata.get("target")
|
|
151
|
+
found = [engine for engine in cls.engines() if engine.target == target]
|
|
152
|
+
if not found:
|
|
153
|
+
fail("target", f"No world database engine stores contracts compiled to {target!r}")
|
|
154
|
+
return found[0]
|
|
155
|
+
|
|
156
|
+
@classmethod
|
|
157
|
+
def create(cls, path: str | Path, contract: Contract) -> "WorldDatabase":
|
|
158
|
+
return cls.for_contract(contract)(Path(path), contract, create=True)
|
|
159
|
+
|
|
160
|
+
@classmethod
|
|
161
|
+
def open(cls, path: str | Path, contract: Contract, *, audit: bool = True) -> "WorldDatabase":
|
|
162
|
+
"""``audit=False`` only for a file the store wrote and a digest verified; the contract check always runs."""
|
|
163
|
+
return cls.for_contract(contract)(Path(path), contract, audit=audit)
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def open_copy(cls, source: str | Path, contract: Contract, destination: str | Path) -> "WorldDatabase":
|
|
167
|
+
"""Only for a snapshot the store wrote and a digest verified: the copy opens unaudited."""
|
|
168
|
+
cls.clone_file(Path(source), Path(destination))
|
|
169
|
+
return cls.open(destination, contract, audit=False)
|
|
170
|
+
|
|
171
|
+
@staticmethod
|
|
172
|
+
def clone_file(source: Path, destination: Path) -> None:
|
|
173
|
+
if not stat.S_ISREG(source.stat().st_mode):
|
|
174
|
+
raise shutil.SpecialFileError(f"{source} is not a regular file")
|
|
175
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
176
|
+
if destination.exists() and destination.samefile(source):
|
|
177
|
+
raise shutil.SameFileError(f"{source} and {destination} are the same file")
|
|
178
|
+
if not _reflink(source, destination):
|
|
179
|
+
shutil.copyfile(source, destination)
|
|
180
|
+
|
|
181
|
+
@classmethod
|
|
182
|
+
@abstractmethod
|
|
183
|
+
def ddl(cls, entities: dict) -> str: ...
|
|
184
|
+
|
|
185
|
+
@staticmethod
|
|
186
|
+
@abstractmethod
|
|
187
|
+
def split_statements(script: str) -> list[str]: ...
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
@abstractmethod
|
|
191
|
+
def missing_table(error: BaseException) -> str | None: ...
|
|
192
|
+
|
|
193
|
+
@staticmethod
|
|
194
|
+
@abstractmethod
|
|
195
|
+
def exceeds_limit(error: BaseException) -> bool: ...
|
|
196
|
+
|
|
197
|
+
@staticmethod
|
|
198
|
+
@abstractmethod
|
|
199
|
+
def carry_entries(path: Path, entries: list[str]) -> None: ...
|
|
200
|
+
|
|
201
|
+
@staticmethod
|
|
202
|
+
@abstractmethod
|
|
203
|
+
def stored_lock(path: Path) -> dict: ...
|
|
204
|
+
|
|
205
|
+
@staticmethod
|
|
206
|
+
@abstractmethod
|
|
207
|
+
def copy_rows(source: Path, destination: Path, ledger: Iterable[str], entities: Iterable[str]) -> None:
|
|
208
|
+
"""In one transaction, relationships checked at commit."""
|
|
209
|
+
|
|
210
|
+
@staticmethod
|
|
211
|
+
@abstractmethod
|
|
212
|
+
def take_entries(path: Path) -> list[str]: ...
|
|
213
|
+
|
|
214
|
+
@staticmethod
|
|
215
|
+
@abstractmethod
|
|
216
|
+
def write_side_tables(path: Path, tables: Iterable[SideTable]) -> None: ...
|
|
217
|
+
|
|
218
|
+
@staticmethod
|
|
219
|
+
@abstractmethod
|
|
220
|
+
def identifiers(sql: str) -> frozenset[str]:
|
|
221
|
+
"""The names a statement uses outside its comments and string literals, lowercased."""
|
|
222
|
+
|
|
223
|
+
@staticmethod
|
|
224
|
+
@abstractmethod
|
|
225
|
+
def drop_indexes(path: Path, names: Iterable[str]) -> None: ...
|
|
226
|
+
|
|
227
|
+
@staticmethod
|
|
228
|
+
@abstractmethod
|
|
229
|
+
def recover_copy(source: Path, destination: Path) -> None: ...
|
|
230
|
+
|
|
231
|
+
@classmethod
|
|
232
|
+
@abstractmethod
|
|
233
|
+
def match_text(cls, column: str) -> str: ...
|
|
234
|
+
|
|
235
|
+
@abstractmethod
|
|
236
|
+
def _open_connection(self) -> None: ...
|
|
237
|
+
|
|
238
|
+
@abstractmethod
|
|
239
|
+
def _enforce_strict_dialect(self) -> None: ...
|
|
240
|
+
|
|
241
|
+
@abstractmethod
|
|
242
|
+
def _execute(self, sql: str, params: Any = ()) -> Any: ...
|
|
243
|
+
|
|
244
|
+
@abstractmethod
|
|
245
|
+
def _executemany(self, sql: str, rows: Any) -> Any: ...
|
|
246
|
+
|
|
247
|
+
@abstractmethod
|
|
248
|
+
def _execute_script(self, script: str) -> None: ...
|
|
249
|
+
|
|
250
|
+
@abstractmethod
|
|
251
|
+
def _schema_objects(self) -> dict[str, str]: ...
|
|
252
|
+
|
|
253
|
+
@abstractmethod
|
|
254
|
+
def _objects_of(self, script: str) -> dict[str, str]: ...
|
|
255
|
+
|
|
256
|
+
@abstractmethod
|
|
257
|
+
def _register_key_function(self, function: Callable[[Any, Any], str | None]) -> None: ...
|
|
258
|
+
|
|
259
|
+
@abstractmethod
|
|
260
|
+
def _install_guard(self) -> None:
|
|
261
|
+
"""Route every access the engine makes through :meth:`_guard_allows`."""
|
|
262
|
+
|
|
263
|
+
@abstractmethod
|
|
264
|
+
def _begin_write(self) -> None:
|
|
265
|
+
"""Begin a transaction that holds the write lock from its first statement."""
|
|
266
|
+
|
|
267
|
+
@abstractmethod
|
|
268
|
+
def _commit(self) -> None: ...
|
|
269
|
+
|
|
270
|
+
@abstractmethod
|
|
271
|
+
def _rollback(self) -> None: ...
|
|
272
|
+
|
|
273
|
+
@abstractmethod
|
|
274
|
+
def _savepoint(self, name: str) -> None: ...
|
|
275
|
+
|
|
276
|
+
@abstractmethod
|
|
277
|
+
def _release(self, name: str) -> None: ...
|
|
278
|
+
|
|
279
|
+
@abstractmethod
|
|
280
|
+
def _rollback_to(self, name: str) -> None: ...
|
|
281
|
+
|
|
282
|
+
@abstractmethod
|
|
283
|
+
def page_clause(self, limit: int | None, offset: int) -> tuple[str, tuple]: ...
|
|
284
|
+
|
|
285
|
+
@abstractmethod
|
|
286
|
+
def _relationships_broken(self) -> bool: ...
|
|
287
|
+
|
|
288
|
+
@abstractmethod
|
|
289
|
+
def _is_constraint_violation(self, error: BaseException) -> bool: ...
|
|
290
|
+
|
|
291
|
+
@abstractmethod
|
|
292
|
+
def _drop_staging(self) -> None: ...
|
|
293
|
+
|
|
294
|
+
@abstractmethod
|
|
295
|
+
def _attach_file(self, name: str, path: str | Path) -> None: ...
|
|
296
|
+
|
|
297
|
+
@abstractmethod
|
|
298
|
+
def _detach_file(self, name: str) -> None: ...
|
|
299
|
+
|
|
300
|
+
@abstractmethod
|
|
301
|
+
def _write_copy(self, destination: Path) -> None: ...
|
|
302
|
+
|
|
303
|
+
@abstractmethod
|
|
304
|
+
def _close_connection(self) -> None: ...
|
|
305
|
+
|
|
306
|
+
@abstractmethod
|
|
307
|
+
def _set_budget(self, budget: QueryBudget | None, deadline: float | None) -> None: ...
|
|
308
|
+
|
|
309
|
+
@property
|
|
310
|
+
@abstractmethod
|
|
311
|
+
def insertion_order(self) -> str: ...
|
|
312
|
+
|
|
313
|
+
def _schema_script(self) -> str:
|
|
314
|
+
return self.contract.artifacts["db/schema.sql"].decode() + BOOKKEEPING_DDL
|
|
315
|
+
|
|
316
|
+
def _create_schema(self) -> None:
|
|
317
|
+
self._execute_script(self._schema_script())
|
|
318
|
+
self._execute("INSERT INTO _gateway_contract VALUES (?)", (canonical(self.contract.lock),))
|
|
319
|
+
|
|
320
|
+
def _check_contract(self) -> None:
|
|
321
|
+
try:
|
|
322
|
+
# The storage contract, never the whole lock: a tool or package edit touches no row.
|
|
323
|
+
changed = self.contract.storage.changes_from(self._stored_contract())
|
|
324
|
+
if changed:
|
|
325
|
+
message = (
|
|
326
|
+
f"State belongs to a different schema: {' and '.join(changed)} changed; no automatic migration"
|
|
327
|
+
)
|
|
328
|
+
fail("state_contract", message, changed=changed)
|
|
329
|
+
# The live DDL is compared with freshly compiled DDL, not a stored hash a tampered file could carry.
|
|
330
|
+
actual = self._schema_objects()
|
|
331
|
+
reads = ReadIndexes.of(self, ReadIndexes.parse(actual)).by_name()
|
|
332
|
+
built = self._objects_of(self._schema_script() + "".join(f"{sql};\n" for sql in reads.values()))
|
|
333
|
+
expected = {name: sql for name, sql in built.items() if name not in reads}
|
|
334
|
+
extra = {name: sql for name, sql in actual.items() if name not in expected}
|
|
335
|
+
if any(actual.get(k) != v for k, v in expected.items()) or any(built.get(n) != v for n, v in extra.items()):
|
|
336
|
+
fail(
|
|
337
|
+
"state_contract",
|
|
338
|
+
"State DDL differs from the compiled storage contract",
|
|
339
|
+
)
|
|
340
|
+
except self.native_error as error:
|
|
341
|
+
raise ContractError("state_contract", "State has no compatible schema contract") from error
|
|
342
|
+
|
|
343
|
+
def _stored_contract(self) -> StorageContract:
|
|
344
|
+
return StorageContract.of(self.lock_from(self._execute(STORED_LOCK_QUERY).fetchall()))
|
|
345
|
+
|
|
346
|
+
@staticmethod
|
|
347
|
+
def lock_from(rows: Iterable[Any]) -> dict:
|
|
348
|
+
try:
|
|
349
|
+
(lock,) = [json.loads(text) for (text,) in rows]
|
|
350
|
+
except (TypeError, ValueError):
|
|
351
|
+
fail("state_contract", "State records no single readable contract")
|
|
352
|
+
return lock
|
|
353
|
+
|
|
354
|
+
def _guard_allows(self, access) -> bool:
|
|
355
|
+
return self._guard.allows(access)
|
|
356
|
+
|
|
357
|
+
def _sql_key(self, entity: Any, payload: Any) -> str | None:
|
|
358
|
+
"""NULL on an invalid entity or payload, so NOT NULL rejects the row; the reason is attached to its error."""
|
|
359
|
+
try:
|
|
360
|
+
value = payload
|
|
361
|
+
if isinstance(payload, (str, bytes)):
|
|
362
|
+
value = parse_json(payload)
|
|
363
|
+
if not isinstance(value, dict):
|
|
364
|
+
fail("key", "gateway_key needs a JSON object payload", entity=entity)
|
|
365
|
+
return self._key(entity, value)
|
|
366
|
+
except (ContractError, ValueError, TypeError) as error:
|
|
367
|
+
detail = getattr(error, "detail", None)
|
|
368
|
+
self._key_error = detail or {"code": "key", "message": str(error)}
|
|
369
|
+
return None
|
|
370
|
+
|
|
371
|
+
def _trusted(self):
|
|
372
|
+
return self._guard.trusted()
|
|
373
|
+
|
|
374
|
+
@contextmanager
|
|
375
|
+
def transaction(self) -> Iterator["WorldDatabase"]:
|
|
376
|
+
with self._lock:
|
|
377
|
+
if self._guard.ingesting:
|
|
378
|
+
fail(
|
|
379
|
+
"transaction",
|
|
380
|
+
"Use SQL inside an ingest session; store mutations belong outside",
|
|
381
|
+
)
|
|
382
|
+
outer = self._depth == 0
|
|
383
|
+
savepoint = f"gateway_batch_{self._depth}"
|
|
384
|
+
with self._trusted():
|
|
385
|
+
self._enter_level(outer, savepoint)
|
|
386
|
+
if outer:
|
|
387
|
+
self._touched = set()
|
|
388
|
+
self._depth += 1
|
|
389
|
+
try:
|
|
390
|
+
yield self
|
|
391
|
+
if outer and self._touched:
|
|
392
|
+
self._audit.touched(self._touched)
|
|
393
|
+
with self._trusted():
|
|
394
|
+
self._leave_level(outer, savepoint)
|
|
395
|
+
except BaseException:
|
|
396
|
+
with self._trusted():
|
|
397
|
+
self._abort_level(outer, savepoint)
|
|
398
|
+
raise
|
|
399
|
+
finally:
|
|
400
|
+
self._depth -= 1
|
|
401
|
+
|
|
402
|
+
def _enter_level(self, outer: bool, savepoint: str) -> None:
|
|
403
|
+
if outer:
|
|
404
|
+
self._begin_write()
|
|
405
|
+
return
|
|
406
|
+
self._savepoint(savepoint)
|
|
407
|
+
|
|
408
|
+
def _leave_level(self, outer: bool, savepoint: str) -> None:
|
|
409
|
+
if outer:
|
|
410
|
+
self._commit()
|
|
411
|
+
return
|
|
412
|
+
self._release(savepoint)
|
|
413
|
+
|
|
414
|
+
def _abort_level(self, outer: bool, savepoint: str) -> None:
|
|
415
|
+
if outer:
|
|
416
|
+
self._rollback()
|
|
417
|
+
return
|
|
418
|
+
self._rollback_to(savepoint)
|
|
419
|
+
self._release(savepoint)
|
|
420
|
+
|
|
421
|
+
def _entity(self, name: str) -> dict:
|
|
422
|
+
if not isinstance(name, str) or name not in self.contract.entities:
|
|
423
|
+
fail("entity", "Unknown entity type", entity=name)
|
|
424
|
+
return self.contract.entities[name]
|
|
425
|
+
|
|
426
|
+
def _key(self, name: str, value: dict, *, exact: bool = False) -> str:
|
|
427
|
+
entity = self._entity(name)
|
|
428
|
+
key = PrimaryKey.of(entity)
|
|
429
|
+
if key.missing(value) is not None or (exact and set(value) != set(key.fields)):
|
|
430
|
+
fail("key", "Supply exactly the declared primary-key fields", entity=name)
|
|
431
|
+
values = key.values(value)
|
|
432
|
+
for field, item in zip(key.fields, values):
|
|
433
|
+
validate(
|
|
434
|
+
field_schema(entity["schema"], field, entity["location"]),
|
|
435
|
+
item,
|
|
436
|
+
location=f"{entity['location']}/schema/properties/{field}",
|
|
437
|
+
)
|
|
438
|
+
# JSON Schema integers include 1.0: the key's identity normalizes integral floats, never the payload.
|
|
439
|
+
return canonical([int(item) if type(item) is float else item for item in values])
|
|
440
|
+
|
|
441
|
+
def get(self, entity: str, key: dict) -> dict | None:
|
|
442
|
+
with self._lock:
|
|
443
|
+
encoded_key = self._key(entity, key, exact=True)
|
|
444
|
+
with self._trusted():
|
|
445
|
+
row = self._execute(
|
|
446
|
+
f"SELECT payload FROM {quote(entity)} WHERE _key = ?",
|
|
447
|
+
(encoded_key,),
|
|
448
|
+
).fetchone()
|
|
449
|
+
return json.loads(row[0]) if row else None
|
|
450
|
+
|
|
451
|
+
def list(self, entity: str) -> list[dict]:
|
|
452
|
+
with self._lock:
|
|
453
|
+
self._entity(entity)
|
|
454
|
+
with self._trusted():
|
|
455
|
+
rows = self._execute(f"SELECT payload FROM {quote(entity)} ORDER BY _key").fetchall()
|
|
456
|
+
return [json.loads(row[0]) for row in rows]
|
|
457
|
+
|
|
458
|
+
def list_where(
|
|
459
|
+
self, entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0
|
|
460
|
+
) -> tuple[list[dict], int]:
|
|
461
|
+
"""A page of ``entity``'s rows whose every ``where`` field holds one of its values, and the total match count.
|
|
462
|
+
Values compare as text, case-insensitively; a list field matches when any element does, a missing one never."""
|
|
463
|
+
with self._lock:
|
|
464
|
+
properties = self._entity(entity)["schema"].get("properties", {})
|
|
465
|
+
unknown = sorted(set(where) - set(properties))
|
|
466
|
+
if unknown:
|
|
467
|
+
fail(
|
|
468
|
+
"field",
|
|
469
|
+
f"Unknown field {', '.join(unknown)} on {entity}; its fields are {', '.join(sorted(properties))}",
|
|
470
|
+
entity=entity,
|
|
471
|
+
fields=sorted(properties),
|
|
472
|
+
)
|
|
473
|
+
if offset < 0:
|
|
474
|
+
fail("input", "offset must be zero or a positive integer")
|
|
475
|
+
wanted = {field: [str(value) for value in values] for field, values in where.items()}
|
|
476
|
+
return self.select(entity, RowMatch.on_fields(wanted), limit, offset)
|
|
477
|
+
|
|
478
|
+
def select(self, entity: str, match: RowMatch, limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]:
|
|
479
|
+
"""Equals ``match.select(self.list(entity))`` sliced to a page, and its length, read by the engine."""
|
|
480
|
+
with self._lock:
|
|
481
|
+
self._entity(entity)
|
|
482
|
+
return Selection(self, entity, match).page(limit, offset)
|
|
483
|
+
|
|
484
|
+
def index_reads(self, extra: Iterable[tuple[str, tuple[str, ...]]] = ()) -> None:
|
|
485
|
+
with self._lock, self._trusted():
|
|
486
|
+
held = self._schema_objects()
|
|
487
|
+
for name, statement in ReadIndexes.of(self, extra).by_name().items():
|
|
488
|
+
if name not in held:
|
|
489
|
+
self._execute(statement)
|
|
490
|
+
|
|
491
|
+
def attach(self, name: str, path: str | Path, entities: Iterable[str], *, bare: bool = False) -> None:
|
|
492
|
+
"""Only ``entities`` are readable and nothing is writable: a tool cannot reach into another world's store."""
|
|
493
|
+
if self._guard.is_attached(name):
|
|
494
|
+
fail("attach", f"'{name}' is attached already")
|
|
495
|
+
with self._lock, self._trusted():
|
|
496
|
+
self._attach_file(name, path)
|
|
497
|
+
self._guard.attach(name, frozenset(entities), bare=bare)
|
|
498
|
+
|
|
499
|
+
def detach(self, name: str) -> None:
|
|
500
|
+
if not self._guard.is_attached(name):
|
|
501
|
+
return
|
|
502
|
+
self._guard.detach(name)
|
|
503
|
+
with self._lock, self._trusted():
|
|
504
|
+
self._detach_file(name)
|
|
505
|
+
|
|
506
|
+
def reads_of(self, name: str) -> int:
|
|
507
|
+
return self._guard.reads(name)
|
|
508
|
+
|
|
509
|
+
def columns_read_of(self, name: str) -> frozenset[str]:
|
|
510
|
+
return self._guard.columns(name)
|
|
511
|
+
|
|
512
|
+
@property
|
|
513
|
+
def attached(self) -> dict[str, frozenset[str]]:
|
|
514
|
+
return dict(self._guard.attached)
|
|
515
|
+
|
|
516
|
+
def read_connection(self) -> ReadConnection:
|
|
517
|
+
"""A tool handler's ``conn``: reads through the read-only guard, no writes."""
|
|
518
|
+
return ReadConnection(self)
|
|
519
|
+
|
|
520
|
+
def key_of(self, entity: str, row: dict) -> str:
|
|
521
|
+
with self._lock:
|
|
522
|
+
return self._key(entity, row)
|
|
523
|
+
|
|
524
|
+
def keys(self, entity: str) -> set[str]:
|
|
525
|
+
with self._lock:
|
|
526
|
+
self._entity(entity)
|
|
527
|
+
with self._trusted():
|
|
528
|
+
return {row[0] for row in self._execute(f"SELECT _key FROM {quote(entity)}")}
|
|
529
|
+
|
|
530
|
+
def set_task_context(self, scenario_id: str) -> None:
|
|
531
|
+
self._write_runtime(RuntimeRows.task_context(scenario_id))
|
|
532
|
+
|
|
533
|
+
def set_run_parameters(self, parameters: dict[str, str]) -> None:
|
|
534
|
+
self._write_runtime(RuntimeRows.run_parameters(parameters))
|
|
535
|
+
|
|
536
|
+
def record_conversation_event(
|
|
537
|
+
self, scenario_id: str, user_turn_index: int, event_type: str, actor: str, name: str | None, payload_json: str
|
|
538
|
+
) -> None:
|
|
539
|
+
self._write_runtime(
|
|
540
|
+
RuntimeRows.conversation_event(scenario_id, user_turn_index, event_type, actor, name, payload_json)
|
|
541
|
+
)
|
|
542
|
+
|
|
543
|
+
def _write_runtime(self, statements: list[Statement]) -> None:
|
|
544
|
+
"""Runtime rows are not entities, so no audit runs."""
|
|
545
|
+
with self._lock:
|
|
546
|
+
if self._guard.ingesting:
|
|
547
|
+
fail("transaction", "Runtime rows are written outside an ingest session")
|
|
548
|
+
outer = self._depth == 0
|
|
549
|
+
savepoint = f"gateway_runtime_{self._depth}"
|
|
550
|
+
with self._trusted():
|
|
551
|
+
self._enter_level(outer, savepoint)
|
|
552
|
+
try:
|
|
553
|
+
for sql, params in statements:
|
|
554
|
+
self._execute(sql, params)
|
|
555
|
+
self._leave_level(outer, savepoint)
|
|
556
|
+
except BaseException:
|
|
557
|
+
self._abort_level(outer, savepoint)
|
|
558
|
+
raise
|
|
559
|
+
|
|
560
|
+
def query(self, sql: str, params: tuple = (), *, private: bool = False, limit: int | None = None) -> Rows:
|
|
561
|
+
"""Read-only SQL over domain tables, enforced by the guard."""
|
|
562
|
+
with self._lock, self._guard.reading(private):
|
|
563
|
+
cursor = None
|
|
564
|
+
try:
|
|
565
|
+
cursor = self._execute(sql, params)
|
|
566
|
+
return Row.all(cursor.description, self._fetch(cursor, limit))
|
|
567
|
+
except self.native_error as error:
|
|
568
|
+
raise self._refusal(error) from error
|
|
569
|
+
finally:
|
|
570
|
+
if cursor is not None:
|
|
571
|
+
cursor.close()
|
|
572
|
+
|
|
573
|
+
def _fetch(self, cursor: Any, limit: int | None) -> list:
|
|
574
|
+
if self._budget is None:
|
|
575
|
+
return cursor.fetchall() if limit is None else cursor.fetchmany(limit)
|
|
576
|
+
rows: list = []
|
|
577
|
+
spent = 0
|
|
578
|
+
while limit is None or len(rows) < limit:
|
|
579
|
+
row = cursor.fetchone()
|
|
580
|
+
if row is None:
|
|
581
|
+
break
|
|
582
|
+
spent += QueryBudget.size_of(row)
|
|
583
|
+
if spent > self._budget.answer_bytes:
|
|
584
|
+
fail("size", f"The answer passes {self._budget.answer_bytes} bytes; select fewer columns or rows")
|
|
585
|
+
rows.append(row)
|
|
586
|
+
return rows
|
|
587
|
+
|
|
588
|
+
def _refusal(self, error: BaseException) -> ContractError:
|
|
589
|
+
if self._deadline is not None and time.monotonic() >= self._deadline:
|
|
590
|
+
return ContractError("time", "The query ran past its time budget; narrow it or add a LIMIT")
|
|
591
|
+
if self.exceeds_limit(error):
|
|
592
|
+
cap = f" of {self._budget.value_bytes} bytes" if self._budget is not None else ""
|
|
593
|
+
return ContractError("size", f"A value passes the size limit{cap}; select fewer or smaller columns")
|
|
594
|
+
# The engine's words ("no such column") reach the author; a bare refusal grades as a silent zero.
|
|
595
|
+
return ContractError("read_only", f"Query failed or attempted a forbidden operation: {error}")
|
|
596
|
+
|
|
597
|
+
@contextmanager
|
|
598
|
+
def bounded(self, budget: QueryBudget) -> Iterator["WorldDatabase"]:
|
|
599
|
+
with self._lock:
|
|
600
|
+
if self._budget is not None:
|
|
601
|
+
fail("budget", "A query budget is already in force on this connection")
|
|
602
|
+
try:
|
|
603
|
+
self._budget = budget
|
|
604
|
+
self._deadline = time.monotonic() + budget.seconds
|
|
605
|
+
self._set_budget(budget, self._deadline)
|
|
606
|
+
yield self
|
|
607
|
+
finally:
|
|
608
|
+
self._budget = None
|
|
609
|
+
self._deadline = None
|
|
610
|
+
self._set_budget(None, None)
|
|
611
|
+
|
|
612
|
+
def public_copy(self, destination: str | Path) -> "WorldDatabase":
|
|
613
|
+
"""Holds no bookkeeping or private runtime rows, so a read the guard misses finds nothing private to return."""
|
|
614
|
+
with self._lock:
|
|
615
|
+
if self._depth:
|
|
616
|
+
fail("transaction", "Cannot copy uncommitted state")
|
|
617
|
+
dest = Path(destination)
|
|
618
|
+
if dest.exists():
|
|
619
|
+
fail("state_exists", "Copy destination already exists")
|
|
620
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
621
|
+
self._write_copy(dest)
|
|
622
|
+
copy = type(self)(dest, self.contract, audit=False)
|
|
623
|
+
try:
|
|
624
|
+
copy._forget_private()
|
|
625
|
+
except BaseException:
|
|
626
|
+
copy._close_connection()
|
|
627
|
+
raise
|
|
628
|
+
return copy
|
|
629
|
+
|
|
630
|
+
def _forget_private(self) -> None:
|
|
631
|
+
with self._lock, self._trusted():
|
|
632
|
+
self._begin_write()
|
|
633
|
+
for table in PRIVATE_TABLE_ORDER:
|
|
634
|
+
self._execute(f"DELETE FROM {quote(table)}")
|
|
635
|
+
self._commit()
|
|
636
|
+
|
|
637
|
+
def _mutate(self, mutation: dict) -> str:
|
|
638
|
+
name = mutation["entity"]
|
|
639
|
+
entity = self._entity(name)
|
|
640
|
+
operation = mutation["op"]
|
|
641
|
+
if operation == "insert":
|
|
642
|
+
value = mutation["value"]
|
|
643
|
+
validate(entity["schema"], value, location=f"{entity['location']}/schema")
|
|
644
|
+
key = self._key(name, value)
|
|
645
|
+
else:
|
|
646
|
+
key = self._key(name, mutation["key"], exact=True)
|
|
647
|
+
value = self._merged(name, entity, key, mutation)
|
|
648
|
+
self._touched.add((name, key))
|
|
649
|
+
try:
|
|
650
|
+
with self._trusted():
|
|
651
|
+
self._write_row(operation, name, key, value)
|
|
652
|
+
except self.native_error as error:
|
|
653
|
+
if not self._is_constraint_violation(error):
|
|
654
|
+
raise
|
|
655
|
+
raise ContractError(
|
|
656
|
+
"unique",
|
|
657
|
+
"Primary or unique key conflicts with existing state",
|
|
658
|
+
entity=name,
|
|
659
|
+
key=json.loads(key),
|
|
660
|
+
schema_path=f"{entity['location']}/unique",
|
|
661
|
+
) from error
|
|
662
|
+
return key
|
|
663
|
+
|
|
664
|
+
def _merged(self, name: str, entity: dict, key: str, mutation: dict) -> dict | None:
|
|
665
|
+
current = self.get(name, mutation["key"])
|
|
666
|
+
if current is None:
|
|
667
|
+
fail(
|
|
668
|
+
"not_found",
|
|
669
|
+
"No entity exists at this key",
|
|
670
|
+
entity=name,
|
|
671
|
+
key=mutation["key"],
|
|
672
|
+
)
|
|
673
|
+
if mutation["op"] != "update":
|
|
674
|
+
return None
|
|
675
|
+
if not isinstance(mutation["value"], dict):
|
|
676
|
+
fail("type", "An update must be an object")
|
|
677
|
+
value = {**current, **mutation["value"]}
|
|
678
|
+
validate(entity["schema"], value, location=f"{entity['location']}/schema")
|
|
679
|
+
if self._key(name, value) != key:
|
|
680
|
+
fail(
|
|
681
|
+
"key_immutable",
|
|
682
|
+
"Updates cannot change primary keys",
|
|
683
|
+
entity=name,
|
|
684
|
+
)
|
|
685
|
+
return value
|
|
686
|
+
|
|
687
|
+
def _write_row(self, operation: str, name: str, key: str, value: dict | None) -> None:
|
|
688
|
+
table = quote(name)
|
|
689
|
+
if operation == "insert":
|
|
690
|
+
self._execute(f"INSERT INTO {table} (_key, payload) VALUES (?, ?)", (key, canonical(value)))
|
|
691
|
+
return
|
|
692
|
+
if operation == "update":
|
|
693
|
+
self._execute(f"UPDATE {table} SET payload = ? WHERE _key = ?", (canonical(value), key))
|
|
694
|
+
return
|
|
695
|
+
self._execute(f"DELETE FROM {table} WHERE _key = ?", (key,))
|
|
696
|
+
|
|
697
|
+
def apply_batch(self, batch: dict) -> dict:
|
|
698
|
+
"""Atomic rejection; exact retries no-op, conflicting ID reuse rejects."""
|
|
699
|
+
members(
|
|
700
|
+
batch,
|
|
701
|
+
{"id", "target", "mapping", "coverage", "mutations"},
|
|
702
|
+
{"id", "target", "mapping", "coverage", "mutations"},
|
|
703
|
+
"/batch",
|
|
704
|
+
)
|
|
705
|
+
nonempty(batch["id"], "/batch/id")
|
|
706
|
+
if not self.contract.targets(batch["target"]):
|
|
707
|
+
fail(
|
|
708
|
+
"batch_target",
|
|
709
|
+
"Batch targets a different domain identity/version/hash",
|
|
710
|
+
"/batch/target",
|
|
711
|
+
)
|
|
712
|
+
members(batch["mapping"], {"id", "version"}, {"id", "version"}, "/batch/mapping")
|
|
713
|
+
for key, value in batch["mapping"].items():
|
|
714
|
+
nonempty(value, f"/batch/mapping/{key}")
|
|
715
|
+
if batch["coverage"] not in ("partial", "complete"):
|
|
716
|
+
fail(
|
|
717
|
+
"coverage",
|
|
718
|
+
"coverage must be partial or complete (neither implies deletion)",
|
|
719
|
+
"/batch/coverage",
|
|
720
|
+
)
|
|
721
|
+
if not isinstance(batch["mutations"], list):
|
|
722
|
+
fail("batch", "mutations must be an array", "/batch/mutations")
|
|
723
|
+
batch_hash = digest(encoded(batch))
|
|
724
|
+
with self.transaction():
|
|
725
|
+
with self._trusted():
|
|
726
|
+
prior = self._execute("SELECT digest FROM _gateway_batches WHERE id = ?", (batch["id"],)).fetchone()
|
|
727
|
+
if prior:
|
|
728
|
+
if prior[0] != batch_hash:
|
|
729
|
+
fail(
|
|
730
|
+
"idempotency_conflict",
|
|
731
|
+
"Batch ID was already applied with different content",
|
|
732
|
+
"/batch/id",
|
|
733
|
+
batch=batch["id"],
|
|
734
|
+
)
|
|
735
|
+
return {"status": "already_applied", "batch": batch["id"], "applied": 0}
|
|
736
|
+
with self._trusted():
|
|
737
|
+
self._execute(
|
|
738
|
+
"INSERT INTO _gateway_batches VALUES (?, ?, ?)",
|
|
739
|
+
(
|
|
740
|
+
batch["id"],
|
|
741
|
+
batch_hash,
|
|
742
|
+
canonical({k: v for k, v in batch.items() if k != "mutations"}),
|
|
743
|
+
),
|
|
744
|
+
)
|
|
745
|
+
for index, mutation in enumerate(batch["mutations"]):
|
|
746
|
+
self._apply_mutation(batch["id"], index, mutation)
|
|
747
|
+
return {
|
|
748
|
+
"status": "applied",
|
|
749
|
+
"batch": batch["id"],
|
|
750
|
+
"applied": len(batch["mutations"]),
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
def _apply_mutation(self, batch_id: str, index: int, mutation: Any) -> None:
|
|
754
|
+
location = f"/batch/mutations/{index}"
|
|
755
|
+
try:
|
|
756
|
+
members(
|
|
757
|
+
mutation,
|
|
758
|
+
{"op", "entity", "key", "value", "source"},
|
|
759
|
+
{"op", "entity", "source"},
|
|
760
|
+
location,
|
|
761
|
+
)
|
|
762
|
+
op = mutation["op"]
|
|
763
|
+
if not isinstance(op, str):
|
|
764
|
+
fail(
|
|
765
|
+
"operation",
|
|
766
|
+
"Operation must be insert, update or delete",
|
|
767
|
+
location,
|
|
768
|
+
)
|
|
769
|
+
required = {
|
|
770
|
+
"insert": {"value"},
|
|
771
|
+
"update": {"key", "value"},
|
|
772
|
+
"delete": {"key"},
|
|
773
|
+
}.get(op)
|
|
774
|
+
if required is None:
|
|
775
|
+
fail(
|
|
776
|
+
"operation",
|
|
777
|
+
"Only insert, update and delete are supported",
|
|
778
|
+
location,
|
|
779
|
+
)
|
|
780
|
+
members(
|
|
781
|
+
mutation,
|
|
782
|
+
{"op", "entity", "source"} | required,
|
|
783
|
+
{"op", "entity", "source"} | required,
|
|
784
|
+
location,
|
|
785
|
+
)
|
|
786
|
+
validate_source(mutation["source"], f"{location}/source")
|
|
787
|
+
key = self._mutate(mutation)
|
|
788
|
+
with self._trusted():
|
|
789
|
+
self._execute(
|
|
790
|
+
"INSERT INTO _gateway_lineage VALUES (?, ?, ?, ?, ?)",
|
|
791
|
+
(
|
|
792
|
+
batch_id,
|
|
793
|
+
index,
|
|
794
|
+
mutation["entity"],
|
|
795
|
+
key,
|
|
796
|
+
canonical(mutation["source"]),
|
|
797
|
+
),
|
|
798
|
+
)
|
|
799
|
+
except ContractError as error:
|
|
800
|
+
raise error.contextualize(**self._mutation_context(batch_id, index, mutation, location))
|
|
801
|
+
|
|
802
|
+
@staticmethod
|
|
803
|
+
def _mutation_context(batch_id: str, index: int, mutation: Any, location: str) -> dict:
|
|
804
|
+
context = {"batch": batch_id, "index": index}
|
|
805
|
+
if not isinstance(mutation, dict):
|
|
806
|
+
return context
|
|
807
|
+
context["entity"] = mutation.get("entity")
|
|
808
|
+
if "key" in mutation:
|
|
809
|
+
context["key"] = mutation["key"]
|
|
810
|
+
# A source is echoed back only once it has passed metadata validation.
|
|
811
|
+
try:
|
|
812
|
+
validate_source(mutation.get("source"), location)
|
|
813
|
+
context["source"] = mutation["source"]
|
|
814
|
+
except ContractError:
|
|
815
|
+
pass
|
|
816
|
+
return context
|
|
817
|
+
|
|
818
|
+
def _single(self, op: str, entity: str, *, source: dict | None = None, **values):
|
|
819
|
+
self.apply_batch(
|
|
820
|
+
{
|
|
821
|
+
"id": uuid.uuid4().hex,
|
|
822
|
+
"target": self.contract.target,
|
|
823
|
+
"mapping": {"id": "gateway-store", "version": "1"},
|
|
824
|
+
"coverage": "partial",
|
|
825
|
+
"mutations": [
|
|
826
|
+
{
|
|
827
|
+
"op": op,
|
|
828
|
+
"entity": entity,
|
|
829
|
+
**values,
|
|
830
|
+
"source": (source if source is not None else {"record": f"store.{op}"}),
|
|
831
|
+
}
|
|
832
|
+
],
|
|
833
|
+
}
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
def insert(self, entity: str, value: dict, *, source: dict | None = None) -> dict:
|
|
837
|
+
self._single("insert", entity, value=value, source=source)
|
|
838
|
+
return json.loads(canonical(value))
|
|
839
|
+
|
|
840
|
+
def update(self, entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict:
|
|
841
|
+
with self._lock:
|
|
842
|
+
self._single("update", entity, key=key, value=patch, source=source)
|
|
843
|
+
return self.get(entity, key)
|
|
844
|
+
|
|
845
|
+
def delete(self, entity: str, key: dict, *, source: dict | None = None) -> None:
|
|
846
|
+
self._single("delete", entity, key=key, source=source)
|
|
847
|
+
|
|
848
|
+
@contextmanager
|
|
849
|
+
def ingest(
|
|
850
|
+
self,
|
|
851
|
+
*,
|
|
852
|
+
source: dict,
|
|
853
|
+
mapping: dict,
|
|
854
|
+
batch_id: str | None = None,
|
|
855
|
+
limit: int = 50,
|
|
856
|
+
) -> Iterator[IngestConnection]:
|
|
857
|
+
"""Commits only if every row conforms (up to ``limit`` violations reported together). Known gap (#474): a TEMP
|
|
858
|
+
object named like a main table still shadows it."""
|
|
859
|
+
session = IngestSession(self, source=source, mapping=mapping, batch_id=batch_id, limit=limit)
|
|
860
|
+
yield from session.run()
|
|
861
|
+
|
|
862
|
+
def lineage(self, entity: str, key: dict) -> list[dict]:
|
|
863
|
+
with self._lock:
|
|
864
|
+
encoded_key = self._key(entity, key, exact=True)
|
|
865
|
+
with self._trusted():
|
|
866
|
+
rows = self._execute(
|
|
867
|
+
"SELECT b.metadata, l.ordinal, l.source "
|
|
868
|
+
"FROM _gateway_lineage l "
|
|
869
|
+
"JOIN _gateway_batches b ON b.id=l.batch "
|
|
870
|
+
"WHERE l.entity=? AND l.key=? "
|
|
871
|
+
f"ORDER BY b.{self.insertion_order}, l.ordinal",
|
|
872
|
+
(entity, encoded_key),
|
|
873
|
+
).fetchall()
|
|
874
|
+
return [
|
|
875
|
+
{
|
|
876
|
+
"batch": json.loads(row[0]),
|
|
877
|
+
"index": row[1],
|
|
878
|
+
"source": json.loads(row[2]),
|
|
879
|
+
}
|
|
880
|
+
for row in rows
|
|
881
|
+
]
|
|
882
|
+
|
|
883
|
+
def audit(self) -> None:
|
|
884
|
+
with self._lock:
|
|
885
|
+
violations = self._audit.violations(limit=1)
|
|
886
|
+
if violations:
|
|
887
|
+
raise violations[0]
|
|
888
|
+
|
|
889
|
+
def counts(self) -> dict[str, int]:
|
|
890
|
+
"""Rows per entity, counted by the engine: no payload is decoded."""
|
|
891
|
+
with self._lock, self._trusted():
|
|
892
|
+
return {
|
|
893
|
+
name: self._execute(f"SELECT COUNT(*) FROM {quote(name)}").fetchone()[0]
|
|
894
|
+
for name in self.contract.entities
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
def snapshot(self, destination: str | Path) -> Path:
|
|
898
|
+
"""Drops the :class:`ReadIndexes`, so its bytes do not depend on where it was taken. The engine may let the
|
|
899
|
+
lock go between copy steps; :meth:`close` waits for a copy in progress."""
|
|
900
|
+
dest = Path(destination)
|
|
901
|
+
with self._lock:
|
|
902
|
+
if self._depth:
|
|
903
|
+
fail("transaction", "Cannot snapshot uncommitted state")
|
|
904
|
+
if self._closed:
|
|
905
|
+
fail("state_closed", "Cannot snapshot a closed store")
|
|
906
|
+
if dest.exists():
|
|
907
|
+
fail("state_exists", "Snapshot destination already exists")
|
|
908
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
909
|
+
self._copies += 1
|
|
910
|
+
try:
|
|
911
|
+
self._write_copy(dest)
|
|
912
|
+
with self._lock:
|
|
913
|
+
held = self._schema_objects()
|
|
914
|
+
self.drop_indexes(dest, ReadIndexes.named(held))
|
|
915
|
+
except BaseException:
|
|
916
|
+
dest.unlink(missing_ok=True)
|
|
917
|
+
raise
|
|
918
|
+
finally:
|
|
919
|
+
with self._lock:
|
|
920
|
+
self._copies -= 1
|
|
921
|
+
self._copied.notify_all()
|
|
922
|
+
return dest
|
|
923
|
+
|
|
924
|
+
def close(self) -> None:
|
|
925
|
+
with self._lock:
|
|
926
|
+
if self._depth:
|
|
927
|
+
fail("transaction", "Cannot close an active tool/batch transaction")
|
|
928
|
+
self._copied.wait_for(lambda: not self._copies)
|
|
929
|
+
self._close_connection()
|
|
930
|
+
self._closed = True
|
|
931
|
+
|
|
932
|
+
def __enter__(self):
|
|
933
|
+
return self
|
|
934
|
+
|
|
935
|
+
def __exit__(self, *args):
|
|
936
|
+
self.close()
|