@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -18,8 +18,9 @@ way that kind seeds:
|
|
|
18
18
|
are generated or per-task; the import refuses and names what to edit.
|
|
19
19
|
|
|
20
20
|
The same row planning seeds a *live* session: :func:`seed_batch` builds the
|
|
21
|
-
batch a running store applies atomically, and
|
|
22
|
-
the inserts
|
|
21
|
+
batch a running store applies atomically, and
|
|
22
|
+
:func:`~gatewaysdk.environment.legacy_sqlite.seed_sqlite` executes the inserts
|
|
23
|
+
against a mock-tools task's live database.
|
|
23
24
|
"""
|
|
24
25
|
|
|
25
26
|
from __future__ import annotations
|
|
@@ -28,22 +29,22 @@ import hashlib
|
|
|
28
29
|
import json
|
|
29
30
|
import re
|
|
30
31
|
import shutil
|
|
31
|
-
import sqlite3
|
|
32
32
|
import tempfile
|
|
33
33
|
import uuid
|
|
34
34
|
from contextlib import contextmanager
|
|
35
35
|
from pathlib import Path
|
|
36
36
|
from typing import Any, Iterable, Iterator
|
|
37
37
|
|
|
38
|
+
from ..legacy_sqlite import MODES, LegacySQLiteBacking, schema_columns, seed_statements
|
|
38
39
|
from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
|
|
39
40
|
from .cast import EntityCasters
|
|
40
41
|
from .compiler import bundle_mode, check_bundle, compile_bundle
|
|
42
|
+
from .database import RowMatch, WorldDatabase
|
|
43
|
+
from .primary_key import PrimaryKey
|
|
41
44
|
from .proof import SCRATCH_IGNORE, RowProof
|
|
42
|
-
from .store import SQLiteStore
|
|
43
45
|
from .tree_import import refuse_unpublishable_journal
|
|
44
|
-
from .validation import ContractError, digest, encoded, fail, journal_paths, read_json
|
|
46
|
+
from .validation import ContractError, digest, encoded, fail, journal_paths, read_json
|
|
45
47
|
|
|
46
|
-
MODES = ("append", "replace")
|
|
47
48
|
MERGES = ("upsert", "fill")
|
|
48
49
|
#: The inline lane's caps (the platform's seed and import calls share them): a local
|
|
49
50
|
#: directory import above either is refused; larger data is a platform batch.
|
|
@@ -58,9 +59,6 @@ MAPPING = {"id": "data-import", "version": "1"}
|
|
|
58
59
|
Rows = dict[str, list[dict]]
|
|
59
60
|
|
|
60
61
|
|
|
61
|
-
# ---------------------------------------------------------------- rows files
|
|
62
|
-
|
|
63
|
-
|
|
64
62
|
def parse_rows(
|
|
65
63
|
paths: Iterable[str | Path],
|
|
66
64
|
*,
|
|
@@ -94,9 +92,6 @@ def parse_rows(
|
|
|
94
92
|
return rows
|
|
95
93
|
|
|
96
94
|
|
|
97
|
-
# ---------------------------------------------------------------- world kinds
|
|
98
|
-
|
|
99
|
-
|
|
100
95
|
def world_kind(root: str | Path) -> str:
|
|
101
96
|
"""The kind the worlds paths report (``World.from_dir(...).reported_kind``):
|
|
102
97
|
``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
|
|
@@ -131,9 +126,6 @@ def _refuse_kind(kind: str, root: Path) -> None:
|
|
|
131
126
|
)
|
|
132
127
|
|
|
133
128
|
|
|
134
|
-
# ---------------------------------------------------------------- import
|
|
135
|
-
|
|
136
|
-
|
|
137
129
|
def import_rows(
|
|
138
130
|
root: str | Path,
|
|
139
131
|
rows: Rows,
|
|
@@ -201,7 +193,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
|
|
|
201
193
|
the row's index in the file, carrying the full ``refusals`` list — each
|
|
202
194
|
naming its entity, index, primary key and field — the shape the batch lane
|
|
203
195
|
reports."""
|
|
204
|
-
from .batch import shape_refusal
|
|
196
|
+
from .batch import raise_shape_refusal, shape_refusal
|
|
205
197
|
|
|
206
198
|
contract = compile_bundle(root, snapshots=False)
|
|
207
199
|
refusals = [
|
|
@@ -214,22 +206,10 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
|
|
|
214
206
|
return
|
|
215
207
|
first = refusals[0]
|
|
216
208
|
entity, index = first["entity"], first["index"]
|
|
217
|
-
row = rows[entity][index]
|
|
218
209
|
total = sum(len(listed) for listed in rows.values())
|
|
219
210
|
summary = f"{len(refusals)} of {total} row(s) refused; {first['pointer']}: " if len(refusals) > 1 else ""
|
|
220
211
|
context = {"entity": entity, "index": index, "refused": len(refusals), "rows": total, "refusals": refusals}
|
|
221
|
-
|
|
222
|
-
fail("entity", summary + first["detail"], f"/{entity}", **context)
|
|
223
|
-
if not isinstance(row, dict):
|
|
224
|
-
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
225
|
-
spec = contract.entities[entity]
|
|
226
|
-
if any(name not in row for name in spec["primary_key"]):
|
|
227
|
-
fail("primary_key", summary + first["detail"], first["pointer"], **context)
|
|
228
|
-
try:
|
|
229
|
-
validate(spec["schema"], row, location=f"{spec['location']}/schema")
|
|
230
|
-
except ContractError as error:
|
|
231
|
-
raise error.contextualize(message=summary + error.detail["message"], **context)
|
|
232
|
-
fail("rows", summary + first["detail"], first["pointer"], **context)
|
|
212
|
+
raise_shape_refusal(contract, first, rows[entity][index], summary, context, f"/{entity}")
|
|
233
213
|
|
|
234
214
|
|
|
235
215
|
def _merged(root: Path, rows: Rows, merge: str) -> Rows:
|
|
@@ -266,16 +246,9 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
|
|
|
266
246
|
the next journal instead: a replace deletes rows those journals hold, and
|
|
267
247
|
an append upserts against them, rather than being overwritten on replay.
|
|
268
248
|
"""
|
|
269
|
-
from .snapshot import
|
|
249
|
+
from .snapshot import refuse_pinned_rows
|
|
270
250
|
|
|
271
|
-
|
|
272
|
-
fail(
|
|
273
|
-
"snapshot_pinned",
|
|
274
|
-
"This world's data is a hosted snapshot (db/snapshot.json), which its data files "
|
|
275
|
-
"cannot reproduce: import rows through the platform (`gateway worlds data import "
|
|
276
|
-
"<slug>` or import_world_data) instead of into the checkout",
|
|
277
|
-
"/db/snapshot.json",
|
|
278
|
-
)
|
|
251
|
+
refuse_pinned_rows(root)
|
|
279
252
|
contract = compile_bundle(root)
|
|
280
253
|
journals = journal_paths(root)
|
|
281
254
|
initial_path = root / "data/initial.json"
|
|
@@ -287,8 +260,7 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
|
|
|
287
260
|
if not fresh:
|
|
288
261
|
return {"kind": KIND_SCHEMA, "batch": prior_id, "entity_counts": entity_counts(root), "changed": False}
|
|
289
262
|
if mode == "replace" and not refreshed:
|
|
290
|
-
# Folded into initial.json
|
|
291
|
-
# entities' old mutations are dropped and the rows inserted afresh.
|
|
263
|
+
# Folded into initial.json: the named entities' old mutations are dropped and the rows inserted afresh.
|
|
292
264
|
prior = [m for m in prior if m.get("entity") not in rows]
|
|
293
265
|
fresh = _plan(contract, rows, mode, _held(contract, prior))
|
|
294
266
|
match = re.search(r"\.v(\d+)$", prior_id)
|
|
@@ -323,15 +295,15 @@ def _import_schema(root: Path, rows: Rows, mode: str, batch_id: str | None) -> d
|
|
|
323
295
|
(staged / written).parent.mkdir(parents=True, exist_ok=True)
|
|
324
296
|
(staged / written).write_bytes(payload)
|
|
325
297
|
# Building the staged snapshot is the proof: every row goes through the store.
|
|
326
|
-
|
|
298
|
+
contract = compile_bundle(staged, snapshots=False)
|
|
299
|
+
counts = build_default(staged, contract)["entityCounts"]
|
|
327
300
|
(root / written).parent.mkdir(parents=True, exist_ok=True)
|
|
328
301
|
(root / written).write_bytes(payload)
|
|
329
|
-
# The proven snapshot moves
|
|
330
|
-
|
|
331
|
-
target = default_snapshot(root)
|
|
302
|
+
# The proven snapshot moves in; compile then finds it current and only rebuilds the scenarios.
|
|
303
|
+
target = default_snapshot(root, contract)
|
|
332
304
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
333
|
-
shutil.move(str(default_snapshot(staged)), str(target))
|
|
334
|
-
shutil.move(str(sidecar_path(default_snapshot(staged))), str(sidecar_path(target)))
|
|
305
|
+
shutil.move(str(default_snapshot(staged, contract)), str(target))
|
|
306
|
+
shutil.move(str(sidecar_path(default_snapshot(staged, contract))), str(sidecar_path(target)))
|
|
335
307
|
compile_bundle(root)
|
|
336
308
|
check_bundle(root)
|
|
337
309
|
return {"kind": KIND_SCHEMA, "batch": batch["id"], "entity_counts": counts, "changed": True}
|
|
@@ -363,10 +335,7 @@ def _held(contract: Any, mutations: list[dict]) -> dict[tuple[str, tuple], dict]
|
|
|
363
335
|
|
|
364
336
|
|
|
365
337
|
def _key(contract: Any, entity: str, row: dict) -> tuple | None:
|
|
366
|
-
|
|
367
|
-
if not fields or any(field not in row for field in fields):
|
|
368
|
-
return None
|
|
369
|
-
return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
|
|
338
|
+
return PrimaryKey.in_contract(contract, entity).identity(row)
|
|
370
339
|
|
|
371
340
|
|
|
372
341
|
def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], dict]) -> list[dict]:
|
|
@@ -389,12 +358,11 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
|
|
|
389
358
|
}
|
|
390
359
|
for (entity, key), value in list(held.items()):
|
|
391
360
|
if entity in rows and (entity, key) not in incoming:
|
|
392
|
-
fields = contract.entities[entity]["primary_key"]
|
|
393
361
|
mutations.append(
|
|
394
362
|
{
|
|
395
363
|
"op": "delete",
|
|
396
364
|
"entity": entity,
|
|
397
|
-
"key":
|
|
365
|
+
"key": PrimaryKey.in_contract(contract, entity).key(value),
|
|
398
366
|
"source": {"record": f"data-import:{entity}:replace"},
|
|
399
367
|
}
|
|
400
368
|
)
|
|
@@ -404,13 +372,13 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
|
|
|
404
372
|
fail(
|
|
405
373
|
"entity", f"Unknown entity {entity}; the world has {', '.join(sorted(contract.entities))}", f"/{entity}"
|
|
406
374
|
)
|
|
407
|
-
|
|
375
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
408
376
|
for index, row in enumerate(listed):
|
|
409
377
|
if not isinstance(row, dict):
|
|
410
378
|
fail("rows", "A row must be an object", f"/{entity}/{index}")
|
|
411
|
-
missing =
|
|
412
|
-
if missing:
|
|
413
|
-
fail("primary_key", f"Row is missing primary key field {missing
|
|
379
|
+
missing = primary.missing(row)
|
|
380
|
+
if missing is not None:
|
|
381
|
+
fail("primary_key", f"Row is missing primary key field {missing!r}", f"/{entity}/{index}")
|
|
414
382
|
key = _key(contract, entity, row)
|
|
415
383
|
assert key is not None
|
|
416
384
|
existing = held.get((entity, key))
|
|
@@ -424,7 +392,7 @@ def _plan(contract: Any, rows: Rows, mode: str, held: dict[tuple[str, tuple], di
|
|
|
424
392
|
"source": {"record": f"data-import:{entity}:{identity}"},
|
|
425
393
|
}
|
|
426
394
|
if existing is not None:
|
|
427
|
-
mutation["key"] =
|
|
395
|
+
mutation["key"] = primary.key(row)
|
|
428
396
|
mutations.append(mutation)
|
|
429
397
|
held[(entity, key)] = row
|
|
430
398
|
return mutations
|
|
@@ -445,22 +413,18 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
|
|
|
445
413
|
if entity not in contract.entities:
|
|
446
414
|
continue
|
|
447
415
|
if mode == "replace":
|
|
448
|
-
# Every held row of a replaced entity goes; the plan needs them all.
|
|
449
416
|
for value in store.list(entity):
|
|
450
417
|
key = _key(contract, entity, value)
|
|
451
418
|
if key is not None:
|
|
452
419
|
held[(entity, key)] = value
|
|
453
420
|
continue
|
|
454
|
-
|
|
455
|
-
fields = contract.entities[entity]["primary_key"]
|
|
421
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
456
422
|
for row in listed:
|
|
457
|
-
|
|
458
|
-
continue
|
|
459
|
-
key = _key(contract, entity, row)
|
|
423
|
+
key = _key(contract, entity, row) if isinstance(row, dict) else None
|
|
460
424
|
if key is None or (entity, key) in held:
|
|
461
425
|
continue
|
|
462
426
|
try:
|
|
463
|
-
value = store.get(entity,
|
|
427
|
+
value = store.get(entity, primary.key(row))
|
|
464
428
|
except ContractError:
|
|
465
429
|
continue # a key the contract refuses is refused by the plan, with the field named
|
|
466
430
|
if value is not None:
|
|
@@ -475,20 +439,11 @@ def seed_batch(store: Any, contract: Any, rows: Rows, mode: str = "append", *, b
|
|
|
475
439
|
}
|
|
476
440
|
|
|
477
441
|
|
|
478
|
-
# ---------------------------------------------------------------- mock-tools
|
|
479
|
-
|
|
480
|
-
|
|
481
442
|
def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
|
|
482
443
|
schema = root / "db/schema.sql"
|
|
483
444
|
if not schema.is_file():
|
|
484
445
|
fail("schema", "mock-tools world has no db/schema.sql to read tables from", "db/schema.sql")
|
|
485
|
-
|
|
486
|
-
try:
|
|
487
|
-
probe.executescript(schema.read_text(encoding="utf-8"))
|
|
488
|
-
except sqlite3.Error as error:
|
|
489
|
-
fail("schema", f"db/schema.sql does not load: {error}", "db/schema.sql")
|
|
490
|
-
columns = sqlite_columns(probe)
|
|
491
|
-
statements = sqlite_statements(columns, rows, mode)
|
|
446
|
+
statements = seed_statements(schema_columns(schema), rows, mode)
|
|
492
447
|
if not statements:
|
|
493
448
|
return {"kind": KIND_MOCK_TOOLS, "entity_counts": entity_counts(root), "changed": False}
|
|
494
449
|
seed = root / "db/seed.sql"
|
|
@@ -504,117 +459,17 @@ def _import_mock_tools(root: Path, rows: Rows, mode: str) -> dict:
|
|
|
504
459
|
(staged / "db/seed.sql").write_text(appended, encoding="utf-8")
|
|
505
460
|
database = Path(scratch) / "world.db"
|
|
506
461
|
materialize_template(staged, database, require_sql=True)
|
|
507
|
-
counts =
|
|
462
|
+
counts = _legacy_counts(database)
|
|
508
463
|
seed.write_text(appended, encoding="utf-8")
|
|
509
464
|
return {"kind": KIND_MOCK_TOOLS, "entity_counts": counts, "changed": True, "statements": len(statements)}
|
|
510
465
|
|
|
511
466
|
|
|
512
|
-
def
|
|
513
|
-
|
|
514
|
-
tables = [
|
|
515
|
-
row[0]
|
|
516
|
-
for row in conn.execute("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'")
|
|
517
|
-
]
|
|
518
|
-
return {table: [row[1] for row in conn.execute(f"PRAGMA table_info({sql_ident(table)})")] for table in tables}
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
def sql_ident(name: str) -> str:
|
|
522
|
-
"""A SQLite identifier, quoted: a table or column named by a schema is a
|
|
523
|
-
name, never SQL, whatever characters it carries."""
|
|
524
|
-
return '"' + name.replace('"', '""') + '"'
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
def _check_sqlite_rows(columns: dict[str, list[str]], rows: Rows) -> None:
|
|
528
|
-
for table, listed in rows.items():
|
|
529
|
-
if table not in columns:
|
|
530
|
-
fail(
|
|
531
|
-
"table",
|
|
532
|
-
f"No table {table} in db/schema.sql; tables: {', '.join(sorted(columns)) or 'none'}",
|
|
533
|
-
f"/{table}",
|
|
534
|
-
)
|
|
535
|
-
for index, row in enumerate(listed):
|
|
536
|
-
if not isinstance(row, dict):
|
|
537
|
-
fail("rows", "A row must be an object", f"/{table}/{index}")
|
|
538
|
-
for column, value in row.items():
|
|
539
|
-
if column not in columns[table]:
|
|
540
|
-
fail(
|
|
541
|
-
"column",
|
|
542
|
-
f"Column {column} is not in table {table}; columns: {', '.join(columns[table])}",
|
|
543
|
-
f"/{table}/{index}/{column}",
|
|
544
|
-
)
|
|
545
|
-
if not isinstance(value, (str, int, float, bool, type(None))):
|
|
546
|
-
fail(
|
|
547
|
-
"value",
|
|
548
|
-
f"Column {column} of {table} takes a scalar; encode nested values as text yourself",
|
|
549
|
-
f"/{table}/{index}/{column}",
|
|
550
|
-
)
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
def sqlite_statements(columns: dict[str, list[str]], rows: Rows, mode: str = "append") -> list[str]:
|
|
554
|
-
"""SQL text that seeds ``rows`` into tables with these ``columns``.
|
|
555
|
-
|
|
556
|
-
``append`` writes ``INSERT OR REPLACE`` (an upsert by the table's own
|
|
557
|
-
primary key); ``replace`` prefixes ``DELETE FROM`` for each named table and
|
|
558
|
-
writes plain inserts.
|
|
559
|
-
"""
|
|
560
|
-
if mode not in MODES:
|
|
561
|
-
fail("mode", f"mode must be one of {', '.join(MODES)}", "/mode")
|
|
562
|
-
_check_sqlite_rows(columns, rows)
|
|
563
|
-
statements: list[str] = []
|
|
564
|
-
verb = "INSERT" if mode == "replace" else "INSERT OR REPLACE"
|
|
565
|
-
for table, listed in rows.items():
|
|
566
|
-
if mode == "replace":
|
|
567
|
-
statements.append(f"DELETE FROM {sql_ident(table)};")
|
|
568
|
-
for row in listed:
|
|
569
|
-
if not row:
|
|
570
|
-
statements.append(f"{verb} INTO {sql_ident(table)} DEFAULT VALUES;")
|
|
571
|
-
continue
|
|
572
|
-
names = ", ".join(sql_ident(column) for column in row)
|
|
573
|
-
values = ", ".join(_sql_literal(value) for value in row.values())
|
|
574
|
-
statements.append(f"{verb} INTO {sql_ident(table)} ({names}) VALUES ({values});")
|
|
575
|
-
return statements
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
def _sql_literal(value: Any) -> str:
|
|
579
|
-
if value is None:
|
|
580
|
-
return "NULL"
|
|
581
|
-
if isinstance(value, bool):
|
|
582
|
-
return "1" if value else "0"
|
|
583
|
-
if isinstance(value, (int, float)):
|
|
584
|
-
return repr(value)
|
|
585
|
-
return "'" + str(value).replace("'", "''") + "'"
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
def seed_sqlite(conn: sqlite3.Connection, rows: Rows, mode: str = "append") -> dict[str, int]:
|
|
589
|
-
"""Execute the seed for ``rows`` against a live sqlite, in one transaction.
|
|
590
|
-
|
|
591
|
-
Columns come from the live database itself. A refused row rolls the whole
|
|
592
|
-
seed back. Returns the row count of every touched table afterwards.
|
|
593
|
-
"""
|
|
594
|
-
statements = sqlite_statements(sqlite_columns(conn), rows, mode)
|
|
467
|
+
def _legacy_counts(database: Path) -> dict[str, int]:
|
|
468
|
+
backing = LegacySQLiteBacking(database)
|
|
595
469
|
try:
|
|
596
|
-
|
|
597
|
-
for statement in statements:
|
|
598
|
-
conn.execute(statement)
|
|
599
|
-
conn.commit()
|
|
600
|
-
except sqlite3.Error as error:
|
|
601
|
-
conn.rollback()
|
|
602
|
-
fail("sqlite", f"seed refused by the database: {error}", "/rows")
|
|
603
|
-
return {table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0] for table in rows}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
def _sqlite_counts(database: Path) -> dict[str, int]:
|
|
607
|
-
conn = sqlite3.connect(database)
|
|
608
|
-
try:
|
|
609
|
-
return {
|
|
610
|
-
table: conn.execute(f"SELECT COUNT(*) FROM {sql_ident(table)}").fetchone()[0]
|
|
611
|
-
for table in sqlite_columns(conn)
|
|
612
|
-
}
|
|
470
|
+
return backing.counts()
|
|
613
471
|
finally:
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
# ---------------------------------------------------------------- show
|
|
472
|
+
backing.close()
|
|
618
473
|
|
|
619
474
|
|
|
620
475
|
def entity_counts(root: str | Path) -> dict[str, int]:
|
|
@@ -628,33 +483,33 @@ def entity_counts(root: str | Path) -> dict[str, int]:
|
|
|
628
483
|
if kind not in (KIND_SCHEMA, KIND_MOCK_TOOLS):
|
|
629
484
|
_refuse_kind(kind, root)
|
|
630
485
|
if kind == KIND_SCHEMA:
|
|
631
|
-
from .snapshot import
|
|
486
|
+
from .snapshot import held_default
|
|
632
487
|
|
|
633
488
|
contract = check_bundle(root)
|
|
634
|
-
|
|
635
|
-
if
|
|
636
|
-
return dict(
|
|
489
|
+
held = held_default(root, contract)
|
|
490
|
+
if held is not None:
|
|
491
|
+
return dict(held[1]["entityCounts"])
|
|
637
492
|
from .._materialize import materialize_template
|
|
638
493
|
|
|
639
494
|
with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
|
|
640
495
|
database = Path(scratch) / "world.db"
|
|
641
496
|
materialize_template(root, database, require_sql=kind == KIND_MOCK_TOOLS)
|
|
642
497
|
if kind == KIND_SCHEMA:
|
|
643
|
-
with
|
|
498
|
+
with WorldDatabase.open(database, contract) as store:
|
|
644
499
|
return store.counts()
|
|
645
|
-
return
|
|
500
|
+
return _legacy_counts(database)
|
|
646
501
|
|
|
647
502
|
|
|
648
503
|
@contextmanager
|
|
649
|
-
def
|
|
504
|
+
def schema_store(root: Path, contract: Any, scenario: str | None = None) -> Iterator[WorldDatabase]:
|
|
650
505
|
"""The rows of a schema world, read only: the compiled snapshot opened in place
|
|
651
|
-
when it is current (
|
|
652
|
-
else the data files applied in a scratch
|
|
653
|
-
``scenario``, the rows a session opened in
|
|
654
|
-
from: the template with the scenario's seed overlay
|
|
655
|
-
built in scratch (the compiled
|
|
506
|
+
when it is current (the default snapshot is this data through this contract, or a
|
|
507
|
+
pinned tree's copy in the snapshot store), else the data files applied in a scratch
|
|
508
|
+
copy, as a served world would. With ``scenario``, the rows a session opened in
|
|
509
|
+
``scenarios/<scenario>/`` starts from: the template with the scenario's seed overlay
|
|
510
|
+
replacing whole entities, built in scratch (the compiled scenario snapshot is not
|
|
656
511
|
consulted, so the answer is the tree as it is now)."""
|
|
657
|
-
from .snapshot import SCENARIOS,
|
|
512
|
+
from .snapshot import SCENARIOS, held_default, scenario_dirs, scenario_overlay
|
|
658
513
|
|
|
659
514
|
overlay: dict[str, list] | None = None
|
|
660
515
|
if scenario is not None:
|
|
@@ -666,8 +521,9 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
|
|
|
666
521
|
f"/{SCENARIOS}/{scenario}",
|
|
667
522
|
)
|
|
668
523
|
overlay = scenario_overlay(scenarios[scenario])
|
|
669
|
-
|
|
670
|
-
|
|
524
|
+
held = held_default(root, contract) if overlay is None else None
|
|
525
|
+
if held is not None:
|
|
526
|
+
with WorldDatabase.open(held[0], contract, audit=False) as store:
|
|
671
527
|
yield store
|
|
672
528
|
return
|
|
673
529
|
from .._materialize import materialize_template
|
|
@@ -675,7 +531,7 @@ def _schema_store(root: Path, contract: Any, scenario: str | None = None) -> Ite
|
|
|
675
531
|
with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
|
|
676
532
|
database = Path(scratch) / "world.db"
|
|
677
533
|
materialize_template(root, database)
|
|
678
|
-
with
|
|
534
|
+
with WorldDatabase.open(database, contract) as store:
|
|
679
535
|
if overlay:
|
|
680
536
|
store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{scenario}"))
|
|
681
537
|
yield store
|
|
@@ -699,15 +555,11 @@ def _table_rows(root: Path, entity: str) -> tuple[list[dict], list[str]]:
|
|
|
699
555
|
with tempfile.TemporaryDirectory(prefix="gateway-show-") as scratch:
|
|
700
556
|
database = Path(scratch) / "world.db"
|
|
701
557
|
materialize_template(root, database, require_sql=True)
|
|
702
|
-
|
|
558
|
+
backing = LegacySQLiteBacking(database)
|
|
703
559
|
try:
|
|
704
|
-
|
|
705
|
-
if entity not in columns:
|
|
706
|
-
fail("table", f"No table {entity}; tables: {', '.join(sorted(columns)) or 'none'}", f"/{entity}")
|
|
707
|
-
conn.row_factory = sqlite3.Row
|
|
708
|
-
return [dict(row) for row in conn.execute(f"SELECT * FROM {sql_ident(entity)}")], columns[entity]
|
|
560
|
+
return backing.table_rows(entity)
|
|
709
561
|
finally:
|
|
710
|
-
|
|
562
|
+
backing.close()
|
|
711
563
|
|
|
712
564
|
|
|
713
565
|
def _seed_rows(root: Path, entity: str) -> list[dict]:
|
|
@@ -729,7 +581,7 @@ def entity_rows(root: str | Path, entity: str) -> list[dict]:
|
|
|
729
581
|
return _table_rows(root, entity)[0]
|
|
730
582
|
if kind != KIND_SCHEMA:
|
|
731
583
|
_refuse_kind(kind, root)
|
|
732
|
-
with
|
|
584
|
+
with schema_store(root, _schema_contract(root, entity)) as store:
|
|
733
585
|
return list(store.list(entity))
|
|
734
586
|
|
|
735
587
|
|
|
@@ -738,9 +590,7 @@ def _listed(value: Any) -> list:
|
|
|
738
590
|
|
|
739
591
|
|
|
740
592
|
def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]], entity: str) -> list[dict]:
|
|
741
|
-
"""The rows every ``wanted`` field of which matches, by the store's rule (
|
|
742
|
-
from .api import _matches
|
|
743
|
-
|
|
593
|
+
"""The rows every ``wanted`` field of which matches, by the store's rule (:class:`RowMatch`)."""
|
|
744
594
|
unknown = sorted(set(wanted) - set(fields))
|
|
745
595
|
if unknown:
|
|
746
596
|
fail(
|
|
@@ -749,8 +599,7 @@ def _filtered(rows: list[dict], fields: list[str], wanted: dict[str, list[str]],
|
|
|
749
599
|
entity=entity,
|
|
750
600
|
fields=sorted(fields),
|
|
751
601
|
)
|
|
752
|
-
|
|
753
|
-
return [row for row in rows if _matches(row, filters, wanted)]
|
|
602
|
+
return RowMatch.on_fields(wanted).select(rows)
|
|
754
603
|
|
|
755
604
|
|
|
756
605
|
def entity_query(
|
|
@@ -789,10 +638,8 @@ def entity_query(
|
|
|
789
638
|
limit = min(limit, QUERY_LIMIT)
|
|
790
639
|
resolved: dict[str, list] | None = None
|
|
791
640
|
wanted = {field: _listed(values) for field, values in where.items()}
|
|
792
|
-
# A cross-world
|
|
793
|
-
#
|
|
794
|
-
# field to its chain, validated up front so an unknown name refuses the whole
|
|
795
|
-
# query rather than one field silently matching nothing.
|
|
641
|
+
# A cross-world ``transform`` normalises the plaintext BEFORE this world's redaction; chains are
|
|
642
|
+
# validated up front so an unknown name refuses the query instead of one field matching nothing.
|
|
796
643
|
from .transforms import apply_transforms, parse_transform
|
|
797
644
|
|
|
798
645
|
chains: dict[str, list[str]] = {}
|
|
@@ -827,7 +674,7 @@ def entity_query(
|
|
|
827
674
|
wanted = {field: [str(value) for value in values] for field, values in wanted.items()}
|
|
828
675
|
kind = world_kind(root)
|
|
829
676
|
if kind == KIND_SCHEMA:
|
|
830
|
-
with
|
|
677
|
+
with schema_store(root, _schema_contract(root, entity), scenario) as store:
|
|
831
678
|
rows, total = store.list_where(entity, wanted, limit, offset)
|
|
832
679
|
else:
|
|
833
680
|
if scenario is not None:
|
|
@@ -992,8 +839,5 @@ __all__ = [
|
|
|
992
839
|
"import_rows",
|
|
993
840
|
"parse_rows",
|
|
994
841
|
"seed_batch",
|
|
995
|
-
"seed_sqlite",
|
|
996
|
-
"sqlite_columns",
|
|
997
|
-
"sqlite_statements",
|
|
998
842
|
"world_kind",
|
|
999
843
|
]
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""The world database: a contract-bound store with engine subclasses."""
|
|
2
|
+
|
|
3
|
+
from .base import WorldDatabase
|
|
4
|
+
from .budget import QueryBudget
|
|
5
|
+
from .connections import IngestConnection, ReadConnection, ReadCursor
|
|
6
|
+
from .guard import Access, AccessKind, Guard
|
|
7
|
+
from .matching import RowMatch
|
|
8
|
+
from .rows import Row, Rows
|
|
9
|
+
from .sqlite import SQLiteWorldDatabase
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Access",
|
|
13
|
+
"AccessKind",
|
|
14
|
+
"Guard",
|
|
15
|
+
"IngestConnection",
|
|
16
|
+
"QueryBudget",
|
|
17
|
+
"ReadConnection",
|
|
18
|
+
"ReadCursor",
|
|
19
|
+
"Row",
|
|
20
|
+
"Rows",
|
|
21
|
+
"RowMatch",
|
|
22
|
+
"SQLiteWorldDatabase",
|
|
23
|
+
"WorldDatabase",
|
|
24
|
+
]
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""Persisted rows checked against the contract: payload, key identity, references."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
from ..compiler import payload_value, quote
|
|
9
|
+
from ..validation import ContractError, fail, parse_json, validate
|
|
10
|
+
|
|
11
|
+
if TYPE_CHECKING:
|
|
12
|
+
from .base import WorldDatabase
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ContractAudit:
|
|
16
|
+
"""Reads a database's rows as the store wrote them and names every row the
|
|
17
|
+
contract refuses, with the batch that last wrote it."""
|
|
18
|
+
|
|
19
|
+
def __init__(self, database: "WorldDatabase") -> None:
|
|
20
|
+
self._database = database
|
|
21
|
+
|
|
22
|
+
def violations(self, limit: int | None = None) -> list[ContractError]:
|
|
23
|
+
"""Contract violations in persisted state, at most ``limit`` of them."""
|
|
24
|
+
database = self._database
|
|
25
|
+
found: list[ContractError] = []
|
|
26
|
+
with database._trusted():
|
|
27
|
+
rows = [
|
|
28
|
+
(name, entity, row)
|
|
29
|
+
for name, entity in database.contract.entities.items()
|
|
30
|
+
for row in database._execute(f"SELECT _key, payload FROM {quote(name)} ORDER BY _key").fetchall()
|
|
31
|
+
]
|
|
32
|
+
for name, entity, row in rows:
|
|
33
|
+
error = self.row(name, entity, row)
|
|
34
|
+
if error is None:
|
|
35
|
+
continue
|
|
36
|
+
found.append(error)
|
|
37
|
+
if limit is not None and len(found) >= limit:
|
|
38
|
+
return found
|
|
39
|
+
with database._trusted():
|
|
40
|
+
if database._relationships_broken():
|
|
41
|
+
found.append(self.relationship_failure())
|
|
42
|
+
return found
|
|
43
|
+
|
|
44
|
+
def touched(self, touched: set[tuple[str, str]]) -> None:
|
|
45
|
+
"""The commit-time audit of one transaction: the rows it wrote, re-read from
|
|
46
|
+
storage and checked like a full audit, plus the engine's own relationship check
|
|
47
|
+
over the whole state (a delete that orphans an untouched row shows up there).
|
|
48
|
+
Rows the transaction never touched cannot have changed, so a transaction's
|
|
49
|
+
cost follows its size, not the size of the world."""
|
|
50
|
+
database = self._database
|
|
51
|
+
with database._trusted():
|
|
52
|
+
for name, key in sorted(touched):
|
|
53
|
+
self._touched_row(name, key)
|
|
54
|
+
if database._relationships_broken():
|
|
55
|
+
raise self.relationship_failure()
|
|
56
|
+
|
|
57
|
+
def _touched_row(self, name: str, key: str) -> None:
|
|
58
|
+
database = self._database
|
|
59
|
+
row = database._execute(f"SELECT _key, payload FROM {quote(name)} WHERE _key = ?", (key,)).fetchone()
|
|
60
|
+
if row is None:
|
|
61
|
+
return
|
|
62
|
+
error = self.row(name, database.contract.entities[name], row)
|
|
63
|
+
if error is not None:
|
|
64
|
+
raise error
|
|
65
|
+
|
|
66
|
+
def relationship_failure(self) -> ContractError:
|
|
67
|
+
return ContractError("foreign_key", f"Persisted {self._database.engine_label} relationship check failed")
|
|
68
|
+
|
|
69
|
+
def row(self, name: str, entity: dict, row) -> ContractError | None:
|
|
70
|
+
"""The contract violation for one persisted row, with lineage context."""
|
|
71
|
+
try:
|
|
72
|
+
value = parse_json(row[1], f"{entity['location']}/schema")
|
|
73
|
+
except ContractError as error:
|
|
74
|
+
return error.contextualize(entity=name)
|
|
75
|
+
except (ValueError, TypeError):
|
|
76
|
+
return ContractError("state_payload", "Stored entity payload is not JSON", entity=name)
|
|
77
|
+
try:
|
|
78
|
+
validate(entity["schema"], value, location=f"{entity['location']}/schema")
|
|
79
|
+
if self._database._key(name, value) != row[0]:
|
|
80
|
+
fail("key", "Stored key differs from payload identity", entity=name)
|
|
81
|
+
for rel in entity["relationships"]:
|
|
82
|
+
self._relationship(entity, rel, value)
|
|
83
|
+
except ContractError as error:
|
|
84
|
+
return error.contextualize(**self._context(name, row[0]))
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
def _relationship(self, entity: dict, rel: dict, value: dict) -> None:
|
|
88
|
+
keys = [payload_value(value, field) for field in rel["fields"]]
|
|
89
|
+
if all(k is None for k in keys):
|
|
90
|
+
return
|
|
91
|
+
if any(k is None for k in keys):
|
|
92
|
+
fail(
|
|
93
|
+
"foreign_key",
|
|
94
|
+
"Composite references must be wholly present or wholly absent/null",
|
|
95
|
+
f"{entity['location']}/relationships",
|
|
96
|
+
)
|
|
97
|
+
target = self._database.contract.entities[rel["target"]]
|
|
98
|
+
target_key = dict(zip(target["primary_key"], keys))
|
|
99
|
+
if self._database.get(rel["target"], target_key) is None:
|
|
100
|
+
fail(
|
|
101
|
+
"foreign_key",
|
|
102
|
+
"Relationship target does not exist; incomplete evidence requires an explicit mapping decision",
|
|
103
|
+
f"{entity['location']}/relationships",
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
def _context(self, name: str, stored_key: str) -> dict:
|
|
107
|
+
database = self._database
|
|
108
|
+
try:
|
|
109
|
+
diagnostic_key = json.loads(stored_key)
|
|
110
|
+
except (ValueError, TypeError):
|
|
111
|
+
diagnostic_key = None
|
|
112
|
+
context = {"entity": name, "key": diagnostic_key}
|
|
113
|
+
with database._trusted():
|
|
114
|
+
lineage = database._execute(
|
|
115
|
+
"SELECT batch, ordinal, source FROM _gateway_lineage "
|
|
116
|
+
f"WHERE entity=? AND key=? ORDER BY {database.insertion_order} DESC LIMIT 1",
|
|
117
|
+
(name, stored_key),
|
|
118
|
+
).fetchone()
|
|
119
|
+
if lineage:
|
|
120
|
+
context.update(batch=lineage[0], index=lineage[1], source=json.loads(lineage[2]))
|
|
121
|
+
return context
|