@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -1,30 +1,33 @@
|
|
|
1
|
-
"""One authored authority -> deterministic
|
|
1
|
+
"""One authored authority -> deterministic storage DDL, runtime, docs and tools."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import re
|
|
7
|
-
from dataclasses import dataclass
|
|
7
|
+
from dataclasses import dataclass, replace
|
|
8
8
|
from pathlib import Path
|
|
9
9
|
from typing import Any
|
|
10
10
|
|
|
11
11
|
from .._toml import tomllib
|
|
12
|
+
from .tree_import import PUBLICATION_FILE_BYTES
|
|
12
13
|
from .validation import (
|
|
14
|
+
FIELD_NAME,
|
|
13
15
|
LANGUAGE,
|
|
14
16
|
MAX_ENTITY_NAME,
|
|
15
17
|
ContractError,
|
|
18
|
+
InlineBudgetExceeded,
|
|
16
19
|
Resolver,
|
|
17
20
|
canonical,
|
|
18
21
|
digest,
|
|
19
22
|
encoded,
|
|
20
23
|
fail,
|
|
24
|
+
local_definition,
|
|
21
25
|
members,
|
|
22
26
|
nonempty,
|
|
23
27
|
publication_digest,
|
|
24
28
|
)
|
|
25
29
|
|
|
26
30
|
COMPILER = "gateway-world-compiler/1"
|
|
27
|
-
TARGET = "sqlite-json/1"
|
|
28
31
|
GENERATED = (
|
|
29
32
|
"db/schema.sql",
|
|
30
33
|
"schema/runtime.json",
|
|
@@ -33,8 +36,7 @@ GENERATED = (
|
|
|
33
36
|
)
|
|
34
37
|
LOCK = "schema/lock.json"
|
|
35
38
|
ROOT = "schema/world.json"
|
|
36
|
-
#: What [environment] kind a schema world declares
|
|
37
|
-
#: label existed carry the runtime's engine name, `mock-tools`, and still load.
|
|
39
|
+
#: What [environment] kind a schema world declares; older trees carry `mock-tools` and still load.
|
|
38
40
|
SCHEMA_KIND = "schema"
|
|
39
41
|
SCHEMA_KINDS = frozenset({SCHEMA_KIND, "mock-tools"})
|
|
40
42
|
RESERVED = ("_", "sqlite_", "gateway_", "benchmark_")
|
|
@@ -73,13 +75,14 @@ def identifier(value: Any, location: str) -> str:
|
|
|
73
75
|
def field_identifier(value: Any, location: str) -> str:
|
|
74
76
|
if (
|
|
75
77
|
not isinstance(value, str)
|
|
76
|
-
or not
|
|
78
|
+
or not FIELD_NAME.fullmatch(value)
|
|
77
79
|
or value.lower() in {"_key", "payload"}
|
|
78
80
|
or value.lower().startswith(FIELD_RESERVED)
|
|
79
81
|
):
|
|
80
82
|
fail(
|
|
81
83
|
"identity",
|
|
82
|
-
"
|
|
84
|
+
"Name a field with letters, digits, _ and - (a letter or _ first); _key, payload, sqlite_, "
|
|
85
|
+
"gateway_ and benchmark_ are reserved",
|
|
83
86
|
location,
|
|
84
87
|
)
|
|
85
88
|
return value
|
|
@@ -103,19 +106,12 @@ def json_pointer_parts(value: str, location: str, *, code: str) -> list[str]:
|
|
|
103
106
|
return parts
|
|
104
107
|
|
|
105
108
|
|
|
106
|
-
def field_schema(
|
|
107
|
-
schema: dict,
|
|
108
|
-
field: str,
|
|
109
|
-
location: str,
|
|
110
|
-
*,
|
|
111
|
-
code: str = "key",
|
|
112
|
-
nested: bool = True,
|
|
113
|
-
) -> dict:
|
|
114
|
-
if field.startswith("/") and not nested:
|
|
115
|
-
fail(code, "Primary keys must use direct property names", location)
|
|
109
|
+
def field_schema(schema: dict, field: str, location: str, *, code: str = "key") -> dict:
|
|
116
110
|
parts = json_pointer_parts(field, location, code=code)
|
|
117
111
|
|
|
118
112
|
def resolve(current: dict, remaining: list[str]) -> dict | None:
|
|
113
|
+
if "$ref" in current:
|
|
114
|
+
return resolve(local_definition(schema.get("$defs", {}), current["$ref"], location), remaining)
|
|
119
115
|
if not remaining:
|
|
120
116
|
return current
|
|
121
117
|
part, *rest = remaining
|
|
@@ -131,8 +127,6 @@ def field_schema(
|
|
|
131
127
|
|
|
132
128
|
resolved = resolve(schema, parts)
|
|
133
129
|
if resolved is None:
|
|
134
|
-
# Authors reach for `owner.user_id` first (Brex retrospective); the
|
|
135
|
-
# message has to teach the pointer syntax, not just refuse.
|
|
136
130
|
fail(
|
|
137
131
|
code,
|
|
138
132
|
f"Field path {field!r} does not resolve through object properties; a nested field "
|
|
@@ -142,6 +136,19 @@ def field_schema(
|
|
|
142
136
|
return resolved
|
|
143
137
|
|
|
144
138
|
|
|
139
|
+
def required_path(schema: dict, field: str, location: str) -> bool:
|
|
140
|
+
"""Whether every step of ``field`` (a property name or a JSON Pointer) is
|
|
141
|
+
required by the object that holds it, so every row carries the value."""
|
|
142
|
+
current = schema
|
|
143
|
+
for part in json_pointer_parts(field, location, code="key"):
|
|
144
|
+
while "$ref" in current:
|
|
145
|
+
current = local_definition(schema.get("$defs", {}), current["$ref"], location)
|
|
146
|
+
if part not in current.get("required", []) or part not in current.get("properties", {}):
|
|
147
|
+
return False
|
|
148
|
+
current = current["properties"][part]
|
|
149
|
+
return True
|
|
150
|
+
|
|
151
|
+
|
|
145
152
|
def payload_value(value: dict, field: str) -> Any:
|
|
146
153
|
current: Any = value
|
|
147
154
|
for part in json_pointer_parts(field, field, code="key"):
|
|
@@ -151,22 +158,21 @@ def payload_value(value: dict, field: str) -> Any:
|
|
|
151
158
|
return current
|
|
152
159
|
|
|
153
160
|
|
|
154
|
-
def
|
|
155
|
-
|
|
156
|
-
|
|
161
|
+
def _manifest(bundle: Path) -> dict:
|
|
162
|
+
path = bundle / "gateway-env.toml"
|
|
163
|
+
try:
|
|
164
|
+
return tomllib.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
165
|
+
except (OSError, ValueError) as error:
|
|
166
|
+
raise ContractError("manifest", f"Cannot parse gateway-env.toml: {error}") from error
|
|
157
167
|
|
|
158
168
|
|
|
159
169
|
def bundle_mode(bundle: Path) -> dict | None:
|
|
160
170
|
"""Markers never downgrade to legacy when an authority has disappeared."""
|
|
161
171
|
bundle = Path(bundle)
|
|
162
|
-
path = bundle / "gateway-env.toml"
|
|
163
172
|
# exists() is False for a dangling symlink; a broken authority must fail,
|
|
164
173
|
# never quietly run as an unconstrained legacy bundle.
|
|
165
174
|
markers = any((bundle / p).exists() or (bundle / p).is_symlink() for p in (ROOT, LOCK, "schema/runtime.json"))
|
|
166
|
-
|
|
167
|
-
manifest = tomllib.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
|
|
168
|
-
except (OSError, ValueError) as error:
|
|
169
|
-
raise ContractError("manifest", f"Cannot parse gateway-env.toml: {error}") from error
|
|
175
|
+
manifest = _manifest(bundle)
|
|
170
176
|
declaration = manifest.get("schema")
|
|
171
177
|
if declaration is None and not markers:
|
|
172
178
|
return None
|
|
@@ -181,9 +187,8 @@ def bundle_mode(bundle: Path) -> dict | None:
|
|
|
181
187
|
env = manifest.get("environment", manifest.get("env", {}))
|
|
182
188
|
if not isinstance(env, dict) or env.get("kind", SCHEMA_KIND) not in SCHEMA_KINDS:
|
|
183
189
|
fail("bundle_mode", f"A schema world declares [environment] kind = '{SCHEMA_KIND}'")
|
|
190
|
+
storage_engine(manifest)
|
|
184
191
|
db = manifest.get("db", {})
|
|
185
|
-
if not isinstance(db, dict) or db.get("engine", "sqlite") != "sqlite" or env.get("db", "sqlite") != "sqlite":
|
|
186
|
-
fail("target", "Schema worlds require SQLite")
|
|
187
192
|
if any(k in db for k in ("schema", "seed", "path")):
|
|
188
193
|
fail(
|
|
189
194
|
"unsupported",
|
|
@@ -205,6 +210,24 @@ def bundle_mode(bundle: Path) -> dict | None:
|
|
|
205
210
|
return manifest
|
|
206
211
|
|
|
207
212
|
|
|
213
|
+
def storage_engine(manifest: dict) -> type:
|
|
214
|
+
"""The world database engine a schema world's manifest names (``[environment] db``,
|
|
215
|
+
``[db] engine``), else the default; every name it writes must be an engine, and the same one."""
|
|
216
|
+
from .database import WorldDatabase
|
|
217
|
+
|
|
218
|
+
table = "environment" if "environment" in manifest else "env"
|
|
219
|
+
env = manifest.get(table, {})
|
|
220
|
+
db = manifest.get("db", {})
|
|
221
|
+
if not isinstance(db, dict):
|
|
222
|
+
fail("target", f"{WorldDatabase.requirement()}; [db] must be a table naming the engine")
|
|
223
|
+
env_db = env.get("db") if isinstance(env, dict) else None
|
|
224
|
+
written = ((f"[{table}] db", env_db), ("[db] engine", db.get("engine")))
|
|
225
|
+
named = [(key, name, WorldDatabase.engine_named(name)) for key, name in written if name is not None]
|
|
226
|
+
if len({engine for _, _, engine in named}) > 1:
|
|
227
|
+
fail("target", " and ".join(f"{key} = {name!r}" for key, name, _ in named) + " name different engines")
|
|
228
|
+
return named[0][2] if named else WorldDatabase.default_engine()
|
|
229
|
+
|
|
230
|
+
|
|
208
231
|
def require_legacy_bundle(bundle: str | Path, runtime: str) -> None:
|
|
209
232
|
"""Explicit rejection for alternate execution paths without store support."""
|
|
210
233
|
if bundle_mode(Path(bundle)) is not None:
|
|
@@ -220,7 +243,6 @@ def _key_fields(
|
|
|
220
243
|
location: str,
|
|
221
244
|
*,
|
|
222
245
|
required: bool = False,
|
|
223
|
-
nested: bool = True,
|
|
224
246
|
code: str = "key",
|
|
225
247
|
) -> list[str]:
|
|
226
248
|
if (
|
|
@@ -231,22 +253,85 @@ def _key_fields(
|
|
|
231
253
|
):
|
|
232
254
|
fail(code, "Keys require a nonempty list of distinct property paths", location)
|
|
233
255
|
for field in fields:
|
|
234
|
-
prop = field_schema(schema, field, f"{location}/{field}", code=code
|
|
256
|
+
prop = field_schema(schema, field, f"{location}/{field}", code=code)
|
|
235
257
|
types = prop.get("type", [])
|
|
236
258
|
types = [types] if isinstance(types, str) else types
|
|
237
259
|
if not set(types) or set(types) - {"string", "integer", "null"} or not set(types) & {"string", "integer"}:
|
|
238
260
|
fail(code, "Key paths must resolve to string/integer properties", location)
|
|
239
261
|
if any(k in prop for k in ("allOf", "oneOf", "anyOf")):
|
|
240
262
|
fail(code, "Composition is not supported on key fields", location)
|
|
241
|
-
if required and (
|
|
263
|
+
if required and (not required_path(schema, field, location) or "null" in types):
|
|
242
264
|
fail(
|
|
243
265
|
code,
|
|
244
|
-
"Primary-key fields must be required and non-null",
|
|
266
|
+
"Primary-key fields must be required and non-null, at every step of a nested path",
|
|
245
267
|
f"{location}/{field}",
|
|
246
268
|
)
|
|
247
269
|
return fields
|
|
248
270
|
|
|
249
271
|
|
|
272
|
+
#: ``(field, words, show both values)`` for every part of a :class:`StorageContract`, in
|
|
273
|
+
#: the order a refusal names them.
|
|
274
|
+
STORAGE_PARTS = (
|
|
275
|
+
("language", "the schema language", True),
|
|
276
|
+
("annotations", "the x-gateway version", True),
|
|
277
|
+
("compiler", "the compiler", True),
|
|
278
|
+
("target", "the storage target", True),
|
|
279
|
+
("domain_id", "the domain $id", True),
|
|
280
|
+
("domain_version", "x-gateway.domain_version", True),
|
|
281
|
+
("domain_hash", "the entities, fields, keys or relationships (their descriptions included)", False),
|
|
282
|
+
("sql", "the generated SQL (db/schema.sql)", False),
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@dataclass(frozen=True)
|
|
287
|
+
class StorageContract:
|
|
288
|
+
"""What a world's stored rows were validated against, and so what a data snapshot
|
|
289
|
+
stays valid for: the storage format, the domain (``$id``, ``domain_version`` and the
|
|
290
|
+
compiled entity schemas with their fields, keys, relationships and the annotations on
|
|
291
|
+
them) and the SQL it compiles to. The rest of the lock (tools, the package version,
|
|
292
|
+
the authored documents' hashes, the generated docs) never touches a stored row, so
|
|
293
|
+
changing it keeps every snapshot. Each part is the set of values that store rows
|
|
294
|
+
alike: only the domain hash has two, the entities compiled with their shared
|
|
295
|
+
definitions inlined or under ``$defs`` (the compiler switches with the contract's size)."""
|
|
296
|
+
|
|
297
|
+
language: frozenset
|
|
298
|
+
annotations: frozenset
|
|
299
|
+
compiler: frozenset
|
|
300
|
+
target: frozenset
|
|
301
|
+
domain_id: frozenset
|
|
302
|
+
domain_version: frozenset
|
|
303
|
+
domain_hash: frozenset
|
|
304
|
+
sql: frozenset
|
|
305
|
+
|
|
306
|
+
@classmethod
|
|
307
|
+
def of(cls, lock: Any, domain_hashes: frozenset = frozenset()) -> "StorageContract":
|
|
308
|
+
try:
|
|
309
|
+
domain = lock["domain"]
|
|
310
|
+
parts = (
|
|
311
|
+
lock["language"],
|
|
312
|
+
lock["annotations"],
|
|
313
|
+
lock["compiler"],
|
|
314
|
+
lock["target"],
|
|
315
|
+
domain["id"],
|
|
316
|
+
domain["version"],
|
|
317
|
+
domain["hash"],
|
|
318
|
+
lock["artifacts"]["db/schema.sql"],
|
|
319
|
+
)
|
|
320
|
+
except (KeyError, TypeError):
|
|
321
|
+
fail("state_contract", "The lock does not name the contract its rows are stored under")
|
|
322
|
+
stored = cls(*(frozenset({part}) for part in parts))
|
|
323
|
+
return replace(stored, domain_hash=stored.domain_hash | domain_hashes)
|
|
324
|
+
|
|
325
|
+
def changes_from(self, stored: "StorageContract") -> list[str]:
|
|
326
|
+
"""What differs from the contract ``stored`` rows were written under, in the words a
|
|
327
|
+
reader acts on; empty when every stored row carries over."""
|
|
328
|
+
return [
|
|
329
|
+
f"{words} ({min(getattr(stored, name))} → {min(getattr(self, name))})" if shown else words
|
|
330
|
+
for name, words, shown in STORAGE_PARTS
|
|
331
|
+
if getattr(self, name).isdisjoint(getattr(stored, name))
|
|
332
|
+
]
|
|
333
|
+
|
|
334
|
+
|
|
250
335
|
@dataclass(frozen=True)
|
|
251
336
|
class Contract:
|
|
252
337
|
metadata: dict
|
|
@@ -255,11 +340,25 @@ class Contract:
|
|
|
255
340
|
#: Structured, non-fatal findings (``recursive_reference``: a $defs cycle no
|
|
256
341
|
#: entity reaches); never part of the lock or the artifacts.
|
|
257
342
|
warnings: tuple[dict, ...] = ()
|
|
343
|
+
#: The domain hash of these entities in the resolver mode the build did not use
|
|
344
|
+
#: (:func:`build`); never part of the lock or the artifacts.
|
|
345
|
+
domain_hashes: frozenset = frozenset()
|
|
258
346
|
|
|
259
347
|
@property
|
|
260
348
|
def target(self) -> dict:
|
|
261
349
|
return self.metadata["domain"]
|
|
262
350
|
|
|
351
|
+
@property
|
|
352
|
+
def storage(self) -> StorageContract:
|
|
353
|
+
return StorageContract.of(self.lock, self.domain_hashes)
|
|
354
|
+
|
|
355
|
+
def targets(self, target: Any) -> bool:
|
|
356
|
+
"""Whether rows written against ``target`` are this contract's: the same domain ``$id``
|
|
357
|
+
and version, its entity hash in either resolver form."""
|
|
358
|
+
if not isinstance(target, dict):
|
|
359
|
+
return False
|
|
360
|
+
return {**target, "hash": self.target["hash"]} == self.target and target.get("hash") in self.storage.domain_hash
|
|
361
|
+
|
|
263
362
|
@property
|
|
264
363
|
def entities(self) -> dict:
|
|
265
364
|
return self.metadata["entities"]
|
|
@@ -268,16 +367,63 @@ class Contract:
|
|
|
268
367
|
def tools(self) -> dict:
|
|
269
368
|
return self.metadata["tools"]
|
|
270
369
|
|
|
370
|
+
@property
|
|
371
|
+
def publishable(self) -> bool:
|
|
372
|
+
return all(len(body) <= PUBLICATION_FILE_BYTES for body in self.artifacts.values())
|
|
373
|
+
|
|
271
374
|
|
|
272
375
|
def build(bundle: Path) -> Contract:
|
|
376
|
+
"""Inline every reference, as this compiler always has, so a world that publishes
|
|
377
|
+
compiles byte for byte as before (its lock and its snapshots' contract stay valid).
|
|
378
|
+
A contract whose inlined files could not be published shares its definitions. Either
|
|
379
|
+
way the contract also knows its entities' domain hash in the other mode, so a tool edit
|
|
380
|
+
that moves a world across that line keeps the rows stored under the first."""
|
|
273
381
|
bundle = Path(bundle)
|
|
382
|
+
try:
|
|
383
|
+
inlined = _build(bundle, Resolver(bundle, budget=PUBLICATION_FILE_BYTES))
|
|
384
|
+
except InlineBudgetExceeded:
|
|
385
|
+
inlined = None
|
|
386
|
+
if inlined is not None and inlined.publishable:
|
|
387
|
+
return replace(inlined, domain_hashes=_entity_hashes(bundle, inlined, Resolver(bundle, shared=True)))
|
|
388
|
+
shared = _build(bundle, Resolver(bundle, shared=True))
|
|
389
|
+
other = (
|
|
390
|
+
frozenset({inlined.target["hash"]})
|
|
391
|
+
if inlined is not None
|
|
392
|
+
else _entity_hashes(bundle, shared, Resolver(bundle, budget=PUBLICATION_FILE_BYTES))
|
|
393
|
+
)
|
|
394
|
+
return replace(shared, domain_hashes=other)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _domain_hash(entities: dict, target: str) -> str:
|
|
398
|
+
return digest(encoded({"language": LANGUAGE, "target": target, "entities": entities}))
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _entity_hashes(bundle: Path, contract: Contract, resolver: Resolver) -> frozenset:
|
|
402
|
+
"""The domain hash of ``contract``'s entities as ``resolver`` compiles them, resolved in
|
|
403
|
+
the order a build resolves them; empty when that representation cannot be built (the
|
|
404
|
+
inlining budget, a shared name), since it only ever widens what rows carry over."""
|
|
405
|
+
authored = resolver.load("world.json")["x-gateway"]["entities"]
|
|
406
|
+
try:
|
|
407
|
+
entities = {
|
|
408
|
+
name: {
|
|
409
|
+
**entity,
|
|
410
|
+
"schema": resolver.standalone(authored[name]["schema"], "world.json", f"{entity['location']}/schema"),
|
|
411
|
+
}
|
|
412
|
+
for name, entity in contract.entities.items()
|
|
413
|
+
}
|
|
414
|
+
except (InlineBudgetExceeded, ContractError):
|
|
415
|
+
return frozenset()
|
|
416
|
+
return frozenset({_domain_hash(entities, contract.lock["target"])})
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _build(bundle: Path, resolver: Resolver) -> Contract:
|
|
274
420
|
manifest = bundle_mode(bundle)
|
|
275
421
|
if manifest is None:
|
|
276
422
|
fail("language", "Not a schema-backed bundle; declare [schema] language")
|
|
423
|
+
engine = storage_engine(manifest)
|
|
277
424
|
from .cycles import ReferenceCycles
|
|
278
425
|
from .route_bindings import check_route_bindings
|
|
279
426
|
|
|
280
|
-
resolver = Resolver(bundle)
|
|
281
427
|
cycles = ReferenceCycles(resolver)
|
|
282
428
|
warnings: list[dict] = []
|
|
283
429
|
document = resolver.load("world.json")
|
|
@@ -319,7 +465,7 @@ def build(bundle: Path) -> Contract:
|
|
|
319
465
|
location,
|
|
320
466
|
)
|
|
321
467
|
if name.lower() in seen:
|
|
322
|
-
fail("identity", "Entity names collide under
|
|
468
|
+
fail("identity", "Entity names collide under SQL case folding", location)
|
|
323
469
|
seen.add(name.lower())
|
|
324
470
|
members(
|
|
325
471
|
entity,
|
|
@@ -327,7 +473,7 @@ def build(bundle: Path) -> Contract:
|
|
|
327
473
|
{"schema", "primary_key"},
|
|
328
474
|
location,
|
|
329
475
|
)
|
|
330
|
-
payload = resolver.
|
|
476
|
+
payload = resolver.standalone(entity["schema"], "world.json", f"{location}/schema")
|
|
331
477
|
warnings.extend(cycles.reached(name, entity["schema"], f"{location}/schema"))
|
|
332
478
|
if (
|
|
333
479
|
payload.get("type") != "object"
|
|
@@ -349,13 +495,7 @@ def build(bundle: Path) -> Contract:
|
|
|
349
495
|
location,
|
|
350
496
|
)
|
|
351
497
|
field_names.add(field.lower())
|
|
352
|
-
primary = _key_fields(
|
|
353
|
-
entity["primary_key"],
|
|
354
|
-
payload,
|
|
355
|
-
f"{location}/primary_key",
|
|
356
|
-
required=True,
|
|
357
|
-
nested=False,
|
|
358
|
-
)
|
|
498
|
+
primary = _key_fields(entity["primary_key"], payload, f"{location}/primary_key", required=True)
|
|
359
499
|
unique = entity.get("unique", [])
|
|
360
500
|
if not isinstance(unique, list):
|
|
361
501
|
fail("key", "unique must be a list of key lists", location)
|
|
@@ -411,14 +551,9 @@ def build(bundle: Path) -> Contract:
|
|
|
411
551
|
location,
|
|
412
552
|
)
|
|
413
553
|
for source, dest in zip(rel["fields"], target["primary_key"]):
|
|
414
|
-
source_type = field_schema(
|
|
415
|
-
entity["schema"],
|
|
416
|
-
source,
|
|
417
|
-
location,
|
|
418
|
-
code="relationship",
|
|
419
|
-
)["type"]
|
|
554
|
+
source_type = field_schema(entity["schema"], source, location, code="relationship")["type"]
|
|
420
555
|
source_types = set(source_type if isinstance(source_type, list) else [source_type]) - {"null"}
|
|
421
|
-
target_type = target["schema"]
|
|
556
|
+
target_type = field_schema(target["schema"], dest, location, code="relationship")["type"]
|
|
422
557
|
target_types = set(target_type if isinstance(target_type, list) else [target_type])
|
|
423
558
|
if source_types != target_types:
|
|
424
559
|
fail(
|
|
@@ -437,8 +572,8 @@ def build(bundle: Path) -> Contract:
|
|
|
437
572
|
members(tool, {"input", "output", "description"}, {"input", "output"}, location)
|
|
438
573
|
if "description" in tool and not isinstance(tool["description"], str):
|
|
439
574
|
fail("tool", "Tool description must be text", location)
|
|
440
|
-
inputs = resolver.
|
|
441
|
-
outputs = resolver.
|
|
575
|
+
inputs = resolver.standalone(tool["input"], "world.json", f"{location}/input")
|
|
576
|
+
outputs = resolver.standalone(tool["output"], "world.json", f"{location}/output")
|
|
442
577
|
if inputs.get("type") != "object":
|
|
443
578
|
fail("tool", "Tool inputs must resolve to an object schema", location)
|
|
444
579
|
compiled_tools[name] = {
|
|
@@ -467,22 +602,28 @@ def build(bundle: Path) -> Contract:
|
|
|
467
602
|
}
|
|
468
603
|
for name, doc in sorted(resolver.documents.items())
|
|
469
604
|
}
|
|
470
|
-
#
|
|
605
|
+
# The domain is what rows are checked against (StorageContract): adding tools, package
|
|
606
|
+
# behaviour or a mapping to it would invalidate every data snapshot on a tool edit.
|
|
471
607
|
domain = {
|
|
472
608
|
"id": document["$id"],
|
|
473
609
|
"version": annotation["domain_version"],
|
|
474
|
-
"hash":
|
|
610
|
+
"hash": _domain_hash(compiled, engine.target),
|
|
475
611
|
}
|
|
476
612
|
metadata = {
|
|
477
613
|
"language": LANGUAGE,
|
|
478
614
|
"compiler": COMPILER,
|
|
479
|
-
"target":
|
|
615
|
+
"target": engine.target,
|
|
480
616
|
"domain": domain,
|
|
481
617
|
"package": package,
|
|
482
618
|
"entities": compiled,
|
|
483
619
|
"tools": compiled_tools,
|
|
484
620
|
}
|
|
485
|
-
sql =
|
|
621
|
+
sql = engine.ddl(compiled)
|
|
622
|
+
shared = {}
|
|
623
|
+
for schema in [entity["schema"] for entity in compiled.values()] + [
|
|
624
|
+
tool[side] for tool in compiled_tools.values() for side in ("input_schema", "output_schema")
|
|
625
|
+
]:
|
|
626
|
+
shared.update(schema.get("$defs", {}))
|
|
486
627
|
docs = [
|
|
487
628
|
f"# {domain['id']} ({domain['version']})",
|
|
488
629
|
"",
|
|
@@ -510,7 +651,7 @@ def build(bundle: Path) -> Contract:
|
|
|
510
651
|
f"Relationships: {canonical(entity['relationships'])}",
|
|
511
652
|
"",
|
|
512
653
|
"```json",
|
|
513
|
-
json.dumps(entity["schema"], sort_keys=True, ensure_ascii=False, indent=2),
|
|
654
|
+
json.dumps(_without_defs(entity["schema"]), sort_keys=True, ensure_ascii=False, indent=2),
|
|
514
655
|
"```",
|
|
515
656
|
"",
|
|
516
657
|
]
|
|
@@ -520,10 +661,34 @@ def build(bundle: Path) -> Contract:
|
|
|
520
661
|
"## Tool contracts",
|
|
521
662
|
"",
|
|
522
663
|
"```json",
|
|
523
|
-
json.dumps(
|
|
664
|
+
json.dumps(
|
|
665
|
+
{
|
|
666
|
+
name: {
|
|
667
|
+
key: _without_defs(value) if key.endswith("_schema") else value for key, value in tool.items()
|
|
668
|
+
}
|
|
669
|
+
for name, tool in compiled_tools.items()
|
|
670
|
+
},
|
|
671
|
+
sort_keys=True,
|
|
672
|
+
ensure_ascii=False,
|
|
673
|
+
indent=2,
|
|
674
|
+
),
|
|
675
|
+
"```",
|
|
676
|
+
"",
|
|
677
|
+
]
|
|
678
|
+
)
|
|
679
|
+
docs.extend(
|
|
680
|
+
[
|
|
681
|
+
"## Definitions",
|
|
682
|
+
"",
|
|
683
|
+
"Each `#/$defs/<name>` reference above, documented once.",
|
|
684
|
+
"",
|
|
685
|
+
"```json",
|
|
686
|
+
json.dumps(shared, sort_keys=True, ensure_ascii=False, indent=2),
|
|
524
687
|
"```",
|
|
525
688
|
"",
|
|
526
689
|
]
|
|
690
|
+
if shared
|
|
691
|
+
else []
|
|
527
692
|
)
|
|
528
693
|
artifacts = {
|
|
529
694
|
"db/schema.sql": sql.encode(),
|
|
@@ -535,7 +700,7 @@ def build(bundle: Path) -> Contract:
|
|
|
535
700
|
"language": LANGUAGE,
|
|
536
701
|
"annotations": 1,
|
|
537
702
|
"compiler": COMPILER,
|
|
538
|
-
"target":
|
|
703
|
+
"target": engine.target,
|
|
539
704
|
"domain": domain,
|
|
540
705
|
"package": package,
|
|
541
706
|
"inputs": inputs,
|
|
@@ -550,9 +715,13 @@ def build(bundle: Path) -> Contract:
|
|
|
550
715
|
SCALARS = {"string", "integer", "number", "boolean"}
|
|
551
716
|
|
|
552
717
|
|
|
718
|
+
def _without_defs(schema: dict) -> dict:
|
|
719
|
+
return {key: value for key, value in schema.items() if key != "$defs"}
|
|
720
|
+
|
|
721
|
+
|
|
553
722
|
def scalar_types(prop: dict) -> set[str]:
|
|
554
723
|
"""The scalar types a property can hold, or empty when it may hold an object
|
|
555
|
-
or array (no column then:
|
|
724
|
+
or array (no column then: a projection would hand SQL a JSON text).
|
|
556
725
|
|
|
557
726
|
Compositions count too: a nullable enum reference, ``anyOf [{$ref Status},
|
|
558
727
|
{type: null}]``, is a string column. Brex's card ``status`` was declared
|
|
@@ -572,88 +741,76 @@ def scalar_types(prop: dict) -> set[str]:
|
|
|
572
741
|
return found if found and found <= SCALARS else set()
|
|
573
742
|
|
|
574
743
|
|
|
575
|
-
def _ddl(entities: dict) -> str:
|
|
576
|
-
lines = [
|
|
577
|
-
"-- Generated by gateway-world-compiler/1; do not edit.",
|
|
578
|
-
"PRAGMA foreign_keys = ON;",
|
|
579
|
-
]
|
|
580
|
-
for name, entity in entities.items():
|
|
581
|
-
columns = [
|
|
582
|
-
'"_key" TEXT PRIMARY KEY NOT NULL',
|
|
583
|
-
'"payload" TEXT NOT NULL CHECK(json_valid("payload"))',
|
|
584
|
-
]
|
|
585
|
-
for field, prop in sorted(entity["schema"].get("properties", {}).items()):
|
|
586
|
-
# JSON remains the source of truth. Projections make verifier SQL useful,
|
|
587
|
-
# and provide SQLite UNIQUE/FK enforcement without affinity coercion.
|
|
588
|
-
if scalar_types(prop):
|
|
589
|
-
columns.append(
|
|
590
|
-
f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", '
|
|
591
|
-
f"{quote_sql(_json_path(field))})) VIRTUAL"
|
|
592
|
-
)
|
|
593
|
-
projected = set(entity["schema"].get("properties", {}))
|
|
594
|
-
constrained = {field for fields in [entity["primary_key"], *entity["unique"]] for field in fields} | {
|
|
595
|
-
field for relationship in entity["relationships"] for field in relationship["fields"]
|
|
596
|
-
}
|
|
597
|
-
for field in sorted(constrained - projected):
|
|
598
|
-
columns.append(
|
|
599
|
-
f'{quote(field)} GENERATED ALWAYS AS (json_extract("payload", '
|
|
600
|
-
f"{quote_sql(_json_path(field))})) VIRTUAL"
|
|
601
|
-
)
|
|
602
|
-
for fields in [entity["primary_key"], *entity["unique"]]:
|
|
603
|
-
columns.append(f"UNIQUE ({', '.join(quote(f) for f in fields)})")
|
|
604
|
-
for rel in entity["relationships"]:
|
|
605
|
-
target = entities[rel["target"]]
|
|
606
|
-
columns.append(
|
|
607
|
-
f"FOREIGN KEY ({', '.join(quote(f) for f in rel['fields'])}) "
|
|
608
|
-
f"REFERENCES {quote(rel['target'])} "
|
|
609
|
-
f"({', '.join(quote(f) for f in target['primary_key'])}) "
|
|
610
|
-
"ON DELETE NO ACTION DEFERRABLE INITIALLY DEFERRED"
|
|
611
|
-
)
|
|
612
|
-
lines.append(f"CREATE TABLE {quote(name)} (\n " + ",\n ".join(columns) + "\n);")
|
|
613
|
-
return "\n".join(lines) + "\n"
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
def quote_sql(value: str) -> str:
|
|
617
|
-
return "'" + value.replace("'", "''") + "'"
|
|
618
|
-
|
|
619
|
-
|
|
620
744
|
def compile_bundle(bundle: str | Path, *, snapshots: bool = True) -> Contract:
|
|
621
745
|
"""Generate artifacts; never overwrite authored handlers or seed data.
|
|
622
746
|
|
|
623
747
|
With ``snapshots`` (the default) the data snapshots are built too, once the
|
|
624
|
-
tree has its ``data/initial.json``: ``db
|
|
625
|
-
each
|
|
748
|
+
tree has its ``data/initial.json``: ``db/<snapshot name>`` and one per scenario,
|
|
749
|
+
each kept when it already holds this batch in rows the contract may open. That
|
|
626
750
|
build is the proof the data applies through the contract. Data the contract
|
|
627
751
|
refuses (an author changed the model and has not re-ingested yet) leaves no
|
|
628
752
|
snapshot behind at all; the refusal is reported where the data is next used
|
|
629
|
-
(`gateway worlds schema compile` reports it at once, see ``compile_world``).
|
|
753
|
+
(`gateway worlds schema compile` reports it at once, see ``compile_world``).
|
|
754
|
+
|
|
755
|
+
A tree pinned to a hosted snapshot (``db/snapshot.json``) compiles only a contract
|
|
756
|
+
its rows carry over to: a domain change they cannot take, or a pin this tree cannot
|
|
757
|
+
prove, is refused before anything is written (:class:`CompilePlan`) and never
|
|
758
|
+
discarded as stale data; any other edit keeps the pin."""
|
|
630
759
|
bundle = Path(bundle)
|
|
631
|
-
contract =
|
|
632
|
-
bind_connection(bundle, contract)
|
|
633
|
-
for path, content in {**contract.artifacts, LOCK: encoded(contract.lock)}.items():
|
|
634
|
-
destination = bundle / path
|
|
635
|
-
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
636
|
-
destination.write_bytes(content)
|
|
637
|
-
# The generated ui follows the contract the way the lock does: a world
|
|
638
|
-
# whose connector.toml declares `[ui]` gets its index.html, app.js and
|
|
639
|
-
# contract.json rewritten from what was just compiled (:mod:`ui`).
|
|
640
|
-
from .ui import generate_ui
|
|
641
|
-
|
|
642
|
-
generate_ui(bundle, contract)
|
|
760
|
+
contract = CompilePlan.of(bundle).write()
|
|
643
761
|
if snapshots and (bundle / "data/initial.json").is_file():
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
try:
|
|
647
|
-
build_all(bundle, contract)
|
|
648
|
-
except ContractError:
|
|
649
|
-
discard_all(bundle)
|
|
762
|
+
_build_snapshots_or_discard(bundle, contract)
|
|
650
763
|
return contract
|
|
651
764
|
|
|
652
765
|
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
"
|
|
656
|
-
|
|
766
|
+
@dataclass(frozen=True)
|
|
767
|
+
class CompilePlan:
|
|
768
|
+
"""What a compile writes, decided before its first write: the contract, and the pin as
|
|
769
|
+
it must read afterwards (``None`` when it stays as it is). Every refusal is raised by
|
|
770
|
+
:meth:`of`, so a refused compile leaves the tree as it was."""
|
|
771
|
+
|
|
772
|
+
bundle: Path
|
|
773
|
+
contract: Contract
|
|
774
|
+
pin: Any
|
|
775
|
+
|
|
776
|
+
@classmethod
|
|
777
|
+
def of(cls, bundle: Path) -> "CompilePlan":
|
|
778
|
+
from .snapshot import SnapshotPin
|
|
779
|
+
|
|
780
|
+
contract = build(bundle)
|
|
781
|
+
bind_connection(bundle, contract)
|
|
782
|
+
pin = SnapshotPin.of(bundle)
|
|
783
|
+
return cls(bundle, contract, None if pin is None else pin.carried(contract))
|
|
784
|
+
|
|
785
|
+
def write(self) -> Contract:
|
|
786
|
+
if self.pin is not None:
|
|
787
|
+
self.pin.write()
|
|
788
|
+
for path, content in {**self.contract.artifacts, LOCK: encoded(self.contract.lock)}.items():
|
|
789
|
+
destination = self.bundle / path
|
|
790
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
791
|
+
destination.write_bytes(content)
|
|
792
|
+
from .ui import generate_ui
|
|
793
|
+
|
|
794
|
+
generate_ui(self.bundle, self.contract)
|
|
795
|
+
return self.contract
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _build_snapshots_or_discard(bundle: Path, contract: Contract) -> None:
|
|
799
|
+
"""The snapshots, best effort: data the contract refuses leaves none behind, while a
|
|
800
|
+
pin refusal is raised, never discarded as stale data."""
|
|
801
|
+
from .snapshot import PIN_REFUSALS, build_all, discard_all
|
|
802
|
+
|
|
803
|
+
try:
|
|
804
|
+
build_all(bundle, contract)
|
|
805
|
+
except ContractError as error:
|
|
806
|
+
if error.detail["code"] in PIN_REFUSALS:
|
|
807
|
+
raise
|
|
808
|
+
discard_all(bundle)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
JOURNAL_REMEDY = (
|
|
812
|
+
"re-import its rows against the new contract (`gateway worlds data import <dir> <rows.json>`), "
|
|
813
|
+
"or restore the schema it was written against"
|
|
657
814
|
)
|
|
658
815
|
|
|
659
816
|
|
|
@@ -667,51 +824,69 @@ def _journals_off_target(bundle: Path, contract: Contract) -> list[tuple[Path, s
|
|
|
667
824
|
location = str(path.relative_to(bundle))
|
|
668
825
|
if not isinstance(journal, dict) or not isinstance(journal.get("target"), dict):
|
|
669
826
|
fail("batch", "Journal must be a batch object with a target", location)
|
|
670
|
-
if journal["target"]
|
|
827
|
+
if not contract.targets(journal["target"]):
|
|
671
828
|
off.append((path, location, journal))
|
|
672
829
|
return off
|
|
673
830
|
|
|
674
831
|
|
|
832
|
+
def _carried_rows(bundle: Path, contract: Contract) -> list[tuple[Path, str, dict, Any]]:
|
|
833
|
+
"""Every journal off this contract's target, with the migration of its rows (None for
|
|
834
|
+
a journal that holds none, which follows any contract)."""
|
|
835
|
+
from .domain_migration import DomainMigration
|
|
836
|
+
|
|
837
|
+
recorded = DomainMigration.recorded(bundle)
|
|
838
|
+
|
|
839
|
+
def migration(journal: dict) -> Any:
|
|
840
|
+
if journal.get("mutations") == []:
|
|
841
|
+
return None
|
|
842
|
+
return DomainMigration.of_target(journal["target"], contract, recorded)
|
|
843
|
+
|
|
844
|
+
off = _journals_off_target(bundle, contract)
|
|
845
|
+
return [(path, location, journal, migration(journal)) for path, location, journal in off]
|
|
846
|
+
|
|
847
|
+
|
|
675
848
|
def stale_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
676
|
-
"""The journals ``compile`` will refuse: rows written against
|
|
677
|
-
|
|
678
|
-
compile is the step that cannot succeed.
|
|
849
|
+
"""The journals ``compile`` will refuse: rows written against a domain this contract
|
|
850
|
+
does not only add to. Read-only; what ``check`` names instead of \"run compile\" when
|
|
851
|
+
compile is the step that cannot succeed. Call it before compile rewrites
|
|
852
|
+
``schema/runtime.json``, which proves the domain the rows were written under."""
|
|
679
853
|
return [
|
|
680
854
|
location
|
|
681
|
-
for _path, location,
|
|
682
|
-
if
|
|
855
|
+
for _path, location, _journal, migration in _carried_rows(Path(bundle), contract)
|
|
856
|
+
if migration is not None and not migration.carries
|
|
683
857
|
]
|
|
684
858
|
|
|
685
859
|
|
|
686
|
-
def
|
|
687
|
-
"""Re-stamp every journal
|
|
688
|
-
target,
|
|
860
|
+
def retarget_journals(bundle: str | Path, contract: Contract) -> list[str]:
|
|
861
|
+
"""Re-stamp every journal written under another domain onto the compiled contract's
|
|
862
|
+
target, after refusing, before any rewrite, one whose rows would not carry over.
|
|
689
863
|
|
|
690
|
-
An
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
contract. A journal with rows is never migrated silently; the message names
|
|
695
|
-
the fix. Returns the bundle-relative paths rewritten."""
|
|
864
|
+
An empty journal has nothing to migrate, so it follows the contract; a journal with
|
|
865
|
+
rows follows it when the contract only adds to the domain they were written under
|
|
866
|
+
(:class:`~.domain_migration.DomainMigration`), and is refused, naming the fix,
|
|
867
|
+
otherwise. Returns the bundle-relative paths rewritten."""
|
|
696
868
|
bundle = Path(bundle)
|
|
697
|
-
|
|
698
|
-
for
|
|
699
|
-
if
|
|
700
|
-
|
|
869
|
+
carried = _carried_rows(bundle, contract)
|
|
870
|
+
for _path, location, _journal, migration in carried:
|
|
871
|
+
if migration is not None:
|
|
872
|
+
migration.require("batch_target", location, JOURNAL_REMEDY)
|
|
873
|
+
for path, _location, journal, _migration in carried:
|
|
701
874
|
journal["target"] = contract.target
|
|
702
875
|
path.write_bytes(encoded(journal))
|
|
703
|
-
|
|
704
|
-
return retargeted
|
|
876
|
+
return [location for _path, location, _journal, _migration in carried]
|
|
705
877
|
|
|
706
878
|
|
|
707
879
|
def compile_world(bundle: str | Path) -> tuple[Contract, dict]:
|
|
708
880
|
"""``compile_bundle`` plus the snapshots, refusing loudly when the data does not
|
|
709
|
-
apply through the contract: what the CLI runs, so an author sees the row.
|
|
710
|
-
|
|
881
|
+
apply through the contract: what the CLI runs, so an author sees the row. Journals
|
|
882
|
+
follow the contract when their rows carry over (``retarget_journals``) and refuse it,
|
|
883
|
+
before anything is written, otherwise."""
|
|
711
884
|
from .snapshot import build_all
|
|
712
885
|
|
|
713
|
-
|
|
714
|
-
|
|
886
|
+
bundle = Path(bundle)
|
|
887
|
+
plan = CompilePlan.of(bundle)
|
|
888
|
+
retarget_journals(bundle, plan.contract)
|
|
889
|
+
contract = plan.write()
|
|
715
890
|
return contract, build_all(bundle, contract)
|
|
716
891
|
|
|
717
892
|
|
|
@@ -729,8 +904,9 @@ def bind_connection(bundle: Path, contract: Contract) -> None:
|
|
|
729
904
|
|
|
730
905
|
def check_bundle(bundle: str | Path) -> Contract:
|
|
731
906
|
"""The contract, provided every generated artifact and every snapshot present in
|
|
732
|
-
the tree matches it; a snapshot's
|
|
733
|
-
file's
|
|
907
|
+
the tree matches it; a snapshot's rows must be stored under this contract's
|
|
908
|
+
:class:`StorageContract` and the file must be its sidecar's sha256. Absent snapshots
|
|
909
|
+
are not required: the legacy path serves."""
|
|
734
910
|
bundle = Path(bundle)
|
|
735
911
|
contract = build(bundle)
|
|
736
912
|
bind_connection(bundle, contract)
|
|
@@ -757,7 +933,7 @@ def _stale_artifact_remedy(bundle: Path, contract: Contract) -> str:
|
|
|
757
933
|
return (
|
|
758
934
|
"Missing, edited or incompatible contract artifact; the tree changed "
|
|
759
935
|
"since its last compile, so run `gateway worlds schema compile <dir>` "
|
|
760
|
-
"(
|
|
936
|
+
"(stored rows carry over only when the change adds to their domain)"
|
|
761
937
|
)
|
|
762
938
|
return (
|
|
763
939
|
"Missing, edited or incompatible contract artifact; the schema changed since "
|