@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
"""A route its declaration answers alone: ``[operations.api]`` action ``read``,
|
|
2
|
+
``create``, ``update`` or ``delete`` over the route's entity, no handler.
|
|
3
|
+
|
|
4
|
+
Its rows are the ones its filter selects: a declared ``filter``, else each path
|
|
5
|
+
segment naming a field of the entity, and for a read each query parameter naming
|
|
6
|
+
one. They are read through the store's selection (``WorldDatabase.select``), so a
|
|
7
|
+
keyed or related read decodes its page and never every row of the entity.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from typing import TYPE_CHECKING, Any
|
|
14
|
+
|
|
15
|
+
from .database.matching import RowMatch, dig
|
|
16
|
+
from .envelope import Page, Paging, page_headers, single, wrap
|
|
17
|
+
from .validation import ContractError
|
|
18
|
+
|
|
19
|
+
if TYPE_CHECKING:
|
|
20
|
+
from .api import ApiContext, OperationSettings
|
|
21
|
+
from .connector import Operation
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MisdeclaredRoute(ContractError):
|
|
25
|
+
"""A declared route that cannot answer the request it took: the world author's bug, answered
|
|
26
|
+
as one (the 400 a handler's bug gets), never as the contract refusing the caller's write."""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DeclaredRoute:
|
|
30
|
+
"""One request to a declared route; :meth:`answer` is its action's answer."""
|
|
31
|
+
|
|
32
|
+
def __init__(
|
|
33
|
+
self,
|
|
34
|
+
op: OperationSettings,
|
|
35
|
+
operation: Operation,
|
|
36
|
+
ctx: ApiContext,
|
|
37
|
+
args: dict[str, str],
|
|
38
|
+
query: dict[str, str],
|
|
39
|
+
body: Any,
|
|
40
|
+
) -> None:
|
|
41
|
+
self.op = op
|
|
42
|
+
self.operation = operation
|
|
43
|
+
self.ctx = ctx
|
|
44
|
+
self.args = args
|
|
45
|
+
self.query = query
|
|
46
|
+
self.body = body
|
|
47
|
+
self.entity = operation.entity or ""
|
|
48
|
+
|
|
49
|
+
def answer(self) -> Any:
|
|
50
|
+
if not self.operation.entity:
|
|
51
|
+
return 501, {"error": "not_modelled", "operation": self.operation.name}
|
|
52
|
+
return ACTIONS[self.op.action](self)
|
|
53
|
+
|
|
54
|
+
def _create(self) -> Any:
|
|
55
|
+
return self.op.status, self._one(self.ctx.insert(self.entity, self._bound(self.body)))
|
|
56
|
+
|
|
57
|
+
def _read(self) -> Any:
|
|
58
|
+
match = self._match()
|
|
59
|
+
if self.op.single:
|
|
60
|
+
rows, _ = self.ctx.select(self.entity, match, limit=1)
|
|
61
|
+
return (
|
|
62
|
+
(self.op.status, single(rows[0], self.op.envelope, self.op.wrapper)) if rows else self.ctx.not_found()
|
|
63
|
+
)
|
|
64
|
+
paging = self._paging()
|
|
65
|
+
offset, limit = paging.window(self.query)
|
|
66
|
+
rows, total = self.ctx.select(self.entity, match, limit=limit, offset=offset)
|
|
67
|
+
page = Page.of(rows, total, offset, limit, paging.style)
|
|
68
|
+
settings = self.ctx.settings
|
|
69
|
+
body = wrap(
|
|
70
|
+
page,
|
|
71
|
+
settings,
|
|
72
|
+
self.op.envelope,
|
|
73
|
+
paging=paging,
|
|
74
|
+
link=self.ctx.page_link,
|
|
75
|
+
omit_done_cursor=settings.omits_done_cursor(self.op),
|
|
76
|
+
wrapper=self.op.wrapper,
|
|
77
|
+
)
|
|
78
|
+
return self.op.status, body, page_headers(page, settings, paging, self.ctx.page_link)
|
|
79
|
+
|
|
80
|
+
def _update(self) -> Any:
|
|
81
|
+
row = self._only()
|
|
82
|
+
if row is None:
|
|
83
|
+
return self.ctx.not_found()
|
|
84
|
+
written = self.ctx.update(self.entity, self.ctx.key_of(self.entity, row), self._bound(self.body, current=row))
|
|
85
|
+
return self.op.status, self._one(written)
|
|
86
|
+
|
|
87
|
+
def _delete(self) -> Any:
|
|
88
|
+
row = self._only()
|
|
89
|
+
if row is None:
|
|
90
|
+
return self.ctx.not_found()
|
|
91
|
+
self.ctx.delete(self.entity, self.ctx.key_of(self.entity, row))
|
|
92
|
+
return self.op.status, None
|
|
93
|
+
|
|
94
|
+
def _only(self) -> dict | None:
|
|
95
|
+
"""The one row the route writes, or None when its filter selects none or several."""
|
|
96
|
+
rows, total = self.ctx.select(self.entity, self._match(), limit=1)
|
|
97
|
+
return rows[0] if total == 1 else None
|
|
98
|
+
|
|
99
|
+
def _match(self) -> RowMatch:
|
|
100
|
+
op, operation = self.op, self.operation
|
|
101
|
+
selection = op.filter or self._segment_filter()
|
|
102
|
+
if not selection and (op.action in ("update", "delete") or (op.single and operation.placeholders)):
|
|
103
|
+
raise MisdeclaredRoute(
|
|
104
|
+
"spec",
|
|
105
|
+
f"route {operation.name} finds its row by nothing: its path names no field of {self.entity} and it "
|
|
106
|
+
"declares no api.filter; map a path argument to a field with api.filter, or answer the route with "
|
|
107
|
+
'action = "handler"',
|
|
108
|
+
schema_path=f"/operations/{operation.name}/api/filter",
|
|
109
|
+
)
|
|
110
|
+
filters = selection
|
|
111
|
+
if not op.filter and op.action == "read":
|
|
112
|
+
filters = {**self._query_field_filter(), **filters}
|
|
113
|
+
# Every value sent filters: ``status[]=a&status[]=b`` reads as either, never as a with b dropped.
|
|
114
|
+
values: dict[str, str | list[str]] = {
|
|
115
|
+
**{k: v for k, v in self.ctx.query_all.items() if k in operation.params or k in filters},
|
|
116
|
+
**self.args,
|
|
117
|
+
}
|
|
118
|
+
return RowMatch(filters, values, op.match or {})
|
|
119
|
+
|
|
120
|
+
def _segment_filter(self) -> dict[str, str]:
|
|
121
|
+
"""The filter a route with no ``filter`` of its own implies: each ``{segment}`` of its
|
|
122
|
+
path that names a top-level field of its entity narrows to the rows carrying that value
|
|
123
|
+
(``GET /wallets/{id}``, ``GET /records/by-email/{email}``). A segment naming no field
|
|
124
|
+
(``/users/{userId}/labels``, a scope the caller must send) implies nothing; a declared
|
|
125
|
+
``filter`` says it all itself. Twin: ``route_bindings.implied_filters``."""
|
|
126
|
+
fields = self.ctx.fields_of(self.entity)
|
|
127
|
+
return {name: name for name in self.operation.placeholders if name in fields}
|
|
128
|
+
|
|
129
|
+
def _query_field_filter(self) -> dict[str, str]:
|
|
130
|
+
"""What a read route with no ``filter`` implies from its query string: each
|
|
131
|
+
parameter that names a top-level field of the entity narrows to rows carrying
|
|
132
|
+
that value (``GET /accounts?status=frozen``), the way every vendor's list
|
|
133
|
+
endpoint reads. Paging parameters and names the entity does not declare are
|
|
134
|
+
left to the pager and the handler."""
|
|
135
|
+
fields = self.ctx.fields_of(self.entity)
|
|
136
|
+
skipped = self.ctx.settings.paging.params
|
|
137
|
+
return {name: name for name in self.ctx.query_all if name in fields and name not in skipped}
|
|
138
|
+
|
|
139
|
+
def _paging(self) -> Paging:
|
|
140
|
+
default = self.ctx.settings.paging
|
|
141
|
+
if self.op.paging is None:
|
|
142
|
+
return default
|
|
143
|
+
return Paging(
|
|
144
|
+
self.op.paging, default.limit_param, default.offset_param, default.cursor_param, default.page_size
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
def _one(self, row: dict) -> Any:
|
|
148
|
+
"""A written row answered the route's way: alone (at its own envelope), or as a list of one."""
|
|
149
|
+
if self.op.single:
|
|
150
|
+
return single(row, self.op.envelope, self.op.wrapper)
|
|
151
|
+
return wrap([row], self.ctx.settings, self.op.envelope, wrapper=self.op.wrapper)
|
|
152
|
+
|
|
153
|
+
def _bound(self, body: Any, current: dict | None = None) -> dict:
|
|
154
|
+
"""The row a create or update writes: defaults, then the body (mapped or as-is), then generated fields."""
|
|
155
|
+
sent = body if isinstance(body, dict) else {}
|
|
156
|
+
row: dict = json.loads(json.dumps(self.op.defaults)) if current is None else {}
|
|
157
|
+
self._fill(row, sent, current)
|
|
158
|
+
if current is None:
|
|
159
|
+
self._generate(row)
|
|
160
|
+
return row
|
|
161
|
+
|
|
162
|
+
def _fill(self, row: dict, body: dict, current: dict | None) -> None:
|
|
163
|
+
if not self.op.body:
|
|
164
|
+
row.update(body)
|
|
165
|
+
return
|
|
166
|
+
for field_name, path in self.op.body.items():
|
|
167
|
+
value = dig(body, path)
|
|
168
|
+
if value is not None:
|
|
169
|
+
_assign(row, field_name, value, current)
|
|
170
|
+
|
|
171
|
+
def _generate(self, row: dict) -> None:
|
|
172
|
+
for field_name, kind in self.op.generate.items():
|
|
173
|
+
if row.get(field_name) is None:
|
|
174
|
+
row[field_name] = GENERATED[kind](self, field_name)
|
|
175
|
+
|
|
176
|
+
def _next_int(self, field_name: str) -> int:
|
|
177
|
+
existing = [r.get(field_name) for r in self.ctx.rows(self.entity)]
|
|
178
|
+
return max([v for v in existing if isinstance(v, int)] + [0]) + 1
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
ACTIONS = {
|
|
182
|
+
"read": DeclaredRoute._read,
|
|
183
|
+
"create": DeclaredRoute._create,
|
|
184
|
+
"update": DeclaredRoute._update,
|
|
185
|
+
"delete": DeclaredRoute._delete,
|
|
186
|
+
}
|
|
187
|
+
GENERATED = {
|
|
188
|
+
"uuid": lambda route, _field: route.ctx.uuid(),
|
|
189
|
+
"now": lambda route, _field: route.ctx.now(),
|
|
190
|
+
"int": DeclaredRoute._next_int,
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _assign(row: dict, target: str, value: Any, current: dict | None) -> None:
|
|
195
|
+
"""Assign a dotted target (``attributes.note``), keeping the rest of a nested
|
|
196
|
+
object an update leaves alone."""
|
|
197
|
+
parts = target.split(".")
|
|
198
|
+
if len(parts) == 1:
|
|
199
|
+
row[target] = value
|
|
200
|
+
return
|
|
201
|
+
head = parts[0]
|
|
202
|
+
if head not in row:
|
|
203
|
+
base = current.get(head) if current else None
|
|
204
|
+
row[head] = json.loads(json.dumps(base)) if isinstance(base, dict) else {}
|
|
205
|
+
node = row[head]
|
|
206
|
+
for part in parts[1:-1]:
|
|
207
|
+
if not isinstance(node.get(part), dict):
|
|
208
|
+
node[part] = {}
|
|
209
|
+
node = node[part]
|
|
210
|
+
node[parts[-1]] = value
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Rows stored under one domain, carried to a contract that only adds to it.
|
|
2
|
+
|
|
3
|
+
Stored rows (a data snapshot, a journal) were validated against the domain they were written
|
|
4
|
+
under. A contract that only adds to that domain — a new entity, a new field no row has to carry,
|
|
5
|
+
another ``domain_version`` — takes every one of them as it is, so they carry over: a journal is
|
|
6
|
+
re-targeted, a store is copied row for row (its batch ledger and lineage with it) into a store of
|
|
7
|
+
the new contract. Any other change to the stored domain is refused, naming each change.
|
|
8
|
+
|
|
9
|
+
The stored domain is proven by its compiled entities, which must hash to the domain the rows name:
|
|
10
|
+
a tree records them in ``schema/runtime.json`` until its next compile, and ``db/snapshot.json``
|
|
11
|
+
keeps them, beside the ``lock``, once the contract has moved off it.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from dataclasses import dataclass
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any, Iterable
|
|
20
|
+
|
|
21
|
+
from .compiler import STORAGE_PARTS, Contract, StorageContract, _domain_hash
|
|
22
|
+
from .database import WorldDatabase
|
|
23
|
+
from .database.guard import RUNTIME_TABLES
|
|
24
|
+
from .validation import canonical, fail
|
|
25
|
+
|
|
26
|
+
RUNTIME = "schema/runtime.json"
|
|
27
|
+
PIN = "db/snapshot.json"
|
|
28
|
+
FORMAT_PARTS = frozenset({"language", "annotations", "compiler", "target"})
|
|
29
|
+
ENTITY_KEYS = (
|
|
30
|
+
("primary_key", "the primary key"),
|
|
31
|
+
("unique", "the unique keys"),
|
|
32
|
+
("relationships", "the relationships"),
|
|
33
|
+
)
|
|
34
|
+
SCHEMA_PARTS = frozenset({"properties", "required", "$defs"})
|
|
35
|
+
LEDGER_TABLES = ("_gateway_batches", "_gateway_lineage", *sorted(RUNTIME_TABLES))
|
|
36
|
+
ADDITIVE = "a new entity, a new field that is not required, or another domain_version"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _proves(target: dict, entities: Any, storage: str) -> bool:
|
|
40
|
+
return isinstance(entities, dict) and _domain_hash(entities, storage) == target.get("hash")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _read_entities(path: Path, key: str) -> Any:
|
|
44
|
+
try:
|
|
45
|
+
document = json.loads(path.read_text(encoding="utf-8"))
|
|
46
|
+
except (OSError, ValueError):
|
|
47
|
+
return None
|
|
48
|
+
return document.get(key) if isinstance(document, dict) else None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(frozen=True)
|
|
52
|
+
class DomainMigration:
|
|
53
|
+
"""The one rule for whether rows stored under ``target`` (a domain's ``id``, ``version`` and
|
|
54
|
+
entity ``hash``) carry over to ``contract``: only when the change adds to that domain. The
|
|
55
|
+
stored ``entities`` are None when nothing proves them; ``lock`` is the stored lock when the
|
|
56
|
+
rows live in a store, which is then the only kind that can be migrated."""
|
|
57
|
+
|
|
58
|
+
target: dict
|
|
59
|
+
entities: dict | None
|
|
60
|
+
contract: Contract
|
|
61
|
+
lock: dict | None = None
|
|
62
|
+
format: tuple[str, ...] = ()
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def recorded(world: str | Path) -> tuple[Any, ...]:
|
|
66
|
+
"""Where ``world`` may still record the entities its stored rows were written under."""
|
|
67
|
+
world = Path(world)
|
|
68
|
+
return (_read_entities(world / PIN, "entities"), _read_entities(world / RUNTIME, "entities"))
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def of_target(
|
|
72
|
+
cls, target: dict, contract: Contract, candidates: Iterable[Any], storage: str | None = None
|
|
73
|
+
) -> "DomainMigration":
|
|
74
|
+
storage = storage or contract.lock["target"]
|
|
75
|
+
proven = next((entities for entities in candidates if _proves(target, entities, storage)), None)
|
|
76
|
+
return cls(target, proven, contract)
|
|
77
|
+
|
|
78
|
+
@classmethod
|
|
79
|
+
def of_lock(cls, lock: dict, contract: Contract, candidates: Iterable[Any]) -> "DomainMigration":
|
|
80
|
+
stored, current = StorageContract.of(lock), contract.storage
|
|
81
|
+
changed = tuple(
|
|
82
|
+
f"{words} ({min(getattr(stored, name))} → {min(getattr(current, name))})"
|
|
83
|
+
for name, words, _shown in STORAGE_PARTS
|
|
84
|
+
if name in FORMAT_PARTS and getattr(current, name).isdisjoint(getattr(stored, name))
|
|
85
|
+
)
|
|
86
|
+
migration = cls.of_target(lock["domain"], contract, candidates, lock.get("target"))
|
|
87
|
+
return cls(migration.target, migration.entities, contract, lock, changed)
|
|
88
|
+
|
|
89
|
+
@classmethod
|
|
90
|
+
def carry(cls, store: Path, contract: Contract, world: str | Path) -> dict[str, int] | None:
|
|
91
|
+
"""The store at ``store`` carried in place to ``contract`` when its rows were stored under
|
|
92
|
+
a domain the contract only adds to (its counts); None when it already opens under it."""
|
|
93
|
+
lock = WorldDatabase.for_contract(contract).stored_lock(store)
|
|
94
|
+
migration = cls.of_lock(lock, contract, cls.recorded(world))
|
|
95
|
+
if not migration.needed:
|
|
96
|
+
return None
|
|
97
|
+
migration.require(
|
|
98
|
+
"state_contract",
|
|
99
|
+
f"/db/{store.name}",
|
|
100
|
+
"Import the rows into the world again, or restore the domain they were stored under",
|
|
101
|
+
)
|
|
102
|
+
fresh = store.with_name(f".{store.name}.migrating")
|
|
103
|
+
fresh.unlink(missing_ok=True)
|
|
104
|
+
counts = migration.migrate(store, fresh)
|
|
105
|
+
fresh.replace(store)
|
|
106
|
+
return counts
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def needed(self) -> bool:
|
|
110
|
+
return bool(self.format) or not self.contract.targets(self.target)
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def carries(self) -> bool:
|
|
114
|
+
return self.entities is not None and not self.refusals
|
|
115
|
+
|
|
116
|
+
@property
|
|
117
|
+
def additions(self) -> list[str]:
|
|
118
|
+
if self.entities is None:
|
|
119
|
+
return []
|
|
120
|
+
current = self.contract.entities
|
|
121
|
+
found = [f"entity {name}" for name in sorted(set(current) - set(self.entities))]
|
|
122
|
+
for name, stored in sorted(self.entities.items()):
|
|
123
|
+
now = current.get(name)
|
|
124
|
+
if now is None:
|
|
125
|
+
continue
|
|
126
|
+
held = stored["schema"].get("properties", {})
|
|
127
|
+
found.extend(f"field {name}.{f}" for f in sorted(set(now["schema"].get("properties", {})) - set(held)))
|
|
128
|
+
version = self.contract.target["version"]
|
|
129
|
+
if self.target.get("version") != version:
|
|
130
|
+
found.append(f"domain_version {self.target.get('version')} → {version}")
|
|
131
|
+
return found
|
|
132
|
+
|
|
133
|
+
@property
|
|
134
|
+
def refusals(self) -> list[str]:
|
|
135
|
+
found = [f"changes {words}" for words in self.format]
|
|
136
|
+
if self.target.get("id") != self.contract.target["id"]:
|
|
137
|
+
found.append(f"changes the domain $id ({self.target.get('id')} → {self.contract.target['id']})")
|
|
138
|
+
if self.entities is None:
|
|
139
|
+
return found
|
|
140
|
+
current = self.contract.entities
|
|
141
|
+
for name, stored in sorted(self.entities.items()):
|
|
142
|
+
found.extend(_entity_refusals(name, stored, current.get(name)))
|
|
143
|
+
return found
|
|
144
|
+
|
|
145
|
+
def require(self, code: str, pointer: str, remedy: str) -> None:
|
|
146
|
+
"""Refuse unless the stored rows carry over, saying what to do about it."""
|
|
147
|
+
if not self.needed:
|
|
148
|
+
return
|
|
149
|
+
found = self.refusals
|
|
150
|
+
if found:
|
|
151
|
+
fail(
|
|
152
|
+
code,
|
|
153
|
+
f"The rows here were stored under the previous domain, and this change {'; '.join(found)}. "
|
|
154
|
+
f"Stored rows carry over only when a change adds to their domain ({ADDITIVE}): "
|
|
155
|
+
f"keep the rest as it was, or: {remedy}",
|
|
156
|
+
pointer,
|
|
157
|
+
changed=found,
|
|
158
|
+
)
|
|
159
|
+
if self.entities is None:
|
|
160
|
+
fail(
|
|
161
|
+
code,
|
|
162
|
+
f"This change moves the domain the rows here were stored under, and nothing in the tree "
|
|
163
|
+
f"records that domain's entities any more, so the runtime cannot tell whether the change only "
|
|
164
|
+
f"adds to it ({ADDITIVE}). Compile the change on the tree the stored rows came from (a fresh "
|
|
165
|
+
f"`gateway worlds sandbox open <slug>`, or `gateway bench pull <slug>`), or: {remedy}",
|
|
166
|
+
pointer,
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
def view(self) -> dict:
|
|
170
|
+
return {"from": self.lock_hash, "adds": self.additions}
|
|
171
|
+
|
|
172
|
+
@property
|
|
173
|
+
def lock_hash(self) -> str:
|
|
174
|
+
from .snapshot import lock_hash
|
|
175
|
+
|
|
176
|
+
return lock_hash(self.lock) if self.lock is not None else self.target.get("hash", "")
|
|
177
|
+
|
|
178
|
+
def migrate(self, source: Path, destination: Path) -> dict[str, int]:
|
|
179
|
+
"""``source`` (a store of the stored lock) copied into a new store at ``destination`` under
|
|
180
|
+
the contract: every row, the batch ledger and the lineage, byte for byte."""
|
|
181
|
+
if self.lock is None or not self.carries:
|
|
182
|
+
fail("state_contract", "Only a store whose stored domain is proven and only added to can be migrated")
|
|
183
|
+
engine = WorldDatabase.for_contract(self.contract)
|
|
184
|
+
with WorldDatabase.open(source, self._stored_contract(), audit=False) as held:
|
|
185
|
+
counts = held.counts()
|
|
186
|
+
try:
|
|
187
|
+
WorldDatabase.create(destination, self.contract).close()
|
|
188
|
+
engine.copy_rows(source, destination, LEDGER_TABLES, sorted(self.entities))
|
|
189
|
+
with WorldDatabase.open(destination, self.contract, audit=False) as moved:
|
|
190
|
+
carried = moved.counts()
|
|
191
|
+
except BaseException:
|
|
192
|
+
destination.unlink(missing_ok=True)
|
|
193
|
+
raise
|
|
194
|
+
if any(carried.get(name) != count for name, count in counts.items()):
|
|
195
|
+
destination.unlink(missing_ok=True)
|
|
196
|
+
fail("state_contract", "The migrated store does not hold every stored row")
|
|
197
|
+
return carried
|
|
198
|
+
|
|
199
|
+
def _stored_contract(self) -> Contract:
|
|
200
|
+
target = self.lock["target"]
|
|
201
|
+
metadata = {"target": target, "domain": self.lock["domain"], "entities": self.entities, "tools": {}}
|
|
202
|
+
ddl = WorldDatabase.for_contract(Contract(metadata, {}, self.lock)).ddl(self.entities)
|
|
203
|
+
return Contract(metadata, {"db/schema.sql": ddl.encode()}, self.lock)
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def _entity_refusals(name: str, stored: dict, current: dict | None) -> list[str]:
|
|
207
|
+
if current is None:
|
|
208
|
+
return [f"removes entity {name}"]
|
|
209
|
+
found = [
|
|
210
|
+
f"changes {words} of {name}" for key, words in ENTITY_KEYS if canonical(stored[key]) != canonical(current[key])
|
|
211
|
+
]
|
|
212
|
+
return found + _schema_refusals(name, stored["schema"], current["schema"])
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _schema_refusals(name: str, stored: dict, current: dict) -> list[str]:
|
|
216
|
+
held, now = stored.get("properties", {}), current.get("properties", {})
|
|
217
|
+
required, requiring = set(stored.get("required", [])), set(current.get("required", []))
|
|
218
|
+
found = [f"removes field {name}.{f}" for f in sorted(set(held) - set(now))]
|
|
219
|
+
found += [
|
|
220
|
+
f"changes field {name}.{f}" for f in sorted(set(held) & set(now)) if canonical(held[f]) != canonical(now[f])
|
|
221
|
+
]
|
|
222
|
+
found += [f"requires field {name}.{f}" for f in sorted(requiring - required)]
|
|
223
|
+
found += [f"stops requiring field {name}.{f}" for f in sorted((required - requiring) & set(now))]
|
|
224
|
+
defs, defined = stored.get("$defs", {}), current.get("$defs", {})
|
|
225
|
+
found += [
|
|
226
|
+
f"changes the definition {d} {name} stores"
|
|
227
|
+
for d in sorted(defs)
|
|
228
|
+
if canonical(defs[d]) != canonical(defined.get(d))
|
|
229
|
+
]
|
|
230
|
+
rest = {key: value for key, value in stored.items() if key not in SCHEMA_PARTS}
|
|
231
|
+
if canonical(rest) != canonical({key: value for key, value in current.items() if key not in SCHEMA_PARTS}):
|
|
232
|
+
found.append(f"changes the schema of {name}")
|
|
233
|
+
return found
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
__all__ = ["DomainMigration"]
|