@withgateway/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2880 -2032
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2982 -2135
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +180 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +172 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1309 -540
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1287 -532
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +177 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +169 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +69 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +69 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +249 -120
- package/dist/worlds/index.d.ts +249 -120
- package/dist/worlds/index.js +1240 -484
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1231 -476
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -514
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +19 -11
- package/skills/connector-schema-authoring/connector-worlds.md +13 -4
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +14 -14
- package/skills/worlds-getting-started/SKILL.md +12 -12
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -44,13 +44,14 @@ from .validation import fail
|
|
|
44
44
|
PIN = "conform"
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
# ---------------------------------------------------------------- the document
|
|
48
|
-
|
|
49
|
-
|
|
50
47
|
def load_openapi(path: str | Path) -> dict:
|
|
51
48
|
return parse_openapi(Path(path).read_text(encoding="utf-8"), str(path))
|
|
52
49
|
|
|
53
50
|
|
|
51
|
+
#: Word for word the TypeScript CLI's ``YAML_NEEDS_PYYAML`` (client.ts), which refuses before sending.
|
|
52
|
+
YAML_NEEDS_PYYAML = "Reading a YAML document needs PyYAML in the Python that runs the schema runtime: pip install pyyaml there, set GATEWAY_PYTHON to an interpreter that has it, or convert the document to JSON"
|
|
53
|
+
|
|
54
|
+
|
|
54
55
|
def parse_openapi(text: str, name: str, *, location: str = "/conform/openapi") -> dict:
|
|
55
56
|
"""The one OpenAPI reader: YAML when the name says so or the text is not JSON,
|
|
56
57
|
JSON otherwise. ``conform`` and ``schema init --from-openapi`` both read through
|
|
@@ -72,8 +73,8 @@ def _parse_json(text: str, name: str, location: str) -> Any:
|
|
|
72
73
|
def _parse_yaml(text: str, name: str, location: str) -> Any:
|
|
73
74
|
try:
|
|
74
75
|
import yaml # optional at runtime; the SDK depends on it, the vendored server does not
|
|
75
|
-
except ImportError:
|
|
76
|
-
fail("conform",
|
|
76
|
+
except ImportError:
|
|
77
|
+
fail("conform", YAML_NEEDS_PYYAML, location)
|
|
77
78
|
try:
|
|
78
79
|
return yaml.safe_load(text)
|
|
79
80
|
except yaml.YAMLError as error:
|
|
@@ -118,9 +119,6 @@ def find_operation(document: dict, prefixes: list[str], operation: Operation) ->
|
|
|
118
119
|
return None
|
|
119
120
|
|
|
120
121
|
|
|
121
|
-
# ---------------------------------------------------------------- lenient validation
|
|
122
|
-
|
|
123
|
-
|
|
124
122
|
class Checker:
|
|
125
123
|
def __init__(self, document: dict):
|
|
126
124
|
self.document = document
|
|
@@ -300,9 +298,6 @@ class Checker:
|
|
|
300
298
|
return None
|
|
301
299
|
|
|
302
300
|
|
|
303
|
-
# ---------------------------------------------------------------- the run
|
|
304
|
-
|
|
305
|
-
|
|
306
301
|
@dataclass
|
|
307
302
|
class Outcome:
|
|
308
303
|
operation: str
|
|
@@ -415,9 +410,10 @@ def _hints(spec, settings) -> dict[str, tuple[str, str]]:
|
|
|
415
410
|
|
|
416
411
|
|
|
417
412
|
def _first_value(rows: list[dict], path: str) -> Any:
|
|
413
|
+
parts = path[1:].split("/") if path.startswith("/") else path.split(".")
|
|
418
414
|
for row in rows:
|
|
419
415
|
found: Any = row
|
|
420
|
-
for part in
|
|
416
|
+
for part in parts:
|
|
421
417
|
found = found.get(part) if isinstance(found, dict) else None
|
|
422
418
|
if found is not None and not isinstance(found, (dict, list)):
|
|
423
419
|
return found
|
|
@@ -75,9 +75,10 @@ from pathlib import Path
|
|
|
75
75
|
from typing import Any, Callable, Iterable
|
|
76
76
|
|
|
77
77
|
from .compiler import check_bundle, compile_bundle
|
|
78
|
-
from .
|
|
78
|
+
from .database import WorldDatabase
|
|
79
|
+
from .primary_key import PrimaryKey
|
|
79
80
|
from .transforms import apply_transforms
|
|
80
|
-
from .validation import ContractError, fail, operation_name, parse_toml
|
|
81
|
+
from .validation import FIELD_NAME, ContractError, fail, operation_name, parse_toml
|
|
81
82
|
|
|
82
83
|
SPEC = "connector.toml"
|
|
83
84
|
CAPTURES = "captures"
|
|
@@ -86,16 +87,13 @@ SCHEMES = ("header", "bearer", "basic", "query", "oauth2", "none")
|
|
|
86
87
|
DEFAULT_TOKEN_PATH = "/oauth/token"
|
|
87
88
|
METHODS = ("GET", "POST", "PUT", "PATCH", "DELETE")
|
|
88
89
|
SLUG = re.compile(r"^[a-z][a-z0-9-]*$")
|
|
89
|
-
#:
|
|
90
|
-
#:
|
|
91
|
-
|
|
92
|
-
#:
|
|
93
|
-
|
|
94
|
-
#: the name cannot capture, match or filter on it.
|
|
95
|
-
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*(\[\])?$")
|
|
90
|
+
#: A declared query parameter as the vendor names it: dotted (``query.cond``,
|
|
91
|
+
#: ``filter.overallStatus``), and ending in ``[]`` where the vendor repeats the key
|
|
92
|
+
#: (``expand[]=a&expand[]=b``). A world that cannot declare the name cannot capture,
|
|
93
|
+
#: match or filter on it.
|
|
94
|
+
PARAM = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*(\[\])?$")
|
|
96
95
|
PLACEHOLDER = re.compile(r"\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
97
|
-
|
|
98
|
-
#: key (``expand[]=a&expand[]=b``) survives; a dict is accepted wherever pairs are.
|
|
96
|
+
# Pairs, not a dict, so a repeated key survives.
|
|
99
97
|
Params = list[tuple[str, str]]
|
|
100
98
|
MAX_BODY = 8 * 1024 * 1024
|
|
101
99
|
RECORDED_HEADERS = ("content-type", "retry-after", "x-ratelimit-limit", "x-ratelimit-remaining", "x-ratelimit-reset")
|
|
@@ -128,13 +126,9 @@ class Operation:
|
|
|
128
126
|
results: str
|
|
129
127
|
drop: tuple[str, ...] = ()
|
|
130
128
|
explode: tuple[Explode, ...] = ()
|
|
131
|
-
|
|
132
|
-
#: statement only knows its account from the ``{id}`` it was fetched under (Brex
|
|
133
|
-
#: retrospective); the captured row never carries it, so ingest stamps it on.
|
|
129
|
+
# A captured row may lack the path argument it was fetched under, so ingest stamps it on.
|
|
134
130
|
carry_args: dict[str, str] = field(default_factory=dict)
|
|
135
|
-
|
|
136
|
-
#: is one of them and the one a capture sends. Slack's Web API takes a read as
|
|
137
|
-
#: GET or POST alike, and its clients post everything.
|
|
131
|
+
# Slack's Web API takes a read as GET or POST alike, and its clients post everything.
|
|
138
132
|
methods: tuple[str, ...] = ()
|
|
139
133
|
|
|
140
134
|
def __post_init__(self) -> None:
|
|
@@ -153,9 +147,7 @@ class Spec:
|
|
|
153
147
|
scheme: str
|
|
154
148
|
auth_name: str | None
|
|
155
149
|
secret_env: str | None
|
|
156
|
-
extra_headers: dict[str, str]
|
|
157
|
-
#: oauth2 only: where the client-credentials grant is exchanged (absolute, or a
|
|
158
|
-
#: path under base_url), the env var naming the client id, and the scope asked for.
|
|
150
|
+
extra_headers: dict[str, str]
|
|
159
151
|
token_url: str | None
|
|
160
152
|
client_id_env: str | None
|
|
161
153
|
scope: str | None
|
|
@@ -167,8 +159,8 @@ class Spec:
|
|
|
167
159
|
cursor_header: str | None
|
|
168
160
|
hash_fields: frozenset[str]
|
|
169
161
|
drop_fields: frozenset[str]
|
|
170
|
-
round_fields: dict[str, int] = field(default_factory=dict)
|
|
171
|
-
preserve_fields: frozenset[str] = frozenset()
|
|
162
|
+
round_fields: dict[str, int] = field(default_factory=dict)
|
|
163
|
+
preserve_fields: frozenset[str] = frozenset()
|
|
172
164
|
operations: dict[str, Operation] = field(default_factory=dict)
|
|
173
165
|
|
|
174
166
|
def operation(self, name: str) -> Operation:
|
|
@@ -200,9 +192,6 @@ class Spec:
|
|
|
200
192
|
)
|
|
201
193
|
|
|
202
194
|
|
|
203
|
-
# ---------------------------------------------------------------- spec
|
|
204
|
-
|
|
205
|
-
|
|
206
195
|
def _table(value: Any, location: str) -> dict:
|
|
207
196
|
if not isinstance(value, dict):
|
|
208
197
|
fail("spec", "Expected a table", location)
|
|
@@ -232,7 +221,7 @@ def parse_redact(value: Any, location: str) -> RedactTable:
|
|
|
232
221
|
drop_fields = frozenset(_paths(redact, "drop", location))
|
|
233
222
|
rounding = redact.get("round", {})
|
|
234
223
|
if not isinstance(rounding, dict) or not all(
|
|
235
|
-
isinstance(k, str) and
|
|
224
|
+
isinstance(k, str) and FIELD_NAME.match(k) and isinstance(v, int) and not isinstance(v, bool) and 0 <= v <= 10
|
|
236
225
|
for k, v in rounding.items()
|
|
237
226
|
):
|
|
238
227
|
fail("spec", "redact.round maps numeric field names to the decimals kept (0-10)", f"{location}/round")
|
|
@@ -249,7 +238,7 @@ def _paths(table: dict, key: str, location: str) -> tuple[str, ...]:
|
|
|
249
238
|
catalog's ``assets`` count map keeps its ``email`` key while records lose theirs."""
|
|
250
239
|
value = table.get(key, [])
|
|
251
240
|
if not isinstance(value, list) or not all(
|
|
252
|
-
isinstance(v, str) and all(part == "*" or
|
|
241
|
+
isinstance(v, str) and all(part == "*" or FIELD_NAME.match(part) for part in v.split(".")) and v for v in value
|
|
253
242
|
):
|
|
254
243
|
fail("spec", f"{key} must be a list of field names or dotted paths", f"{location}/{key}")
|
|
255
244
|
return tuple(value)
|
|
@@ -271,7 +260,7 @@ def _verbs(table: dict, location: str) -> tuple[str, ...]:
|
|
|
271
260
|
return verbs
|
|
272
261
|
|
|
273
262
|
|
|
274
|
-
def _names(table: dict, key: str, location: str, pattern: re.Pattern =
|
|
263
|
+
def _names(table: dict, key: str, location: str, pattern: re.Pattern = FIELD_NAME) -> tuple[str, ...]:
|
|
275
264
|
value = table.get(key, [])
|
|
276
265
|
if not isinstance(value, list) or not all(isinstance(v, str) and pattern.match(v) for v in value):
|
|
277
266
|
fail("spec", f"{key} must be a list of field names", f"{location}/{key}")
|
|
@@ -509,9 +498,6 @@ def load_spec(world: str | Path) -> Spec:
|
|
|
509
498
|
return bind_spec(spec, read_json(world_json))
|
|
510
499
|
|
|
511
500
|
|
|
512
|
-
# ---------------------------------------------------------------- requests
|
|
513
|
-
|
|
514
|
-
|
|
515
501
|
def _secret_value(spec: Spec, secret: str | None) -> str:
|
|
516
502
|
if secret is not None:
|
|
517
503
|
return secret
|
|
@@ -633,9 +619,6 @@ def _open(request: urllib.request.Request, timeout: float) -> Response:
|
|
|
633
619
|
fail("transport", f"Could not reach the connection: {error}", "/capture")
|
|
634
620
|
|
|
635
621
|
|
|
636
|
-
# ---------------------------------------------------------------- redaction
|
|
637
|
-
|
|
638
|
-
|
|
639
622
|
def _named(key: str, here: str, names: frozenset[str]) -> bool:
|
|
640
623
|
return key in names or here in names
|
|
641
624
|
|
|
@@ -759,7 +742,6 @@ def redact(value: Any, spec: Spec, path: str = "") -> Any:
|
|
|
759
742
|
elif _named(key, here, spec.preserve_fields) and item is not None:
|
|
760
743
|
out[key] = _shaped(spec.slug, item)
|
|
761
744
|
elif key in spec.round_fields and isinstance(item, (int, float)) and not isinstance(item, bool):
|
|
762
|
-
# Coordinates and the like: keep the shape and the type, lose the precision.
|
|
763
745
|
out[key] = round(item, spec.round_fields[key])
|
|
764
746
|
else:
|
|
765
747
|
out[key] = redact(item, spec, here)
|
|
@@ -777,9 +759,6 @@ def _dig(body: Any, path: str) -> Any:
|
|
|
777
759
|
return body
|
|
778
760
|
|
|
779
761
|
|
|
780
|
-
# ---------------------------------------------------------------- capture
|
|
781
|
-
|
|
782
|
-
|
|
783
762
|
def capture(
|
|
784
763
|
world: str | Path,
|
|
785
764
|
op: str,
|
|
@@ -825,8 +804,7 @@ def capture(
|
|
|
825
804
|
sleep(pause)
|
|
826
805
|
page_params = list(params)
|
|
827
806
|
if cursor and cursor.startswith(("https://", "http://")):
|
|
828
|
-
#
|
|
829
|
-
# query, never its host — the page must come from the connection described.
|
|
807
|
+
# Follow a next-page URL's query, never its host: the page must come from the connection described.
|
|
830
808
|
following = urllib.parse.urlsplit(cursor)
|
|
831
809
|
if following.netloc != urllib.parse.urlsplit(spec.base_url).netloc:
|
|
832
810
|
fail("capture", f"The next-page URL points at {following.netloc}, not the connection", "/capture")
|
|
@@ -898,22 +876,12 @@ def _append_manifest(root: Path, path: Path, record: dict, sha256: str) -> None:
|
|
|
898
876
|
manifest_path.write_text(json.dumps(entries, indent=1) + "\n")
|
|
899
877
|
|
|
900
878
|
|
|
901
|
-
# ---------------------------------------------------------------- ingest
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
def _key(contract: Any, entity: str, row: dict) -> tuple | None:
|
|
905
|
-
fields = contract.entities.get(entity, {}).get("primary_key", [])
|
|
906
|
-
if not fields or any(field not in row for field in fields):
|
|
907
|
-
return None
|
|
908
|
-
return tuple(json.dumps(row[field], sort_keys=True) for field in fields)
|
|
909
|
-
|
|
910
|
-
|
|
911
879
|
def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list[dict]:
|
|
912
880
|
"""Insert what is new, update what changed, skip what the world already holds."""
|
|
913
881
|
held: dict[tuple[str, tuple], dict] = {}
|
|
914
882
|
for mutation in prior:
|
|
915
883
|
if mutation.get("op") in ("insert", "update") and isinstance(mutation.get("value"), dict):
|
|
916
|
-
key =
|
|
884
|
+
key = PrimaryKey.in_contract(contract, mutation["entity"]).identity(mutation["value"])
|
|
917
885
|
if key is not None:
|
|
918
886
|
held[(mutation["entity"], key)] = mutation["value"]
|
|
919
887
|
mutations: list[dict] = []
|
|
@@ -926,7 +894,8 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
|
|
|
926
894
|
if token in seen:
|
|
927
895
|
continue
|
|
928
896
|
seen[token] = entry["file"]
|
|
929
|
-
|
|
897
|
+
primary = PrimaryKey.in_contract(contract, entity)
|
|
898
|
+
key = primary.identity(row)
|
|
930
899
|
existing = held.get((entity, key)) if key is not None else None
|
|
931
900
|
if existing == row:
|
|
932
901
|
continue
|
|
@@ -940,8 +909,7 @@ def _mutations(root: Path, spec: Spec, contract: Any, prior: list[dict]) -> list
|
|
|
940
909
|
},
|
|
941
910
|
}
|
|
942
911
|
if existing is not None:
|
|
943
|
-
|
|
944
|
-
mutation["key"] = {field: row[field] for field in fields}
|
|
912
|
+
mutation["key"] = primary.key(row)
|
|
945
913
|
mutations.append(mutation)
|
|
946
914
|
return mutations
|
|
947
915
|
|
|
@@ -996,9 +964,7 @@ def response_rows(
|
|
|
996
964
|
for raw_row in rows:
|
|
997
965
|
if not isinstance(raw_row, dict):
|
|
998
966
|
fail("ingest", f"A row in {where} is not an object", pointer)
|
|
999
|
-
# The request is the authority for
|
|
1000
|
-
# names the field itself is overwritten. A parameter the capture was made
|
|
1001
|
-
# without is not stamped; the contract then says whether the row lacks it.
|
|
967
|
+
# The request is the authority for carried fields: a row naming the field itself is overwritten.
|
|
1002
968
|
carried = {name: context[source] for name, source in operation.carry_args.items() if source in context}
|
|
1003
969
|
out.extend(_project(operation, {**raw_row, **carried}, where, pointer))
|
|
1004
970
|
return out
|
|
@@ -1074,17 +1040,16 @@ def _prove(root: Path, contract: Any, batch: dict) -> dict[str, int]:
|
|
|
1074
1040
|
"""Apply the batch in a scratch copy of the world; the entity counts it leaves, or the
|
|
1075
1041
|
store's refusal. The world on disk is never touched."""
|
|
1076
1042
|
from .._materialize import materialize_template # only ingest needs the parent package
|
|
1043
|
+
from .snapshot import build_names
|
|
1077
1044
|
|
|
1078
1045
|
with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
|
|
1079
1046
|
staged = Path(scratch) / "world"
|
|
1080
|
-
shutil.copytree(
|
|
1081
|
-
root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", "state.sqlite", "state.json")
|
|
1082
|
-
)
|
|
1047
|
+
shutil.copytree(root, staged, ignore=shutil.ignore_patterns(CAPTURES, "__pycache__", *build_names()))
|
|
1083
1048
|
(staged / "data/initial.json").write_text(json.dumps(batch, sort_keys=True, indent=1) + "\n")
|
|
1084
1049
|
compile_bundle(staged)
|
|
1085
1050
|
database = Path(scratch) / "world.db"
|
|
1086
1051
|
materialize_template(staged, database)
|
|
1087
|
-
with
|
|
1052
|
+
with WorldDatabase.open(database, check_bundle(staged)) as store:
|
|
1088
1053
|
store.audit()
|
|
1089
1054
|
return {entity: len(store.list(entity)) for entity in contract.entities}
|
|
1090
1055
|
|
|
@@ -1168,5 +1133,5 @@ def _counts(root: Path, contract: Any) -> dict[str, int]:
|
|
|
1168
1133
|
with tempfile.TemporaryDirectory(prefix="gateway-ingest-") as scratch:
|
|
1169
1134
|
database = Path(scratch) / "world.db"
|
|
1170
1135
|
materialize_template(root, database)
|
|
1171
|
-
with
|
|
1136
|
+
with WorldDatabase.open(database, contract) as store:
|
|
1172
1137
|
return {entity: len(store.list(entity)) for entity in contract.entities}
|