@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -65,9 +65,8 @@ def import_data(request: dict) -> dict:
|
|
|
65
65
|
**report,
|
|
66
66
|
**{key: result[key] for key in ("proof", "deferred") if key in result},
|
|
67
67
|
}
|
|
68
|
-
if result["kind"] == KIND_SCHEMA and snapshots.
|
|
69
|
-
#
|
|
70
|
-
# served over them. Rebuild so the tree that travels back is one thing.
|
|
68
|
+
if result["kind"] == KIND_SCHEMA and snapshots.has_default_snapshot(world):
|
|
69
|
+
# A snapshot built before this import would be served over the grown journals.
|
|
71
70
|
from .compiler import build
|
|
72
71
|
from .snapshot import build_all
|
|
73
72
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
"""The bridge's read commands over a world's rows: ``counts``
|
|
1
|
+
"""The bridge's read commands over a world's rows: ``counts``, ``query``, ``sql``
|
|
2
|
+
and ``verify-legacy``.
|
|
2
3
|
|
|
3
|
-
|
|
4
|
+
Each opens the world through the ``open_world`` collaborator the dispatcher hands
|
|
4
5
|
them (:func:`platform._world_for`): inline ``files`` (small trees), a
|
|
5
6
|
``workdir`` the platform prepared with the tree and any checked-out snapshot
|
|
6
7
|
already on disk (large trees; nothing travels through the request), or a local
|
|
@@ -14,10 +15,16 @@ from __future__ import annotations
|
|
|
14
15
|
|
|
15
16
|
import tempfile
|
|
16
17
|
from pathlib import Path
|
|
17
|
-
from typing import Callable
|
|
18
|
+
from typing import Any, Callable
|
|
18
19
|
|
|
19
|
-
from
|
|
20
|
+
from ..legacy_sqlite import LegacyVerifiers
|
|
21
|
+
from .compiler import bundle_mode
|
|
22
|
+
from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
|
|
20
23
|
from .validation import ContractError, fail
|
|
24
|
+
from .world_sql import WorldSql
|
|
25
|
+
|
|
26
|
+
#: The memory one `sql` call may add to the bridge process, past what it already holds.
|
|
27
|
+
SQL_MEMORY_BYTES = 512 * 1024 * 1024
|
|
21
28
|
|
|
22
29
|
# (request, staging directory) -> the world root to read; the staging directory
|
|
23
30
|
# is where inline ``files`` are written and is gone when the command returns.
|
|
@@ -51,7 +58,7 @@ def query(request: dict, open_world: WorldOpener) -> dict:
|
|
|
51
58
|
a list of values), a page at a time (``limit``, ``offset``). With ``resolve``,
|
|
52
59
|
each value is plaintext and is looked up in the forms the world's ``[redact]``
|
|
53
60
|
policy stores (:func:`data.entity_query`). A tree that carries its compiled
|
|
54
|
-
``db
|
|
61
|
+
``db/<snapshot name>`` (a checked-out snapshot) answers from the snapshot, as
|
|
55
62
|
``counts`` does; otherwise from its data files. With ``across`` (local runtime
|
|
56
63
|
only: ``world`` is a directory here, so its ``[dependencies]`` paths resolve) the
|
|
57
64
|
answer is :func:`data.entity_query_across`'s groups, one per linked world a
|
|
@@ -93,3 +100,52 @@ def query(request: dict, open_world: WorldOpener) -> dict:
|
|
|
93
100
|
root, entity, where, **page, resolve=resolve, where_transform=where_transform, scenario=scenario
|
|
94
101
|
)
|
|
95
102
|
return {"status": "ok", "kind": world_kind(root), **report}
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def sql(request: dict, open_world: WorldOpener) -> dict:
|
|
106
|
+
"""``sql``: one read-only statement over a schema world's rows (:class:`WorldSql`),
|
|
107
|
+
from its current snapshot or its data files, optionally as a ``scenario`` starts."""
|
|
108
|
+
scenario = request.get("scenario")
|
|
109
|
+
if scenario is not None and (not isinstance(scenario, str) or not scenario):
|
|
110
|
+
fail("input", "scenario must be a non-empty string", "/scenario")
|
|
111
|
+
with tempfile.TemporaryDirectory(prefix="gateway-sql-") as directory:
|
|
112
|
+
root = open_world(request, Path(directory))
|
|
113
|
+
answer = WorldSql(root, scenario).run(request.get("sql"), request.get("limit", QUERY_DEFAULT_LIMIT))
|
|
114
|
+
return {"status": "ok", "kind": world_kind(root), **answer}
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limits: Any = None) -> None:
|
|
118
|
+
"""For a ``sql`` request, cap this one-shot bridge process's address space at what it holds now
|
|
119
|
+
plus :data:`SQL_MEMORY_BYTES`, never raising a soft or hard limit it already has: a caller's
|
|
120
|
+
statement cannot take the host's memory however wide its rows are. Called by the bridge's process
|
|
121
|
+
entry only, never by an in-process ``execute``, because the cap lasts for the life of the process.
|
|
122
|
+
Applies where ``/proc`` exists (Linux: the platform, and a TypeScript CLI's local bridge there);
|
|
123
|
+
``status`` and ``limits`` (the ``resource`` module) are injectable for tests."""
|
|
124
|
+
if not isinstance(request, dict) or request.get("command") != "sql":
|
|
125
|
+
return
|
|
126
|
+
if not status.is_file():
|
|
127
|
+
return
|
|
128
|
+
if limits is None:
|
|
129
|
+
import resource as limits
|
|
130
|
+
sizes = [int(line.split()[1]) * 1024 for line in status.read_text().splitlines() if line.startswith("VmSize:")]
|
|
131
|
+
if not sizes:
|
|
132
|
+
fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
|
|
133
|
+
cap = sizes[0] + SQL_MEMORY_BYTES
|
|
134
|
+
soft, hard = limits.getrlimit(limits.RLIMIT_AS)
|
|
135
|
+
hard = _lower(cap, hard, limits.RLIM_INFINITY)
|
|
136
|
+
limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _lower(cap: int, limit: int, infinity: int) -> int:
|
|
140
|
+
return cap if limit == infinity else min(cap, limit)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def verify_legacy(request: dict, open_world: WorldOpener) -> dict:
|
|
144
|
+
"""``verify-legacy``: a db/schema.sql world's SQL verifiers, each scored on a fresh copy
|
|
145
|
+
of its seeded state (:class:`LegacyVerifiers`). A schema world grades through its
|
|
146
|
+
contract instead (``World.verify``)."""
|
|
147
|
+
with tempfile.TemporaryDirectory(prefix="gateway-verify-") as directory:
|
|
148
|
+
root = open_world(request, Path(directory))
|
|
149
|
+
if bundle_mode(root) is not None:
|
|
150
|
+
fail("kind", "verify-legacy scores a db/schema.sql world; a schema world grades through World.verify")
|
|
151
|
+
return {"status": "ok", "verifiers": LegacyVerifiers(root).scores()}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""An entity's primary key, read off a row or a key.
|
|
2
|
+
|
|
3
|
+
Each field of a key is a property name (``id``) or a JSON Pointer to a scalar inside
|
|
4
|
+
the row (``/protocolSection/identificationModule/nctId``). A row carries a nested
|
|
5
|
+
field inside its objects; a key — what ``store.get``, ``update`` and ``delete`` take —
|
|
6
|
+
carries every field flat under its own name. :class:`PrimaryKey` reads both, so the
|
|
7
|
+
store, the data import, the batch lane, ingest, extract, grading and the served API
|
|
8
|
+
agree on what a row's key is."""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
from dataclasses import dataclass
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from .compiler import json_pointer_parts
|
|
17
|
+
|
|
18
|
+
_ABSENT = object()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class PrimaryKey:
|
|
23
|
+
fields: tuple[str, ...]
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def of(cls, entity: dict) -> PrimaryKey:
|
|
27
|
+
return cls(tuple(entity.get("primary_key") or ()))
|
|
28
|
+
|
|
29
|
+
@classmethod
|
|
30
|
+
def in_contract(cls, contract: Any, entity: str) -> PrimaryKey:
|
|
31
|
+
return cls.of(contract.entities.get(entity, {}))
|
|
32
|
+
|
|
33
|
+
def missing(self, row: Any) -> str | None:
|
|
34
|
+
"""The first key field ``row`` does not carry, or None when it carries them all."""
|
|
35
|
+
if not isinstance(row, dict):
|
|
36
|
+
return self.fields[0] if self.fields else None
|
|
37
|
+
return next((field for field in self.fields if _read(row, field) is _ABSENT), None)
|
|
38
|
+
|
|
39
|
+
def values(self, row: dict) -> list[Any]:
|
|
40
|
+
return [None if (value := _read(row, field)) is _ABSENT else value for field in self.fields]
|
|
41
|
+
|
|
42
|
+
def key(self, row: dict) -> dict[str, Any]:
|
|
43
|
+
"""The key a store lookup takes for ``row``: every field under its own name."""
|
|
44
|
+
return dict(zip(self.fields, self.values(row)))
|
|
45
|
+
|
|
46
|
+
def identity(self, row: Any) -> tuple[str, ...] | None:
|
|
47
|
+
"""The row's key as comparable text; None without a key or when a field is missing."""
|
|
48
|
+
if not self.fields or self.missing(row) is not None:
|
|
49
|
+
return None
|
|
50
|
+
return tuple(json.dumps(value, sort_keys=True) for value in self.values(row))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _read(row: dict, field: str) -> Any:
|
|
54
|
+
if field in row:
|
|
55
|
+
return row[field]
|
|
56
|
+
current: Any = row
|
|
57
|
+
for part in json_pointer_parts(field, field, code="key"):
|
|
58
|
+
if not isinstance(current, dict) or part not in current:
|
|
59
|
+
return _ABSENT
|
|
60
|
+
current = current[part]
|
|
61
|
+
return current
|
|
@@ -8,7 +8,7 @@ What the rows are proven *against* depends on where the world's data lives, and
|
|
|
8
8
|
are the data, so the rows are imported into a scratch copy of the tree through
|
|
9
9
|
the same gate the real import runs;
|
|
10
10
|
* :class:`SnapshotProof` — a checkout pinned to a hosted snapshot that carries the
|
|
11
|
-
snapshot itself (
|
|
11
|
+
snapshot itself (the default snapshot current against the pin): the platform's
|
|
12
12
|
batch lane runs over a private copy of those bytes, so references, unique keys
|
|
13
13
|
and merges are checked against the rows the platform holds;
|
|
14
14
|
* :class:`ContractProof` — a pinned checkout without its snapshot: only what needs
|
|
@@ -26,15 +26,20 @@ import shutil
|
|
|
26
26
|
import tempfile
|
|
27
27
|
from contextlib import contextmanager
|
|
28
28
|
from pathlib import Path
|
|
29
|
-
from typing import Any, Iterable, Iterator
|
|
29
|
+
from typing import TYPE_CHECKING, Any, Iterable, Iterator
|
|
30
30
|
|
|
31
31
|
from .batch import ChunkSpec, build_batch, chunk_rows, validate_chunks
|
|
32
32
|
from .compiler import compile_bundle
|
|
33
|
+
from .database import WorldDatabase
|
|
34
|
+
from .snapshot import build_names
|
|
33
35
|
from .validation import fail
|
|
34
36
|
|
|
37
|
+
if TYPE_CHECKING:
|
|
38
|
+
from .snapshot import SnapshotPin
|
|
39
|
+
|
|
35
40
|
Rows = dict[str, list[dict]]
|
|
36
41
|
#: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
|
|
37
|
-
SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway",
|
|
42
|
+
SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", *build_names())
|
|
38
43
|
#: What a state-independent proof leaves to the platform batch.
|
|
39
44
|
DEFERRED = ("references", "unique")
|
|
40
45
|
|
|
@@ -61,10 +66,10 @@ class RowProof:
|
|
|
61
66
|
|
|
62
67
|
@classmethod
|
|
63
68
|
def of(cls, root: str | Path) -> "RowProof":
|
|
64
|
-
from .snapshot import
|
|
69
|
+
from .snapshot import SnapshotPin, current_default
|
|
65
70
|
|
|
66
71
|
root = Path(root)
|
|
67
|
-
pin =
|
|
72
|
+
pin = SnapshotPin.of(root)
|
|
68
73
|
if pin is None:
|
|
69
74
|
return FilesProof(root)
|
|
70
75
|
contract = compile_bundle(root, snapshots=False)
|
|
@@ -95,8 +100,9 @@ class FilesProof(RowProof):
|
|
|
95
100
|
with proving_copy(self.root) as staged, tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
|
|
96
101
|
contract = compile_bundle(staged, snapshots=False)
|
|
97
102
|
build_default(staged, contract)
|
|
98
|
-
|
|
99
|
-
|
|
103
|
+
snapshot = default_snapshot(staged, contract)
|
|
104
|
+
parent = Path(scratch) / snapshot.name
|
|
105
|
+
WorldDatabase.clone_file(snapshot, parent)
|
|
100
106
|
return self._report(_batch(contract, parent, chunks, mode, merge, atomic, "check", EMPTY_DIGEST, redactor))
|
|
101
107
|
|
|
102
108
|
def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
|
|
@@ -108,7 +114,7 @@ class FilesProof(RowProof):
|
|
|
108
114
|
|
|
109
115
|
|
|
110
116
|
class _PinnedProof(RowProof):
|
|
111
|
-
def __init__(self, root: Path, pin:
|
|
117
|
+
def __init__(self, root: Path, pin: SnapshotPin, contract: Any):
|
|
112
118
|
super().__init__(root)
|
|
113
119
|
self.pin = pin
|
|
114
120
|
self.contract = contract
|
|
@@ -143,10 +149,11 @@ class SnapshotProof(_PinnedProof):
|
|
|
143
149
|
from .snapshot import default_snapshot
|
|
144
150
|
|
|
145
151
|
with tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
|
|
146
|
-
|
|
147
|
-
|
|
152
|
+
snapshot = default_snapshot(self.root, self.contract)
|
|
153
|
+
parent = Path(scratch) / snapshot.name
|
|
154
|
+
WorldDatabase.clone_file(snapshot, parent)
|
|
148
155
|
return self._report(
|
|
149
|
-
_batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin
|
|
156
|
+
_batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin.batch_digest, redactor)
|
|
150
157
|
)
|
|
151
158
|
|
|
152
159
|
|
|
@@ -157,7 +164,7 @@ class ContractProof(_PinnedProof):
|
|
|
157
164
|
self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
|
|
158
165
|
) -> dict:
|
|
159
166
|
shaped = validate_chunks(self.contract, chunks, redactor=redactor)
|
|
160
|
-
counts = self.pin.get("entityCounts")
|
|
167
|
+
counts = self.pin.fields.get("entityCounts")
|
|
161
168
|
return self._report(
|
|
162
169
|
{
|
|
163
170
|
"rows": shaped["rows"],
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
"""A route request's body as a world reads it: one :class:`RequestBody` kind per
|
|
2
|
+
media type, decided in :meth:`RequestBody.of` alone.
|
|
3
|
+
|
|
4
|
+
``fields`` is what a route reads, a handler's ``body`` and a declared route's
|
|
5
|
+
input: a JSON value; a form's fields, each name's first value; a multipart
|
|
6
|
+
form's fields, each name's first part, a file part read as ``{"filename",
|
|
7
|
+
"content_type", "size"}`` with its bytes in ``files``; a multipart/related
|
|
8
|
+
body's first part as JSON, the parts after it in ``files`` keyed ``"1"``,
|
|
9
|
+
``"2"``... A body of any other type is an upload (``application/octet-stream``,
|
|
10
|
+
``text/csv``, ``image/png``): its fields are its JSON when it is JSON, since
|
|
11
|
+
clients label JSON many ways, and else None, which a declared route refuses as
|
|
12
|
+
the vendor's 400 and a handler route reads as ``{}`` beside the bytes
|
|
13
|
+
(``ctx.raw``). JSON that does not parse under a JSON label, and a multipart
|
|
14
|
+
body that does not split, are None to every route. Every part is found in time
|
|
15
|
+
linear in the body; a body of more than :data:`MAX_PARTS` parts, or a part
|
|
16
|
+
whose headers pass :data:`MAX_PART_HEAD` bytes, does not split.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import re
|
|
23
|
+
import urllib.parse
|
|
24
|
+
from abc import ABC, abstractmethod
|
|
25
|
+
from dataclasses import dataclass
|
|
26
|
+
from functools import cached_property
|
|
27
|
+
from typing import Any, ClassVar, Mapping
|
|
28
|
+
|
|
29
|
+
MAX_PARTS = 1000
|
|
30
|
+
MAX_PART_HEAD = 8 * 1024
|
|
31
|
+
_BOUNDARY = re.compile(r"""(?:^|;)\s*boundary=(?:"([^"\r\n]{1,70})"|([^\s;"]{1,70}))""", re.IGNORECASE)
|
|
32
|
+
_PARAM = re.compile(r""";\s*([A-Za-z0-9!#$%&'*+.^_`|~-]+)\s*=\s*("(?:[^"\\]|\\.)*"|[^;]*)""")
|
|
33
|
+
_CRLF = b"\r\n"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def json_value(raw: bytes) -> Any:
|
|
37
|
+
"""``raw`` as JSON: ``{}`` when empty, None when it is not JSON."""
|
|
38
|
+
if not raw:
|
|
39
|
+
return {}
|
|
40
|
+
try:
|
|
41
|
+
return json.loads(raw.decode("utf-8"))
|
|
42
|
+
except (ValueError, RecursionError):
|
|
43
|
+
return None
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def form_fields(raw: bytes) -> dict[str, str]:
|
|
47
|
+
"""``raw`` as an ``application/x-www-form-urlencoded`` form, each name's first value."""
|
|
48
|
+
pairs = urllib.parse.parse_qs(raw.decode("utf-8", "replace"), keep_blank_values=True)
|
|
49
|
+
return {name: values[0] for name, values in pairs.items()}
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass(frozen=True)
|
|
53
|
+
class Upload:
|
|
54
|
+
"""One file a request carried: its name and type as the client gave them, and its bytes."""
|
|
55
|
+
|
|
56
|
+
filename: str | None
|
|
57
|
+
content_type: str | None
|
|
58
|
+
content: bytes
|
|
59
|
+
|
|
60
|
+
def described(self) -> dict:
|
|
61
|
+
return {"filename": self.filename, "content_type": self.content_type, "size": len(self.content)}
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(frozen=True)
|
|
65
|
+
class Part:
|
|
66
|
+
"""One part of a multipart body: its headers, names lower-cased, and its bytes."""
|
|
67
|
+
|
|
68
|
+
headers: dict[str, str]
|
|
69
|
+
content: bytes
|
|
70
|
+
|
|
71
|
+
@cached_property
|
|
72
|
+
def disposition(self) -> dict[str, str]:
|
|
73
|
+
params = _PARAM.findall(";" + self.headers.get("content-disposition", "").partition(";")[2])
|
|
74
|
+
return {name.lower(): _unquoted(value.strip()) for name, value in params}
|
|
75
|
+
|
|
76
|
+
@property
|
|
77
|
+
def name(self) -> str | None:
|
|
78
|
+
return self.disposition.get("name")
|
|
79
|
+
|
|
80
|
+
@property
|
|
81
|
+
def is_file(self) -> bool:
|
|
82
|
+
return "filename" in self.disposition
|
|
83
|
+
|
|
84
|
+
def upload(self) -> Upload:
|
|
85
|
+
return Upload(self.disposition.get("filename"), self.headers.get("content-type"), self.content)
|
|
86
|
+
|
|
87
|
+
def field(self) -> Any:
|
|
88
|
+
return self.upload().described() if self.is_file else self.content.decode("utf-8", "replace")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _unquoted(value: str) -> str:
|
|
92
|
+
quoted = len(value) > 1 and value[0] == value[-1] == '"'
|
|
93
|
+
return re.sub(r"\\(.)", r"\1", value[1:-1]) if quoted else value
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _part(chunk: bytes) -> Part | None:
|
|
97
|
+
"""One part from what follows its delimiter: transport padding, a line break, its
|
|
98
|
+
headers and a blank line, then its bytes."""
|
|
99
|
+
padded = chunk.lstrip(b" \t")
|
|
100
|
+
if not padded.startswith(_CRLF):
|
|
101
|
+
return None
|
|
102
|
+
body = padded[2:]
|
|
103
|
+
if body.startswith(_CRLF):
|
|
104
|
+
return Part({}, body[2:])
|
|
105
|
+
head, separator, content = body.partition(_CRLF + _CRLF)
|
|
106
|
+
if not separator or len(head) > MAX_PART_HEAD:
|
|
107
|
+
return None
|
|
108
|
+
named = [line.partition(":") for line in head.decode("utf-8", "replace").split("\r\n")]
|
|
109
|
+
return Part({name.strip().lower(): value.strip() for name, _, value in named}, content)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def multipart_parts(raw: bytes, content_type: str) -> list[Part] | None:
|
|
113
|
+
"""The parts of a multipart body labelled ``content_type``, or None when it does not split."""
|
|
114
|
+
found = _BOUNDARY.search(content_type)
|
|
115
|
+
if found is None:
|
|
116
|
+
return None
|
|
117
|
+
delimiter = _CRLF + b"--" + (found.group(1) or found.group(2)).encode("latin-1")
|
|
118
|
+
chunks = (_CRLF + raw).split(delimiter, MAX_PARTS + 2)
|
|
119
|
+
if not 2 < len(chunks) <= MAX_PARTS + 2 or not chunks[-1].startswith(b"--"):
|
|
120
|
+
return None
|
|
121
|
+
parts = [_part(chunk) for chunk in chunks[1:-1]]
|
|
122
|
+
return None if None in parts else parts
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _first_by_name(parts: list[Part]) -> dict[str, Part]:
|
|
126
|
+
named: dict[str, Part] = {}
|
|
127
|
+
for part in parts:
|
|
128
|
+
named.setdefault(part.name or "", part)
|
|
129
|
+
named.pop("", None)
|
|
130
|
+
return named
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class RequestBody(ABC):
|
|
135
|
+
"""A request's body (module docstring): the bytes as sent and the label they came under."""
|
|
136
|
+
|
|
137
|
+
raw: bytes
|
|
138
|
+
content_type: str = ""
|
|
139
|
+
media: ClassVar[tuple[str, ...]] = ()
|
|
140
|
+
uploads: ClassVar[bool] = False
|
|
141
|
+
|
|
142
|
+
@staticmethod
|
|
143
|
+
def of(raw: bytes, headers: Mapping[str, str]) -> RequestBody:
|
|
144
|
+
"""The kind ``headers``' content type names, a new one for every request: the one
|
|
145
|
+
place a body's kind is decided. An empty body is ``{}`` whatever its label."""
|
|
146
|
+
content_type = headers.get("content-type") or ""
|
|
147
|
+
media = content_type.partition(";")[0].strip().lower()
|
|
148
|
+
labelled = KINDS.get(media) or (UploadBody if media and not media.endswith("+json") else JsonBody)
|
|
149
|
+
return (labelled if raw else JsonBody)(raw, content_type)
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
@abstractmethod
|
|
153
|
+
def fields(self) -> Any:
|
|
154
|
+
"""What a route reads (module docstring), or None when the body cannot be read so."""
|
|
155
|
+
|
|
156
|
+
@property
|
|
157
|
+
def files(self) -> dict[str, Upload]:
|
|
158
|
+
"""The files the body carried, by the field or position that names each."""
|
|
159
|
+
return {}
|
|
160
|
+
|
|
161
|
+
def fields_for(self, handler: bool) -> Any:
|
|
162
|
+
"""What one route reads: :attr:`fields`, or for a handler route ``{}`` beside an
|
|
163
|
+
upload's bytes; None when the route cannot read it."""
|
|
164
|
+
return {} if self.fields is None and self.uploads and handler else self.fields
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
@dataclass(frozen=True)
|
|
168
|
+
class JsonBody(RequestBody):
|
|
169
|
+
media = ("application/json",)
|
|
170
|
+
|
|
171
|
+
@cached_property
|
|
172
|
+
def fields(self) -> Any:
|
|
173
|
+
return json_value(self.raw)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@dataclass(frozen=True)
|
|
177
|
+
class UploadBody(RequestBody):
|
|
178
|
+
uploads = True
|
|
179
|
+
|
|
180
|
+
@cached_property
|
|
181
|
+
def fields(self) -> Any:
|
|
182
|
+
return json_value(self.raw)
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
@dataclass(frozen=True)
|
|
186
|
+
class FormBody(RequestBody):
|
|
187
|
+
"""A form, or JSON under a form's label: no form field starts with ``{`` or ``[``,
|
|
188
|
+
and some clients label every POST a form."""
|
|
189
|
+
|
|
190
|
+
media = ("application/x-www-form-urlencoded",)
|
|
191
|
+
|
|
192
|
+
@cached_property
|
|
193
|
+
def fields(self) -> Any:
|
|
194
|
+
return json_value(self.raw) if self.raw.lstrip().startswith((b"{", b"[")) else form_fields(self.raw)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@dataclass(frozen=True)
|
|
198
|
+
class MultipartFormBody(RequestBody):
|
|
199
|
+
media = ("multipart/form-data",)
|
|
200
|
+
|
|
201
|
+
@cached_property
|
|
202
|
+
def named(self) -> dict[str, Part] | None:
|
|
203
|
+
parts = multipart_parts(self.raw, self.content_type)
|
|
204
|
+
return None if parts is None else _first_by_name(parts)
|
|
205
|
+
|
|
206
|
+
@cached_property
|
|
207
|
+
def fields(self) -> Any:
|
|
208
|
+
return None if self.named is None else {name: part.field() for name, part in self.named.items()}
|
|
209
|
+
|
|
210
|
+
@property
|
|
211
|
+
def files(self) -> dict[str, Upload]:
|
|
212
|
+
return {name: part.upload() for name, part in (self.named or {}).items() if part.is_file}
|
|
213
|
+
|
|
214
|
+
def fields_for(self, handler: bool) -> Any:
|
|
215
|
+
"""A form with a file reaches a handler alone: a declared route reads no file."""
|
|
216
|
+
return None if self.files and not handler else self.fields
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
@dataclass(frozen=True)
|
|
220
|
+
class MultipartRelatedBody(RequestBody):
|
|
221
|
+
media = ("multipart/related",)
|
|
222
|
+
|
|
223
|
+
@cached_property
|
|
224
|
+
def parts(self) -> list[Part]:
|
|
225
|
+
return multipart_parts(self.raw, self.content_type) or []
|
|
226
|
+
|
|
227
|
+
@cached_property
|
|
228
|
+
def fields(self) -> Any:
|
|
229
|
+
return json_value(self.parts[0].content) if self.parts else None
|
|
230
|
+
|
|
231
|
+
@property
|
|
232
|
+
def files(self) -> dict[str, Upload]:
|
|
233
|
+
return {str(index): part.upload() for index, part in enumerate(self.parts[1:], start=1)}
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
KINDS: dict[str, type[RequestBody]] = {
|
|
237
|
+
media: kind for kind in (JsonBody, FormBody, MultipartFormBody, MultipartRelatedBody) for media in kind.media
|
|
238
|
+
}
|