@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -65,9 +65,8 @@ def import_data(request: dict) -> dict:
65
65
  **report,
66
66
  **{key: result[key] for key in ("proof", "deferred") if key in result},
67
67
  }
68
- if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
69
- # The journals grew; a snapshot built before this import would be
70
- # served over them. Rebuild so the tree that travels back is one thing.
68
+ if result["kind"] == KIND_SCHEMA and snapshots.has_default_snapshot(world):
69
+ # A snapshot built before this import would be served over the grown journals.
71
70
  from .compiler import build
72
71
  from .snapshot import build_all
73
72
 
@@ -1,6 +1,7 @@
1
- """The bridge's read commands over a world's rows: ``counts`` and ``query``.
1
+ """The bridge's read commands over a world's rows: ``counts``, ``query``, ``sql``
2
+ and ``verify-legacy``.
2
3
 
3
- Both open the world through the ``open_world`` collaborator the dispatcher hands
4
+ Each opens the world through the ``open_world`` collaborator the dispatcher hands
4
5
  them (:func:`platform._world_for`): inline ``files`` (small trees), a
5
6
  ``workdir`` the platform prepared with the tree and any checked-out snapshot
6
7
  already on disk (large trees; nothing travels through the request), or a local
@@ -14,10 +15,16 @@ from __future__ import annotations
14
15
 
15
16
  import tempfile
16
17
  from pathlib import Path
17
- from typing import Callable
18
+ from typing import Any, Callable
18
19
 
19
- from .data import entity_counts, entity_query, entity_query_across, entity_rows, world_kind
20
+ from ..legacy_sqlite import LegacyVerifiers
21
+ from .compiler import bundle_mode
22
+ from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
20
23
  from .validation import ContractError, fail
24
+ from .world_sql import WorldSql
25
+
26
+ #: The memory one `sql` call may add to the bridge process, past what it already holds.
27
+ SQL_MEMORY_BYTES = 512 * 1024 * 1024
21
28
 
22
29
  # (request, staging directory) -> the world root to read; the staging directory
23
30
  # is where inline ``files`` are written and is gone when the command returns.
@@ -51,7 +58,7 @@ def query(request: dict, open_world: WorldOpener) -> dict:
51
58
  a list of values), a page at a time (``limit``, ``offset``). With ``resolve``,
52
59
  each value is plaintext and is looked up in the forms the world's ``[redact]``
53
60
  policy stores (:func:`data.entity_query`). A tree that carries its compiled
54
- ``db/state.sqlite`` (a checked-out snapshot) answers from the snapshot, as
61
+ ``db/<snapshot name>`` (a checked-out snapshot) answers from the snapshot, as
55
62
  ``counts`` does; otherwise from its data files. With ``across`` (local runtime
56
63
  only: ``world`` is a directory here, so its ``[dependencies]`` paths resolve) the
57
64
  answer is :func:`data.entity_query_across`'s groups, one per linked world a
@@ -93,3 +100,52 @@ def query(request: dict, open_world: WorldOpener) -> dict:
93
100
  root, entity, where, **page, resolve=resolve, where_transform=where_transform, scenario=scenario
94
101
  )
95
102
  return {"status": "ok", "kind": world_kind(root), **report}
103
+
104
+
105
+ def sql(request: dict, open_world: WorldOpener) -> dict:
106
+ """``sql``: one read-only statement over a schema world's rows (:class:`WorldSql`),
107
+ from its current snapshot or its data files, optionally as a ``scenario`` starts."""
108
+ scenario = request.get("scenario")
109
+ if scenario is not None and (not isinstance(scenario, str) or not scenario):
110
+ fail("input", "scenario must be a non-empty string", "/scenario")
111
+ with tempfile.TemporaryDirectory(prefix="gateway-sql-") as directory:
112
+ root = open_world(request, Path(directory))
113
+ answer = WorldSql(root, scenario).run(request.get("sql"), request.get("limit", QUERY_DEFAULT_LIMIT))
114
+ return {"status": "ok", "kind": world_kind(root), **answer}
115
+
116
+
117
+ def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limits: Any = None) -> None:
118
+ """For a ``sql`` request, cap this one-shot bridge process's address space at what it holds now
119
+ plus :data:`SQL_MEMORY_BYTES`, never raising a soft or hard limit it already has: a caller's
120
+ statement cannot take the host's memory however wide its rows are. Called by the bridge's process
121
+ entry only, never by an in-process ``execute``, because the cap lasts for the life of the process.
122
+ Applies where ``/proc`` exists (Linux: the platform, and a TypeScript CLI's local bridge there);
123
+ ``status`` and ``limits`` (the ``resource`` module) are injectable for tests."""
124
+ if not isinstance(request, dict) or request.get("command") != "sql":
125
+ return
126
+ if not status.is_file():
127
+ return
128
+ if limits is None:
129
+ import resource as limits
130
+ sizes = [int(line.split()[1]) * 1024 for line in status.read_text().splitlines() if line.startswith("VmSize:")]
131
+ if not sizes:
132
+ fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
133
+ cap = sizes[0] + SQL_MEMORY_BYTES
134
+ soft, hard = limits.getrlimit(limits.RLIMIT_AS)
135
+ hard = _lower(cap, hard, limits.RLIM_INFINITY)
136
+ limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
137
+
138
+
139
+ def _lower(cap: int, limit: int, infinity: int) -> int:
140
+ return cap if limit == infinity else min(cap, limit)
141
+
142
+
143
+ def verify_legacy(request: dict, open_world: WorldOpener) -> dict:
144
+ """``verify-legacy``: a db/schema.sql world's SQL verifiers, each scored on a fresh copy
145
+ of its seeded state (:class:`LegacyVerifiers`). A schema world grades through its
146
+ contract instead (``World.verify``)."""
147
+ with tempfile.TemporaryDirectory(prefix="gateway-verify-") as directory:
148
+ root = open_world(request, Path(directory))
149
+ if bundle_mode(root) is not None:
150
+ fail("kind", "verify-legacy scores a db/schema.sql world; a schema world grades through World.verify")
151
+ return {"status": "ok", "verifiers": LegacyVerifiers(root).scores()}
@@ -0,0 +1,61 @@
1
+ """An entity's primary key, read off a row or a key.
2
+
3
+ Each field of a key is a property name (``id``) or a JSON Pointer to a scalar inside
4
+ the row (``/protocolSection/identificationModule/nctId``). A row carries a nested
5
+ field inside its objects; a key — what ``store.get``, ``update`` and ``delete`` take —
6
+ carries every field flat under its own name. :class:`PrimaryKey` reads both, so the
7
+ store, the data import, the batch lane, ingest, extract, grading and the served API
8
+ agree on what a row's key is."""
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from dataclasses import dataclass
14
+ from typing import Any
15
+
16
+ from .compiler import json_pointer_parts
17
+
18
+ _ABSENT = object()
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class PrimaryKey:
23
+ fields: tuple[str, ...]
24
+
25
+ @classmethod
26
+ def of(cls, entity: dict) -> PrimaryKey:
27
+ return cls(tuple(entity.get("primary_key") or ()))
28
+
29
+ @classmethod
30
+ def in_contract(cls, contract: Any, entity: str) -> PrimaryKey:
31
+ return cls.of(contract.entities.get(entity, {}))
32
+
33
+ def missing(self, row: Any) -> str | None:
34
+ """The first key field ``row`` does not carry, or None when it carries them all."""
35
+ if not isinstance(row, dict):
36
+ return self.fields[0] if self.fields else None
37
+ return next((field for field in self.fields if _read(row, field) is _ABSENT), None)
38
+
39
+ def values(self, row: dict) -> list[Any]:
40
+ return [None if (value := _read(row, field)) is _ABSENT else value for field in self.fields]
41
+
42
+ def key(self, row: dict) -> dict[str, Any]:
43
+ """The key a store lookup takes for ``row``: every field under its own name."""
44
+ return dict(zip(self.fields, self.values(row)))
45
+
46
+ def identity(self, row: Any) -> tuple[str, ...] | None:
47
+ """The row's key as comparable text; None without a key or when a field is missing."""
48
+ if not self.fields or self.missing(row) is not None:
49
+ return None
50
+ return tuple(json.dumps(value, sort_keys=True) for value in self.values(row))
51
+
52
+
53
+ def _read(row: dict, field: str) -> Any:
54
+ if field in row:
55
+ return row[field]
56
+ current: Any = row
57
+ for part in json_pointer_parts(field, field, code="key"):
58
+ if not isinstance(current, dict) or part not in current:
59
+ return _ABSENT
60
+ current = current[part]
61
+ return current
@@ -8,7 +8,7 @@ What the rows are proven *against* depends on where the world's data lives, and
8
8
  are the data, so the rows are imported into a scratch copy of the tree through
9
9
  the same gate the real import runs;
10
10
  * :class:`SnapshotProof` — a checkout pinned to a hosted snapshot that carries the
11
- snapshot itself (``db/state.sqlite`` current against the pin): the platform's
11
+ snapshot itself (the default snapshot current against the pin): the platform's
12
12
  batch lane runs over a private copy of those bytes, so references, unique keys
13
13
  and merges are checked against the rows the platform holds;
14
14
  * :class:`ContractProof` — a pinned checkout without its snapshot: only what needs
@@ -26,15 +26,20 @@ import shutil
26
26
  import tempfile
27
27
  from contextlib import contextmanager
28
28
  from pathlib import Path
29
- from typing import Any, Iterable, Iterator
29
+ from typing import TYPE_CHECKING, Any, Iterable, Iterator
30
30
 
31
31
  from .batch import ChunkSpec, build_batch, chunk_rows, validate_chunks
32
32
  from .compiler import compile_bundle
33
+ from .database import WorldDatabase
34
+ from .snapshot import build_names
33
35
  from .validation import fail
34
36
 
37
+ if TYPE_CHECKING:
38
+ from .snapshot import SnapshotPin
39
+
35
40
  Rows = dict[str, list[dict]]
36
41
  #: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
37
- SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
42
+ SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", *build_names())
38
43
  #: What a state-independent proof leaves to the platform batch.
39
44
  DEFERRED = ("references", "unique")
40
45
 
@@ -61,10 +66,10 @@ class RowProof:
61
66
 
62
67
  @classmethod
63
68
  def of(cls, root: str | Path) -> "RowProof":
64
- from .snapshot import current_default, pinned_default
69
+ from .snapshot import SnapshotPin, current_default
65
70
 
66
71
  root = Path(root)
67
- pin = pinned_default(root)
72
+ pin = SnapshotPin.of(root)
68
73
  if pin is None:
69
74
  return FilesProof(root)
70
75
  contract = compile_bundle(root, snapshots=False)
@@ -95,8 +100,9 @@ class FilesProof(RowProof):
95
100
  with proving_copy(self.root) as staged, tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
96
101
  contract = compile_bundle(staged, snapshots=False)
97
102
  build_default(staged, contract)
98
- parent = Path(scratch) / "state.sqlite"
99
- shutil.copyfile(default_snapshot(staged), parent)
103
+ snapshot = default_snapshot(staged, contract)
104
+ parent = Path(scratch) / snapshot.name
105
+ WorldDatabase.clone_file(snapshot, parent)
100
106
  return self._report(_batch(contract, parent, chunks, mode, merge, atomic, "check", EMPTY_DIGEST, redactor))
101
107
 
102
108
  def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
@@ -108,7 +114,7 @@ class FilesProof(RowProof):
108
114
 
109
115
 
110
116
  class _PinnedProof(RowProof):
111
- def __init__(self, root: Path, pin: dict, contract: Any):
117
+ def __init__(self, root: Path, pin: SnapshotPin, contract: Any):
112
118
  super().__init__(root)
113
119
  self.pin = pin
114
120
  self.contract = contract
@@ -143,10 +149,11 @@ class SnapshotProof(_PinnedProof):
143
149
  from .snapshot import default_snapshot
144
150
 
145
151
  with tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
146
- parent = Path(scratch) / "state.sqlite"
147
- shutil.copyfile(default_snapshot(self.root), parent)
152
+ snapshot = default_snapshot(self.root, self.contract)
153
+ parent = Path(scratch) / snapshot.name
154
+ WorldDatabase.clone_file(snapshot, parent)
148
155
  return self._report(
149
- _batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin["batchDigest"], redactor)
156
+ _batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin.batch_digest, redactor)
150
157
  )
151
158
 
152
159
 
@@ -157,7 +164,7 @@ class ContractProof(_PinnedProof):
157
164
  self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
158
165
  ) -> dict:
159
166
  shaped = validate_chunks(self.contract, chunks, redactor=redactor)
160
- counts = self.pin.get("entityCounts")
167
+ counts = self.pin.fields.get("entityCounts")
161
168
  return self._report(
162
169
  {
163
170
  "rows": shaped["rows"],
@@ -0,0 +1,238 @@
1
+ """A route request's body as a world reads it: one :class:`RequestBody` kind per
2
+ media type, decided in :meth:`RequestBody.of` alone.
3
+
4
+ ``fields`` is what a route reads, a handler's ``body`` and a declared route's
5
+ input: a JSON value; a form's fields, each name's first value; a multipart
6
+ form's fields, each name's first part, a file part read as ``{"filename",
7
+ "content_type", "size"}`` with its bytes in ``files``; a multipart/related
8
+ body's first part as JSON, the parts after it in ``files`` keyed ``"1"``,
9
+ ``"2"``... A body of any other type is an upload (``application/octet-stream``,
10
+ ``text/csv``, ``image/png``): its fields are its JSON when it is JSON, since
11
+ clients label JSON many ways, and else None, which a declared route refuses as
12
+ the vendor's 400 and a handler route reads as ``{}`` beside the bytes
13
+ (``ctx.raw``). JSON that does not parse under a JSON label, and a multipart
14
+ body that does not split, are None to every route. Every part is found in time
15
+ linear in the body; a body of more than :data:`MAX_PARTS` parts, or a part
16
+ whose headers pass :data:`MAX_PART_HEAD` bytes, does not split.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import json
22
+ import re
23
+ import urllib.parse
24
+ from abc import ABC, abstractmethod
25
+ from dataclasses import dataclass
26
+ from functools import cached_property
27
+ from typing import Any, ClassVar, Mapping
28
+
29
+ MAX_PARTS = 1000
30
+ MAX_PART_HEAD = 8 * 1024
31
+ _BOUNDARY = re.compile(r"""(?:^|;)\s*boundary=(?:"([^"\r\n]{1,70})"|([^\s;"]{1,70}))""", re.IGNORECASE)
32
+ _PARAM = re.compile(r""";\s*([A-Za-z0-9!#$%&'*+.^_`|~-]+)\s*=\s*("(?:[^"\\]|\\.)*"|[^;]*)""")
33
+ _CRLF = b"\r\n"
34
+
35
+
36
+ def json_value(raw: bytes) -> Any:
37
+ """``raw`` as JSON: ``{}`` when empty, None when it is not JSON."""
38
+ if not raw:
39
+ return {}
40
+ try:
41
+ return json.loads(raw.decode("utf-8"))
42
+ except (ValueError, RecursionError):
43
+ return None
44
+
45
+
46
+ def form_fields(raw: bytes) -> dict[str, str]:
47
+ """``raw`` as an ``application/x-www-form-urlencoded`` form, each name's first value."""
48
+ pairs = urllib.parse.parse_qs(raw.decode("utf-8", "replace"), keep_blank_values=True)
49
+ return {name: values[0] for name, values in pairs.items()}
50
+
51
+
52
+ @dataclass(frozen=True)
53
+ class Upload:
54
+ """One file a request carried: its name and type as the client gave them, and its bytes."""
55
+
56
+ filename: str | None
57
+ content_type: str | None
58
+ content: bytes
59
+
60
+ def described(self) -> dict:
61
+ return {"filename": self.filename, "content_type": self.content_type, "size": len(self.content)}
62
+
63
+
64
+ @dataclass(frozen=True)
65
+ class Part:
66
+ """One part of a multipart body: its headers, names lower-cased, and its bytes."""
67
+
68
+ headers: dict[str, str]
69
+ content: bytes
70
+
71
+ @cached_property
72
+ def disposition(self) -> dict[str, str]:
73
+ params = _PARAM.findall(";" + self.headers.get("content-disposition", "").partition(";")[2])
74
+ return {name.lower(): _unquoted(value.strip()) for name, value in params}
75
+
76
+ @property
77
+ def name(self) -> str | None:
78
+ return self.disposition.get("name")
79
+
80
+ @property
81
+ def is_file(self) -> bool:
82
+ return "filename" in self.disposition
83
+
84
+ def upload(self) -> Upload:
85
+ return Upload(self.disposition.get("filename"), self.headers.get("content-type"), self.content)
86
+
87
+ def field(self) -> Any:
88
+ return self.upload().described() if self.is_file else self.content.decode("utf-8", "replace")
89
+
90
+
91
+ def _unquoted(value: str) -> str:
92
+ quoted = len(value) > 1 and value[0] == value[-1] == '"'
93
+ return re.sub(r"\\(.)", r"\1", value[1:-1]) if quoted else value
94
+
95
+
96
+ def _part(chunk: bytes) -> Part | None:
97
+ """One part from what follows its delimiter: transport padding, a line break, its
98
+ headers and a blank line, then its bytes."""
99
+ padded = chunk.lstrip(b" \t")
100
+ if not padded.startswith(_CRLF):
101
+ return None
102
+ body = padded[2:]
103
+ if body.startswith(_CRLF):
104
+ return Part({}, body[2:])
105
+ head, separator, content = body.partition(_CRLF + _CRLF)
106
+ if not separator or len(head) > MAX_PART_HEAD:
107
+ return None
108
+ named = [line.partition(":") for line in head.decode("utf-8", "replace").split("\r\n")]
109
+ return Part({name.strip().lower(): value.strip() for name, _, value in named}, content)
110
+
111
+
112
+ def multipart_parts(raw: bytes, content_type: str) -> list[Part] | None:
113
+ """The parts of a multipart body labelled ``content_type``, or None when it does not split."""
114
+ found = _BOUNDARY.search(content_type)
115
+ if found is None:
116
+ return None
117
+ delimiter = _CRLF + b"--" + (found.group(1) or found.group(2)).encode("latin-1")
118
+ chunks = (_CRLF + raw).split(delimiter, MAX_PARTS + 2)
119
+ if not 2 < len(chunks) <= MAX_PARTS + 2 or not chunks[-1].startswith(b"--"):
120
+ return None
121
+ parts = [_part(chunk) for chunk in chunks[1:-1]]
122
+ return None if None in parts else parts
123
+
124
+
125
+ def _first_by_name(parts: list[Part]) -> dict[str, Part]:
126
+ named: dict[str, Part] = {}
127
+ for part in parts:
128
+ named.setdefault(part.name or "", part)
129
+ named.pop("", None)
130
+ return named
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class RequestBody(ABC):
135
+ """A request's body (module docstring): the bytes as sent and the label they came under."""
136
+
137
+ raw: bytes
138
+ content_type: str = ""
139
+ media: ClassVar[tuple[str, ...]] = ()
140
+ uploads: ClassVar[bool] = False
141
+
142
+ @staticmethod
143
+ def of(raw: bytes, headers: Mapping[str, str]) -> RequestBody:
144
+ """The kind ``headers``' content type names, a new one for every request: the one
145
+ place a body's kind is decided. An empty body is ``{}`` whatever its label."""
146
+ content_type = headers.get("content-type") or ""
147
+ media = content_type.partition(";")[0].strip().lower()
148
+ labelled = KINDS.get(media) or (UploadBody if media and not media.endswith("+json") else JsonBody)
149
+ return (labelled if raw else JsonBody)(raw, content_type)
150
+
151
+ @property
152
+ @abstractmethod
153
+ def fields(self) -> Any:
154
+ """What a route reads (module docstring), or None when the body cannot be read so."""
155
+
156
+ @property
157
+ def files(self) -> dict[str, Upload]:
158
+ """The files the body carried, by the field or position that names each."""
159
+ return {}
160
+
161
+ def fields_for(self, handler: bool) -> Any:
162
+ """What one route reads: :attr:`fields`, or for a handler route ``{}`` beside an
163
+ upload's bytes; None when the route cannot read it."""
164
+ return {} if self.fields is None and self.uploads and handler else self.fields
165
+
166
+
167
+ @dataclass(frozen=True)
168
+ class JsonBody(RequestBody):
169
+ media = ("application/json",)
170
+
171
+ @cached_property
172
+ def fields(self) -> Any:
173
+ return json_value(self.raw)
174
+
175
+
176
+ @dataclass(frozen=True)
177
+ class UploadBody(RequestBody):
178
+ uploads = True
179
+
180
+ @cached_property
181
+ def fields(self) -> Any:
182
+ return json_value(self.raw)
183
+
184
+
185
+ @dataclass(frozen=True)
186
+ class FormBody(RequestBody):
187
+ """A form, or JSON under a form's label: no form field starts with ``{`` or ``[``,
188
+ and some clients label every POST a form."""
189
+
190
+ media = ("application/x-www-form-urlencoded",)
191
+
192
+ @cached_property
193
+ def fields(self) -> Any:
194
+ return json_value(self.raw) if self.raw.lstrip().startswith((b"{", b"[")) else form_fields(self.raw)
195
+
196
+
197
+ @dataclass(frozen=True)
198
+ class MultipartFormBody(RequestBody):
199
+ media = ("multipart/form-data",)
200
+
201
+ @cached_property
202
+ def named(self) -> dict[str, Part] | None:
203
+ parts = multipart_parts(self.raw, self.content_type)
204
+ return None if parts is None else _first_by_name(parts)
205
+
206
+ @cached_property
207
+ def fields(self) -> Any:
208
+ return None if self.named is None else {name: part.field() for name, part in self.named.items()}
209
+
210
+ @property
211
+ def files(self) -> dict[str, Upload]:
212
+ return {name: part.upload() for name, part in (self.named or {}).items() if part.is_file}
213
+
214
+ def fields_for(self, handler: bool) -> Any:
215
+ """A form with a file reaches a handler alone: a declared route reads no file."""
216
+ return None if self.files and not handler else self.fields
217
+
218
+
219
+ @dataclass(frozen=True)
220
+ class MultipartRelatedBody(RequestBody):
221
+ media = ("multipart/related",)
222
+
223
+ @cached_property
224
+ def parts(self) -> list[Part]:
225
+ return multipart_parts(self.raw, self.content_type) or []
226
+
227
+ @cached_property
228
+ def fields(self) -> Any:
229
+ return json_value(self.parts[0].content) if self.parts else None
230
+
231
+ @property
232
+ def files(self) -> dict[str, Upload]:
233
+ return {str(index): part.upload() for index, part in enumerate(self.parts[1:], start=1)}
234
+
235
+
236
+ KINDS: dict[str, type[RequestBody]] = {
237
+ media: kind for kind in (JsonBody, FormBody, MultipartFormBody, MultipartRelatedBody) for media in kind.media
238
+ }