@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -65,9 +65,8 @@ def import_data(request: dict) -> dict:
65
65
  **report,
66
66
  **{key: result[key] for key in ("proof", "deferred") if key in result},
67
67
  }
68
- if result["kind"] == KIND_SCHEMA and snapshots.default_snapshot(world).is_file():
69
- # The journals grew; a snapshot built before this import would be
70
- # served over them. Rebuild so the tree that travels back is one thing.
68
+ if result["kind"] == KIND_SCHEMA and snapshots.has_default_snapshot(world):
69
+ # A snapshot built before this import would be served over the grown journals.
71
70
  from .compiler import build
72
71
  from .snapshot import build_all
73
72
 
@@ -1,6 +1,7 @@
1
- """The bridge's read commands over a world's rows: ``counts`` and ``query``.
1
+ """The bridge's read commands over a world's rows: ``counts``, ``query``, ``sql``
2
+ and ``verify-legacy``.
2
3
 
3
- Both open the world through the ``open_world`` collaborator the dispatcher hands
4
+ Each opens the world through the ``open_world`` collaborator the dispatcher hands
4
5
  them (:func:`platform._world_for`): inline ``files`` (small trees), a
5
6
  ``workdir`` the platform prepared with the tree and any checked-out snapshot
6
7
  already on disk (large trees; nothing travels through the request), or a local
@@ -14,10 +15,16 @@ from __future__ import annotations
14
15
 
15
16
  import tempfile
16
17
  from pathlib import Path
17
- from typing import Callable
18
+ from typing import Any, Callable
18
19
 
19
- from .data import entity_counts, entity_query, entity_query_across, entity_rows, world_kind
20
+ from ..legacy_sqlite import LegacyVerifiers
21
+ from .compiler import bundle_mode
22
+ from .data import QUERY_DEFAULT_LIMIT, entity_counts, entity_query, entity_query_across, entity_rows, world_kind
20
23
  from .validation import ContractError, fail
24
+ from .world_sql import WorldSql
25
+
26
+ #: The memory one `sql` call may add to the bridge process, past what it already holds.
27
+ SQL_MEMORY_BYTES = 512 * 1024 * 1024
21
28
 
22
29
  # (request, staging directory) -> the world root to read; the staging directory
23
30
  # is where inline ``files`` are written and is gone when the command returns.
@@ -51,7 +58,7 @@ def query(request: dict, open_world: WorldOpener) -> dict:
51
58
  a list of values), a page at a time (``limit``, ``offset``). With ``resolve``,
52
59
  each value is plaintext and is looked up in the forms the world's ``[redact]``
53
60
  policy stores (:func:`data.entity_query`). A tree that carries its compiled
54
- ``db/state.sqlite`` (a checked-out snapshot) answers from the snapshot, as
61
+ ``db/<snapshot name>`` (a checked-out snapshot) answers from the snapshot, as
55
62
  ``counts`` does; otherwise from its data files. With ``across`` (local runtime
56
63
  only: ``world`` is a directory here, so its ``[dependencies]`` paths resolve) the
57
64
  answer is :func:`data.entity_query_across`'s groups, one per linked world a
@@ -93,3 +100,52 @@ def query(request: dict, open_world: WorldOpener) -> dict:
93
100
  root, entity, where, **page, resolve=resolve, where_transform=where_transform, scenario=scenario
94
101
  )
95
102
  return {"status": "ok", "kind": world_kind(root), **report}
103
+
104
+
105
+ def sql(request: dict, open_world: WorldOpener) -> dict:
106
+ """``sql``: one read-only statement over a schema world's rows (:class:`WorldSql`),
107
+ from its current snapshot or its data files, optionally as a ``scenario`` starts."""
108
+ scenario = request.get("scenario")
109
+ if scenario is not None and (not isinstance(scenario, str) or not scenario):
110
+ fail("input", "scenario must be a non-empty string", "/scenario")
111
+ with tempfile.TemporaryDirectory(prefix="gateway-sql-") as directory:
112
+ root = open_world(request, Path(directory))
113
+ answer = WorldSql(root, scenario).run(request.get("sql"), request.get("limit", QUERY_DEFAULT_LIMIT))
114
+ return {"status": "ok", "kind": world_kind(root), **answer}
115
+
116
+
117
+ def cap_memory_for(request: Any, status: Path = Path("/proc/self/status"), limits: Any = None) -> None:
118
+ """For a ``sql`` request, cap this one-shot bridge process's address space at what it holds now
119
+ plus :data:`SQL_MEMORY_BYTES`, never raising a soft or hard limit it already has: a caller's
120
+ statement cannot take the host's memory however wide its rows are. Called by the bridge's process
121
+ entry only, never by an in-process ``execute``, because the cap lasts for the life of the process.
122
+ Applies where ``/proc`` exists (Linux: the platform, and a TypeScript CLI's local bridge there);
123
+ ``status`` and ``limits`` (the ``resource`` module) are injectable for tests."""
124
+ if not isinstance(request, dict) or request.get("command") != "sql":
125
+ return
126
+ if not status.is_file():
127
+ return
128
+ if limits is None:
129
+ import resource as limits
130
+ sizes = [int(line.split()[1]) * 1024 for line in status.read_text().splitlines() if line.startswith("VmSize:")]
131
+ if not sizes:
132
+ fail("runtime", "This process cannot read its own memory size, so it refuses to run caller SQL")
133
+ cap = sizes[0] + SQL_MEMORY_BYTES
134
+ soft, hard = limits.getrlimit(limits.RLIMIT_AS)
135
+ hard = _lower(cap, hard, limits.RLIM_INFINITY)
136
+ limits.setrlimit(limits.RLIMIT_AS, (_lower(hard, soft, limits.RLIM_INFINITY), hard))
137
+
138
+
139
+ def _lower(cap: int, limit: int, infinity: int) -> int:
140
+ return cap if limit == infinity else min(cap, limit)
141
+
142
+
143
+ def verify_legacy(request: dict, open_world: WorldOpener) -> dict:
144
+ """``verify-legacy``: a db/schema.sql world's SQL verifiers, each scored on a fresh copy
145
+ of its seeded state (:class:`LegacyVerifiers`). A schema world grades through its
146
+ contract instead (``World.verify``)."""
147
+ with tempfile.TemporaryDirectory(prefix="gateway-verify-") as directory:
148
+ root = open_world(request, Path(directory))
149
+ if bundle_mode(root) is not None:
150
+ fail("kind", "verify-legacy scores a db/schema.sql world; a schema world grades through World.verify")
151
+ return {"status": "ok", "verifiers": LegacyVerifiers(root).scores()}
@@ -0,0 +1,61 @@
1
+ """An entity's primary key, read off a row or a key.
2
+
3
+ Each field of a key is a property name (``id``) or a JSON Pointer to a scalar inside
4
+ the row (``/protocolSection/identificationModule/nctId``). A row carries a nested
5
+ field inside its objects; a key — what ``store.get``, ``update`` and ``delete`` take —
6
+ carries every field flat under its own name. :class:`PrimaryKey` reads both, so the
7
+ store, the data import, the batch lane, ingest, extract, grading and the served API
8
+ agree on what a row's key is."""
9
+
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ from dataclasses import dataclass
14
+ from typing import Any
15
+
16
+ from .compiler import json_pointer_parts
17
+
18
+ _ABSENT = object()
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class PrimaryKey:
23
+ fields: tuple[str, ...]
24
+
25
+ @classmethod
26
+ def of(cls, entity: dict) -> PrimaryKey:
27
+ return cls(tuple(entity.get("primary_key") or ()))
28
+
29
+ @classmethod
30
+ def in_contract(cls, contract: Any, entity: str) -> PrimaryKey:
31
+ return cls.of(contract.entities.get(entity, {}))
32
+
33
+ def missing(self, row: Any) -> str | None:
34
+ """The first key field ``row`` does not carry, or None when it carries them all."""
35
+ if not isinstance(row, dict):
36
+ return self.fields[0] if self.fields else None
37
+ return next((field for field in self.fields if _read(row, field) is _ABSENT), None)
38
+
39
+ def values(self, row: dict) -> list[Any]:
40
+ return [None if (value := _read(row, field)) is _ABSENT else value for field in self.fields]
41
+
42
+ def key(self, row: dict) -> dict[str, Any]:
43
+ """The key a store lookup takes for ``row``: every field under its own name."""
44
+ return dict(zip(self.fields, self.values(row)))
45
+
46
+ def identity(self, row: Any) -> tuple[str, ...] | None:
47
+ """The row's key as comparable text; None without a key or when a field is missing."""
48
+ if not self.fields or self.missing(row) is not None:
49
+ return None
50
+ return tuple(json.dumps(value, sort_keys=True) for value in self.values(row))
51
+
52
+
53
+ def _read(row: dict, field: str) -> Any:
54
+ if field in row:
55
+ return row[field]
56
+ current: Any = row
57
+ for part in json_pointer_parts(field, field, code="key"):
58
+ if not isinstance(current, dict) or part not in current:
59
+ return _ABSENT
60
+ current = current[part]
61
+ return current
@@ -8,7 +8,7 @@ What the rows are proven *against* depends on where the world's data lives, and
8
8
  are the data, so the rows are imported into a scratch copy of the tree through
9
9
  the same gate the real import runs;
10
10
  * :class:`SnapshotProof` — a checkout pinned to a hosted snapshot that carries the
11
- snapshot itself (``db/state.sqlite`` current against the pin): the platform's
11
+ snapshot itself (the default snapshot current against the pin): the platform's
12
12
  batch lane runs over a private copy of those bytes, so references, unique keys
13
13
  and merges are checked against the rows the platform holds;
14
14
  * :class:`ContractProof` — a pinned checkout without its snapshot: only what needs
@@ -26,15 +26,20 @@ import shutil
26
26
  import tempfile
27
27
  from contextlib import contextmanager
28
28
  from pathlib import Path
29
- from typing import Any, Iterable, Iterator
29
+ from typing import TYPE_CHECKING, Any, Iterable, Iterator
30
30
 
31
31
  from .batch import ChunkSpec, build_batch, chunk_rows, validate_chunks
32
32
  from .compiler import compile_bundle
33
+ from .database import WorldDatabase
34
+ from .snapshot import build_names
33
35
  from .validation import fail
34
36
 
37
+ if TYPE_CHECKING:
38
+ from .snapshot import SnapshotPin
39
+
35
40
  Rows = dict[str, list[dict]]
36
41
  #: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
37
- SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
42
+ SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", *build_names())
38
43
  #: What a state-independent proof leaves to the platform batch.
39
44
  DEFERRED = ("references", "unique")
40
45
 
@@ -61,10 +66,10 @@ class RowProof:
61
66
 
62
67
  @classmethod
63
68
  def of(cls, root: str | Path) -> "RowProof":
64
- from .snapshot import current_default, pinned_default
69
+ from .snapshot import SnapshotPin, current_default
65
70
 
66
71
  root = Path(root)
67
- pin = pinned_default(root)
72
+ pin = SnapshotPin.of(root)
68
73
  if pin is None:
69
74
  return FilesProof(root)
70
75
  contract = compile_bundle(root, snapshots=False)
@@ -95,8 +100,9 @@ class FilesProof(RowProof):
95
100
  with proving_copy(self.root) as staged, tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
96
101
  contract = compile_bundle(staged, snapshots=False)
97
102
  build_default(staged, contract)
98
- parent = Path(scratch) / "state.sqlite"
99
- shutil.copyfile(default_snapshot(staged), parent)
103
+ snapshot = default_snapshot(staged, contract)
104
+ parent = Path(scratch) / snapshot.name
105
+ WorldDatabase.clone_file(snapshot, parent)
100
106
  return self._report(_batch(contract, parent, chunks, mode, merge, atomic, "check", EMPTY_DIGEST, redactor))
101
107
 
102
108
  def run(self, rows: Rows, *, mode: str, merge: str, batch_id: str | None, redact: str) -> dict:
@@ -108,7 +114,7 @@ class FilesProof(RowProof):
108
114
 
109
115
 
110
116
  class _PinnedProof(RowProof):
111
- def __init__(self, root: Path, pin: dict, contract: Any):
117
+ def __init__(self, root: Path, pin: SnapshotPin, contract: Any):
112
118
  super().__init__(root)
113
119
  self.pin = pin
114
120
  self.contract = contract
@@ -143,10 +149,11 @@ class SnapshotProof(_PinnedProof):
143
149
  from .snapshot import default_snapshot
144
150
 
145
151
  with tempfile.TemporaryDirectory(prefix="gateway-check-") as scratch:
146
- parent = Path(scratch) / "state.sqlite"
147
- shutil.copyfile(default_snapshot(self.root), parent)
152
+ snapshot = default_snapshot(self.root, self.contract)
153
+ parent = Path(scratch) / snapshot.name
154
+ WorldDatabase.clone_file(snapshot, parent)
148
155
  return self._report(
149
- _batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin["batchDigest"], redactor)
156
+ _batch(self.contract, parent, chunks, mode, merge, atomic, "check", self.pin.batch_digest, redactor)
150
157
  )
151
158
 
152
159
 
@@ -157,7 +164,7 @@ class ContractProof(_PinnedProof):
157
164
  self, chunks: list[ChunkSpec], *, mode: str, merge: str, atomic: bool = True, redactor: Any = None
158
165
  ) -> dict:
159
166
  shaped = validate_chunks(self.contract, chunks, redactor=redactor)
160
- counts = self.pin.get("entityCounts")
167
+ counts = self.pin.fields.get("entityCounts")
161
168
  return self._report(
162
169
  {
163
170
  "rows": shaped["rows"],