@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,81 @@
1
+ """Read-only SQL over a schema world's rows, through its world database's guard."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import math
7
+ import tempfile
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from .compiler import check_bundle
12
+ from .data import KIND_SCHEMA, QUERY_DEFAULT_LIMIT, QUERY_LIMIT, schema_store, world_kind
13
+ from .database import QueryBudget, WorldDatabase
14
+ from .validation import fail
15
+
16
+
17
+ #: What one caller's statement may cost: 10 s, a value of 4 MiB, an answer of 16 MiB as JSON.
18
+ SQL_BUDGET = QueryBudget(seconds=10.0, value_bytes=4 * 1024 * 1024, answer_bytes=16 * 1024 * 1024)
19
+
20
+
21
+ class WorldSql:
22
+ """One read-only statement against the rows a schema world holds: its current
23
+ snapshot, or its data files applied in scratch, as ``worlds data query`` reads
24
+ them. The statement runs on a private copy that holds no bookkeeping or private
25
+ runtime rows (``WorldDatabase.public_copy``), so the guard refusing writes,
26
+ ATTACH and unsafe functions is not the only thing between it and provenance; it
27
+ runs under :data:`SQL_BUDGET` (``WorldDatabase.bounded``): time, value size and the bytes of
28
+ the whole answer as JSON; the platform's bridge also caps the process's memory."""
29
+
30
+ def __init__(self, root: str | Path, scenario: str | None = None) -> None:
31
+ self.root = Path(root)
32
+ self.scenario = scenario
33
+
34
+ def run(self, sql: str, limit: int = QUERY_DEFAULT_LIMIT) -> dict:
35
+ """``{engine, columns, rows, count, limit, truncated}``: at most ``limit`` rows
36
+ (capped at :data:`QUERY_LIMIT`), each a list of values in column order."""
37
+ if not isinstance(sql, str) or not sql.strip():
38
+ fail("input", "sql must be one non-empty SQL statement", "/sql")
39
+ if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
40
+ fail("input", "limit must be a positive integer", "/limit")
41
+ limit = min(limit, QUERY_LIMIT)
42
+ kind = world_kind(self.root)
43
+ if kind != KIND_SCHEMA:
44
+ fail(
45
+ "kind",
46
+ f"worlds data sql reads a schema world's database; this world is {kind}. "
47
+ "A db/schema.sql world is a SQLite file: read it with sqlite3",
48
+ )
49
+ contract = check_bundle(self.root)
50
+ engine = WorldDatabase.for_contract(contract)
51
+ statements = engine.split_statements(sql)
52
+ if len(statements) != 1:
53
+ fail("input", f"Send exactly one SQL statement; this text has {len(statements)}", "/sql")
54
+ with tempfile.TemporaryDirectory(prefix="gateway-sql-") as scratch:
55
+ with schema_store(self.root, contract, self.scenario) as store:
56
+ readable = store.public_copy(Path(scratch) / engine.snapshot_name)
57
+ with readable, readable.bounded(SQL_BUDGET):
58
+ rows = self._read(readable, statements[0], limit + 1)
59
+ return {
60
+ "engine": engine.engine,
61
+ "columns": list(rows.columns),
62
+ "rows": [[_json_value(value) for value in row] for row in rows[:limit]],
63
+ "count": min(len(rows), limit),
64
+ "limit": limit,
65
+ "truncated": len(rows) > limit,
66
+ }
67
+
68
+ @staticmethod
69
+ def _read(readable: WorldDatabase, statement: str, limit: int) -> Any:
70
+ try:
71
+ return readable.query(statement, limit=limit)
72
+ except MemoryError:
73
+ fail("size", "The statement needs more memory than one read may take; select fewer or smaller columns")
74
+
75
+
76
+ def _json_value(value: Any) -> Any:
77
+ if isinstance(value, (bytes, bytearray, memoryview)):
78
+ return {"base64": base64.b64encode(bytes(value)).decode("ascii")}
79
+ if isinstance(value, float) and not math.isfinite(value):
80
+ return str(value)
81
+ return value
@@ -13,7 +13,8 @@ name order, then the pytest files, so a case sees what the cases before it
13
13
  wrote. A world with no ``tests/`` directory passes with nothing run: tests are
14
14
  optional, and "none" is a fact the report states, not a failure. The exception
15
15
  is a world whose ``gateway-env.toml`` declares ``[actions] on_push = ["tests"]``
16
- and ships no case: it promised a suite, so a missing or empty ``tests/`` FAILS.
16
+ and ships no case, or none that ran (every pytest case skipped, none collected):
17
+ it promised a suite, so that FAILS. A skipped test never counts as passed.
17
18
 
18
19
  Three case shapes, all discovered from ``tests/``:
19
20
 
@@ -48,7 +49,12 @@ Three case shapes, all discovered from ``tests/``:
48
49
 
49
50
  * ``test_*.py`` / ``*_test.py``: pytest, with ``WORLD_DIR`` (the world), ``WORLD_URL`` (the
50
51
  local server: the session verbs under ``/__session/``, the vendor routes when the world
51
- has them) and ``WORLD_API_KEY`` in the environment. They run under the first interpreter
52
+ has them), ``WORLD_API_KEY``, ``WORLD_ADMIN_TOKEN`` (the session's key to the runner's
53
+ ``/__world/*`` routes, sent as ``X-World-Admin-Token``) and, for a world that declares
54
+ ``[ui]``, ``WORLD_UI_URL`` (the page on its own origin at its entry, its ``api_path``
55
+ answered beside it with the world's credential, the layout the World Host gives a session's
56
+ ui; no token in it) in the environment. There is no browser beside them on the platform, so a
57
+ test there checks the page over HTTP. They run under the first interpreter
52
58
  that has pytest — this one, then ``$VIRTUAL_ENV``, a ``.venv`` beside the world or in the
53
59
  working directory, then ``python3`` on PATH — and the report names which. Python tests
54
60
  present with no pytest anywhere is a FAILURE naming the interpreter and the install
@@ -57,8 +63,8 @@ Three case shapes, all discovered from ``tests/``:
57
63
 
58
64
  from __future__ import annotations
59
65
 
60
- import json
61
66
  import importlib.util
67
+ import json
62
68
  import os
63
69
  import shutil
64
70
  import subprocess
@@ -67,33 +73,33 @@ import tempfile
67
73
  import urllib.error
68
74
  import urllib.parse
69
75
  import urllib.request
70
- from dataclasses import asdict, dataclass, field
76
+ from dataclasses import asdict, dataclass, field, replace
71
77
  from pathlib import Path
72
78
  from typing import Any
73
79
 
74
80
  from .._toml import tomllib
81
+ from ..world_session_contract import ADMIN_TOKEN_ENV
75
82
  from .api import load_api_settings
76
83
  from .conform import PIN, _credentials
77
84
  from .connector import load_spec
78
85
  from .local import LocalSession, open_world
79
86
  from .validation import fail
87
+ from .workers import INHERITED_ENV
80
88
 
81
89
  TESTS_DIR = "tests"
82
90
  MANIFEST = "gateway-env.toml"
83
91
  CASE_TIMEOUT_S = 30
84
92
  PYTEST_TIMEOUT_S = 600
85
93
  PROBE_TIMEOUT_S = 30
94
+ PYTEST_NO_TESTS_COLLECTED = 5
86
95
 
87
96
 
88
97
  @dataclass
89
98
  class CaseResult:
90
99
  name: str
91
100
  file: str
92
- status: str # passed | failed | error
93
- # What the case is for and where it belongs, as the author wrote them: the
94
- # Tests tab groups a report by `group` (a route family, a workflow step)
95
- # and shows `description` under the name, so a reader sees what is being
96
- # asserted and why, not only a status per line.
101
+ status: str
102
+ # The Tests tab groups a report by `group` and shows `description` under each case.
97
103
  description: str = ""
98
104
  group: str = ""
99
105
  request: dict = field(default_factory=dict)
@@ -104,7 +110,7 @@ class CaseResult:
104
110
 
105
111
  @dataclass
106
112
  class PythonResult:
107
- status: str # passed | failed | none
113
+ status: str
108
114
  files: list[str] = field(default_factory=list)
109
115
  tests: int = 0
110
116
  failures: int = 0
@@ -112,9 +118,12 @@ class PythonResult:
112
118
  skipped: int = 0
113
119
  reason: str | None = None
114
120
  output: str = ""
115
- #: The interpreter pytest ran under, when it ran.
116
121
  interpreter: str | None = None
117
122
 
123
+ @property
124
+ def ran_nothing(self) -> bool:
125
+ return self.status != "failed" and self.tests == self.skipped
126
+
118
127
 
119
128
  @dataclass
120
129
  class Report:
@@ -130,7 +139,7 @@ class Report:
130
139
  if self.python.status == "failed":
131
140
  failed += max(1, self.python.failures + self.python.errors)
132
141
  if self.python.status == "passed":
133
- passed += self.python.tests
142
+ passed += self.python.tests - self.python.skipped
134
143
  return {
135
144
  "total": passed + failed,
136
145
  "passed": passed,
@@ -149,9 +158,6 @@ class Report:
149
158
  }
150
159
 
151
160
 
152
- # ---------------------------------------------------------------- discovery
153
-
154
-
155
161
  EXPECT_KEYS = {"status", "json", "length", "contains", "headers"}
156
162
 
157
163
 
@@ -178,11 +184,13 @@ def discover(root: str | Path) -> Discovered:
178
184
  fail("tests", f"{file}: not valid JSON ({error})", f"/tests/{path.name}")
179
185
  listed = document.get("cases") if isinstance(document, dict) else document
180
186
  if not isinstance(listed, list):
181
- fail("tests", f"{file}: expected a list of cases or {{\"cases\": [...]}}", f"/tests/{path.name}")
187
+ fail("tests", f'{file}: expected a list of cases or {{"cases": [...]}}', f"/tests/{path.name}")
182
188
  named = document.get("scenario") if isinstance(document, dict) else None
183
189
  if named is not None:
184
190
  if not isinstance(named, str) or not named:
185
- fail("tests", f"{file}: scenario must be the name of one of the world's scenarios", f"/tests/{path.name}")
191
+ fail(
192
+ "tests", f"{file}: scenario must be the name of one of the world's scenarios", f"/tests/{path.name}"
193
+ )
186
194
  if scenario is not None and scenario[1] != named:
187
195
  fail(
188
196
  "tests",
@@ -194,13 +202,25 @@ def discover(root: str | Path) -> Discovered:
194
202
  for index, case in enumerate(listed):
195
203
  where = f"{file}[{index}]"
196
204
  if not isinstance(case, dict) or not isinstance(case.get("expect"), dict):
197
- fail("tests", f"{where}: a case is {{name, request, expect}} or {{name, tool, args, expect}}", f"/tests/{path.name}")
205
+ fail(
206
+ "tests",
207
+ f"{where}: a case is {{name, request, expect}} or {{name, tool, args, expect}}",
208
+ f"/tests/{path.name}",
209
+ )
198
210
  unknown = sorted(set(case["expect"]) - EXPECT_KEYS)
199
211
  if unknown:
200
- fail("tests", f"{where}: expect has no {', '.join(unknown)}; it takes {', '.join(sorted(EXPECT_KEYS))}", f"/tests/{path.name}")
212
+ fail(
213
+ "tests",
214
+ f"{where}: expect has no {', '.join(unknown)}; it takes {', '.join(sorted(EXPECT_KEYS))}",
215
+ f"/tests/{path.name}",
216
+ )
201
217
  http, tool = isinstance(case.get("request"), dict), isinstance(case.get("tool"), str)
202
218
  if http == tool:
203
- fail("tests", f"{where}: a case has exactly one of request (an HTTP call) or tool (a tool call)", f"/tests/{path.name}")
219
+ fail(
220
+ "tests",
221
+ f"{where}: a case has exactly one of request (an HTTP call) or tool (a tool call)",
222
+ f"/tests/{path.name}",
223
+ )
204
224
  if tool and case.get("args") is not None and not isinstance(case.get("args"), dict):
205
225
  fail("tests", f"{where}: args must be an object of the tool's arguments", f"/tests/{path.name}")
206
226
  cases.append((file, case))
@@ -223,9 +243,6 @@ def declares_on_push_tests(root: str | Path) -> bool:
223
243
  return isinstance(on_push, list) and "tests" in on_push
224
244
 
225
245
 
226
- # ---------------------------------------------------------------- predicates
227
-
228
-
229
246
  def _subset(expected: Any, actual: Any, at: str, problems: list[str]) -> None:
230
247
  if isinstance(expected, dict):
231
248
  if not isinstance(actual, dict):
@@ -305,9 +322,6 @@ def check(expect: dict, status: int, headers: dict[str, str], raw: bytes) -> lis
305
322
  return problems
306
323
 
307
324
 
308
- # ---------------------------------------------------------------- the run
309
-
310
-
311
325
  def run(world: str | Path, *, scenario: str | None = None) -> Report:
312
326
  """Open the world as one local session and run every test it ships."""
313
327
  root = Path(world)
@@ -315,8 +329,6 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
315
329
  cases, python_files = found.cases, found.python
316
330
  if not cases and not python_files:
317
331
  if declares_on_push_tests(root):
318
- # The manifest promised a suite on every push; shipping none is the
319
- # defect the on-push run exists to catch, so it fails here too.
320
332
  where = "an empty tests/ directory" if (root / TESTS_DIR).is_dir() else "no tests/ directory"
321
333
  reason = (
322
334
  f'{MANIFEST} declares [actions] on_push = ["tests"] but the world ships {where}; '
@@ -346,12 +358,21 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
346
358
  results.append(_run_tool_case(session, file, case))
347
359
  else:
348
360
  results.append(_run_case(base, prefix, spec, credentials, file, case))
349
- python = _run_python(root, python_files, base)
361
+ python = _run_python(root, python_files, session)
350
362
  finally:
351
363
  session.close()
364
+ if not results and python.ran_nothing and declares_on_push_tests(root):
365
+ python = _unkept_promise(python)
352
366
  return Report(str(root), results, python, scenario)
353
367
 
354
368
 
369
+ def _unkept_promise(python: PythonResult) -> PythonResult:
370
+ why = python.reason or f"{python.skipped} skipped"
371
+ return replace(
372
+ python, status="failed", reason=f'{MANIFEST} declares [actions] on_push = ["tests"] but no test ran ({why})'
373
+ )
374
+
375
+
355
376
  def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
356
377
  tool = case["tool"]
357
378
  args = case.get("args") or {}
@@ -365,8 +386,6 @@ def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
365
386
  request={"tool": tool, "args": args},
366
387
  expected=expect,
367
388
  )
368
- # A refusal (an unknown tool, arguments off the schema) is the session's 400
369
- # answer, and a case may expect it.
370
389
  answer = session.call(tool, args)
371
390
  status = int(answer.get("status", 0))
372
391
  raw = json.dumps(answer.get("body")).encode("utf-8")
@@ -392,9 +411,7 @@ def _run_case(base: str, prefix: str, spec, credentials: dict[str, str], file: s
392
411
  method = str(request.get("method") or "GET").upper()
393
412
  path = str(request.get("path") or "/")
394
413
  if prefix and not path.startswith(prefix + "/") and path != prefix and not path.startswith("/__"):
395
- # a case may name the vendor path bare, or with the base url's prefix;
396
- # the world's own routes (/__session/, /__ui/, the ui's api_path) are
397
- # never under the vendor's prefix
414
+ # the world's own /__ routes (session, ui, the ui's api_path) are never under the vendor's prefix
398
415
  path = prefix + (path if path.startswith("/") else "/" + path)
399
416
  query = dict(request.get("query") or {})
400
417
  if request.get("auth", True) and spec.scheme == "query":
@@ -452,7 +469,7 @@ def has_pytest(interpreter: str) -> bool:
452
469
  return probe.returncode == 0
453
470
 
454
471
 
455
- def _run_python(root: Path, files: list[Path], base: str) -> PythonResult:
472
+ def _run_python(root: Path, files: list[Path], session: LocalSession) -> PythonResult:
456
473
  if not files:
457
474
  return PythonResult(status="none")
458
475
  names = [str(p.relative_to(root)) for p in files]
@@ -472,12 +489,15 @@ def _run_python(root: Path, files: list[Path], base: str) -> PythonResult:
472
489
  with tempfile.TemporaryDirectory(prefix="gateway-world-tests-") as scratch:
473
490
  junit = Path(scratch) / "junit.xml"
474
491
  env = {
475
- **{k: v for k, v in os.environ.items() if k in ("PATH", "LANG", "LC_ALL", "TZ", "HOME", "PYTHONPATH", "VIRTUAL_ENV")},
492
+ **{name: os.environ[name] for name in (*INHERITED_ENV, "HOME") if name in os.environ},
476
493
  "WORLD_DIR": str(root),
477
- "WORLD_URL": base,
494
+ "WORLD_URL": session.base_url or "",
478
495
  "WORLD_API_KEY": PIN,
496
+ ADMIN_TOKEN_ENV: session.admin.value or "",
479
497
  "PYTHONDONTWRITEBYTECODE": "1",
480
498
  }
499
+ if session.ui_url is not None:
500
+ env["WORLD_UI_URL"] = session.ui_url
481
501
  try:
482
502
  proc = subprocess.run(
483
503
  [interpreter, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--junitxml={junit}", *names],
@@ -496,9 +516,20 @@ def _run_python(root: Path, files: list[Path], base: str) -> PythonResult:
496
516
  reason = "pytest wrote no junit report"
497
517
  return PythonResult(status="failed", files=names, reason=reason, output=output, interpreter=interpreter)
498
518
  tests, failures, errors, skipped = counts
499
- if proc.returncode == 5: # pytest: no tests collected
519
+ if proc.returncode == PYTEST_NO_TESTS_COLLECTED:
500
520
  reason = "no tests collected"
501
521
  return PythonResult(status="none", files=names, reason=reason, output=output, interpreter=interpreter)
522
+ if proc.returncode == 0 and tests == skipped:
523
+ reason = f"{skipped} skipped"
524
+ return PythonResult(
525
+ status="skipped",
526
+ files=names,
527
+ tests=tests,
528
+ skipped=skipped,
529
+ reason=reason,
530
+ output=output,
531
+ interpreter=interpreter,
532
+ )
502
533
  status = "passed" if proc.returncode == 0 else "failed"
503
534
  return PythonResult(
504
535
  status=status,
@@ -527,4 +558,5 @@ def _junit_counts(path: Path) -> tuple[int, int, int, int] | None:
527
558
  failures += int(suite.get("failures") or 0)
528
559
  errors += int(suite.get("errors") or 0)
529
560
  skipped += int(suite.get("skipped") or 0)
530
- return total, failures, errors, skipped
561
+ xfailed = sum(1 for marker in tree.iter("skipped") if marker.get("type") == "pytest.xfail")
562
+ return total, failures, errors, skipped - xfailed
@@ -0,0 +1,57 @@
1
+ """The one score rule every grader answers by: a finite number from 0 to 1.
2
+
3
+ A passed check scores 1, a failed one 0, a fraction is partial credit and
4
+ ``None`` is ungraded. Anything else (1.7, -0.5, NaN, Infinity) is not a score:
5
+ a reader refuses it by key and value instead of filing it, since two
6
+ out-of-range checks can average to a perfect 1.0 and NaN has no JSON form the
7
+ platform can read. Stdlib only, inside ``gatewaysdk.environment`` because that
8
+ package is vendored on its own (the World Host, the batch runner). The
9
+ TypeScript twin is ``sdk-ts/src/worlds/grade.ts``; both are pinned by
10
+ ``gatewaysdk/fixtures/tasks/grade-vectors.json``.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import json
16
+ import math
17
+ from collections.abc import Iterable
18
+ from typing import Any, Tuple
19
+
20
+
21
+ class Score:
22
+ RULE = "a score is a number from 0 to 1"
23
+ LOW = 0.0
24
+ HIGH = 1.0
25
+
26
+ @staticmethod
27
+ def finite(value: float) -> bool:
28
+ """Whether a float holds it: not NaN or Infinity, and not an integer past
29
+ float range, which ``math.isfinite`` cannot even convert."""
30
+ try:
31
+ return math.isfinite(value)
32
+ except OverflowError:
33
+ return False
34
+
35
+ @staticmethod
36
+ def admits(value: float) -> bool:
37
+ return Score.finite(value) and Score.LOW <= value <= Score.HIGH
38
+
39
+ @staticmethod
40
+ def shown(value: float) -> str:
41
+ """The value as a grader's JSON prints it: ``1.7``, ``NaN``, ``Infinity``."""
42
+ return json.dumps(value)
43
+
44
+ @staticmethod
45
+ def problem(scores: Iterable[Tuple[str, Any]]) -> str | None:
46
+ """Every number among ``scores`` that is not a score, as ``'<key>' = <value>``
47
+ and the rule, or ``None`` when all of them are. Pairs, not a mapping, so
48
+ that no key hides another. A boolean is the 1 or 0 it claims; a string or
49
+ ``None`` is not a number and is not judged here."""
50
+ refused = [
51
+ f"{key!r} = {Score.shown(value)}"
52
+ for key, value in scores
53
+ if isinstance(value, (int, float)) and not Score.admits(value)
54
+ ]
55
+ if not refused:
56
+ return None
57
+ return f"{', '.join(refused)}; {Score.RULE}"
@@ -0,0 +1,36 @@
1
+ """What a secret looks like on its own, whatever field it sits in.
2
+
3
+ ``SECRET_VALUE`` is the key shapes the browser recorder
4
+ (``world_browser.redact_action_args``, twinned by the TypeScript recorders) and
5
+ the world call log (``schema.call_log.CallRedaction``) both redact: a provider
6
+ key by its prefix, an AWS access key id, a bearer header. ``SECRET_TOKEN`` and
7
+ :func:`basic_credentials` add what the call log also redacts: a JSON Web Token,
8
+ a PEM private key, GitHub fine-grained and app tokens, a Google API key, Stripe
9
+ restricted keys and webhook secrets, GitLab and npm tokens, an AWS temporary
10
+ key id, Slack app and refresh tokens, and a Basic authorization value.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import base64
16
+ import re
17
+
18
+ SECRET_VALUE = re.compile(r"^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})")
19
+ SECRET_TOKEN = re.compile(
20
+ r"^(?:eyJ[-_A-Za-z0-9]{8,}\.eyJ[-_A-Za-z0-9]{8,}\.[-_A-Za-z0-9]+$"
21
+ r"|github_pat_\w{20,}|gh[ousr]_[A-Za-z0-9]{20,}|AIza[-\w]{35}|rk_(?:live|test)_\w{8,}|whsec_\w{8,}"
22
+ r"|glpat-[-\w]{20,}|npm_[A-Za-z0-9]{36}|ASIA[0-9A-Z]{16}|xapp-[-\w]{20,}|xoxe\.[-.\w]{20,})"
23
+ r"|-----BEGIN [A-Z ]*PRIVATE KEY-----"
24
+ )
25
+ _BASIC = re.compile(r"^[Bb]asic ((?:[A-Za-z0-9+/]{4})+(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?)$")
26
+
27
+
28
+ def basic_credentials(value: str) -> bool:
29
+ """Whether ``value`` is a Basic authorization value: base64 of ``user:password``."""
30
+ found = _BASIC.match(value)
31
+ if not found:
32
+ return False
33
+ try:
34
+ return ":" in base64.b64decode(found.group(1), validate=True).decode("utf-8")
35
+ except ValueError:
36
+ return False
@@ -23,14 +23,17 @@ from dataclasses import dataclass
23
23
  from functools import cached_property
24
24
  from pathlib import Path
25
25
  from typing import ClassVar
26
+
26
27
  from ._toml import tomllib
28
+ from .errors import WorldStartError
29
+ from .world_session_contract import API_BUNDLE_RUN_REFUSAL
27
30
 
28
31
  try: # optional: only consulted when a tree ships a root .gitignore
29
32
  from gitignore_parser import parse_gitignore
30
33
  except ImportError: # pragma: no cover - the runner and the sandbox vendor this module alone
31
34
  parse_gitignore = None # type: ignore[assignment]
32
35
 
33
- # ── Markers (twin of packages/shared/src/world/markers.ts) ──────────────────
36
+ # Twin of packages/shared/src/world/markers.ts.
34
37
 
35
38
  WORLD_MANIFEST = "gateway-env.toml"
36
39
  API_BUNDLE_MANIFEST = "gateway-api-world.toml"
@@ -41,6 +44,7 @@ BENCHMARK_CONFIG = "benchmark.json"
41
44
  WORLD_SCHEMA_MARKERS = ("schema/world.json", "schema/runtime.json", "schema/lock.json")
42
45
  WORLD_ROOT_MARKERS = (CONNECTOR_SPEC, *WORLD_SCHEMA_MARKERS)
43
46
  SCHEMA_MANIFEST_KIND = "schema"
47
+ LEGACY_DB_ENGINE = "sqlite"
44
48
  SCHEMA_MANIFEST_KINDS = frozenset({SCHEMA_MANIFEST_KIND, "mock-tools"})
45
49
 
46
50
  KIND_MOCK_TOOLS = "mock-tools"
@@ -229,8 +233,6 @@ class WorldTree:
229
233
  def has_connector(self) -> bool:
230
234
  return self.has(self.world_path(CONNECTOR_SPEC))
231
235
 
232
- # ── The archive: the file set a version IS ───────────────────────────────
233
- #
234
236
  # Vendored from Prime Intellect's ``prime`` CLI (MIT; THIRD_PARTY_LICENSES)
235
237
  # so a container hashed here resolves to the same logical version on their
236
238
  # Environments Hub: SHA-256 over ``file:<relpath>`` + raw bytes, in sorted
@@ -445,6 +447,34 @@ class World:
445
447
  def is_schema(self) -> bool:
446
448
  return self.schema_mode == "schema"
447
449
 
450
+ @property
451
+ def agent_run_refusal(self) -> str | None:
452
+ """Why the benchmark runner's agent cannot run this tree, or None."""
453
+ return None
454
+
455
+ def db_engine(self, manifest: dict) -> str:
456
+ """The engine the tree's state runs on. Raw SQL in ``db/schema.sql`` and
457
+ ``db/seed.sql`` is SQLite's, so a contract-less world names no other."""
458
+ engine = self._declared_db_engine(manifest)
459
+ if engine != LEGACY_DB_ENGINE:
460
+ raise WorldStartError(
461
+ f"db engine {engine!r} is not available to a db/schema.sql world, which runs on "
462
+ f"{LEGACY_DB_ENGINE}; a schema world chooses its engine (`gateway worlds schema init`)."
463
+ )
464
+ return engine
465
+
466
+ @staticmethod
467
+ def _declared_db_engine(manifest: dict) -> str:
468
+ env = next(
469
+ (manifest[key] for key in ("environment", "env") if isinstance(manifest.get(key), dict)),
470
+ {},
471
+ )
472
+ db = manifest.get("db")
473
+ if isinstance(db, str):
474
+ return db
475
+ named = db.get("engine") if isinstance(db, dict) else None
476
+ return named or env.get("db") or LEGACY_DB_ENGINE
477
+
448
478
 
449
479
  class SchemaWorld(World):
450
480
  """A gateway-world/1 contract tree stored as itself."""
@@ -456,6 +486,11 @@ class SchemaWorld(World):
456
486
  def reported_kind(self) -> str:
457
487
  return SCHEMA_MANIFEST_KIND
458
488
 
489
+ def db_engine(self, manifest: dict) -> str:
490
+ from .schema.compiler import storage_engine
491
+
492
+ return storage_engine(manifest).engine
493
+
459
494
  @property
460
495
  def tools_engine(self) -> bool:
461
496
  return True
@@ -481,6 +516,10 @@ class ApiBundle(World):
481
516
  def task_names(self) -> list[str]:
482
517
  return self.tree.scenario_names()
483
518
 
519
+ @property
520
+ def agent_run_refusal(self) -> str:
521
+ return API_BUNDLE_RUN_REFUSAL
522
+
484
523
 
485
524
  class ContainerBench(World):
486
525
  @property
@@ -35,6 +35,37 @@ EXPLORATORY_TASK_SOURCE = "none"
35
35
  #: ``reward: null``.
36
36
  NO_GRADER_REASON = "no grader"
37
37
 
38
+ #: Where a world's server and its runner find the token for the runner's
39
+ #: ``/__world/*`` routes: set by the platform on a session pod's two
40
+ #: containers, by the Batch runner on a world it boots.
41
+ ADMIN_TOKEN_ENV = "WORLD_ADMIN_TOKEN"
42
+
43
+ #: The code and words of a call by a route's name: a world's routes answer
44
+ #: over its HTTP api, never through the tool door.
45
+ NOT_A_TOOL = "not_a_tool"
46
+ ROUTE_NOT_A_TOOL = "'{name}' is a route of this world, not a tool; call it over the session's api URL"
47
+
48
+
49
+ def route_not_a_tool(name: str) -> str:
50
+ return ROUTE_NOT_A_TOOL.format(name=name)
51
+
52
+
53
+ #: Why the benchmark runner's agent refuses an api bundle: it serves no tools.
54
+ API_BUNDLE_RUN_REFUSAL = (
55
+ "an api bundle serves no tools: its routes answer over its HTTP api and are never an agent's tools, so "
56
+ "the benchmark runner's agent has nothing to call. Migrate it to a schema world (`gateway worlds migrate "
57
+ "<slug> --apply`) and declare its tools in x-gateway.tools, or drive its routes over HTTP from your own "
58
+ "agent on a session (`gateway worlds session open <slug> --surface api`)"
59
+ )
60
+
61
+
62
+ def still_starting_hint(session_id: str) -> str:
63
+ """What ``session open`` adds when its wait ran out: the session keeps opening."""
64
+ return (
65
+ f"It keeps opening: poll `gateway worlds session status {session_id}` until ready, "
66
+ f"or end it with `gateway worlds session close {session_id}`."
67
+ )
68
+
38
69
 
39
70
  def idle_warning(idle_expires_at: str | None, *, now: str | None = None) -> str | None:
40
71
  """The line ``session status`` prints when an unpinned session has under
@@ -42,11 +73,7 @@ def idle_warning(idle_expires_at: str | None, *, now: str | None = None) -> str
42
73
  if not idle_expires_at:
43
74
  return None
44
75
  expires = datetime.fromisoformat(idle_expires_at.replace("Z", "+00:00"))
45
- current = (
46
- datetime.fromisoformat(now.replace("Z", "+00:00"))
47
- if now
48
- else datetime.now(timezone.utc)
49
- )
76
+ current = datetime.fromisoformat(now.replace("Z", "+00:00")) if now else datetime.now(timezone.utc)
50
77
  left_s = (expires - current).total_seconds()
51
78
  if left_s >= IDLE_WARNING_MINUTES * 60:
52
79
  return None
@@ -407,8 +407,8 @@ reward: 0 {"ticket_closed":0}
407
407
  reward is for the untouched world; an agent's rollout is what `benchmarks run` grades. Drive one by hand: …
408
408
  ```
409
409
 
410
- A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with SQLite's
411
- words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
410
+ A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with the database
411
+ engine's words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
412
412
  before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
413
413
  print and the run is marked not executed, exit 0.
414
414