@withgateway/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2880 -2032
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2982 -2135
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +180 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +172 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1309 -540
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1287 -532
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +177 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +169 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +69 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +69 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +249 -120
- package/dist/worlds/index.d.ts +249 -120
- package/dist/worlds/index.js +1240 -484
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1231 -476
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -514
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +19 -11
- package/skills/connector-schema-authoring/connector-worlds.md +13 -4
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +14 -14
- package/skills/worlds-getting-started/SKILL.md +12 -12
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"""Read-only SQL over a schema world's rows, through its world database's guard."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import math
|
|
7
|
+
import tempfile
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from .compiler import check_bundle
|
|
12
|
+
from .data import KIND_SCHEMA, QUERY_DEFAULT_LIMIT, QUERY_LIMIT, schema_store, world_kind
|
|
13
|
+
from .database import QueryBudget, WorldDatabase
|
|
14
|
+
from .validation import fail
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
#: What one caller's statement may cost: 10 s, a value of 4 MiB, an answer of 16 MiB as JSON.
|
|
18
|
+
SQL_BUDGET = QueryBudget(seconds=10.0, value_bytes=4 * 1024 * 1024, answer_bytes=16 * 1024 * 1024)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class WorldSql:
|
|
22
|
+
"""One read-only statement against the rows a schema world holds: its current
|
|
23
|
+
snapshot, or its data files applied in scratch, as ``worlds data query`` reads
|
|
24
|
+
them. The statement runs on a private copy that holds no bookkeeping or private
|
|
25
|
+
runtime rows (``WorldDatabase.public_copy``), so the guard refusing writes,
|
|
26
|
+
ATTACH and unsafe functions is not the only thing between it and provenance; it
|
|
27
|
+
runs under :data:`SQL_BUDGET` (``WorldDatabase.bounded``): time, value size and the bytes of
|
|
28
|
+
the whole answer as JSON; the platform's bridge also caps the process's memory."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, root: str | Path, scenario: str | None = None) -> None:
|
|
31
|
+
self.root = Path(root)
|
|
32
|
+
self.scenario = scenario
|
|
33
|
+
|
|
34
|
+
def run(self, sql: str, limit: int = QUERY_DEFAULT_LIMIT) -> dict:
|
|
35
|
+
"""``{engine, columns, rows, count, limit, truncated}``: at most ``limit`` rows
|
|
36
|
+
(capped at :data:`QUERY_LIMIT`), each a list of values in column order."""
|
|
37
|
+
if not isinstance(sql, str) or not sql.strip():
|
|
38
|
+
fail("input", "sql must be one non-empty SQL statement", "/sql")
|
|
39
|
+
if isinstance(limit, bool) or not isinstance(limit, int) or limit < 1:
|
|
40
|
+
fail("input", "limit must be a positive integer", "/limit")
|
|
41
|
+
limit = min(limit, QUERY_LIMIT)
|
|
42
|
+
kind = world_kind(self.root)
|
|
43
|
+
if kind != KIND_SCHEMA:
|
|
44
|
+
fail(
|
|
45
|
+
"kind",
|
|
46
|
+
f"worlds data sql reads a schema world's database; this world is {kind}. "
|
|
47
|
+
"A db/schema.sql world is a SQLite file: read it with sqlite3",
|
|
48
|
+
)
|
|
49
|
+
contract = check_bundle(self.root)
|
|
50
|
+
engine = WorldDatabase.for_contract(contract)
|
|
51
|
+
statements = engine.split_statements(sql)
|
|
52
|
+
if len(statements) != 1:
|
|
53
|
+
fail("input", f"Send exactly one SQL statement; this text has {len(statements)}", "/sql")
|
|
54
|
+
with tempfile.TemporaryDirectory(prefix="gateway-sql-") as scratch:
|
|
55
|
+
with schema_store(self.root, contract, self.scenario) as store:
|
|
56
|
+
readable = store.public_copy(Path(scratch) / engine.snapshot_name)
|
|
57
|
+
with readable, readable.bounded(SQL_BUDGET):
|
|
58
|
+
rows = self._read(readable, statements[0], limit + 1)
|
|
59
|
+
return {
|
|
60
|
+
"engine": engine.engine,
|
|
61
|
+
"columns": list(rows.columns),
|
|
62
|
+
"rows": [[_json_value(value) for value in row] for row in rows[:limit]],
|
|
63
|
+
"count": min(len(rows), limit),
|
|
64
|
+
"limit": limit,
|
|
65
|
+
"truncated": len(rows) > limit,
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
@staticmethod
|
|
69
|
+
def _read(readable: WorldDatabase, statement: str, limit: int) -> Any:
|
|
70
|
+
try:
|
|
71
|
+
return readable.query(statement, limit=limit)
|
|
72
|
+
except MemoryError:
|
|
73
|
+
fail("size", "The statement needs more memory than one read may take; select fewer or smaller columns")
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _json_value(value: Any) -> Any:
|
|
77
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
78
|
+
return {"base64": base64.b64encode(bytes(value)).decode("ascii")}
|
|
79
|
+
if isinstance(value, float) and not math.isfinite(value):
|
|
80
|
+
return str(value)
|
|
81
|
+
return value
|
|
@@ -13,7 +13,8 @@ name order, then the pytest files, so a case sees what the cases before it
|
|
|
13
13
|
wrote. A world with no ``tests/`` directory passes with nothing run: tests are
|
|
14
14
|
optional, and "none" is a fact the report states, not a failure. The exception
|
|
15
15
|
is a world whose ``gateway-env.toml`` declares ``[actions] on_push = ["tests"]``
|
|
16
|
-
and ships no case
|
|
16
|
+
and ships no case, or none that ran (every pytest case skipped, none collected):
|
|
17
|
+
it promised a suite, so that FAILS. A skipped test never counts as passed.
|
|
17
18
|
|
|
18
19
|
Three case shapes, all discovered from ``tests/``:
|
|
19
20
|
|
|
@@ -48,7 +49,12 @@ Three case shapes, all discovered from ``tests/``:
|
|
|
48
49
|
|
|
49
50
|
* ``test_*.py`` / ``*_test.py``: pytest, with ``WORLD_DIR`` (the world), ``WORLD_URL`` (the
|
|
50
51
|
local server: the session verbs under ``/__session/``, the vendor routes when the world
|
|
51
|
-
has them)
|
|
52
|
+
has them), ``WORLD_API_KEY``, ``WORLD_ADMIN_TOKEN`` (the session's key to the runner's
|
|
53
|
+
``/__world/*`` routes, sent as ``X-World-Admin-Token``) and, for a world that declares
|
|
54
|
+
``[ui]``, ``WORLD_UI_URL`` (the page on its own origin at its entry, its ``api_path``
|
|
55
|
+
answered beside it with the world's credential, the layout the World Host gives a session's
|
|
56
|
+
ui; no token in it) in the environment. There is no browser beside them on the platform, so a
|
|
57
|
+
test there checks the page over HTTP. They run under the first interpreter
|
|
52
58
|
that has pytest — this one, then ``$VIRTUAL_ENV``, a ``.venv`` beside the world or in the
|
|
53
59
|
working directory, then ``python3`` on PATH — and the report names which. Python tests
|
|
54
60
|
present with no pytest anywhere is a FAILURE naming the interpreter and the install
|
|
@@ -57,8 +63,8 @@ Three case shapes, all discovered from ``tests/``:
|
|
|
57
63
|
|
|
58
64
|
from __future__ import annotations
|
|
59
65
|
|
|
60
|
-
import json
|
|
61
66
|
import importlib.util
|
|
67
|
+
import json
|
|
62
68
|
import os
|
|
63
69
|
import shutil
|
|
64
70
|
import subprocess
|
|
@@ -67,33 +73,33 @@ import tempfile
|
|
|
67
73
|
import urllib.error
|
|
68
74
|
import urllib.parse
|
|
69
75
|
import urllib.request
|
|
70
|
-
from dataclasses import asdict, dataclass, field
|
|
76
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
71
77
|
from pathlib import Path
|
|
72
78
|
from typing import Any
|
|
73
79
|
|
|
74
80
|
from .._toml import tomllib
|
|
81
|
+
from ..world_session_contract import ADMIN_TOKEN_ENV
|
|
75
82
|
from .api import load_api_settings
|
|
76
83
|
from .conform import PIN, _credentials
|
|
77
84
|
from .connector import load_spec
|
|
78
85
|
from .local import LocalSession, open_world
|
|
79
86
|
from .validation import fail
|
|
87
|
+
from .workers import INHERITED_ENV
|
|
80
88
|
|
|
81
89
|
TESTS_DIR = "tests"
|
|
82
90
|
MANIFEST = "gateway-env.toml"
|
|
83
91
|
CASE_TIMEOUT_S = 30
|
|
84
92
|
PYTEST_TIMEOUT_S = 600
|
|
85
93
|
PROBE_TIMEOUT_S = 30
|
|
94
|
+
PYTEST_NO_TESTS_COLLECTED = 5
|
|
86
95
|
|
|
87
96
|
|
|
88
97
|
@dataclass
|
|
89
98
|
class CaseResult:
|
|
90
99
|
name: str
|
|
91
100
|
file: str
|
|
92
|
-
status: str
|
|
93
|
-
#
|
|
94
|
-
# Tests tab groups a report by `group` (a route family, a workflow step)
|
|
95
|
-
# and shows `description` under the name, so a reader sees what is being
|
|
96
|
-
# asserted and why, not only a status per line.
|
|
101
|
+
status: str
|
|
102
|
+
# The Tests tab groups a report by `group` and shows `description` under each case.
|
|
97
103
|
description: str = ""
|
|
98
104
|
group: str = ""
|
|
99
105
|
request: dict = field(default_factory=dict)
|
|
@@ -104,7 +110,7 @@ class CaseResult:
|
|
|
104
110
|
|
|
105
111
|
@dataclass
|
|
106
112
|
class PythonResult:
|
|
107
|
-
status: str
|
|
113
|
+
status: str
|
|
108
114
|
files: list[str] = field(default_factory=list)
|
|
109
115
|
tests: int = 0
|
|
110
116
|
failures: int = 0
|
|
@@ -112,9 +118,12 @@ class PythonResult:
|
|
|
112
118
|
skipped: int = 0
|
|
113
119
|
reason: str | None = None
|
|
114
120
|
output: str = ""
|
|
115
|
-
#: The interpreter pytest ran under, when it ran.
|
|
116
121
|
interpreter: str | None = None
|
|
117
122
|
|
|
123
|
+
@property
|
|
124
|
+
def ran_nothing(self) -> bool:
|
|
125
|
+
return self.status != "failed" and self.tests == self.skipped
|
|
126
|
+
|
|
118
127
|
|
|
119
128
|
@dataclass
|
|
120
129
|
class Report:
|
|
@@ -130,7 +139,7 @@ class Report:
|
|
|
130
139
|
if self.python.status == "failed":
|
|
131
140
|
failed += max(1, self.python.failures + self.python.errors)
|
|
132
141
|
if self.python.status == "passed":
|
|
133
|
-
passed += self.python.tests
|
|
142
|
+
passed += self.python.tests - self.python.skipped
|
|
134
143
|
return {
|
|
135
144
|
"total": passed + failed,
|
|
136
145
|
"passed": passed,
|
|
@@ -149,9 +158,6 @@ class Report:
|
|
|
149
158
|
}
|
|
150
159
|
|
|
151
160
|
|
|
152
|
-
# ---------------------------------------------------------------- discovery
|
|
153
|
-
|
|
154
|
-
|
|
155
161
|
EXPECT_KEYS = {"status", "json", "length", "contains", "headers"}
|
|
156
162
|
|
|
157
163
|
|
|
@@ -178,11 +184,13 @@ def discover(root: str | Path) -> Discovered:
|
|
|
178
184
|
fail("tests", f"{file}: not valid JSON ({error})", f"/tests/{path.name}")
|
|
179
185
|
listed = document.get("cases") if isinstance(document, dict) else document
|
|
180
186
|
if not isinstance(listed, list):
|
|
181
|
-
fail("tests", f
|
|
187
|
+
fail("tests", f'{file}: expected a list of cases or {{"cases": [...]}}', f"/tests/{path.name}")
|
|
182
188
|
named = document.get("scenario") if isinstance(document, dict) else None
|
|
183
189
|
if named is not None:
|
|
184
190
|
if not isinstance(named, str) or not named:
|
|
185
|
-
fail(
|
|
191
|
+
fail(
|
|
192
|
+
"tests", f"{file}: scenario must be the name of one of the world's scenarios", f"/tests/{path.name}"
|
|
193
|
+
)
|
|
186
194
|
if scenario is not None and scenario[1] != named:
|
|
187
195
|
fail(
|
|
188
196
|
"tests",
|
|
@@ -194,13 +202,25 @@ def discover(root: str | Path) -> Discovered:
|
|
|
194
202
|
for index, case in enumerate(listed):
|
|
195
203
|
where = f"{file}[{index}]"
|
|
196
204
|
if not isinstance(case, dict) or not isinstance(case.get("expect"), dict):
|
|
197
|
-
fail(
|
|
205
|
+
fail(
|
|
206
|
+
"tests",
|
|
207
|
+
f"{where}: a case is {{name, request, expect}} or {{name, tool, args, expect}}",
|
|
208
|
+
f"/tests/{path.name}",
|
|
209
|
+
)
|
|
198
210
|
unknown = sorted(set(case["expect"]) - EXPECT_KEYS)
|
|
199
211
|
if unknown:
|
|
200
|
-
fail(
|
|
212
|
+
fail(
|
|
213
|
+
"tests",
|
|
214
|
+
f"{where}: expect has no {', '.join(unknown)}; it takes {', '.join(sorted(EXPECT_KEYS))}",
|
|
215
|
+
f"/tests/{path.name}",
|
|
216
|
+
)
|
|
201
217
|
http, tool = isinstance(case.get("request"), dict), isinstance(case.get("tool"), str)
|
|
202
218
|
if http == tool:
|
|
203
|
-
fail(
|
|
219
|
+
fail(
|
|
220
|
+
"tests",
|
|
221
|
+
f"{where}: a case has exactly one of request (an HTTP call) or tool (a tool call)",
|
|
222
|
+
f"/tests/{path.name}",
|
|
223
|
+
)
|
|
204
224
|
if tool and case.get("args") is not None and not isinstance(case.get("args"), dict):
|
|
205
225
|
fail("tests", f"{where}: args must be an object of the tool's arguments", f"/tests/{path.name}")
|
|
206
226
|
cases.append((file, case))
|
|
@@ -223,9 +243,6 @@ def declares_on_push_tests(root: str | Path) -> bool:
|
|
|
223
243
|
return isinstance(on_push, list) and "tests" in on_push
|
|
224
244
|
|
|
225
245
|
|
|
226
|
-
# ---------------------------------------------------------------- predicates
|
|
227
|
-
|
|
228
|
-
|
|
229
246
|
def _subset(expected: Any, actual: Any, at: str, problems: list[str]) -> None:
|
|
230
247
|
if isinstance(expected, dict):
|
|
231
248
|
if not isinstance(actual, dict):
|
|
@@ -305,9 +322,6 @@ def check(expect: dict, status: int, headers: dict[str, str], raw: bytes) -> lis
|
|
|
305
322
|
return problems
|
|
306
323
|
|
|
307
324
|
|
|
308
|
-
# ---------------------------------------------------------------- the run
|
|
309
|
-
|
|
310
|
-
|
|
311
325
|
def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
312
326
|
"""Open the world as one local session and run every test it ships."""
|
|
313
327
|
root = Path(world)
|
|
@@ -315,8 +329,6 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
|
315
329
|
cases, python_files = found.cases, found.python
|
|
316
330
|
if not cases and not python_files:
|
|
317
331
|
if declares_on_push_tests(root):
|
|
318
|
-
# The manifest promised a suite on every push; shipping none is the
|
|
319
|
-
# defect the on-push run exists to catch, so it fails here too.
|
|
320
332
|
where = "an empty tests/ directory" if (root / TESTS_DIR).is_dir() else "no tests/ directory"
|
|
321
333
|
reason = (
|
|
322
334
|
f'{MANIFEST} declares [actions] on_push = ["tests"] but the world ships {where}; '
|
|
@@ -346,12 +358,21 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
|
346
358
|
results.append(_run_tool_case(session, file, case))
|
|
347
359
|
else:
|
|
348
360
|
results.append(_run_case(base, prefix, spec, credentials, file, case))
|
|
349
|
-
python = _run_python(root, python_files,
|
|
361
|
+
python = _run_python(root, python_files, session)
|
|
350
362
|
finally:
|
|
351
363
|
session.close()
|
|
364
|
+
if not results and python.ran_nothing and declares_on_push_tests(root):
|
|
365
|
+
python = _unkept_promise(python)
|
|
352
366
|
return Report(str(root), results, python, scenario)
|
|
353
367
|
|
|
354
368
|
|
|
369
|
+
def _unkept_promise(python: PythonResult) -> PythonResult:
|
|
370
|
+
why = python.reason or f"{python.skipped} skipped"
|
|
371
|
+
return replace(
|
|
372
|
+
python, status="failed", reason=f'{MANIFEST} declares [actions] on_push = ["tests"] but no test ran ({why})'
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
355
376
|
def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
|
|
356
377
|
tool = case["tool"]
|
|
357
378
|
args = case.get("args") or {}
|
|
@@ -365,8 +386,6 @@ def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
|
|
|
365
386
|
request={"tool": tool, "args": args},
|
|
366
387
|
expected=expect,
|
|
367
388
|
)
|
|
368
|
-
# A refusal (an unknown tool, arguments off the schema) is the session's 400
|
|
369
|
-
# answer, and a case may expect it.
|
|
370
389
|
answer = session.call(tool, args)
|
|
371
390
|
status = int(answer.get("status", 0))
|
|
372
391
|
raw = json.dumps(answer.get("body")).encode("utf-8")
|
|
@@ -392,9 +411,7 @@ def _run_case(base: str, prefix: str, spec, credentials: dict[str, str], file: s
|
|
|
392
411
|
method = str(request.get("method") or "GET").upper()
|
|
393
412
|
path = str(request.get("path") or "/")
|
|
394
413
|
if prefix and not path.startswith(prefix + "/") and path != prefix and not path.startswith("/__"):
|
|
395
|
-
#
|
|
396
|
-
# the world's own routes (/__session/, /__ui/, the ui's api_path) are
|
|
397
|
-
# never under the vendor's prefix
|
|
414
|
+
# the world's own /__ routes (session, ui, the ui's api_path) are never under the vendor's prefix
|
|
398
415
|
path = prefix + (path if path.startswith("/") else "/" + path)
|
|
399
416
|
query = dict(request.get("query") or {})
|
|
400
417
|
if request.get("auth", True) and spec.scheme == "query":
|
|
@@ -452,7 +469,7 @@ def has_pytest(interpreter: str) -> bool:
|
|
|
452
469
|
return probe.returncode == 0
|
|
453
470
|
|
|
454
471
|
|
|
455
|
-
def _run_python(root: Path, files: list[Path],
|
|
472
|
+
def _run_python(root: Path, files: list[Path], session: LocalSession) -> PythonResult:
|
|
456
473
|
if not files:
|
|
457
474
|
return PythonResult(status="none")
|
|
458
475
|
names = [str(p.relative_to(root)) for p in files]
|
|
@@ -472,12 +489,15 @@ def _run_python(root: Path, files: list[Path], base: str) -> PythonResult:
|
|
|
472
489
|
with tempfile.TemporaryDirectory(prefix="gateway-world-tests-") as scratch:
|
|
473
490
|
junit = Path(scratch) / "junit.xml"
|
|
474
491
|
env = {
|
|
475
|
-
**{
|
|
492
|
+
**{name: os.environ[name] for name in (*INHERITED_ENV, "HOME") if name in os.environ},
|
|
476
493
|
"WORLD_DIR": str(root),
|
|
477
|
-
"WORLD_URL":
|
|
494
|
+
"WORLD_URL": session.base_url or "",
|
|
478
495
|
"WORLD_API_KEY": PIN,
|
|
496
|
+
ADMIN_TOKEN_ENV: session.admin.value or "",
|
|
479
497
|
"PYTHONDONTWRITEBYTECODE": "1",
|
|
480
498
|
}
|
|
499
|
+
if session.ui_url is not None:
|
|
500
|
+
env["WORLD_UI_URL"] = session.ui_url
|
|
481
501
|
try:
|
|
482
502
|
proc = subprocess.run(
|
|
483
503
|
[interpreter, "-m", "pytest", "-q", "-p", "no:cacheprovider", f"--junitxml={junit}", *names],
|
|
@@ -496,9 +516,20 @@ def _run_python(root: Path, files: list[Path], base: str) -> PythonResult:
|
|
|
496
516
|
reason = "pytest wrote no junit report"
|
|
497
517
|
return PythonResult(status="failed", files=names, reason=reason, output=output, interpreter=interpreter)
|
|
498
518
|
tests, failures, errors, skipped = counts
|
|
499
|
-
if proc.returncode ==
|
|
519
|
+
if proc.returncode == PYTEST_NO_TESTS_COLLECTED:
|
|
500
520
|
reason = "no tests collected"
|
|
501
521
|
return PythonResult(status="none", files=names, reason=reason, output=output, interpreter=interpreter)
|
|
522
|
+
if proc.returncode == 0 and tests == skipped:
|
|
523
|
+
reason = f"{skipped} skipped"
|
|
524
|
+
return PythonResult(
|
|
525
|
+
status="skipped",
|
|
526
|
+
files=names,
|
|
527
|
+
tests=tests,
|
|
528
|
+
skipped=skipped,
|
|
529
|
+
reason=reason,
|
|
530
|
+
output=output,
|
|
531
|
+
interpreter=interpreter,
|
|
532
|
+
)
|
|
502
533
|
status = "passed" if proc.returncode == 0 else "failed"
|
|
503
534
|
return PythonResult(
|
|
504
535
|
status=status,
|
|
@@ -527,4 +558,5 @@ def _junit_counts(path: Path) -> tuple[int, int, int, int] | None:
|
|
|
527
558
|
failures += int(suite.get("failures") or 0)
|
|
528
559
|
errors += int(suite.get("errors") or 0)
|
|
529
560
|
skipped += int(suite.get("skipped") or 0)
|
|
530
|
-
|
|
561
|
+
xfailed = sum(1 for marker in tree.iter("skipped") if marker.get("type") == "pytest.xfail")
|
|
562
|
+
return total, failures, errors, skipped - xfailed
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""The one score rule every grader answers by: a finite number from 0 to 1.
|
|
2
|
+
|
|
3
|
+
A passed check scores 1, a failed one 0, a fraction is partial credit and
|
|
4
|
+
``None`` is ungraded. Anything else (1.7, -0.5, NaN, Infinity) is not a score:
|
|
5
|
+
a reader refuses it by key and value instead of filing it, since two
|
|
6
|
+
out-of-range checks can average to a perfect 1.0 and NaN has no JSON form the
|
|
7
|
+
platform can read. Stdlib only, inside ``gatewaysdk.environment`` because that
|
|
8
|
+
package is vendored on its own (the World Host, the batch runner). The
|
|
9
|
+
TypeScript twin is ``sdk-ts/src/worlds/grade.ts``; both are pinned by
|
|
10
|
+
``gatewaysdk/fixtures/tasks/grade-vectors.json``.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import json
|
|
16
|
+
import math
|
|
17
|
+
from collections.abc import Iterable
|
|
18
|
+
from typing import Any, Tuple
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Score:
|
|
22
|
+
RULE = "a score is a number from 0 to 1"
|
|
23
|
+
LOW = 0.0
|
|
24
|
+
HIGH = 1.0
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
def finite(value: float) -> bool:
|
|
28
|
+
"""Whether a float holds it: not NaN or Infinity, and not an integer past
|
|
29
|
+
float range, which ``math.isfinite`` cannot even convert."""
|
|
30
|
+
try:
|
|
31
|
+
return math.isfinite(value)
|
|
32
|
+
except OverflowError:
|
|
33
|
+
return False
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def admits(value: float) -> bool:
|
|
37
|
+
return Score.finite(value) and Score.LOW <= value <= Score.HIGH
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def shown(value: float) -> str:
|
|
41
|
+
"""The value as a grader's JSON prints it: ``1.7``, ``NaN``, ``Infinity``."""
|
|
42
|
+
return json.dumps(value)
|
|
43
|
+
|
|
44
|
+
@staticmethod
|
|
45
|
+
def problem(scores: Iterable[Tuple[str, Any]]) -> str | None:
|
|
46
|
+
"""Every number among ``scores`` that is not a score, as ``'<key>' = <value>``
|
|
47
|
+
and the rule, or ``None`` when all of them are. Pairs, not a mapping, so
|
|
48
|
+
that no key hides another. A boolean is the 1 or 0 it claims; a string or
|
|
49
|
+
``None`` is not a number and is not judged here."""
|
|
50
|
+
refused = [
|
|
51
|
+
f"{key!r} = {Score.shown(value)}"
|
|
52
|
+
for key, value in scores
|
|
53
|
+
if isinstance(value, (int, float)) and not Score.admits(value)
|
|
54
|
+
]
|
|
55
|
+
if not refused:
|
|
56
|
+
return None
|
|
57
|
+
return f"{', '.join(refused)}; {Score.RULE}"
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""What a secret looks like on its own, whatever field it sits in.
|
|
2
|
+
|
|
3
|
+
``SECRET_VALUE`` is the key shapes the browser recorder
|
|
4
|
+
(``world_browser.redact_action_args``, twinned by the TypeScript recorders) and
|
|
5
|
+
the world call log (``schema.call_log.CallRedaction``) both redact: a provider
|
|
6
|
+
key by its prefix, an AWS access key id, a bearer header. ``SECRET_TOKEN`` and
|
|
7
|
+
:func:`basic_credentials` add what the call log also redacts: a JSON Web Token,
|
|
8
|
+
a PEM private key, GitHub fine-grained and app tokens, a Google API key, Stripe
|
|
9
|
+
restricted keys and webhook secrets, GitLab and npm tokens, an AWS temporary
|
|
10
|
+
key id, Slack app and refresh tokens, and a Basic authorization value.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import base64
|
|
16
|
+
import re
|
|
17
|
+
|
|
18
|
+
SECRET_VALUE = re.compile(r"^((sk|pk|ghp|xox[abp])[-_][-_A-Za-z0-9]{8,}|AKIA[0-9A-Z]{12,}|Bearer [-_.A-Za-z0-9]{8,})")
|
|
19
|
+
SECRET_TOKEN = re.compile(
|
|
20
|
+
r"^(?:eyJ[-_A-Za-z0-9]{8,}\.eyJ[-_A-Za-z0-9]{8,}\.[-_A-Za-z0-9]+$"
|
|
21
|
+
r"|github_pat_\w{20,}|gh[ousr]_[A-Za-z0-9]{20,}|AIza[-\w]{35}|rk_(?:live|test)_\w{8,}|whsec_\w{8,}"
|
|
22
|
+
r"|glpat-[-\w]{20,}|npm_[A-Za-z0-9]{36}|ASIA[0-9A-Z]{16}|xapp-[-\w]{20,}|xoxe\.[-.\w]{20,})"
|
|
23
|
+
r"|-----BEGIN [A-Z ]*PRIVATE KEY-----"
|
|
24
|
+
)
|
|
25
|
+
_BASIC = re.compile(r"^[Bb]asic ((?:[A-Za-z0-9+/]{4})+(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?)$")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def basic_credentials(value: str) -> bool:
|
|
29
|
+
"""Whether ``value`` is a Basic authorization value: base64 of ``user:password``."""
|
|
30
|
+
found = _BASIC.match(value)
|
|
31
|
+
if not found:
|
|
32
|
+
return False
|
|
33
|
+
try:
|
|
34
|
+
return ":" in base64.b64decode(found.group(1), validate=True).decode("utf-8")
|
|
35
|
+
except ValueError:
|
|
36
|
+
return False
|
|
@@ -23,14 +23,17 @@ from dataclasses import dataclass
|
|
|
23
23
|
from functools import cached_property
|
|
24
24
|
from pathlib import Path
|
|
25
25
|
from typing import ClassVar
|
|
26
|
+
|
|
26
27
|
from ._toml import tomllib
|
|
28
|
+
from .errors import WorldStartError
|
|
29
|
+
from .world_session_contract import API_BUNDLE_RUN_REFUSAL
|
|
27
30
|
|
|
28
31
|
try: # optional: only consulted when a tree ships a root .gitignore
|
|
29
32
|
from gitignore_parser import parse_gitignore
|
|
30
33
|
except ImportError: # pragma: no cover - the runner and the sandbox vendor this module alone
|
|
31
34
|
parse_gitignore = None # type: ignore[assignment]
|
|
32
35
|
|
|
33
|
-
#
|
|
36
|
+
# Twin of packages/shared/src/world/markers.ts.
|
|
34
37
|
|
|
35
38
|
WORLD_MANIFEST = "gateway-env.toml"
|
|
36
39
|
API_BUNDLE_MANIFEST = "gateway-api-world.toml"
|
|
@@ -41,6 +44,7 @@ BENCHMARK_CONFIG = "benchmark.json"
|
|
|
41
44
|
WORLD_SCHEMA_MARKERS = ("schema/world.json", "schema/runtime.json", "schema/lock.json")
|
|
42
45
|
WORLD_ROOT_MARKERS = (CONNECTOR_SPEC, *WORLD_SCHEMA_MARKERS)
|
|
43
46
|
SCHEMA_MANIFEST_KIND = "schema"
|
|
47
|
+
LEGACY_DB_ENGINE = "sqlite"
|
|
44
48
|
SCHEMA_MANIFEST_KINDS = frozenset({SCHEMA_MANIFEST_KIND, "mock-tools"})
|
|
45
49
|
|
|
46
50
|
KIND_MOCK_TOOLS = "mock-tools"
|
|
@@ -229,8 +233,6 @@ class WorldTree:
|
|
|
229
233
|
def has_connector(self) -> bool:
|
|
230
234
|
return self.has(self.world_path(CONNECTOR_SPEC))
|
|
231
235
|
|
|
232
|
-
# ── The archive: the file set a version IS ───────────────────────────────
|
|
233
|
-
#
|
|
234
236
|
# Vendored from Prime Intellect's ``prime`` CLI (MIT; THIRD_PARTY_LICENSES)
|
|
235
237
|
# so a container hashed here resolves to the same logical version on their
|
|
236
238
|
# Environments Hub: SHA-256 over ``file:<relpath>`` + raw bytes, in sorted
|
|
@@ -445,6 +447,34 @@ class World:
|
|
|
445
447
|
def is_schema(self) -> bool:
|
|
446
448
|
return self.schema_mode == "schema"
|
|
447
449
|
|
|
450
|
+
@property
|
|
451
|
+
def agent_run_refusal(self) -> str | None:
|
|
452
|
+
"""Why the benchmark runner's agent cannot run this tree, or None."""
|
|
453
|
+
return None
|
|
454
|
+
|
|
455
|
+
def db_engine(self, manifest: dict) -> str:
|
|
456
|
+
"""The engine the tree's state runs on. Raw SQL in ``db/schema.sql`` and
|
|
457
|
+
``db/seed.sql`` is SQLite's, so a contract-less world names no other."""
|
|
458
|
+
engine = self._declared_db_engine(manifest)
|
|
459
|
+
if engine != LEGACY_DB_ENGINE:
|
|
460
|
+
raise WorldStartError(
|
|
461
|
+
f"db engine {engine!r} is not available to a db/schema.sql world, which runs on "
|
|
462
|
+
f"{LEGACY_DB_ENGINE}; a schema world chooses its engine (`gateway worlds schema init`)."
|
|
463
|
+
)
|
|
464
|
+
return engine
|
|
465
|
+
|
|
466
|
+
@staticmethod
|
|
467
|
+
def _declared_db_engine(manifest: dict) -> str:
|
|
468
|
+
env = next(
|
|
469
|
+
(manifest[key] for key in ("environment", "env") if isinstance(manifest.get(key), dict)),
|
|
470
|
+
{},
|
|
471
|
+
)
|
|
472
|
+
db = manifest.get("db")
|
|
473
|
+
if isinstance(db, str):
|
|
474
|
+
return db
|
|
475
|
+
named = db.get("engine") if isinstance(db, dict) else None
|
|
476
|
+
return named or env.get("db") or LEGACY_DB_ENGINE
|
|
477
|
+
|
|
448
478
|
|
|
449
479
|
class SchemaWorld(World):
|
|
450
480
|
"""A gateway-world/1 contract tree stored as itself."""
|
|
@@ -456,6 +486,11 @@ class SchemaWorld(World):
|
|
|
456
486
|
def reported_kind(self) -> str:
|
|
457
487
|
return SCHEMA_MANIFEST_KIND
|
|
458
488
|
|
|
489
|
+
def db_engine(self, manifest: dict) -> str:
|
|
490
|
+
from .schema.compiler import storage_engine
|
|
491
|
+
|
|
492
|
+
return storage_engine(manifest).engine
|
|
493
|
+
|
|
459
494
|
@property
|
|
460
495
|
def tools_engine(self) -> bool:
|
|
461
496
|
return True
|
|
@@ -481,6 +516,10 @@ class ApiBundle(World):
|
|
|
481
516
|
def task_names(self) -> list[str]:
|
|
482
517
|
return self.tree.scenario_names()
|
|
483
518
|
|
|
519
|
+
@property
|
|
520
|
+
def agent_run_refusal(self) -> str:
|
|
521
|
+
return API_BUNDLE_RUN_REFUSAL
|
|
522
|
+
|
|
484
523
|
|
|
485
524
|
class ContainerBench(World):
|
|
486
525
|
@property
|
|
@@ -35,6 +35,37 @@ EXPLORATORY_TASK_SOURCE = "none"
|
|
|
35
35
|
#: ``reward: null``.
|
|
36
36
|
NO_GRADER_REASON = "no grader"
|
|
37
37
|
|
|
38
|
+
#: Where a world's server and its runner find the token for the runner's
|
|
39
|
+
#: ``/__world/*`` routes: set by the platform on a session pod's two
|
|
40
|
+
#: containers, by the Batch runner on a world it boots.
|
|
41
|
+
ADMIN_TOKEN_ENV = "WORLD_ADMIN_TOKEN"
|
|
42
|
+
|
|
43
|
+
#: The code and words of a call by a route's name: a world's routes answer
|
|
44
|
+
#: over its HTTP api, never through the tool door.
|
|
45
|
+
NOT_A_TOOL = "not_a_tool"
|
|
46
|
+
ROUTE_NOT_A_TOOL = "'{name}' is a route of this world, not a tool; call it over the session's api URL"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def route_not_a_tool(name: str) -> str:
|
|
50
|
+
return ROUTE_NOT_A_TOOL.format(name=name)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
#: Why the benchmark runner's agent refuses an api bundle: it serves no tools.
|
|
54
|
+
API_BUNDLE_RUN_REFUSAL = (
|
|
55
|
+
"an api bundle serves no tools: its routes answer over its HTTP api and are never an agent's tools, so "
|
|
56
|
+
"the benchmark runner's agent has nothing to call. Migrate it to a schema world (`gateway worlds migrate "
|
|
57
|
+
"<slug> --apply`) and declare its tools in x-gateway.tools, or drive its routes over HTTP from your own "
|
|
58
|
+
"agent on a session (`gateway worlds session open <slug> --surface api`)"
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def still_starting_hint(session_id: str) -> str:
|
|
63
|
+
"""What ``session open`` adds when its wait ran out: the session keeps opening."""
|
|
64
|
+
return (
|
|
65
|
+
f"It keeps opening: poll `gateway worlds session status {session_id}` until ready, "
|
|
66
|
+
f"or end it with `gateway worlds session close {session_id}`."
|
|
67
|
+
)
|
|
68
|
+
|
|
38
69
|
|
|
39
70
|
def idle_warning(idle_expires_at: str | None, *, now: str | None = None) -> str | None:
|
|
40
71
|
"""The line ``session status`` prints when an unpinned session has under
|
|
@@ -42,11 +73,7 @@ def idle_warning(idle_expires_at: str | None, *, now: str | None = None) -> str
|
|
|
42
73
|
if not idle_expires_at:
|
|
43
74
|
return None
|
|
44
75
|
expires = datetime.fromisoformat(idle_expires_at.replace("Z", "+00:00"))
|
|
45
|
-
current = (
|
|
46
|
-
datetime.fromisoformat(now.replace("Z", "+00:00"))
|
|
47
|
-
if now
|
|
48
|
-
else datetime.now(timezone.utc)
|
|
49
|
-
)
|
|
76
|
+
current = datetime.fromisoformat(now.replace("Z", "+00:00")) if now else datetime.now(timezone.utc)
|
|
50
77
|
left_s = (expires - current).total_seconds()
|
|
51
78
|
if left_s >= IDLE_WARNING_MINUTES * 60:
|
|
52
79
|
return None
|
|
@@ -407,8 +407,8 @@ reward: 0 {"ticket_closed":0}
|
|
|
407
407
|
reward is for the untouched world; an agent's rollout is what `benchmarks run` grades. Drive one by hand: …
|
|
408
408
|
```
|
|
409
409
|
|
|
410
|
-
A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with
|
|
411
|
-
words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
|
|
410
|
+
A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with the database
|
|
411
|
+
engine's words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
|
|
412
412
|
before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
|
|
413
413
|
print and the run is marked not executed, exit 0.
|
|
414
414
|
|