@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -20,6 +20,9 @@ from typing import TYPE_CHECKING, Any
|
|
|
20
20
|
if TYPE_CHECKING:
|
|
21
21
|
from .api import ApiSettings
|
|
22
22
|
|
|
23
|
+
#: The largest offset or page size a request can ask for: what a store takes as one integer.
|
|
24
|
+
MAX_ROWS = 2**63 - 1
|
|
25
|
+
|
|
23
26
|
|
|
24
27
|
@dataclass(frozen=True)
|
|
25
28
|
class Paging:
|
|
@@ -29,6 +32,21 @@ class Paging:
|
|
|
29
32
|
cursor_param: str = "cursor"
|
|
30
33
|
page_size: int = 100
|
|
31
34
|
|
|
35
|
+
@property
|
|
36
|
+
def params(self) -> frozenset[str]:
|
|
37
|
+
"""The query parameters the pager reads, which never filter a route's rows."""
|
|
38
|
+
return frozenset({self.limit_param, self.offset_param, self.cursor_param})
|
|
39
|
+
|
|
40
|
+
def window(self, query: dict[str, str]) -> tuple[int, int | None]:
|
|
41
|
+
"""Where the page a request asks for starts and how many rows it holds; every row
|
|
42
|
+
(``None``) for a route that does not page."""
|
|
43
|
+
if self.style == "none":
|
|
44
|
+
return 0, None
|
|
45
|
+
limit = _int_param(query.get(self.limit_param), self.page_size)
|
|
46
|
+
if self.style == "offset":
|
|
47
|
+
return _int_param(query.get(self.offset_param), 0), limit
|
|
48
|
+
return _cursor_offset(query.get(self.cursor_param)), limit
|
|
49
|
+
|
|
32
50
|
|
|
33
51
|
@dataclass(frozen=True)
|
|
34
52
|
class Page:
|
|
@@ -47,6 +65,11 @@ class Page:
|
|
|
47
65
|
def previous_offset(self) -> int | None:
|
|
48
66
|
return max(self.offset - self.limit, 0) if self.style != "none" and self.offset > 0 else None
|
|
49
67
|
|
|
68
|
+
@classmethod
|
|
69
|
+
def of(cls, rows: list[dict], total: int, offset: int, limit: int | None, style: str) -> Page:
|
|
70
|
+
"""The page :meth:`Paging.window` asked for, ``rows`` already cut to it."""
|
|
71
|
+
return cls(rows, total, offset, total if limit is None else limit, style)
|
|
72
|
+
|
|
50
73
|
|
|
51
74
|
def _encode_cursor(offset: int) -> str:
|
|
52
75
|
return base64.urlsafe_b64encode(str(offset).encode()).decode().rstrip("=")
|
|
@@ -56,23 +79,16 @@ def _cursor_offset(raw: str | None) -> int:
|
|
|
56
79
|
if not raw:
|
|
57
80
|
return 0
|
|
58
81
|
try:
|
|
59
|
-
return int(base64.urlsafe_b64decode(raw + "=" * (-len(raw) % 4)).decode())
|
|
82
|
+
return min(max(int(base64.urlsafe_b64decode(raw + "=" * (-len(raw) % 4)).decode()), 0), MAX_ROWS)
|
|
60
83
|
except (ValueError, UnicodeDecodeError):
|
|
61
84
|
return 0
|
|
62
85
|
|
|
63
86
|
|
|
64
87
|
def paginate(rows: list[dict], query: dict[str, str], paging: Paging) -> Page:
|
|
65
88
|
"""One page of rows with the offsets the next and previous pages start at."""
|
|
66
|
-
|
|
67
|
-
if
|
|
68
|
-
|
|
69
|
-
limit = _int_param(query.get(paging.limit_param), paging.page_size)
|
|
70
|
-
offset = (
|
|
71
|
-
_int_param(query.get(paging.offset_param), 0)
|
|
72
|
-
if paging.style == "offset"
|
|
73
|
-
else _cursor_offset(query.get(paging.cursor_param))
|
|
74
|
-
)
|
|
75
|
-
return Page(rows[offset : offset + limit], total, offset, limit, paging.style)
|
|
89
|
+
offset, limit = paging.window(query)
|
|
90
|
+
end = None if limit is None else offset + limit
|
|
91
|
+
return Page.of(rows[offset:end], len(rows), offset, limit, paging.style)
|
|
76
92
|
|
|
77
93
|
|
|
78
94
|
def _int_param(value: str | None, default: int) -> int:
|
|
@@ -80,31 +96,63 @@ def _int_param(value: str | None, default: int) -> int:
|
|
|
80
96
|
parsed = int(value) if value is not None else default
|
|
81
97
|
except ValueError:
|
|
82
98
|
return default
|
|
83
|
-
return parsed if parsed >= 0 else default
|
|
99
|
+
return min(parsed, MAX_ROWS) if parsed >= 0 else default
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True)
|
|
103
|
+
class PageLink:
|
|
104
|
+
"""The URL a page was asked at, from which its neighbours are built: the request's origin and
|
|
105
|
+
path and every value of its query, in order, less ``dropped`` (the parameters a credential the
|
|
106
|
+
door presented rides in, never the caller's to see again)."""
|
|
107
|
+
|
|
108
|
+
base: str
|
|
109
|
+
query: tuple[tuple[str, tuple[str, ...]], ...] = ()
|
|
84
110
|
|
|
111
|
+
@classmethod
|
|
112
|
+
def of(cls, base: str, query_all: dict[str, list[str]], dropped: frozenset[str] = frozenset()) -> PageLink:
|
|
113
|
+
return cls(base, tuple((name, tuple(values)) for name, values in query_all.items() if name not in dropped))
|
|
85
114
|
|
|
86
|
-
|
|
115
|
+
@property
|
|
116
|
+
def first_values(self) -> tuple[str, dict[str, str]]:
|
|
117
|
+
"""``(url, {name: first value})``, the link a world's handler reads as ``ctx.link``."""
|
|
118
|
+
return self.base, {name: values[0] for name, values in self.query}
|
|
119
|
+
|
|
120
|
+
def at(self, page: Page, offset: int, paging: Paging) -> str:
|
|
121
|
+
"""The neighbouring page at ``offset``: every value the request sent, its paging parameters
|
|
122
|
+
set where the request named them, else after them."""
|
|
123
|
+
following: dict[str, Any] = dict(self.query)
|
|
124
|
+
if page.style == "offset":
|
|
125
|
+
following[paging.offset_param] = str(offset)
|
|
126
|
+
else:
|
|
127
|
+
following[paging.cursor_param] = _encode_cursor(offset)
|
|
128
|
+
following[paging.limit_param] = str(page.limit)
|
|
129
|
+
return f"{self.base}?{urllib.parse.urlencode(following, doseq=True)}"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _page_ref(page: Page, offset: int | None, paging: Paging, link: PageLink | None) -> Any:
|
|
87
133
|
"""What the vendor's client is handed for a neighbouring page: '' or None when there is
|
|
88
134
|
none, a cursor token, or — with ``page_links`` — the absolute URL DRF-style vendors answer."""
|
|
89
135
|
if offset is None:
|
|
90
136
|
return None if link is not None else ""
|
|
91
137
|
if link is None:
|
|
92
138
|
return _encode_cursor(offset)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"""The page's cursor/total/previous as response headers, when the vendor sends them that way."""
|
|
105
|
-
if not settings.page_headers:
|
|
106
|
-
return None
|
|
139
|
+
return link.at(page, offset, paging)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def page_headers(
|
|
143
|
+
page: Page,
|
|
144
|
+
settings: ApiSettings,
|
|
145
|
+
paging: Paging | None = None,
|
|
146
|
+
link: PageLink | None = None,
|
|
147
|
+
) -> dict[str, str] | None:
|
|
148
|
+
"""The page's cursor/total/previous as response headers, when the vendor sends them that way,
|
|
149
|
+
and with ``link_header`` its neighbours as an RFC 8288 ``Link`` on the request's own origin."""
|
|
107
150
|
paging = paging or settings.paging
|
|
151
|
+
headers = {**_named_headers(page, settings, paging), **_link_header(page, settings, paging, link)}
|
|
152
|
+
return headers or None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _named_headers(page: Page, settings: ApiSettings, paging: Paging) -> dict[str, str]:
|
|
108
156
|
values = {
|
|
109
157
|
"cursor": _page_ref(page, page.next_offset, paging, None),
|
|
110
158
|
"total": str(page.total),
|
|
@@ -113,13 +161,23 @@ def page_headers(page: Page, settings: ApiSettings, paging: Paging | None = None
|
|
|
113
161
|
return {header: str(values[key]) for key, header in settings.page_headers.items() if values[key] not in ("", None)}
|
|
114
162
|
|
|
115
163
|
|
|
164
|
+
def _link_header(page: Page, settings: ApiSettings, paging: Paging, link: PageLink | None) -> dict[str, str]:
|
|
165
|
+
"""``Link: <next>; rel="next", <prev>; rel="prev"`` (GitHub's shape), each URL the request's
|
|
166
|
+
own with the page's paging parameters set; none unless the world declares ``link_header``."""
|
|
167
|
+
if not settings.link_header or link is None:
|
|
168
|
+
return {}
|
|
169
|
+
neighbours = (("next", page.next_offset), ("prev", page.previous_offset))
|
|
170
|
+
relations = [f'<{_page_ref(page, at, paging, link)}>; rel="{rel}"' for rel, at in neighbours if at is not None]
|
|
171
|
+
return {"Link": ", ".join(relations)} if relations else {}
|
|
172
|
+
|
|
173
|
+
|
|
116
174
|
def wrap(
|
|
117
175
|
page: Page | list[dict],
|
|
118
176
|
settings: ApiSettings,
|
|
119
177
|
envelope: str | None = None,
|
|
120
178
|
*,
|
|
121
179
|
paging: Paging | None = None,
|
|
122
|
-
link:
|
|
180
|
+
link: PageLink | None = None,
|
|
123
181
|
omit_done_cursor: bool = False,
|
|
124
182
|
wrapper: dict | None = None,
|
|
125
183
|
) -> Any:
|
|
@@ -1,22 +1,3 @@
|
|
|
1
|
-
"""One grader path: a task's grader over a world's end state.
|
|
2
|
-
|
|
3
|
-
A task grades one of three ways — ``assertions`` (declarative checks, in
|
|
4
|
-
process), ``python`` (a ``verify.py`` fed the state on stdin, printing the
|
|
5
|
-
rewards on stdout) or ``rubric`` (judged by the platform; here, pending). The
|
|
6
|
-
World Host grades a session this way and the batch runner grades a stored
|
|
7
|
-
task this way, whether the world is stored as itself (a schema world) or
|
|
8
|
-
packaged as an api bundle: :class:`TaskGrader` is the one place the three
|
|
9
|
-
kinds are told apart, and :func:`keyed_entities` is the one shape of the state
|
|
10
|
-
they read (``{entity: {primary key: row}}``, the api runtime's dump).
|
|
11
|
-
|
|
12
|
-
The two hosts differ only in how they run world-authored code: the World Host
|
|
13
|
-
confines the verifier like a worker (:class:`VerifierSandbox` subclass in
|
|
14
|
-
:mod:`workers`), the runner's container is the confinement. A verifier that
|
|
15
|
-
cannot deliver a grade — missing, timed out, crashed, printed no JSON — is a
|
|
16
|
-
:class:`GraderError`; the host turns it into a refusal, the runner into a
|
|
17
|
-
null-graded sample that names the reason. Null is "ungraded", never zero.
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
1
|
from __future__ import annotations
|
|
21
2
|
|
|
22
3
|
import json
|
|
@@ -29,53 +10,54 @@ from typing import Any, Protocol
|
|
|
29
10
|
|
|
30
11
|
from ..score import Score
|
|
31
12
|
from ..world_session_contract import NO_GRADER_REASON
|
|
13
|
+
from .assertions import Assertions
|
|
14
|
+
from .journal import NOT_RECORDED
|
|
32
15
|
from .primary_key import PrimaryKey
|
|
33
|
-
from .tasks import
|
|
16
|
+
from .tasks import CALL_LOG, JOURNAL, JOURNAL_META, GraderError
|
|
34
17
|
|
|
35
18
|
GRADER_KINDS = ("assertions", "python", "rubric")
|
|
36
19
|
VERIFY_TIMEOUT_S = 30.0
|
|
20
|
+
JOURNAL_READS = (JOURNAL, CALL_LOG)
|
|
37
21
|
|
|
38
22
|
|
|
39
23
|
@dataclass(frozen=True)
|
|
40
24
|
class Grade:
|
|
41
|
-
"""
|
|
42
|
-
|
|
43
|
-
platform still owes (a rubric); ``reason`` says why a grade is null when
|
|
44
|
-
no grader ran at all (an exploratory session, a task without one)."""
|
|
25
|
+
"""``reward`` is None when no grader decided it (never 0) and 0 when one failed closed; ``pending`` means the
|
|
26
|
+
platform still owes the grade (a rubric)."""
|
|
45
27
|
|
|
46
28
|
reward: float | None
|
|
47
29
|
rewards: dict[str, float | None] = field(default_factory=dict)
|
|
48
30
|
pending: bool = False
|
|
49
31
|
grader: str | None = None
|
|
50
32
|
reason: str | None = None
|
|
33
|
+
reasons: dict[str, str] = field(default_factory=dict)
|
|
34
|
+
checks: dict[str, dict] = field(default_factory=dict)
|
|
51
35
|
|
|
52
36
|
def as_dict(self) -> dict[str, Any]:
|
|
53
|
-
"""The wire shape ``grade`` answers with (and ``verify.py`` prints)."""
|
|
54
37
|
out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
|
|
55
38
|
if self.pending:
|
|
56
39
|
out["pending"] = True
|
|
57
40
|
out["grader"] = self.grader
|
|
58
41
|
if self.reason is not None:
|
|
59
42
|
out["reason"] = self.reason
|
|
43
|
+
if self.reasons:
|
|
44
|
+
out["reasons"] = dict(self.reasons)
|
|
45
|
+
if self.checks:
|
|
46
|
+
out["checks"] = dict(self.checks)
|
|
60
47
|
return out
|
|
61
48
|
|
|
49
|
+
@staticmethod
|
|
50
|
+
def closed(reason: str) -> Grade:
|
|
51
|
+
return Grade(0.0, reason=reason)
|
|
52
|
+
|
|
62
53
|
@staticmethod
|
|
63
54
|
def ungraded() -> Grade:
|
|
64
|
-
"""The grade of a session with no grader: null, and it says so."""
|
|
65
55
|
return Grade(None, reason=NO_GRADER_REASON)
|
|
66
56
|
|
|
67
57
|
@staticmethod
|
|
68
58
|
def of_rewards(printed: Mapping[str, Any]) -> Grade:
|
|
69
|
-
"""
|
|
70
|
-
|
|
71
|
-
boolean as the 1.0 / 0.0 it claims; a string or null is an ungraded
|
|
72
|
-
check, the World Host's contract for annotations). A nested object or
|
|
73
|
-
array is a :class:`GraderError` naming the key: read as one null check,
|
|
74
|
-
a nested ``rewards`` map would lose every score inside it while the
|
|
75
|
-
grade still answers. So is a number outside 0..1 or a non-finite one
|
|
76
|
-
(``json.dumps(nan)`` prints ``NaN``), by key and value: out-of-range
|
|
77
|
-
checks average into a perfect 1.0, and NaN is not JSON the platform
|
|
78
|
-
can read."""
|
|
59
|
+
"""A nested value would hide the scores inside it, an out-of-range one would average into 1.0 and NaN is not
|
|
60
|
+
JSON, so each is refused by key; a string or null is an ungraded check."""
|
|
79
61
|
refused = [f"{key!r} as {_kind(value)}" for key, value in printed.items() if not _is_score(value)]
|
|
80
62
|
if refused:
|
|
81
63
|
raise GraderError(
|
|
@@ -103,19 +85,15 @@ def _kind(value: Any) -> str:
|
|
|
103
85
|
|
|
104
86
|
|
|
105
87
|
def _number(value: Any) -> float | None:
|
|
106
|
-
"""A
|
|
107
|
-
tracing reward emitter's convention), anything else (NaN and Infinity
|
|
108
|
-
included) null."""
|
|
88
|
+
"""A boolean is the 1.0 / 0.0 it claims, the tracing reward emitter's convention."""
|
|
109
89
|
if isinstance(value, (int, float)) and Score.finite(value):
|
|
110
90
|
return float(value)
|
|
111
91
|
return None
|
|
112
92
|
|
|
113
93
|
|
|
114
94
|
class VerifierSandbox:
|
|
115
|
-
"""
|
|
116
|
-
|
|
117
|
-
is itself the confinement (the batch runner). A host that must confine the
|
|
118
|
-
verifier further subclasses this."""
|
|
95
|
+
"""Runs verify.py with the inherited environment, safe only where its container is the confinement (the batch
|
|
96
|
+
runner); the World Host confines it further in a subclass."""
|
|
119
97
|
|
|
120
98
|
def command(self, script: Path) -> list[str]:
|
|
121
99
|
return [sys.executable, str(script)]
|
|
@@ -125,21 +103,6 @@ class VerifierSandbox:
|
|
|
125
103
|
|
|
126
104
|
|
|
127
105
|
class TaskGrader:
|
|
128
|
-
"""A task's grader, ready to grade a state dump.
|
|
129
|
-
|
|
130
|
-
``grader`` is the spec's normalised ``grader`` table. A python grader's
|
|
131
|
-
source must already be on disk at ``verify`` (the session's or the run's
|
|
132
|
-
``<task>/checks/verify.py``); a bundled scenario's ``verify.py`` is graded
|
|
133
|
-
as a python grader with no table of its own.
|
|
134
|
-
|
|
135
|
-
A grader with a ``scope`` is a CROSS-WORLD VERIFIER: the same table, read
|
|
136
|
-
over several of a multi-world task's worlds at once. ``scope`` names the
|
|
137
|
-
aliases (the first is the session it runs on); :meth:`evidence` builds
|
|
138
|
-
the dump it grades, every scoped world's rows as ``<alias>.<entity>`` and
|
|
139
|
-
its calls as ``<alias>.calls`` — the naming a world's dependencies use, so
|
|
140
|
-
a check or a ``verify.py`` reads it unchanged. An unscoped grader's
|
|
141
|
-
evidence is the session's own state, untouched."""
|
|
142
|
-
|
|
143
106
|
def __init__(
|
|
144
107
|
self,
|
|
145
108
|
grader: Mapping[str, Any],
|
|
@@ -150,6 +113,7 @@ class TaskGrader:
|
|
|
150
113
|
default_timeout_s: float = VERIFY_TIMEOUT_S,
|
|
151
114
|
scope: Iterable[str] = (),
|
|
152
115
|
name: str | None = None,
|
|
116
|
+
explain: bool = False,
|
|
153
117
|
) -> None:
|
|
154
118
|
kind = grader.get("kind")
|
|
155
119
|
if kind not in GRADER_KINDS:
|
|
@@ -161,23 +125,16 @@ class TaskGrader:
|
|
|
161
125
|
self.home = home
|
|
162
126
|
self.scope: tuple[str, ...] = tuple(scope)
|
|
163
127
|
self.name = name
|
|
128
|
+
self.explain = explain
|
|
164
129
|
timeout = grader.get("timeoutSeconds")
|
|
165
130
|
own = None if isinstance(timeout, bool) else _number(timeout)
|
|
166
131
|
self.timeout_s = own if own is not None and own > 0 else float(default_timeout_s)
|
|
167
132
|
|
|
168
133
|
@property
|
|
169
134
|
def anchor(self) -> str | None:
|
|
170
|
-
"""The alias whose session runs a scoped grader; None when unscoped."""
|
|
171
135
|
return self.scope[0] if self.scope else None
|
|
172
136
|
|
|
173
137
|
def evidence(self, own: Mapping[str, Any], context: Mapping[str, Mapping[str, Any]]) -> dict:
|
|
174
|
-
"""The dump this grader reads. Unscoped: ``own`` as it is. Scoped:
|
|
175
|
-
every scoped world's state under ``<alias>.<key>`` — the anchor's from
|
|
176
|
-
``own``, the others from ``context`` (``{alias: state}``, what the
|
|
177
|
-
platform gathered from those sessions) — with the agent's ``report``
|
|
178
|
-
kept at the top. A scoped alias ``context`` lacks, or a check over an
|
|
179
|
-
entity no scoped world holds, is a :class:`GraderError`: ungraded with
|
|
180
|
-
the reason, never zero."""
|
|
181
138
|
if not self.scope:
|
|
182
139
|
return dict(own)
|
|
183
140
|
missing = [alias for alias in self.scope[1:] if alias not in context]
|
|
@@ -192,9 +149,7 @@ class TaskGrader:
|
|
|
192
149
|
dump.update({f"{alias}.{key}": value for key, value in states[alias].items() if key != "report"})
|
|
193
150
|
if "report" in own:
|
|
194
151
|
dump["report"] = own["report"]
|
|
195
|
-
# No
|
|
196
|
-
# only its own), and a check over a key the dump lacks selects nothing —
|
|
197
|
-
# which `absent` and `count: eq 0` would pass on a typo.
|
|
152
|
+
# No contract vets a cross-world entity, and one the dump lacks selects nothing: absent would pass on a typo.
|
|
198
153
|
unknown = [check["entity"] for check in self.grader.get("checks", ()) if check["entity"] not in dump]
|
|
199
154
|
if unknown:
|
|
200
155
|
raise GraderError(
|
|
@@ -210,7 +165,6 @@ class TaskGrader:
|
|
|
210
165
|
home: Path | None = None,
|
|
211
166
|
timeout_s: float = VERIFY_TIMEOUT_S,
|
|
212
167
|
) -> TaskGrader:
|
|
213
|
-
"""A bundled scenario's ``verify.py``: a python grader with the host's timeout."""
|
|
214
168
|
return TaskGrader(
|
|
215
169
|
{"kind": "python"},
|
|
216
170
|
verify=verify,
|
|
@@ -221,9 +175,13 @@ class TaskGrader:
|
|
|
221
175
|
|
|
222
176
|
@property
|
|
223
177
|
def pending(self) -> bool:
|
|
224
|
-
"""Whether the platform grades this task (a rubric), not this process."""
|
|
225
178
|
return self.kind == "rubric"
|
|
226
179
|
|
|
180
|
+
@property
|
|
181
|
+
def reads_journal(self) -> bool:
|
|
182
|
+
checks = self.grader.get("checks", ())
|
|
183
|
+
return self.grader.get("readsJournal") is True or any(check["entity"] in JOURNAL_READS for check in checks)
|
|
184
|
+
|
|
227
185
|
def grade(self, state: Mapping[str, Any]) -> Grade:
|
|
228
186
|
graders: dict[str, Callable[[Mapping[str, Any]], Grade]] = {
|
|
229
187
|
"assertions": self._assertions,
|
|
@@ -233,13 +191,17 @@ class TaskGrader:
|
|
|
233
191
|
return graders[self.kind](state)
|
|
234
192
|
|
|
235
193
|
def _assertions(self, state: Mapping[str, Any]) -> Grade:
|
|
236
|
-
graded =
|
|
237
|
-
|
|
194
|
+
graded = Assertions(self.grader["checks"]).grade(state)
|
|
195
|
+
checks = {key: outcome.as_dict() for key, outcome in graded.outcomes.items()} if self.explain else {}
|
|
196
|
+
return Grade(reward=graded.reward, rewards=graded.rewards, reasons=graded.reasons, checks=checks)
|
|
238
197
|
|
|
239
198
|
def _rubric(self, state: Mapping[str, Any]) -> Grade:
|
|
240
199
|
return Grade(reward=None, rewards={}, pending=True, grader="rubric")
|
|
241
200
|
|
|
242
201
|
def _python(self, state: Mapping[str, Any]) -> Grade:
|
|
202
|
+
refused = self._incomplete_journal(state)
|
|
203
|
+
if refused is not None:
|
|
204
|
+
return Grade.closed(refused)
|
|
243
205
|
verify = self.verify
|
|
244
206
|
if verify is None or not verify.is_file():
|
|
245
207
|
raise GraderError(f"the task's verify.py is missing at {verify}")
|
|
@@ -268,15 +230,34 @@ class TaskGrader:
|
|
|
268
230
|
raise GraderError("verify.py rewards output is not an object")
|
|
269
231
|
return Grade.of_rewards(printed)
|
|
270
232
|
|
|
233
|
+
def _incomplete_journal(self, state: Mapping[str, Any]) -> str | None:
|
|
234
|
+
if not self.grader.get("readsJournal"):
|
|
235
|
+
return None
|
|
236
|
+
if JOURNAL not in state:
|
|
237
|
+
return f"journal incomplete ({NOT_RECORDED}): this verify.py reads the journal"
|
|
238
|
+
if self.grader.get("acceptsIncompleteJournal"):
|
|
239
|
+
return None
|
|
240
|
+
meta = state.get(JOURNAL_META)
|
|
241
|
+
meta = meta if isinstance(meta, Mapping) else {}
|
|
242
|
+
withheld = meta.get("withheld") or 0
|
|
243
|
+
if meta.get("complete") is True and not withheld:
|
|
244
|
+
return None
|
|
245
|
+
why = (
|
|
246
|
+
meta.get("reason") or "it may be missing requests"
|
|
247
|
+
if meta.get("complete") is not True
|
|
248
|
+
else f"the bodies of {withheld} requests were withheld past the grade's budget"
|
|
249
|
+
)
|
|
250
|
+
return (
|
|
251
|
+
f"journal incomplete ({why}): this verify.py reads the journal, "
|
|
252
|
+
"so it does not grade one that may be missing requests; declare acceptsIncompleteJournal: true to grade it"
|
|
253
|
+
)
|
|
254
|
+
|
|
271
255
|
|
|
272
256
|
def keyed_entities(
|
|
273
257
|
entities: Mapping[str, Mapping[str, Any]],
|
|
274
258
|
rows_of: Callable[[str], Iterable[Mapping[str, Any]]],
|
|
275
259
|
) -> dict:
|
|
276
|
-
"""
|
|
277
|
-
name, its rows keyed by their primary key (``a:b`` for a composite key) —
|
|
278
|
-
the api runtime's ``/__world/state`` dump, so a check or a ``verify.py``
|
|
279
|
-
written against one shape of the world grades the other."""
|
|
260
|
+
"""Twin of the api runtime's ``/__world/state`` dump, so a check grades either shape of a world."""
|
|
280
261
|
out: dict[str, dict[str, Any]] = {}
|
|
281
262
|
for entity, spec in entities.items():
|
|
282
263
|
key = PrimaryKey.of(spec)
|
|
@@ -285,9 +266,6 @@ def keyed_entities(
|
|
|
285
266
|
|
|
286
267
|
|
|
287
268
|
class StoredWorld(Protocol):
|
|
288
|
-
"""What a grader needs of a live world: its contract and the store holding
|
|
289
|
-
its rows — a World Host session's world and the batch runner's are both."""
|
|
290
|
-
|
|
291
269
|
@property
|
|
292
270
|
def contract(self) -> Any: ...
|
|
293
271
|
|
|
@@ -296,12 +274,6 @@ class StoredWorld(Protocol):
|
|
|
296
274
|
|
|
297
275
|
|
|
298
276
|
def world_state(root: StoredWorld | None, dependencies: Mapping[str, StoredWorld]) -> dict:
|
|
299
|
-
"""The rows as a task's grader reads them, on the host and in the batch
|
|
300
|
-
runner alike: the root's entities by name and each dependency's as
|
|
301
|
-
``<alias>.<entity>``, every entity's rows keyed by primary key — the api
|
|
302
|
-
runtime's dump, so one ``verify.py`` grades the world in either shape. A
|
|
303
|
-
world without a contract (a legacy tools bundle) contributes nothing, as
|
|
304
|
-
does a task set that owns no root world (``root`` None)."""
|
|
305
277
|
out: dict[str, Any] = dict(_keyed_rows_of(root))
|
|
306
278
|
for alias, world in dependencies.items():
|
|
307
279
|
for entity, rows in _keyed_rows_of(world).items():
|
|
@@ -5,6 +5,8 @@ A tool or a route whose handler raises is answered the same way on every door:
|
|
|
5
5
|
* ``AttributeError`` / ``TypeError`` / ``NameError`` are the author's bug
|
|
6
6
|
(``store.upsert`` on a store with no ``upsert``, a misspelled name, a
|
|
7
7
|
wrong argument shape) — status **400**, ``error: "handler"``.
|
|
8
|
+
* ``Refusal(status, body)`` is the vendor saying no (not found, invalid,
|
|
9
|
+
forbidden) — its own 4xx status and body, as the caller should see them.
|
|
8
10
|
* Anything else is a failure inside the handler's own logic — status **500**,
|
|
9
11
|
``error: "engine"``.
|
|
10
12
|
|
|
@@ -36,7 +38,20 @@ HANDLER_BUG_ERRORS: tuple[type[BaseException], ...] = (
|
|
|
36
38
|
NameError,
|
|
37
39
|
)
|
|
38
40
|
|
|
41
|
+
|
|
42
|
+
class Refusal(Exception):
|
|
43
|
+
"""A handler's 4xx answer, raised: the body goes out as given, never checked against the tool's output."""
|
|
44
|
+
|
|
45
|
+
def __init__(self, status: int, body: Any = None):
|
|
46
|
+
if isinstance(status, bool) or not isinstance(status, int) or not 400 <= status <= 499:
|
|
47
|
+
raise TypeError(f"Refusal takes a status from 400 to 499, not {status!r}")
|
|
48
|
+
super().__init__(f"refused with {status}")
|
|
49
|
+
self.status = status
|
|
50
|
+
self.body = body
|
|
51
|
+
|
|
52
|
+
|
|
39
53
|
_DETAIL_LIMIT = 2000
|
|
54
|
+
_FIELD_LIMIT = 256
|
|
40
55
|
_SURROGATE = re.compile("[\ud800-\udfff]")
|
|
41
56
|
|
|
42
57
|
|
|
@@ -50,11 +65,26 @@ def handler_bug_body(handler: str, error: BaseException, about: str = "") -> dic
|
|
|
50
65
|
return {"error": "handler", "detail": f"{about}{_detail(error)}"[:_DETAIL_LIMIT], "handler": handler}
|
|
51
66
|
|
|
52
67
|
|
|
68
|
+
def log_line(message: str) -> None:
|
|
69
|
+
"""One line on the host's stderr, which is its pod's log."""
|
|
70
|
+
print(f"world host: {message}", file=sys.stderr, flush=True)
|
|
71
|
+
|
|
72
|
+
|
|
53
73
|
def log_request_failure(error: BaseException, method: str, path: str) -> None:
|
|
54
74
|
"""One stderr line for a failure met answering a request: its class, the
|
|
55
75
|
method and the path's first segment only, since a later one can be a credential."""
|
|
56
76
|
segment = path.lstrip("/").split("/", 1)[0]
|
|
57
|
-
|
|
77
|
+
log_line(f"{type(error).__name__} on {method} /{segment}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def log_contract_refusal(world: str, operation: str, error: ContractError) -> None:
|
|
81
|
+
"""One stderr line for a write the contract refused, whose caller is answered the world's
|
|
82
|
+
declared ``invalid`` alone: the rule and where it applies, bounded and ``repr``-escaped, never a
|
|
83
|
+
value. Where it applies is the contract's schema path, or, for a value JSON cannot hold, the
|
|
84
|
+
body's own path to it, so a key the caller chose can appear, escaped; ``path`` is never logged."""
|
|
85
|
+
rule = str(error.detail.get("code") or "")[:_FIELD_LIMIT]
|
|
86
|
+
where = str(error.detail.get("schema_path") or "")[:_FIELD_LIMIT]
|
|
87
|
+
log_line(f"{world}: {operation} refused by the contract: {rule!r} at {where!r}")
|
|
58
88
|
|
|
59
89
|
|
|
60
90
|
def request_failure_body(error: BaseException) -> dict:
|
|
@@ -68,12 +98,24 @@ def engine_failure_body(handler: str, error: BaseException) -> dict:
|
|
|
68
98
|
|
|
69
99
|
|
|
70
100
|
def failure_answer(handler: str, error: Exception) -> tuple[int, dict]:
|
|
71
|
-
"""A handler's raise as its ``(status, body)``: its author's bug or the engine's failure."""
|
|
101
|
+
"""A handler's raise as its ``(status, body)``: a refusal, its author's bug or the engine's failure."""
|
|
102
|
+
if isinstance(error, Refusal):
|
|
103
|
+
return refusal_answer(handler, error)
|
|
72
104
|
if isinstance(error, HANDLER_BUG_ERRORS):
|
|
73
105
|
return 400, handler_bug_body(handler, error)
|
|
74
106
|
return 500, engine_failure_body(handler, error)
|
|
75
107
|
|
|
76
108
|
|
|
109
|
+
def refusal_answer(handler: str, refusal: Refusal) -> tuple[int, Any]:
|
|
110
|
+
"""A refusal as it is answered, its body checked now (it may have changed since the raise)."""
|
|
111
|
+
if isinstance(refusal.body, bytes):
|
|
112
|
+
return 400, handler_bug_body(handler, TypeError("a Refusal body is JSON, not bytes"))
|
|
113
|
+
problem = unencodable(refusal.body)
|
|
114
|
+
if problem is not None:
|
|
115
|
+
return 400, handler_bug_body(handler, problem, "a Refusal body is not JSON: ")
|
|
116
|
+
return refusal.status, refusal.body
|
|
117
|
+
|
|
118
|
+
|
|
77
119
|
def unencodable(body: Any) -> Exception | None:
|
|
78
120
|
"""Why a body cannot go out as JSON, or None when it can (bytes go out as they are)."""
|
|
79
121
|
if isinstance(body, bytes):
|
|
@@ -160,14 +202,19 @@ def normalize_answer(result: Any, default_status: int | None) -> Any:
|
|
|
160
202
|
|
|
161
203
|
def handled_answer(name: str, handler: Callable[..., Any], request: tuple, default_status: int | None) -> Any:
|
|
162
204
|
"""A handler's answer, or the answer for how it failed (``handler_errors``):
|
|
163
|
-
a raise is its 400 or 500, an answer
|
|
164
|
-
the contract raises
|
|
205
|
+
a raise is its 400 or 500, an answer of the wrong shape or one JSON cannot
|
|
206
|
+
carry its 400. A refusal the contract raises while the handler runs
|
|
207
|
+
(``ContractError``) is the route's to answer."""
|
|
165
208
|
try:
|
|
166
|
-
|
|
209
|
+
result = handler(*request)
|
|
167
210
|
except ContractError:
|
|
168
211
|
raise
|
|
169
212
|
except Exception as error:
|
|
170
213
|
return (*failure_answer(name, error), None)
|
|
214
|
+
try:
|
|
215
|
+
response = normalize_answer(result, default_status)
|
|
216
|
+
except ContractError as error:
|
|
217
|
+
return 400, handler_bug_body(name, error), None
|
|
171
218
|
if response is None:
|
|
172
219
|
return None
|
|
173
220
|
refused = unencodable_answer(name, response)
|