@withgateway/sdk 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +13267 -5869
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +13272 -5874
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +83 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -28
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -0
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +4355 -0
- package/dist/hooks/index.js.map +1 -0
- package/dist/hooks/index.mjs +4243 -0
- package/dist/hooks/index.mjs.map +1 -0
- package/dist/{index-Dpo-t8iO.d.ts → index-1xnxt0Cc.d.ts} +58 -2
- package/dist/index-B0R3IgsV.d.mts +1279 -0
- package/dist/index-B0R3IgsV.d.ts +1279 -0
- package/dist/{index-B5HUQQDE.d.mts → index-C1wDEwH4.d.mts} +58 -2
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +6698 -1606
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +6793 -1713
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +72 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +68 -17
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +47 -3
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +42 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +390 -27
- package/dist/worlds/index.d.ts +390 -27
- package/dist/worlds/index.js +2291 -366
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +2281 -356
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +7 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/_materialize.py +3 -3
- package/runtime/gatewaysdk/environment/schema/api.py +153 -20
- package/runtime/gatewaysdk/environment/schema/compiler.py +14 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +171 -535
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +156 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +6 -5
- package/runtime/gatewaysdk/environment/schema/tasks.py +12 -2
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +44 -13
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +156 -0
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""One grader path: a task's grader over a world's end state.
|
|
2
|
+
|
|
3
|
+
A task grades one of three ways — ``assertions`` (declarative checks, in
|
|
4
|
+
process), ``python`` (a ``verify.py`` fed the state on stdin, printing the
|
|
5
|
+
rewards on stdout) or ``rubric`` (judged by the platform; here, pending). The
|
|
6
|
+
World Host grades a session this way and the batch runner grades a stored
|
|
7
|
+
task this way, whether the world is stored as itself (a schema world) or
|
|
8
|
+
packaged as an api bundle: :class:`TaskGrader` is the one place the three
|
|
9
|
+
kinds are told apart, and :func:`keyed_entities` is the one shape of the state
|
|
10
|
+
they read (``{entity: {primary key: row}}``, the api runtime's dump).
|
|
11
|
+
|
|
12
|
+
The two hosts differ only in how they run world-authored code: the World Host
|
|
13
|
+
confines the verifier like a worker (:class:`VerifierSandbox` subclass in
|
|
14
|
+
:mod:`workers`), the runner's container is the confinement. A verifier that
|
|
15
|
+
cannot deliver a grade — missing, timed out, crashed, printed no JSON — is a
|
|
16
|
+
:class:`GraderError`; the host turns it into a refusal, the runner into a
|
|
17
|
+
null-graded sample that names the reason. Null is "ungraded", never zero.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any, Protocol
|
|
29
|
+
|
|
30
|
+
from .tasks import evaluate_assertions
|
|
31
|
+
|
|
32
|
+
GRADER_KINDS = ("assertions", "python", "rubric")
|
|
33
|
+
VERIFY_TIMEOUT_S = 30.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class GraderError(Exception):
|
|
37
|
+
"""A grader that could not deliver a grade, with the reason a reader acts on."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Grade:
|
|
42
|
+
"""A task's grade: the scalar ``reward`` (None when ungraded) and every
|
|
43
|
+
named dimension under ``rewards`` (nulls kept). ``pending`` is a grade the
|
|
44
|
+
platform still owes (a rubric)."""
|
|
45
|
+
|
|
46
|
+
reward: float | None
|
|
47
|
+
rewards: dict[str, float | None] = field(default_factory=dict)
|
|
48
|
+
pending: bool = False
|
|
49
|
+
grader: str | None = None
|
|
50
|
+
|
|
51
|
+
def as_dict(self) -> dict[str, Any]:
|
|
52
|
+
"""The wire shape ``grade`` answers with (and ``verify.py`` prints)."""
|
|
53
|
+
out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
|
|
54
|
+
if self.pending:
|
|
55
|
+
out["pending"] = True
|
|
56
|
+
out["grader"] = self.grader
|
|
57
|
+
return out
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def of_rewards(printed: Mapping[str, Any]) -> "Grade":
|
|
61
|
+
"""The grade a verifier printed: the scalar under ``reward``, every
|
|
62
|
+
other key a dimension (non-numeric ones null)."""
|
|
63
|
+
return Grade(
|
|
64
|
+
reward=_number(printed.get("reward")),
|
|
65
|
+
rewards={key: _number(value) for key, value in printed.items() if key != "reward"},
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _number(value: Any) -> float | None:
|
|
70
|
+
if isinstance(value, (int, float)):
|
|
71
|
+
return float(value)
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class VerifierSandbox:
|
|
76
|
+
"""How a ``verify.py`` is run: the argv and the environment. The base runs
|
|
77
|
+
it with this interpreter and the inherited environment — a container that
|
|
78
|
+
is itself the confinement (the batch runner). A host that must confine the
|
|
79
|
+
verifier further subclasses this."""
|
|
80
|
+
|
|
81
|
+
def command(self, script: Path) -> list[str]:
|
|
82
|
+
return [sys.executable, str(script)]
|
|
83
|
+
|
|
84
|
+
def environment(self, home: Path) -> dict[str, str] | None:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class TaskGrader:
|
|
89
|
+
"""A task's grader, ready to grade a state dump.
|
|
90
|
+
|
|
91
|
+
``grader`` is the spec's normalised ``grader`` table. A python grader's
|
|
92
|
+
source must already be on disk at ``verify`` (the session's or the run's
|
|
93
|
+
``<task>/checks/verify.py``); a bundled scenario's ``verify.py`` is graded
|
|
94
|
+
as a python grader with no table of its own."""
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
grader: Mapping[str, Any],
|
|
99
|
+
*,
|
|
100
|
+
verify: Path | None = None,
|
|
101
|
+
sandbox: VerifierSandbox | None = None,
|
|
102
|
+
home: Path | None = None,
|
|
103
|
+
default_timeout_s: float = VERIFY_TIMEOUT_S,
|
|
104
|
+
) -> None:
|
|
105
|
+
kind = grader.get("kind")
|
|
106
|
+
if kind not in GRADER_KINDS:
|
|
107
|
+
raise GraderError(f"unknown grader kind {kind!r}; a task grades by {', '.join(GRADER_KINDS)}")
|
|
108
|
+
self.grader = dict(grader)
|
|
109
|
+
self.kind: str = kind
|
|
110
|
+
self.verify = verify
|
|
111
|
+
self.sandbox = sandbox or VerifierSandbox()
|
|
112
|
+
self.home = home
|
|
113
|
+
timeout = grader.get("timeoutSeconds")
|
|
114
|
+
own = None if isinstance(timeout, bool) else _number(timeout)
|
|
115
|
+
self.timeout_s = own if own is not None and own > 0 else float(default_timeout_s)
|
|
116
|
+
|
|
117
|
+
@staticmethod
|
|
118
|
+
def for_verifier(
|
|
119
|
+
verify: Path,
|
|
120
|
+
*,
|
|
121
|
+
sandbox: VerifierSandbox | None = None,
|
|
122
|
+
home: Path | None = None,
|
|
123
|
+
timeout_s: float = VERIFY_TIMEOUT_S,
|
|
124
|
+
) -> "TaskGrader":
|
|
125
|
+
"""A bundled scenario's ``verify.py``: a python grader with the host's timeout."""
|
|
126
|
+
return TaskGrader({"kind": "python"}, verify=verify, sandbox=sandbox, home=home, default_timeout_s=timeout_s)
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def pending(self) -> bool:
|
|
130
|
+
"""Whether the platform grades this task (a rubric), not this process."""
|
|
131
|
+
return self.kind == "rubric"
|
|
132
|
+
|
|
133
|
+
def grade(self, state: Mapping[str, Any]) -> Grade:
|
|
134
|
+
graders: dict[str, Callable[[Mapping[str, Any]], Grade]] = {
|
|
135
|
+
"assertions": self._assertions,
|
|
136
|
+
"python": self._python,
|
|
137
|
+
"rubric": self._rubric,
|
|
138
|
+
}
|
|
139
|
+
return graders[self.kind](state)
|
|
140
|
+
|
|
141
|
+
def _assertions(self, state: Mapping[str, Any]) -> Grade:
|
|
142
|
+
graded = evaluate_assertions(self.grader["checks"], dict(state))
|
|
143
|
+
return Grade(reward=graded["reward"], rewards=graded["rewards"])
|
|
144
|
+
|
|
145
|
+
def _rubric(self, state: Mapping[str, Any]) -> Grade:
|
|
146
|
+
return Grade(reward=None, rewards={}, pending=True, grader="rubric")
|
|
147
|
+
|
|
148
|
+
def _python(self, state: Mapping[str, Any]) -> Grade:
|
|
149
|
+
verify = self.verify
|
|
150
|
+
if verify is None or not verify.is_file():
|
|
151
|
+
raise GraderError(f"the task's verify.py is missing at {verify}")
|
|
152
|
+
home = self.home if self.home is not None else verify.parent
|
|
153
|
+
home.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
try:
|
|
155
|
+
proc = subprocess.run(
|
|
156
|
+
self.sandbox.command(verify),
|
|
157
|
+
input=json.dumps(state),
|
|
158
|
+
check=False,
|
|
159
|
+
capture_output=True,
|
|
160
|
+
text=True,
|
|
161
|
+
timeout=self.timeout_s,
|
|
162
|
+
cwd=str(verify.parent),
|
|
163
|
+
env=self.sandbox.environment(home),
|
|
164
|
+
)
|
|
165
|
+
except subprocess.TimeoutExpired:
|
|
166
|
+
raise GraderError(f"verify.py timed out after {self.timeout_s:g}s") from None
|
|
167
|
+
if proc.returncode != 0:
|
|
168
|
+
raise GraderError(f"verify.py exited {proc.returncode}: {proc.stderr.strip()[-800:]}")
|
|
169
|
+
try:
|
|
170
|
+
printed = json.loads(proc.stdout.strip().splitlines()[-1])
|
|
171
|
+
except (json.JSONDecodeError, IndexError):
|
|
172
|
+
raise GraderError(f"verify.py printed no rewards JSON: {proc.stdout.strip()[-400:]}") from None
|
|
173
|
+
if not isinstance(printed, dict):
|
|
174
|
+
raise GraderError("verify.py rewards output is not an object")
|
|
175
|
+
return Grade.of_rewards(printed)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def keyed_entities(
|
|
179
|
+
entities: Mapping[str, Mapping[str, Any]], rows_of: Callable[[str], Iterable[Mapping[str, Any]]]
|
|
180
|
+
) -> dict:
|
|
181
|
+
"""A world's rows the way a grader reads them: every contract entity by
|
|
182
|
+
name, its rows keyed by their primary key (``a:b`` for a composite key) —
|
|
183
|
+
the api runtime's ``/__world/state`` dump, so a check or a ``verify.py``
|
|
184
|
+
written against one shape of the world grades the other."""
|
|
185
|
+
out: dict[str, dict[str, Any]] = {}
|
|
186
|
+
for entity, spec in entities.items():
|
|
187
|
+
keyed: dict[str, Any] = {}
|
|
188
|
+
for row in rows_of(entity):
|
|
189
|
+
keyed[":".join(str(row.get(name)) for name in spec["primary_key"])] = row
|
|
190
|
+
out[entity] = keyed
|
|
191
|
+
return out
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class StoredWorld(Protocol):
|
|
195
|
+
"""What a grader needs of a live world: its contract and the store holding
|
|
196
|
+
its rows — a World Host session's world and the batch runner's are both."""
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def contract(self) -> Any: ...
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def store(self) -> Any: ...
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def world_state(root: StoredWorld | None, dependencies: Mapping[str, StoredWorld]) -> dict:
|
|
206
|
+
"""The rows as a task's grader reads them, on the host and in the batch
|
|
207
|
+
runner alike: the root's entities by name and each dependency's as
|
|
208
|
+
``<alias>.<entity>``, every entity's rows keyed by primary key — the api
|
|
209
|
+
runtime's dump, so one ``verify.py`` grades the world in either shape. A
|
|
210
|
+
world without a contract (a legacy tools bundle) contributes nothing, as
|
|
211
|
+
does a task set that owns no root world (``root`` None)."""
|
|
212
|
+
out: dict[str, Any] = dict(_keyed_rows_of(root))
|
|
213
|
+
for alias, world in dependencies.items():
|
|
214
|
+
for entity, rows in _keyed_rows_of(world).items():
|
|
215
|
+
out[f"{alias}.{entity}"] = rows
|
|
216
|
+
return out
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _keyed_rows_of(world: StoredWorld | None) -> dict[str, dict]:
|
|
220
|
+
if world is None:
|
|
221
|
+
return {}
|
|
222
|
+
store, contract = world.store, world.contract
|
|
223
|
+
if store is None or contract is None:
|
|
224
|
+
return {}
|
|
225
|
+
return keyed_entities(contract.entities, store.list)
|