@withgateway/sdk 0.2.4 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +1798 -1713
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2412 -2327
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +47 -31
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +43 -27
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +390 -287
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +375 -271
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +36 -20
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +32 -16
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +2 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +2 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +26 -22
- package/dist/worlds/index.d.ts +26 -22
- package/dist/worlds/index.js +307 -203
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +297 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +2 -1
- package/runtime/gatewaysdk/environment/_bundle.py +14 -24
- package/runtime/gatewaysdk/environment/schema/api.py +37 -18
- package/runtime/gatewaysdk/environment/schema/data.py +21 -19
- package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
- package/runtime/gatewaysdk/environment/schema/host.py +156 -546
- package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
- package/runtime/gatewaysdk/environment/schema/local.py +32 -126
- package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
- package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +6 -4
- package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
- package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
- package/runtime/gatewaysdk/environment/world.py +391 -0
- package/skills/hooks-getting-started/SKILL.md +6 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@withgateway/sdk",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"module": "./dist/index.mjs",
|
|
@@ -104,6 +104,7 @@
|
|
|
104
104
|
"typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
|
|
105
105
|
"test": "vitest",
|
|
106
106
|
"test:run": "vitest run",
|
|
107
|
+
"test:integration": "vitest run --config vitest.integration.config.ts",
|
|
107
108
|
"lint": "eslint src --ext .ts",
|
|
108
109
|
"clean": "rm -rf dist",
|
|
109
110
|
"example": "tsx examples/test-ab-experiment.ts",
|
|
@@ -1,11 +1,9 @@
|
|
|
1
1
|
"""Bundle inspection: kind detection, manifest parsing, handler loading.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
- **harbor** — a tree of Harbor task dirs (``tasks/*/task.toml`` or
|
|
8
|
-
``*/task.toml``), mirroring ``benchmark_hub.harbor_adapter.detect_env_kind``.
|
|
3
|
+
Kind detection is :func:`gatewaysdk.environment.world.World.from_dir`; the
|
|
4
|
+
kinds ``gwenv`` runs in-process are **mock-tools** (``gateway-env.toml`` at the
|
|
5
|
+
bundle root, ``tools/*.py`` handlers, ``db/schema.sql``, ``verifiers/*.sql``)
|
|
6
|
+
and **harbor** (a tree of task dirs, ``tasks/*/task.toml`` or ``*/task.toml``).
|
|
9
7
|
|
|
10
8
|
Handler loading executes each ``tools/*.py`` with this SDK's
|
|
11
9
|
:mod:`gatewaysdk.environment.runtime` injected into ``sys.modules`` as
|
|
@@ -27,30 +25,22 @@ from typing import Any, Callable, Dict, List, Optional
|
|
|
27
25
|
|
|
28
26
|
from . import runtime
|
|
29
27
|
from .errors import EnvironmentNotReady, WorldStartError
|
|
30
|
-
|
|
31
|
-
KIND_MOCK_TOOLS = "mock-tools"
|
|
32
|
-
KIND_API = "api"
|
|
33
|
-
KIND_HARBOR = "harbor"
|
|
28
|
+
from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
|
|
34
29
|
|
|
35
30
|
|
|
36
31
|
def detect_kind(bundle_path: Path) -> str:
|
|
37
|
-
"""
|
|
38
|
-
|
|
39
|
-
dirs (same glob heuristics as the runner's ``detect_env_kind``); else the
|
|
40
|
-
bundle is not runnable here."""
|
|
32
|
+
"""The tree's kind (``World.from_dir``), for a tree ``gwenv`` can run
|
|
33
|
+
in-process; a container benchmark package is not runnable here."""
|
|
41
34
|
from .schema import bundle_mode
|
|
42
35
|
|
|
43
36
|
bundle_mode(bundle_path) # Orphan/invalid schema declarations never downgrade.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
52
|
-
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
53
|
-
)
|
|
37
|
+
world = World.from_dir(bundle_path)
|
|
38
|
+
if not world.local_runtime:
|
|
39
|
+
raise EnvironmentNotReady(
|
|
40
|
+
f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
|
|
41
|
+
"a harbor task tree (no */task.toml) — nothing gwenv can run locally."
|
|
42
|
+
)
|
|
43
|
+
return world.kind
|
|
54
44
|
|
|
55
45
|
|
|
56
46
|
# ---------------------------------------------------------------------------
|
|
@@ -107,6 +107,7 @@ from typing import Any, Callable
|
|
|
107
107
|
from . import snapshot as snapshots
|
|
108
108
|
from .compiler import check_bundle, compile_bundle
|
|
109
109
|
from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
|
|
110
|
+
from .grading import keyed_entities
|
|
110
111
|
from .store import SQLiteStore
|
|
111
112
|
from .validation import ContractError, fail, read_json
|
|
112
113
|
|
|
@@ -719,15 +720,12 @@ class WorldState:
|
|
|
719
720
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
720
721
|
return {name: row.get(name) for name in self.contract.entities[entity]["primary_key"]}
|
|
721
722
|
|
|
723
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
724
|
+
schema = self.contract.entities[entity].get("schema") or {}
|
|
725
|
+
return set((schema.get("properties") or {}).keys())
|
|
726
|
+
|
|
722
727
|
def dump(self) -> dict:
|
|
723
|
-
|
|
724
|
-
for entity, spec in self.contract.entities.items():
|
|
725
|
-
keyed = {}
|
|
726
|
-
for row in self.rows(entity):
|
|
727
|
-
key = ":".join(str(row.get(field)) for field in spec["primary_key"])
|
|
728
|
-
keyed[key] = row
|
|
729
|
-
out[entity] = keyed
|
|
730
|
-
return out
|
|
728
|
+
return keyed_entities(self.contract.entities, self.rows)
|
|
731
729
|
|
|
732
730
|
def describe_snapshot(self) -> dict | None:
|
|
733
731
|
"""What ``/__world/state`` reports beside the rows: the snapshot in use, or null."""
|
|
@@ -791,6 +789,17 @@ def _key_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str,
|
|
|
791
789
|
return {name: name for name in operation.placeholders if name in keys}
|
|
792
790
|
|
|
793
791
|
|
|
792
|
+
def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
|
|
793
|
+
"""What a read route with no ``filter`` implies from its query string: each
|
|
794
|
+
parameter that names a top-level field of the entity narrows to rows carrying
|
|
795
|
+
that value (``GET /accounts?status=frozen``), the way every vendor's list
|
|
796
|
+
endpoint reads. Paging parameters and names the entity does not declare are
|
|
797
|
+
left to the pager and the handler."""
|
|
798
|
+
fields = ctx.fields_of(entity)
|
|
799
|
+
paging = {ctx.settings.paging.limit_param, ctx.settings.paging.offset_param, ctx.settings.paging.cursor_param}
|
|
800
|
+
return {name: name for name in sent if name in fields and name not in paging}
|
|
801
|
+
|
|
802
|
+
|
|
794
803
|
def _compare(have: str, wanted: str, mode: str) -> bool:
|
|
795
804
|
if mode == "contains":
|
|
796
805
|
return wanted in have
|
|
@@ -976,6 +985,9 @@ class ApiContext:
|
|
|
976
985
|
def key_of(self, entity: str, row: dict) -> dict:
|
|
977
986
|
return self.state.key_of(entity, row)
|
|
978
987
|
|
|
988
|
+
def fields_of(self, entity: str) -> set[str]:
|
|
989
|
+
return self.state.fields_of(entity)
|
|
990
|
+
|
|
979
991
|
# -- the live feed
|
|
980
992
|
def emit(self, event: dict) -> dict:
|
|
981
993
|
"""Record what this write changed for a page that polls (:meth:`WorldState.emit`)."""
|
|
@@ -1339,12 +1351,17 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
|
|
|
1339
1351
|
for key, value in path_params.items():
|
|
1340
1352
|
path = path.replace("{" + str(key) + "}", urllib.parse.quote(str(value), safe=""))
|
|
1341
1353
|
query = args.get("query") if isinstance(args.get("query"), dict) else {}
|
|
1342
|
-
if query:
|
|
1343
|
-
path = f"{path}?" + urllib.parse.urlencode(query)
|
|
1344
1354
|
body = args.get("body") if isinstance(args.get("body"), dict) else None
|
|
1345
|
-
if body
|
|
1346
|
-
|
|
1355
|
+
leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
|
|
1356
|
+
if operation.method == "GET" and leftovers:
|
|
1357
|
+
# A GET carries no body: a call written like a tool's (``list_accounts {"status": "frozen"}``)
|
|
1358
|
+
# means its query string, so the filter it asks for reaches the route.
|
|
1359
|
+
query = {**{k: v for k, v in leftovers.items() if not isinstance(v, (dict, list))}, **query}
|
|
1360
|
+
body = None
|
|
1361
|
+
elif body is None:
|
|
1347
1362
|
body = leftovers or None
|
|
1363
|
+
if query:
|
|
1364
|
+
path = f"{path}?" + urllib.parse.urlencode({k: str(v) for k, v in query.items()})
|
|
1348
1365
|
raw = json.dumps(body).encode("utf-8") if body is not None and operation.method != "GET" else b""
|
|
1349
1366
|
base = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
|
|
1350
1367
|
return operation.method, f"{base}{path}", raw
|
|
@@ -1627,17 +1644,19 @@ class WorldApp:
|
|
|
1627
1644
|
if not operation.entity:
|
|
1628
1645
|
return 501, {"error": "not_modelled", "operation": operation.name}
|
|
1629
1646
|
entity = operation.entity
|
|
1630
|
-
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1631
|
-
# reads as "either", not as "a" with "b" silently dropped.
|
|
1632
|
-
values: dict[str, str | list[str]] = {
|
|
1633
|
-
**{k: v for k, v in ctx.query_all.items() if k in operation.params},
|
|
1634
|
-
**args,
|
|
1635
|
-
}
|
|
1636
1647
|
omit_done_cursor = ctx.settings.omits_done_cursor(op)
|
|
1637
1648
|
if op.action == "create":
|
|
1638
1649
|
row = ctx.insert(entity, _bind(op, ctx, body))
|
|
1639
1650
|
return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
|
|
1640
1651
|
filters = op.filter or _key_filter(operation, ctx, entity)
|
|
1652
|
+
if not op.filter and op.action == "read":
|
|
1653
|
+
filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
|
|
1654
|
+
# A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
|
|
1655
|
+
# reads as "either", not as "a" with "b" silently dropped.
|
|
1656
|
+
values: dict[str, str | list[str]] = {
|
|
1657
|
+
**{k: v for k, v in ctx.query_all.items() if k in operation.params or k in filters},
|
|
1658
|
+
**args,
|
|
1659
|
+
}
|
|
1641
1660
|
if op.action in ("update", "delete") and not filters:
|
|
1642
1661
|
fail(
|
|
1643
1662
|
"spec",
|
|
@@ -35,7 +35,9 @@ from contextlib import contextmanager
|
|
|
35
35
|
from pathlib import Path
|
|
36
36
|
from typing import Any, Iterable, Iterator
|
|
37
37
|
|
|
38
|
+
from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
|
|
38
39
|
from .compiler import bundle_mode, check_bundle, compile_bundle
|
|
40
|
+
from .proof import SCRATCH_IGNORE, RowProof
|
|
39
41
|
from .store import SQLiteStore
|
|
40
42
|
from .validation import ContractError, digest, encoded, fail, journal_paths, read_json, validate
|
|
41
43
|
|
|
@@ -48,14 +50,8 @@ INLINE_BYTES = 4 * 1024 * 1024
|
|
|
48
50
|
#: The most rows one ``entity_query`` page returns, the vendor pages' cap.
|
|
49
51
|
QUERY_LIMIT = 1000
|
|
50
52
|
QUERY_DEFAULT_LIMIT = 100
|
|
51
|
-
KIND_SCHEMA =
|
|
52
|
-
KIND_MOCK_TOOLS = "mock-tools"
|
|
53
|
-
KIND_API = "api"
|
|
54
|
-
KIND_HARBOR = "harbor"
|
|
55
|
-
KIND_UNKNOWN = "unknown"
|
|
53
|
+
KIND_SCHEMA = SCHEMA_MANIFEST_KIND
|
|
56
54
|
MAPPING = {"id": "data-import", "version": "1"}
|
|
57
|
-
#: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
|
|
58
|
-
SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
|
|
59
55
|
|
|
60
56
|
Rows = dict[str, list[dict]]
|
|
61
57
|
|
|
@@ -87,17 +83,13 @@ def parse_rows(paths: Iterable[str | Path], *, entity: str | None = None) -> Row
|
|
|
87
83
|
|
|
88
84
|
|
|
89
85
|
def world_kind(root: str | Path) -> str:
|
|
90
|
-
"""
|
|
86
|
+
"""The kind the worlds paths report (``World.from_dir(...).reported_kind``):
|
|
87
|
+
``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
|
|
88
|
+
``verifiers-v1``. An invalid schema declaration refuses here rather than
|
|
89
|
+
downgrading the tree."""
|
|
91
90
|
root = Path(root)
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if (root / "gateway-api-world.toml").is_file():
|
|
95
|
-
return KIND_API
|
|
96
|
-
if (root / "gateway-env.toml").is_file():
|
|
97
|
-
return KIND_MOCK_TOOLS
|
|
98
|
-
if list(root.glob("tasks/*/task.toml")) or list(root.glob("*/task.toml")):
|
|
99
|
-
return KIND_HARBOR
|
|
100
|
-
return KIND_UNKNOWN
|
|
91
|
+
bundle_mode(root)
|
|
92
|
+
return World.from_dir(root).reported_kind
|
|
101
93
|
|
|
102
94
|
|
|
103
95
|
def _refuse_kind(kind: str, root: Path) -> None:
|
|
@@ -135,6 +127,7 @@ def import_rows(
|
|
|
135
127
|
merge: str = "upsert",
|
|
136
128
|
batch_id: str | None = None,
|
|
137
129
|
redact: str = "off",
|
|
130
|
+
dry_run: bool = False,
|
|
138
131
|
) -> dict:
|
|
139
132
|
"""Write ``rows`` into the world at ``root`` the way its kind seeds.
|
|
140
133
|
|
|
@@ -150,6 +143,13 @@ def import_rows(
|
|
|
150
143
|
``redact`` — ``off`` (default) stores the rows as given; ``apply`` runs them
|
|
151
144
|
through the world's ``[redact]`` policy first; ``refuse`` rejects a row that
|
|
152
145
|
carries plaintext in a redacted field (:mod:`.redaction`).
|
|
146
|
+
|
|
147
|
+
``dry_run`` proves the rows and writes nothing (``changed`` always ``False``):
|
|
148
|
+
the :class:`.proof.RowProof` the tree calls for — the same gate over a scratch
|
|
149
|
+
copy of an authored tree, the platform's batch lane over the hosted snapshot a
|
|
150
|
+
pinned checkout carries, or shape-only with references and unique keys
|
|
151
|
+
``deferred`` to the platform when the pinned checkout lacks its snapshot. A
|
|
152
|
+
pinned checkout refuses a real import: the rows belong on the platform.
|
|
153
153
|
"""
|
|
154
154
|
from .redaction import parse_mode, redact_rows, require_policy
|
|
155
155
|
|
|
@@ -160,6 +160,8 @@ def import_rows(
|
|
|
160
160
|
fail("merge", f"merge must be one of {', '.join(MERGES)}", "/merge")
|
|
161
161
|
if not isinstance(rows, dict) or not all(isinstance(v, list) for v in rows.values()):
|
|
162
162
|
fail("rows", "rows must map entity names to arrays of rows", "/rows")
|
|
163
|
+
if dry_run:
|
|
164
|
+
return RowProof.of(root).run(rows, mode=mode, merge=merge, batch_id=batch_id, redact=redact)
|
|
163
165
|
redact = parse_mode(redact)
|
|
164
166
|
rows = redact_rows(require_policy(root, redact), rows, redact)
|
|
165
167
|
kind = world_kind(root)
|
|
@@ -186,7 +188,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
|
|
|
186
188
|
reports."""
|
|
187
189
|
from .batch import shape_refusal
|
|
188
190
|
|
|
189
|
-
contract = compile_bundle(root)
|
|
191
|
+
contract = compile_bundle(root, snapshots=False)
|
|
190
192
|
refusals = [
|
|
191
193
|
refused
|
|
192
194
|
for entity, listed in rows.items()
|
|
@@ -220,7 +222,7 @@ def _merged(root: Path, rows: Rows, merge: str) -> Rows:
|
|
|
220
222
|
plan will refuse (not an object, key missing) is left in place for it."""
|
|
221
223
|
if merge != "fill":
|
|
222
224
|
return rows
|
|
223
|
-
contract = compile_bundle(root)
|
|
225
|
+
contract = compile_bundle(root, snapshots=False)
|
|
224
226
|
settled: Rows = {}
|
|
225
227
|
for entity, listed in rows.items():
|
|
226
228
|
first: dict[tuple, dict] = {}
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"""One grader path: a task's grader over a world's end state.
|
|
2
|
+
|
|
3
|
+
A task grades one of three ways — ``assertions`` (declarative checks, in
|
|
4
|
+
process), ``python`` (a ``verify.py`` fed the state on stdin, printing the
|
|
5
|
+
rewards on stdout) or ``rubric`` (judged by the platform; here, pending). The
|
|
6
|
+
World Host grades a session this way and the batch runner grades a stored
|
|
7
|
+
task this way, whether the world is stored as itself (a schema world) or
|
|
8
|
+
packaged as an api bundle: :class:`TaskGrader` is the one place the three
|
|
9
|
+
kinds are told apart, and :func:`keyed_entities` is the one shape of the state
|
|
10
|
+
they read (``{entity: {primary key: row}}``, the api runtime's dump).
|
|
11
|
+
|
|
12
|
+
The two hosts differ only in how they run world-authored code: the World Host
|
|
13
|
+
confines the verifier like a worker (:class:`VerifierSandbox` subclass in
|
|
14
|
+
:mod:`workers`), the runner's container is the confinement. A verifier that
|
|
15
|
+
cannot deliver a grade — missing, timed out, crashed, printed no JSON — is a
|
|
16
|
+
:class:`GraderError`; the host turns it into a refusal, the runner into a
|
|
17
|
+
null-graded sample that names the reason. Null is "ungraded", never zero.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
from collections.abc import Callable, Iterable, Mapping
|
|
26
|
+
from dataclasses import dataclass, field
|
|
27
|
+
from pathlib import Path
|
|
28
|
+
from typing import Any, Protocol
|
|
29
|
+
|
|
30
|
+
from .tasks import evaluate_assertions
|
|
31
|
+
|
|
32
|
+
GRADER_KINDS = ("assertions", "python", "rubric")
|
|
33
|
+
VERIFY_TIMEOUT_S = 30.0
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class GraderError(Exception):
|
|
37
|
+
"""A grader that could not deliver a grade, with the reason a reader acts on."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True)
|
|
41
|
+
class Grade:
|
|
42
|
+
"""A task's grade: the scalar ``reward`` (None when ungraded) and every
|
|
43
|
+
named dimension under ``rewards`` (nulls kept). ``pending`` is a grade the
|
|
44
|
+
platform still owes (a rubric)."""
|
|
45
|
+
|
|
46
|
+
reward: float | None
|
|
47
|
+
rewards: dict[str, float | None] = field(default_factory=dict)
|
|
48
|
+
pending: bool = False
|
|
49
|
+
grader: str | None = None
|
|
50
|
+
|
|
51
|
+
def as_dict(self) -> dict[str, Any]:
|
|
52
|
+
"""The wire shape ``grade`` answers with (and ``verify.py`` prints)."""
|
|
53
|
+
out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
|
|
54
|
+
if self.pending:
|
|
55
|
+
out["pending"] = True
|
|
56
|
+
out["grader"] = self.grader
|
|
57
|
+
return out
|
|
58
|
+
|
|
59
|
+
@staticmethod
|
|
60
|
+
def of_rewards(printed: Mapping[str, Any]) -> "Grade":
|
|
61
|
+
"""The grade a verifier printed: the scalar under ``reward``, every
|
|
62
|
+
other key a dimension (non-numeric ones null)."""
|
|
63
|
+
return Grade(
|
|
64
|
+
reward=_number(printed.get("reward")),
|
|
65
|
+
rewards={key: _number(value) for key, value in printed.items() if key != "reward"},
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _number(value: Any) -> float | None:
|
|
70
|
+
if isinstance(value, (int, float)):
|
|
71
|
+
return float(value)
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class VerifierSandbox:
|
|
76
|
+
"""How a ``verify.py`` is run: the argv and the environment. The base runs
|
|
77
|
+
it with this interpreter and the inherited environment — a container that
|
|
78
|
+
is itself the confinement (the batch runner). A host that must confine the
|
|
79
|
+
verifier further subclasses this."""
|
|
80
|
+
|
|
81
|
+
def command(self, script: Path) -> list[str]:
|
|
82
|
+
return [sys.executable, str(script)]
|
|
83
|
+
|
|
84
|
+
def environment(self, home: Path) -> dict[str, str] | None:
|
|
85
|
+
return None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class TaskGrader:
|
|
89
|
+
"""A task's grader, ready to grade a state dump.
|
|
90
|
+
|
|
91
|
+
``grader`` is the spec's normalised ``grader`` table. A python grader's
|
|
92
|
+
source must already be on disk at ``verify`` (the session's or the run's
|
|
93
|
+
``<task>/checks/verify.py``); a bundled scenario's ``verify.py`` is graded
|
|
94
|
+
as a python grader with no table of its own."""
|
|
95
|
+
|
|
96
|
+
def __init__(
|
|
97
|
+
self,
|
|
98
|
+
grader: Mapping[str, Any],
|
|
99
|
+
*,
|
|
100
|
+
verify: Path | None = None,
|
|
101
|
+
sandbox: VerifierSandbox | None = None,
|
|
102
|
+
home: Path | None = None,
|
|
103
|
+
default_timeout_s: float = VERIFY_TIMEOUT_S,
|
|
104
|
+
) -> None:
|
|
105
|
+
kind = grader.get("kind")
|
|
106
|
+
if kind not in GRADER_KINDS:
|
|
107
|
+
raise GraderError(f"unknown grader kind {kind!r}; a task grades by {', '.join(GRADER_KINDS)}")
|
|
108
|
+
self.grader = dict(grader)
|
|
109
|
+
self.kind: str = kind
|
|
110
|
+
self.verify = verify
|
|
111
|
+
self.sandbox = sandbox or VerifierSandbox()
|
|
112
|
+
self.home = home
|
|
113
|
+
timeout = grader.get("timeoutSeconds")
|
|
114
|
+
own = None if isinstance(timeout, bool) else _number(timeout)
|
|
115
|
+
self.timeout_s = own if own is not None and own > 0 else float(default_timeout_s)
|
|
116
|
+
|
|
117
|
+
@staticmethod
|
|
118
|
+
def for_verifier(
|
|
119
|
+
verify: Path,
|
|
120
|
+
*,
|
|
121
|
+
sandbox: VerifierSandbox | None = None,
|
|
122
|
+
home: Path | None = None,
|
|
123
|
+
timeout_s: float = VERIFY_TIMEOUT_S,
|
|
124
|
+
) -> "TaskGrader":
|
|
125
|
+
"""A bundled scenario's ``verify.py``: a python grader with the host's timeout."""
|
|
126
|
+
return TaskGrader({"kind": "python"}, verify=verify, sandbox=sandbox, home=home, default_timeout_s=timeout_s)
|
|
127
|
+
|
|
128
|
+
@property
|
|
129
|
+
def pending(self) -> bool:
|
|
130
|
+
"""Whether the platform grades this task (a rubric), not this process."""
|
|
131
|
+
return self.kind == "rubric"
|
|
132
|
+
|
|
133
|
+
def grade(self, state: Mapping[str, Any]) -> Grade:
|
|
134
|
+
graders: dict[str, Callable[[Mapping[str, Any]], Grade]] = {
|
|
135
|
+
"assertions": self._assertions,
|
|
136
|
+
"python": self._python,
|
|
137
|
+
"rubric": self._rubric,
|
|
138
|
+
}
|
|
139
|
+
return graders[self.kind](state)
|
|
140
|
+
|
|
141
|
+
def _assertions(self, state: Mapping[str, Any]) -> Grade:
|
|
142
|
+
graded = evaluate_assertions(self.grader["checks"], dict(state))
|
|
143
|
+
return Grade(reward=graded["reward"], rewards=graded["rewards"])
|
|
144
|
+
|
|
145
|
+
def _rubric(self, state: Mapping[str, Any]) -> Grade:
|
|
146
|
+
return Grade(reward=None, rewards={}, pending=True, grader="rubric")
|
|
147
|
+
|
|
148
|
+
def _python(self, state: Mapping[str, Any]) -> Grade:
|
|
149
|
+
verify = self.verify
|
|
150
|
+
if verify is None or not verify.is_file():
|
|
151
|
+
raise GraderError(f"the task's verify.py is missing at {verify}")
|
|
152
|
+
home = self.home if self.home is not None else verify.parent
|
|
153
|
+
home.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
try:
|
|
155
|
+
proc = subprocess.run(
|
|
156
|
+
self.sandbox.command(verify),
|
|
157
|
+
input=json.dumps(state),
|
|
158
|
+
check=False,
|
|
159
|
+
capture_output=True,
|
|
160
|
+
text=True,
|
|
161
|
+
timeout=self.timeout_s,
|
|
162
|
+
cwd=str(verify.parent),
|
|
163
|
+
env=self.sandbox.environment(home),
|
|
164
|
+
)
|
|
165
|
+
except subprocess.TimeoutExpired:
|
|
166
|
+
raise GraderError(f"verify.py timed out after {self.timeout_s:g}s") from None
|
|
167
|
+
if proc.returncode != 0:
|
|
168
|
+
raise GraderError(f"verify.py exited {proc.returncode}: {proc.stderr.strip()[-800:]}")
|
|
169
|
+
try:
|
|
170
|
+
printed = json.loads(proc.stdout.strip().splitlines()[-1])
|
|
171
|
+
except (json.JSONDecodeError, IndexError):
|
|
172
|
+
raise GraderError(f"verify.py printed no rewards JSON: {proc.stdout.strip()[-400:]}") from None
|
|
173
|
+
if not isinstance(printed, dict):
|
|
174
|
+
raise GraderError("verify.py rewards output is not an object")
|
|
175
|
+
return Grade.of_rewards(printed)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def keyed_entities(
|
|
179
|
+
entities: Mapping[str, Mapping[str, Any]], rows_of: Callable[[str], Iterable[Mapping[str, Any]]]
|
|
180
|
+
) -> dict:
|
|
181
|
+
"""A world's rows the way a grader reads them: every contract entity by
|
|
182
|
+
name, its rows keyed by their primary key (``a:b`` for a composite key) —
|
|
183
|
+
the api runtime's ``/__world/state`` dump, so a check or a ``verify.py``
|
|
184
|
+
written against one shape of the world grades the other."""
|
|
185
|
+
out: dict[str, dict[str, Any]] = {}
|
|
186
|
+
for entity, spec in entities.items():
|
|
187
|
+
keyed: dict[str, Any] = {}
|
|
188
|
+
for row in rows_of(entity):
|
|
189
|
+
keyed[":".join(str(row.get(name)) for name in spec["primary_key"])] = row
|
|
190
|
+
out[entity] = keyed
|
|
191
|
+
return out
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
class StoredWorld(Protocol):
|
|
195
|
+
"""What a grader needs of a live world: its contract and the store holding
|
|
196
|
+
its rows — a World Host session's world and the batch runner's are both."""
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def contract(self) -> Any: ...
|
|
200
|
+
|
|
201
|
+
@property
|
|
202
|
+
def store(self) -> Any: ...
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def world_state(root: StoredWorld | None, dependencies: Mapping[str, StoredWorld]) -> dict:
|
|
206
|
+
"""The rows as a task's grader reads them, on the host and in the batch
|
|
207
|
+
runner alike: the root's entities by name and each dependency's as
|
|
208
|
+
``<alias>.<entity>``, every entity's rows keyed by primary key — the api
|
|
209
|
+
runtime's dump, so one ``verify.py`` grades the world in either shape. A
|
|
210
|
+
world without a contract (a legacy tools bundle) contributes nothing, as
|
|
211
|
+
does a task set that owns no root world (``root`` None)."""
|
|
212
|
+
out: dict[str, Any] = dict(_keyed_rows_of(root))
|
|
213
|
+
for alias, world in dependencies.items():
|
|
214
|
+
for entity, rows in _keyed_rows_of(world).items():
|
|
215
|
+
out[f"{alias}.{entity}"] = rows
|
|
216
|
+
return out
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def _keyed_rows_of(world: StoredWorld | None) -> dict[str, dict]:
|
|
220
|
+
if world is None:
|
|
221
|
+
return {}
|
|
222
|
+
store, contract = world.store, world.contract
|
|
223
|
+
if store is None or contract is None:
|
|
224
|
+
return {}
|
|
225
|
+
return keyed_entities(contract.entities, store.list)
|