@withgateway/sdk 0.2.4 → 0.2.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/cli.js +1798 -1713
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +2412 -2327
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +47 -31
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +43 -27
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/index.d.mts +1 -1
  10. package/dist/index.d.ts +1 -1
  11. package/dist/index.js +390 -287
  12. package/dist/index.js.map +1 -1
  13. package/dist/index.mjs +375 -271
  14. package/dist/index.mjs.map +1 -1
  15. package/dist/traces/index.js +36 -20
  16. package/dist/traces/index.js.map +1 -1
  17. package/dist/traces/index.mjs +32 -16
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +2 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +2 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +26 -22
  24. package/dist/worlds/index.d.ts +26 -22
  25. package/dist/worlds/index.js +307 -203
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +297 -193
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +2 -1
  30. package/runtime/gatewaysdk/environment/_bundle.py +14 -24
  31. package/runtime/gatewaysdk/environment/schema/api.py +37 -18
  32. package/runtime/gatewaysdk/environment/schema/data.py +21 -19
  33. package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
  34. package/runtime/gatewaysdk/environment/schema/host.py +156 -546
  35. package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
  36. package/runtime/gatewaysdk/environment/schema/local.py +32 -126
  37. package/runtime/gatewaysdk/environment/schema/platform.py +14 -22
  38. package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
  39. package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
  40. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  41. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +6 -4
  42. package/runtime/gatewaysdk/environment/schema/tools_world.py +75 -23
  43. package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
  44. package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
  45. package/runtime/gatewaysdk/environment/world.py +391 -0
  46. package/skills/hooks-getting-started/SKILL.md +6 -4
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@withgateway/sdk",
3
- "version": "0.2.4",
3
+ "version": "0.2.5",
4
4
  "description": "Gateway SDK for TypeScript - A/B experiments, tracing, and observability",
5
5
  "main": "./dist/index.js",
6
6
  "module": "./dist/index.mjs",
@@ -104,6 +104,7 @@
104
104
  "typecheck:examples": "tsc -p tsconfig.examples.json --noEmit",
105
105
  "test": "vitest",
106
106
  "test:run": "vitest run",
107
+ "test:integration": "vitest run --config vitest.integration.config.ts",
107
108
  "lint": "eslint src --ext .ts",
108
109
  "clean": "rm -rf dist",
109
110
  "example": "tsx examples/test-ab-experiment.ts",
@@ -1,11 +1,9 @@
1
1
  """Bundle inspection: kind detection, manifest parsing, handler loading.
2
2
 
3
- Two structurally-detected environment kinds (spec §3.1 + addendum):
4
-
5
- - **mock-tools** — ``gateway-env.toml`` at the bundle root, ``tools/*.py``
6
- handlers, ``db/schema.sql`` (+ ``db/seed.sql``), ``verifiers/*.sql``.
7
- - **harbor** — a tree of Harbor task dirs (``tasks/*/task.toml`` or
8
- ``*/task.toml``), mirroring ``benchmark_hub.harbor_adapter.detect_env_kind``.
3
+ Kind detection is :func:`gatewaysdk.environment.world.World.from_dir`; the
4
+ kinds ``gwenv`` runs in-process are **mock-tools** (``gateway-env.toml`` at the
5
+ bundle root, ``tools/*.py`` handlers, ``db/schema.sql``, ``verifiers/*.sql``)
6
+ and **harbor** (a tree of task dirs, ``tasks/*/task.toml`` or ``*/task.toml``).
9
7
 
10
8
  Handler loading executes each ``tools/*.py`` with this SDK's
11
9
  :mod:`gatewaysdk.environment.runtime` injected into ``sys.modules`` as
@@ -27,30 +25,22 @@ from typing import Any, Callable, Dict, List, Optional
27
25
 
28
26
  from . import runtime
29
27
  from .errors import EnvironmentNotReady, WorldStartError
30
-
31
- KIND_MOCK_TOOLS = "mock-tools"
32
- KIND_API = "api"
33
- KIND_HARBOR = "harbor"
28
+ from .world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, World # noqa: F401
34
29
 
35
30
 
36
31
  def detect_kind(bundle_path: Path) -> str:
37
- """"api" iff ``gateway-api-world.toml`` sits at the root; else "mock-tools"
38
- iff ``gateway-env.toml`` does; else "harbor" iff the tree holds Harbor task
39
- dirs (same glob heuristics as the runner's ``detect_env_kind``); else the
40
- bundle is not runnable here."""
32
+ """The tree's kind (``World.from_dir``), for a tree ``gwenv`` can run
33
+ in-process; a container benchmark package is not runnable here."""
41
34
  from .schema import bundle_mode
42
35
 
43
36
  bundle_mode(bundle_path) # Orphan/invalid schema declarations never downgrade.
44
- if (bundle_path / "gateway-api-world.toml").is_file():
45
- return KIND_API
46
- if (bundle_path / "gateway-env.toml").is_file():
47
- return KIND_MOCK_TOOLS
48
- if list(bundle_path.glob("tasks/*/task.toml")) or list(bundle_path.glob("*/task.toml")):
49
- return KIND_HARBOR
50
- raise EnvironmentNotReady(
51
- f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
52
- "a harbor task tree (no */task.toml) — nothing gwenv can run locally."
53
- )
37
+ world = World.from_dir(bundle_path)
38
+ if not world.local_runtime:
39
+ raise EnvironmentNotReady(
40
+ f"{bundle_path} is neither a mock-tools bundle (no gateway-env.toml) nor "
41
+ "a harbor task tree (no */task.toml) — nothing gwenv can run locally."
42
+ )
43
+ return world.kind
54
44
 
55
45
 
56
46
  # ---------------------------------------------------------------------------
@@ -107,6 +107,7 @@ from typing import Any, Callable
107
107
  from . import snapshot as snapshots
108
108
  from .compiler import check_bundle, compile_bundle
109
109
  from .connector import DEFAULT_TOKEN_PATH, SPEC, Operation, Spec, load_spec, stored_forms
110
+ from .grading import keyed_entities
110
111
  from .store import SQLiteStore
111
112
  from .validation import ContractError, fail, read_json
112
113
 
@@ -719,15 +720,12 @@ class WorldState:
719
720
  def key_of(self, entity: str, row: dict) -> dict:
720
721
  return {name: row.get(name) for name in self.contract.entities[entity]["primary_key"]}
721
722
 
723
+ def fields_of(self, entity: str) -> set[str]:
724
+ schema = self.contract.entities[entity].get("schema") or {}
725
+ return set((schema.get("properties") or {}).keys())
726
+
722
727
  def dump(self) -> dict:
723
- out: dict[str, dict[str, dict]] = {}
724
- for entity, spec in self.contract.entities.items():
725
- keyed = {}
726
- for row in self.rows(entity):
727
- key = ":".join(str(row.get(field)) for field in spec["primary_key"])
728
- keyed[key] = row
729
- out[entity] = keyed
730
- return out
728
+ return keyed_entities(self.contract.entities, self.rows)
731
729
 
732
730
  def describe_snapshot(self) -> dict | None:
733
731
  """What ``/__world/state`` reports beside the rows: the snapshot in use, or null."""
@@ -791,6 +789,17 @@ def _key_filter(operation: Operation, ctx: ApiContext, entity: str) -> dict[str,
791
789
  return {name: name for name in operation.placeholders if name in keys}
792
790
 
793
791
 
792
+ def _query_field_filter(ctx: ApiContext, entity: str, sent: dict[str, list[str]]) -> dict[str, str]:
793
+ """What a read route with no ``filter`` implies from its query string: each
794
+ parameter that names a top-level field of the entity narrows to rows carrying
795
+ that value (``GET /accounts?status=frozen``), the way every vendor's list
796
+ endpoint reads. Paging parameters and names the entity does not declare are
797
+ left to the pager and the handler."""
798
+ fields = ctx.fields_of(entity)
799
+ paging = {ctx.settings.paging.limit_param, ctx.settings.paging.offset_param, ctx.settings.paging.cursor_param}
800
+ return {name: name for name in sent if name in fields and name not in paging}
801
+
802
+
794
803
  def _compare(have: str, wanted: str, mode: str) -> bool:
795
804
  if mode == "contains":
796
805
  return wanted in have
@@ -976,6 +985,9 @@ class ApiContext:
976
985
  def key_of(self, entity: str, row: dict) -> dict:
977
986
  return self.state.key_of(entity, row)
978
987
 
988
+ def fields_of(self, entity: str) -> set[str]:
989
+ return self.state.fields_of(entity)
990
+
979
991
  # -- the live feed
980
992
  def emit(self, event: dict) -> dict:
981
993
  """Record what this write changed for a page that polls (:meth:`WorldState.emit`)."""
@@ -1339,12 +1351,17 @@ def operation_request(spec: Spec, operation: Operation, args: dict) -> tuple[str
1339
1351
  for key, value in path_params.items():
1340
1352
  path = path.replace("{" + str(key) + "}", urllib.parse.quote(str(value), safe=""))
1341
1353
  query = args.get("query") if isinstance(args.get("query"), dict) else {}
1342
- if query:
1343
- path = f"{path}?" + urllib.parse.urlencode(query)
1344
1354
  body = args.get("body") if isinstance(args.get("body"), dict) else None
1345
- if body is None:
1346
- leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
1355
+ leftovers = {k: v for k, v in args.items() if k not in ("path", "query", "body") and k not in consumed}
1356
+ if operation.method == "GET" and leftovers:
1357
+ # A GET carries no body: a call written like a tool's (``list_accounts {"status": "frozen"}``)
1358
+ # means its query string, so the filter it asks for reaches the route.
1359
+ query = {**{k: v for k, v in leftovers.items() if not isinstance(v, (dict, list))}, **query}
1360
+ body = None
1361
+ elif body is None:
1347
1362
  body = leftovers or None
1363
+ if query:
1364
+ path = f"{path}?" + urllib.parse.urlencode({k: str(v) for k, v in query.items()})
1348
1365
  raw = json.dumps(body).encode("utf-8") if body is not None and operation.method != "GET" else b""
1349
1366
  base = urllib.parse.urlsplit(spec.base_url).path.rstrip("/")
1350
1367
  return operation.method, f"{base}{path}", raw
@@ -1627,17 +1644,19 @@ class WorldApp:
1627
1644
  if not operation.entity:
1628
1645
  return 501, {"error": "not_modelled", "operation": operation.name}
1629
1646
  entity = operation.entity
1630
- # A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
1631
- # reads as "either", not as "a" with "b" silently dropped.
1632
- values: dict[str, str | list[str]] = {
1633
- **{k: v for k, v in ctx.query_all.items() if k in operation.params},
1634
- **args,
1635
- }
1636
1647
  omit_done_cursor = ctx.settings.omits_done_cursor(op)
1637
1648
  if op.action == "create":
1638
1649
  row = ctx.insert(entity, _bind(op, ctx, body))
1639
1650
  return op.status, row if op.single else wrap([row], ctx.settings, op.envelope)
1640
1651
  filters = op.filter or _key_filter(operation, ctx, entity)
1652
+ if not op.filter and op.action == "read":
1653
+ filters = {**_query_field_filter(ctx, entity, ctx.query_all), **filters}
1654
+ # A declared parameter filters on every value the client sent, so ``status[]=a&status[]=b``
1655
+ # reads as "either", not as "a" with "b" silently dropped.
1656
+ values: dict[str, str | list[str]] = {
1657
+ **{k: v for k, v in ctx.query_all.items() if k in operation.params or k in filters},
1658
+ **args,
1659
+ }
1641
1660
  if op.action in ("update", "delete") and not filters:
1642
1661
  fail(
1643
1662
  "spec",
@@ -35,7 +35,9 @@ from contextlib import contextmanager
35
35
  from pathlib import Path
36
36
  from typing import Any, Iterable, Iterator
37
37
 
38
+ from ..world import KIND_API, KIND_HARBOR, KIND_MOCK_TOOLS, SCHEMA_MANIFEST_KIND, World
38
39
  from .compiler import bundle_mode, check_bundle, compile_bundle
40
+ from .proof import SCRATCH_IGNORE, RowProof
39
41
  from .store import SQLiteStore
40
42
  from .validation import ContractError, digest, encoded, fail, journal_paths, read_json, validate
41
43
 
@@ -48,14 +50,8 @@ INLINE_BYTES = 4 * 1024 * 1024
48
50
  #: The most rows one ``entity_query`` page returns, the vendor pages' cap.
49
51
  QUERY_LIMIT = 1000
50
52
  QUERY_DEFAULT_LIMIT = 100
51
- KIND_SCHEMA = "schema"
52
- KIND_MOCK_TOOLS = "mock-tools"
53
- KIND_API = "api"
54
- KIND_HARBOR = "harbor"
55
- KIND_UNKNOWN = "unknown"
53
+ KIND_SCHEMA = SCHEMA_MANIFEST_KIND
56
54
  MAPPING = {"id": "data-import", "version": "1"}
57
- #: Left out of scratch copies: captures, caches, and the snapshots (rebuilt from the data).
58
- SCRATCH_IGNORE = ("captures", "__pycache__", ".gateway", "state.sqlite", "state.json", ".state.sqlite.*")
59
55
 
60
56
  Rows = dict[str, list[dict]]
61
57
 
@@ -87,17 +83,13 @@ def parse_rows(paths: Iterable[str | Path], *, entity: str | None = None) -> Row
87
83
 
88
84
 
89
85
  def world_kind(root: str | Path) -> str:
90
- """``schema`` (gateway-world/1), ``mock-tools``, ``api``, ``harbor`` or ``unknown``."""
86
+ """The kind the worlds paths report (``World.from_dir(...).reported_kind``):
87
+ ``schema``, ``mock-tools``, ``api``, ``harbor``, ``verifiers`` or
88
+ ``verifiers-v1``. An invalid schema declaration refuses here rather than
89
+ downgrading the tree."""
91
90
  root = Path(root)
92
- if bundle_mode(root) is not None:
93
- return KIND_SCHEMA
94
- if (root / "gateway-api-world.toml").is_file():
95
- return KIND_API
96
- if (root / "gateway-env.toml").is_file():
97
- return KIND_MOCK_TOOLS
98
- if list(root.glob("tasks/*/task.toml")) or list(root.glob("*/task.toml")):
99
- return KIND_HARBOR
100
- return KIND_UNKNOWN
91
+ bundle_mode(root)
92
+ return World.from_dir(root).reported_kind
101
93
 
102
94
 
103
95
  def _refuse_kind(kind: str, root: Path) -> None:
@@ -135,6 +127,7 @@ def import_rows(
135
127
  merge: str = "upsert",
136
128
  batch_id: str | None = None,
137
129
  redact: str = "off",
130
+ dry_run: bool = False,
138
131
  ) -> dict:
139
132
  """Write ``rows`` into the world at ``root`` the way its kind seeds.
140
133
 
@@ -150,6 +143,13 @@ def import_rows(
150
143
  ``redact`` — ``off`` (default) stores the rows as given; ``apply`` runs them
151
144
  through the world's ``[redact]`` policy first; ``refuse`` rejects a row that
152
145
  carries plaintext in a redacted field (:mod:`.redaction`).
146
+
147
+ ``dry_run`` proves the rows and writes nothing (``changed`` always ``False``):
148
+ the :class:`.proof.RowProof` the tree calls for — the same gate over a scratch
149
+ copy of an authored tree, the platform's batch lane over the hosted snapshot a
150
+ pinned checkout carries, or shape-only with references and unique keys
151
+ ``deferred`` to the platform when the pinned checkout lacks its snapshot. A
152
+ pinned checkout refuses a real import: the rows belong on the platform.
153
153
  """
154
154
  from .redaction import parse_mode, redact_rows, require_policy
155
155
 
@@ -160,6 +160,8 @@ def import_rows(
160
160
  fail("merge", f"merge must be one of {', '.join(MERGES)}", "/merge")
161
161
  if not isinstance(rows, dict) or not all(isinstance(v, list) for v in rows.values()):
162
162
  fail("rows", "rows must map entity names to arrays of rows", "/rows")
163
+ if dry_run:
164
+ return RowProof.of(root).run(rows, mode=mode, merge=merge, batch_id=batch_id, redact=redact)
163
165
  redact = parse_mode(redact)
164
166
  rows = redact_rows(require_policy(root, redact), rows, redact)
165
167
  kind = world_kind(root)
@@ -186,7 +188,7 @@ def _refuse_shapes(root: Path, rows: Rows) -> None:
186
188
  reports."""
187
189
  from .batch import shape_refusal
188
190
 
189
- contract = compile_bundle(root)
191
+ contract = compile_bundle(root, snapshots=False)
190
192
  refusals = [
191
193
  refused
192
194
  for entity, listed in rows.items()
@@ -220,7 +222,7 @@ def _merged(root: Path, rows: Rows, merge: str) -> Rows:
220
222
  plan will refuse (not an object, key missing) is left in place for it."""
221
223
  if merge != "fill":
222
224
  return rows
223
- contract = compile_bundle(root)
225
+ contract = compile_bundle(root, snapshots=False)
224
226
  settled: Rows = {}
225
227
  for entity, listed in rows.items():
226
228
  first: dict[tuple, dict] = {}
@@ -0,0 +1,225 @@
1
+ """One grader path: a task's grader over a world's end state.
2
+
3
+ A task grades one of three ways — ``assertions`` (declarative checks, in
4
+ process), ``python`` (a ``verify.py`` fed the state on stdin, printing the
5
+ rewards on stdout) or ``rubric`` (judged by the platform; here, pending). The
6
+ World Host grades a session this way and the batch runner grades a stored
7
+ task this way, whether the world is stored as itself (a schema world) or
8
+ packaged as an api bundle: :class:`TaskGrader` is the one place the three
9
+ kinds are told apart, and :func:`keyed_entities` is the one shape of the state
10
+ they read (``{entity: {primary key: row}}``, the api runtime's dump).
11
+
12
+ The two hosts differ only in how they run world-authored code: the World Host
13
+ confines the verifier like a worker (:class:`VerifierSandbox` subclass in
14
+ :mod:`workers`), the runner's container is the confinement. A verifier that
15
+ cannot deliver a grade — missing, timed out, crashed, printed no JSON — is a
16
+ :class:`GraderError`; the host turns it into a refusal, the runner into a
17
+ null-graded sample that names the reason. Null is "ungraded", never zero.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import subprocess
24
+ import sys
25
+ from collections.abc import Callable, Iterable, Mapping
26
+ from dataclasses import dataclass, field
27
+ from pathlib import Path
28
+ from typing import Any, Protocol
29
+
30
+ from .tasks import evaluate_assertions
31
+
32
+ GRADER_KINDS = ("assertions", "python", "rubric")
33
+ VERIFY_TIMEOUT_S = 30.0
34
+
35
+
36
+ class GraderError(Exception):
37
+ """A grader that could not deliver a grade, with the reason a reader acts on."""
38
+
39
+
40
+ @dataclass(frozen=True)
41
+ class Grade:
42
+ """A task's grade: the scalar ``reward`` (None when ungraded) and every
43
+ named dimension under ``rewards`` (nulls kept). ``pending`` is a grade the
44
+ platform still owes (a rubric)."""
45
+
46
+ reward: float | None
47
+ rewards: dict[str, float | None] = field(default_factory=dict)
48
+ pending: bool = False
49
+ grader: str | None = None
50
+
51
+ def as_dict(self) -> dict[str, Any]:
52
+ """The wire shape ``grade`` answers with (and ``verify.py`` prints)."""
53
+ out: dict[str, Any] = {"reward": self.reward, "rewards": dict(self.rewards)}
54
+ if self.pending:
55
+ out["pending"] = True
56
+ out["grader"] = self.grader
57
+ return out
58
+
59
+ @staticmethod
60
+ def of_rewards(printed: Mapping[str, Any]) -> "Grade":
61
+ """The grade a verifier printed: the scalar under ``reward``, every
62
+ other key a dimension (non-numeric ones null)."""
63
+ return Grade(
64
+ reward=_number(printed.get("reward")),
65
+ rewards={key: _number(value) for key, value in printed.items() if key != "reward"},
66
+ )
67
+
68
+
69
+ def _number(value: Any) -> float | None:
70
+ if isinstance(value, (int, float)):
71
+ return float(value)
72
+ return None
73
+
74
+
75
+ class VerifierSandbox:
76
+ """How a ``verify.py`` is run: the argv and the environment. The base runs
77
+ it with this interpreter and the inherited environment — a container that
78
+ is itself the confinement (the batch runner). A host that must confine the
79
+ verifier further subclasses this."""
80
+
81
+ def command(self, script: Path) -> list[str]:
82
+ return [sys.executable, str(script)]
83
+
84
+ def environment(self, home: Path) -> dict[str, str] | None:
85
+ return None
86
+
87
+
88
+ class TaskGrader:
89
+ """A task's grader, ready to grade a state dump.
90
+
91
+ ``grader`` is the spec's normalised ``grader`` table. A python grader's
92
+ source must already be on disk at ``verify`` (the session's or the run's
93
+ ``<task>/checks/verify.py``); a bundled scenario's ``verify.py`` is graded
94
+ as a python grader with no table of its own."""
95
+
96
+ def __init__(
97
+ self,
98
+ grader: Mapping[str, Any],
99
+ *,
100
+ verify: Path | None = None,
101
+ sandbox: VerifierSandbox | None = None,
102
+ home: Path | None = None,
103
+ default_timeout_s: float = VERIFY_TIMEOUT_S,
104
+ ) -> None:
105
+ kind = grader.get("kind")
106
+ if kind not in GRADER_KINDS:
107
+ raise GraderError(f"unknown grader kind {kind!r}; a task grades by {', '.join(GRADER_KINDS)}")
108
+ self.grader = dict(grader)
109
+ self.kind: str = kind
110
+ self.verify = verify
111
+ self.sandbox = sandbox or VerifierSandbox()
112
+ self.home = home
113
+ timeout = grader.get("timeoutSeconds")
114
+ own = None if isinstance(timeout, bool) else _number(timeout)
115
+ self.timeout_s = own if own is not None and own > 0 else float(default_timeout_s)
116
+
117
+ @staticmethod
118
+ def for_verifier(
119
+ verify: Path,
120
+ *,
121
+ sandbox: VerifierSandbox | None = None,
122
+ home: Path | None = None,
123
+ timeout_s: float = VERIFY_TIMEOUT_S,
124
+ ) -> "TaskGrader":
125
+ """A bundled scenario's ``verify.py``: a python grader with the host's timeout."""
126
+ return TaskGrader({"kind": "python"}, verify=verify, sandbox=sandbox, home=home, default_timeout_s=timeout_s)
127
+
128
+ @property
129
+ def pending(self) -> bool:
130
+ """Whether the platform grades this task (a rubric), not this process."""
131
+ return self.kind == "rubric"
132
+
133
+ def grade(self, state: Mapping[str, Any]) -> Grade:
134
+ graders: dict[str, Callable[[Mapping[str, Any]], Grade]] = {
135
+ "assertions": self._assertions,
136
+ "python": self._python,
137
+ "rubric": self._rubric,
138
+ }
139
+ return graders[self.kind](state)
140
+
141
+ def _assertions(self, state: Mapping[str, Any]) -> Grade:
142
+ graded = evaluate_assertions(self.grader["checks"], dict(state))
143
+ return Grade(reward=graded["reward"], rewards=graded["rewards"])
144
+
145
+ def _rubric(self, state: Mapping[str, Any]) -> Grade:
146
+ return Grade(reward=None, rewards={}, pending=True, grader="rubric")
147
+
148
+ def _python(self, state: Mapping[str, Any]) -> Grade:
149
+ verify = self.verify
150
+ if verify is None or not verify.is_file():
151
+ raise GraderError(f"the task's verify.py is missing at {verify}")
152
+ home = self.home if self.home is not None else verify.parent
153
+ home.mkdir(parents=True, exist_ok=True)
154
+ try:
155
+ proc = subprocess.run(
156
+ self.sandbox.command(verify),
157
+ input=json.dumps(state),
158
+ check=False,
159
+ capture_output=True,
160
+ text=True,
161
+ timeout=self.timeout_s,
162
+ cwd=str(verify.parent),
163
+ env=self.sandbox.environment(home),
164
+ )
165
+ except subprocess.TimeoutExpired:
166
+ raise GraderError(f"verify.py timed out after {self.timeout_s:g}s") from None
167
+ if proc.returncode != 0:
168
+ raise GraderError(f"verify.py exited {proc.returncode}: {proc.stderr.strip()[-800:]}")
169
+ try:
170
+ printed = json.loads(proc.stdout.strip().splitlines()[-1])
171
+ except (json.JSONDecodeError, IndexError):
172
+ raise GraderError(f"verify.py printed no rewards JSON: {proc.stdout.strip()[-400:]}") from None
173
+ if not isinstance(printed, dict):
174
+ raise GraderError("verify.py rewards output is not an object")
175
+ return Grade.of_rewards(printed)
176
+
177
+
178
+ def keyed_entities(
179
+ entities: Mapping[str, Mapping[str, Any]], rows_of: Callable[[str], Iterable[Mapping[str, Any]]]
180
+ ) -> dict:
181
+ """A world's rows the way a grader reads them: every contract entity by
182
+ name, its rows keyed by their primary key (``a:b`` for a composite key) —
183
+ the api runtime's ``/__world/state`` dump, so a check or a ``verify.py``
184
+ written against one shape of the world grades the other."""
185
+ out: dict[str, dict[str, Any]] = {}
186
+ for entity, spec in entities.items():
187
+ keyed: dict[str, Any] = {}
188
+ for row in rows_of(entity):
189
+ keyed[":".join(str(row.get(name)) for name in spec["primary_key"])] = row
190
+ out[entity] = keyed
191
+ return out
192
+
193
+
194
+ class StoredWorld(Protocol):
195
+ """What a grader needs of a live world: its contract and the store holding
196
+ its rows — a World Host session's world and the batch runner's are both."""
197
+
198
+ @property
199
+ def contract(self) -> Any: ...
200
+
201
+ @property
202
+ def store(self) -> Any: ...
203
+
204
+
205
+ def world_state(root: StoredWorld | None, dependencies: Mapping[str, StoredWorld]) -> dict:
206
+ """The rows as a task's grader reads them, on the host and in the batch
207
+ runner alike: the root's entities by name and each dependency's as
208
+ ``<alias>.<entity>``, every entity's rows keyed by primary key — the api
209
+ runtime's dump, so one ``verify.py`` grades the world in either shape. A
210
+ world without a contract (a legacy tools bundle) contributes nothing, as
211
+ does a task set that owns no root world (``root`` None)."""
212
+ out: dict[str, Any] = dict(_keyed_rows_of(root))
213
+ for alias, world in dependencies.items():
214
+ for entity, rows in _keyed_rows_of(world).items():
215
+ out[f"{alias}.{entity}"] = rows
216
+ return out
217
+
218
+
219
+ def _keyed_rows_of(world: StoredWorld | None) -> dict[str, dict]:
220
+ if world is None:
221
+ return {}
222
+ store, contract = world.store, world.contract
223
+ if store is None or contract is None:
224
+ return {}
225
+ return keyed_entities(contract.entities, store.list)