@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -1,31 +1,16 @@
1
- """Session tasks: a task supplied at open instead of read from the bundle.
2
-
3
- A bundled task is ``scenarios/<name>/`` in the version (instruction, seed
4
- overlay, ``checks/verify.py``). A session task is the same three things sent
5
- with the open, for one session, so one published api world serves any number
6
- of end users each with their own goal, starting rows and pass criteria.
7
-
8
- This module is the host's half of the contract whose other half is
9
- ``gateway/packages/shared/src/domain/worldTasks.ts`` (the platform validates
10
- with Zod, the host validates again here: it trusts nothing it did not check).
11
- Keep the two in step by hand; the limits and the check grammar are the same.
12
-
13
- * :func:`parse_task_spec` — the spec, checked and normalised.
14
- * :func:`write_task_dir` — the spec on disk under the session
15
- (``task.json``, ``instruction.md``, ``checks/verify.py``), where a grade
16
- and a restart find it.
17
- * :func:`evaluate_assertions` — the declarative grader, pure Python over
18
- the state dump, so a check cannot read outside its session.
19
- """
1
+ """Twin of gateway/packages/shared/src/domain/worldTasks.ts, pinned by conformance/world-journal.json; the host
2
+ validates again, trusting nothing it did not check."""
20
3
 
21
4
  from __future__ import annotations
22
5
 
23
6
  import json
7
+ import math
24
8
  import re
25
9
  from dataclasses import dataclass
26
10
  from pathlib import Path
27
11
  from typing import Any, Iterable
28
12
 
13
+ from ..reward_keys import RewardKeys
29
14
  from .validation import fail
30
15
 
31
16
  # The caps, as the platform's WORLD_TASK_LIMITS.
@@ -42,7 +27,6 @@ RUBRIC_CRITERIA = 50
42
27
  RUBRIC_CRITERION_BYTES = 2 * 1024
43
28
  TOOLS = 200
44
29
  TOOL_NAME_CHARS = 200
45
- #: Cross-world verifiers one task may carry.
46
30
  VERIFIERS = 32
47
31
 
48
32
  SEED_MODES = ("append", "replace")
@@ -55,7 +39,6 @@ _TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
55
39
 
56
40
  TASK_DIR = "task"
57
41
  TASK_FILE = "task.json"
58
- #: Under the task dir, a python verifier's source: ``verifiers/<name>/verify.py``.
59
42
  VERIFIERS_DIR = "verifiers"
60
43
 
61
44
 
@@ -70,9 +53,7 @@ def _scalar(value: Any) -> bool:
70
53
 
71
54
  @dataclass(frozen=True)
72
55
  class TaskSpec:
73
- """A validated session task. ``raw`` is the normalised spec as a dict: what
74
- the platform sent, defaults filled, and what is written to ``task.json``
75
- and compared when an open is retried."""
56
+ """``raw`` is the normalised spec, written to task.json and compared when an open is retried."""
76
57
 
77
58
  instruction: str
78
59
  name: str | None
@@ -82,8 +63,6 @@ class TaskSpec:
82
63
  raw: dict
83
64
  #: The tools the session may call, or None for every tool the world serves.
84
65
  tools: tuple[str, ...] | None = None
85
- #: Cross-world verifiers anchored on this session: each a grader table
86
- #: with ``name`` and ``scope`` (see ``TaskGrader.evidence``).
87
66
  verifiers: tuple[dict, ...] = ()
88
67
 
89
68
  @property
@@ -92,59 +71,73 @@ class TaskSpec:
92
71
 
93
72
  @property
94
73
  def graded_on_platform(self) -> bool:
95
- """A rubric task is judged by the platform's inference path, never here."""
96
74
  return self.grader_kind == "rubric"
97
75
 
98
76
  def allows_tool(self, tool: str) -> bool:
99
77
  return self.tools is None or tool in self.tools
100
78
 
101
79
  def entities(self) -> list[str]:
102
- """Every entity the task names, seed rows and assertion checks alike, in
103
- order of first mention. A world that defines none of a name cannot start
104
- or grade this task: a seed row has no table, and a check over a table
105
- that does not exist selects no rows, which an ``absent`` or ``count:
106
- eq 0`` check would pass on a typo."""
107
80
  named: list[str] = []
108
81
  if self.seed is not None:
109
82
  named.extend(self.seed["rows"])
110
83
  if self.grader is not None and self.grader["kind"] == "assertions":
111
84
  named.extend(check["entity"] for check in self.grader["checks"])
112
- # A verifier's checks name other worlds' entities as well; only the
113
- # ones under this session's own alias are this world's to define.
85
+ # A session task is named for its alias: only checks under it name this world's entities.
114
86
  own = f"{self.name}."
115
87
  for verifier in self.verifiers:
116
88
  named.extend(
117
- check["entity"][len(own) :]
118
- for check in verifier.get("checks", ())
119
- if check["entity"].startswith(own)
89
+ check["entity"][len(own) :] for check in verifier.get("checks", ()) if check["entity"].startswith(own)
120
90
  )
121
91
  return list(dict.fromkeys(named))
122
92
 
123
93
 
124
- #: Keys a state dump carries beside its entities, which a grader reads like an
125
- #: entity: ``calls`` (the session's call log) and ``snapshot``. A world cannot
126
- #: declare an entity by either name (the compiler refuses it), so a check on
127
- #: one is never a typo for a row entity; a seed may not target them.
128
- RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
129
94
  CALL_LOG = "calls"
95
+ JOURNAL = "gateway_requests"
96
+ JOURNAL_META = "gateway_journal"
97
+ RESERVED_STATE_KEYS = frozenset({CALL_LOG, "snapshot", JOURNAL})
130
98
  DOOR_FIELD = "via"
99
+ WITHHELD_FIELD = "withheld"
100
+ ROW_FIELDS = ("found", "rows")
101
+ ORDER_KEYS = ("before", "after")
102
+ JOURNAL_FIELDS = (
103
+ "seq",
104
+ "id",
105
+ "at",
106
+ "durationMs",
107
+ "via",
108
+ "operation",
109
+ "method",
110
+ "path",
111
+ "rows",
112
+ "found",
113
+ "request.body",
114
+ "request.bodyBytes",
115
+ "request.truncated",
116
+ "request.contentType",
117
+ "response.status",
118
+ "response.body",
119
+ "response.bodyBytes",
120
+ "response.truncated",
121
+ "response.contentType",
122
+ )
123
+ JOURNAL_FIELD_PREFIXES = ("query.", "request.headers.", "request.body.", "response.body.")
131
124
 
132
125
 
133
126
  class GraderError(Exception):
134
- """A grader that could not deliver a grade, with the reason a reader acts on."""
127
+ """A grader that cannot deliver a grade: the host refuses, the runner records null with the reason, never 0."""
135
128
 
136
129
 
137
130
  def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
138
- """The entities ``spec`` names that ``defined`` (a world contract's) lacks.
139
- A grader's check on a reserved state key is not one of them."""
140
131
  known = set(defined)
141
132
  seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
142
- return [entity for entity in spec.entities()
143
- if entity not in known and not (entity in RESERVED_STATE_KEYS and entity not in seeded)]
133
+ return [
134
+ entity
135
+ for entity in spec.entities()
136
+ if entity not in known and not (entity in RESERVED_STATE_KEYS and entity not in seeded)
137
+ ]
144
138
 
145
139
 
146
140
  def parse_task_spec(value: Any) -> TaskSpec:
147
- """The spec checked for shape and size; every refusal names its path."""
148
141
  if not isinstance(value, dict):
149
142
  fail("task", "a task is a json object", "/task")
150
143
  unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata", "verifiers"}
@@ -163,7 +156,11 @@ def parse_task_spec(value: Any) -> TaskSpec:
163
156
  tools = _parse_tools(value.get("tools"))
164
157
  verifiers = _parse_verifiers(value.get("verifiers"), name)
165
158
  if verifiers and grader is not None and grader["kind"] == "rubric":
166
- fail("task", "a task with verifiers grades by assertions, python or nothing: a rubric never reaches the host's grade", "/task/grader/kind")
159
+ fail(
160
+ "task",
161
+ "a task with verifiers grades by assertions, python or nothing: a rubric never reaches the host's grade",
162
+ "/task/grader/kind",
163
+ )
167
164
  metadata = value.get("metadata")
168
165
  if metadata is None:
169
166
  metadata = {}
@@ -200,13 +197,7 @@ _VERIFIER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
200
197
 
201
198
 
202
199
  def _parse_verifiers(value: Any, task_name: str | None) -> tuple[dict, ...]:
203
- """``verifiers``: cross-world graders, each ``{name, scope, ...grader}``
204
- (the grader through :func:`_parse_grader`, so the check grammar and the
205
- caps are defined once). A scope names two or more distinct aliases; the
206
- first is this session, so it must be the task's own name — a session task
207
- is what one alias of a multi-world task opens on, named for that alias.
208
- Every assertions check names ``<alias>.<entity>`` with the alias in
209
- scope. Mirrors ``WorldTaskVerifierSchema``."""
200
+ """Mirrors ``WorldTaskVerifierSchema``."""
210
201
  if value is None:
211
202
  return ()
212
203
  at = "/task/verifiers"
@@ -227,27 +218,48 @@ def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
227
218
  if not isinstance(name, str) or not _VERIFIER_NAME.match(name) or len(name) > NAME_CHARS:
228
219
  fail("task", "a verifier name is letters, digits, _ . -", f"{at}/name")
229
220
  scope = row.get("scope")
230
- if not isinstance(scope, list) or len(scope) < 2 or any(not isinstance(a, str) or not _NAME.match(a) for a in scope):
221
+ if (
222
+ not isinstance(scope, list)
223
+ or len(scope) < 2
224
+ or any(not isinstance(a, str) or not _NAME.match(a) for a in scope)
225
+ ):
231
226
  fail("task", "a verifier's scope names two or more world aliases", f"{at}/scope")
232
227
  if len(set(scope)) != len(scope):
233
228
  fail("task", "a verifier's scope lists each alias once", f"{at}/scope")
229
+ reserved = [alias for alias in scope if RewardKeys.reserved(alias)]
230
+ if reserved:
231
+ fail("task", RewardKeys.refusal(reserved[0]), f"{at}/scope")
234
232
  if scope[0] != task_name:
235
- fail("task", f"verifier '{name}' runs on the session of its first scope alias, so the task's name must be '{scope[0]}'", f"{at}/scope")
233
+ fail(
234
+ "task",
235
+ f"verifier '{name}' runs on the session of its first scope alias, so the task's name must be '{scope[0]}'",
236
+ f"{at}/scope",
237
+ )
236
238
  grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
237
239
  if grader is None or grader["kind"] == "rubric":
238
240
  fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
241
+ if grader.get("readsJournal"):
242
+ fail("task", CROSS_JOURNAL, f"{at}/readsJournal")
239
243
  for j, check in enumerate(grader.get("checks", ())):
240
- if check["entity"].split(".", 1)[0] not in scope or "." not in check["entity"]:
241
- fail(
242
- "task",
243
- f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
244
- f"{at}/checks/{j}/entity",
245
- )
244
+ _cross_world_entity(check["entity"], scope, f"{at}/checks/{j}/entity")
246
245
  return {"name": name, "scope": list(scope), **grader}
247
246
 
248
247
 
248
+ CROSS_JOURNAL = f"a cross-world verifier reads <alias>.{CALL_LOG}; {JOURNAL} checks go on a world's own grader"
249
+
250
+
251
+ def _cross_world_entity(entity: str, scope: list, at: str) -> None:
252
+ if entity.split(".", 1)[0] not in scope or "." not in entity:
253
+ fail(
254
+ "task",
255
+ f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
256
+ at,
257
+ )
258
+ if _on_the_journal(entity):
259
+ fail("task", CROSS_JOURNAL, at)
260
+
261
+
249
262
  def verifier_source(task_dir: Path, name: str) -> Path:
250
- """Where :func:`write_task_dir` puts a python verifier's ``verify.py``."""
251
263
  return Path(task_dir) / VERIFIERS_DIR / name / "verify.py"
252
264
 
253
265
 
@@ -266,21 +278,15 @@ def _parse_tools(tools: Any) -> tuple[str, ...] | None:
266
278
  return tuple(tools)
267
279
 
268
280
 
269
- def unknown_task_tools(spec: TaskSpec, served: Iterable[str]) -> list[str]:
270
- """The tools ``spec`` allows that the world does not serve. A name the
271
- world never had would make the allow-list smaller than the author meant
272
- without anyone noticing, so it is refused at open."""
273
- if spec.tools is None:
274
- return []
275
- known = set(served)
276
- return [tool for tool in spec.tools if tool not in known]
277
-
278
-
279
281
  def _parse_seed(seed: Any) -> dict | None:
280
282
  if seed is None:
281
283
  return None
282
284
  if not isinstance(seed, dict) or set(seed) - {"mode", "rows", "redact"}:
283
- fail("task", "seed is {mode?: append|replace, redact?: off|apply|refuse, rows: {entity: [row, ...]}}", "/task/seed")
285
+ fail(
286
+ "task",
287
+ "seed is {mode?: append|replace, redact?: off|apply|refuse, rows: {entity: [row, ...]}}",
288
+ "/task/seed",
289
+ )
284
290
  mode = seed.get("mode", "append")
285
291
  if mode not in SEED_MODES:
286
292
  fail("task", f"seed mode must be one of {', '.join(SEED_MODES)}", "/task/seed/mode")
@@ -313,24 +319,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
313
319
  fail("task", "grader is a json object with a kind", at)
314
320
  kind = grader.get("kind")
315
321
  if kind == "python":
316
- if set(grader) - {"kind", "source", "timeoutSeconds"}:
317
- fail("task", "a python grader is {kind, source, timeoutSeconds?}", at)
318
- source = grader.get("source")
319
- if not isinstance(source, str) or not source:
320
- fail("task", "a python grader needs its verify.py source", f"{at}/source")
321
- if _bytes(source) > PYTHON_SOURCE_BYTES:
322
- fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
323
- out: dict = {"kind": "python", "source": source}
324
- timeout = grader.get("timeoutSeconds")
325
- if timeout is not None:
326
- if (
327
- isinstance(timeout, bool)
328
- or not isinstance(timeout, (int, float))
329
- or not 0 < timeout <= PYTHON_TIMEOUT_S
330
- ):
331
- fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
332
- out["timeoutSeconds"] = float(timeout)
333
- return out
322
+ return _parse_python_grader(grader, at)
334
323
  if kind == "assertions":
335
324
  if set(grader) - {"kind", "checks"}:
336
325
  fail("task", "an assertions grader is {kind, checks}", at)
@@ -340,9 +329,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
340
329
  if len(checks) > CHECKS:
341
330
  fail("task", f"an assertions grader has at most {CHECKS} checks", f"{at}/checks")
342
331
  parsed = [_parse_check(check, f"{at}/checks/{i}") for i, check in enumerate(checks)]
343
- # Every check owns one reward key, its id or its position; the keys are
344
- # distinct as a set, so an explicit `check_2` cannot shadow the third
345
- # check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
332
+ # Mirrors WorldTaskAssertionsGraderSchema.
346
333
  seen: set[str] = set()
347
334
  for i, check in enumerate(parsed):
348
335
  key = check_key(check, i)
@@ -363,12 +350,50 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
363
350
  if kind == "rubric":
364
351
  return _parse_rubric(grader, at)
365
352
  fail("task", "grader.kind must be assertions, python or rubric", f"{at}/kind")
366
- raise AssertionError # fail() never returns
353
+ raise AssertionError
354
+
355
+
356
+ PYTHON_GRADER_KEYS = frozenset({"kind", "source", "timeoutSeconds", "readsJournal", "acceptsIncompleteJournal"})
357
+ JOURNAL_FLAGS = ("readsJournal", "acceptsIncompleteJournal")
358
+
359
+
360
+ def _parse_python_grader(grader: dict, at: str) -> dict:
361
+ """``readsJournal`` hands verify.py the session's journal; ``acceptsIncompleteJournal`` grades one that lost
362
+ requests instead of failing closed."""
363
+ if set(grader) - PYTHON_GRADER_KEYS:
364
+ fail("task", "a python grader is {kind, source, timeoutSeconds?, readsJournal?, acceptsIncompleteJournal?}", at)
365
+ source = grader.get("source")
366
+ if not isinstance(source, str) or not source:
367
+ fail("task", "a python grader needs its verify.py source", f"{at}/source")
368
+ if _bytes(source) > PYTHON_SOURCE_BYTES:
369
+ fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
370
+ out: dict = {"kind": "python", "source": source}
371
+ timeout = grader.get("timeoutSeconds")
372
+ if timeout is not None:
373
+ out["timeoutSeconds"] = _timeout(timeout, at)
374
+ for flag in JOURNAL_FLAGS:
375
+ out.update(_flag(grader, flag, at))
376
+ if out.get("acceptsIncompleteJournal") and not out.get("readsJournal"):
377
+ fail("task", "acceptsIncompleteJournal only goes with readsJournal: true", f"{at}/acceptsIncompleteJournal")
378
+ return out
379
+
380
+
381
+ def _timeout(value: Any, at: str) -> float:
382
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= PYTHON_TIMEOUT_S:
383
+ fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
384
+ return float(value)
385
+
386
+
387
+ def _flag(grader: dict, name: str, at: str) -> dict:
388
+ value = grader.get(name)
389
+ if value is None or value is False:
390
+ return {}
391
+ if value is not True:
392
+ fail("task", f"{name} is true or absent", f"{at}/{name}")
393
+ return {name: True}
367
394
 
368
395
 
369
396
  def _parse_rubric(grader: dict, at: str) -> dict:
370
- """An LLM judge's brief. Checked here for shape only: the host stores it
371
- with the task and never runs it; the platform grades it."""
372
397
  if set(grader) - {"kind", "rubric", "criteria", "model"}:
373
398
  fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", at)
374
399
  rubric = grader.get("rubric")
@@ -422,81 +447,166 @@ def _parse_rubric(grader: dict, at: str) -> dict:
422
447
 
423
448
 
424
449
  def check_key(check: dict, index: int) -> str:
425
- """The reward key of a check: its id, else its position as ``check_<i>``."""
426
450
  return check.get("id") or f"check_{index}"
427
451
 
428
452
 
453
+ CHECK_KEYS = frozenset({"id", "weight", "entity", "where", "assert", "count", "field", "op", "value", *ORDER_KEYS})
454
+ PREDICATES = ("all", "any")
455
+
456
+
429
457
  def _parse_check(check: Any, at: str) -> dict:
430
458
  if not isinstance(check, dict):
431
459
  fail("task", "a check is a json object", at)
432
- unknown = set(check) - {"id", "weight", "entity", "where", "assert", "count", "field", "op", "value"}
460
+ unknown = set(check) - CHECK_KEYS
433
461
  if unknown:
434
462
  fail("task", f"unknown check field(s): {', '.join(sorted(unknown))}", at)
435
463
  out: dict = {}
436
- check_id = check.get("id")
437
- if check_id is not None:
438
- if not isinstance(check_id, str) or not _CHECK_ID.match(check_id) or len(check_id) > 120:
439
- fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
440
- out["id"] = check_id
441
- weight = check.get("weight", 1)
442
- if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0 or weight == float("inf"):
443
- fail("task", "weight is a positive number", f"{at}/weight")
444
- out["weight"] = float(weight)
464
+ if check.get("id") is not None:
465
+ out["id"] = _check_id(check["id"], at)
466
+ out["weight"] = _weight(check.get("weight", 1), at)
445
467
  entity = check.get("entity")
446
468
  if not isinstance(entity, str) or not entity or len(entity) > 120:
447
469
  fail("task", "a check names an entity", f"{at}/entity")
448
470
  out["entity"] = entity
449
- where = check.get("where")
450
- if where is not None:
451
- if not isinstance(where, dict) or any(
452
- not isinstance(k, str) or not k or not _scalar(v) for k, v in where.items()
453
- ):
454
- fail("task", "where is {field: scalar, ...}", f"{at}/where")
455
- out["where"] = where
471
+ for key in ("where", *ORDER_KEYS):
472
+ selection = _selection(check.get(key), key, f"{at}/{key}")
473
+ if selection is not None:
474
+ out[key] = selection
456
475
  assertion = check.get("assert", "exists")
457
476
  if assertion not in ASSERTIONS:
458
477
  fail("task", f"assert must be one of {', '.join(ASSERTIONS)}", f"{at}/assert")
459
478
  out["assert"] = assertion
479
+ out.update(_count(check, assertion, at))
480
+ out.update(_field_test(check, assertion, at))
481
+ _check_reads(out, at)
482
+ return out
483
+
484
+
485
+ def _check_id(value: Any, at: str) -> str:
486
+ if not isinstance(value, str) or not _CHECK_ID.match(value) or len(value) > 120:
487
+ fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
488
+ return value
489
+
490
+
491
+ def _weight(value: Any, at: str) -> float:
492
+ if isinstance(value, bool) or not isinstance(value, (int, float)) or not value > 0 or value == float("inf"):
493
+ fail("task", "weight is a positive number", f"{at}/weight")
494
+ return float(value)
495
+
496
+
497
+ def _selection(value: Any, key: str, at: str) -> dict | None:
498
+ if value is None:
499
+ return None
500
+ if not isinstance(value, dict) or any(not isinstance(k, str) or not k or not _scalar(v) for k, v in value.items()):
501
+ fail("task", f"{key} is {{field: scalar, ...}}", at)
502
+ return value
503
+
504
+
505
+ def _count(check: dict, assertion: str, at: str) -> dict:
460
506
  count = check.get("count")
461
- if assertion == "count":
462
- if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
463
- fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
464
- for key, bound in count.items():
465
- if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
466
- fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
467
- out["count"] = count
468
- elif count is not None:
507
+ if assertion != "count" and count is not None:
469
508
  fail("task", "count only goes with assert: count", f"{at}/count")
470
- field = check.get("field")
471
- op = check.get("op", "eq")
472
- value = check.get("value")
473
- if assertion in ("all", "any"):
474
- if not isinstance(field, str) or not field:
475
- fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
476
- if op not in FIELD_OPS:
477
- fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
478
- if "value" not in check:
479
- fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
480
- if op == "in":
481
- if not isinstance(value, list) or any(not _scalar(v) for v in value):
482
- fail("task", "op: in compares against a list of scalars", f"{at}/value")
483
- elif not _scalar(value):
484
- fail("task", f"op: {op} compares against one scalar", f"{at}/value")
485
- out["field"], out["op"], out["value"] = field, op, value
486
- elif field is not None or "value" in check or "op" in check:
509
+ if assertion != "count":
510
+ return {}
511
+ if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
512
+ fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
513
+ for key, bound in count.items():
514
+ _count_bound(key, bound, at)
515
+ return {"count": count}
516
+
517
+
518
+ def _count_bound(key: str, bound: Any, at: str) -> None:
519
+ if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
520
+ fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
521
+
522
+
523
+ def _field_test(check: dict, assertion: str, at: str) -> dict:
524
+ field, op, value = check.get("field"), check.get("op", "eq"), check.get("value")
525
+ if assertion not in PREDICATES and (field is not None or "value" in check or "op" in check):
487
526
  fail("task", "field/op/value only go with assert: all or any", f"{at}/field")
488
- return out
527
+ if assertion not in PREDICATES:
528
+ return {}
529
+ if not isinstance(field, str) or not field:
530
+ fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
531
+ if op not in FIELD_OPS:
532
+ fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
533
+ if "value" not in check:
534
+ fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
535
+ _compared(op, value, at)
536
+ return {"field": field, "op": op, "value": value}
537
+
538
+
539
+ def _compared(op: str, value: Any, at: str) -> None:
540
+ if op == "in" and (not isinstance(value, list) or any(not _scalar(v) for v in value)):
541
+ fail("task", "op: in compares against a list of scalars", f"{at}/value")
542
+ if op != "in" and not _scalar(value):
543
+ fail("task", f"op: {op} compares against one scalar", f"{at}/value")
544
+
545
+
546
+ def _on_the_journal(entity: str) -> bool:
547
+ return entity == JOURNAL or entity.endswith(f".{JOURNAL}")
548
+
549
+
550
+ def _ordered_log(entity: str) -> bool:
551
+ return _on_the_journal(entity) or entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
552
+
553
+
554
+ def _check_reads(check: dict, at: str) -> None:
555
+ entity = check["entity"]
556
+ ordered = [key for key in ORDER_KEYS if key in check]
557
+ if ordered and not _ordered_log(entity):
558
+ fail(
559
+ "task",
560
+ f"{ordered[0]} orders requests in time, so it goes on a {JOURNAL} or {CALL_LOG} check, not on {entity}",
561
+ f"{at}/{ordered[0]}",
562
+ )
563
+ if entity == JOURNAL_META or entity.endswith(f".{JOURNAL_META}"):
564
+ fail("task", f"{JOURNAL_META} is the journal's summary, not its rows; check {JOURNAL} instead", f"{at}/entity")
565
+ if _on_the_journal(entity):
566
+ _journal_paths(check, at)
567
+
568
+
569
+ def _journal_paths(check: dict, at: str) -> None:
570
+ named = [(key, path) for key in ("where", *ORDER_KEYS) for path in check.get(key, {})]
571
+ named += [("field", check["field"])] if "field" in check else []
572
+ for key, path in named:
573
+ _journal_path(path, f"{at}/{key}")
574
+
489
575
 
576
+ def _distance(a: str, b: str) -> int:
577
+ previous = list(range(len(b) + 1))
578
+ for i, left in enumerate(a, 1):
579
+ current = [i]
580
+ for j, right in enumerate(b, 1):
581
+ current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (left != right)))
582
+ previous = current
583
+ return previous[-1]
490
584
 
491
- # --------------------------------------------------------------------------- #
492
- # On disk
493
- # --------------------------------------------------------------------------- #
585
+
586
+ def nearest_journal_field(path: str) -> str | None:
587
+ """Twin of ``nearestJournalField`` in worldJournal.ts."""
588
+ best, best_distance = None, math.inf
589
+ for candidate in (*JOURNAL_FIELDS, *(prefix[:-1] for prefix in JOURNAL_FIELD_PREFIXES)):
590
+ d = _distance(path, candidate)
591
+ if d < best_distance and d <= 3 and d < len(candidate) / 2:
592
+ best, best_distance = candidate, d
593
+ return best
594
+
595
+
596
+ def _journal_path(path: str, at: str) -> None:
597
+ if path in JOURNAL_FIELDS or path.startswith(JOURNAL_FIELD_PREFIXES):
598
+ return
599
+ near = nearest_journal_field(path)
600
+ hint = f"did you mean {near}? " if near else ""
601
+ fail(
602
+ "task",
603
+ f"{JOURNAL} has no field {path}; {hint}match on {', '.join(JOURNAL_FIELDS)} "
604
+ f"or a path under {', '.join(JOURNAL_FIELD_PREFIXES)}",
605
+ at,
606
+ )
494
607
 
495
608
 
496
609
  def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
497
- """The task under the session: ``task.json`` (the whole spec),
498
- ``instruction.md``, and for a python grader ``checks/verify.py`` where
499
- the bundled layout has it, so the one verifier path runs both."""
500
610
  root = Path(session_dir) / TASK_DIR
501
611
  root.mkdir(parents=True, exist_ok=True)
502
612
  (root / TASK_FILE).write_text(json.dumps(spec.raw, indent=2, sort_keys=True) + "\n", encoding="utf-8")
@@ -515,139 +625,20 @@ def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
515
625
 
516
626
 
517
627
  def read_task_dir(session_dir: Path) -> TaskSpec | None:
518
- """The task a session directory holds, or None for a bundled-task session."""
519
628
  path = Path(session_dir) / TASK_DIR / TASK_FILE
520
629
  if not path.is_file():
521
630
  return None
522
631
  return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
523
632
 
524
633
 
525
- # --------------------------------------------------------------------------- #
526
- # The assertions grader
527
- # --------------------------------------------------------------------------- #
528
-
529
-
530
- def _lookup(row: Any, path: str) -> tuple[bool, Any]:
531
- """``(found, value)`` of a dotted path into a row."""
532
- current = row
533
- for part in path.split("."):
534
- if not isinstance(current, dict) or part not in current:
535
- return False, None
536
- current = current[part]
537
- return True, current
538
-
539
-
540
- def _same(actual: Any, expected: Any) -> bool:
541
- """JSON equality: ``true`` is not ``1``, as it is not in the schema."""
542
- if isinstance(actual, bool) or isinstance(expected, bool):
543
- return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
544
- return actual == expected
545
-
546
-
547
- def _matches_where(row: dict, where: dict | None) -> bool:
548
- for path, expected in (where or {}).items():
549
- found, actual = _lookup(row, path)
550
- if not found or not _same(actual, expected):
551
- return False
552
- return True
553
-
554
-
555
- def _ordered(actual: Any, expected: Any) -> bool:
556
- """Whether the two compare with ``<``: both numbers (not bools) or both strings."""
557
- numbers = (int, float)
558
- if isinstance(actual, bool) or isinstance(expected, bool):
559
- return False
560
- if isinstance(actual, numbers) and isinstance(expected, numbers):
561
- return True
562
- return isinstance(actual, str) and isinstance(expected, str)
563
-
564
-
565
- def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
566
- found, actual = _lookup(row, field)
567
- if op == "eq":
568
- return found and _same(actual, expected)
569
- if op == "neq":
570
- return not found or not _same(actual, expected)
571
- if op == "in":
572
- return found and any(_same(actual, option) for option in expected)
573
- if op == "contains":
574
- if not found:
575
- return False
576
- if isinstance(actual, str):
577
- return isinstance(expected, str) and expected in actual
578
- if isinstance(actual, list):
579
- return any(_same(item, expected) for item in actual)
580
- return False
581
- if not found or not _ordered(actual, expected):
582
- return False
583
- if op == "gt":
584
- return actual > expected
585
- if op == "gte":
586
- return actual >= expected
587
- if op == "lt":
588
- return actual < expected
589
- return actual <= expected # lte
590
-
591
-
592
- def _rows_of(state: dict, entity: str) -> list[dict]:
593
- """The rows of one entity from a state dump: ``{entity: {key: row}}`` (the
594
- verifier's shape) or ``{entity: [row, ...]}``."""
595
- held = state.get(entity)
596
- if isinstance(held, dict):
597
- return [row for row in held.values() if isinstance(row, dict)]
598
- if isinstance(held, list):
599
- return [row for row in held if isinstance(row, dict)]
600
- return []
634
+ def _on_the_call_log(check: dict) -> bool:
635
+ entity = check["entity"]
636
+ return entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
601
637
 
602
638
 
603
- def _names_the_door(check: dict) -> bool:
604
- entity = check["entity"]
605
- on_call_log = entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
606
- return on_call_log and (check.get("field") == DOOR_FIELD or DOOR_FIELD in (check.get("where") or {}))
639
+ def _names(check: dict, field: str) -> bool:
640
+ return check.get("field") == field or field in (check.get("where") or {})
607
641
 
608
642
 
609
- def _require_the_door(check: dict, state: dict, rows: list[dict]) -> None:
610
- """A door check over no call log, or a log that does not record the door,
611
- would select nothing (``absent`` passes) or miss the key (``neq`` passes):
612
- ungraded."""
613
- if not _names_the_door(check):
614
- return
615
- if check["entity"] not in state or any(DOOR_FIELD not in row for row in rows):
616
- raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
617
-
618
-
619
- def evaluate_check(check: dict, state: dict) -> bool:
620
- rows = _rows_of(state, check["entity"])
621
- _require_the_door(check, state, rows)
622
- selected = [row for row in rows if _matches_where(row, check.get("where"))]
623
- assertion = check["assert"]
624
- if assertion == "exists":
625
- return bool(selected)
626
- if assertion == "absent":
627
- return not selected
628
- if assertion == "count":
629
- bound = check["count"]
630
- n = len(selected)
631
- return all(
632
- (key != "eq" or n == value) and (key != "gte" or n >= value) and (key != "lte" or n <= value)
633
- for key, value in bound.items()
634
- )
635
- tests = (_predicate(row, check["field"], check["op"], check["value"]) for row in selected)
636
- if assertion == "all":
637
- return bool(selected) and all(tests)
638
- return any(tests) # any
639
-
640
-
641
- def evaluate_assertions(checks: list[dict], state: dict) -> dict:
642
- """The grade of an assertions grader over a state dump: the weighted share
643
- of checks that passed as ``reward``, each check's 0/1 under its id (or its
644
- position) in ``rewards``. The runner's shape, as ``verify.py`` prints it."""
645
- rewards: dict[str, float] = {}
646
- total = 0.0
647
- passed = 0.0
648
- for i, check in enumerate(checks):
649
- ok = evaluate_check(check, state)
650
- rewards[check_key(check, i)] = 1.0 if ok else 0.0
651
- total += check["weight"]
652
- passed += check["weight"] if ok else 0.0
653
- return {"reward": (passed / total) if total else None, "rewards": rewards}
643
+ def _names_the_door(check: dict) -> bool:
644
+ return _on_the_call_log(check) and _names(check, DOOR_FIELD)