@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -1,22 +1,5 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
A bundled task is ``scenarios/<name>/`` in the version (instruction, seed
|
|
4
|
-
overlay, ``checks/verify.py``). A session task is the same three things sent
|
|
5
|
-
with the open, for one session, so one published api world serves any number
|
|
6
|
-
of end users each with their own goal, starting rows and pass criteria.
|
|
7
|
-
|
|
8
|
-
This module is the host's half of the contract whose other half is
|
|
9
|
-
``gateway/packages/shared/src/domain/worldTasks.ts`` (the platform validates
|
|
10
|
-
with Zod, the host validates again here: it trusts nothing it did not check).
|
|
11
|
-
Keep the two in step by hand; the limits and the check grammar are the same.
|
|
12
|
-
|
|
13
|
-
* :func:`parse_task_spec` — the spec, checked and normalised.
|
|
14
|
-
* :func:`write_task_dir` — the spec on disk under the session
|
|
15
|
-
(``task.json``, ``instruction.md``, ``checks/verify.py``), where a grade
|
|
16
|
-
and a restart find it.
|
|
17
|
-
* :func:`evaluate_assertions` — the declarative grader, pure Python over
|
|
18
|
-
the state dump, so a check cannot read outside its session.
|
|
19
|
-
"""
|
|
1
|
+
"""Twin of gateway/packages/shared/src/domain/worldTasks.ts, pinned by conformance/world-journal.json; the host
|
|
2
|
+
validates again, trusting nothing it did not check."""
|
|
20
3
|
|
|
21
4
|
from __future__ import annotations
|
|
22
5
|
|
|
@@ -44,7 +27,6 @@ RUBRIC_CRITERIA = 50
|
|
|
44
27
|
RUBRIC_CRITERION_BYTES = 2 * 1024
|
|
45
28
|
TOOLS = 200
|
|
46
29
|
TOOL_NAME_CHARS = 200
|
|
47
|
-
#: Cross-world verifiers one task may carry.
|
|
48
30
|
VERIFIERS = 32
|
|
49
31
|
|
|
50
32
|
SEED_MODES = ("append", "replace")
|
|
@@ -57,7 +39,6 @@ _TOOL_NAME = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.:-]*$")
|
|
|
57
39
|
|
|
58
40
|
TASK_DIR = "task"
|
|
59
41
|
TASK_FILE = "task.json"
|
|
60
|
-
#: Under the task dir, a python verifier's source: ``verifiers/<name>/verify.py``.
|
|
61
42
|
VERIFIERS_DIR = "verifiers"
|
|
62
43
|
|
|
63
44
|
|
|
@@ -72,9 +53,7 @@ def _scalar(value: Any) -> bool:
|
|
|
72
53
|
|
|
73
54
|
@dataclass(frozen=True)
|
|
74
55
|
class TaskSpec:
|
|
75
|
-
"""
|
|
76
|
-
the platform sent, defaults filled, and what is written to ``task.json``
|
|
77
|
-
and compared when an open is retried."""
|
|
56
|
+
"""``raw`` is the normalised spec, written to task.json and compared when an open is retried."""
|
|
78
57
|
|
|
79
58
|
instruction: str
|
|
80
59
|
name: str | None
|
|
@@ -84,8 +63,6 @@ class TaskSpec:
|
|
|
84
63
|
raw: dict
|
|
85
64
|
#: The tools the session may call, or None for every tool the world serves.
|
|
86
65
|
tools: tuple[str, ...] | None = None
|
|
87
|
-
#: Cross-world verifiers anchored on this session: each a grader table
|
|
88
|
-
#: with ``name`` and ``scope`` (see ``TaskGrader.evidence``).
|
|
89
66
|
verifiers: tuple[dict, ...] = ()
|
|
90
67
|
|
|
91
68
|
@property
|
|
@@ -94,25 +71,18 @@ class TaskSpec:
|
|
|
94
71
|
|
|
95
72
|
@property
|
|
96
73
|
def graded_on_platform(self) -> bool:
|
|
97
|
-
"""A rubric task is judged by the platform's inference path, never here."""
|
|
98
74
|
return self.grader_kind == "rubric"
|
|
99
75
|
|
|
100
76
|
def allows_tool(self, tool: str) -> bool:
|
|
101
77
|
return self.tools is None or tool in self.tools
|
|
102
78
|
|
|
103
79
|
def entities(self) -> list[str]:
|
|
104
|
-
"""Every entity the task names, seed rows and assertion checks alike, in
|
|
105
|
-
order of first mention. A world that defines none of a name cannot start
|
|
106
|
-
or grade this task: a seed row has no table, and a check over a table
|
|
107
|
-
that does not exist selects no rows, which an ``absent`` or ``count:
|
|
108
|
-
eq 0`` check would pass on a typo."""
|
|
109
80
|
named: list[str] = []
|
|
110
81
|
if self.seed is not None:
|
|
111
82
|
named.extend(self.seed["rows"])
|
|
112
83
|
if self.grader is not None and self.grader["kind"] == "assertions":
|
|
113
84
|
named.extend(check["entity"] for check in self.grader["checks"])
|
|
114
|
-
# A
|
|
115
|
-
# ones under this session's own alias are this world's to define.
|
|
85
|
+
# A session task is named for its alias: only checks under it name this world's entities.
|
|
116
86
|
own = f"{self.name}."
|
|
117
87
|
for verifier in self.verifiers:
|
|
118
88
|
named.extend(
|
|
@@ -121,24 +91,43 @@ class TaskSpec:
|
|
|
121
91
|
return list(dict.fromkeys(named))
|
|
122
92
|
|
|
123
93
|
|
|
124
|
-
#: Keys a state dump carries beside its entities, which a grader reads like an
|
|
125
|
-
#: entity: ``calls`` (the session's call log) and ``snapshot``. A world cannot
|
|
126
|
-
#: declare an entity by either name (the compiler refuses it), so a check on
|
|
127
|
-
#: one is never a typo for a row entity; a seed may not target them.
|
|
128
|
-
RESERVED_STATE_KEYS = frozenset({"calls", "snapshot"})
|
|
129
94
|
CALL_LOG = "calls"
|
|
95
|
+
JOURNAL = "gateway_requests"
|
|
96
|
+
JOURNAL_META = "gateway_journal"
|
|
97
|
+
RESERVED_STATE_KEYS = frozenset({CALL_LOG, "snapshot", JOURNAL})
|
|
130
98
|
DOOR_FIELD = "via"
|
|
131
99
|
WITHHELD_FIELD = "withheld"
|
|
132
100
|
ROW_FIELDS = ("found", "rows")
|
|
101
|
+
ORDER_KEYS = ("before", "after")
|
|
102
|
+
JOURNAL_FIELDS = (
|
|
103
|
+
"seq",
|
|
104
|
+
"id",
|
|
105
|
+
"at",
|
|
106
|
+
"durationMs",
|
|
107
|
+
"via",
|
|
108
|
+
"operation",
|
|
109
|
+
"method",
|
|
110
|
+
"path",
|
|
111
|
+
"rows",
|
|
112
|
+
"found",
|
|
113
|
+
"request.body",
|
|
114
|
+
"request.bodyBytes",
|
|
115
|
+
"request.truncated",
|
|
116
|
+
"request.contentType",
|
|
117
|
+
"response.status",
|
|
118
|
+
"response.body",
|
|
119
|
+
"response.bodyBytes",
|
|
120
|
+
"response.truncated",
|
|
121
|
+
"response.contentType",
|
|
122
|
+
)
|
|
123
|
+
JOURNAL_FIELD_PREFIXES = ("query.", "request.headers.", "request.body.", "response.body.")
|
|
133
124
|
|
|
134
125
|
|
|
135
126
|
class GraderError(Exception):
|
|
136
|
-
"""A grader that
|
|
127
|
+
"""A grader that cannot deliver a grade: the host refuses, the runner records null with the reason, never 0."""
|
|
137
128
|
|
|
138
129
|
|
|
139
130
|
def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
|
|
140
|
-
"""The entities ``spec`` names that ``defined`` (a world contract's) lacks.
|
|
141
|
-
A grader's check on a reserved state key is not one of them."""
|
|
142
131
|
known = set(defined)
|
|
143
132
|
seeded = set((spec.seed or {}).get("rows", {})) if isinstance(spec.seed, dict) else set()
|
|
144
133
|
return [
|
|
@@ -149,7 +138,6 @@ def unknown_task_entities(spec: TaskSpec, defined: Iterable[str]) -> list[str]:
|
|
|
149
138
|
|
|
150
139
|
|
|
151
140
|
def parse_task_spec(value: Any) -> TaskSpec:
|
|
152
|
-
"""The spec checked for shape and size; every refusal names its path."""
|
|
153
141
|
if not isinstance(value, dict):
|
|
154
142
|
fail("task", "a task is a json object", "/task")
|
|
155
143
|
unknown = set(value) - {"name", "instruction", "seed", "grader", "tools", "metadata", "verifiers"}
|
|
@@ -209,13 +197,7 @@ _VERIFIER_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
|
209
197
|
|
|
210
198
|
|
|
211
199
|
def _parse_verifiers(value: Any, task_name: str | None) -> tuple[dict, ...]:
|
|
212
|
-
"""
|
|
213
|
-
(the grader through :func:`_parse_grader`, so the check grammar and the
|
|
214
|
-
caps are defined once). A scope names two or more distinct aliases; the
|
|
215
|
-
first is this session, so it must be the task's own name — a session task
|
|
216
|
-
is what one alias of a multi-world task opens on, named for that alias.
|
|
217
|
-
Every assertions check names ``<alias>.<entity>`` with the alias in
|
|
218
|
-
scope. Mirrors ``WorldTaskVerifierSchema``."""
|
|
200
|
+
"""Mirrors ``WorldTaskVerifierSchema``."""
|
|
219
201
|
if value is None:
|
|
220
202
|
return ()
|
|
221
203
|
at = "/task/verifiers"
|
|
@@ -256,18 +238,28 @@ def _parse_verifier(row: Any, at: str, task_name: str | None) -> dict:
|
|
|
256
238
|
grader = _parse_grader({k: v for k, v in row.items() if k not in ("name", "scope")}, at)
|
|
257
239
|
if grader is None or grader["kind"] == "rubric":
|
|
258
240
|
fail("task", "a verifier grades by assertions or python (a rubric judge reads one world)", f"{at}/kind")
|
|
241
|
+
if grader.get("readsJournal"):
|
|
242
|
+
fail("task", CROSS_JOURNAL, f"{at}/readsJournal")
|
|
259
243
|
for j, check in enumerate(grader.get("checks", ())):
|
|
260
|
-
|
|
261
|
-
fail(
|
|
262
|
-
"task",
|
|
263
|
-
f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
|
|
264
|
-
f"{at}/checks/{j}/entity",
|
|
265
|
-
)
|
|
244
|
+
_cross_world_entity(check["entity"], scope, f"{at}/checks/{j}/entity")
|
|
266
245
|
return {"name": name, "scope": list(scope), **grader}
|
|
267
246
|
|
|
268
247
|
|
|
248
|
+
CROSS_JOURNAL = f"a cross-world verifier reads <alias>.{CALL_LOG}; {JOURNAL} checks go on a world's own grader"
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _cross_world_entity(entity: str, scope: list, at: str) -> None:
|
|
252
|
+
if entity.split(".", 1)[0] not in scope or "." not in entity:
|
|
253
|
+
fail(
|
|
254
|
+
"task",
|
|
255
|
+
f"a cross-world check names its entity as <alias>.<entity> (or <alias>.calls) with the alias in scope ({', '.join(scope)})",
|
|
256
|
+
at,
|
|
257
|
+
)
|
|
258
|
+
if _on_the_journal(entity):
|
|
259
|
+
fail("task", CROSS_JOURNAL, at)
|
|
260
|
+
|
|
261
|
+
|
|
269
262
|
def verifier_source(task_dir: Path, name: str) -> Path:
|
|
270
|
-
"""Where :func:`write_task_dir` puts a python verifier's ``verify.py``."""
|
|
271
263
|
return Path(task_dir) / VERIFIERS_DIR / name / "verify.py"
|
|
272
264
|
|
|
273
265
|
|
|
@@ -327,24 +319,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
327
319
|
fail("task", "grader is a json object with a kind", at)
|
|
328
320
|
kind = grader.get("kind")
|
|
329
321
|
if kind == "python":
|
|
330
|
-
|
|
331
|
-
fail("task", "a python grader is {kind, source, timeoutSeconds?}", at)
|
|
332
|
-
source = grader.get("source")
|
|
333
|
-
if not isinstance(source, str) or not source:
|
|
334
|
-
fail("task", "a python grader needs its verify.py source", f"{at}/source")
|
|
335
|
-
if _bytes(source) > PYTHON_SOURCE_BYTES:
|
|
336
|
-
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
|
|
337
|
-
out: dict = {"kind": "python", "source": source}
|
|
338
|
-
timeout = grader.get("timeoutSeconds")
|
|
339
|
-
if timeout is not None:
|
|
340
|
-
if (
|
|
341
|
-
isinstance(timeout, bool)
|
|
342
|
-
or not isinstance(timeout, (int, float))
|
|
343
|
-
or not 0 < timeout <= PYTHON_TIMEOUT_S
|
|
344
|
-
):
|
|
345
|
-
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
|
|
346
|
-
out["timeoutSeconds"] = float(timeout)
|
|
347
|
-
return out
|
|
322
|
+
return _parse_python_grader(grader, at)
|
|
348
323
|
if kind == "assertions":
|
|
349
324
|
if set(grader) - {"kind", "checks"}:
|
|
350
325
|
fail("task", "an assertions grader is {kind, checks}", at)
|
|
@@ -354,9 +329,7 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
354
329
|
if len(checks) > CHECKS:
|
|
355
330
|
fail("task", f"an assertions grader has at most {CHECKS} checks", f"{at}/checks")
|
|
356
331
|
parsed = [_parse_check(check, f"{at}/checks/{i}") for i, check in enumerate(checks)]
|
|
357
|
-
#
|
|
358
|
-
# distinct as a set, so an explicit `check_2` cannot shadow the third
|
|
359
|
-
# check's positional one. Mirrors WorldTaskAssertionsGraderSchema.
|
|
332
|
+
# Mirrors WorldTaskAssertionsGraderSchema.
|
|
360
333
|
seen: set[str] = set()
|
|
361
334
|
for i, check in enumerate(parsed):
|
|
362
335
|
key = check_key(check, i)
|
|
@@ -377,12 +350,50 @@ def _parse_grader(grader: Any, at: str = "/task/grader") -> dict | None:
|
|
|
377
350
|
if kind == "rubric":
|
|
378
351
|
return _parse_rubric(grader, at)
|
|
379
352
|
fail("task", "grader.kind must be assertions, python or rubric", f"{at}/kind")
|
|
380
|
-
raise AssertionError
|
|
353
|
+
raise AssertionError
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
PYTHON_GRADER_KEYS = frozenset({"kind", "source", "timeoutSeconds", "readsJournal", "acceptsIncompleteJournal"})
|
|
357
|
+
JOURNAL_FLAGS = ("readsJournal", "acceptsIncompleteJournal")
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _parse_python_grader(grader: dict, at: str) -> dict:
|
|
361
|
+
"""``readsJournal`` hands verify.py the session's journal; ``acceptsIncompleteJournal`` grades one that lost
|
|
362
|
+
requests instead of failing closed."""
|
|
363
|
+
if set(grader) - PYTHON_GRADER_KEYS:
|
|
364
|
+
fail("task", "a python grader is {kind, source, timeoutSeconds?, readsJournal?, acceptsIncompleteJournal?}", at)
|
|
365
|
+
source = grader.get("source")
|
|
366
|
+
if not isinstance(source, str) or not source:
|
|
367
|
+
fail("task", "a python grader needs its verify.py source", f"{at}/source")
|
|
368
|
+
if _bytes(source) > PYTHON_SOURCE_BYTES:
|
|
369
|
+
fail("task", f"a python grader is at most {PYTHON_SOURCE_BYTES} bytes", f"{at}/source")
|
|
370
|
+
out: dict = {"kind": "python", "source": source}
|
|
371
|
+
timeout = grader.get("timeoutSeconds")
|
|
372
|
+
if timeout is not None:
|
|
373
|
+
out["timeoutSeconds"] = _timeout(timeout, at)
|
|
374
|
+
for flag in JOURNAL_FLAGS:
|
|
375
|
+
out.update(_flag(grader, flag, at))
|
|
376
|
+
if out.get("acceptsIncompleteJournal") and not out.get("readsJournal"):
|
|
377
|
+
fail("task", "acceptsIncompleteJournal only goes with readsJournal: true", f"{at}/acceptsIncompleteJournal")
|
|
378
|
+
return out
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _timeout(value: Any, at: str) -> float:
|
|
382
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not 0 < value <= PYTHON_TIMEOUT_S:
|
|
383
|
+
fail("task", f"timeoutSeconds is a number in (0, {PYTHON_TIMEOUT_S:g}]", f"{at}/timeoutSeconds")
|
|
384
|
+
return float(value)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _flag(grader: dict, name: str, at: str) -> dict:
|
|
388
|
+
value = grader.get(name)
|
|
389
|
+
if value is None or value is False:
|
|
390
|
+
return {}
|
|
391
|
+
if value is not True:
|
|
392
|
+
fail("task", f"{name} is true or absent", f"{at}/{name}")
|
|
393
|
+
return {name: True}
|
|
381
394
|
|
|
382
395
|
|
|
383
396
|
def _parse_rubric(grader: dict, at: str) -> dict:
|
|
384
|
-
"""An LLM judge's brief. Checked here for shape only: the host stores it
|
|
385
|
-
with the task and never runs it; the platform grades it."""
|
|
386
397
|
if set(grader) - {"kind", "rubric", "criteria", "model"}:
|
|
387
398
|
fail("task", "a rubric grader is {kind, rubric, criteria?, model?}", at)
|
|
388
399
|
rubric = grader.get("rubric")
|
|
@@ -436,76 +447,166 @@ def _parse_rubric(grader: dict, at: str) -> dict:
|
|
|
436
447
|
|
|
437
448
|
|
|
438
449
|
def check_key(check: dict, index: int) -> str:
|
|
439
|
-
"""The reward key of a check: its id, else its position as ``check_<i>``."""
|
|
440
450
|
return check.get("id") or f"check_{index}"
|
|
441
451
|
|
|
442
452
|
|
|
453
|
+
CHECK_KEYS = frozenset({"id", "weight", "entity", "where", "assert", "count", "field", "op", "value", *ORDER_KEYS})
|
|
454
|
+
PREDICATES = ("all", "any")
|
|
455
|
+
|
|
456
|
+
|
|
443
457
|
def _parse_check(check: Any, at: str) -> dict:
|
|
444
458
|
if not isinstance(check, dict):
|
|
445
459
|
fail("task", "a check is a json object", at)
|
|
446
|
-
unknown = set(check) -
|
|
460
|
+
unknown = set(check) - CHECK_KEYS
|
|
447
461
|
if unknown:
|
|
448
462
|
fail("task", f"unknown check field(s): {', '.join(sorted(unknown))}", at)
|
|
449
463
|
out: dict = {}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
|
|
454
|
-
out["id"] = check_id
|
|
455
|
-
weight = check.get("weight", 1)
|
|
456
|
-
if isinstance(weight, bool) or not isinstance(weight, (int, float)) or not weight > 0 or weight == float("inf"):
|
|
457
|
-
fail("task", "weight is a positive number", f"{at}/weight")
|
|
458
|
-
out["weight"] = float(weight)
|
|
464
|
+
if check.get("id") is not None:
|
|
465
|
+
out["id"] = _check_id(check["id"], at)
|
|
466
|
+
out["weight"] = _weight(check.get("weight", 1), at)
|
|
459
467
|
entity = check.get("entity")
|
|
460
468
|
if not isinstance(entity, str) or not entity or len(entity) > 120:
|
|
461
469
|
fail("task", "a check names an entity", f"{at}/entity")
|
|
462
470
|
out["entity"] = entity
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
if
|
|
466
|
-
|
|
467
|
-
):
|
|
468
|
-
fail("task", "where is {field: scalar, ...}", f"{at}/where")
|
|
469
|
-
out["where"] = where
|
|
471
|
+
for key in ("where", *ORDER_KEYS):
|
|
472
|
+
selection = _selection(check.get(key), key, f"{at}/{key}")
|
|
473
|
+
if selection is not None:
|
|
474
|
+
out[key] = selection
|
|
470
475
|
assertion = check.get("assert", "exists")
|
|
471
476
|
if assertion not in ASSERTIONS:
|
|
472
477
|
fail("task", f"assert must be one of {', '.join(ASSERTIONS)}", f"{at}/assert")
|
|
473
478
|
out["assert"] = assertion
|
|
479
|
+
out.update(_count(check, assertion, at))
|
|
480
|
+
out.update(_field_test(check, assertion, at))
|
|
481
|
+
_check_reads(out, at)
|
|
482
|
+
return out
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _check_id(value: Any, at: str) -> str:
|
|
486
|
+
if not isinstance(value, str) or not _CHECK_ID.match(value) or len(value) > 120:
|
|
487
|
+
fail("task", "a check id is letters, digits, _ . -", f"{at}/id")
|
|
488
|
+
return value
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _weight(value: Any, at: str) -> float:
|
|
492
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not value > 0 or value == float("inf"):
|
|
493
|
+
fail("task", "weight is a positive number", f"{at}/weight")
|
|
494
|
+
return float(value)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _selection(value: Any, key: str, at: str) -> dict | None:
|
|
498
|
+
if value is None:
|
|
499
|
+
return None
|
|
500
|
+
if not isinstance(value, dict) or any(not isinstance(k, str) or not k or not _scalar(v) for k, v in value.items()):
|
|
501
|
+
fail("task", f"{key} is {{field: scalar, ...}}", at)
|
|
502
|
+
return value
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _count(check: dict, assertion: str, at: str) -> dict:
|
|
474
506
|
count = check.get("count")
|
|
475
|
-
if assertion
|
|
476
|
-
if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
|
|
477
|
-
fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
|
|
478
|
-
for key, bound in count.items():
|
|
479
|
-
if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
|
|
480
|
-
fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
|
|
481
|
-
out["count"] = count
|
|
482
|
-
elif count is not None:
|
|
507
|
+
if assertion != "count" and count is not None:
|
|
483
508
|
fail("task", "count only goes with assert: count", f"{at}/count")
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
509
|
+
if assertion != "count":
|
|
510
|
+
return {}
|
|
511
|
+
if not isinstance(count, dict) or set(count) - {"eq", "gte", "lte"} or not count:
|
|
512
|
+
fail("task", "assert: count needs a count bound {eq|gte|lte: n}", f"{at}/count")
|
|
513
|
+
for key, bound in count.items():
|
|
514
|
+
_count_bound(key, bound, at)
|
|
515
|
+
return {"count": count}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _count_bound(key: str, bound: Any, at: str) -> None:
|
|
519
|
+
if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0:
|
|
520
|
+
fail("task", f"count.{key} is a non-negative integer", f"{at}/count/{key}")
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
def _field_test(check: dict, assertion: str, at: str) -> dict:
|
|
524
|
+
field, op, value = check.get("field"), check.get("op", "eq"), check.get("value")
|
|
525
|
+
if assertion not in PREDICATES and (field is not None or "value" in check or "op" in check):
|
|
501
526
|
fail("task", "field/op/value only go with assert: all or any", f"{at}/field")
|
|
502
|
-
|
|
527
|
+
if assertion not in PREDICATES:
|
|
528
|
+
return {}
|
|
529
|
+
if not isinstance(field, str) or not field:
|
|
530
|
+
fail("task", f"assert: {assertion} needs a field to test", f"{at}/field")
|
|
531
|
+
if op not in FIELD_OPS:
|
|
532
|
+
fail("task", f"op must be one of {', '.join(FIELD_OPS)}", f"{at}/op")
|
|
533
|
+
if "value" not in check:
|
|
534
|
+
fail("task", f"assert: {assertion} needs a value to compare with", f"{at}/value")
|
|
535
|
+
_compared(op, value, at)
|
|
536
|
+
return {"field": field, "op": op, "value": value}
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _compared(op: str, value: Any, at: str) -> None:
|
|
540
|
+
if op == "in" and (not isinstance(value, list) or any(not _scalar(v) for v in value)):
|
|
541
|
+
fail("task", "op: in compares against a list of scalars", f"{at}/value")
|
|
542
|
+
if op != "in" and not _scalar(value):
|
|
543
|
+
fail("task", f"op: {op} compares against one scalar", f"{at}/value")
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def _on_the_journal(entity: str) -> bool:
|
|
547
|
+
return entity == JOURNAL or entity.endswith(f".{JOURNAL}")
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _ordered_log(entity: str) -> bool:
|
|
551
|
+
return _on_the_journal(entity) or entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
def _check_reads(check: dict, at: str) -> None:
|
|
555
|
+
entity = check["entity"]
|
|
556
|
+
ordered = [key for key in ORDER_KEYS if key in check]
|
|
557
|
+
if ordered and not _ordered_log(entity):
|
|
558
|
+
fail(
|
|
559
|
+
"task",
|
|
560
|
+
f"{ordered[0]} orders requests in time, so it goes on a {JOURNAL} or {CALL_LOG} check, not on {entity}",
|
|
561
|
+
f"{at}/{ordered[0]}",
|
|
562
|
+
)
|
|
563
|
+
if entity == JOURNAL_META or entity.endswith(f".{JOURNAL_META}"):
|
|
564
|
+
fail("task", f"{JOURNAL_META} is the journal's summary, not its rows; check {JOURNAL} instead", f"{at}/entity")
|
|
565
|
+
if _on_the_journal(entity):
|
|
566
|
+
_journal_paths(check, at)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def _journal_paths(check: dict, at: str) -> None:
|
|
570
|
+
named = [(key, path) for key in ("where", *ORDER_KEYS) for path in check.get(key, {})]
|
|
571
|
+
named += [("field", check["field"])] if "field" in check else []
|
|
572
|
+
for key, path in named:
|
|
573
|
+
_journal_path(path, f"{at}/{key}")
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _distance(a: str, b: str) -> int:
|
|
577
|
+
previous = list(range(len(b) + 1))
|
|
578
|
+
for i, left in enumerate(a, 1):
|
|
579
|
+
current = [i]
|
|
580
|
+
for j, right in enumerate(b, 1):
|
|
581
|
+
current.append(min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + (left != right)))
|
|
582
|
+
previous = current
|
|
583
|
+
return previous[-1]
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def nearest_journal_field(path: str) -> str | None:
|
|
587
|
+
"""Twin of ``nearestJournalField`` in worldJournal.ts."""
|
|
588
|
+
best, best_distance = None, math.inf
|
|
589
|
+
for candidate in (*JOURNAL_FIELDS, *(prefix[:-1] for prefix in JOURNAL_FIELD_PREFIXES)):
|
|
590
|
+
d = _distance(path, candidate)
|
|
591
|
+
if d < best_distance and d <= 3 and d < len(candidate) / 2:
|
|
592
|
+
best, best_distance = candidate, d
|
|
593
|
+
return best
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def _journal_path(path: str, at: str) -> None:
|
|
597
|
+
if path in JOURNAL_FIELDS or path.startswith(JOURNAL_FIELD_PREFIXES):
|
|
598
|
+
return
|
|
599
|
+
near = nearest_journal_field(path)
|
|
600
|
+
hint = f"did you mean {near}? " if near else ""
|
|
601
|
+
fail(
|
|
602
|
+
"task",
|
|
603
|
+
f"{JOURNAL} has no field {path}; {hint}match on {', '.join(JOURNAL_FIELDS)} "
|
|
604
|
+
f"or a path under {', '.join(JOURNAL_FIELD_PREFIXES)}",
|
|
605
|
+
at,
|
|
606
|
+
)
|
|
503
607
|
|
|
504
608
|
|
|
505
609
|
def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
506
|
-
"""The task under the session: ``task.json`` (the whole spec),
|
|
507
|
-
``instruction.md``, and for a python grader ``checks/verify.py`` where
|
|
508
|
-
the bundled layout has it, so the one verifier path runs both."""
|
|
509
610
|
root = Path(session_dir) / TASK_DIR
|
|
510
611
|
root.mkdir(parents=True, exist_ok=True)
|
|
511
612
|
(root / TASK_FILE).write_text(json.dumps(spec.raw, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
@@ -524,86 +625,12 @@ def write_task_dir(spec: TaskSpec, session_dir: Path) -> Path:
|
|
|
524
625
|
|
|
525
626
|
|
|
526
627
|
def read_task_dir(session_dir: Path) -> TaskSpec | None:
|
|
527
|
-
"""The task a session directory holds, or None for a bundled-task session."""
|
|
528
628
|
path = Path(session_dir) / TASK_DIR / TASK_FILE
|
|
529
629
|
if not path.is_file():
|
|
530
630
|
return None
|
|
531
631
|
return parse_task_spec(json.loads(path.read_text(encoding="utf-8")))
|
|
532
632
|
|
|
533
633
|
|
|
534
|
-
def _lookup(row: Any, path: str) -> tuple[bool, Any]:
|
|
535
|
-
"""``(found, value)`` of a dotted path into a row."""
|
|
536
|
-
current = row
|
|
537
|
-
for part in path.split("."):
|
|
538
|
-
if not isinstance(current, dict) or part not in current:
|
|
539
|
-
return False, None
|
|
540
|
-
current = current[part]
|
|
541
|
-
return True, current
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
def _same(actual: Any, expected: Any) -> bool:
|
|
545
|
-
"""JSON equality: ``true`` is not ``1``, as it is not in the schema."""
|
|
546
|
-
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
547
|
-
return isinstance(actual, bool) and isinstance(expected, bool) and actual is expected
|
|
548
|
-
return actual == expected
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
def _matches_where(row: dict, where: dict | None) -> bool:
|
|
552
|
-
for path, expected in (where or {}).items():
|
|
553
|
-
found, actual = _lookup(row, path)
|
|
554
|
-
if not found or not _same(actual, expected):
|
|
555
|
-
return False
|
|
556
|
-
return True
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
def _ordered(actual: Any, expected: Any) -> bool:
|
|
560
|
-
"""Whether the two compare with ``<``: both numbers (not bools) or both strings."""
|
|
561
|
-
numbers = (int, float)
|
|
562
|
-
if isinstance(actual, bool) or isinstance(expected, bool):
|
|
563
|
-
return False
|
|
564
|
-
if isinstance(actual, numbers) and isinstance(expected, numbers):
|
|
565
|
-
return True
|
|
566
|
-
return isinstance(actual, str) and isinstance(expected, str)
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
def _predicate(row: dict, field: str, op: str, expected: Any) -> bool:
|
|
570
|
-
found, actual = _lookup(row, field)
|
|
571
|
-
if op == "eq":
|
|
572
|
-
return found and _same(actual, expected)
|
|
573
|
-
if op == "neq":
|
|
574
|
-
return not found or not _same(actual, expected)
|
|
575
|
-
if op == "in":
|
|
576
|
-
return found and any(_same(actual, option) for option in expected)
|
|
577
|
-
if op == "contains":
|
|
578
|
-
if not found:
|
|
579
|
-
return False
|
|
580
|
-
if isinstance(actual, str):
|
|
581
|
-
return isinstance(expected, str) and expected in actual
|
|
582
|
-
if isinstance(actual, list):
|
|
583
|
-
return any(_same(item, expected) for item in actual)
|
|
584
|
-
return False
|
|
585
|
-
if not found or not _ordered(actual, expected):
|
|
586
|
-
return False
|
|
587
|
-
if op == "gt":
|
|
588
|
-
return actual > expected
|
|
589
|
-
if op == "gte":
|
|
590
|
-
return actual >= expected
|
|
591
|
-
if op == "lt":
|
|
592
|
-
return actual < expected
|
|
593
|
-
return actual <= expected
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
def _rows_of(state: dict, entity: str) -> list[dict]:
|
|
597
|
-
"""The rows of one entity from a state dump: ``{entity: {key: row}}`` (the
|
|
598
|
-
verifier's shape) or ``{entity: [row, ...]}``."""
|
|
599
|
-
held = state.get(entity)
|
|
600
|
-
if isinstance(held, dict):
|
|
601
|
-
return [row for row in held.values() if isinstance(row, dict)]
|
|
602
|
-
if isinstance(held, list):
|
|
603
|
-
return [row for row in held if isinstance(row, dict)]
|
|
604
|
-
return []
|
|
605
|
-
|
|
606
|
-
|
|
607
634
|
def _on_the_call_log(check: dict) -> bool:
|
|
608
635
|
entity = check["entity"]
|
|
609
636
|
return entity == CALL_LOG or entity.endswith(f".{CALL_LOG}")
|
|
@@ -615,124 +642,3 @@ def _names(check: dict, field: str) -> bool:
|
|
|
615
642
|
|
|
616
643
|
def _names_the_door(check: dict) -> bool:
|
|
617
644
|
return _on_the_call_log(check) and _names(check, DOOR_FIELD)
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
def _require_the_log(check: dict, state: dict, rows: list[dict]) -> None:
|
|
621
|
-
"""A check over a call log the engine does not keep would select nothing
|
|
622
|
-
(``absent`` passes) and one over a door a log does not record would miss
|
|
623
|
-
the key (``neq`` passes): each is ungraded."""
|
|
624
|
-
if not _on_the_call_log(check):
|
|
625
|
-
return
|
|
626
|
-
if _names_the_door(check) and (check["entity"] not in state or any(DOOR_FIELD not in row for row in rows)):
|
|
627
|
-
raise GraderError(f"this engine's call log does not record the door ({DOOR_FIELD})")
|
|
628
|
-
if check["entity"] not in state:
|
|
629
|
-
raise GraderError(f"this engine keeps no call log ({check['entity']})")
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
def _unknown_fields(row: dict) -> tuple[str, ...]:
|
|
633
|
-
"""What a call-log entry cannot say: the request fields it withheld (a
|
|
634
|
-
request too large to keep), and ``found`` / ``rows`` where its answer
|
|
635
|
-
declared no rows (``found`` null)."""
|
|
636
|
-
withheld = tuple(row.get(WITHHELD_FIELD) or ())
|
|
637
|
-
return withheld + ROW_FIELDS if row.get("found") is None else withheld
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
def _reads(unknown: tuple[str, ...], path: str) -> bool:
|
|
641
|
-
return any(path == field or path.startswith(f"{field}.") for field in unknown)
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
def _possible(check: dict, row: dict, unknown: tuple[str, ...]) -> tuple[bool | None, bool | None]:
|
|
645
|
-
"""Whether a row is selected and whether it passes the check's predicate,
|
|
646
|
-
each None where the row cannot say."""
|
|
647
|
-
where = check.get("where") or {}
|
|
648
|
-
known = {path: value for path, value in where.items() if not _reads(unknown, path)}
|
|
649
|
-
if not _matches_where(row, known):
|
|
650
|
-
return False, None
|
|
651
|
-
selected = None if len(known) < len(where) else True
|
|
652
|
-
return selected, _passes(check, row, unknown)
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
def _passes(check: dict, row: dict, unknown: tuple[str, ...]) -> bool | None:
|
|
656
|
-
"""Whether a row passes the check's predicate, None where it cannot say. A
|
|
657
|
-
withheld list still holds what it lists, so a ``contains`` it already
|
|
658
|
-
satisfies passes: more items cannot take it away."""
|
|
659
|
-
field = check.get("field")
|
|
660
|
-
if field is None:
|
|
661
|
-
return None
|
|
662
|
-
holds = _predicate(row, field, check["op"], check["value"])
|
|
663
|
-
if not _reads(unknown, field):
|
|
664
|
-
return holds
|
|
665
|
-
partial = isinstance(_lookup(row, field)[1], list)
|
|
666
|
-
return True if holds and partial and check["op"] == "contains" else None
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
def _dropped(rows: list[dict]) -> int:
|
|
670
|
-
"""How many calls a bounded log no longer keeps: the first kept ``seq`` says."""
|
|
671
|
-
kept = [row["seq"] for row in rows if isinstance(row.get("seq"), int)]
|
|
672
|
-
return min(kept) - 1 if kept else 0
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
def _interval(check: dict) -> tuple[float, float]:
|
|
676
|
-
assertion = check["assert"]
|
|
677
|
-
if assertion == "exists":
|
|
678
|
-
return 1, math.inf
|
|
679
|
-
if assertion == "absent":
|
|
680
|
-
return 0, 0
|
|
681
|
-
bound = check["count"]
|
|
682
|
-
return max(bound.get("gte", 0), bound.get("eq", 0)), min(bound.get("lte", math.inf), bound.get("eq", math.inf))
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
def _outcomes(check: dict, possible: list[tuple[bool | None, bool | None]], dropped: int) -> set[bool]:
|
|
686
|
-
"""Every result the check could have over what the rows cannot say. A call
|
|
687
|
-
the log no longer keeps is selected (or, under a ``where``, perhaps) and
|
|
688
|
-
passes or fails, unknown."""
|
|
689
|
-
gone_sure, gone_maybe = (0, dropped) if check.get("where") else (dropped, 0)
|
|
690
|
-
sure = [passes for selected, passes in possible if selected is True] + [None] * min(gone_sure, 1)
|
|
691
|
-
maybe = [passes for selected, passes in possible if selected is None] + [None] * min(gone_maybe, 1)
|
|
692
|
-
least = len(sure) - min(gone_sure, 1) + gone_sure
|
|
693
|
-
most = least + len(maybe) - min(gone_maybe, 1) + gone_maybe
|
|
694
|
-
assertion = check["assert"]
|
|
695
|
-
if assertion in ("exists", "absent", "count"):
|
|
696
|
-
low, high = _interval(check)
|
|
697
|
-
can_true, can_false = max(low, least) <= min(high, most), not (low <= least and most <= high)
|
|
698
|
-
elif assertion == "all":
|
|
699
|
-
can_true = False not in sure and (least > 0 or any(passes is not False for passes in maybe))
|
|
700
|
-
can_false = least == 0 or any(passes is not True for passes in sure + maybe)
|
|
701
|
-
else:
|
|
702
|
-
can_true = any(passes is not False for passes in sure + maybe)
|
|
703
|
-
can_false = True not in sure
|
|
704
|
-
return {outcome for outcome, holds in ((True, can_true), (False, can_false)) if holds}
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
def evaluate_check(check: dict, state: dict) -> bool:
|
|
708
|
-
"""One check over a state dump. Over a call log, a row may not say what the
|
|
709
|
-
check reads (a withheld request, a ``found`` null, a call the log no longer
|
|
710
|
-
keeps): the check grades whenever the rows that can say decide it, and is
|
|
711
|
-
ungraded (:class:`GraderError`) only where what they cannot say could
|
|
712
|
-
change the result."""
|
|
713
|
-
rows = _rows_of(state, check["entity"])
|
|
714
|
-
_require_the_log(check, state, rows)
|
|
715
|
-
on_log = _on_the_call_log(check)
|
|
716
|
-
possible = [_possible(check, row, _unknown_fields(row) if on_log else ()) for row in rows]
|
|
717
|
-
outcomes = _outcomes(check, possible, _dropped(rows) if on_log else 0)
|
|
718
|
-
if len(outcomes) > 1:
|
|
719
|
-
raise GraderError(
|
|
720
|
-
"calls this check reads cannot say what it asks (a request too large to keep, an answer that "
|
|
721
|
-
"declared no rows, or a call the log no longer keeps), and the rest do not decide it: ungraded"
|
|
722
|
-
)
|
|
723
|
-
return outcomes.pop()
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
def evaluate_assertions(checks: list[dict], state: dict) -> dict:
|
|
727
|
-
"""The grade of an assertions grader over a state dump: the weighted share
|
|
728
|
-
of checks that passed as ``reward``, each check's 0/1 under its id (or its
|
|
729
|
-
position) in ``rewards``. The runner's shape, as ``verify.py`` prints it."""
|
|
730
|
-
rewards: dict[str, float] = {}
|
|
731
|
-
total = 0.0
|
|
732
|
-
passed = 0.0
|
|
733
|
-
for i, check in enumerate(checks):
|
|
734
|
-
ok = evaluate_check(check, state)
|
|
735
|
-
rewards[check_key(check, i)] = 1.0 if ok else 0.0
|
|
736
|
-
total += check["weight"]
|
|
737
|
-
passed += check["weight"] if ok else 0.0
|
|
738
|
-
return {"reward": (passed / total) if total else None, "rewards": rewards}
|